testguard-cli 0.4.0 → 0.6.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (51) hide show
  1. package/CHANGELOG.md +291 -0
  2. package/README.md +321 -27
  3. package/package.json +1 -1
  4. package/spec/GATE-SEMANTICS.md +109 -1
  5. package/spec/README.md +4 -2
  6. package/spec/lib/validate.mjs +75 -1
  7. package/spec/schemas/baseline.schema.json +8 -0
  8. package/spec/schemas/brief.schema.json +11 -1
  9. package/spec/schemas/common.schema.json +32 -1
  10. package/spec/schemas/evidence.schema.json +374 -63
  11. package/spec/schemas/gate.schema.json +52 -0
  12. package/spec/schemas/replay.schema.json +85 -0
  13. package/spec/schemas/status.schema.json +269 -45
  14. package/src/admit/admit.mjs +17 -0
  15. package/src/baseline/baseline.mjs +31 -2
  16. package/src/brief/brief.mjs +96 -8
  17. package/src/claims/removed.mjs +161 -0
  18. package/src/cli.mjs +69 -12
  19. package/src/commands/admit.mjs +116 -0
  20. package/src/commands/baseline.mjs +17 -3
  21. package/src/commands/brief.mjs +26 -12
  22. package/src/commands/claims.mjs +35 -6
  23. package/src/commands/gate.mjs +47 -0
  24. package/src/commands/init.mjs +19 -6
  25. package/src/commands/mcp.mjs +12 -0
  26. package/src/commands/probe.mjs +12 -3
  27. package/src/commands/replay.mjs +63 -0
  28. package/src/commands/status.mjs +4 -1
  29. package/src/gate/changed.mjs +222 -0
  30. package/src/git.mjs +5 -0
  31. package/src/init/init.mjs +182 -29
  32. package/src/init/templates/SKILL.md +21 -3
  33. package/src/mcp/server.mjs +104 -0
  34. package/src/mcp/tools.mjs +114 -0
  35. package/src/probe/contention.mjs +60 -0
  36. package/src/probe/discover.mjs +12 -3
  37. package/src/probe/independence.mjs +59 -0
  38. package/src/probe/mocks.mjs +120 -0
  39. package/src/probe/probe.mjs +126 -18
  40. package/src/probe/rank.mjs +73 -15
  41. package/src/probe/runners/index.mjs +49 -5
  42. package/src/probe/runners/jest.mjs +3 -3
  43. package/src/probe/runners/playwright.mjs +133 -0
  44. package/src/probe/runners/shared.mjs +70 -13
  45. package/src/probe/runners/vitest.mjs +4 -3
  46. package/src/render.mjs +2 -2
  47. package/src/replay/label.mjs +64 -0
  48. package/src/replay/replay.mjs +309 -0
  49. package/src/scaffold/producers.mjs +217 -6
  50. package/src/scaffold/scaffold.mjs +9 -3
  51. package/src/status/status.mjs +88 -7
package/src/init/init.mjs CHANGED
@@ -1,68 +1,198 @@
1
- import { existsSync, mkdirSync, readFileSync, writeFileSync } from 'node:fs';
2
- import { join, dirname } from 'node:path';
1
+ import { existsSync, mkdirSync, readFileSync, writeFileSync, realpathSync } from 'node:fs';
2
+ import { join, dirname, relative, resolve, sep } from 'node:path';
3
3
  import { fileURLToPath } from 'node:url';
4
+ import { spawnSync } from 'node:child_process';
4
5
 
5
6
  const TEMPLATES = join(dirname(fileURLToPath(import.meta.url)), 'templates');
6
- const HOOK_CMD = 'npx -y testguard-cli brief --text';
7
7
  const AGENTS_BEGIN = '<!-- testguard:begin -->';
8
8
  const AGENTS_END = '<!-- testguard:end -->';
9
- const AGENTS_BLOCK = `${AGENTS_BEGIN}
9
+ const GITIGNORE_LINES = ['.testguard/evidence.json', '.testguard/evidence-provisional.json', '.testguard/evidence-partial.json', '.testguard/brief.json', '.testguard/gate.json', '.testguard/scaffold-*.json'];
10
+
11
+ /**
12
+ * The session-start hook command. It runs from the git root (where the agent
13
+ * session lives) and must never fetch from the network: a local install of
14
+ * the project (or of the root) is preferred, then a `testguard` already on
15
+ * PATH, then nothing — the hook exits 0 with no output rather than break a
16
+ * session.
17
+ *
18
+ * No form of `npx` appears here, and `--no-install` is not an exception.
19
+ * Measured: in a project with nothing installed,
20
+ * `npx --no-install --loglevel=http testguard-cli --version` logs
21
+ * `npm http fetch GET 200 https://registry.npmjs.org/testguard-cli`, and
22
+ * against an unreachable registry it exits non-zero. npm resolves the
23
+ * packument before it decides not to install, so `--no-install` is quiet, not
24
+ * offline. A `command -v` lookup covers the global-install case with no
25
+ * network at all, which is the whole point in a repository that pins and
26
+ * audits its dependencies.
27
+ */
28
+ export function hookCommand(dir) {
29
+ const arg = dir === '.' ? '' : ` ${dir}`;
30
+ const candidates = dir === '.' ? ['node_modules/.bin/testguard'] : [`${dir}/node_modules/.bin/testguard`, 'node_modules/.bin/testguard'];
31
+ // The PATH branch is braced: `a || b && c` binds as `(a || b) && c` in sh,
32
+ // so an unbraced `&&` would run the brief a second time whenever the local
33
+ // binary succeeded.
34
+ const onPath = `{ command -v testguard >/dev/null 2>&1 && testguard brief --text${arg} 2>/dev/null; }`;
35
+ return [...candidates.map((c) => `${c} brief --text${arg} 2>/dev/null`), onPath, 'true'].join(' || ');
36
+ }
37
+
38
+ function agentsBlock(entries) {
39
+ const lines = entries.map((dir) => (dir === '.'
40
+ ? '- This project: claims in `testguard.claims.json`; run `testguard status --json` and follow `next`.'
41
+ : `- \`${dir}/\`: claims in \`${dir}/testguard.claims.json\`; run \`testguard status --json ${dir}\` and follow \`next\`.`));
42
+ return `${AGENTS_BEGIN}
10
43
  ## TestGuard
11
44
 
12
- This project's tests are verified by [TestGuard](https://github.com/raccioly/testguard).
13
- Before writing or changing tests, run \`testguard status --json\` and follow \`next\`.
45
+ This repository's tests are verified by [TestGuard](https://github.com/raccioly/testguard).
46
+ Before writing or changing tests, run \`testguard status --json [dir]\` and follow \`next\`.
14
47
  The full operating loop and the verdict table are in \`.claude/skills/testguard/SKILL.md\`.
15
48
  Never make a fault die by editing \`testguard.claims.json\`; write the test. Claim edits are recorded in the evidence.
49
+
50
+ ${lines.join('\n')}
16
51
  ${AGENTS_END}`;
17
- const GITIGNORE_LINES = ['.testguard/evidence.json', '.testguard/evidence-provisional.json', '.testguard/evidence-partial.json', '.testguard/brief.json', '.testguard/scaffold-*.json'];
52
+ }
53
+
54
+ /** The git root of `dir`, or null outside a repository. */
55
+ export function gitRoot(dir) {
56
+ const r = spawnSync('git', ['rev-parse', '--show-toplevel'], { cwd: dir, encoding: 'utf8' });
57
+ return r.status === 0 ? r.stdout.trim() : null;
58
+ }
59
+
60
+ /** Is `file` (absolute) ignored by git in `root`? False outside a repository. */
61
+ export function isGitIgnored(root, file) {
62
+ if (!root) return false;
63
+ return spawnSync('git', ['check-ignore', '-q', '--', file], { cwd: root }).status === 0;
64
+ }
18
65
 
19
66
  /**
20
- * Install the agent operating layer into a consumer project: the skill, the
21
- * session-start hook, an AGENTS.md section and the .gitignore lines.
22
- * Idempotent: re-running changes nothing unless --force replaces the skill.
67
+ * Install the operating layer.
68
+ *
69
+ * Two layers, two places. The AGENT layer — the skill, the session-start
70
+ * hook, the AGENTS.md section — goes to the git root, because that is where
71
+ * an agent session runs; installed in a subdirectory it never fires and is
72
+ * never read (a field report installed it in a backend/ folder and got
73
+ * nothing). The PROJECT layer — the `.gitignore` lines beside the claims
74
+ * file — stays in the project directory. `here: true` keeps everything in
75
+ * the project directory for a subdirectory that is its own agent root.
76
+ *
77
+ * Idempotent per project: a second project in the same repository adds a
78
+ * hook line and an AGENTS.md bullet, never duplicates. Every written file is
79
+ * checked against .gitignore; an ignored file is reported as ignored, never
80
+ * as something to commit.
23
81
  */
24
- export function initProject({ projectDir, force = false }) {
82
+ const CI_EVIDENCE_HELPERS = {
83
+ github: `#!/bin/sh
84
+ # Fetch the evidence CI wrote on a branch, then brief from it. Run it when you
85
+ # want it: the session-start hook never touches the network, and this script
86
+ # exits 0 with a message whenever anything is missing.
87
+ BRANCH="\${1:-main}"
88
+ OUT=.testguard/ci
89
+ command -v gh >/dev/null 2>&1 || { echo "gh is not installed; see README > Read CI's evidence locally" >&2; exit 0; }
90
+ command -v testguard >/dev/null 2>&1 || { echo "testguard is not on PATH; install it or run npx testguard-cli brief --evidence <file>" >&2; exit 0; }
91
+ mkdir -p "$OUT" || exit 0
92
+ gh run download --name "testguard-evidence-$BRANCH" --dir "$OUT" >/dev/null 2>&1 || { echo "no testguard-evidence-$BRANCH artifact to download" >&2; exit 0; }
93
+ exec testguard brief . --text --evidence "$OUT/ci-self-evidence.json"
94
+ `,
95
+ gitlab: `#!/bin/sh
96
+ # Fetch the evidence CI wrote on a branch, then brief from it. Run it when you
97
+ # want it: the session-start hook never touches the network, and this script
98
+ # exits 0 with a message whenever anything is missing.
99
+ BRANCH="\${1:-main}"
100
+ OUT=.testguard/ci
101
+ command -v glab >/dev/null 2>&1 || { echo "glab is not installed; see README > Read CI's evidence locally" >&2; exit 0; }
102
+ command -v testguard >/dev/null 2>&1 || { echo "testguard is not on PATH; install it or run npx testguard-cli brief --evidence <file>" >&2; exit 0; }
103
+ mkdir -p "$OUT" || exit 0
104
+ glab ci artifact "$BRANCH" testguard:probe --path "$OUT/" >/dev/null 2>&1 || { echo "no testguard:probe artifact to download for $BRANCH" >&2; exit 0; }
105
+ exec testguard brief . --text --evidence "$OUT/.testguard/evidence.json"
106
+ `,
107
+ };
108
+
109
+ export function initProject({ projectDir, force = false, here = false, ciEvidence, mcp = false }) {
110
+ projectDir = realpathSync(resolve(projectDir));
111
+ const root = here ? null : gitRoot(projectDir);
112
+ const agentRoot = root ?? projectDir;
113
+ const ignoreRoot = root ?? gitRoot(projectDir);
114
+ const dir = (relative(agentRoot, projectDir).split(sep).join('/')) || '.';
25
115
  const done = [];
116
+ const notes = [];
117
+ let mcpConfig;
26
118
  const skipped = [];
119
+ const warnings = [];
120
+ const rel = (p) => relative(agentRoot, p).split(sep).join('/');
121
+ const note = (p) => { if (isGitIgnored(ignoreRoot, p)) warnings.push(`${rel(p)} is ignored by .gitignore — it was written, but git will not track it and agents in a fresh clone will never see it; unignore it or move it`); };
27
122
 
28
- const skillDir = join(projectDir, '.claude', 'skills', 'testguard');
123
+ const skillDir = join(agentRoot, '.claude', 'skills', 'testguard');
29
124
  const skillPath = join(skillDir, 'SKILL.md');
30
125
  if (!existsSync(skillPath) || force) {
126
+ const replaced = force && existsSync(skillPath);
31
127
  mkdirSync(skillDir, { recursive: true });
32
128
  writeFileSync(skillPath, readFileSync(join(TEMPLATES, 'SKILL.md'), 'utf8'));
33
- done.push(`.claude/skills/testguard/SKILL.md${force && existsSync(skillPath) ? ' (replaced)' : ''}`);
129
+ done.push(`${rel(skillPath)}${replaced ? ' (replaced)' : ''}`);
130
+ note(skillPath);
34
131
  } else {
35
- skipped.push('.claude/skills/testguard/SKILL.md exists (use --force to replace)');
132
+ skipped.push(`${rel(skillPath)} exists (use --force to replace)`);
36
133
  }
37
134
 
38
- const settingsPath = join(projectDir, '.claude', 'settings.json');
135
+ const settingsPath = join(agentRoot, '.claude', 'settings.json');
39
136
  let settings = {};
40
137
  if (existsSync(settingsPath)) {
41
138
  try {
42
139
  settings = JSON.parse(readFileSync(settingsPath, 'utf8'));
43
140
  } catch (e) {
44
- throw new Error(`.claude/settings.json is not valid JSON (${e.message}); fix it before init can add the hook`);
141
+ throw new Error(`${rel(settingsPath)} is not valid JSON (${e.message}); fix it before init can add the hook`);
45
142
  }
46
143
  }
47
144
  settings.hooks ??= {};
48
145
  settings.hooks.SessionStart ??= [];
49
- const already = JSON.stringify(settings.hooks.SessionStart).includes(HOOK_CMD);
50
- if (already) {
51
- skipped.push('SessionStart hook already present in .claude/settings.json');
146
+ const cmd = hookCommand(dir);
147
+ const hasCmd = settings.hooks.SessionStart.some((e) => JSON.stringify(e).includes(cmd));
148
+ // Any earlier hook that reached for npx — `-y` fetched outright, and
149
+ // `--no-install` still resolves the packument from the registry — is replaced.
150
+ // Which project a hook is for is the token right after `brief --text`, if
151
+ // that token is an argument rather than a redirect or a shell operator.
152
+ // Testing for any non-space there mistook `brief --text 2>/dev/null` for a
153
+ // hook belonging to a directory, so a legacy root hook was never replaced.
154
+ const hookDir = (text) => {
155
+ const m = /brief --text(?:\s+([^\s"|;&]+))?/.exec(text);
156
+ const token = m?.[1];
157
+ return !token || token.startsWith('2>') || token.startsWith('>') ? '.' : token;
158
+ };
159
+ const legacy = settings.hooks.SessionStart.findIndex((e) => {
160
+ const text = JSON.stringify(e);
161
+ return /npx\s+(-y|--no-install)[^"]*brief --text/.test(text) && hookDir(text) === dir;
162
+ });
163
+ if (hasCmd) {
164
+ skipped.push(`SessionStart hook for ${dir === '.' ? 'this project' : dir} already present in ${rel(settingsPath)}`);
52
165
  } else {
53
- settings.hooks.SessionStart.push({ hooks: [{ type: 'command', command: HOOK_CMD }] });
166
+ if (legacy !== -1) {
167
+ settings.hooks.SessionStart.splice(legacy, 1);
168
+ done.push(`${rel(settingsPath)}: replaced the network-reaching npx hook with a local-then-PATH one`);
169
+ }
170
+ settings.hooks.SessionStart.push({ hooks: [{ type: 'command', command: cmd }] });
54
171
  mkdirSync(dirname(settingsPath), { recursive: true });
55
172
  writeFileSync(settingsPath, JSON.stringify(settings, null, 2) + '\n');
56
- done.push('.claude/settings.json: SessionStart hook → brief --text');
173
+ done.push(`${rel(settingsPath)}: SessionStart hook → brief --text${dir === '.' ? '' : ` ${dir}`} (local install first, npx --no-install fallback, never a fetch)`);
174
+ note(settingsPath);
57
175
  }
58
176
 
59
- const agentsPath = join(projectDir, 'AGENTS.md');
177
+ const agentsPath = join(agentRoot, 'AGENTS.md');
60
178
  const agents = existsSync(agentsPath) ? readFileSync(agentsPath, 'utf8') : '';
61
- if (agents.includes(AGENTS_BEGIN)) {
62
- skipped.push('AGENTS.md already has the TestGuard section');
179
+ const begin = agents.indexOf(AGENTS_BEGIN);
180
+ const end = agents.indexOf(AGENTS_END);
181
+ if (begin !== -1 && end !== -1) {
182
+ const block = agents.slice(begin, end + AGENTS_END.length);
183
+ const entries = [...block.matchAll(/^- (?:This project|`([^`]+)\/`)/gm)].map((m) => m[1] ?? '.');
184
+ const legacyBlock = entries.length === 0; // a pre-0.6 section with no project list
185
+ if (entries.includes(dir)) {
186
+ skipped.push(`AGENTS.md already has the TestGuard section for ${dir === '.' ? 'this project' : dir}`);
187
+ } else {
188
+ writeFileSync(agentsPath, agents.slice(0, begin) + agentsBlock([...entries, dir]) + agents.slice(end + AGENTS_END.length));
189
+ done.push(legacyBlock ? `AGENTS.md: TestGuard section rewritten to list ${dir}` : `AGENTS.md: TestGuard section now lists ${dir}`);
190
+ note(agentsPath);
191
+ }
63
192
  } else {
64
- writeFileSync(agentsPath, (agents ? agents.replace(/\s*$/, '\n\n') : '# Agent instructions\n\n') + AGENTS_BLOCK + '\n');
193
+ writeFileSync(agentsPath, (agents ? agents.replace(/\s*$/, '\n\n') : '# Agent instructions\n\n') + agentsBlock([dir]) + '\n');
65
194
  done.push(agents ? 'AGENTS.md: TestGuard section appended' : 'AGENTS.md created with the TestGuard section');
195
+ note(agentsPath);
66
196
  }
67
197
 
68
198
  const giPath = join(projectDir, '.gitignore');
@@ -71,12 +201,35 @@ export function initProject({ projectDir, force = false }) {
71
201
  const dirIgnored = entries.some((x) => /^\/?\.testguard\/?$/.test(x));
72
202
  const missing = dirIgnored ? [] : GITIGNORE_LINES.filter((l) => !entries.includes(l));
73
203
  if (dirIgnored) {
74
- skipped.push('.gitignore ignores .testguard/ entirely — note that baseline.json should be committed; ignore only the regenerated files if you adopt a baseline');
204
+ skipped.push(`${rel(giPath)} ignores .testguard/ entirely — note that baseline.json should be committed; ignore only the regenerated files if you adopt a baseline`);
75
205
  } else if (missing.length) {
76
206
  writeFileSync(giPath, (gi ? gi.replace(/\s*$/, '\n') : '') + '# TestGuard: regenerated per run (baseline.json IS committed)\n' + missing.join('\n') + '\n');
77
- done.push(`.gitignore: ${missing.length} line${missing.length === 1 ? '' : 's'} added`);
207
+ done.push(`${rel(giPath)}: ${missing.length} line${missing.length === 1 ? '' : 's'} added`);
78
208
  } else {
79
- skipped.push('.gitignore already ignores the regenerated files');
209
+ skipped.push(`${rel(giPath)} already ignores the regenerated files`);
210
+ }
211
+ if (ciEvidence) {
212
+ const helperPath = join(projectDir, '.testguard', 'fetch-ci-evidence.sh');
213
+ const body = CI_EVIDENCE_HELPERS[ciEvidence];
214
+ if (!body) throw new Error(`--ci-evidence must be github or gitlab, not ${ciEvidence}`);
215
+ if (!existsSync(helperPath) || force) {
216
+ mkdirSync(dirname(helperPath), { recursive: true });
217
+ writeFileSync(helperPath, body, { mode: 0o755 });
218
+ done.push(`.testguard/fetch-ci-evidence.sh (${ciEvidence}) — run it on demand; the session-start hook stays offline`);
219
+ } else {
220
+ skipped.push('.testguard/fetch-ci-evidence.sh exists (use --force to replace)');
221
+ }
222
+ }
223
+ if (mcp) {
224
+ // Printed, never written: a harness's own config is the person's file
225
+ // (and often global), so init shows the snippet and lets them place it.
226
+ const cmd = { command: 'npx', args: ['-y', 'testguard-cli', 'mcp'] };
227
+ mcpConfig = {
228
+ 'Claude Code — .mcp.json in the project root, or `claude mcp add`': { mcpServers: { testguard: cmd } },
229
+ 'Cursor — .cursor/mcp.json': { mcpServers: { testguard: cmd } },
230
+ 'Codex CLI — ~/.codex/config.toml': '[mcp_servers.testguard]\ncommand = "npx"\nargs = ["-y", "testguard-cli", "mcp"]',
231
+ };
232
+ notes.push('MCP: five read-only tools (status, brief, claims, evidence, next_command). Nothing there runs a probe.');
80
233
  }
81
- return { done, skipped };
234
+ return { done, skipped, warnings, agentRoot, projectDir, dir, ...(mcpConfig ? { mcpConfig } : {}) };
82
235
  }
@@ -21,13 +21,31 @@ which files exist. `state` is one of:
21
21
  | state | meaning | you do |
22
22
  |---|---|---|
23
23
  | `no-claims` | no `testguard.claims.json` | `testguard scaffold <file>` for a file with guards; replace every `TODO:` statement with what the code guarantees; keep or drop each proposal; move the claims into `testguard.claims.json` |
24
+ | `unclaimed-changes` | files you changed carry no claim (only when a reference is known: `--changed <ref>`, or CI's base branch) | `next.file` names the first; `testguard scaffold <file>` and state the claim, or add a `testguard.ignore.json` path entry with a reason a reviewer will accept. **Before** writing more code. |
24
25
  | `unprobed` | claims never probed | `testguard probe` |
25
26
  | `evidence-stale` | code, tests or claims changed since the evidence | `testguard probe --include-dirty` (or `--claim <ID>` for one) |
26
27
  | `provisional-only` | only `--confirm 1` evidence exists | `testguard probe --confirm 3` |
27
28
  | `unproven` | a finding is not covered by the baseline | `next.target` names it; see the verdict table |
28
29
  | `clean` | everything killed or baselined | `testguard baseline` if `next` says so; otherwise nothing |
29
30
 
30
- Exit codes: `0` clean · `1` unproven claims (or drift) · `2` precondition failed / nothing to do yet · `3` usage.
31
+ Exit codes: `0` clean · `1` unproven claims (or drift, or unclaimed changes) · `2` precondition failed / nothing to do yet · `3` usage.
32
+
33
+ ## Every change needs a claim
34
+
35
+ `probe` verifies only the claims that exist; it is silent about unclaimed
36
+ code by construction. So, for every source file you create or change:
37
+
38
+ 1. `testguard gate --changed HEAD --include-dirty` (the pre-commit shape; in a
39
+ PR, `--changed origin/<base>`). One unclaimed file exits `1`.
40
+ 2. For each `UNCLAIMED` file: `testguard scaffold <file>` proposes faults;
41
+ state what the code guarantees; add the claim, its faults and its
42
+ `defendedBy` to `testguard.claims.json`. A new test file must be named in
43
+ some claim's `defendedBy`, or it is unclaimed too.
44
+ 3. Only when a file genuinely carries nothing to claim (generated code, a
45
+ thin wrapper whose logic is claimed elsewhere): a `path` entry in
46
+ `testguard.ignore.json` with a reason and, where possible, an `expires`.
47
+ The gate prints every entry it relied on; a reviewer reads them.
48
+ 4. Then `testguard admit <test-file> --claim <ID>` for the new claim's test (or `testguard probe --claim <ID> --include-dirty` for the full report).
31
49
 
32
50
  ## Verdict → action
33
51
 
@@ -45,8 +63,8 @@ Exit codes: `0` clean · `1` unproven claims (or drift) · `2` precondition fail
45
63
 
46
64
  1. `testguard status --json` → take `next.target`.
47
65
  2. Write the test. It must **fail when the fault is applied and pass on HEAD**. To check the first half by hand: apply `find` → `replace` in the target file, run the defender, restore.
48
- 3. `testguard probe --claim <ID> --include-dirty` — probes your uncommitted test without touching the tree. `--confirm 1` gives a fast **provisional** signal; a `?` on a verdict means unconfirmed.
49
- 4. When it is `killed` at `--confirm 3`, commit the test.
66
+ 3. `testguard admit <test-file> --claim <ID>` — the two gates as one verb, on your uncommitted test, without touching the tree: `ADMITTED` (exit 0) means the test is green on HEAD and fails on every fault of the claim, 3/3; `NOT ADMITTED` (exit 1) names the first blocking fault and what to do. `--fault <FID>` judges one fault; `--confirm 1` gives a fast **provisional** `ADMITTED?` that must be confirmed at 3 before you commit. (`testguard probe --claim <ID> --include-dirty` is the same run with the full report.)
67
+ 4. When it is `ADMITTED` at `--confirm 3`, commit the test.
50
68
  5. When everything is killed or baselined, `testguard baseline` if `next` says so, and commit `.testguard/baseline.json`.
51
69
 
52
70
  ## Rules that are not yours to bend
@@ -0,0 +1,104 @@
1
+ import { TOOLS } from './tools.mjs';
2
+
3
+ /**
4
+ * A Model Context Protocol server over stdio, hand-rolled.
5
+ *
6
+ * Why not the official SDK: this tool has one exact-pinned runtime dependency
7
+ * and intends to keep it that way (AGENTS.md rule 8). The surface here is
8
+ * three methods and five read-only tools and will not grow, so the protocol
9
+ * is written out rather than imported. The version is pinned below and the
10
+ * conformance is covered by tests that speak the wire format.
11
+ *
12
+ * JSON-RPC 2.0 framing: one message per line (newline-delimited JSON), which
13
+ * is what stdio transports use. A notification (no `id`) is never answered.
14
+ */
15
+ export const PROTOCOL_VERSION = '2025-06-18';
16
+
17
+ const ERROR = { parse: -32700, invalidRequest: -32600, methodNotFound: -32601, invalidParams: -32602, internal: -32603 };
18
+
19
+ const ok = (id, result) => ({ jsonrpc: '2.0', id, result });
20
+ const err = (id, code, message, data) => ({ jsonrpc: '2.0', id, error: { code, message, ...(data ? { data } : {}) } });
21
+
22
+ /** Everything an agent may ask for, with the version stamped onto each call. */
23
+ export function handle(message, { version = '0.0.0' } = {}) {
24
+ if (message === null || typeof message !== 'object' || Array.isArray(message)) return err(null, ERROR.invalidRequest, 'a request must be a JSON object');
25
+ const { id, method, params } = message;
26
+ const isNotification = id === undefined || id === null;
27
+
28
+ switch (method) {
29
+ case 'initialize':
30
+ return isNotification ? null : ok(id, {
31
+ protocolVersion: PROTOCOL_VERSION,
32
+ // Read-only: no prompts, no resources, no sampling. Saying so is part
33
+ // of the contract — nothing here can change a claims file.
34
+ capabilities: { tools: { listChanged: false } },
35
+ serverInfo: { name: 'testguard', version },
36
+ instructions: 'Read testguard_status first: it carries the one next action. Run the command it names in your own terminal — these tools never run a probe, because a probe is long-running, budgeted, and the person should see it happen.',
37
+ });
38
+
39
+ case 'notifications/initialized':
40
+ case 'notifications/cancelled':
41
+ return null;
42
+
43
+ case 'ping':
44
+ return isNotification ? null : ok(id, {});
45
+
46
+ case 'tools/list':
47
+ return isNotification ? null : ok(id, { tools: TOOLS.map(({ name, description, inputSchema }) => ({ name, description, inputSchema, annotations: { readOnlyHint: true, destructiveHint: false, idempotentHint: true, openWorldHint: false } })) });
48
+
49
+ case 'tools/call': {
50
+ if (isNotification) return null;
51
+ const tool = TOOLS.find((t) => t.name === params?.name);
52
+ if (!tool) return err(id, ERROR.invalidParams, `unknown tool: ${params?.name}`);
53
+ try {
54
+ const result = tool.handler({ ...(params?.arguments ?? {}), __version: version });
55
+ return ok(id, { content: [{ type: 'text', text: JSON.stringify(result, null, 2) }], structuredContent: result, isError: false });
56
+ } catch (e) {
57
+ // A tool failure is a tool result, not a protocol error: the agent
58
+ // should read the reason and act, not lose the connection.
59
+ return ok(id, { content: [{ type: 'text', text: `${tool.name} failed: ${e.message}` }], isError: true });
60
+ }
61
+ }
62
+
63
+ default:
64
+ return isNotification ? null : err(id, ERROR.methodNotFound, `unknown method: ${method}`);
65
+ }
66
+ }
67
+
68
+ /**
69
+ * Speak the protocol on a pair of streams. Resolves when input ends, so the
70
+ * caller can await a clean shutdown.
71
+ */
72
+ export function serve({ input = process.stdin, output = process.stdout, version = '0.0.0' } = {}) {
73
+ return new Promise((resolveDone) => {
74
+ let buffer = '';
75
+ const write = (msg) => {
76
+ if (msg) output.write(JSON.stringify(msg) + '\n');
77
+ };
78
+ input.setEncoding?.('utf8');
79
+ input.on('data', (chunk) => {
80
+ buffer += chunk;
81
+ let nl;
82
+ while ((nl = buffer.indexOf('\n')) !== -1) {
83
+ const line = buffer.slice(0, nl).trim();
84
+ buffer = buffer.slice(nl + 1);
85
+ if (!line) continue;
86
+ let message;
87
+ try {
88
+ message = JSON.parse(line);
89
+ } catch {
90
+ write(err(null, ERROR.parse, 'invalid JSON'));
91
+ continue;
92
+ }
93
+ try {
94
+ write(handle(message, { version }));
95
+ } catch (e) {
96
+ // The server outlives any single bad request.
97
+ write(err(message?.id ?? null, ERROR.internal, e.message));
98
+ }
99
+ }
100
+ });
101
+ input.on('end', () => resolveDone(0));
102
+ input.on('close', () => resolveDone(0));
103
+ });
104
+ }
@@ -0,0 +1,114 @@
1
+ import { existsSync } from 'node:fs';
2
+ import { join, resolve } from 'node:path';
3
+ import { computeStatus } from '../status/status.mjs';
4
+ import { buildBrief, buildUnclaimedBrief } from '../brief/brief.mjs';
5
+ import { loadClaims, defaultClaimsPath } from '../claims/load.mjs';
6
+ import { scanAnnotations, reconcile } from '../claims/annotations.mjs';
7
+ import { computeRemovedClaims } from '../claims/removed.mjs';
8
+ import { readSpecDoc } from '../evidence/writer.mjs';
9
+ import { resolveChangedRef, withChangedRef } from '../gate/changed.mjs';
10
+
11
+ /**
12
+ * The read-only surface an agent needs, in any harness.
13
+ *
14
+ * TestGuard's operating loop currently lives in a Claude Code skill and a
15
+ * session-start hook, and both vanish the moment the harness is something
16
+ * else. These tools expose the documents the tool already computes, so the
17
+ * loop survives that change.
18
+ *
19
+ * Every tool is read-only and none of them probes. A probe is long-running
20
+ * and budgeted, and the human should see it happen: `next_command` hands back
21
+ * the exact shell line for the agent to run in its own terminal instead.
22
+ */
23
+ const projectOf = (args) => resolve(args?.dir ?? '.');
24
+ const evidenceOf = (args) => (args?.evidence ? resolve(args.evidence) : undefined);
25
+
26
+ const DIR = { type: 'string', description: 'Project directory (where testguard.claims.json lives). Defaults to the current directory.' };
27
+ const EVIDENCE = { type: 'string', description: "Read this evidence document instead of the project's own — CI's, fetched as an artifact." };
28
+ const CHANGED = { type: 'string', description: 'A git reference to measure the change against, so claim coverage of the change is included.' };
29
+
30
+ const statusFor = (args) => {
31
+ const projectDir = projectOf(args);
32
+ return withChangedRef(resolveChangedRef({ explicit: args?.changed }), (changedRef) =>
33
+ computeStatus({ projectDir, toolVersion: args?.__version ?? '0.0.0', changedRef, evidence: evidenceOf(args) }));
34
+ };
35
+
36
+ export const TOOLS = [
37
+ {
38
+ name: 'testguard_status',
39
+ description: 'Where this project stands and the ONE next action, as the machine-readable status document: state, next {action, command, why}, counts, stale inputs, faults edited since they were probed, and the ranked findings. This is the single source of truth every other rendering derives from — read it before deciding anything.',
40
+ inputSchema: { type: 'object', properties: { dir: DIR, changed: CHANGED, evidence: EVIDENCE }, additionalProperties: false },
41
+ handler: (args) => statusFor(args),
42
+ },
43
+ {
44
+ name: 'testguard_brief',
45
+ description: 'Where the test suite is blind, ranked and capped, with one actionable hint per finding and unclaimed changed files first. Read this before writing or changing code, which is what the session-start hook does for harnesses that have one.',
46
+ inputSchema: { type: 'object', properties: { dir: DIR, changed: CHANGED, evidence: EVIDENCE, max: { type: 'integer', minimum: 1, maximum: 50 } }, additionalProperties: false },
47
+ handler: (args) => {
48
+ const projectDir = projectOf(args);
49
+ const evPath = evidenceOf(args) ?? join(projectDir, '.testguard', 'evidence.json');
50
+ const status = (() => {
51
+ try {
52
+ return statusFor(args);
53
+ } catch {
54
+ return undefined; // a brief must never fail because status could not be computed
55
+ }
56
+ })();
57
+ if (!existsSync(evPath)) {
58
+ if (status?.changes?.uncovered?.length) {
59
+ return buildUnclaimedBrief({ tool: { name: 'testguard', version: args?.__version ?? '0.0.0' }, next: status.next, changes: status.changes });
60
+ }
61
+ return { note: `no evidence at ${evPath}: nothing has been probed yet`, next: status?.next };
62
+ }
63
+ const evidence = readSpecDoc('evidence', evPath);
64
+ const basePath = join(projectDir, '.testguard', 'baseline.json');
65
+ const baseline = existsSync(basePath) ? readSpecDoc('baseline', basePath) : undefined;
66
+ return buildBrief(evidence, baseline, { max: args?.max ?? 20, next: status?.next, changes: status?.changes });
67
+ },
68
+ },
69
+ {
70
+ name: 'testguard_claims',
71
+ description: "What this project claims must be true, each with its faults and defenders, plus drift against `@claim` annotations in source and — when `since` is given — every claim or fault that existed at that reference and does not now. A claim that disappeared is invisible to every other check.",
72
+ inputSchema: { type: 'object', properties: { dir: DIR, since: { type: 'string', description: 'A git reference: report claims and faults removed since it.' } }, additionalProperties: false },
73
+ handler: (args) => {
74
+ const projectDir = projectOf(args);
75
+ const path = defaultClaimsPath(projectDir);
76
+ if (!existsSync(path)) return { path, claims: [], note: 'no claims file in this project yet' };
77
+ const claims = loadClaims(path);
78
+ const drift = reconcile(claims, scanAnnotations(projectDir));
79
+ const removed = args?.since
80
+ ? computeRemovedClaims({ projectDir, ref: args.since, claimsPath: path, current: claims, evidencePath: join(projectDir, '.testguard', 'evidence.json'), toolVersion: args?.__version ?? '0.0.0' })
81
+ : undefined;
82
+ return { path, claims, drift, ...(removed ? { removed } : {}) };
83
+ },
84
+ },
85
+ {
86
+ name: 'testguard_evidence',
87
+ description: 'One run\'s findings on disk: every verdict with the runs that produced it. Pass `claim` (and optionally `fault`) for a single record instead of the whole document, which can be large.',
88
+ inputSchema: {
89
+ type: 'object',
90
+ properties: { dir: DIR, evidence: EVIDENCE, claim: { type: 'string', description: 'Return only this claim\'s records.' }, fault: { type: 'string', description: 'With `claim`: only this fault\'s record.' } },
91
+ additionalProperties: false,
92
+ },
93
+ handler: (args) => {
94
+ const projectDir = projectOf(args);
95
+ const evPath = evidenceOf(args) ?? join(projectDir, '.testguard', 'evidence.json');
96
+ if (!existsSync(evPath)) return { note: `no evidence at ${evPath}: run a probe first`, path: evPath };
97
+ const doc = readSpecDoc('evidence', evPath);
98
+ if (!args?.claim) return doc;
99
+ const records = doc.records.filter((r) => r.claim.id === args.claim && (!args.fault || r.subject.id === args.fault));
100
+ return { ...doc, records, filtered: { claim: args.claim, ...(args.fault ? { fault: args.fault } : {}) } };
101
+ },
102
+ },
103
+ {
104
+ name: 'testguard_next_command',
105
+ description: 'The exact shell command to run next, with the reason. Run it yourself in a terminal: a probe is long-running and budgeted, and the person should see it happen. This tool never runs anything.',
106
+ inputSchema: { type: 'object', properties: { dir: DIR, changed: CHANGED, evidence: EVIDENCE }, additionalProperties: false },
107
+ handler: (args) => {
108
+ const doc = statusFor(args);
109
+ return { state: doc.state, action: doc.next.action, command: doc.next.command, why: doc.next.why, ...(doc.next.target ? { target: doc.next.target } : {}), ...(doc.next.file ? { file: doc.next.file } : {}) };
110
+ },
111
+ },
112
+ ];
113
+
114
+ export const TOOL_NAMES = TOOLS.map((t) => t.name);
@@ -0,0 +1,60 @@
1
+ import { spawnSync } from 'node:child_process';
2
+
3
+ /**
4
+ * Concurrent test runners (issue #26).
5
+ *
6
+ * A probe runs the defenders 3× clean and 3× with each fault applied. Heavy
7
+ * suites time out under parallel CPU load, and a timeout is a verdict —
8
+ * `TIMEOUT` or `FLAKY-DEFENDER` — that says nothing about the claim. A field
9
+ * report serialised everything by hand for exactly this reason. We cannot
10
+ * prevent the contention, but we can name it: warn before the first run, and
11
+ * record it on the evidence so a later reader of a flaky verdict knows the
12
+ * run was contended.
13
+ *
14
+ * Best effort by design: the detector never fails a probe, and a process list
15
+ * it cannot read is simply no detection.
16
+ */
17
+
18
+ const RUNNER_RE = /\b(vitest|jest|playwright|mocha|ava|karma|cypress)\b/i;
19
+ // Our own child processes and this process are not contention.
20
+ const SELF_RE = /testguard/i;
21
+
22
+ /** Rows of the process table as `{ pid, command }`, or [] when it cannot be read. */
23
+ export function processList({ platform = process.platform, run = spawnSync } = {}) {
24
+ try {
25
+ const r = platform === 'win32'
26
+ ? run('wmic', ['process', 'get', 'ProcessId,CommandLine', '/format:csv'], { encoding: 'utf8', timeout: 5000 })
27
+ : run('ps', ['-Ao', 'pid=,args='], { encoding: 'utf8', timeout: 5000 });
28
+ if (r.status !== 0 || !r.stdout) return [];
29
+ return r.stdout.split('\n').map((line) => {
30
+ const m = platform === 'win32' ? /^[^,]*,(.*),(\d+)\s*$/.exec(line) : /^\s*(\d+)\s+(.*)$/.exec(line);
31
+ if (!m) return null;
32
+ const [pid, command] = platform === 'win32' ? [m[2], m[1]] : [m[1], m[2]];
33
+ return { pid: Number(pid), command: command.trim() };
34
+ }).filter(Boolean);
35
+ } catch {
36
+ return [];
37
+ }
38
+ }
39
+
40
+ /**
41
+ * Test runners already running, other than ours. Returns
42
+ * `{ detected, runners: [{ pid, command }] }`; `detected` is false when
43
+ * nothing was found or the process list could not be read.
44
+ */
45
+ export function detectContention({ self = process.pid, ...opts } = {}) {
46
+ const rows = processList(opts);
47
+ const runners = rows
48
+ .filter((p) => p.pid !== self && RUNNER_RE.test(p.command) && !SELF_RE.test(p.command))
49
+ // `ps` lists the grep/ps itself and any shell wrapper; a runner's command
50
+ // line always names its own binary, so require it to look like an exec.
51
+ .filter((p) => !/^\s*(ps|grep|wmic)\b/.test(p.command))
52
+ .map((p) => ({ pid: p.pid, command: p.command.length > 200 ? p.command.slice(0, 197) + '…' : p.command }));
53
+ return { detected: runners.length > 0, runners };
54
+ }
55
+
56
+ /** One line for the operator: what is running, and what it costs. */
57
+ export const contentionWarning = (c) =>
58
+ `${c.runners.length} test runner${c.runners.length === 1 ? ' is' : 's are'} already running (${c.runners.map((r) => `pid ${r.pid}`).join(', ')}); ` +
59
+ 'defenders run N times each here, and a suite that times out under parallel load yields TIMEOUT or FLAKY-DEFENDER — verdicts about the load, not the claim. ' +
60
+ 'Wait for it to finish, or pass --serial to run one test file at a time.';
@@ -1,12 +1,21 @@
1
1
  import { join } from 'node:path';
2
2
  import { listTestFiles } from './runners/shared.mjs';
3
3
  import { fileImports } from './rank.mjs';
4
+ import { classifyDefenders } from './mocks.mjs';
4
5
 
5
6
  /**
6
7
  * When a claim declares no defenders: the test files that import the fault's
7
- * target file, directly, by relative path or through a resolved alias. This
8
- * is what `nocover` measures against — no test file even imports the source.
8
+ * target file — directly, by relative path or through a resolved alias — and
9
+ * do NOT mock it. A file that mocks the target cannot detect any fault in it;
10
+ * counting it would make `nocover` under-report and waste probe runs on files
11
+ * that cannot fail. `nocover` therefore means exactly "no test file imports
12
+ * this source without mocking it".
9
13
  */
14
+ export function discoverDefendersDetailed(projectDir, targetRel) {
15
+ const importing = listTestFiles(projectDir).filter((t) => fileImports(projectDir, join(projectDir, t), targetRel));
16
+ return { importing, ...classifyDefenders(projectDir, targetRel, importing) };
17
+ }
18
+
10
19
  export function discoverDefenders(projectDir, targetRel) {
11
- return listTestFiles(projectDir).filter((t) => fileImports(projectDir, join(projectDir, t), targetRel));
20
+ return discoverDefendersDetailed(projectDir, targetRel).canDetect;
12
21
  }