testguard-cli 0.5.0 → 0.7.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (65) hide show
  1. package/CHANGELOG.md +628 -0
  2. package/README.md +536 -42
  3. package/package.json +2 -2
  4. package/spec/GATE-SEMANTICS.md +166 -2
  5. package/spec/README.md +29 -10
  6. package/spec/lib/validate.mjs +140 -8
  7. package/spec/schemas/baseline.schema.json +13 -5
  8. package/spec/schemas/brief.schema.json +12 -12
  9. package/spec/schemas/calibration.schema.json +5 -5
  10. package/spec/schemas/claims.schema.json +122 -25
  11. package/spec/schemas/common.schema.json +131 -29
  12. package/spec/schemas/evidence.schema.json +476 -69
  13. package/spec/schemas/gate.schema.json +12 -12
  14. package/spec/schemas/ignore.schema.json +4 -4
  15. package/spec/schemas/replay.schema.json +85 -0
  16. package/spec/schemas/status.schema.json +258 -56
  17. package/src/admit/admit.mjs +17 -0
  18. package/src/baseline/baseline.mjs +31 -2
  19. package/src/brief/brief.mjs +71 -7
  20. package/src/claims/removed.mjs +161 -0
  21. package/src/cli.mjs +68 -10
  22. package/src/commands/admit.mjs +116 -0
  23. package/src/commands/baseline.mjs +17 -3
  24. package/src/commands/brief.mjs +13 -7
  25. package/src/commands/claims.mjs +55 -6
  26. package/src/commands/init.mjs +19 -6
  27. package/src/commands/mcp.mjs +12 -0
  28. package/src/commands/probe.mjs +35 -7
  29. package/src/commands/replay.mjs +63 -0
  30. package/src/commands/status.mjs +3 -1
  31. package/src/gate/changed.mjs +9 -2
  32. package/src/git.mjs +5 -0
  33. package/src/init/init.mjs +182 -29
  34. package/src/init/templates/SKILL.md +3 -3
  35. package/src/mcp/server.mjs +104 -0
  36. package/src/mcp/tools.mjs +114 -0
  37. package/src/probe/attribution.mjs +123 -0
  38. package/src/probe/classify.mjs +18 -2
  39. package/src/probe/contention.mjs +60 -0
  40. package/src/probe/cost.mjs +145 -0
  41. package/src/probe/discover.mjs +20 -3
  42. package/src/probe/independence.mjs +59 -0
  43. package/src/probe/inject.mjs +41 -9
  44. package/src/probe/mocks.mjs +122 -0
  45. package/src/probe/probe.mjs +331 -40
  46. package/src/probe/progress.mjs +87 -0
  47. package/src/probe/pyimports.mjs +287 -0
  48. package/src/probe/rank.mjs +75 -15
  49. package/src/probe/runners/index.mjs +79 -8
  50. package/src/probe/runners/jest.mjs +15 -3
  51. package/src/probe/runners/playwright.mjs +142 -0
  52. package/src/probe/runners/python/_testguard_pytest_plugin.py +132 -0
  53. package/src/probe/runners/python/_testguard_report.py +141 -0
  54. package/src/probe/runners/python/_testguard_unittest_main.py +168 -0
  55. package/src/probe/runners/python.mjs +316 -0
  56. package/src/probe/runners/shared.mjs +80 -14
  57. package/src/probe/runners/vitest.mjs +16 -3
  58. package/src/render.mjs +2 -2
  59. package/src/replay/label.mjs +64 -0
  60. package/src/replay/replay.mjs +310 -0
  61. package/src/scaffold/producers.mjs +217 -6
  62. package/src/scaffold/producers.python.mjs +209 -0
  63. package/src/scaffold/scaffold.mjs +90 -30
  64. package/src/status/status.mjs +47 -6
  65. package/src/util/glob.mjs +7 -1
package/src/init/init.mjs CHANGED
@@ -1,68 +1,198 @@
1
- import { existsSync, mkdirSync, readFileSync, writeFileSync } from 'node:fs';
2
- import { join, dirname } from 'node:path';
1
+ import { existsSync, mkdirSync, readFileSync, writeFileSync, realpathSync } from 'node:fs';
2
+ import { join, dirname, relative, resolve, sep } from 'node:path';
3
3
  import { fileURLToPath } from 'node:url';
4
+ import { spawnSync } from 'node:child_process';
4
5
 
5
6
  const TEMPLATES = join(dirname(fileURLToPath(import.meta.url)), 'templates');
6
- const HOOK_CMD = 'npx -y testguard-cli brief --text';
7
7
  const AGENTS_BEGIN = '<!-- testguard:begin -->';
8
8
  const AGENTS_END = '<!-- testguard:end -->';
9
- const AGENTS_BLOCK = `${AGENTS_BEGIN}
9
+ const GITIGNORE_LINES = ['.testguard/evidence.json', '.testguard/evidence-provisional.json', '.testguard/evidence-partial.json', '.testguard/brief.json', '.testguard/gate.json', '.testguard/scaffold-*.json'];
10
+
11
+ /**
12
+ * The session-start hook command. It runs from the git root (where the agent
13
+ * session lives) and must never fetch from the network: a local install of
14
+ * the project (or of the root) is preferred, then a `testguard` already on
15
+ * PATH, then nothing — the hook exits 0 with no output rather than break a
16
+ * session.
17
+ *
18
+ * No form of `npx` appears here, and `--no-install` is not an exception.
19
+ * Measured: in a project with nothing installed,
20
+ * `npx --no-install --loglevel=http testguard-cli --version` logs
21
+ * `npm http fetch GET 200 https://registry.npmjs.org/testguard-cli`, and
22
+ * against an unreachable registry it exits non-zero. npm resolves the
23
+ * packument before it decides not to install, so `--no-install` is quiet, not
24
+ * offline. A `command -v` lookup covers the global-install case with no
25
+ * network at all, which is the whole point in a repository that pins and
26
+ * audits its dependencies.
27
+ */
28
+ export function hookCommand(dir) {
29
+ const arg = dir === '.' ? '' : ` ${dir}`;
30
+ const candidates = dir === '.' ? ['node_modules/.bin/testguard'] : [`${dir}/node_modules/.bin/testguard`, 'node_modules/.bin/testguard'];
31
+ // The PATH branch is braced: `a || b && c` binds as `(a || b) && c` in sh,
32
+ // so an unbraced `&&` would run the brief a second time whenever the local
33
+ // binary succeeded.
34
+ const onPath = `{ command -v testguard >/dev/null 2>&1 && testguard brief --text${arg} 2>/dev/null; }`;
35
+ return [...candidates.map((c) => `${c} brief --text${arg} 2>/dev/null`), onPath, 'true'].join(' || ');
36
+ }
37
+
38
+ function agentsBlock(entries) {
39
+ const lines = entries.map((dir) => (dir === '.'
40
+ ? '- This project: claims in `testguard.claims.json`; run `testguard status --json` and follow `next`.'
41
+ : `- \`${dir}/\`: claims in \`${dir}/testguard.claims.json\`; run \`testguard status --json ${dir}\` and follow \`next\`.`));
42
+ return `${AGENTS_BEGIN}
10
43
  ## TestGuard
11
44
 
12
- This project's tests are verified by [TestGuard](https://github.com/raccioly/testguard).
13
- Before writing or changing tests, run \`testguard status --json\` and follow \`next\`.
45
+ This repository's tests are verified by [TestGuard](https://github.com/raccioly/testguard).
46
+ Before writing or changing tests, run \`testguard status --json [dir]\` and follow \`next\`.
14
47
  The full operating loop and the verdict table are in \`.claude/skills/testguard/SKILL.md\`.
15
48
  Never make a fault die by editing \`testguard.claims.json\`; write the test. Claim edits are recorded in the evidence.
49
+
50
+ ${lines.join('\n')}
16
51
  ${AGENTS_END}`;
17
- const GITIGNORE_LINES = ['.testguard/evidence.json', '.testguard/evidence-provisional.json', '.testguard/evidence-partial.json', '.testguard/brief.json', '.testguard/gate.json', '.testguard/scaffold-*.json'];
52
+ }
53
+
54
+ /** The git root of `dir`, or null outside a repository. */
55
+ export function gitRoot(dir) {
56
+ const r = spawnSync('git', ['rev-parse', '--show-toplevel'], { cwd: dir, encoding: 'utf8' });
57
+ return r.status === 0 ? r.stdout.trim() : null;
58
+ }
59
+
60
+ /** Is `file` (absolute) ignored by git in `root`? False outside a repository. */
61
+ export function isGitIgnored(root, file) {
62
+ if (!root) return false;
63
+ return spawnSync('git', ['check-ignore', '-q', '--', file], { cwd: root }).status === 0;
64
+ }
18
65
 
19
66
  /**
20
- * Install the agent operating layer into a consumer project: the skill, the
21
- * session-start hook, an AGENTS.md section and the .gitignore lines.
22
- * Idempotent: re-running changes nothing unless --force replaces the skill.
67
+ * Install the operating layer.
68
+ *
69
+ * Two layers, two places. The AGENT layer — the skill, the session-start
70
+ * hook, the AGENTS.md section — goes to the git root, because that is where
71
+ * an agent session runs; installed in a subdirectory it never fires and is
72
+ * never read (a field report installed it in a backend/ folder and got
73
+ * nothing). The PROJECT layer — the `.gitignore` lines beside the claims
74
+ * file — stays in the project directory. `here: true` keeps everything in
75
+ * the project directory for a subdirectory that is its own agent root.
76
+ *
77
+ * Idempotent per project: a second project in the same repository adds a
78
+ * hook line and an AGENTS.md bullet, never duplicates. Every written file is
79
+ * checked against .gitignore; an ignored file is reported as ignored, never
80
+ * as something to commit.
23
81
  */
24
- export function initProject({ projectDir, force = false }) {
82
+ const CI_EVIDENCE_HELPERS = {
83
+ github: `#!/bin/sh
84
+ # Fetch the evidence CI wrote on a branch, then brief from it. Run it when you
85
+ # want it: the session-start hook never touches the network, and this script
86
+ # exits 0 with a message whenever anything is missing.
87
+ BRANCH="\${1:-main}"
88
+ OUT=.testguard/ci
89
+ command -v gh >/dev/null 2>&1 || { echo "gh is not installed; see README > Read CI's evidence locally" >&2; exit 0; }
90
+ command -v testguard >/dev/null 2>&1 || { echo "testguard is not on PATH; install it or run npx testguard-cli brief --evidence <file>" >&2; exit 0; }
91
+ mkdir -p "$OUT" || exit 0
92
+ gh run download --name "testguard-evidence-$BRANCH" --dir "$OUT" >/dev/null 2>&1 || { echo "no testguard-evidence-$BRANCH artifact to download" >&2; exit 0; }
93
+ exec testguard brief . --text --evidence "$OUT/ci-self-evidence.json"
94
+ `,
95
+ gitlab: `#!/bin/sh
96
+ # Fetch the evidence CI wrote on a branch, then brief from it. Run it when you
97
+ # want it: the session-start hook never touches the network, and this script
98
+ # exits 0 with a message whenever anything is missing.
99
+ BRANCH="\${1:-main}"
100
+ OUT=.testguard/ci
101
+ command -v glab >/dev/null 2>&1 || { echo "glab is not installed; see README > Read CI's evidence locally" >&2; exit 0; }
102
+ command -v testguard >/dev/null 2>&1 || { echo "testguard is not on PATH; install it or run npx testguard-cli brief --evidence <file>" >&2; exit 0; }
103
+ mkdir -p "$OUT" || exit 0
104
+ glab ci artifact "$BRANCH" testguard:probe --path "$OUT/" >/dev/null 2>&1 || { echo "no testguard:probe artifact to download for $BRANCH" >&2; exit 0; }
105
+ exec testguard brief . --text --evidence "$OUT/.testguard/evidence.json"
106
+ `,
107
+ };
108
+
109
+ export function initProject({ projectDir, force = false, here = false, ciEvidence, mcp = false }) {
110
+ projectDir = realpathSync(resolve(projectDir));
111
+ const root = here ? null : gitRoot(projectDir);
112
+ const agentRoot = root ?? projectDir;
113
+ const ignoreRoot = root ?? gitRoot(projectDir);
114
+ const dir = (relative(agentRoot, projectDir).split(sep).join('/')) || '.';
25
115
  const done = [];
116
+ const notes = [];
117
+ let mcpConfig;
26
118
  const skipped = [];
119
+ const warnings = [];
120
+ const rel = (p) => relative(agentRoot, p).split(sep).join('/');
121
+ const note = (p) => { if (isGitIgnored(ignoreRoot, p)) warnings.push(`${rel(p)} is ignored by .gitignore — it was written, but git will not track it and agents in a fresh clone will never see it; unignore it or move it`); };
27
122
 
28
- const skillDir = join(projectDir, '.claude', 'skills', 'testguard');
123
+ const skillDir = join(agentRoot, '.claude', 'skills', 'testguard');
29
124
  const skillPath = join(skillDir, 'SKILL.md');
30
125
  if (!existsSync(skillPath) || force) {
126
+ const replaced = force && existsSync(skillPath);
31
127
  mkdirSync(skillDir, { recursive: true });
32
128
  writeFileSync(skillPath, readFileSync(join(TEMPLATES, 'SKILL.md'), 'utf8'));
33
- done.push(`.claude/skills/testguard/SKILL.md${force && existsSync(skillPath) ? ' (replaced)' : ''}`);
129
+ done.push(`${rel(skillPath)}${replaced ? ' (replaced)' : ''}`);
130
+ note(skillPath);
34
131
  } else {
35
- skipped.push('.claude/skills/testguard/SKILL.md exists (use --force to replace)');
132
+ skipped.push(`${rel(skillPath)} exists (use --force to replace)`);
36
133
  }
37
134
 
38
- const settingsPath = join(projectDir, '.claude', 'settings.json');
135
+ const settingsPath = join(agentRoot, '.claude', 'settings.json');
39
136
  let settings = {};
40
137
  if (existsSync(settingsPath)) {
41
138
  try {
42
139
  settings = JSON.parse(readFileSync(settingsPath, 'utf8'));
43
140
  } catch (e) {
44
- throw new Error(`.claude/settings.json is not valid JSON (${e.message}); fix it before init can add the hook`);
141
+ throw new Error(`${rel(settingsPath)} is not valid JSON (${e.message}); fix it before init can add the hook`);
45
142
  }
46
143
  }
47
144
  settings.hooks ??= {};
48
145
  settings.hooks.SessionStart ??= [];
49
- const already = JSON.stringify(settings.hooks.SessionStart).includes(HOOK_CMD);
50
- if (already) {
51
- skipped.push('SessionStart hook already present in .claude/settings.json');
146
+ const cmd = hookCommand(dir);
147
+ const hasCmd = settings.hooks.SessionStart.some((e) => JSON.stringify(e).includes(cmd));
148
+ // Any earlier hook that reached for npx — `-y` fetched outright, and
149
+ // `--no-install` still resolves the packument from the registry — is replaced.
150
+ // Which project a hook is for is the token right after `brief --text`, if
151
+ // that token is an argument rather than a redirect or a shell operator.
152
+ // Testing for any non-space there mistook `brief --text 2>/dev/null` for a
153
+ // hook belonging to a directory, so a legacy root hook was never replaced.
154
+ const hookDir = (text) => {
155
+ const m = /brief --text(?:\s+([^\s"|;&]+))?/.exec(text);
156
+ const token = m?.[1];
157
+ return !token || token.startsWith('2>') || token.startsWith('>') ? '.' : token;
158
+ };
159
+ const legacy = settings.hooks.SessionStart.findIndex((e) => {
160
+ const text = JSON.stringify(e);
161
+ return /npx\s+(-y|--no-install)[^"]*brief --text/.test(text) && hookDir(text) === dir;
162
+ });
163
+ if (hasCmd) {
164
+ skipped.push(`SessionStart hook for ${dir === '.' ? 'this project' : dir} already present in ${rel(settingsPath)}`);
52
165
  } else {
53
- settings.hooks.SessionStart.push({ hooks: [{ type: 'command', command: HOOK_CMD }] });
166
+ if (legacy !== -1) {
167
+ settings.hooks.SessionStart.splice(legacy, 1);
168
+ done.push(`${rel(settingsPath)}: replaced the network-reaching npx hook with a local-then-PATH one`);
169
+ }
170
+ settings.hooks.SessionStart.push({ hooks: [{ type: 'command', command: cmd }] });
54
171
  mkdirSync(dirname(settingsPath), { recursive: true });
55
172
  writeFileSync(settingsPath, JSON.stringify(settings, null, 2) + '\n');
56
- done.push('.claude/settings.json: SessionStart hook → brief --text');
173
+ done.push(`${rel(settingsPath)}: SessionStart hook → brief --text${dir === '.' ? '' : ` ${dir}`} (local install first, npx --no-install fallback, never a fetch)`);
174
+ note(settingsPath);
57
175
  }
58
176
 
59
- const agentsPath = join(projectDir, 'AGENTS.md');
177
+ const agentsPath = join(agentRoot, 'AGENTS.md');
60
178
  const agents = existsSync(agentsPath) ? readFileSync(agentsPath, 'utf8') : '';
61
- if (agents.includes(AGENTS_BEGIN)) {
62
- skipped.push('AGENTS.md already has the TestGuard section');
179
+ const begin = agents.indexOf(AGENTS_BEGIN);
180
+ const end = agents.indexOf(AGENTS_END);
181
+ if (begin !== -1 && end !== -1) {
182
+ const block = agents.slice(begin, end + AGENTS_END.length);
183
+ const entries = [...block.matchAll(/^- (?:This project|`([^`]+)\/`)/gm)].map((m) => m[1] ?? '.');
184
+ const legacyBlock = entries.length === 0; // a pre-0.6 section with no project list
185
+ if (entries.includes(dir)) {
186
+ skipped.push(`AGENTS.md already has the TestGuard section for ${dir === '.' ? 'this project' : dir}`);
187
+ } else {
188
+ writeFileSync(agentsPath, agents.slice(0, begin) + agentsBlock([...entries, dir]) + agents.slice(end + AGENTS_END.length));
189
+ done.push(legacyBlock ? `AGENTS.md: TestGuard section rewritten to list ${dir}` : `AGENTS.md: TestGuard section now lists ${dir}`);
190
+ note(agentsPath);
191
+ }
63
192
  } else {
64
- writeFileSync(agentsPath, (agents ? agents.replace(/\s*$/, '\n\n') : '# Agent instructions\n\n') + AGENTS_BLOCK + '\n');
193
+ writeFileSync(agentsPath, (agents ? agents.replace(/\s*$/, '\n\n') : '# Agent instructions\n\n') + agentsBlock([dir]) + '\n');
65
194
  done.push(agents ? 'AGENTS.md: TestGuard section appended' : 'AGENTS.md created with the TestGuard section');
195
+ note(agentsPath);
66
196
  }
67
197
 
68
198
  const giPath = join(projectDir, '.gitignore');
@@ -71,12 +201,35 @@ export function initProject({ projectDir, force = false }) {
71
201
  const dirIgnored = entries.some((x) => /^\/?\.testguard\/?$/.test(x));
72
202
  const missing = dirIgnored ? [] : GITIGNORE_LINES.filter((l) => !entries.includes(l));
73
203
  if (dirIgnored) {
74
- skipped.push('.gitignore ignores .testguard/ entirely — note that baseline.json should be committed; ignore only the regenerated files if you adopt a baseline');
204
+ skipped.push(`${rel(giPath)} ignores .testguard/ entirely — note that baseline.json should be committed; ignore only the regenerated files if you adopt a baseline`);
75
205
  } else if (missing.length) {
76
206
  writeFileSync(giPath, (gi ? gi.replace(/\s*$/, '\n') : '') + '# TestGuard: regenerated per run (baseline.json IS committed)\n' + missing.join('\n') + '\n');
77
- done.push(`.gitignore: ${missing.length} line${missing.length === 1 ? '' : 's'} added`);
207
+ done.push(`${rel(giPath)}: ${missing.length} line${missing.length === 1 ? '' : 's'} added`);
78
208
  } else {
79
- skipped.push('.gitignore already ignores the regenerated files');
209
+ skipped.push(`${rel(giPath)} already ignores the regenerated files`);
210
+ }
211
+ if (ciEvidence) {
212
+ const helperPath = join(projectDir, '.testguard', 'fetch-ci-evidence.sh');
213
+ const body = CI_EVIDENCE_HELPERS[ciEvidence];
214
+ if (!body) throw new Error(`--ci-evidence must be github or gitlab, not ${ciEvidence}`);
215
+ if (!existsSync(helperPath) || force) {
216
+ mkdirSync(dirname(helperPath), { recursive: true });
217
+ writeFileSync(helperPath, body, { mode: 0o755 });
218
+ done.push(`.testguard/fetch-ci-evidence.sh (${ciEvidence}) — run it on demand; the session-start hook stays offline`);
219
+ } else {
220
+ skipped.push('.testguard/fetch-ci-evidence.sh exists (use --force to replace)');
221
+ }
222
+ }
223
+ if (mcp) {
224
+ // Printed, never written: a harness's own config is the person's file
225
+ // (and often global), so init shows the snippet and lets them place it.
226
+ const cmd = { command: 'npx', args: ['-y', 'testguard-cli', 'mcp'] };
227
+ mcpConfig = {
228
+ 'Claude Code — .mcp.json in the project root, or `claude mcp add`': { mcpServers: { testguard: cmd } },
229
+ 'Cursor — .cursor/mcp.json': { mcpServers: { testguard: cmd } },
230
+ 'Codex CLI — ~/.codex/config.toml': '[mcp_servers.testguard]\ncommand = "npx"\nargs = ["-y", "testguard-cli", "mcp"]',
231
+ };
232
+ notes.push('MCP: five read-only tools (status, brief, claims, evidence, next_command). Nothing there runs a probe.');
80
233
  }
81
- return { done, skipped };
234
+ return { done, skipped, warnings, agentRoot, projectDir, dir, ...(mcpConfig ? { mcpConfig } : {}) };
82
235
  }
@@ -45,7 +45,7 @@ code by construction. So, for every source file you create or change:
45
45
  thin wrapper whose logic is claimed elsewhere): a `path` entry in
46
46
  `testguard.ignore.json` with a reason and, where possible, an `expires`.
47
47
  The gate prints every entry it relied on; a reviewer reads them.
48
- 4. Then `testguard probe --claim <ID> --include-dirty` for the new claim.
48
+ 4. Then `testguard admit <test-file> --claim <ID>` for the new claim's test (or `testguard probe --claim <ID> --include-dirty` for the full report).
49
49
 
50
50
  ## Verdict → action
51
51
 
@@ -63,8 +63,8 @@ code by construction. So, for every source file you create or change:
63
63
 
64
64
  1. `testguard status --json` → take `next.target`.
65
65
  2. Write the test. It must **fail when the fault is applied and pass on HEAD**. To check the first half by hand: apply `find` → `replace` in the target file, run the defender, restore.
66
- 3. `testguard probe --claim <ID> --include-dirty` — probes your uncommitted test without touching the tree. `--confirm 1` gives a fast **provisional** signal; a `?` on a verdict means unconfirmed.
67
- 4. When it is `killed` at `--confirm 3`, commit the test.
66
+ 3. `testguard admit <test-file> --claim <ID>` — the two gates as one verb, on your uncommitted test, without touching the tree: `ADMITTED` (exit 0) means the test is green on HEAD and fails on every fault of the claim, 3/3; `NOT ADMITTED` (exit 1) names the first blocking fault and what to do. `--fault <FID>` judges one fault; `--confirm 1` gives a fast **provisional** `ADMITTED?` that must be confirmed at 3 before you commit. (`testguard probe --claim <ID> --include-dirty` is the same run with the full report.)
67
+ 4. When it is `ADMITTED` at `--confirm 3`, commit the test.
68
68
  5. When everything is killed or baselined, `testguard baseline` if `next` says so, and commit `.testguard/baseline.json`.
69
69
 
70
70
  ## Rules that are not yours to bend
@@ -0,0 +1,104 @@
1
+ import { TOOLS } from './tools.mjs';
2
+
3
+ /**
4
+ * A Model Context Protocol server over stdio, hand-rolled.
5
+ *
6
+ * Why not the official SDK: this tool has one exact-pinned runtime dependency
7
+ * and intends to keep it that way (AGENTS.md rule 8). The surface here is
8
+ * three methods and five read-only tools and will not grow, so the protocol
9
+ * is written out rather than imported. The version is pinned below and the
10
+ * conformance is covered by tests that speak the wire format.
11
+ *
12
+ * JSON-RPC 2.0 framing: one message per line (newline-delimited JSON), which
13
+ * is what stdio transports use. A notification (no `id`) is never answered.
14
+ */
15
+ export const PROTOCOL_VERSION = '2025-06-18';
16
+
17
+ const ERROR = { parse: -32700, invalidRequest: -32600, methodNotFound: -32601, invalidParams: -32602, internal: -32603 };
18
+
19
+ const ok = (id, result) => ({ jsonrpc: '2.0', id, result });
20
+ const err = (id, code, message, data) => ({ jsonrpc: '2.0', id, error: { code, message, ...(data ? { data } : {}) } });
21
+
22
+ /** Everything an agent may ask for, with the version stamped onto each call. */
23
+ export function handle(message, { version = '0.0.0' } = {}) {
24
+ if (message === null || typeof message !== 'object' || Array.isArray(message)) return err(null, ERROR.invalidRequest, 'a request must be a JSON object');
25
+ const { id, method, params } = message;
26
+ const isNotification = id === undefined || id === null;
27
+
28
+ switch (method) {
29
+ case 'initialize':
30
+ return isNotification ? null : ok(id, {
31
+ protocolVersion: PROTOCOL_VERSION,
32
+ // Read-only: no prompts, no resources, no sampling. Saying so is part
33
+ // of the contract — nothing here can change a claims file.
34
+ capabilities: { tools: { listChanged: false } },
35
+ serverInfo: { name: 'testguard', version },
36
+ instructions: 'Read testguard_status first: it carries the one next action. Run the command it names in your own terminal — these tools never run a probe, because a probe is long-running, budgeted, and the person should see it happen.',
37
+ });
38
+
39
+ case 'notifications/initialized':
40
+ case 'notifications/cancelled':
41
+ return null;
42
+
43
+ case 'ping':
44
+ return isNotification ? null : ok(id, {});
45
+
46
+ case 'tools/list':
47
+ return isNotification ? null : ok(id, { tools: TOOLS.map(({ name, description, inputSchema }) => ({ name, description, inputSchema, annotations: { readOnlyHint: true, destructiveHint: false, idempotentHint: true, openWorldHint: false } })) });
48
+
49
+ case 'tools/call': {
50
+ if (isNotification) return null;
51
+ const tool = TOOLS.find((t) => t.name === params?.name);
52
+ if (!tool) return err(id, ERROR.invalidParams, `unknown tool: ${params?.name}`);
53
+ try {
54
+ const result = tool.handler({ ...(params?.arguments ?? {}), __version: version });
55
+ return ok(id, { content: [{ type: 'text', text: JSON.stringify(result, null, 2) }], structuredContent: result, isError: false });
56
+ } catch (e) {
57
+ // A tool failure is a tool result, not a protocol error: the agent
58
+ // should read the reason and act, not lose the connection.
59
+ return ok(id, { content: [{ type: 'text', text: `${tool.name} failed: ${e.message}` }], isError: true });
60
+ }
61
+ }
62
+
63
+ default:
64
+ return isNotification ? null : err(id, ERROR.methodNotFound, `unknown method: ${method}`);
65
+ }
66
+ }
67
+
68
+ /**
69
+ * Speak the protocol on a pair of streams. Resolves when input ends, so the
70
+ * caller can await a clean shutdown.
71
+ */
72
+ export function serve({ input = process.stdin, output = process.stdout, version = '0.0.0' } = {}) {
73
+ return new Promise((resolveDone) => {
74
+ let buffer = '';
75
+ const write = (msg) => {
76
+ if (msg) output.write(JSON.stringify(msg) + '\n');
77
+ };
78
+ input.setEncoding?.('utf8');
79
+ input.on('data', (chunk) => {
80
+ buffer += chunk;
81
+ let nl;
82
+ while ((nl = buffer.indexOf('\n')) !== -1) {
83
+ const line = buffer.slice(0, nl).trim();
84
+ buffer = buffer.slice(nl + 1);
85
+ if (!line) continue;
86
+ let message;
87
+ try {
88
+ message = JSON.parse(line);
89
+ } catch {
90
+ write(err(null, ERROR.parse, 'invalid JSON'));
91
+ continue;
92
+ }
93
+ try {
94
+ write(handle(message, { version }));
95
+ } catch (e) {
96
+ // The server outlives any single bad request.
97
+ write(err(message?.id ?? null, ERROR.internal, e.message));
98
+ }
99
+ }
100
+ });
101
+ input.on('end', () => resolveDone(0));
102
+ input.on('close', () => resolveDone(0));
103
+ });
104
+ }
@@ -0,0 +1,114 @@
1
+ import { existsSync } from 'node:fs';
2
+ import { join, resolve } from 'node:path';
3
+ import { computeStatus } from '../status/status.mjs';
4
+ import { buildBrief, buildUnclaimedBrief } from '../brief/brief.mjs';
5
+ import { loadClaims, defaultClaimsPath } from '../claims/load.mjs';
6
+ import { scanAnnotations, reconcile } from '../claims/annotations.mjs';
7
+ import { computeRemovedClaims } from '../claims/removed.mjs';
8
+ import { readSpecDoc } from '../evidence/writer.mjs';
9
+ import { resolveChangedRef, withChangedRef } from '../gate/changed.mjs';
10
+
11
+ /**
12
+ * The read-only surface an agent needs, in any harness.
13
+ *
14
+ * TestGuard's operating loop currently lives in a Claude Code skill and a
15
+ * session-start hook, and both vanish the moment the harness is something
16
+ * else. These tools expose the documents the tool already computes, so the
17
+ * loop survives that change.
18
+ *
19
+ * Every tool is read-only and none of them probes. A probe is long-running
20
+ * and budgeted, and the human should see it happen: `next_command` hands back
21
+ * the exact shell line for the agent to run in its own terminal instead.
22
+ */
23
+ const projectOf = (args) => resolve(args?.dir ?? '.');
24
+ const evidenceOf = (args) => (args?.evidence ? resolve(args.evidence) : undefined);
25
+
26
+ const DIR = { type: 'string', description: 'Project directory (where testguard.claims.json lives). Defaults to the current directory.' };
27
+ const EVIDENCE = { type: 'string', description: "Read this evidence document instead of the project's own — CI's, fetched as an artifact." };
28
+ const CHANGED = { type: 'string', description: 'A git reference to measure the change against, so claim coverage of the change is included.' };
29
+
30
+ const statusFor = (args) => {
31
+ const projectDir = projectOf(args);
32
+ return withChangedRef(resolveChangedRef({ explicit: args?.changed }), (changedRef) =>
33
+ computeStatus({ projectDir, toolVersion: args?.__version ?? '0.0.0', changedRef, evidence: evidenceOf(args) }));
34
+ };
35
+
36
+ export const TOOLS = [
37
+ {
38
+ name: 'testguard_status',
39
+ description: 'Where this project stands and the ONE next action, as the machine-readable status document: state, next {action, command, why}, counts, stale inputs, faults edited since they were probed, and the ranked findings. This is the single source of truth every other rendering derives from — read it before deciding anything.',
40
+ inputSchema: { type: 'object', properties: { dir: DIR, changed: CHANGED, evidence: EVIDENCE }, additionalProperties: false },
41
+ handler: (args) => statusFor(args),
42
+ },
43
+ {
44
+ name: 'testguard_brief',
45
+ description: 'Where the test suite is blind, ranked and capped, with one actionable hint per finding and unclaimed changed files first. Read this before writing or changing code, which is what the session-start hook does for harnesses that have one.',
46
+ inputSchema: { type: 'object', properties: { dir: DIR, changed: CHANGED, evidence: EVIDENCE, max: { type: 'integer', minimum: 1, maximum: 50 } }, additionalProperties: false },
47
+ handler: (args) => {
48
+ const projectDir = projectOf(args);
49
+ const evPath = evidenceOf(args) ?? join(projectDir, '.testguard', 'evidence.json');
50
+ const status = (() => {
51
+ try {
52
+ return statusFor(args);
53
+ } catch {
54
+ return undefined; // a brief must never fail because status could not be computed
55
+ }
56
+ })();
57
+ if (!existsSync(evPath)) {
58
+ if (status?.changes?.uncovered?.length) {
59
+ return buildUnclaimedBrief({ tool: { name: 'testguard', version: args?.__version ?? '0.0.0' }, next: status.next, changes: status.changes });
60
+ }
61
+ return { note: `no evidence at ${evPath}: nothing has been probed yet`, next: status?.next };
62
+ }
63
+ const evidence = readSpecDoc('evidence', evPath);
64
+ const basePath = join(projectDir, '.testguard', 'baseline.json');
65
+ const baseline = existsSync(basePath) ? readSpecDoc('baseline', basePath) : undefined;
66
+ return buildBrief(evidence, baseline, { max: args?.max ?? 20, next: status?.next, changes: status?.changes });
67
+ },
68
+ },
69
+ {
70
+ name: 'testguard_claims',
71
+ description: "What this project claims must be true, each with its faults and defenders, plus drift against `@claim` annotations in source and — when `since` is given — every claim or fault that existed at that reference and does not now. A claim that disappeared is invisible to every other check.",
72
+ inputSchema: { type: 'object', properties: { dir: DIR, since: { type: 'string', description: 'A git reference: report claims and faults removed since it.' } }, additionalProperties: false },
73
+ handler: (args) => {
74
+ const projectDir = projectOf(args);
75
+ const path = defaultClaimsPath(projectDir);
76
+ if (!existsSync(path)) return { path, claims: [], note: 'no claims file in this project yet' };
77
+ const claims = loadClaims(path);
78
+ const drift = reconcile(claims, scanAnnotations(projectDir));
79
+ const removed = args?.since
80
+ ? computeRemovedClaims({ projectDir, ref: args.since, claimsPath: path, current: claims, evidencePath: join(projectDir, '.testguard', 'evidence.json'), toolVersion: args?.__version ?? '0.0.0' })
81
+ : undefined;
82
+ return { path, claims, drift, ...(removed ? { removed } : {}) };
83
+ },
84
+ },
85
+ {
86
+ name: 'testguard_evidence',
87
+ description: 'One run\'s findings on disk: every verdict with the runs that produced it. Pass `claim` (and optionally `fault`) for a single record instead of the whole document, which can be large.',
88
+ inputSchema: {
89
+ type: 'object',
90
+ properties: { dir: DIR, evidence: EVIDENCE, claim: { type: 'string', description: 'Return only this claim\'s records.' }, fault: { type: 'string', description: 'With `claim`: only this fault\'s record.' } },
91
+ additionalProperties: false,
92
+ },
93
+ handler: (args) => {
94
+ const projectDir = projectOf(args);
95
+ const evPath = evidenceOf(args) ?? join(projectDir, '.testguard', 'evidence.json');
96
+ if (!existsSync(evPath)) return { note: `no evidence at ${evPath}: run a probe first`, path: evPath };
97
+ const doc = readSpecDoc('evidence', evPath);
98
+ if (!args?.claim) return doc;
99
+ const records = doc.records.filter((r) => r.claim.id === args.claim && (!args.fault || r.subject.id === args.fault));
100
+ return { ...doc, records, filtered: { claim: args.claim, ...(args.fault ? { fault: args.fault } : {}) } };
101
+ },
102
+ },
103
+ {
104
+ name: 'testguard_next_command',
105
+ description: 'The exact shell command to run next, with the reason. Run it yourself in a terminal: a probe is long-running and budgeted, and the person should see it happen. This tool never runs anything.',
106
+ inputSchema: { type: 'object', properties: { dir: DIR, changed: CHANGED, evidence: EVIDENCE }, additionalProperties: false },
107
+ handler: (args) => {
108
+ const doc = statusFor(args);
109
+ return { state: doc.state, action: doc.next.action, command: doc.next.command, why: doc.next.why, ...(doc.next.target ? { target: doc.next.target } : {}), ...(doc.next.file ? { file: doc.next.file } : {}) };
110
+ },
111
+ },
112
+ ];
113
+
114
+ export const TOOL_NAMES = TOOLS.map((t) => t.name);
@@ -0,0 +1,123 @@
1
+ /**
2
+ * The decisions escalation and flake measurement make, as pure functions.
3
+ *
4
+ * These used to live inline in `probeOne`, which meant the only thing that
5
+ * could falsify them was a full fixture probe — a 50-second acceptance test
6
+ * standing in for a rule you can state in three lines. `classify()` has always
7
+ * been pure for this reason: the order of its checks *is* the spec, and every
8
+ * branch is unit-testable without a runner. Same principle here.
9
+ *
10
+ * Nothing in this file runs a test, touches the filesystem, or knows what a
11
+ * worktree is. Everything it needs is passed in.
12
+ */
13
+
14
+ const isKill = (r) => r.outcome === 'fail' && (r.assertionFailures ?? 0) > 0;
15
+
16
+ /**
17
+ * Fold one escalation run into the running attribution state.
18
+ *
19
+ * A test is an undeclared killer only if it fails in **every** run: a single
20
+ * run cannot say, because a test that is flaky elsewhere in the suite would
21
+ * take the credit. So the killer set is an intersection, seeded by the first
22
+ * run rather than by the empty set.
23
+ *
24
+ * Once the intersection is empty no further run can name a killer, so the
25
+ * caller may stop early. Stopping can only fail to *name* a killer, never
26
+ * upgrade a verdict, which is the pessimistic side and therefore the safe one.
27
+ */
28
+ export function foldEscalationRun(state, { run, failedTests }) {
29
+ const runs = [...state.runs, run];
30
+ const killers = state.killers === null ? new Set(failedTests) : new Set([...state.killers].filter((t) => failedTests.includes(t)));
31
+ return { runs, killers, stoppedEarly: killers.size === 0 ? 'no-common-failure' : undefined };
32
+ }
33
+
34
+ /** The initial attribution state, before any escalation run. */
35
+ export const escalationStart = () => ({ runs: [], killers: null, stoppedEarly: undefined });
36
+
37
+ /**
38
+ * What escalation concluded. `undeclaredKillers` is named only when a test
39
+ * failed in all N runs AND every run was a genuine kill (an assertion
40
+ * failure). A suite that errored or timed out proves nothing about the fault,
41
+ * so it never earns an attribution.
42
+ */
43
+ export function escalationResult(state, confirmRuns) {
44
+ const complete = state.runs.length === confirmRuns;
45
+ const attributable = state.killers !== null && state.killers.size > 0 && complete && state.runs.every(isKill);
46
+ return {
47
+ escalated: true,
48
+ escalationRuns: state.runs,
49
+ ...(state.stoppedEarly ? { escalationStoppedEarly: state.stoppedEarly } : {}),
50
+ ...(attributable ? { reason: 'killed-by-undeclared-tests', undeclaredKillers: [...state.killers].sort() } : {}),
51
+ };
52
+ }
53
+
54
+ /**
55
+ * The defenders' flake rate on unmodified source.
56
+ *
57
+ * Only runs that **actually ran** carry information. A load error or a timeout
58
+ * measures the environment, not the suite's stability, and counting either as
59
+ * a "failure" would invent flakiness that was never observed. Returns
60
+ * `undefined` when nothing was measurable, so the caller omits the field
61
+ * rather than recording a rate of zero it cannot support.
62
+ */
63
+ export function flakeRate(runs) {
64
+ const measured = runs.filter((r) => r.outcome === 'pass' || r.outcome === 'fail');
65
+ if (!measured.length) return undefined;
66
+ return { runs: measured.length, failures: measured.filter((r) => r.outcome === 'fail').length };
67
+ }
68
+
69
+ /**
70
+ * The tests that actually failed on the fault, by file. Only these are its
71
+ * killers; everything else in a run is noise for attribution purposes.
72
+ */
73
+ export function killersFromRuns(probeRuns) {
74
+ return [...new Set(probeRuns.flatMap((r) => r.failedTests ?? []).map((t) => t.split('::')[0]).filter(Boolean))];
75
+ }
76
+
77
+ /**
78
+ * The `subject` of an evidence record. `contentHash` is what makes a fault edit
79
+ * visible: weakening a fault after it survived changes this hash, and `status`
80
+ * reports the change with the verdict the fault used to have. It is deliberately
81
+ * NOT part of the fingerprint — repairing a rotted anchor must not churn the
82
+ * baseline — which is exactly why it has to be recorded separately.
83
+ */
84
+ export function subjectOf(fault, sha256) {
85
+ return {
86
+ kind: 'fault',
87
+ id: fault.id,
88
+ description: fault.description,
89
+ file: fault.file,
90
+ faultClass: fault.faultClass,
91
+ producedBy: fault.producedBy,
92
+ contentHash: sha256(`${fault.find}\n${fault.replace}`),
93
+ };
94
+ }
95
+
96
+ /**
97
+ * May a prior record stand in for this one, or must the fault be probed again?
98
+ *
99
+ * Same source, same defenders, same N — and the SAME FAULT. The last is not a
100
+ * refinement: a verdict is an answer about one specific `find`/`replace`, and
101
+ * once those change the old answer is not an answer to the new question.
102
+ * Without it, weakening a fault that survived keeps its old verdict, and
103
+ * repairing a rotted anchor keeps `unverifiable` — the evidence reporting a
104
+ * result that was never measured against the fault the file now contains.
105
+ * `status` flags the changed hash afterwards, but by then the evidence has
106
+ * already said something untrue.
107
+ *
108
+ * A prior that predates `contentHash` cannot be compared, so it is re-probed.
109
+ * The cost of re-measuring is a run; the cost of the other direction is a
110
+ * verdict nobody measured.
111
+ */
112
+ export function isReusable(prior, current) {
113
+ if (!prior || !prior.subject?.contentHash) return false;
114
+ // A record that exists because the probe threw is a statement about the run,
115
+ // not about the code. Reusing it would make one disturbed run permanent.
116
+ if (prior.detail?.reason === 'probe-error') return false;
117
+ const same = (x, y) => JSON.stringify(x) === JSON.stringify(y);
118
+ return prior.inputs.targetHash === current.inputs.targetHash
119
+ && same(prior.inputs.defenderHashes, current.inputs.defenderHashes)
120
+ && same(prior.defenders.requested, current.requested)
121
+ && same(prior.defenders.resolved, current.resolved)
122
+ && prior.subject.contentHash === current.contentHash;
123
+ }