vigiles 2.3.0 → 2.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +214 -190
- package/dist/agent-result.d.ts +40 -0
- package/dist/agent-result.js +97 -0
- package/dist/agent-runtime.d.ts +64 -0
- package/dist/agent-runtime.js +147 -0
- package/dist/cli.js +155 -1
- package/dist/compile.d.ts +32 -3
- package/dist/compile.js +268 -0
- package/dist/eval-cache.d.ts +33 -0
- package/dist/eval-cache.js +94 -0
- package/dist/eval.d.ts +180 -9
- package/dist/eval.js +319 -57
- package/dist/harness-assert.d.ts +175 -6
- package/dist/harness-assert.js +355 -4
- package/dist/harness-test.d.ts +130 -5
- package/dist/harness-test.js +205 -32
- package/dist/judge.js +2 -0
- package/dist/linters.d.ts +6 -0
- package/dist/linters.js +1 -0
- package/dist/mcp.d.ts +48 -0
- package/dist/mcp.js +247 -0
- package/dist/mock-entry.d.ts +2 -0
- package/dist/mock-entry.js +36 -0
- package/dist/mock-model.d.ts +29 -0
- package/dist/mock-model.js +40 -0
- package/dist/plugin-loader.js +51 -17
- package/dist/sandbox.d.ts +76 -0
- package/dist/sandbox.js +241 -0
- package/dist/spec.d.ts +130 -0
- package/dist/spec.js +55 -0
- package/dist/stats.d.ts +49 -0
- package/dist/stats.js +109 -0
- package/package.json +7 -3
package/dist/harness-test.js
CHANGED
|
@@ -1,6 +1,11 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
-
exports.resolveHarness = exports.loadPlugin = exports.scriptModel = void 0;
|
|
3
|
+
exports.sandboxAvailable = exports.specTrusted = exports.decideSandbox = exports.resolveHarness = exports.loadPlugin = exports.scriptModel = void 0;
|
|
4
|
+
exports.parseToolCalls = parseToolCalls;
|
|
5
|
+
exports.parseResultEvent = parseResultEvent;
|
|
6
|
+
exports.parseOutput = parseOutput;
|
|
7
|
+
exports.parseHooks = parseHooks;
|
|
8
|
+
exports.buildClaudeArgs = buildClaudeArgs;
|
|
4
9
|
exports.claudeAvailable = claudeAvailable;
|
|
5
10
|
exports.runHarnessTest = runHarnessTest;
|
|
6
11
|
/**
|
|
@@ -27,10 +32,11 @@ exports.runHarnessTest = runHarnessTest;
|
|
|
27
32
|
* The "steps" are the scripted model turns — their real home is deterministic
|
|
28
33
|
* harness testing, not production enforcement.
|
|
29
34
|
*
|
|
30
|
-
* Note: the
|
|
31
|
-
* Edit/Write tools
|
|
32
|
-
*
|
|
33
|
-
*
|
|
35
|
+
* Note: the mock drives Bash and Stop hooks, and — verified on claude 2.1.169 —
|
|
36
|
+
* the Edit/Write tools too (allowlisted past the permission prompt), so their
|
|
37
|
+
* PreToolUse/PostToolUse hooks fire in this tier. The events the mock can't
|
|
38
|
+
* trigger (PreCompact / Notification / SessionEnd / SubagentStop) belong to the
|
|
39
|
+
* `runHook` unit tier.
|
|
34
40
|
*/
|
|
35
41
|
const node_child_process_1 = require("node:child_process");
|
|
36
42
|
const node_fs_1 = require("node:fs");
|
|
@@ -38,11 +44,159 @@ const node_os_1 = require("node:os");
|
|
|
38
44
|
const node_path_1 = require("node:path");
|
|
39
45
|
const mock_model_js_1 = require("./mock-model.js");
|
|
40
46
|
const plugin_loader_js_1 = require("./plugin-loader.js");
|
|
47
|
+
const sandbox_js_1 = require("./sandbox.js");
|
|
41
48
|
var mock_model_js_2 = require("./mock-model.js");
|
|
42
49
|
Object.defineProperty(exports, "scriptModel", { enumerable: true, get: function () { return mock_model_js_2.scriptModel; } });
|
|
43
50
|
var plugin_loader_js_2 = require("./plugin-loader.js");
|
|
44
51
|
Object.defineProperty(exports, "loadPlugin", { enumerable: true, get: function () { return plugin_loader_js_2.loadPlugin; } });
|
|
45
52
|
Object.defineProperty(exports, "resolveHarness", { enumerable: true, get: function () { return plugin_loader_js_2.resolveHarness; } });
|
|
53
|
+
var sandbox_js_2 = require("./sandbox.js");
|
|
54
|
+
Object.defineProperty(exports, "decideSandbox", { enumerable: true, get: function () { return sandbox_js_2.decideSandbox; } });
|
|
55
|
+
Object.defineProperty(exports, "specTrusted", { enumerable: true, get: function () { return sandbox_js_2.specTrusted; } });
|
|
56
|
+
Object.defineProperty(exports, "sandboxAvailable", { enumerable: true, get: function () { return sandbox_js_2.sandboxAvailable; } });
|
|
57
|
+
function contentText(content) {
|
|
58
|
+
if (typeof content === "string")
|
|
59
|
+
return content;
|
|
60
|
+
if (!Array.isArray(content))
|
|
61
|
+
return "";
|
|
62
|
+
return content
|
|
63
|
+
.map((b) => {
|
|
64
|
+
if (typeof b === "string")
|
|
65
|
+
return b;
|
|
66
|
+
const t = b.text;
|
|
67
|
+
return typeof t === "string" ? t : "";
|
|
68
|
+
})
|
|
69
|
+
.join("");
|
|
70
|
+
}
|
|
71
|
+
/**
|
|
72
|
+
* Parse `--output-format stream-json` (the `transcript: true` output) into the
|
|
73
|
+
* tools the agent invoked, each joined to its result by id. Returns [] for the
|
|
74
|
+
* non-stream `json` output. The seam that lets a test assert on the agent's
|
|
75
|
+
* actions, not a brittle stdout substring.
|
|
76
|
+
*/
|
|
77
|
+
function parseToolCalls(streamJson) {
|
|
78
|
+
const uses = [];
|
|
79
|
+
const results = new Map();
|
|
80
|
+
for (const line of streamJson.split("\n")) {
|
|
81
|
+
if (!line.trim())
|
|
82
|
+
continue;
|
|
83
|
+
let evt;
|
|
84
|
+
try {
|
|
85
|
+
evt = JSON.parse(line);
|
|
86
|
+
}
|
|
87
|
+
catch {
|
|
88
|
+
continue;
|
|
89
|
+
}
|
|
90
|
+
const content = evt.message?.content;
|
|
91
|
+
if (!Array.isArray(content))
|
|
92
|
+
continue;
|
|
93
|
+
for (const b of content) {
|
|
94
|
+
if (b.type === "tool_use" && typeof b.name === "string") {
|
|
95
|
+
const id = typeof b.id === "string" ? b.id : "";
|
|
96
|
+
uses.push({ id, name: b.name, input: b.input });
|
|
97
|
+
}
|
|
98
|
+
else if (b.type === "tool_result") {
|
|
99
|
+
const id = typeof b.tool_use_id === "string" ? b.tool_use_id : "";
|
|
100
|
+
results.set(id, {
|
|
101
|
+
text: contentText(b.content),
|
|
102
|
+
isError: b.is_error === true,
|
|
103
|
+
});
|
|
104
|
+
}
|
|
105
|
+
}
|
|
106
|
+
}
|
|
107
|
+
return uses.map((u) => ({
|
|
108
|
+
name: u.name,
|
|
109
|
+
input: u.input,
|
|
110
|
+
resultText: results.get(u.id)?.text ?? "",
|
|
111
|
+
isError: results.get(u.id)?.isError ?? false,
|
|
112
|
+
}));
|
|
113
|
+
}
|
|
114
|
+
/**
|
|
115
|
+
* The terminal `result` event — present in BOTH `--output-format` shapes (a
|
|
116
|
+
* `{type:"result", …}` line in stream-json, the single object in `json`), or
|
|
117
|
+
* null. The seam for the final answer + turn count without parsing twice.
|
|
118
|
+
*/
|
|
119
|
+
function parseResultEvent(stdout) {
|
|
120
|
+
for (const line of stdout.split("\n")) {
|
|
121
|
+
if (!line.trim())
|
|
122
|
+
continue;
|
|
123
|
+
let evt;
|
|
124
|
+
try {
|
|
125
|
+
evt = JSON.parse(line);
|
|
126
|
+
}
|
|
127
|
+
catch {
|
|
128
|
+
continue;
|
|
129
|
+
}
|
|
130
|
+
if (evt.type === "result")
|
|
131
|
+
return evt;
|
|
132
|
+
}
|
|
133
|
+
return null;
|
|
134
|
+
}
|
|
135
|
+
/** The agent's final answer text from a transcript / result object, or "". */
|
|
136
|
+
function parseOutput(stdout) {
|
|
137
|
+
const result = parseResultEvent(stdout)?.result;
|
|
138
|
+
return typeof result === "string" ? result : "";
|
|
139
|
+
}
|
|
140
|
+
/**
|
|
141
|
+
* The hooks that fired, recorded from the CLI's `hook_response` stream events
|
|
142
|
+
* (`--output-format stream-json`). Each carries the hook name/event, its exit
|
|
143
|
+
* code, and whether it blocked — the honest record vs. inferring from marker
|
|
144
|
+
* files. Returns [] for the non-stream `json` output (no per-hook events).
|
|
145
|
+
*/
|
|
146
|
+
function toHookFire(evt) {
|
|
147
|
+
const exitCode = typeof evt.exit_code === "number" ? evt.exit_code : undefined;
|
|
148
|
+
return {
|
|
149
|
+
name: typeof evt.hook_name === "string" ? evt.hook_name : "",
|
|
150
|
+
event: typeof evt.hook_event === "string" ? evt.hook_event : "",
|
|
151
|
+
exitCode,
|
|
152
|
+
blocked: evt.outcome === "error" || (exitCode !== undefined && exitCode !== 0),
|
|
153
|
+
output: typeof evt.output === "string" ? evt.output : "",
|
|
154
|
+
};
|
|
155
|
+
}
|
|
156
|
+
function parseHooks(stdout) {
|
|
157
|
+
const hooks = [];
|
|
158
|
+
for (const line of stdout.split("\n")) {
|
|
159
|
+
if (!line.trim())
|
|
160
|
+
continue;
|
|
161
|
+
let evt;
|
|
162
|
+
try {
|
|
163
|
+
evt = JSON.parse(line);
|
|
164
|
+
}
|
|
165
|
+
catch {
|
|
166
|
+
continue;
|
|
167
|
+
}
|
|
168
|
+
if (evt.type === "system" && evt.subtype === "hook_response") {
|
|
169
|
+
hooks.push(toHookFire(evt));
|
|
170
|
+
}
|
|
171
|
+
}
|
|
172
|
+
return hooks;
|
|
173
|
+
}
|
|
174
|
+
/**
|
|
175
|
+
* The `claude` CLI argv for a harness run (shared by the direct and sandboxed
|
|
176
|
+
* paths). `ANTHROPIC_BASE_URL` is set by the caller's environment / wrapper, not
|
|
177
|
+
* here. Pure, so the arg shape is unit-tested.
|
|
178
|
+
*/
|
|
179
|
+
function buildClaudeArgs(spec, hasSettings) {
|
|
180
|
+
const tools = spec.allowedTools ?? ["Read", "Edit", "Write", "Bash"];
|
|
181
|
+
return [
|
|
182
|
+
"-p",
|
|
183
|
+
spec.prompt ?? "go",
|
|
184
|
+
...(spec.transcript
|
|
185
|
+
? ["--output-format", "stream-json", "--verbose"]
|
|
186
|
+
: ["--output-format", "json"]),
|
|
187
|
+
"--model",
|
|
188
|
+
"claude-sonnet-4-5",
|
|
189
|
+
...(spec.pluginDir !== undefined
|
|
190
|
+
? ["--plugin-dir", (0, node_path_1.resolve)(spec.pluginDir)]
|
|
191
|
+
: []),
|
|
192
|
+
...(hasSettings ? ["--settings", "settings.json"] : []),
|
|
193
|
+
"--allowedTools",
|
|
194
|
+
...tools,
|
|
195
|
+
];
|
|
196
|
+
}
|
|
197
|
+
/* v8 ignore start -- spawns the real claude CLI + filesystem; exercised by the
|
|
198
|
+
claude-backed suite, excluded from the deterministic coverage gate (the parse
|
|
199
|
+
helpers above carry the testable logic). */
|
|
46
200
|
/** Whether the `claude` CLI is available — harness tests need it. */
|
|
47
201
|
function claudeAvailable() {
|
|
48
202
|
try {
|
|
@@ -92,8 +246,20 @@ function spawnClaude(args, cwd, baseUrl, timeoutMs) {
|
|
|
92
246
|
/**
|
|
93
247
|
* Run the real `claude` CLI against a scripted mock model, with the given
|
|
94
248
|
* fixture and settings (hooks). Deterministic — same script, same result.
|
|
249
|
+
*
|
|
250
|
+
* Safe by default: an external `plugin` / `pluginDir` brings in untrusted
|
|
251
|
+
* third-party hooks and is confined under bubblewrap (`spec.sandbox`, default
|
|
252
|
+
* `"auto"`); if no sandbox is available the run REFUSES rather than executing
|
|
253
|
+
* unconfined. See `src/sandbox.ts`.
|
|
95
254
|
*/
|
|
96
255
|
async function runHarnessTest(spec) {
|
|
256
|
+
const decision = (0, sandbox_js_1.decideSandbox)({
|
|
257
|
+
trusted: (0, sandbox_js_1.specTrusted)(spec),
|
|
258
|
+
mode: spec.sandbox ?? "auto",
|
|
259
|
+
available: (0, sandbox_js_1.sandboxAvailable)(),
|
|
260
|
+
});
|
|
261
|
+
if (decision.action === "throw")
|
|
262
|
+
throw new Error(decision.reason);
|
|
97
263
|
const cwd = (0, node_fs_1.mkdtempSync)((0, node_path_1.join)((0, node_os_1.tmpdir)(), "vigiles-harness-"));
|
|
98
264
|
const { files, settings } = (0, plugin_loader_js_1.resolveHarness)({
|
|
99
265
|
plugin: spec.plugin,
|
|
@@ -101,38 +267,45 @@ async function runHarnessTest(spec) {
|
|
|
101
267
|
files: spec.files,
|
|
102
268
|
});
|
|
103
269
|
writeFixture(cwd, files, settings);
|
|
270
|
+
const args = buildClaudeArgs(spec, settings !== undefined);
|
|
271
|
+
const timeoutMs = spec.timeoutMs ?? 60000;
|
|
272
|
+
const build = (out, turns, modelRequests) => ({
|
|
273
|
+
exitCode: out.code,
|
|
274
|
+
stdout: out.stdout,
|
|
275
|
+
stderr: out.stderr ?? "",
|
|
276
|
+
cwd,
|
|
277
|
+
turns,
|
|
278
|
+
toolCalls: parseToolCalls(out.stdout),
|
|
279
|
+
hooks: parseHooks(out.stdout),
|
|
280
|
+
output: parseOutput(out.stdout),
|
|
281
|
+
modelRequests,
|
|
282
|
+
file: (p) => {
|
|
283
|
+
const f = (0, node_path_1.resolve)(cwd, p);
|
|
284
|
+
return (0, node_fs_1.existsSync)(f) ? (0, node_fs_1.readFileSync)(f, "utf-8") : null;
|
|
285
|
+
},
|
|
286
|
+
cleanup: () => {
|
|
287
|
+
(0, node_fs_1.rmSync)(cwd, { recursive: true, force: true });
|
|
288
|
+
},
|
|
289
|
+
});
|
|
290
|
+
// Confined path: the mock is co-launched inside the sandbox's netns.
|
|
291
|
+
if (decision.action === "sandbox") {
|
|
292
|
+
const out = await (0, sandbox_js_1.runSandboxed)({
|
|
293
|
+
cwd,
|
|
294
|
+
claudeArgs: args,
|
|
295
|
+
script: spec.model,
|
|
296
|
+
timeoutMs,
|
|
297
|
+
});
|
|
298
|
+
return build(out, out.requests.length, out.requests);
|
|
299
|
+
}
|
|
300
|
+
// Direct path: mock runs in this process; claude reaches it over localhost.
|
|
104
301
|
const mock = await (0, mock_model_js_1.startMock)(spec.model);
|
|
105
302
|
try {
|
|
106
|
-
const
|
|
107
|
-
|
|
108
|
-
"-p",
|
|
109
|
-
spec.prompt ?? "go",
|
|
110
|
-
"--output-format",
|
|
111
|
-
"json",
|
|
112
|
-
"--model",
|
|
113
|
-
"claude-sonnet-4-5",
|
|
114
|
-
...(settings !== undefined ? ["--settings", "settings.json"] : []),
|
|
115
|
-
"--allowedTools",
|
|
116
|
-
...tools,
|
|
117
|
-
];
|
|
118
|
-
const out = await spawnClaude(args, cwd, mock.url, spec.timeoutMs ?? 60000);
|
|
119
|
-
return {
|
|
120
|
-
exitCode: out.code,
|
|
121
|
-
stdout: out.stdout,
|
|
122
|
-
stderr: out.stderr,
|
|
123
|
-
cwd,
|
|
124
|
-
turns: mock.count,
|
|
125
|
-
file: (p) => {
|
|
126
|
-
const f = (0, node_path_1.resolve)(cwd, p);
|
|
127
|
-
return (0, node_fs_1.existsSync)(f) ? (0, node_fs_1.readFileSync)(f, "utf-8") : null;
|
|
128
|
-
},
|
|
129
|
-
cleanup: () => {
|
|
130
|
-
(0, node_fs_1.rmSync)(cwd, { recursive: true, force: true });
|
|
131
|
-
},
|
|
132
|
-
};
|
|
303
|
+
const out = await spawnClaude(args, cwd, mock.url, timeoutMs);
|
|
304
|
+
return build(out, mock.count, [...mock.requests]);
|
|
133
305
|
}
|
|
134
306
|
finally {
|
|
135
307
|
mock.close();
|
|
136
308
|
}
|
|
137
309
|
}
|
|
310
|
+
/* v8 ignore stop */
|
|
138
311
|
//# sourceMappingURL=harness-test.js.map
|
package/dist/judge.js
CHANGED
|
@@ -36,6 +36,7 @@ function firstJsonObject(s) {
|
|
|
36
36
|
return null;
|
|
37
37
|
}
|
|
38
38
|
}
|
|
39
|
+
/* v8 ignore start -- spawns the real claude CLI; parseJudgeOutput holds the logic */
|
|
39
40
|
/** Grade `output` against `rubric` with a model. Synchronous (for `measure`). */
|
|
40
41
|
function judge(opts) {
|
|
41
42
|
const threshold = opts.threshold ?? 0.5;
|
|
@@ -65,6 +66,7 @@ function judge(opts) {
|
|
|
65
66
|
}
|
|
66
67
|
return parseJudgeOutput(res.stdout ?? "", threshold);
|
|
67
68
|
}
|
|
69
|
+
/* v8 ignore stop */
|
|
68
70
|
/**
|
|
69
71
|
* Parse a verdict out of the grader's stdout — pure, so the parsing is testable
|
|
70
72
|
* without a model. Handles `claude --output-format json` (text wrapped in a
|
package/dist/linters.d.ts
CHANGED
|
@@ -24,6 +24,12 @@ export interface DetectedLinter {
|
|
|
24
24
|
}
|
|
25
25
|
/** @internal */ export declare function extractLinterName(enforcedBy: string): string;
|
|
26
26
|
/** @internal */ export declare function extractRuleName(enforcedBy: string): string | null;
|
|
27
|
+
/**
|
|
28
|
+
* Levenshtein distance for short-string typo detection. Rule names are
|
|
29
|
+
* short so edit distance is more appropriate than NCD (which is tuned
|
|
30
|
+
* for longer texts).
|
|
31
|
+
*/
|
|
32
|
+
export declare function editDistance(a: string, b: string): number;
|
|
27
33
|
/** @internal */ export declare function clearCedarCache(): void;
|
|
28
34
|
/**
|
|
29
35
|
* Check a single linter rule reference (e.g., "eslint/no-console").
|
package/dist/linters.js
CHANGED
|
@@ -13,6 +13,7 @@
|
|
|
13
13
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
14
14
|
exports.extractLinterName = extractLinterName;
|
|
15
15
|
exports.extractRuleName = extractRuleName;
|
|
16
|
+
exports.editDistance = editDistance;
|
|
16
17
|
exports.clearCedarCache = clearCedarCache;
|
|
17
18
|
exports.checkLinterRule = checkLinterRule;
|
|
18
19
|
const node_fs_1 = require("node:fs");
|
package/dist/mcp.d.ts
ADDED
|
@@ -0,0 +1,48 @@
|
|
|
1
|
+
export interface McpServerConfig {
|
|
2
|
+
readonly command: string;
|
|
3
|
+
readonly args?: readonly string[];
|
|
4
|
+
readonly env?: Record<string, string>;
|
|
5
|
+
readonly cwd?: string;
|
|
6
|
+
}
|
|
7
|
+
export interface McpToolInfo {
|
|
8
|
+
readonly name: string;
|
|
9
|
+
readonly description?: string;
|
|
10
|
+
}
|
|
11
|
+
/**
|
|
12
|
+
* Start an MCP server over stdio, complete the handshake, and return its tools.
|
|
13
|
+
* Kills the server when done. Throws on spawn/timeout/exit/protocol error.
|
|
14
|
+
*/
|
|
15
|
+
export declare function listMcpTools(server: McpServerConfig, timeoutMs?: number): Promise<McpToolInfo[]>;
|
|
16
|
+
export interface McpRefResult {
|
|
17
|
+
readonly exists: boolean;
|
|
18
|
+
readonly available: string[];
|
|
19
|
+
readonly suggestions: string[];
|
|
20
|
+
}
|
|
21
|
+
/**
|
|
22
|
+
* Verify `toolName` exists on `server`; on a miss, suggest the closest tool names
|
|
23
|
+
* (edit distance) — "did you mean issue_write?".
|
|
24
|
+
*/
|
|
25
|
+
export declare function verifyMcpTool(server: McpServerConfig, toolName: string, timeoutMs?: number): Promise<McpRefResult>;
|
|
26
|
+
export interface McpRef {
|
|
27
|
+
readonly server: string;
|
|
28
|
+
readonly tool: string;
|
|
29
|
+
readonly line: number;
|
|
30
|
+
}
|
|
31
|
+
export type McpRefReason = "server-undeclared" | "server-unreachable" | "tool-missing";
|
|
32
|
+
export interface McpRefError extends McpRef {
|
|
33
|
+
readonly reason: McpRefReason;
|
|
34
|
+
readonly suggestions: string[];
|
|
35
|
+
}
|
|
36
|
+
/** Parse `vigiles:mcp server#tool` marks from a markdown file's inline spans. */
|
|
37
|
+
export declare function parseMcpRefs(markdown: string): McpRef[];
|
|
38
|
+
/** Read `mcpServers` from `.mcp.json` (the stdio-server config map), or `{}`. */
|
|
39
|
+
export declare function loadMcpServers(cwd: string): Record<string, McpServerConfig>;
|
|
40
|
+
/**
|
|
41
|
+
* Verify every `vigiles:mcp server#tool` mark in `markdown` against the live
|
|
42
|
+
* servers in `mcpServers` (each referenced server is started once). A reference
|
|
43
|
+
* to an undeclared server, an unreachable server, or a missing tool is an error.
|
|
44
|
+
*/
|
|
45
|
+
export declare function verifyMcpRefs(markdown: string, mcpServers: Record<string, McpServerConfig>, timeoutMs?: number): Promise<McpRefError[]>;
|
|
46
|
+
/** Human-readable message for an MCP reference error (with "did you mean"). */
|
|
47
|
+
export declare function mcpRefMessage(e: McpRefError): string;
|
|
48
|
+
//# sourceMappingURL=mcp.d.ts.map
|
package/dist/mcp.js
ADDED
|
@@ -0,0 +1,247 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.listMcpTools = listMcpTools;
|
|
4
|
+
exports.verifyMcpTool = verifyMcpTool;
|
|
5
|
+
exports.parseMcpRefs = parseMcpRefs;
|
|
6
|
+
exports.loadMcpServers = loadMcpServers;
|
|
7
|
+
exports.verifyMcpRefs = verifyMcpRefs;
|
|
8
|
+
exports.mcpRefMessage = mcpRefMessage;
|
|
9
|
+
/**
|
|
10
|
+
* Minimal MCP client over stdio — start a server, do the JSON-RPC handshake, and
|
|
11
|
+
* list its tools. This lets vigiles VERIFY a referenced `mcp__server__tool`
|
|
12
|
+
* resolves against the real server (the way `enforce()` resolves a linter rule
|
|
13
|
+
* against its catalog), catching a skill/CLAUDE.md that cites an MCP tool that was
|
|
14
|
+
* renamed or removed — e.g. the GitHub MCP server renaming `create_issue` →
|
|
15
|
+
* `issue_write`, which otherwise fails silently at runtime.
|
|
16
|
+
*
|
|
17
|
+
* MCP stdio transport = newline-delimited JSON-RPC 2.0.
|
|
18
|
+
*/
|
|
19
|
+
const node_child_process_1 = require("node:child_process");
|
|
20
|
+
const node_fs_1 = require("node:fs");
|
|
21
|
+
const node_path_1 = require("node:path");
|
|
22
|
+
const refs_js_1 = require("./refs.js");
|
|
23
|
+
const hash_js_1 = require("./hash.js");
|
|
24
|
+
function dispatch(line, pending) {
|
|
25
|
+
let msg;
|
|
26
|
+
try {
|
|
27
|
+
msg = JSON.parse(line);
|
|
28
|
+
}
|
|
29
|
+
catch {
|
|
30
|
+
return;
|
|
31
|
+
}
|
|
32
|
+
if (typeof msg.id !== "number")
|
|
33
|
+
return;
|
|
34
|
+
const p = pending.get(msg.id);
|
|
35
|
+
if (p) {
|
|
36
|
+
pending.delete(msg.id);
|
|
37
|
+
p.resolve(msg);
|
|
38
|
+
}
|
|
39
|
+
}
|
|
40
|
+
/**
|
|
41
|
+
* Start an MCP server over stdio, complete the handshake, and return its tools.
|
|
42
|
+
* Kills the server when done. Throws on spawn/timeout/exit/protocol error.
|
|
43
|
+
*/
|
|
44
|
+
async function listMcpTools(server, timeoutMs = 10000) {
|
|
45
|
+
const child = (0, node_child_process_1.spawn)(server.command, [...(server.args ?? [])], {
|
|
46
|
+
env: server.env ? { ...process.env, ...server.env } : process.env,
|
|
47
|
+
cwd: server.cwd,
|
|
48
|
+
stdio: ["pipe", "pipe", "ignore"],
|
|
49
|
+
});
|
|
50
|
+
const { stdin, stdout } = child;
|
|
51
|
+
if (!stdin || !stdout) {
|
|
52
|
+
child.kill("SIGKILL");
|
|
53
|
+
throw new Error("failed to open MCP server stdio");
|
|
54
|
+
}
|
|
55
|
+
const pending = new Map();
|
|
56
|
+
const failAll = (err) => {
|
|
57
|
+
for (const [, p] of pending)
|
|
58
|
+
p.reject(err);
|
|
59
|
+
pending.clear();
|
|
60
|
+
};
|
|
61
|
+
let buffer = "";
|
|
62
|
+
stdout.setEncoding("utf-8");
|
|
63
|
+
stdout.on("data", (chunk) => {
|
|
64
|
+
buffer += chunk;
|
|
65
|
+
let nl = buffer.indexOf("\n");
|
|
66
|
+
while (nl >= 0) {
|
|
67
|
+
const line = buffer.slice(0, nl).trim();
|
|
68
|
+
buffer = buffer.slice(nl + 1);
|
|
69
|
+
if (line)
|
|
70
|
+
dispatch(line, pending);
|
|
71
|
+
nl = buffer.indexOf("\n");
|
|
72
|
+
}
|
|
73
|
+
});
|
|
74
|
+
child.on("error", (e) => {
|
|
75
|
+
failAll(e);
|
|
76
|
+
});
|
|
77
|
+
child.on("close", () => {
|
|
78
|
+
failAll(new Error("MCP server exited before responding"));
|
|
79
|
+
});
|
|
80
|
+
const send = (obj) => {
|
|
81
|
+
stdin.write(`${JSON.stringify(obj)}\n`);
|
|
82
|
+
};
|
|
83
|
+
const request = (id, method, params) => new Promise((resolve, reject) => {
|
|
84
|
+
pending.set(id, { resolve, reject });
|
|
85
|
+
send({ jsonrpc: "2.0", id, method, params });
|
|
86
|
+
});
|
|
87
|
+
const timer = setTimeout(() => {
|
|
88
|
+
failAll(new Error(`MCP server timed out after ${String(timeoutMs)}ms`));
|
|
89
|
+
child.kill("SIGKILL");
|
|
90
|
+
}, timeoutMs);
|
|
91
|
+
try {
|
|
92
|
+
await request(1, "initialize", {
|
|
93
|
+
protocolVersion: "2024-11-05",
|
|
94
|
+
capabilities: {},
|
|
95
|
+
clientInfo: { name: "vigiles", version: "0" },
|
|
96
|
+
});
|
|
97
|
+
send({ jsonrpc: "2.0", method: "notifications/initialized" });
|
|
98
|
+
const res = await request(2, "tools/list", {});
|
|
99
|
+
if (res.error) {
|
|
100
|
+
throw new Error(`tools/list failed: ${res.error.message ?? "unknown"}`);
|
|
101
|
+
}
|
|
102
|
+
return (res.result?.tools ?? []).map((t) => ({
|
|
103
|
+
name: t.name,
|
|
104
|
+
description: t.description,
|
|
105
|
+
}));
|
|
106
|
+
}
|
|
107
|
+
finally {
|
|
108
|
+
clearTimeout(timer);
|
|
109
|
+
child.kill("SIGKILL");
|
|
110
|
+
}
|
|
111
|
+
}
|
|
112
|
+
/**
|
|
113
|
+
* Verify `toolName` exists on `server`; on a miss, suggest the closest tool names
|
|
114
|
+
* (edit distance) — "did you mean issue_write?".
|
|
115
|
+
*/
|
|
116
|
+
async function verifyMcpTool(server, toolName, timeoutMs = 10000) {
|
|
117
|
+
const available = (await listMcpTools(server, timeoutMs)).map((t) => t.name);
|
|
118
|
+
const exists = available.includes(toolName);
|
|
119
|
+
return {
|
|
120
|
+
exists,
|
|
121
|
+
available,
|
|
122
|
+
suggestions: exists ? [] : closest(toolName, available),
|
|
123
|
+
};
|
|
124
|
+
}
|
|
125
|
+
function editDistance(a, b) {
|
|
126
|
+
if (a === b)
|
|
127
|
+
return 0;
|
|
128
|
+
const m = a.length;
|
|
129
|
+
const n = b.length;
|
|
130
|
+
if (m === 0)
|
|
131
|
+
return n;
|
|
132
|
+
if (n === 0)
|
|
133
|
+
return m;
|
|
134
|
+
const dp = Array.from({ length: n + 1 }, (_, i) => i);
|
|
135
|
+
for (let i = 1; i <= m; i++) {
|
|
136
|
+
let prev = dp[0];
|
|
137
|
+
dp[0] = i;
|
|
138
|
+
for (let j = 1; j <= n; j++) {
|
|
139
|
+
const tmp = dp[j];
|
|
140
|
+
dp[j] =
|
|
141
|
+
a[i - 1] === b[j - 1] ? prev : 1 + Math.min(prev, dp[j], dp[j - 1]);
|
|
142
|
+
prev = tmp;
|
|
143
|
+
}
|
|
144
|
+
}
|
|
145
|
+
return dp[n];
|
|
146
|
+
}
|
|
147
|
+
function closest(target, candidates, max = 4) {
|
|
148
|
+
return candidates
|
|
149
|
+
.map((c) => ({ c, d: editDistance(target, c) }))
|
|
150
|
+
.filter((x) => x.d <= max)
|
|
151
|
+
.sort((a, b) => a.d - b.d)
|
|
152
|
+
.slice(0, 3)
|
|
153
|
+
.map((x) => x.c);
|
|
154
|
+
}
|
|
155
|
+
// --- MCP tool references in instruction files ------------------------------
|
|
156
|
+
// `vigiles:mcp <server>#<tool>` inside an inline code span — the MCP analogue of
|
|
157
|
+
// the `vigiles:symbol path#name` mark. Self-contained (server + tool in one
|
|
158
|
+
// token) so it binds unambiguously.
|
|
159
|
+
const MCP_MARK = /^vigiles:mcp\s+([\w-]+)#([\w.-]+)$/;
|
|
160
|
+
/** Parse `vigiles:mcp server#tool` marks from a markdown file's inline spans. */
|
|
161
|
+
function parseMcpRefs(markdown) {
|
|
162
|
+
const refs = [];
|
|
163
|
+
for (const span of (0, refs_js_1.inlineSpans)(markdown)) {
|
|
164
|
+
const m = MCP_MARK.exec(span.text);
|
|
165
|
+
if (m)
|
|
166
|
+
refs.push({ server: m[1], tool: m[2], line: span.line });
|
|
167
|
+
}
|
|
168
|
+
return refs;
|
|
169
|
+
}
|
|
170
|
+
/** Read `mcpServers` from `.mcp.json` (the stdio-server config map), or `{}`. */
|
|
171
|
+
function loadMcpServers(cwd) {
|
|
172
|
+
const p = (0, node_path_1.join)(cwd, ".mcp.json");
|
|
173
|
+
if (!(0, node_fs_1.existsSync)(p))
|
|
174
|
+
return {};
|
|
175
|
+
try {
|
|
176
|
+
const json = JSON.parse((0, node_fs_1.readFileSync)(p, "utf-8"));
|
|
177
|
+
return json.mcpServers ?? {};
|
|
178
|
+
}
|
|
179
|
+
catch {
|
|
180
|
+
return {};
|
|
181
|
+
}
|
|
182
|
+
}
|
|
183
|
+
async function verifyOneServer(group, cfg, timeoutMs) {
|
|
184
|
+
if (!cfg) {
|
|
185
|
+
return group.map((r) => ({
|
|
186
|
+
...r,
|
|
187
|
+
reason: "server-undeclared",
|
|
188
|
+
suggestions: [],
|
|
189
|
+
}));
|
|
190
|
+
}
|
|
191
|
+
let available;
|
|
192
|
+
try {
|
|
193
|
+
available = (await listMcpTools(cfg, timeoutMs)).map((t) => t.name);
|
|
194
|
+
}
|
|
195
|
+
catch {
|
|
196
|
+
return group.map((r) => ({
|
|
197
|
+
...r,
|
|
198
|
+
reason: "server-unreachable",
|
|
199
|
+
suggestions: [],
|
|
200
|
+
}));
|
|
201
|
+
}
|
|
202
|
+
const errs = [];
|
|
203
|
+
for (const r of group) {
|
|
204
|
+
if (!available.includes(r.tool)) {
|
|
205
|
+
errs.push({
|
|
206
|
+
...r,
|
|
207
|
+
reason: "tool-missing",
|
|
208
|
+
suggestions: closest(r.tool, available),
|
|
209
|
+
});
|
|
210
|
+
}
|
|
211
|
+
}
|
|
212
|
+
return errs;
|
|
213
|
+
}
|
|
214
|
+
/**
|
|
215
|
+
* Verify every `vigiles:mcp server#tool` mark in `markdown` against the live
|
|
216
|
+
* servers in `mcpServers` (each referenced server is started once). A reference
|
|
217
|
+
* to an undeclared server, an unreachable server, or a missing tool is an error.
|
|
218
|
+
*/
|
|
219
|
+
async function verifyMcpRefs(markdown, mcpServers, timeoutMs = 10000) {
|
|
220
|
+
const byServer = new Map();
|
|
221
|
+
for (const r of parseMcpRefs(markdown)) {
|
|
222
|
+
const arr = byServer.get(r.server) ?? [];
|
|
223
|
+
arr.push(r);
|
|
224
|
+
byServer.set(r.server, arr);
|
|
225
|
+
}
|
|
226
|
+
const all = [];
|
|
227
|
+
for (const [server, group] of byServer) {
|
|
228
|
+
all.push(...(await verifyOneServer(group, mcpServers[server], timeoutMs)));
|
|
229
|
+
}
|
|
230
|
+
return all;
|
|
231
|
+
}
|
|
232
|
+
/** Human-readable message for an MCP reference error (with "did you mean"). */
|
|
233
|
+
function mcpRefMessage(e) {
|
|
234
|
+
switch (e.reason) {
|
|
235
|
+
case "server-undeclared":
|
|
236
|
+
return `MCP server "${e.server}" is not declared in .mcp.json`;
|
|
237
|
+
case "server-unreachable":
|
|
238
|
+
return `MCP server "${e.server}" failed to start`;
|
|
239
|
+
case "tool-missing":
|
|
240
|
+
return `MCP tool "${e.server}#${e.tool}" not found${e.suggestions.length > 0
|
|
241
|
+
? ` — did you mean ${e.suggestions.map((s) => `"${s}"`).join(", ")}?`
|
|
242
|
+
: ""}`;
|
|
243
|
+
default:
|
|
244
|
+
return (0, hash_js_1.assertNever)(e.reason);
|
|
245
|
+
}
|
|
246
|
+
}
|
|
247
|
+
//# sourceMappingURL=mcp.js.map
|
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
/**
|
|
4
|
+
* vigiles — the in-sandbox mock entry.
|
|
5
|
+
*
|
|
6
|
+
* Run as a subprocess INSIDE the bubblewrap network namespace (see
|
|
7
|
+
* `src/sandbox.ts`), so the scripted mock lives on the sandbox's isolated
|
|
8
|
+
* loopback — reachable by the confined `claude`, unreachable from outside.
|
|
9
|
+
* Reads the model script from a file, streams each captured request to an ndjson
|
|
10
|
+
* file the parent reads back (for `trace.modelRequests`), and writes its chosen
|
|
11
|
+
* port so the wrapper can point `ANTHROPIC_BASE_URL` at it.
|
|
12
|
+
*
|
|
13
|
+
* node mock-entry.js <scriptFile> <requestsFile> <portFile>
|
|
14
|
+
*
|
|
15
|
+
* Not unit-tested directly (it's a daemon driven only through a live sandbox);
|
|
16
|
+
* exercised end-to-end by the bwrap-backed integration test.
|
|
17
|
+
*/
|
|
18
|
+
const node_fs_1 = require("node:fs");
|
|
19
|
+
const mock_model_js_1 = require("./mock-model.js");
|
|
20
|
+
void (async () => {
|
|
21
|
+
const [scriptFile, requestsFile, portFile] = process.argv.slice(2);
|
|
22
|
+
if (!scriptFile || !requestsFile || !portFile) {
|
|
23
|
+
process.stderr.write("mock-entry: scriptFile requestsFile portFile\n");
|
|
24
|
+
process.exit(2);
|
|
25
|
+
}
|
|
26
|
+
const turns = JSON.parse((0, node_fs_1.readFileSync)(scriptFile, "utf-8"));
|
|
27
|
+
const handle = await (0, mock_model_js_1.startMock)(turns, {
|
|
28
|
+
onRequest: (req) => {
|
|
29
|
+
(0, node_fs_1.appendFileSync)(requestsFile, JSON.stringify(req) + "\n");
|
|
30
|
+
},
|
|
31
|
+
});
|
|
32
|
+
// Signal readiness last: the wrapper waits for a non-empty port file.
|
|
33
|
+
(0, node_fs_1.writeFileSync)(portFile, new URL(handle.url).port);
|
|
34
|
+
// Stay alive until the wrapper kills us once `claude` has finished.
|
|
35
|
+
})();
|
|
36
|
+
//# sourceMappingURL=mock-entry.js.map
|