vigiles 2.2.0 → 2.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +176 -152
- package/dist/cli.js +91 -1
- package/dist/eval.d.ts +34 -1
- package/dist/eval.js +55 -20
- package/dist/harness-assert.d.ts +109 -0
- package/dist/harness-assert.js +220 -0
- package/dist/harness-test.d.ts +46 -0
- package/dist/harness-test.js +81 -9
- package/dist/jest.d.ts +9 -0
- package/dist/jest.js +23 -0
- package/dist/judge.d.ts +29 -0
- package/dist/judge.js +88 -0
- package/dist/mcp.d.ts +48 -0
- package/dist/mcp.js +247 -0
- package/dist/plugin-loader.d.ts +37 -0
- package/dist/plugin-loader.js +195 -0
- package/dist/run-hook.d.ts +77 -0
- package/dist/run-hook.js +80 -0
- package/dist/run-scripts.d.ts +20 -0
- package/dist/run-scripts.js +70 -0
- package/dist/vitest.d.mts +9 -0
- package/dist/vitest.mjs +22 -0
- package/package.json +37 -5
package/dist/harness-test.js
CHANGED
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
-
exports.scriptModel = void 0;
|
|
3
|
+
exports.resolveHarness = exports.loadPlugin = exports.scriptModel = void 0;
|
|
4
|
+
exports.parseToolCalls = parseToolCalls;
|
|
4
5
|
exports.claudeAvailable = claudeAvailable;
|
|
5
6
|
exports.runHarnessTest = runHarnessTest;
|
|
6
7
|
/**
|
|
@@ -37,8 +38,69 @@ const node_fs_1 = require("node:fs");
|
|
|
37
38
|
const node_os_1 = require("node:os");
|
|
38
39
|
const node_path_1 = require("node:path");
|
|
39
40
|
const mock_model_js_1 = require("./mock-model.js");
|
|
41
|
+
const plugin_loader_js_1 = require("./plugin-loader.js");
|
|
40
42
|
var mock_model_js_2 = require("./mock-model.js");
|
|
41
43
|
Object.defineProperty(exports, "scriptModel", { enumerable: true, get: function () { return mock_model_js_2.scriptModel; } });
|
|
44
|
+
var plugin_loader_js_2 = require("./plugin-loader.js");
|
|
45
|
+
Object.defineProperty(exports, "loadPlugin", { enumerable: true, get: function () { return plugin_loader_js_2.loadPlugin; } });
|
|
46
|
+
Object.defineProperty(exports, "resolveHarness", { enumerable: true, get: function () { return plugin_loader_js_2.resolveHarness; } });
|
|
47
|
+
function contentText(content) {
|
|
48
|
+
if (typeof content === "string")
|
|
49
|
+
return content;
|
|
50
|
+
if (!Array.isArray(content))
|
|
51
|
+
return "";
|
|
52
|
+
return content
|
|
53
|
+
.map((b) => {
|
|
54
|
+
if (typeof b === "string")
|
|
55
|
+
return b;
|
|
56
|
+
const t = b.text;
|
|
57
|
+
return typeof t === "string" ? t : "";
|
|
58
|
+
})
|
|
59
|
+
.join("");
|
|
60
|
+
}
|
|
61
|
+
/**
|
|
62
|
+
* Parse `--output-format stream-json` (the `transcript: true` output) into the
|
|
63
|
+
* tools the agent invoked, each joined to its result by id. Returns [] for the
|
|
64
|
+
* non-stream `json` output. The seam that lets a test assert on the agent's
|
|
65
|
+
* actions, not a brittle stdout substring.
|
|
66
|
+
*/
|
|
67
|
+
function parseToolCalls(streamJson) {
|
|
68
|
+
const uses = [];
|
|
69
|
+
const results = new Map();
|
|
70
|
+
for (const line of streamJson.split("\n")) {
|
|
71
|
+
if (!line.trim())
|
|
72
|
+
continue;
|
|
73
|
+
let evt;
|
|
74
|
+
try {
|
|
75
|
+
evt = JSON.parse(line);
|
|
76
|
+
}
|
|
77
|
+
catch {
|
|
78
|
+
continue;
|
|
79
|
+
}
|
|
80
|
+
const content = evt.message?.content;
|
|
81
|
+
if (!Array.isArray(content))
|
|
82
|
+
continue;
|
|
83
|
+
for (const b of content) {
|
|
84
|
+
if (b.type === "tool_use" && typeof b.name === "string") {
|
|
85
|
+
const id = typeof b.id === "string" ? b.id : "";
|
|
86
|
+
uses.push({ id, name: b.name, input: b.input });
|
|
87
|
+
}
|
|
88
|
+
else if (b.type === "tool_result") {
|
|
89
|
+
const id = typeof b.tool_use_id === "string" ? b.tool_use_id : "";
|
|
90
|
+
results.set(id, {
|
|
91
|
+
text: contentText(b.content),
|
|
92
|
+
isError: b.is_error === true,
|
|
93
|
+
});
|
|
94
|
+
}
|
|
95
|
+
}
|
|
96
|
+
}
|
|
97
|
+
return uses.map((u) => ({
|
|
98
|
+
name: u.name,
|
|
99
|
+
input: u.input,
|
|
100
|
+
resultText: results.get(u.id)?.text ?? "",
|
|
101
|
+
isError: results.get(u.id)?.isError ?? false,
|
|
102
|
+
}));
|
|
103
|
+
}
|
|
42
104
|
/** Whether the `claude` CLI is available — harness tests need it. */
|
|
43
105
|
function claudeAvailable() {
|
|
44
106
|
try {
|
|
@@ -48,17 +110,17 @@ function claudeAvailable() {
|
|
|
48
110
|
return false;
|
|
49
111
|
}
|
|
50
112
|
}
|
|
51
|
-
function writeFixture(cwd,
|
|
52
|
-
for (const [p, content] of Object.entries(
|
|
113
|
+
function writeFixture(cwd, files, settings) {
|
|
114
|
+
for (const [p, content] of Object.entries(files)) {
|
|
53
115
|
const full = (0, node_path_1.resolve)(cwd, p);
|
|
54
116
|
(0, node_fs_1.mkdirSync)((0, node_path_1.dirname)(full), { recursive: true });
|
|
55
117
|
(0, node_fs_1.writeFileSync)(full, content);
|
|
56
118
|
}
|
|
57
|
-
if (
|
|
119
|
+
if (settings !== undefined) {
|
|
58
120
|
// `{cwd}` in any hook command is substituted with the working dir, so a
|
|
59
121
|
// hook can reference an absolute path inside it (hooks don't run with the
|
|
60
122
|
// project dir as cwd).
|
|
61
|
-
const json = JSON.stringify(
|
|
123
|
+
const json = JSON.stringify(settings, null, 2).replaceAll("{cwd}", cwd);
|
|
62
124
|
(0, node_fs_1.writeFileSync)((0, node_path_1.join)(cwd, "settings.json"), json);
|
|
63
125
|
}
|
|
64
126
|
}
|
|
@@ -91,18 +153,27 @@ function spawnClaude(args, cwd, baseUrl, timeoutMs) {
|
|
|
91
153
|
*/
|
|
92
154
|
async function runHarnessTest(spec) {
|
|
93
155
|
const cwd = (0, node_fs_1.mkdtempSync)((0, node_path_1.join)((0, node_os_1.tmpdir)(), "vigiles-harness-"));
|
|
94
|
-
|
|
156
|
+
const { files, settings } = (0, plugin_loader_js_1.resolveHarness)({
|
|
157
|
+
plugin: spec.plugin,
|
|
158
|
+
settings: spec.settings,
|
|
159
|
+
files: spec.files,
|
|
160
|
+
});
|
|
161
|
+
writeFixture(cwd, files, settings);
|
|
95
162
|
const mock = await (0, mock_model_js_1.startMock)(spec.model);
|
|
96
163
|
try {
|
|
97
164
|
const tools = spec.allowedTools ?? ["Read", "Edit", "Write", "Bash"];
|
|
98
165
|
const args = [
|
|
99
166
|
"-p",
|
|
100
167
|
spec.prompt ?? "go",
|
|
101
|
-
|
|
102
|
-
|
|
168
|
+
...(spec.transcript
|
|
169
|
+
? ["--output-format", "stream-json", "--verbose"]
|
|
170
|
+
: ["--output-format", "json"]),
|
|
103
171
|
"--model",
|
|
104
172
|
"claude-sonnet-4-5",
|
|
105
|
-
...(spec.
|
|
173
|
+
...(spec.pluginDir !== undefined
|
|
174
|
+
? ["--plugin-dir", (0, node_path_1.resolve)(spec.pluginDir)]
|
|
175
|
+
: []),
|
|
176
|
+
...(settings !== undefined ? ["--settings", "settings.json"] : []),
|
|
106
177
|
"--allowedTools",
|
|
107
178
|
...tools,
|
|
108
179
|
];
|
|
@@ -113,6 +184,7 @@ async function runHarnessTest(spec) {
|
|
|
113
184
|
stderr: out.stderr,
|
|
114
185
|
cwd,
|
|
115
186
|
turns: mock.count,
|
|
187
|
+
toolCalls: parseToolCalls(out.stdout),
|
|
116
188
|
file: (p) => {
|
|
117
189
|
const f = (0, node_path_1.resolve)(cwd, p);
|
|
118
190
|
return (0, node_fs_1.existsSync)(f) ? (0, node_fs_1.readFileSync)(f, "utf-8") : null;
|
package/dist/jest.d.ts
ADDED
package/dist/jest.js
ADDED
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
/* eslint-disable max-params --
|
|
4
|
+
The matcher signatures mirror the runtime vigilesMatchers (positional args). */
|
|
5
|
+
/**
|
|
6
|
+
* vigiles — jest integration (opt-in).
|
|
7
|
+
*
|
|
8
|
+
* Importing this entry registers the vigiles matchers AND augments jest's types
|
|
9
|
+
* so `toHaveCreated` / `toBeatBaseline` type-check.
|
|
10
|
+
*
|
|
11
|
+
* // jest.config.js → setupFilesAfterEnv: ["vigiles/jest"]
|
|
12
|
+
* // …or at the top of a test file:
|
|
13
|
+
* import "vigiles/jest";
|
|
14
|
+
*
|
|
15
|
+
* expect(result).toHaveCreated("DONE");
|
|
16
|
+
* expect(report).toBeatBaseline("vanilla", "gated", "caught");
|
|
17
|
+
*
|
|
18
|
+
* jest is an optional peer dependency — only jest users load this entry.
|
|
19
|
+
*/
|
|
20
|
+
const globals_1 = require("@jest/globals");
|
|
21
|
+
const harness_assert_js_1 = require("./harness-assert.js");
|
|
22
|
+
globals_1.expect.extend(harness_assert_js_1.vigilesMatchers);
|
|
23
|
+
//# sourceMappingURL=jest.js.map
|
package/dist/judge.d.ts
ADDED
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
export interface JudgeResult {
|
|
2
|
+
/** Score in [0, 1] (clamped). 0 on any failure to obtain a verdict. */
|
|
3
|
+
readonly score: number;
|
|
4
|
+
/** score ≥ threshold (default 0.5). */
|
|
5
|
+
readonly pass: boolean;
|
|
6
|
+
/** The model's one-line rationale, or an error string. */
|
|
7
|
+
readonly reason: string;
|
|
8
|
+
}
|
|
9
|
+
export interface JudgeOptions {
|
|
10
|
+
/** The text to grade. */
|
|
11
|
+
readonly output: string;
|
|
12
|
+
/** The grading rubric — describe what earns a high vs low score. */
|
|
13
|
+
readonly rubric: string;
|
|
14
|
+
/** Model alias. Default "haiku" (cheap; judging is a simple call). */
|
|
15
|
+
readonly model?: string;
|
|
16
|
+
/** pass = score ≥ threshold. Default 0.5. */
|
|
17
|
+
readonly threshold?: number;
|
|
18
|
+
/** Per-call timeout ms. Default 60000. */
|
|
19
|
+
readonly timeoutMs?: number;
|
|
20
|
+
}
|
|
21
|
+
/** Grade `output` against `rubric` with a model. Synchronous (for `measure`). */
|
|
22
|
+
export declare function judge(opts: JudgeOptions): JudgeResult;
|
|
23
|
+
/**
|
|
24
|
+
* Parse a verdict out of the grader's stdout — pure, so the parsing is testable
|
|
25
|
+
* without a model. Handles `claude --output-format json` (text wrapped in a
|
|
26
|
+
* `result` field), bare/prose-wrapped JSON, clamping, and the pass threshold.
|
|
27
|
+
*/
|
|
28
|
+
export declare function parseJudgeOutput(stdout: string, threshold?: number): JudgeResult;
|
|
29
|
+
//# sourceMappingURL=judge.d.ts.map
|
package/dist/judge.js
ADDED
|
@@ -0,0 +1,88 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.judge = judge;
|
|
4
|
+
exports.parseJudgeOutput = parseJudgeOutput;
|
|
5
|
+
/**
|
|
6
|
+
* vigiles — a thin LLM-as-judge for the eval tier.
|
|
7
|
+
*
|
|
8
|
+
* Some outcomes aren't a regex: "is this commit message clear?", "did the SKILL
|
|
9
|
+
* produce a sensible plan?". `judge` grades an output against a rubric with a
|
|
10
|
+
* model and returns a numeric score + pass/fail, for use *inside* an eval's
|
|
11
|
+
* `measure` (which is synchronous — so this shells out via the `claude` CLI
|
|
12
|
+
* synchronously, no extra deps):
|
|
13
|
+
*
|
|
14
|
+
* measure: (ctx) => {
|
|
15
|
+
* const v = judge({ output: ctx.file("PLAN.md") ?? "", rubric:
|
|
16
|
+
* "1 if the plan lists concrete, ordered steps; else 0." });
|
|
17
|
+
* return { quality: v.score, ok: v.pass };
|
|
18
|
+
* }
|
|
19
|
+
*
|
|
20
|
+
* This is deliberately minimal — for datasets, tracing, and dashboards use a
|
|
21
|
+
* dedicated eval platform (Braintrust, DeepEval). vigiles owns the harness A/B,
|
|
22
|
+
* not the judging platform. Needs the `claude` CLI + model auth.
|
|
23
|
+
*/
|
|
24
|
+
const node_child_process_1 = require("node:child_process");
|
|
25
|
+
const clamp01 = (n) => Math.max(0, Math.min(1, n));
|
|
26
|
+
/** Extract the first JSON object from a string (models often wrap it in prose). */
|
|
27
|
+
function firstJsonObject(s) {
|
|
28
|
+
const start = s.indexOf("{");
|
|
29
|
+
const end = s.lastIndexOf("}");
|
|
30
|
+
if (start === -1 || end <= start)
|
|
31
|
+
return null;
|
|
32
|
+
try {
|
|
33
|
+
return JSON.parse(s.slice(start, end + 1));
|
|
34
|
+
}
|
|
35
|
+
catch {
|
|
36
|
+
return null;
|
|
37
|
+
}
|
|
38
|
+
}
|
|
39
|
+
/** Grade `output` against `rubric` with a model. Synchronous (for `measure`). */
|
|
40
|
+
function judge(opts) {
|
|
41
|
+
const threshold = opts.threshold ?? 0.5;
|
|
42
|
+
const prompt = "You are a strict grader. Score the OUTPUT against the RUBRIC. " +
|
|
43
|
+
'Respond with ONLY a JSON object: {"score": <number 0..1>, "reason": "<one line>"}.\n\n' +
|
|
44
|
+
`RUBRIC:\n${opts.rubric}\n\nOUTPUT:\n${opts.output}`;
|
|
45
|
+
let res;
|
|
46
|
+
try {
|
|
47
|
+
res = (0, node_child_process_1.spawnSync)("claude", [
|
|
48
|
+
"-p",
|
|
49
|
+
prompt,
|
|
50
|
+
"--model",
|
|
51
|
+
opts.model ?? "haiku",
|
|
52
|
+
"--output-format",
|
|
53
|
+
"json",
|
|
54
|
+
], { encoding: "utf-8", timeout: opts.timeoutMs ?? 60000 });
|
|
55
|
+
}
|
|
56
|
+
catch (e) {
|
|
57
|
+
return {
|
|
58
|
+
score: 0,
|
|
59
|
+
pass: false,
|
|
60
|
+
reason: `judge spawn failed: ${String(e)}`,
|
|
61
|
+
};
|
|
62
|
+
}
|
|
63
|
+
if (res.status !== 0) {
|
|
64
|
+
return { score: 0, pass: false, reason: "judge: no model output" };
|
|
65
|
+
}
|
|
66
|
+
return parseJudgeOutput(res.stdout ?? "", threshold);
|
|
67
|
+
}
|
|
68
|
+
/**
|
|
69
|
+
* Parse a verdict out of the grader's stdout — pure, so the parsing is testable
|
|
70
|
+
* without a model. Handles `claude --output-format json` (text wrapped in a
|
|
71
|
+
* `result` field), bare/prose-wrapped JSON, clamping, and the pass threshold.
|
|
72
|
+
*/
|
|
73
|
+
function parseJudgeOutput(stdout, threshold = 0.5) {
|
|
74
|
+
if (!stdout)
|
|
75
|
+
return { score: 0, pass: false, reason: "judge: no model output" };
|
|
76
|
+
// claude --output-format json wraps the model text in a `result` field.
|
|
77
|
+
let text = stdout;
|
|
78
|
+
const wrapper = firstJsonObject(stdout);
|
|
79
|
+
if (wrapper && typeof wrapper.result === "string")
|
|
80
|
+
text = wrapper.result;
|
|
81
|
+
const verdict = firstJsonObject(text);
|
|
82
|
+
if (!verdict || typeof verdict.score !== "number") {
|
|
83
|
+
return { score: 0, pass: false, reason: "judge: unparseable verdict" };
|
|
84
|
+
}
|
|
85
|
+
const score = clamp01(verdict.score);
|
|
86
|
+
return { score, pass: score >= threshold, reason: verdict.reason ?? "" };
|
|
87
|
+
}
|
|
88
|
+
//# sourceMappingURL=judge.js.map
|
package/dist/mcp.d.ts
ADDED
|
@@ -0,0 +1,48 @@
|
|
|
1
|
+
export interface McpServerConfig {
|
|
2
|
+
readonly command: string;
|
|
3
|
+
readonly args?: readonly string[];
|
|
4
|
+
readonly env?: Record<string, string>;
|
|
5
|
+
readonly cwd?: string;
|
|
6
|
+
}
|
|
7
|
+
export interface McpToolInfo {
|
|
8
|
+
readonly name: string;
|
|
9
|
+
readonly description?: string;
|
|
10
|
+
}
|
|
11
|
+
/**
|
|
12
|
+
* Start an MCP server over stdio, complete the handshake, and return its tools.
|
|
13
|
+
* Kills the server when done. Throws on spawn/timeout/exit/protocol error.
|
|
14
|
+
*/
|
|
15
|
+
export declare function listMcpTools(server: McpServerConfig, timeoutMs?: number): Promise<McpToolInfo[]>;
|
|
16
|
+
export interface McpRefResult {
|
|
17
|
+
readonly exists: boolean;
|
|
18
|
+
readonly available: string[];
|
|
19
|
+
readonly suggestions: string[];
|
|
20
|
+
}
|
|
21
|
+
/**
|
|
22
|
+
* Verify `toolName` exists on `server`; on a miss, suggest the closest tool names
|
|
23
|
+
* (edit distance) — "did you mean issue_write?".
|
|
24
|
+
*/
|
|
25
|
+
export declare function verifyMcpTool(server: McpServerConfig, toolName: string, timeoutMs?: number): Promise<McpRefResult>;
|
|
26
|
+
export interface McpRef {
|
|
27
|
+
readonly server: string;
|
|
28
|
+
readonly tool: string;
|
|
29
|
+
readonly line: number;
|
|
30
|
+
}
|
|
31
|
+
export type McpRefReason = "server-undeclared" | "server-unreachable" | "tool-missing";
|
|
32
|
+
export interface McpRefError extends McpRef {
|
|
33
|
+
readonly reason: McpRefReason;
|
|
34
|
+
readonly suggestions: string[];
|
|
35
|
+
}
|
|
36
|
+
/** Parse `vigiles:mcp server#tool` marks from a markdown file's inline spans. */
|
|
37
|
+
export declare function parseMcpRefs(markdown: string): McpRef[];
|
|
38
|
+
/** Read `mcpServers` from `.mcp.json` (the stdio-server config map), or `{}`. */
|
|
39
|
+
export declare function loadMcpServers(cwd: string): Record<string, McpServerConfig>;
|
|
40
|
+
/**
|
|
41
|
+
* Verify every `vigiles:mcp server#tool` mark in `markdown` against the live
|
|
42
|
+
* servers in `mcpServers` (each referenced server is started once). A reference
|
|
43
|
+
* to an undeclared server, an unreachable server, or a missing tool is an error.
|
|
44
|
+
*/
|
|
45
|
+
export declare function verifyMcpRefs(markdown: string, mcpServers: Record<string, McpServerConfig>, timeoutMs?: number): Promise<McpRefError[]>;
|
|
46
|
+
/** Human-readable message for an MCP reference error (with "did you mean"). */
|
|
47
|
+
export declare function mcpRefMessage(e: McpRefError): string;
|
|
48
|
+
//# sourceMappingURL=mcp.d.ts.map
|
package/dist/mcp.js
ADDED
|
@@ -0,0 +1,247 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.listMcpTools = listMcpTools;
|
|
4
|
+
exports.verifyMcpTool = verifyMcpTool;
|
|
5
|
+
exports.parseMcpRefs = parseMcpRefs;
|
|
6
|
+
exports.loadMcpServers = loadMcpServers;
|
|
7
|
+
exports.verifyMcpRefs = verifyMcpRefs;
|
|
8
|
+
exports.mcpRefMessage = mcpRefMessage;
|
|
9
|
+
/**
|
|
10
|
+
* Minimal MCP client over stdio — start a server, do the JSON-RPC handshake, and
|
|
11
|
+
* list its tools. This lets vigiles VERIFY a referenced `mcp__server__tool`
|
|
12
|
+
* resolves against the real server (the way `enforce()` resolves a linter rule
|
|
13
|
+
* against its catalog), catching a skill/CLAUDE.md that cites an MCP tool that was
|
|
14
|
+
* renamed or removed — e.g. the GitHub MCP server renaming `create_issue` →
|
|
15
|
+
* `issue_write`, which otherwise fails silently at runtime.
|
|
16
|
+
*
|
|
17
|
+
* MCP stdio transport = newline-delimited JSON-RPC 2.0.
|
|
18
|
+
*/
|
|
19
|
+
const node_child_process_1 = require("node:child_process");
|
|
20
|
+
const node_fs_1 = require("node:fs");
|
|
21
|
+
const node_path_1 = require("node:path");
|
|
22
|
+
const refs_js_1 = require("./refs.js");
|
|
23
|
+
const hash_js_1 = require("./hash.js");
|
|
24
|
+
function dispatch(line, pending) {
|
|
25
|
+
let msg;
|
|
26
|
+
try {
|
|
27
|
+
msg = JSON.parse(line);
|
|
28
|
+
}
|
|
29
|
+
catch {
|
|
30
|
+
return;
|
|
31
|
+
}
|
|
32
|
+
if (typeof msg.id !== "number")
|
|
33
|
+
return;
|
|
34
|
+
const p = pending.get(msg.id);
|
|
35
|
+
if (p) {
|
|
36
|
+
pending.delete(msg.id);
|
|
37
|
+
p.resolve(msg);
|
|
38
|
+
}
|
|
39
|
+
}
|
|
40
|
+
/**
|
|
41
|
+
* Start an MCP server over stdio, complete the handshake, and return its tools.
|
|
42
|
+
* Kills the server when done. Throws on spawn/timeout/exit/protocol error.
|
|
43
|
+
*/
|
|
44
|
+
async function listMcpTools(server, timeoutMs = 10000) {
|
|
45
|
+
const child = (0, node_child_process_1.spawn)(server.command, [...(server.args ?? [])], {
|
|
46
|
+
env: server.env ? { ...process.env, ...server.env } : process.env,
|
|
47
|
+
cwd: server.cwd,
|
|
48
|
+
stdio: ["pipe", "pipe", "ignore"],
|
|
49
|
+
});
|
|
50
|
+
const { stdin, stdout } = child;
|
|
51
|
+
if (!stdin || !stdout) {
|
|
52
|
+
child.kill("SIGKILL");
|
|
53
|
+
throw new Error("failed to open MCP server stdio");
|
|
54
|
+
}
|
|
55
|
+
const pending = new Map();
|
|
56
|
+
const failAll = (err) => {
|
|
57
|
+
for (const [, p] of pending)
|
|
58
|
+
p.reject(err);
|
|
59
|
+
pending.clear();
|
|
60
|
+
};
|
|
61
|
+
let buffer = "";
|
|
62
|
+
stdout.setEncoding("utf-8");
|
|
63
|
+
stdout.on("data", (chunk) => {
|
|
64
|
+
buffer += chunk;
|
|
65
|
+
let nl = buffer.indexOf("\n");
|
|
66
|
+
while (nl >= 0) {
|
|
67
|
+
const line = buffer.slice(0, nl).trim();
|
|
68
|
+
buffer = buffer.slice(nl + 1);
|
|
69
|
+
if (line)
|
|
70
|
+
dispatch(line, pending);
|
|
71
|
+
nl = buffer.indexOf("\n");
|
|
72
|
+
}
|
|
73
|
+
});
|
|
74
|
+
child.on("error", (e) => {
|
|
75
|
+
failAll(e);
|
|
76
|
+
});
|
|
77
|
+
child.on("close", () => {
|
|
78
|
+
failAll(new Error("MCP server exited before responding"));
|
|
79
|
+
});
|
|
80
|
+
const send = (obj) => {
|
|
81
|
+
stdin.write(`${JSON.stringify(obj)}\n`);
|
|
82
|
+
};
|
|
83
|
+
const request = (id, method, params) => new Promise((resolve, reject) => {
|
|
84
|
+
pending.set(id, { resolve, reject });
|
|
85
|
+
send({ jsonrpc: "2.0", id, method, params });
|
|
86
|
+
});
|
|
87
|
+
const timer = setTimeout(() => {
|
|
88
|
+
failAll(new Error(`MCP server timed out after ${String(timeoutMs)}ms`));
|
|
89
|
+
child.kill("SIGKILL");
|
|
90
|
+
}, timeoutMs);
|
|
91
|
+
try {
|
|
92
|
+
await request(1, "initialize", {
|
|
93
|
+
protocolVersion: "2024-11-05",
|
|
94
|
+
capabilities: {},
|
|
95
|
+
clientInfo: { name: "vigiles", version: "0" },
|
|
96
|
+
});
|
|
97
|
+
send({ jsonrpc: "2.0", method: "notifications/initialized" });
|
|
98
|
+
const res = await request(2, "tools/list", {});
|
|
99
|
+
if (res.error) {
|
|
100
|
+
throw new Error(`tools/list failed: ${res.error.message ?? "unknown"}`);
|
|
101
|
+
}
|
|
102
|
+
return (res.result?.tools ?? []).map((t) => ({
|
|
103
|
+
name: t.name,
|
|
104
|
+
description: t.description,
|
|
105
|
+
}));
|
|
106
|
+
}
|
|
107
|
+
finally {
|
|
108
|
+
clearTimeout(timer);
|
|
109
|
+
child.kill("SIGKILL");
|
|
110
|
+
}
|
|
111
|
+
}
|
|
112
|
+
/**
|
|
113
|
+
* Verify `toolName` exists on `server`; on a miss, suggest the closest tool names
|
|
114
|
+
* (edit distance) — "did you mean issue_write?".
|
|
115
|
+
*/
|
|
116
|
+
async function verifyMcpTool(server, toolName, timeoutMs = 10000) {
|
|
117
|
+
const available = (await listMcpTools(server, timeoutMs)).map((t) => t.name);
|
|
118
|
+
const exists = available.includes(toolName);
|
|
119
|
+
return {
|
|
120
|
+
exists,
|
|
121
|
+
available,
|
|
122
|
+
suggestions: exists ? [] : closest(toolName, available),
|
|
123
|
+
};
|
|
124
|
+
}
|
|
125
|
+
function editDistance(a, b) {
|
|
126
|
+
if (a === b)
|
|
127
|
+
return 0;
|
|
128
|
+
const m = a.length;
|
|
129
|
+
const n = b.length;
|
|
130
|
+
if (m === 0)
|
|
131
|
+
return n;
|
|
132
|
+
if (n === 0)
|
|
133
|
+
return m;
|
|
134
|
+
const dp = Array.from({ length: n + 1 }, (_, i) => i);
|
|
135
|
+
for (let i = 1; i <= m; i++) {
|
|
136
|
+
let prev = dp[0];
|
|
137
|
+
dp[0] = i;
|
|
138
|
+
for (let j = 1; j <= n; j++) {
|
|
139
|
+
const tmp = dp[j];
|
|
140
|
+
dp[j] =
|
|
141
|
+
a[i - 1] === b[j - 1] ? prev : 1 + Math.min(prev, dp[j], dp[j - 1]);
|
|
142
|
+
prev = tmp;
|
|
143
|
+
}
|
|
144
|
+
}
|
|
145
|
+
return dp[n];
|
|
146
|
+
}
|
|
147
|
+
function closest(target, candidates, max = 4) {
|
|
148
|
+
return candidates
|
|
149
|
+
.map((c) => ({ c, d: editDistance(target, c) }))
|
|
150
|
+
.filter((x) => x.d <= max)
|
|
151
|
+
.sort((a, b) => a.d - b.d)
|
|
152
|
+
.slice(0, 3)
|
|
153
|
+
.map((x) => x.c);
|
|
154
|
+
}
|
|
155
|
+
// --- MCP tool references in instruction files ------------------------------
|
|
156
|
+
// `vigiles:mcp <server>#<tool>` inside an inline code span — the MCP analogue of
|
|
157
|
+
// the `vigiles:symbol path#name` mark. Self-contained (server + tool in one
|
|
158
|
+
// token) so it binds unambiguously.
|
|
159
|
+
const MCP_MARK = /^vigiles:mcp\s+([\w-]+)#([\w.-]+)$/;
|
|
160
|
+
/** Parse `vigiles:mcp server#tool` marks from a markdown file's inline spans. */
|
|
161
|
+
function parseMcpRefs(markdown) {
|
|
162
|
+
const refs = [];
|
|
163
|
+
for (const span of (0, refs_js_1.inlineSpans)(markdown)) {
|
|
164
|
+
const m = MCP_MARK.exec(span.text);
|
|
165
|
+
if (m)
|
|
166
|
+
refs.push({ server: m[1], tool: m[2], line: span.line });
|
|
167
|
+
}
|
|
168
|
+
return refs;
|
|
169
|
+
}
|
|
170
|
+
/** Read `mcpServers` from `.mcp.json` (the stdio-server config map), or `{}`. */
|
|
171
|
+
function loadMcpServers(cwd) {
|
|
172
|
+
const p = (0, node_path_1.join)(cwd, ".mcp.json");
|
|
173
|
+
if (!(0, node_fs_1.existsSync)(p))
|
|
174
|
+
return {};
|
|
175
|
+
try {
|
|
176
|
+
const json = JSON.parse((0, node_fs_1.readFileSync)(p, "utf-8"));
|
|
177
|
+
return json.mcpServers ?? {};
|
|
178
|
+
}
|
|
179
|
+
catch {
|
|
180
|
+
return {};
|
|
181
|
+
}
|
|
182
|
+
}
|
|
183
|
+
async function verifyOneServer(group, cfg, timeoutMs) {
|
|
184
|
+
if (!cfg) {
|
|
185
|
+
return group.map((r) => ({
|
|
186
|
+
...r,
|
|
187
|
+
reason: "server-undeclared",
|
|
188
|
+
suggestions: [],
|
|
189
|
+
}));
|
|
190
|
+
}
|
|
191
|
+
let available;
|
|
192
|
+
try {
|
|
193
|
+
available = (await listMcpTools(cfg, timeoutMs)).map((t) => t.name);
|
|
194
|
+
}
|
|
195
|
+
catch {
|
|
196
|
+
return group.map((r) => ({
|
|
197
|
+
...r,
|
|
198
|
+
reason: "server-unreachable",
|
|
199
|
+
suggestions: [],
|
|
200
|
+
}));
|
|
201
|
+
}
|
|
202
|
+
const errs = [];
|
|
203
|
+
for (const r of group) {
|
|
204
|
+
if (!available.includes(r.tool)) {
|
|
205
|
+
errs.push({
|
|
206
|
+
...r,
|
|
207
|
+
reason: "tool-missing",
|
|
208
|
+
suggestions: closest(r.tool, available),
|
|
209
|
+
});
|
|
210
|
+
}
|
|
211
|
+
}
|
|
212
|
+
return errs;
|
|
213
|
+
}
|
|
214
|
+
/**
|
|
215
|
+
* Verify every `vigiles:mcp server#tool` mark in `markdown` against the live
|
|
216
|
+
* servers in `mcpServers` (each referenced server is started once). A reference
|
|
217
|
+
* to an undeclared server, an unreachable server, or a missing tool is an error.
|
|
218
|
+
*/
|
|
219
|
+
async function verifyMcpRefs(markdown, mcpServers, timeoutMs = 10000) {
|
|
220
|
+
const byServer = new Map();
|
|
221
|
+
for (const r of parseMcpRefs(markdown)) {
|
|
222
|
+
const arr = byServer.get(r.server) ?? [];
|
|
223
|
+
arr.push(r);
|
|
224
|
+
byServer.set(r.server, arr);
|
|
225
|
+
}
|
|
226
|
+
const all = [];
|
|
227
|
+
for (const [server, group] of byServer) {
|
|
228
|
+
all.push(...(await verifyOneServer(group, mcpServers[server], timeoutMs)));
|
|
229
|
+
}
|
|
230
|
+
return all;
|
|
231
|
+
}
|
|
232
|
+
/** Human-readable message for an MCP reference error (with "did you mean"). */
|
|
233
|
+
function mcpRefMessage(e) {
|
|
234
|
+
switch (e.reason) {
|
|
235
|
+
case "server-undeclared":
|
|
236
|
+
return `MCP server "${e.server}" is not declared in .mcp.json`;
|
|
237
|
+
case "server-unreachable":
|
|
238
|
+
return `MCP server "${e.server}" failed to start`;
|
|
239
|
+
case "tool-missing":
|
|
240
|
+
return `MCP tool "${e.server}#${e.tool}" not found${e.suggestions.length > 0
|
|
241
|
+
? ` — did you mean ${e.suggestions.map((s) => `"${s}"`).join(", ")}?`
|
|
242
|
+
: ""}`;
|
|
243
|
+
default:
|
|
244
|
+
return (0, hash_js_1.assertNever)(e.reason);
|
|
245
|
+
}
|
|
246
|
+
}
|
|
247
|
+
//# sourceMappingURL=mcp.js.map
|
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
export interface LoadedPlugin {
|
|
2
|
+
/** A `.claude/settings.json`-shaped object with hooks resolved. */
|
|
3
|
+
readonly settings: {
|
|
4
|
+
hooks?: unknown;
|
|
5
|
+
};
|
|
6
|
+
/** Files to materialize in the sandbox (CLAUDE.md, skills, agents, commands). */
|
|
7
|
+
readonly files: Record<string, string>;
|
|
8
|
+
/**
|
|
9
|
+
* Surfaces that are present in the plugin but cannot be exercised at the
|
|
10
|
+
* deterministic tier (subagents and slash commands need a real model; MCP
|
|
11
|
+
* servers aren't wired by the loader). Empty when the plugin is fully
|
|
12
|
+
* covered. Surfaced so "load the whole plugin" never silently tests nothing —
|
|
13
|
+
* read it in a test, or just to know what the deterministic run won't reach.
|
|
14
|
+
*/
|
|
15
|
+
readonly warnings: readonly string[];
|
|
16
|
+
}
|
|
17
|
+
/**
|
|
18
|
+
* Load the real harness at `pluginPath`. Returns the resolved settings (hooks),
|
|
19
|
+
* the files (CLAUDE.md + skills + agents + commands) to write into the test
|
|
20
|
+
* sandbox, and `warnings` for surfaces the deterministic tier can't drive. Merge
|
|
21
|
+
* `settings` with any inline settings and spread `files` into the fixture.
|
|
22
|
+
*/
|
|
23
|
+
export declare function loadPlugin(pluginPath: string): LoadedPlugin;
|
|
24
|
+
/**
|
|
25
|
+
* Resolve the effective harness for a test/eval (arm): load the plugin if given,
|
|
26
|
+
* then layer inline settings + files on top. Shared by `runHarnessTest` and
|
|
27
|
+
* `runEval` so both test the assembled machine the same way.
|
|
28
|
+
*/
|
|
29
|
+
export declare function resolveHarness(opts: {
|
|
30
|
+
plugin?: string;
|
|
31
|
+
settings?: unknown;
|
|
32
|
+
files?: Record<string, string>;
|
|
33
|
+
}): {
|
|
34
|
+
settings: unknown;
|
|
35
|
+
files: Record<string, string>;
|
|
36
|
+
};
|
|
37
|
+
//# sourceMappingURL=plugin-loader.d.ts.map
|