vigiles 2.1.1 → 2.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +127 -5
- package/dist/action-gate.d.ts +28 -0
- package/dist/action-gate.js +73 -0
- package/dist/cli.js +450 -75
- package/dist/community-skills.d.ts +22 -0
- package/dist/community-skills.js +86 -0
- package/dist/compile-generator.d.ts +48 -0
- package/dist/compile-generator.js +322 -0
- package/dist/compile.d.ts +3 -0
- package/dist/compile.js +217 -26
- package/dist/eval.d.ts +87 -0
- package/dist/eval.js +208 -0
- package/dist/frontmatter.d.ts +24 -6
- package/dist/frontmatter.js +103 -30
- package/dist/generate-schema.js +10 -0
- package/dist/harness-assert.d.ts +68 -0
- package/dist/harness-assert.js +127 -0
- package/dist/harness-test.d.ts +45 -0
- package/dist/harness-test.js +138 -0
- package/dist/inline.d.ts +22 -4
- package/dist/inline.js +60 -13
- package/dist/jest.d.ts +9 -0
- package/dist/jest.js +23 -0
- package/dist/judge.d.ts +29 -0
- package/dist/judge.js +88 -0
- package/dist/linters.js +28 -0
- package/dist/mock-model.d.ts +31 -0
- package/dist/mock-model.js +189 -0
- package/dist/plugin-loader.d.ts +37 -0
- package/dist/plugin-loader.js +195 -0
- package/dist/refs.d.ts +44 -0
- package/dist/refs.js +144 -0
- package/dist/run-hook.d.ts +77 -0
- package/dist/run-hook.js +80 -0
- package/dist/run-scripts.d.ts +20 -0
- package/dist/run-scripts.js +70 -0
- package/dist/skill-driver.d.ts +77 -0
- package/dist/skill-driver.js +76 -0
- package/dist/skill-runtime.d.ts +101 -0
- package/dist/skill-runtime.js +289 -0
- package/dist/skill-test.d.ts +47 -0
- package/dist/skill-test.js +77 -0
- package/dist/spec.d.ts +90 -4
- package/dist/spec.js +29 -0
- package/dist/symbols.d.ts +30 -0
- package/dist/symbols.js +142 -0
- package/dist/vitest.d.mts +9 -0
- package/dist/vitest.mjs +22 -0
- package/package.json +45 -6
package/dist/frontmatter.js
CHANGED
|
@@ -56,11 +56,11 @@ function findLine(lines, needle, fromIndex) {
|
|
|
56
56
|
return fromIndex + 1;
|
|
57
57
|
}
|
|
58
58
|
/**
|
|
59
|
-
* Navigate a parsed frontmatter document to its `vigiles
|
|
60
|
-
*
|
|
61
|
-
* `vigiles
|
|
59
|
+
* Navigate a parsed frontmatter document to its `vigiles` mapping. Returns
|
|
60
|
+
* "none" when there's nothing for vigiles to check, "error" when the
|
|
61
|
+
* `vigiles` key is present but not a mapping, or the mapping itself.
|
|
62
62
|
*/
|
|
63
|
-
function
|
|
63
|
+
function getVigiles(doc, lines, startLine) {
|
|
64
64
|
if (doc === null || typeof doc !== "object" || Array.isArray(doc)) {
|
|
65
65
|
return { kind: "none" };
|
|
66
66
|
}
|
|
@@ -78,6 +78,13 @@ function lookupEnforce(doc, lines, startLine) {
|
|
|
78
78
|
},
|
|
79
79
|
};
|
|
80
80
|
}
|
|
81
|
+
return { kind: "map", vigiles: vigiles };
|
|
82
|
+
}
|
|
83
|
+
/**
|
|
84
|
+
* Locate the `vigiles.enforce` list. Returns "none" when absent, "error" when
|
|
85
|
+
* present but not a list, or the list with its source line.
|
|
86
|
+
*/
|
|
87
|
+
function lookupEnforce(vigiles, lines, startLine) {
|
|
81
88
|
const enforce = vigiles.enforce;
|
|
82
89
|
if (enforce === undefined)
|
|
83
90
|
return { kind: "none" };
|
|
@@ -93,6 +100,45 @@ function lookupEnforce(doc, lines, startLine) {
|
|
|
93
100
|
}
|
|
94
101
|
return { kind: "list", enforce, enforceLine };
|
|
95
102
|
}
|
|
103
|
+
/**
|
|
104
|
+
* Parse a `vigiles.<key>` list of plain strings (used for `files` and
|
|
105
|
+
* `commands`). Returns located items plus error findings for the wrong shape
|
|
106
|
+
* or non-string entries; an absent key yields empty results.
|
|
107
|
+
*/
|
|
108
|
+
function parseStringList(vigiles, key, lines, startLine) {
|
|
109
|
+
const raw = vigiles[key];
|
|
110
|
+
if (raw === undefined)
|
|
111
|
+
return { items: [], errors: [] };
|
|
112
|
+
const keyLine = findLine(lines, `${key}:`, startLine - 1);
|
|
113
|
+
if (!Array.isArray(raw)) {
|
|
114
|
+
return {
|
|
115
|
+
items: [],
|
|
116
|
+
errors: [
|
|
117
|
+
{
|
|
118
|
+
line: keyLine,
|
|
119
|
+
message: `\`vigiles.${key}\` must be a list of strings.`,
|
|
120
|
+
},
|
|
121
|
+
],
|
|
122
|
+
};
|
|
123
|
+
}
|
|
124
|
+
const items = [];
|
|
125
|
+
const errors = [];
|
|
126
|
+
let cursor = keyLine;
|
|
127
|
+
for (let i = 0; i < raw.length; i++) {
|
|
128
|
+
const v = raw[i];
|
|
129
|
+
if (typeof v !== "string" || v.trim() === "") {
|
|
130
|
+
errors.push({
|
|
131
|
+
line: keyLine,
|
|
132
|
+
message: `vigiles.${key}[${String(i)}] must be a non-empty string.`,
|
|
133
|
+
});
|
|
134
|
+
continue;
|
|
135
|
+
}
|
|
136
|
+
const line = findLine(lines, v, cursor);
|
|
137
|
+
cursor = line;
|
|
138
|
+
items.push({ value: v, line });
|
|
139
|
+
}
|
|
140
|
+
return { items, errors };
|
|
141
|
+
}
|
|
96
142
|
/** Parse one `vigiles.enforce` entry into a rule or an error finding. */
|
|
97
143
|
function parseEntry(entry, index, ctx) {
|
|
98
144
|
if (entry === null || typeof entry !== "object" || Array.isArray(entry)) {
|
|
@@ -126,10 +172,15 @@ function parseEntry(entry, index, ctx) {
|
|
|
126
172
|
}
|
|
127
173
|
return { rule: { linterRule: rule, why, line }, nextCursor: line };
|
|
128
174
|
}
|
|
175
|
+
/** Fresh empty result — callers may push into the arrays, so never shared. */
|
|
176
|
+
function emptyResult() {
|
|
177
|
+
return { rules: [], files: [], commands: [], errors: [] };
|
|
178
|
+
}
|
|
129
179
|
/**
|
|
130
|
-
* Parse `vigiles.enforce` rules
|
|
131
|
-
*
|
|
132
|
-
*
|
|
180
|
+
* Parse `vigiles.enforce` rules, `vigiles.files`, and `vigiles.commands` out
|
|
181
|
+
* of a markdown file's YAML frontmatter. Does not touch the filesystem and
|
|
182
|
+
* does not verify references — callers feed rules into `checkLinterRule` and
|
|
183
|
+
* file/command refs into `validateFileRef` / `validateCommandRef`.
|
|
133
184
|
*
|
|
134
185
|
* A file with no frontmatter, or frontmatter with no `vigiles` key, yields
|
|
135
186
|
* empty results with no errors. Malformed YAML or a malformed `vigiles`
|
|
@@ -138,7 +189,7 @@ function parseEntry(entry, index, ctx) {
|
|
|
138
189
|
function parseFrontmatterRules(content) {
|
|
139
190
|
const fm = extractFrontmatter(content);
|
|
140
191
|
if (!fm)
|
|
141
|
-
return
|
|
192
|
+
return emptyResult();
|
|
142
193
|
const lines = content.split("\n");
|
|
143
194
|
let doc;
|
|
144
195
|
try {
|
|
@@ -149,6 +200,8 @@ function parseFrontmatterRules(content) {
|
|
|
149
200
|
const line = (err.mark?.line ?? 0) + fm.startLine;
|
|
150
201
|
return {
|
|
151
202
|
rules: [],
|
|
203
|
+
files: [],
|
|
204
|
+
commands: [],
|
|
152
205
|
errors: [
|
|
153
206
|
{
|
|
154
207
|
line,
|
|
@@ -157,34 +210,54 @@ function parseFrontmatterRules(content) {
|
|
|
157
210
|
],
|
|
158
211
|
};
|
|
159
212
|
}
|
|
160
|
-
const
|
|
161
|
-
if (
|
|
162
|
-
return
|
|
163
|
-
if (
|
|
164
|
-
return { rules: [], errors: [
|
|
213
|
+
const vig = getVigiles(doc, lines, fm.startLine);
|
|
214
|
+
if (vig.kind === "none")
|
|
215
|
+
return emptyResult();
|
|
216
|
+
if (vig.kind === "error")
|
|
217
|
+
return { rules: [], files: [], commands: [], errors: [vig.error] };
|
|
165
218
|
const rules = [];
|
|
166
219
|
const errors = [];
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
220
|
+
const enforceLookup = lookupEnforce(vig.vigiles, lines, fm.startLine);
|
|
221
|
+
if (enforceLookup.kind === "error") {
|
|
222
|
+
errors.push(enforceLookup.error);
|
|
223
|
+
}
|
|
224
|
+
else if (enforceLookup.kind === "list") {
|
|
225
|
+
let cursor = enforceLookup.enforceLine; // search start: line after `enforce:`
|
|
226
|
+
for (let i = 0; i < enforceLookup.enforce.length; i++) {
|
|
227
|
+
const r = parseEntry(enforceLookup.enforce[i], i, {
|
|
228
|
+
lines,
|
|
229
|
+
enforceLine: enforceLookup.enforceLine,
|
|
230
|
+
cursor,
|
|
231
|
+
});
|
|
232
|
+
cursor = r.nextCursor;
|
|
233
|
+
if (r.rule)
|
|
234
|
+
rules.push(r.rule);
|
|
235
|
+
if (r.error)
|
|
236
|
+
errors.push(r.error);
|
|
237
|
+
}
|
|
179
238
|
}
|
|
180
|
-
|
|
239
|
+
const fileList = parseStringList(vig.vigiles, "files", lines, fm.startLine);
|
|
240
|
+
errors.push(...fileList.errors);
|
|
241
|
+
const files = fileList.items.map((it) => ({
|
|
242
|
+
path: it.value,
|
|
243
|
+
line: it.line,
|
|
244
|
+
}));
|
|
245
|
+
const cmdList = parseStringList(vig.vigiles, "commands", lines, fm.startLine);
|
|
246
|
+
errors.push(...cmdList.errors);
|
|
247
|
+
const commands = cmdList.items.map((it) => ({
|
|
248
|
+
command: it.value,
|
|
249
|
+
line: it.line,
|
|
250
|
+
}));
|
|
251
|
+
return { rules, files, commands, errors };
|
|
181
252
|
}
|
|
182
253
|
/**
|
|
183
|
-
* True if the content has at least one parseable `vigiles
|
|
184
|
-
*
|
|
185
|
-
*
|
|
254
|
+
* True if the content has at least one parseable `vigiles` reference in its
|
|
255
|
+
* frontmatter — an `enforce` rule, a `files` entry, or a `commands` entry.
|
|
256
|
+
* Used by `require-spec` validation to treat frontmatter mode as
|
|
257
|
+
* spec-equivalent, mirroring `hasInlineRules`.
|
|
186
258
|
*/
|
|
187
259
|
function hasFrontmatterRules(content) {
|
|
188
|
-
|
|
260
|
+
const r = parseFrontmatterRules(content);
|
|
261
|
+
return r.rules.length + r.files.length + r.commands.length > 0;
|
|
189
262
|
}
|
|
190
263
|
//# sourceMappingURL=frontmatter.js.map
|
package/dist/generate-schema.js
CHANGED
|
@@ -101,6 +101,16 @@ function generateSchema(options = {}) {
|
|
|
101
101
|
},
|
|
102
102
|
},
|
|
103
103
|
},
|
|
104
|
+
files: {
|
|
105
|
+
type: "array",
|
|
106
|
+
description: "File paths referenced by this instruction file, verified to exist by `vigiles audit`.",
|
|
107
|
+
items: { type: "string" },
|
|
108
|
+
},
|
|
109
|
+
commands: {
|
|
110
|
+
type: "array",
|
|
111
|
+
description: "Commands (npm scripts / script-runner invocations) referenced here, verified by `vigiles audit`.",
|
|
112
|
+
items: { type: "string" },
|
|
113
|
+
},
|
|
104
114
|
},
|
|
105
115
|
},
|
|
106
116
|
},
|
|
@@ -0,0 +1,68 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* vigiles — runner-agnostic helpers for harness tests / evals.
|
|
3
|
+
*
|
|
4
|
+
* `runHarnessTest` and `runEval` are plain async functions that return data, so
|
|
5
|
+
* they already work inside any runner (node:test, vitest, jest, mocha). These
|
|
6
|
+
* helpers remove the last bit of boilerplate without coupling to a runner:
|
|
7
|
+
*
|
|
8
|
+
* - `withHarness` — run a harness test and auto-clean the sandbox (try/finally),
|
|
9
|
+
* so you don't leak temp dirs in `afterEach`.
|
|
10
|
+
* - plain `assert*` helpers that throw — usable in every runner, including
|
|
11
|
+
* node:test which has no `expect.extend`.
|
|
12
|
+
* - `vigilesMatchers` — register with `expect.extend(vigilesMatchers)` for
|
|
13
|
+
* `expect(...).toHaveCreated(...)` sugar. The signature is identical for
|
|
14
|
+
* vitest and jest, so the same object supports both.
|
|
15
|
+
*/
|
|
16
|
+
import { type HarnessTestSpec, type HarnessTestResult } from "./harness-test.js";
|
|
17
|
+
import type { EvalReport } from "./eval.js";
|
|
18
|
+
import type { HookRunResult } from "./run-hook.js";
|
|
19
|
+
/**
|
|
20
|
+
* Run a harness test, hand the result to `fn`, and always clean up the sandbox.
|
|
21
|
+
* Returns whatever `fn` returns. Use this instead of calling `cleanup()` by
|
|
22
|
+
* hand — it survives assertion failures.
|
|
23
|
+
*/
|
|
24
|
+
export declare function withHarness<T>(spec: HarnessTestSpec, fn: (r: HarnessTestResult) => T | Promise<T>): Promise<T>;
|
|
25
|
+
/** Assert the sandbox contains `path` (a hook/agent side-effect file). */
|
|
26
|
+
export declare function assertCreated(r: HarnessTestResult, path: string): void;
|
|
27
|
+
/** Assert the sandbox does NOT contain `path` (e.g. a blocked action's output). */
|
|
28
|
+
export declare function assertNotCreated(r: HarnessTestResult, path: string): void;
|
|
29
|
+
/** Assert the scripted model served at least `n` turns (e.g. a Stop hook forced more). */
|
|
30
|
+
export declare function assertServedTurns(r: HarnessTestResult, n: number): void;
|
|
31
|
+
/** Assert a `runHook` result blocked (exit 2 / decision:block / permission:deny). */
|
|
32
|
+
export declare function assertHookBlocked(r: HookRunResult): void;
|
|
33
|
+
/** Assert a `runHook` result allowed (did not block). */
|
|
34
|
+
export declare function assertHookAllowed(r: HookRunResult): void;
|
|
35
|
+
/** The gap on `metric` between two arms (arm − baseline). */
|
|
36
|
+
export declare function improvement(report: EvalReport, baseline: string, arm: string, metric: string): number;
|
|
37
|
+
/**
|
|
38
|
+
* Assert `arm` beats `baseline` on `metric` by more than `by`. With `by` left at
|
|
39
|
+
* 0 this just asserts a positive gap; pass the combined se to demand the gap
|
|
40
|
+
* clear the noise floor.
|
|
41
|
+
*/
|
|
42
|
+
export declare function assertImproves(report: EvalReport, opts: {
|
|
43
|
+
baseline: string;
|
|
44
|
+
arm: string;
|
|
45
|
+
metric: string;
|
|
46
|
+
by?: number;
|
|
47
|
+
}): void;
|
|
48
|
+
interface MatcherOutput {
|
|
49
|
+
pass: boolean;
|
|
50
|
+
message: () => string;
|
|
51
|
+
}
|
|
52
|
+
/**
|
|
53
|
+
* Custom matchers compatible with both vitest and jest. Register once:
|
|
54
|
+
*
|
|
55
|
+
* import { expect } from "vitest"; // or "@jest/globals"
|
|
56
|
+
* import { vigilesMatchers } from "vigiles/harness-assert";
|
|
57
|
+
* expect.extend(vigilesMatchers);
|
|
58
|
+
*
|
|
59
|
+
* expect(result).toHaveCreated("RESULT");
|
|
60
|
+
* expect(report).toBeatBaseline("vanilla", "gated", "caught");
|
|
61
|
+
*/
|
|
62
|
+
export declare const vigilesMatchers: {
|
|
63
|
+
toHaveCreated(received: HarnessTestResult, path: string): MatcherOutput;
|
|
64
|
+
toBlock(received: HookRunResult): MatcherOutput;
|
|
65
|
+
toBeatBaseline(received: EvalReport, baseline: string, arm: string, metric: string, by?: number): MatcherOutput;
|
|
66
|
+
};
|
|
67
|
+
export {};
|
|
68
|
+
//# sourceMappingURL=harness-assert.d.ts.map
|
|
@@ -0,0 +1,127 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.vigilesMatchers = void 0;
|
|
4
|
+
exports.withHarness = withHarness;
|
|
5
|
+
exports.assertCreated = assertCreated;
|
|
6
|
+
exports.assertNotCreated = assertNotCreated;
|
|
7
|
+
exports.assertServedTurns = assertServedTurns;
|
|
8
|
+
exports.assertHookBlocked = assertHookBlocked;
|
|
9
|
+
exports.assertHookAllowed = assertHookAllowed;
|
|
10
|
+
exports.improvement = improvement;
|
|
11
|
+
exports.assertImproves = assertImproves;
|
|
12
|
+
/**
|
|
13
|
+
* vigiles — runner-agnostic helpers for harness tests / evals.
|
|
14
|
+
*
|
|
15
|
+
* `runHarnessTest` and `runEval` are plain async functions that return data, so
|
|
16
|
+
* they already work inside any runner (node:test, vitest, jest, mocha). These
|
|
17
|
+
* helpers remove the last bit of boilerplate without coupling to a runner:
|
|
18
|
+
*
|
|
19
|
+
* - `withHarness` — run a harness test and auto-clean the sandbox (try/finally),
|
|
20
|
+
* so you don't leak temp dirs in `afterEach`.
|
|
21
|
+
* - plain `assert*` helpers that throw — usable in every runner, including
|
|
22
|
+
* node:test which has no `expect.extend`.
|
|
23
|
+
* - `vigilesMatchers` — register with `expect.extend(vigilesMatchers)` for
|
|
24
|
+
* `expect(...).toHaveCreated(...)` sugar. The signature is identical for
|
|
25
|
+
* vitest and jest, so the same object supports both.
|
|
26
|
+
*/
|
|
27
|
+
const harness_test_js_1 = require("./harness-test.js");
|
|
28
|
+
/**
|
|
29
|
+
* Run a harness test, hand the result to `fn`, and always clean up the sandbox.
|
|
30
|
+
* Returns whatever `fn` returns. Use this instead of calling `cleanup()` by
|
|
31
|
+
* hand — it survives assertion failures.
|
|
32
|
+
*/
|
|
33
|
+
async function withHarness(spec, fn) {
|
|
34
|
+
const r = await (0, harness_test_js_1.runHarnessTest)(spec);
|
|
35
|
+
try {
|
|
36
|
+
return await fn(r);
|
|
37
|
+
}
|
|
38
|
+
finally {
|
|
39
|
+
r.cleanup();
|
|
40
|
+
}
|
|
41
|
+
}
|
|
42
|
+
// --- Plain throwing assertions (any runner) --------------------------------
|
|
43
|
+
function fail(message) {
|
|
44
|
+
throw new Error(message);
|
|
45
|
+
}
|
|
46
|
+
/** Assert the sandbox contains `path` (a hook/agent side-effect file). */
|
|
47
|
+
function assertCreated(r, path) {
|
|
48
|
+
if (r.file(path) === null)
|
|
49
|
+
fail(`expected the run to create ${path}`);
|
|
50
|
+
}
|
|
51
|
+
/** Assert the sandbox does NOT contain `path` (e.g. a blocked action's output). */
|
|
52
|
+
function assertNotCreated(r, path) {
|
|
53
|
+
if (r.file(path) !== null)
|
|
54
|
+
fail(`expected the run NOT to create ${path}`);
|
|
55
|
+
}
|
|
56
|
+
/** Assert the scripted model served at least `n` turns (e.g. a Stop hook forced more). */
|
|
57
|
+
function assertServedTurns(r, n) {
|
|
58
|
+
if (r.turns < n) {
|
|
59
|
+
fail(`expected ≥ ${String(n)} model turns, got ${String(r.turns)}`);
|
|
60
|
+
}
|
|
61
|
+
}
|
|
62
|
+
/** Assert a `runHook` result blocked (exit 2 / decision:block / permission:deny). */
|
|
63
|
+
function assertHookBlocked(r) {
|
|
64
|
+
if (!r.blocked) {
|
|
65
|
+
fail(`expected the hook to block, but it allowed (exit ${String(r.exitCode)})`);
|
|
66
|
+
}
|
|
67
|
+
}
|
|
68
|
+
/** Assert a `runHook` result allowed (did not block). */
|
|
69
|
+
function assertHookAllowed(r) {
|
|
70
|
+
if (r.blocked) {
|
|
71
|
+
fail(`expected the hook to allow, but it blocked (exit ${String(r.exitCode)}, decision ${String(r.decision)})`);
|
|
72
|
+
}
|
|
73
|
+
}
|
|
74
|
+
/** The gap on `metric` between two arms (arm − baseline). */
|
|
75
|
+
function improvement(report, baseline, arm, metric) {
|
|
76
|
+
const a = report.arms[arm]?.metrics[metric] ?? 0;
|
|
77
|
+
const b = report.arms[baseline]?.metrics[metric] ?? 0;
|
|
78
|
+
return a - b;
|
|
79
|
+
}
|
|
80
|
+
/**
|
|
81
|
+
* Assert `arm` beats `baseline` on `metric` by more than `by`. With `by` left at
|
|
82
|
+
* 0 this just asserts a positive gap; pass the combined se to demand the gap
|
|
83
|
+
* clear the noise floor.
|
|
84
|
+
*/
|
|
85
|
+
function assertImproves(report, opts) {
|
|
86
|
+
const by = opts.by ?? 0;
|
|
87
|
+
const delta = improvement(report, opts.baseline, opts.arm, opts.metric);
|
|
88
|
+
if (delta <= by) {
|
|
89
|
+
fail(`expected ${opts.arm} to beat ${opts.baseline} on ${opts.metric} by > ${String(by)}, got ${delta.toFixed(3)}`);
|
|
90
|
+
}
|
|
91
|
+
}
|
|
92
|
+
/**
|
|
93
|
+
* Custom matchers compatible with both vitest and jest. Register once:
|
|
94
|
+
*
|
|
95
|
+
* import { expect } from "vitest"; // or "@jest/globals"
|
|
96
|
+
* import { vigilesMatchers } from "vigiles/harness-assert";
|
|
97
|
+
* expect.extend(vigilesMatchers);
|
|
98
|
+
*
|
|
99
|
+
* expect(result).toHaveCreated("RESULT");
|
|
100
|
+
* expect(report).toBeatBaseline("vanilla", "gated", "caught");
|
|
101
|
+
*/
|
|
102
|
+
exports.vigilesMatchers = {
|
|
103
|
+
toHaveCreated(received, path) {
|
|
104
|
+
const pass = received.file(path) !== null;
|
|
105
|
+
return {
|
|
106
|
+
pass,
|
|
107
|
+
message: () => `expected the run ${pass ? "not " : ""}to create ${path}`,
|
|
108
|
+
};
|
|
109
|
+
},
|
|
110
|
+
toBlock(received) {
|
|
111
|
+
const pass = received.blocked;
|
|
112
|
+
return {
|
|
113
|
+
pass,
|
|
114
|
+
message: () => `expected the hook ${pass ? "not " : ""}to block (exit ${String(received.exitCode)}, decision ${String(received.decision)})`,
|
|
115
|
+
};
|
|
116
|
+
},
|
|
117
|
+
// eslint-disable-next-line max-params -- jest/vitest matchers take positional args
|
|
118
|
+
toBeatBaseline(received, baseline, arm, metric, by = 0) {
|
|
119
|
+
const delta = improvement(received, baseline, arm, metric);
|
|
120
|
+
const pass = delta > by;
|
|
121
|
+
return {
|
|
122
|
+
pass,
|
|
123
|
+
message: () => `expected ${arm} ${pass ? "not " : ""}to beat ${baseline} on ${metric} by > ${String(by)} (got ${delta.toFixed(3)})`,
|
|
124
|
+
};
|
|
125
|
+
},
|
|
126
|
+
};
|
|
127
|
+
//# sourceMappingURL=harness-assert.js.map
|
|
@@ -0,0 +1,45 @@
|
|
|
1
|
+
import { type ModelTurn } from "./mock-model.js";
|
|
2
|
+
export { scriptModel, type ModelTurn } from "./mock-model.js";
|
|
3
|
+
export { loadPlugin, resolveHarness } from "./plugin-loader.js";
|
|
4
|
+
export interface HarnessTestSpec {
|
|
5
|
+
/** Fixture files to write in a fresh temp working dir (path → contents). */
|
|
6
|
+
readonly files?: Record<string, string>;
|
|
7
|
+
/** `.claude/settings.json` contents — the hooks/permissions under test. */
|
|
8
|
+
readonly settings?: unknown;
|
|
9
|
+
/**
|
|
10
|
+
* Path to a real plugin/repo whose harness (hooks + CLAUDE.md + skills) is
|
|
11
|
+
* loaded into the sandbox, so you test the assembled machine, not a retyped
|
|
12
|
+
* subset. Inline `settings`/`files` layer on top. See src/plugin-loader.ts.
|
|
13
|
+
*/
|
|
14
|
+
readonly plugin?: string;
|
|
15
|
+
/** The scripted model turns the agent will take. */
|
|
16
|
+
readonly model: readonly ModelTurn[];
|
|
17
|
+
/** The user prompt. Default: "go". */
|
|
18
|
+
readonly prompt?: string;
|
|
19
|
+
/** Tools the agent may use. Default: Read Edit Write Bash. */
|
|
20
|
+
readonly allowedTools?: readonly string[];
|
|
21
|
+
/** Per-run wall-clock timeout in ms. Default 60000. */
|
|
22
|
+
readonly timeoutMs?: number;
|
|
23
|
+
}
|
|
24
|
+
export interface HarnessTestResult {
|
|
25
|
+
readonly exitCode: number;
|
|
26
|
+
readonly stdout: string;
|
|
27
|
+
/** Hook block messages and diagnostics land here. */
|
|
28
|
+
readonly stderr: string;
|
|
29
|
+
/** The temp working dir (inspect or clean it up). */
|
|
30
|
+
readonly cwd: string;
|
|
31
|
+
/** Number of model turns the agent took (mock turns served). */
|
|
32
|
+
readonly turns: number;
|
|
33
|
+
/** Final contents of a file under the working dir, or null if absent. */
|
|
34
|
+
file(path: string): string | null;
|
|
35
|
+
/** Remove the temp working dir. */
|
|
36
|
+
cleanup(): void;
|
|
37
|
+
}
|
|
38
|
+
/** Whether the `claude` CLI is available — harness tests need it. */
|
|
39
|
+
export declare function claudeAvailable(): boolean;
|
|
40
|
+
/**
|
|
41
|
+
* Run the real `claude` CLI against a scripted mock model, with the given
|
|
42
|
+
* fixture and settings (hooks). Deterministic — same script, same result.
|
|
43
|
+
*/
|
|
44
|
+
export declare function runHarnessTest(spec: HarnessTestSpec): Promise<HarnessTestResult>;
|
|
45
|
+
//# sourceMappingURL=harness-test.d.ts.map
|
|
@@ -0,0 +1,138 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.resolveHarness = exports.loadPlugin = exports.scriptModel = void 0;
|
|
4
|
+
exports.claudeAvailable = claudeAvailable;
|
|
5
|
+
exports.runHarnessTest = runHarnessTest;
|
|
6
|
+
/**
|
|
7
|
+
* vigiles — deterministic Claude Code harness testing.
|
|
8
|
+
*
|
|
9
|
+
* Test what your *harness* does — hooks, settings, skills, instruction files —
|
|
10
|
+
* without paying for or depending on a real model. `runHarnessTest` spins up the
|
|
11
|
+
* real `claude` CLI (so your real hooks and settings fire exactly as in
|
|
12
|
+
* production) but points it at a scripted mock model (`src/mock-model.ts`), so
|
|
13
|
+
* the agent's turns are fixed and the outcome is reproducible. No API key, no
|
|
14
|
+
* cost, CI-friendly.
|
|
15
|
+
*
|
|
16
|
+
* const r = await runHarnessTest({
|
|
17
|
+
* settings: { hooks: { Stop: [{ hooks: [{ type: "command",
|
|
18
|
+
* command: "test -f DONE || { echo 'not done' >&2; exit 2; }" }] }] } },
|
|
19
|
+
* model: scriptModel([
|
|
20
|
+
* { text: "I'm done" }, // tries to stop → blocked
|
|
21
|
+
* { tool: "Bash", input: { command: "touch DONE" } },
|
|
22
|
+
* { text: "now done" },
|
|
23
|
+
* ]),
|
|
24
|
+
* });
|
|
25
|
+
* assert(JSON.parse(r.stdout).num_turns > 1); // the Stop hook fired
|
|
26
|
+
*
|
|
27
|
+
* The "steps" are the scripted model turns — their real home is deterministic
|
|
28
|
+
* harness testing, not production enforcement.
|
|
29
|
+
*
|
|
30
|
+
* Note: the simple mock drives the Bash tool and Stop hooks reliably; the
|
|
31
|
+
* Edit/Write tools are gated in headless mode and don't fire via the mock —
|
|
32
|
+
* drive file actions through Bash, or use the real-model eval tier (`eval.ts`)
|
|
33
|
+
* for Edit/Write hooks.
|
|
34
|
+
*/
|
|
35
|
+
const node_child_process_1 = require("node:child_process");
|
|
36
|
+
const node_fs_1 = require("node:fs");
|
|
37
|
+
const node_os_1 = require("node:os");
|
|
38
|
+
const node_path_1 = require("node:path");
|
|
39
|
+
const mock_model_js_1 = require("./mock-model.js");
|
|
40
|
+
const plugin_loader_js_1 = require("./plugin-loader.js");
|
|
41
|
+
var mock_model_js_2 = require("./mock-model.js");
|
|
42
|
+
Object.defineProperty(exports, "scriptModel", { enumerable: true, get: function () { return mock_model_js_2.scriptModel; } });
|
|
43
|
+
var plugin_loader_js_2 = require("./plugin-loader.js");
|
|
44
|
+
Object.defineProperty(exports, "loadPlugin", { enumerable: true, get: function () { return plugin_loader_js_2.loadPlugin; } });
|
|
45
|
+
Object.defineProperty(exports, "resolveHarness", { enumerable: true, get: function () { return plugin_loader_js_2.resolveHarness; } });
|
|
46
|
+
/** Whether the `claude` CLI is available — harness tests need it. */
|
|
47
|
+
function claudeAvailable() {
|
|
48
|
+
try {
|
|
49
|
+
return (0, node_child_process_1.spawnSync)("claude", ["--version"], { stdio: "ignore" }).status === 0;
|
|
50
|
+
}
|
|
51
|
+
catch {
|
|
52
|
+
return false;
|
|
53
|
+
}
|
|
54
|
+
}
|
|
55
|
+
function writeFixture(cwd, files, settings) {
|
|
56
|
+
for (const [p, content] of Object.entries(files)) {
|
|
57
|
+
const full = (0, node_path_1.resolve)(cwd, p);
|
|
58
|
+
(0, node_fs_1.mkdirSync)((0, node_path_1.dirname)(full), { recursive: true });
|
|
59
|
+
(0, node_fs_1.writeFileSync)(full, content);
|
|
60
|
+
}
|
|
61
|
+
if (settings !== undefined) {
|
|
62
|
+
// `{cwd}` in any hook command is substituted with the working dir, so a
|
|
63
|
+
// hook can reference an absolute path inside it (hooks don't run with the
|
|
64
|
+
// project dir as cwd).
|
|
65
|
+
const json = JSON.stringify(settings, null, 2).replaceAll("{cwd}", cwd);
|
|
66
|
+
(0, node_fs_1.writeFileSync)((0, node_path_1.join)(cwd, "settings.json"), json);
|
|
67
|
+
}
|
|
68
|
+
}
|
|
69
|
+
function spawnClaude(args, cwd, baseUrl, timeoutMs) {
|
|
70
|
+
return new Promise((resolvePromise) => {
|
|
71
|
+
const child = (0, node_child_process_1.spawn)("claude", args, {
|
|
72
|
+
cwd,
|
|
73
|
+
env: {
|
|
74
|
+
...process.env,
|
|
75
|
+
ANTHROPIC_BASE_URL: baseUrl,
|
|
76
|
+
// Any value works — the mock ignores auth; this avoids needing a real key.
|
|
77
|
+
ANTHROPIC_API_KEY: "sk-vigiles-mock",
|
|
78
|
+
},
|
|
79
|
+
stdio: ["ignore", "pipe", "pipe"],
|
|
80
|
+
});
|
|
81
|
+
let stdout = "";
|
|
82
|
+
let stderr = "";
|
|
83
|
+
child.stdout.on("data", (d) => (stdout += d.toString()));
|
|
84
|
+
child.stderr.on("data", (d) => (stderr += d.toString()));
|
|
85
|
+
const timer = setTimeout(() => child.kill("SIGKILL"), timeoutMs);
|
|
86
|
+
child.on("close", (code) => {
|
|
87
|
+
clearTimeout(timer);
|
|
88
|
+
resolvePromise({ code: code ?? 0, stdout, stderr });
|
|
89
|
+
});
|
|
90
|
+
});
|
|
91
|
+
}
|
|
92
|
+
/**
|
|
93
|
+
* Run the real `claude` CLI against a scripted mock model, with the given
|
|
94
|
+
* fixture and settings (hooks). Deterministic — same script, same result.
|
|
95
|
+
*/
|
|
96
|
+
async function runHarnessTest(spec) {
|
|
97
|
+
const cwd = (0, node_fs_1.mkdtempSync)((0, node_path_1.join)((0, node_os_1.tmpdir)(), "vigiles-harness-"));
|
|
98
|
+
const { files, settings } = (0, plugin_loader_js_1.resolveHarness)({
|
|
99
|
+
plugin: spec.plugin,
|
|
100
|
+
settings: spec.settings,
|
|
101
|
+
files: spec.files,
|
|
102
|
+
});
|
|
103
|
+
writeFixture(cwd, files, settings);
|
|
104
|
+
const mock = await (0, mock_model_js_1.startMock)(spec.model);
|
|
105
|
+
try {
|
|
106
|
+
const tools = spec.allowedTools ?? ["Read", "Edit", "Write", "Bash"];
|
|
107
|
+
const args = [
|
|
108
|
+
"-p",
|
|
109
|
+
spec.prompt ?? "go",
|
|
110
|
+
"--output-format",
|
|
111
|
+
"json",
|
|
112
|
+
"--model",
|
|
113
|
+
"claude-sonnet-4-5",
|
|
114
|
+
...(settings !== undefined ? ["--settings", "settings.json"] : []),
|
|
115
|
+
"--allowedTools",
|
|
116
|
+
...tools,
|
|
117
|
+
];
|
|
118
|
+
const out = await spawnClaude(args, cwd, mock.url, spec.timeoutMs ?? 60000);
|
|
119
|
+
return {
|
|
120
|
+
exitCode: out.code,
|
|
121
|
+
stdout: out.stdout,
|
|
122
|
+
stderr: out.stderr,
|
|
123
|
+
cwd,
|
|
124
|
+
turns: mock.count,
|
|
125
|
+
file: (p) => {
|
|
126
|
+
const f = (0, node_path_1.resolve)(cwd, p);
|
|
127
|
+
return (0, node_fs_1.existsSync)(f) ? (0, node_fs_1.readFileSync)(f, "utf-8") : null;
|
|
128
|
+
},
|
|
129
|
+
cleanup: () => {
|
|
130
|
+
(0, node_fs_1.rmSync)(cwd, { recursive: true, force: true });
|
|
131
|
+
},
|
|
132
|
+
};
|
|
133
|
+
}
|
|
134
|
+
finally {
|
|
135
|
+
mock.close();
|
|
136
|
+
}
|
|
137
|
+
}
|
|
138
|
+
//# sourceMappingURL=harness-test.js.map
|
package/dist/inline.d.ts
CHANGED
|
@@ -25,8 +25,24 @@ export interface InlineRule {
|
|
|
25
25
|
/** 1-based line number of the comment in the source file. */
|
|
26
26
|
line: number;
|
|
27
27
|
}
|
|
28
|
+
/** A `<!-- vigiles:file <path> -->` reference (verified to exist). */
|
|
29
|
+
export interface InlineFileRef {
|
|
30
|
+
/** Project-relative path to verify exists. */
|
|
31
|
+
path: string;
|
|
32
|
+
/** 1-based line number of the comment in the source file. */
|
|
33
|
+
line: number;
|
|
34
|
+
}
|
|
35
|
+
/** A `<!-- vigiles:cmd "<command>" -->` reference (npm scripts verified). */
|
|
36
|
+
export interface InlineCmdRef {
|
|
37
|
+
/** Command to verify (npm scripts checked against package.json). */
|
|
38
|
+
command: string;
|
|
39
|
+
/** 1-based line number of the comment in the source file. */
|
|
40
|
+
line: number;
|
|
41
|
+
}
|
|
28
42
|
export interface InlineParseResult {
|
|
29
43
|
rules: InlineRule[];
|
|
44
|
+
files: InlineFileRef[];
|
|
45
|
+
commands: InlineCmdRef[];
|
|
30
46
|
/** Lines that look like vigiles: markers but failed to parse. */
|
|
31
47
|
errors: {
|
|
32
48
|
line: number;
|
|
@@ -46,13 +62,15 @@ export interface InlineParseResult {
|
|
|
46
62
|
*/
|
|
47
63
|
export declare function parseInlineRules(content: string): InlineParseResult;
|
|
48
64
|
/**
|
|
49
|
-
* True if the content contains at least one parseable vigiles
|
|
50
|
-
* rule
|
|
51
|
-
* `require-spec` validation to
|
|
65
|
+
* True if the content contains at least one parseable vigiles inline marker —
|
|
66
|
+
* an `enforce` rule, a `file` reference, or a `cmd` reference (ignoring fenced
|
|
67
|
+
* code blocks and malformed markers). Used by `require-spec` validation to
|
|
68
|
+
* treat inline mode as spec-equivalent: a file that pins even a single path is
|
|
69
|
+
* meaningfully managed.
|
|
52
70
|
*
|
|
53
71
|
* Deliberately delegates to `parseInlineRules` so a loose prefix regex
|
|
54
72
|
* can't satisfy require-spec with a malformed marker that produces no
|
|
55
|
-
* real
|
|
73
|
+
* real reference.
|
|
56
74
|
*/
|
|
57
75
|
export declare function hasInlineRules(content: string): boolean;
|
|
58
76
|
//# sourceMappingURL=inline.d.ts.map
|