vigiles 36.0.0 → 37.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cli-commands.d.ts +1 -1
- package/dist/cli-commands.js +1 -0
- package/dist/cli-main.js +13 -0
- package/dist/cli.js +51 -0
- package/dist/core/eval-driver.d.ts +9 -1
- package/dist/core/eval-spec-parse.d.ts +14 -0
- package/dist/core/eval-spec-parse.js +168 -0
- package/dist/core/run-env.d.ts +46 -1
- package/dist/core/run-env.js +59 -0
- package/dist/core/stub-rules.d.ts +199 -0
- package/dist/core/stub-rules.js +397 -0
- package/dist/eval-cache.d.ts +19 -1
- package/dist/eval-cache.js +1 -1
- package/dist/eval-entry.d.ts +28 -0
- package/dist/eval-entry.js +63 -10
- package/dist/eval-surface.d.ts +2 -0
- package/dist/eval.d.ts +80 -32
- package/dist/eval.js +293 -91
- package/dist/harness-test.d.ts +1 -1
- package/dist/services.d.ts +1 -1
- package/dist/services.js +1 -1
- package/dist/stub.d.ts +57 -0
- package/dist/stub.js +103 -0
- package/dist/test.d.ts +4 -1
- package/dist/test.js +8 -4
- package/dist/tool-stub.d.ts +24 -29
- package/dist/tool-stub.js +63 -74
- package/package.json +1 -1
- package/skills/test-harness/references/observing-a-run.md +13 -12
- package/skills/test-harness/references/writing-tests.md +1 -0
package/dist/stub.d.ts
ADDED
|
@@ -0,0 +1,57 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* `experimental_stub` — a tool stub that answers PER INVOCATION, for the eval
|
|
3
|
+
* tier's `stubs`. The rules and their matching are in `core/stub-rules.ts`; this
|
|
4
|
+
* is the typed helper and the check over a run's recorded calls.
|
|
5
|
+
*
|
|
6
|
+
* ONE experimental root (STABILITY.md "one root per feature"): `rest` and
|
|
7
|
+
* `called` are members, so no stable name rests on an unstable one. The rules
|
|
8
|
+
* themselves are plain data (`ToolStub`): a `.mjs` literal of the same shape is
|
|
9
|
+
* accepted, because the spec boundary parses `stubs` either way.
|
|
10
|
+
*/
|
|
11
|
+
import type { Check } from "./check.js";
|
|
12
|
+
import type { Trace } from "./harness-test.js";
|
|
13
|
+
import { type ArgvPattern, type ArgvToken, type StubRule, type ToolStub } from "./core/stub-rules.js";
|
|
14
|
+
/** Bounds for {@link experimental_stub.called}. */
|
|
15
|
+
export interface StubCalledBounds {
|
|
16
|
+
/** At least this many matching calls. Default 1, or 0 when `max` is given. */
|
|
17
|
+
readonly min?: number;
|
|
18
|
+
/** At most this many. Default: no limit. */
|
|
19
|
+
readonly max?: number;
|
|
20
|
+
/** Order-free tokens after the positional prefix (needs a trailing `rest`). */
|
|
21
|
+
readonly contains?: readonly ArgvToken[];
|
|
22
|
+
}
|
|
23
|
+
/**
|
|
24
|
+
* A check over a run's stub log: the ANSWERED invocations of `name` that match
|
|
25
|
+
* `argv` (and `bounds.contains`) number within `[min, max]`. A call no rule
|
|
26
|
+
* answered is not counted (it is reported as unanswered, and `vigiles eval`
|
|
27
|
+
* fails the run on it). A trace with no stub log
|
|
28
|
+
* FAILS — "called at most 0 times" must not pass because nothing was recorded.
|
|
29
|
+
*/
|
|
30
|
+
declare function called(name: string, argv: ArgvPattern, bounds?: StubCalledBounds): Check<Trace>;
|
|
31
|
+
/** Parse one stub: a throw names the bad field; the value is plain data. */
|
|
32
|
+
declare function stubOf(name: string, rules: readonly [StubRule, ...StubRule[]]): ToolStub;
|
|
33
|
+
/**
|
|
34
|
+
* A stub for the binary `name`: a non-empty list of rules over its argv, first
|
|
35
|
+
* match wins. An invocation no rule answers fails the eval — name each command
|
|
36
|
+
* the agent runs; do not answer everything with one output.
|
|
37
|
+
*
|
|
38
|
+
* ```ts
|
|
39
|
+
* const { rest } = experimental_stub;
|
|
40
|
+
* const gh = experimental_stub("gh", [
|
|
41
|
+
* { argv: ["issue", "create", rest], reply: { kind: "always", stdout: "https://github.com/o/r/issues/9001\n" } },
|
|
42
|
+
* { argv: ["api", /^repos\/o\/r\/issues\?/], reply: { kind: "always", stdout: "[]" } },
|
|
43
|
+
* ]);
|
|
44
|
+
* ```
|
|
45
|
+
*
|
|
46
|
+
* `experimental_stub.rest` — "any further tokens", last position only.
|
|
47
|
+
* `experimental_stub.called(name, argv, { min, max, contains })` — a check over
|
|
48
|
+
* the run's recorded calls.
|
|
49
|
+
*
|
|
50
|
+
* @experimental
|
|
51
|
+
*/
|
|
52
|
+
export declare const experimental_stub: typeof stubOf & {
|
|
53
|
+
rest: import("./core/stub-rules.js").ArgvRest;
|
|
54
|
+
called: typeof called;
|
|
55
|
+
};
|
|
56
|
+
export {};
|
|
57
|
+
//# sourceMappingURL=stub.d.ts.map
|
package/dist/stub.js
ADDED
|
@@ -0,0 +1,103 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.experimental_stub = void 0;
|
|
4
|
+
const eval_spec_parse_js_1 = require("./core/eval-spec-parse.js");
|
|
5
|
+
const stub_rules_js_1 = require("./core/stub-rules.js");
|
|
6
|
+
const result = (pass, message) => ({
|
|
7
|
+
pass,
|
|
8
|
+
score: pass ? 1 : 0,
|
|
9
|
+
message,
|
|
10
|
+
});
|
|
11
|
+
/** The failure for a trace that carries no stub log at all. */
|
|
12
|
+
const noLog = (shown) => result(false, `stub ${shown}: this trace has no stub log — the run declared no stubs, or its tier does not record them`);
|
|
13
|
+
/**
|
|
14
|
+
* How many of a run's stub calls are `name` with an argv matching `argv` (+
|
|
15
|
+
* `contains`), split by whether a rule answered them. Only an ANSWERED call
|
|
16
|
+
* counts toward `called`: "the tool was called" in a measurement means the
|
|
17
|
+
* agent got its answer, so a create that came back as the miss line did not
|
|
18
|
+
* file anything. The run fails on the unanswered call separately.
|
|
19
|
+
*/
|
|
20
|
+
function countCalls(calls, name, argv, contains) {
|
|
21
|
+
const matching = calls.filter((c) => c.tool === name && (0, stub_rules_js_1.matchArgv)({ argv, contains }, c.argv));
|
|
22
|
+
const answered = matching.filter((c) => !(0, stub_rules_js_1.isUnanswered)(c.outcome)).length;
|
|
23
|
+
return { answered, unanswered: matching.length - answered };
|
|
24
|
+
}
|
|
25
|
+
/** The [min, max] a `called` check accepts, and how a report prints it. */
|
|
26
|
+
function rangeOf(bounds) {
|
|
27
|
+
const min = bounds.min ?? (bounds.max === undefined ? 1 : 0);
|
|
28
|
+
const max = bounds.max ?? Number.POSITIVE_INFINITY;
|
|
29
|
+
const text = Number.isFinite(max)
|
|
30
|
+
? `${String(min)}..${String(max)}`
|
|
31
|
+
: `at least ${String(min)}`;
|
|
32
|
+
return { min, max, text };
|
|
33
|
+
}
|
|
34
|
+
/** The verdict of a `called` check over a run's recorded calls. */
|
|
35
|
+
function calledVerdict(calls, shown, expect) {
|
|
36
|
+
const { name, argv, ...bounds } = expect;
|
|
37
|
+
const { min, max, text } = rangeOf(bounds);
|
|
38
|
+
const { answered: n, unanswered } = countCalls(calls, name, argv, bounds.contains);
|
|
39
|
+
const missed = unanswered === 0
|
|
40
|
+
? ""
|
|
41
|
+
: `; ${String(unanswered)} matching call(s) went unanswered and are not counted`;
|
|
42
|
+
const msg = `stub ${shown} called ${String(n)} time(s) (expected ${text})${missed}`;
|
|
43
|
+
return result(n >= min && n <= max, msg);
|
|
44
|
+
}
|
|
45
|
+
/**
|
|
46
|
+
* A check over a run's stub log: the ANSWERED invocations of `name` that match
|
|
47
|
+
* `argv` (and `bounds.contains`) number within `[min, max]`. A call no rule
|
|
48
|
+
* answered is not counted (it is reported as unanswered, and `vigiles eval`
|
|
49
|
+
* fails the run on it). A trace with no stub log
|
|
50
|
+
* FAILS — "called at most 0 times" must not pass because nothing was recorded.
|
|
51
|
+
*/
|
|
52
|
+
function called(name, argv, bounds = {}) {
|
|
53
|
+
// The pattern is parsed the way a rule is (rest position, RegExp flags).
|
|
54
|
+
const rule = { argv, contains: bounds.contains, reply: { kind: "always" } };
|
|
55
|
+
(0, eval_spec_parse_js_1.parseToolStubs)([{ name, rules: [rule] }], "experimental_stub.called");
|
|
56
|
+
const { min, max } = rangeOf(bounds);
|
|
57
|
+
const shown = `${name} ${(0, stub_rules_js_1.describePattern)(argv)}`;
|
|
58
|
+
return {
|
|
59
|
+
kind: "stubCalled",
|
|
60
|
+
eval(t) {
|
|
61
|
+
if (t.stubCalls === undefined)
|
|
62
|
+
return noLog(shown);
|
|
63
|
+
return calledVerdict(t.stubCalls, shown, { name, argv, ...bounds });
|
|
64
|
+
},
|
|
65
|
+
toJSON: () => ({
|
|
66
|
+
kind: "stubCalled",
|
|
67
|
+
tool: name,
|
|
68
|
+
argv: (0, stub_rules_js_1.describePattern)(argv),
|
|
69
|
+
min,
|
|
70
|
+
max: Number.isFinite(max) ? max : null,
|
|
71
|
+
}),
|
|
72
|
+
};
|
|
73
|
+
}
|
|
74
|
+
/** Parse one stub: a throw names the bad field; the value is plain data. */
|
|
75
|
+
function stubOf(name, rules) {
|
|
76
|
+
(0, eval_spec_parse_js_1.parseToolStubs)([{ name, rules }], "experimental_stub");
|
|
77
|
+
return (0, stub_rules_js_1.toToolStub)({ name, rules });
|
|
78
|
+
}
|
|
79
|
+
/**
|
|
80
|
+
* A stub for the binary `name`: a non-empty list of rules over its argv, first
|
|
81
|
+
* match wins. An invocation no rule answers fails the eval — name each command
|
|
82
|
+
* the agent runs; do not answer everything with one output.
|
|
83
|
+
*
|
|
84
|
+
* ```ts
|
|
85
|
+
* const { rest } = experimental_stub;
|
|
86
|
+
* const gh = experimental_stub("gh", [
|
|
87
|
+
* { argv: ["issue", "create", rest], reply: { kind: "always", stdout: "https://github.com/o/r/issues/9001\n" } },
|
|
88
|
+
* { argv: ["api", /^repos\/o\/r\/issues\?/], reply: { kind: "always", stdout: "[]" } },
|
|
89
|
+
* ]);
|
|
90
|
+
* ```
|
|
91
|
+
*
|
|
92
|
+
* `experimental_stub.rest` — "any further tokens", last position only.
|
|
93
|
+
* `experimental_stub.called(name, argv, { min, max, contains })` — a check over
|
|
94
|
+
* the run's recorded calls.
|
|
95
|
+
*
|
|
96
|
+
* @experimental
|
|
97
|
+
*/
|
|
98
|
+
// eslint-disable-next-line functional/immutable-data -- the one experimental root: a callable with members, built the way experimental_agent is
|
|
99
|
+
exports.experimental_stub = Object.assign(stubOf, {
|
|
100
|
+
rest: stub_rules_js_1.ARGV_REST,
|
|
101
|
+
called,
|
|
102
|
+
});
|
|
103
|
+
//# sourceMappingURL=stub.js.map
|
package/dist/test.d.ts
CHANGED
|
@@ -68,7 +68,10 @@ export { DISASTER_CATALOG, verifyGuardrail, unblockedDisasters, assertBlocksDisa
|
|
|
68
68
|
export type { DisasterEvent, DisasterCategory, GuardrailResult, VerifyGuardrailOptions, } from "./guardrail-check.js";
|
|
69
69
|
export { experimental_verifyPluginGuards, experimental_formatPluginGuardReport, } from "./verify-plugin-guards.js";
|
|
70
70
|
export type { PluginGuardReport, SweptHook, SweptHookOutcome, NonCommandHookAction, VerifyPluginGuardsOptions, } from "./verify-plugin-guards.js";
|
|
71
|
-
export
|
|
71
|
+
export { experimental_stub } from "./stub.js";
|
|
72
|
+
export type { StubCalledBounds } from "./stub.js";
|
|
73
|
+
export type { ArgvPattern, ArgvRest, ArgvToken, StubAnswer, StubCall, StubOutcome, StubReply, StubRule, ToolStub, } from "./core/stub-rules.js";
|
|
74
|
+
export type { HomeFiles, HomeSeed, RunEnv } from "./core/run-env.js";
|
|
72
75
|
export { runHarnessTest, runHarness, parseToolCalls, parseSubagents, parseResultEvent, parseOutput, parseHooks, decideSandbox, specTrusted, sandboxAvailable, } from "./harness-test.js";
|
|
73
76
|
export { experimental_outputStyleArms } from "./output-style-arms.js";
|
|
74
77
|
export type { OutputStyleArms } from "./output-style-arms.js";
|
package/dist/test.js
CHANGED
|
@@ -71,8 +71,8 @@ var __exportStar = (this && this.__exportStar) || function(m, exports) {
|
|
|
71
71
|
for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
|
|
72
72
|
};
|
|
73
73
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
74
|
-
exports.
|
|
75
|
-
exports.experimental_makeDockerRuntime = exports.experimental_dockerRuntime = exports.experimental_withServices = exports.experimental_startServices = exports.stubSkillBody = exports.parseClaudeRun = exports.formatTriggerRateReport = exports.formatEvalReport = exports.formatCheckReport = exports.checkReportToJUnit = exports.checkPromptDiversity = exports.assertPromptDiversity = exports.assertRates = exports.defineEval = exports.formatContainment = exports.compareContainment = exports.skillContract = exports.mustNotInclude = exports.mustInclude = exports.commandsIn = exports.experimental_outputStyleArms = exports.sandboxAvailable = exports.specTrusted = exports.decideSandbox = exports.parseHooks = exports.parseOutput = exports.parseResultEvent = exports.parseSubagents = exports.parseToolCalls = void 0;
|
|
74
|
+
exports.runHarnessTest = exports.experimental_stub = exports.experimental_formatPluginGuardReport = exports.experimental_verifyPluginGuards = exports.experimental_alternateSpellings = exports.formatGuardrailReport = exports.assertBlocksDisasters = exports.unblockedDisasters = exports.verifyGuardrail = exports.DISASTER_CATALOG = exports.cacheTokens = exports.outputTokens = exports.inputTokens = exports.tokens = exports.latency = exports.cost = exports.mcp = exports.allowed = exports.blocked = exports.subagent = exports.didNotWrite = exports.wrote = exports.turns = exports.received = exports.hookFired = exports.experimental_replyCount = exports.experimental_eachReply = exports.output = exports.skill = exports.onlyTools = exports.notTool = exports.toolWith = exports.tool = exports.assertChecks = exports.evalChecks = exports.experimental_hookState = exports.loadHook = exports.experimental_assertEmittedOk = exports.experimental_parseEmitted = exports.experimental_emitTool = exports.egressRoutes = exports.fileToolEvents = exports.propertyHook = exports.decideHook = exports.parseHookOutput = exports.runHook = exports.runScript = exports.cleanupTmpDir = exports.makeTmpDir = exports.recordCheck = void 0;
|
|
75
|
+
exports.experimental_makeDockerRuntime = exports.experimental_dockerRuntime = exports.experimental_withServices = exports.experimental_startServices = exports.stubSkillBody = exports.parseClaudeRun = exports.formatTriggerRateReport = exports.formatEvalReport = exports.formatCheckReport = exports.checkReportToJUnit = exports.checkPromptDiversity = exports.assertPromptDiversity = exports.assertRates = exports.defineEval = exports.formatContainment = exports.compareContainment = exports.skillContract = exports.mustNotInclude = exports.mustInclude = exports.commandsIn = exports.experimental_outputStyleArms = exports.sandboxAvailable = exports.specTrusted = exports.decideSandbox = exports.parseHooks = exports.parseOutput = exports.parseResultEvent = exports.parseSubagents = exports.parseToolCalls = exports.runHarness = void 0;
|
|
76
76
|
// --- reporting: how much did this script actually do? ---
|
|
77
77
|
// `vigiles test` can otherwise see only an exit code, so a file that runs NOTHING
|
|
78
78
|
// prints the same `✓` as one that ran and passed (measured 2026-08-08 on a file
|
|
@@ -196,8 +196,12 @@ Object.defineProperty(exports, "experimental_alternateSpellings", { enumerable:
|
|
|
196
196
|
var verify_plugin_guards_js_1 = require("./verify-plugin-guards.js");
|
|
197
197
|
Object.defineProperty(exports, "experimental_verifyPluginGuards", { enumerable: true, get: function () { return verify_plugin_guards_js_1.experimental_verifyPluginGuards; } });
|
|
198
198
|
Object.defineProperty(exports, "experimental_formatPluginGuardReport", { enumerable: true, get: function () { return verify_plugin_guards_js_1.experimental_formatPluginGuardReport; } });
|
|
199
|
-
// Tool stubs on PATH (rung R2):
|
|
200
|
-
|
|
199
|
+
// Tool stubs on PATH (rung R2): a fake binary that answers PER INVOCATION, by
|
|
200
|
+
// rules over its argv. The helper and its check are experimental; the rule
|
|
201
|
+
// shapes are plain data, because a `.mjs` literal of the same shape is accepted
|
|
202
|
+
// at the spec boundary. Writing the shims is internal (`tool-stub.ts`).
|
|
203
|
+
var stub_js_1 = require("./stub.js");
|
|
204
|
+
Object.defineProperty(exports, "experimental_stub", { enumerable: true, get: function () { return stub_js_1.experimental_stub; } });
|
|
201
205
|
// The assembled machine — AGNOSTIC SURFACE ONLY. The Claude-Code transport
|
|
202
206
|
// (`scriptModel`, `claudeCodeDriver`, `buildClaudeArgs`, `claudeAvailable`,
|
|
203
207
|
// `loadPlugin`, `resolveHarness`) is deliberately NOT re-exported here, so this
|
package/dist/tool-stub.d.ts
CHANGED
|
@@ -1,35 +1,30 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
export interface ToolStub {
|
|
7
|
-
/** The binary name to shadow on PATH (e.g. `"gh"`, `"psql"`). */
|
|
8
|
-
readonly name: string;
|
|
9
|
-
/** Canned stdout the fake prints. Default `""`. */
|
|
10
|
-
readonly stdout?: string;
|
|
11
|
-
/** Canned stderr the fake writes to fd 2. Default: none. */
|
|
12
|
-
readonly stderr?: string;
|
|
13
|
-
/** Exit code the fake returns. Default `0`. */
|
|
14
|
-
readonly exitCode?: number;
|
|
1
|
+
import { type StubCall, type ToolStub } from "./core/stub-rules.js";
|
|
2
|
+
/** Where a run's stubs live. `binDir` goes first on the run's PATH. */
|
|
3
|
+
export interface StubDir {
|
|
4
|
+
readonly root: string;
|
|
5
|
+
readonly binDir: string;
|
|
15
6
|
}
|
|
7
|
+
/** How a shim reaches the stub runtime: this node, and this vigiles' `cli.js`. */
|
|
8
|
+
export interface StubLauncher {
|
|
9
|
+
readonly node: string;
|
|
10
|
+
readonly cli: string;
|
|
11
|
+
}
|
|
12
|
+
/** The shim for one stub. Pure. */
|
|
13
|
+
export declare function stubShim(launcher: StubLauncher, root: string, name: string): string;
|
|
16
14
|
/**
|
|
17
|
-
*
|
|
18
|
-
*
|
|
19
|
-
*
|
|
20
|
-
*/
|
|
21
|
-
export declare function renderToolStub(stub: ToolStub): string;
|
|
22
|
-
/**
|
|
23
|
-
* Write one executable POSIX shell stub per entry into `binDir` (which must
|
|
24
|
-
* already exist). Each file is named exactly `stub.name` and `chmod 0o755` so it
|
|
25
|
-
* is directly executable once `binDir` is on PATH. Pure-ish — fs only, no spawn.
|
|
15
|
+
* Write a fresh stub root for `stubs`: a shim and a rules file per stub. The
|
|
16
|
+
* stubs must already be parsed (the spec boundary did it). Remove it with
|
|
17
|
+
* {@link removeStubDir}.
|
|
26
18
|
*/
|
|
27
|
-
export declare function
|
|
19
|
+
export declare function writeStubDir(stubs: readonly ToolStub[], launcher: StubLauncher): StubDir;
|
|
20
|
+
/** Every invocation the run's stubs recorded, in order. Empty when none was called. */
|
|
21
|
+
export declare function readStubCalls(dir: StubDir): readonly StubCall[];
|
|
28
22
|
/**
|
|
29
|
-
*
|
|
30
|
-
*
|
|
31
|
-
*
|
|
32
|
-
* so a `parentDir` cleanup also clears it).
|
|
23
|
+
* Forget every call recorded so far. The log is the stubs' only state (an
|
|
24
|
+
* `inOrder` rule counts its prior answers in it), so a cleared log is a stub
|
|
25
|
+
* that has never been called. Safe when no call was logged yet.
|
|
33
26
|
*/
|
|
34
|
-
export declare function
|
|
27
|
+
export declare function resetStubCalls(dir: StubDir): void;
|
|
28
|
+
/** Remove a stub root. Safe on one already gone. */
|
|
29
|
+
export declare function removeStubDir(dir: StubDir): void;
|
|
35
30
|
//# sourceMappingURL=tool-stub.d.ts.map
|
package/dist/tool-stub.js
CHANGED
|
@@ -1,92 +1,81 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
-
exports.
|
|
4
|
-
exports.
|
|
5
|
-
exports.
|
|
3
|
+
exports.stubShim = stubShim;
|
|
4
|
+
exports.writeStubDir = writeStubDir;
|
|
5
|
+
exports.readStubCalls = readStubCalls;
|
|
6
|
+
exports.resetStubCalls = resetStubCalls;
|
|
7
|
+
exports.removeStubDir = removeStubDir;
|
|
6
8
|
/**
|
|
7
|
-
* vigiles —
|
|
9
|
+
* vigiles — tool stubs on PATH: the EFFECTS half (rung R2 of the eval coverage
|
|
10
|
+
* model). The rules, the decision and the log format are pure, in
|
|
11
|
+
* `core/stub-rules.ts`; this file writes them to disk and reads the log back.
|
|
8
12
|
*
|
|
9
|
-
*
|
|
10
|
-
*
|
|
11
|
-
* a
|
|
12
|
-
* **recorded / author-provided canned result** — a standard PATH-shim +
|
|
13
|
-
* VCR-style record/replay technique, not a novel invention. See
|
|
14
|
-
* `research/eval-coverage-and-isolation.md` (the three-rung model).
|
|
13
|
+
* Layout of one stub root — a temp dir BESIDE the run's working dir, never in
|
|
14
|
+
* it, so the agent does not find vigiles plumbing in its project (models did
|
|
15
|
+
* `ls`/`cat` a stub dir that lived inside the work dir):
|
|
15
16
|
*
|
|
16
|
-
*
|
|
17
|
-
*
|
|
18
|
-
*
|
|
19
|
-
* a documented follow-on — NOT implemented here.
|
|
17
|
+
* <root>/bin/<name> a two-line sh shim, first on the run's PATH
|
|
18
|
+
* <root>/rules/<name>.json the encoded rules (`encodeStub`)
|
|
19
|
+
* <root>/calls.jsonl one line per invocation, every stub of the run
|
|
20
20
|
*
|
|
21
|
-
*
|
|
22
|
-
*
|
|
23
|
-
*
|
|
24
|
-
*
|
|
21
|
+
* The shim execs `node <vigiles cli.js> hook-runtime stub <root> <name> "$@"`,
|
|
22
|
+
* a fast path in the dispatcher that loads only the pure decision — the same
|
|
23
|
+
* shape as the intercept hook, which also calls back into the CLI by absolute
|
|
24
|
+
* path because `npx vigiles` does not resolve from a throwaway dir.
|
|
25
25
|
*
|
|
26
|
-
*
|
|
27
|
-
*
|
|
28
|
-
* sub-command / flags) is a follow-on.
|
|
26
|
+
* The canned answers are author/recorded fixtures, NEVER model-synthesized: a
|
|
27
|
+
* synthesized `gh` output looks plausible and diverges from the real tool.
|
|
29
28
|
*/
|
|
30
29
|
const node_fs_1 = require("node:fs");
|
|
31
|
-
const
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
return Buffer.from(s, "utf-8").toString("base64");
|
|
30
|
+
const stub_rules_js_1 = require("./core/stub-rules.js");
|
|
31
|
+
const tmp_root_js_1 = require("./core/tmp-root.js");
|
|
32
|
+
/** A POSIX single-quoted word: the only quoting `sh` never re-reads. */
|
|
33
|
+
const shQuote = (s) => `'${s.replaceAll("'", `'\\''`)}'`;
|
|
34
|
+
/** The shim for one stub. Pure. */
|
|
35
|
+
function stubShim(launcher, root, name) {
|
|
36
|
+
const words = [
|
|
37
|
+
launcher.node,
|
|
38
|
+
launcher.cli,
|
|
39
|
+
"hook-runtime",
|
|
40
|
+
"stub",
|
|
41
|
+
root,
|
|
42
|
+
name,
|
|
43
|
+
];
|
|
44
|
+
return `#!/bin/sh\nexec ${words.map(shQuote).join(" ")} "$@"\n`;
|
|
47
45
|
}
|
|
48
46
|
/**
|
|
49
|
-
*
|
|
50
|
-
*
|
|
51
|
-
*
|
|
47
|
+
* Write a fresh stub root for `stubs`: a shim and a rules file per stub. The
|
|
48
|
+
* stubs must already be parsed (the spec boundary did it). Remove it with
|
|
49
|
+
* {@link removeStubDir}.
|
|
52
50
|
*/
|
|
53
|
-
function
|
|
54
|
-
const
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
}
|
|
65
|
-
lines.push(`exit ${String(stub.exitCode ?? 0)}`);
|
|
66
|
-
return lines.join("\n") + "\n";
|
|
51
|
+
function writeStubDir(stubs, launcher) {
|
|
52
|
+
const root = (0, tmp_root_js_1.makeTmpDir)("stubs");
|
|
53
|
+
const { binDir, rulesDir } = (0, stub_rules_js_1.stubPaths)(root);
|
|
54
|
+
(0, node_fs_1.mkdirSync)(binDir, { recursive: true });
|
|
55
|
+
(0, node_fs_1.mkdirSync)(rulesDir, { recursive: true });
|
|
56
|
+
stubs.forEach((stub) => {
|
|
57
|
+
const paths = (0, stub_rules_js_1.stubPaths)(root, stub.name);
|
|
58
|
+
(0, node_fs_1.writeFileSync)(paths.rules, (0, stub_rules_js_1.encodeStub)(stub), "utf8");
|
|
59
|
+
(0, node_fs_1.writeFileSync)(paths.bin, stubShim(launcher, root, stub.name), "utf8");
|
|
60
|
+
(0, node_fs_1.chmodSync)(paths.bin, 0o755);
|
|
61
|
+
});
|
|
62
|
+
return { root, binDir };
|
|
67
63
|
}
|
|
68
|
-
/**
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
*/
|
|
73
|
-
function writeToolStubs(binDir, stubs) {
|
|
74
|
-
for (const stub of stubs) {
|
|
75
|
-
const file = (0, node_path_1.join)(binDir, stub.name);
|
|
76
|
-
(0, node_fs_1.writeFileSync)(file, renderToolStub(stub), "utf-8");
|
|
77
|
-
(0, node_fs_1.chmodSync)(file, 0o755);
|
|
78
|
-
}
|
|
64
|
+
/** Every invocation the run's stubs recorded, in order. Empty when none was called. */
|
|
65
|
+
function readStubCalls(dir) {
|
|
66
|
+
const { log } = (0, stub_rules_js_1.stubPaths)(dir.root);
|
|
67
|
+
return (0, node_fs_1.existsSync)(log) ? (0, stub_rules_js_1.parseStubLog)((0, node_fs_1.readFileSync)(log, "utf8")) : [];
|
|
79
68
|
}
|
|
80
69
|
/**
|
|
81
|
-
*
|
|
82
|
-
*
|
|
83
|
-
*
|
|
84
|
-
* so a `parentDir` cleanup also clears it).
|
|
70
|
+
* Forget every call recorded so far. The log is the stubs' only state (an
|
|
71
|
+
* `inOrder` rule counts its prior answers in it), so a cleared log is a stub
|
|
72
|
+
* that has never been called. Safe when no call was logged yet.
|
|
85
73
|
*/
|
|
86
|
-
function
|
|
87
|
-
(0, node_fs_1.
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
74
|
+
function resetStubCalls(dir) {
|
|
75
|
+
(0, node_fs_1.rmSync)((0, stub_rules_js_1.stubPaths)(dir.root).log, { force: true });
|
|
76
|
+
}
|
|
77
|
+
/** Remove a stub root. Safe on one already gone. */
|
|
78
|
+
function removeStubDir(dir) {
|
|
79
|
+
(0, node_fs_1.rmSync)(dir.root, { recursive: true, force: true });
|
|
91
80
|
}
|
|
92
81
|
//# sourceMappingURL=tool-stub.js.map
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "vigiles",
|
|
3
|
-
"version": "
|
|
3
|
+
"version": "37.0.0",
|
|
4
4
|
"description": "Audit, test and measure the harness your AI agent runs on — grade your CLAUDE.md / AGENTS.md, skills, subagents and hooks, run them against a scripted model, and measure whether they actually fire.",
|
|
5
5
|
"keywords": [
|
|
6
6
|
"claude-code",
|
|
@@ -10,18 +10,19 @@ questions are keyed on the **observation** instead — "what did this skill
|
|
|
10
10
|
actually do?" — and they have answers already. Reach for these before building
|
|
11
11
|
anything; every one of them ships today.
|
|
12
12
|
|
|
13
|
-
| The question you're actually asking | Use
|
|
14
|
-
| ----------------------------------------------------------------------- |
|
|
15
|
-
| Which tools did it call, and with what arguments? | `trace.toolCalls` · `tool` / `toolWith` checks · `parseToolCalls` (`vigiles`)
|
|
16
|
-
| Did it call a tool it must not? | `notTool(name)`
|
|
17
|
-
| Did it call **only** tools from a known set? | `onlyTools([...])` — the white-list, symmetric to `assertWroteOnly`
|
|
18
|
-
| Did it stay inside the `allowed-tools` its own frontmatter declares? | `skillContract(dir).surface` — builds that check FROM the declaration
|
|
19
|
-
| What files did the run write? | `filesWritten` · `wrote(path)` / `didNotWrite(path)` · `r.file(path)`
|
|
20
|
-
| Did it write **only** where it was supposed to? | `assertWroteOnly([...])` / `assertNoWrite()` — needs `{ sandbox: "auto" }`
|
|
21
|
-
| Run a tool call but **don't let it execute** — capture the args instead | the `interceptTools` option on `measure` / `runEval` (a `ToolIntercept[]`)
|
|
22
|
-
|
|
|
23
|
-
|
|
|
24
|
-
|
|
|
13
|
+
| The question you're actually asking | Use |
|
|
14
|
+
| ----------------------------------------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------- |
|
|
15
|
+
| Which tools did it call, and with what arguments? | `trace.toolCalls` · `tool` / `toolWith` checks · `parseToolCalls` (`vigiles`) |
|
|
16
|
+
| Did it call a tool it must not? | `notTool(name)` |
|
|
17
|
+
| Did it call **only** tools from a known set? | `onlyTools([...])` — the white-list, symmetric to `assertWroteOnly` |
|
|
18
|
+
| Did it stay inside the `allowed-tools` its own frontmatter declares? | `skillContract(dir).surface` — builds that check FROM the declaration |
|
|
19
|
+
| What files did the run write? | `filesWritten` · `wrote(path)` / `didNotWrite(path)` · `r.file(path)` |
|
|
20
|
+
| Did it write **only** where it was supposed to? | `assertWroteOnly([...])` / `assertNoWrite()` — needs `{ sandbox: "auto" }` |
|
|
21
|
+
| Run a tool call but **don't let it execute** — capture the args instead | the `interceptTools` option on `measure` / `runEval` (a `ToolIntercept[]`) |
|
|
22
|
+
| Let a CLI the agent (or its script) calls **answer** — `gh`, `psql` | `stubs: [experimental_stub(name, rules)]` on `runEval`; read `ctx.stubCalls`, check with `experimental_stub.called(...)` — an argv no rule answers fails the eval |
|
|
23
|
+
| Did a subagent do it, and which one? | `subagent(name, [...])` · `SubagentTrace` |
|
|
24
|
+
| Was it an MCP tool? | `mcp(server, toolName)` |
|
|
25
|
+
| Assert the whole effect boundary deterministically | `assertChecks` + the checks above (see `examples/harness/effect-boundary.harness.mjs`) |
|
|
25
26
|
|
|
26
27
|
`interceptTools` is the one worth knowing about, because it is not obvious it
|
|
27
28
|
exists: it denies a tool its **real execution** via an auto-wired `PreToolUse`
|
|
@@ -90,6 +90,7 @@ import { defineEval, assertSignificant } from "vigiles";
|
|
|
90
90
|
|
|
91
91
|
export default defineEval({
|
|
92
92
|
runEval: {
|
|
93
|
+
env: { kind: "ephemeral" }, // required: a throwaway HOME, or { kind: "inherit", reason }
|
|
93
94
|
arms: { off: {}, on: { pluginDir: "./" } },
|
|
94
95
|
task: "…a task the harness change should affect…",
|
|
95
96
|
measure: (ctx) => ({ ok: /* a bare predicate over the trace */ true }),
|