vigiles 4.0.0 → 4.0.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude-plugin/plugin.json +1 -1
- package/README.md +29 -18
- package/action.yml +1 -1
- package/dist/adapter-conformance.js +1 -1
- package/dist/adapters/claude-code/adapter.js +1 -1
- package/dist/check.d.ts +132 -0
- package/dist/check.js +318 -0
- package/dist/claude-code.d.ts +1 -1
- package/dist/claude-code.js +1 -1
- package/dist/cli.js +133 -55
- package/dist/core/compile.d.ts +1 -1
- package/dist/core/compile.js +1 -1
- package/dist/core/compose.d.ts +1 -1
- package/dist/core/compose.js +1 -1
- package/dist/core/generate-schema.d.ts +1 -1
- package/dist/core/generate-schema.js +4 -4
- package/dist/core/linters.js +2 -2
- package/dist/core/orphans.js +57 -14
- package/dist/core/proofs.js +1 -1
- package/dist/core/refs.d.ts +1 -1
- package/dist/core/refs.js +2 -2
- package/dist/core/sidecar.d.ts +1 -1
- package/dist/core/sidecar.js +1 -1
- package/dist/core/spec.d.ts +1 -1
- package/dist/core/spec.js +1 -1
- package/dist/core/types.d.ts +1 -1
- package/dist/core/validate.js +2 -2
- package/dist/e2e.d.ts +10 -13
- package/dist/e2e.js +10 -17
- package/dist/{adapters/claude-code/eval-cache.d.ts → eval-cache.d.ts} +1 -1
- package/dist/{adapters/claude-code/eval-cache.js → eval-cache.js} +1 -1
- package/dist/{adapters/claude-code/eval.d.ts → eval.d.ts} +217 -2
- package/dist/{adapters/claude-code/eval.js → eval.js} +430 -20
- package/dist/harness-assert.d.ts +11 -8
- package/dist/harness-assert.js +21 -5
- package/dist/{adapters/claude-code/harness-test.d.ts → harness-test.d.ts} +50 -4
- package/dist/{adapters/claude-code/harness-test.js → harness-test.js} +106 -4
- package/dist/integration.d.ts +11 -3
- package/dist/integration.js +12 -3
- package/dist/jest.d.ts +3 -1
- package/dist/jest.js +3 -2
- package/dist/{adapters/claude-code/mock-model.d.ts → mock-model.d.ts} +2 -2
- package/dist/{adapters/claude-code/run-hook.d.ts → run-hook.d.ts} +23 -1
- package/dist/{adapters/claude-code/run-hook.js → run-hook.js} +29 -1
- package/dist/{adapters/claude-code/sandbox.js → sandbox.js} +1 -1
- package/dist/scan.d.ts +1 -1
- package/dist/scan.js +1 -1
- package/dist/setup-plan.d.ts +5 -1
- package/dist/setup-plan.js +11 -1
- package/dist/test-coverage.js +8 -1
- package/dist/testing.d.ts +10 -3
- package/dist/testing.js +15 -3
- package/dist/unit.d.ts +4 -2
- package/dist/unit.js +8 -2
- package/dist/vitest.d.mts +3 -1
- package/hooks/refs-nudge.sh +1 -1
- package/hooks/session-start.sh +3 -3
- package/package.json +8 -7
- package/skills/edit-spec/SKILL.md +21 -10
- package/skills/linter-docs/SKILL.md +23 -0
- package/skills/migrate-to-spec/SKILL.md +1 -1
- package/skills/strengthen/SKILL.md +1 -2
- package/skills/generate-rule/SKILL.md +0 -64
- /package/dist/{adapters/claude-code/egress-entry.d.ts → egress-entry.d.ts} +0 -0
- /package/dist/{adapters/claude-code/egress-entry.js → egress-entry.js} +0 -0
- /package/dist/{adapters/claude-code/egress-proxy.d.ts → egress-proxy.d.ts} +0 -0
- /package/dist/{adapters/claude-code/egress-proxy.js → egress-proxy.js} +0 -0
- /package/dist/{adapters/claude-code/egress.d.ts → egress.d.ts} +0 -0
- /package/dist/{adapters/claude-code/egress.js → egress.js} +0 -0
- /package/dist/{adapters/claude-code/eval-baseline.d.ts → eval-baseline.d.ts} +0 -0
- /package/dist/{adapters/claude-code/eval-baseline.js → eval-baseline.js} +0 -0
- /package/dist/{adapters/claude-code/judge.d.ts → judge.d.ts} +0 -0
- /package/dist/{adapters/claude-code/judge.js → judge.js} +0 -0
- /package/dist/{adapters/claude-code/mock-entry.d.ts → mock-entry.d.ts} +0 -0
- /package/dist/{adapters/claude-code/mock-entry.js → mock-entry.js} +0 -0
- /package/dist/{adapters/claude-code/mock-model.js → mock-model.js} +0 -0
- /package/dist/{adapters/claude-code/sandbox.d.ts → sandbox.d.ts} +0 -0
- /package/dist/{adapters/claude-code/stats.d.ts → stats.d.ts} +0 -0
- /package/dist/{adapters/claude-code/stats.js → stats.js} +0 -0
|
@@ -2,6 +2,7 @@
|
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
3
|
exports.claudeCodeDriver = exports.sandboxAvailable = exports.specTrusted = exports.decideSandbox = exports.resolveHarness = exports.loadPlugin = exports.scriptModel = void 0;
|
|
4
4
|
exports.parseToolCalls = parseToolCalls;
|
|
5
|
+
exports.parseSubagents = parseSubagents;
|
|
5
6
|
exports.parseResultEvent = parseResultEvent;
|
|
6
7
|
exports.parseOutput = parseOutput;
|
|
7
8
|
exports.parseHooks = parseHooks;
|
|
@@ -9,6 +10,7 @@ exports.buildClaudeArgs = buildClaudeArgs;
|
|
|
9
10
|
exports.parseClaudeRun = parseClaudeRun;
|
|
10
11
|
exports.claudeAvailable = claudeAvailable;
|
|
11
12
|
exports.runHarnessTest = runHarnessTest;
|
|
13
|
+
exports.runHarness = runHarness;
|
|
12
14
|
/**
|
|
13
15
|
* vigiles — deterministic Claude Code harness testing.
|
|
14
16
|
*
|
|
@@ -43,14 +45,14 @@ const node_child_process_1 = require("node:child_process");
|
|
|
43
45
|
const node_fs_1 = require("node:fs");
|
|
44
46
|
const node_os_1 = require("node:os");
|
|
45
47
|
const node_path_1 = require("node:path");
|
|
46
|
-
const adapter_conformance_js_1 = require("
|
|
47
|
-
const runtime_js_1 = require("./runtime.js");
|
|
48
|
+
const adapter_conformance_js_1 = require("./adapter-conformance.js");
|
|
49
|
+
const runtime_js_1 = require("./adapters/claude-code/runtime.js");
|
|
48
50
|
const mock_model_js_1 = require("./mock-model.js");
|
|
49
|
-
const plugin_loader_js_1 = require("./plugin-loader.js");
|
|
51
|
+
const plugin_loader_js_1 = require("./adapters/claude-code/plugin-loader.js");
|
|
50
52
|
const sandbox_js_1 = require("./sandbox.js");
|
|
51
53
|
var mock_model_js_2 = require("./mock-model.js");
|
|
52
54
|
Object.defineProperty(exports, "scriptModel", { enumerable: true, get: function () { return mock_model_js_2.scriptModel; } });
|
|
53
|
-
var plugin_loader_js_2 = require("./plugin-loader.js");
|
|
55
|
+
var plugin_loader_js_2 = require("./adapters/claude-code/plugin-loader.js");
|
|
54
56
|
Object.defineProperty(exports, "loadPlugin", { enumerable: true, get: function () { return plugin_loader_js_2.loadPlugin; } });
|
|
55
57
|
Object.defineProperty(exports, "resolveHarness", { enumerable: true, get: function () { return plugin_loader_js_2.resolveHarness; } });
|
|
56
58
|
var sandbox_js_2 = require("./sandbox.js");
|
|
@@ -114,6 +116,82 @@ function parseToolCalls(streamJson) {
|
|
|
114
116
|
isError: results.get(u.id)?.isError ?? false,
|
|
115
117
|
}));
|
|
116
118
|
}
|
|
119
|
+
/**
|
|
120
|
+
* Recover sub-agent runs as nested traces. A subagent-dispatch tool call (the
|
|
121
|
+
* `Agent` tool on the live CLI — older docs say `Task` — carrying an
|
|
122
|
+
* `input.subagent_type`) spawns a subagent whose own events the CLI tags with a
|
|
123
|
+
* top-level `parent_tool_use_id` = the dispatch tool-use id. We group those
|
|
124
|
+
* tagged tool calls under their dispatch, keyed by `subagent_type`. **Schema
|
|
125
|
+
* verified against real claude output** (`parent_tool_use_id` sibling of
|
|
126
|
+
* `message`, `subagent_type` in the dispatch input; tool named `Agent`) — the
|
|
127
|
+
* same `message.content` line shape `parseToolCalls` consumes, and we match the
|
|
128
|
+
* input field NOT the tool name so a future rename can't break it. Pure; empty
|
|
129
|
+
* for a harness that doesn't emit `parent_tool_use_id` (e.g. Codex).
|
|
130
|
+
*/
|
|
131
|
+
function parseSubagents(streamJson) {
|
|
132
|
+
const tasks = new Map(); // dispatch id → subagent name
|
|
133
|
+
const byParent = new Map();
|
|
134
|
+
const groupFor = (parent) => {
|
|
135
|
+
let g = byParent.get(parent);
|
|
136
|
+
if (!g) {
|
|
137
|
+
g = { uses: [], results: new Map() };
|
|
138
|
+
byParent.set(parent, g);
|
|
139
|
+
}
|
|
140
|
+
return g;
|
|
141
|
+
};
|
|
142
|
+
for (const line of streamJson.split("\n")) {
|
|
143
|
+
if (!line.trim())
|
|
144
|
+
continue;
|
|
145
|
+
let evt;
|
|
146
|
+
try {
|
|
147
|
+
evt = JSON.parse(line);
|
|
148
|
+
}
|
|
149
|
+
catch {
|
|
150
|
+
continue;
|
|
151
|
+
}
|
|
152
|
+
const content = evt.message?.content;
|
|
153
|
+
if (!Array.isArray(content))
|
|
154
|
+
continue;
|
|
155
|
+
const parent = typeof evt.parent_tool_use_id === "string"
|
|
156
|
+
? evt.parent_tool_use_id
|
|
157
|
+
: undefined;
|
|
158
|
+
for (const b of content) {
|
|
159
|
+
if (b.type === "tool_use" && typeof b.name === "string") {
|
|
160
|
+
const id = typeof b.id === "string" ? b.id : "";
|
|
161
|
+
if (!parent) {
|
|
162
|
+
// A subagent dispatch is any top-level tool_use whose input carries a
|
|
163
|
+
// `subagent_type` — the dispatch tool is named "Agent" on the live CLI
|
|
164
|
+
// (older docs say "Task"), so match the input field, NOT the tool name,
|
|
165
|
+
// to survive the rename. Confirmed against real claude output.
|
|
166
|
+
const sub = b.input?.subagent_type;
|
|
167
|
+
if (typeof sub === "string")
|
|
168
|
+
tasks.set(id, sub);
|
|
169
|
+
}
|
|
170
|
+
if (parent)
|
|
171
|
+
groupFor(parent).uses.push({ id, name: b.name, input: b.input });
|
|
172
|
+
}
|
|
173
|
+
else if (b.type === "tool_result" && parent) {
|
|
174
|
+
const id = typeof b.tool_use_id === "string" ? b.tool_use_id : "";
|
|
175
|
+
groupFor(parent).results.set(id, {
|
|
176
|
+
text: contentText(b.content),
|
|
177
|
+
isError: b.is_error === true,
|
|
178
|
+
});
|
|
179
|
+
}
|
|
180
|
+
}
|
|
181
|
+
}
|
|
182
|
+
const out = [];
|
|
183
|
+
for (const [taskId, name] of tasks) {
|
|
184
|
+
const g = byParent.get(taskId);
|
|
185
|
+
const toolCalls = (g?.uses ?? []).map((u) => ({
|
|
186
|
+
name: u.name,
|
|
187
|
+
input: u.input,
|
|
188
|
+
resultText: g?.results.get(u.id)?.text ?? "",
|
|
189
|
+
isError: g?.results.get(u.id)?.isError ?? false,
|
|
190
|
+
}));
|
|
191
|
+
out.push({ name, toolCalls });
|
|
192
|
+
}
|
|
193
|
+
return out;
|
|
194
|
+
}
|
|
117
195
|
/**
|
|
118
196
|
* The terminal `result` event — present in BOTH `--output-format` shapes (a
|
|
119
197
|
* `{type:"result", …}` line in stream-json, the single object in `json`), or
|
|
@@ -297,6 +375,7 @@ function makeResult(cwd, out, parsed, turns, modelRequests) {
|
|
|
297
375
|
hooks: parsed.hooks,
|
|
298
376
|
output: parsed.output,
|
|
299
377
|
modelRequests,
|
|
378
|
+
subagents: parseSubagents(out.stdout),
|
|
300
379
|
file: (p) => {
|
|
301
380
|
const f = (0, node_path_1.resolve)(cwd, p);
|
|
302
381
|
return (0, node_fs_1.existsSync)(f) ? (0, node_fs_1.readFileSync)(f, "utf-8") : null;
|
|
@@ -380,6 +459,29 @@ async function runHarnessTest(spec, opts = {}) {
|
|
|
380
459
|
await mock.close();
|
|
381
460
|
}
|
|
382
461
|
}
|
|
462
|
+
/**
|
|
463
|
+
* `runHarness` — the harness-scope entry of the revamped API (Phase 2 of
|
|
464
|
+
* `research/testing-api-design.md`). The harness has two execution scopes, `hook`
|
|
465
|
+
* (`runHook`) and `harness` (the whole assembled agent); today's `integration` /
|
|
466
|
+
* `e2e` / `eval` are all the **harness** scope under realness flags. This entry is
|
|
467
|
+
* the **deterministic** harness run (`model: "mock"`, the default) — the
|
|
468
|
+
* workhorse you gate every commit, with no key. A **real-model** harness run is
|
|
469
|
+
* non-deterministic by definition, so you don't *assert* a single one — you
|
|
470
|
+
* `measure()` it across trials (the eval scope). `egress` is a capability of this
|
|
471
|
+
* scope (the e2e tier), not a separate tier.
|
|
472
|
+
*
|
|
473
|
+
* Behaviour is identical to `runHarnessTest` (which it wraps); the new name +
|
|
474
|
+
* `model` flag make the scope/realness explicit and steer real-model runs to the
|
|
475
|
+
* right tool.
|
|
476
|
+
*/
|
|
477
|
+
async function runHarness(spec, opts = {}) {
|
|
478
|
+
if (opts.model === "real") {
|
|
479
|
+
throw new Error("runHarness runs the harness DETERMINISTICALLY (model: 'mock'). A real-model " +
|
|
480
|
+
"harness run is non-deterministic, so a single one can't be asserted — " +
|
|
481
|
+
"measure it across trials with `measure()` / `runEval` (the eval scope) instead.");
|
|
482
|
+
}
|
|
483
|
+
return runHarnessTest(spec, opts);
|
|
484
|
+
}
|
|
383
485
|
/** Pull the pillar-2 driver off an adapter, asserting it supports testing. */
|
|
384
486
|
function requireDriver(adapter) {
|
|
385
487
|
(0, adapter_conformance_js_1.assertHarnessTestable)(adapter);
|
package/dist/integration.d.ts
CHANGED
|
@@ -8,9 +8,17 @@
|
|
|
8
8
|
* (the plugin loader). Capability contract: needs the **`claude` binary and
|
|
9
9
|
* bubblewrap**, but **no API key and no network**. A `*.integration.test.ts`
|
|
10
10
|
* imports from here.
|
|
11
|
+
*
|
|
12
|
+
* Real **egress** is a CAPABILITY of this scope (the former `e2e` tier), not a
|
|
13
|
+
* separate tier: `egressRoutes()` probes whether allowlisted egress can route,
|
|
14
|
+
* and `runHook` takes `egress: { allow }` for allowlisted real outbound — gated
|
|
15
|
+
* by a routable sandbox + real network (a test self-skips via `egressRoutes()`).
|
|
16
|
+
* `vigiles/e2e` remains as a thin back-compat alias. See
|
|
17
|
+
* `research/testing-api-design.md` Part 4.
|
|
11
18
|
*/
|
|
12
19
|
export * from "./unit.js";
|
|
13
|
-
export * from "./
|
|
14
|
-
export * from "./
|
|
15
|
-
export
|
|
20
|
+
export * from "./harness-test.js";
|
|
21
|
+
export * from "./mock-model.js";
|
|
22
|
+
export type { LoadedPlugin } from "./plugin-loader.js";
|
|
23
|
+
export { egressRoutes } from "./run-hook.js";
|
|
16
24
|
//# sourceMappingURL=integration.d.ts.map
|
package/dist/integration.js
CHANGED
|
@@ -14,6 +14,7 @@ var __exportStar = (this && this.__exportStar) || function(m, exports) {
|
|
|
14
14
|
for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
|
|
15
15
|
};
|
|
16
16
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
17
|
+
exports.egressRoutes = void 0;
|
|
17
18
|
/**
|
|
18
19
|
* `vigiles/integration` — the **deterministic, assembled-machine** tier.
|
|
19
20
|
*
|
|
@@ -24,9 +25,17 @@ Object.defineProperty(exports, "__esModule", { value: true });
|
|
|
24
25
|
* (the plugin loader). Capability contract: needs the **`claude` binary and
|
|
25
26
|
* bubblewrap**, but **no API key and no network**. A `*.integration.test.ts`
|
|
26
27
|
* imports from here.
|
|
28
|
+
*
|
|
29
|
+
* Real **egress** is a CAPABILITY of this scope (the former `e2e` tier), not a
|
|
30
|
+
* separate tier: `egressRoutes()` probes whether allowlisted egress can route,
|
|
31
|
+
* and `runHook` takes `egress: { allow }` for allowlisted real outbound — gated
|
|
32
|
+
* by a routable sandbox + real network (a test self-skips via `egressRoutes()`).
|
|
33
|
+
* `vigiles/e2e` remains as a thin back-compat alias. See
|
|
34
|
+
* `research/testing-api-design.md` Part 4.
|
|
27
35
|
*/
|
|
28
36
|
__exportStar(require("./unit.js"), exports);
|
|
29
|
-
__exportStar(require("./
|
|
30
|
-
__exportStar(require("./
|
|
31
|
-
|
|
37
|
+
__exportStar(require("./harness-test.js"), exports);
|
|
38
|
+
__exportStar(require("./mock-model.js"), exports);
|
|
39
|
+
var run_hook_js_1 = require("./run-hook.js");
|
|
40
|
+
Object.defineProperty(exports, "egressRoutes", { enumerable: true, get: function () { return run_hook_js_1.egressRoutes; } });
|
|
32
41
|
//# sourceMappingURL=integration.js.map
|
package/dist/jest.d.ts
CHANGED
|
@@ -1,9 +1,11 @@
|
|
|
1
|
+
import type { Check } from "./check.js";
|
|
1
2
|
declare module "@jest/expect" {
|
|
2
3
|
interface Matchers<R> {
|
|
3
4
|
toHaveCreated(path: string): R;
|
|
4
5
|
toBlock(): R;
|
|
5
6
|
toBeatBaseline(baseline: string, arm: string, metric: string, by?: number): R;
|
|
7
|
+
toPass(check: Check<any>): R;
|
|
8
|
+
toPassAll(checks: readonly Check<any>[]): R;
|
|
6
9
|
}
|
|
7
10
|
}
|
|
8
|
-
export {};
|
|
9
11
|
//# sourceMappingURL=jest.d.ts.map
|
package/dist/jest.js
CHANGED
|
@@ -1,7 +1,8 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
-
/* eslint-disable max-params --
|
|
4
|
-
The matcher signatures mirror the runtime vigilesMatchers (positional args)
|
|
3
|
+
/* eslint-disable max-params, @typescript-eslint/no-explicit-any --
|
|
4
|
+
The matcher signatures mirror the runtime vigilesMatchers (positional args);
|
|
5
|
+
`Check<any>` matches the runtime generic for the type-only augmentation. */
|
|
5
6
|
/**
|
|
6
7
|
* vigiles — jest integration (opt-in).
|
|
7
8
|
*
|
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
import type { ModelTurn, ModelRequest } from "
|
|
2
|
-
export type { ModelTurn, ModelRequest } from "
|
|
1
|
+
import type { ModelTurn, ModelRequest } from "./core/harness-driver.js";
|
|
2
|
+
export type { ModelTurn, ModelRequest } from "./core/harness-driver.js";
|
|
3
3
|
/** Build a scripted model from an ordered list of turns. */
|
|
4
4
|
export declare function scriptModel(turns: readonly ModelTurn[]): ModelTurn[];
|
|
5
5
|
export interface TurnInfo {
|
|
@@ -1,6 +1,28 @@
|
|
|
1
|
-
import type { HookProtocol } from "
|
|
1
|
+
import type { HookProtocol } from "./core/hook-protocol.js";
|
|
2
2
|
import { type SandboxMode, type EgressAttempt } from "./sandbox.js";
|
|
3
3
|
export type { EgressAttempt };
|
|
4
|
+
/** Result of {@link propertyHook}: the first shrunk counterexample, if any. */
|
|
5
|
+
export interface HookPropertyResult<E> {
|
|
6
|
+
readonly passed: boolean;
|
|
7
|
+
readonly iterations: number;
|
|
8
|
+
/** The (shrunk) event that broke an invariant — present iff `!passed`. */
|
|
9
|
+
readonly counterexample?: E;
|
|
10
|
+
/** Which invariant failed — present iff `!passed`. */
|
|
11
|
+
readonly failedInvariant?: string;
|
|
12
|
+
}
|
|
13
|
+
/**
|
|
14
|
+
* Property-test a hook's decision over generated events. Throws nothing — returns
|
|
15
|
+
* a result you assert on (`assert.ok(r.passed)`), so the counterexample is
|
|
16
|
+
* inspectable. `mutate(event, rng)` produces a variation from the running event;
|
|
17
|
+
* each named invariant is checked against `decide(event)`.
|
|
18
|
+
*/
|
|
19
|
+
export declare function propertyHook<E, D>(opts: {
|
|
20
|
+
readonly seed: E;
|
|
21
|
+
readonly mutate: (event: E, rng: number) => E;
|
|
22
|
+
readonly decide: (event: E) => D;
|
|
23
|
+
readonly invariants: Record<string, (decision: D, event: E) => boolean>;
|
|
24
|
+
readonly iterations?: number;
|
|
25
|
+
}): HookPropertyResult<E>;
|
|
4
26
|
/** A hook event payload (the JSON Claude Code writes to the hook's stdin). */
|
|
5
27
|
export interface HookInput {
|
|
6
28
|
/** e.g. "PreToolUse", "PostToolUse", "Stop", "SessionStart", "PreCompact". */
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.propertyHook = propertyHook;
|
|
3
4
|
exports.parseHookOutput = parseHookOutput;
|
|
4
5
|
exports.decideHook = decideHook;
|
|
5
6
|
exports.runHookWith = runHookWith;
|
|
@@ -38,9 +39,36 @@ const node_child_process_1 = require("node:child_process");
|
|
|
38
39
|
const node_fs_1 = require("node:fs");
|
|
39
40
|
const node_os_1 = require("node:os");
|
|
40
41
|
const node_path_1 = require("node:path");
|
|
41
|
-
const hook_protocol_js_1 = require("./hook-protocol.js");
|
|
42
|
+
const hook_protocol_js_1 = require("./adapters/claude-code/hook-protocol.js");
|
|
42
43
|
const egress_js_1 = require("./egress.js");
|
|
43
44
|
const sandbox_js_1 = require("./sandbox.js");
|
|
45
|
+
const proofs_js_1 = require("./core/proofs.js");
|
|
46
|
+
/**
|
|
47
|
+
* Property-test a hook's decision over generated events. Throws nothing — returns
|
|
48
|
+
* a result you assert on (`assert.ok(r.passed)`), so the counterexample is
|
|
49
|
+
* inspectable. `mutate(event, rng)` produces a variation from the running event;
|
|
50
|
+
* each named invariant is checked against `decide(event)`.
|
|
51
|
+
*/
|
|
52
|
+
function propertyHook(opts) {
|
|
53
|
+
const wrapped = {};
|
|
54
|
+
for (const [name, inv] of Object.entries(opts.invariants)) {
|
|
55
|
+
wrapped[name] = (e) => inv(opts.decide(e), e);
|
|
56
|
+
}
|
|
57
|
+
const r = (0, proofs_js_1.propertyTest)(opts.seed, opts.mutate, wrapped, {
|
|
58
|
+
iterations: opts.iterations ?? 100,
|
|
59
|
+
sequenceLength: 1, // each event is independent — no mutation sequence
|
|
60
|
+
seed: 1,
|
|
61
|
+
});
|
|
62
|
+
if (r.passed)
|
|
63
|
+
return { passed: true, iterations: r.iterations };
|
|
64
|
+
const last = r.failingSequence?.[r.failingSequence.length - 1];
|
|
65
|
+
return {
|
|
66
|
+
passed: false,
|
|
67
|
+
iterations: r.iterations,
|
|
68
|
+
counterexample: r.shrunk ?? last,
|
|
69
|
+
failedInvariant: r.failedInvariant,
|
|
70
|
+
};
|
|
71
|
+
}
|
|
44
72
|
/** Parse stdout as a hook JSON decision (pure, testable without a process). */
|
|
45
73
|
function parseHookOutput(stdout) {
|
|
46
74
|
const s = stdout.trim();
|
|
@@ -35,7 +35,7 @@ const node_child_process_1 = require("node:child_process");
|
|
|
35
35
|
const node_fs_1 = require("node:fs");
|
|
36
36
|
const node_os_1 = require("node:os");
|
|
37
37
|
const node_path_1 = require("node:path");
|
|
38
|
-
const runtime_js_1 = require("./runtime.js");
|
|
38
|
+
const runtime_js_1 = require("./adapters/claude-code/runtime.js");
|
|
39
39
|
let cachedAvailable;
|
|
40
40
|
/**
|
|
41
41
|
* Whether this environment can ACTUALLY confine untrusted code under bubblewrap.
|
package/dist/scan.d.ts
CHANGED
|
@@ -3,7 +3,7 @@
|
|
|
3
3
|
* and what's broken, with **no model and no API key**.
|
|
4
4
|
*
|
|
5
5
|
* This is the deterministic substrate under the plugin/skill leaderboard
|
|
6
|
-
* (research/divergent-bets.md #9) and the harness-aware
|
|
6
|
+
* (research/divergent-bets.md #9) and the harness-aware scan
|
|
7
7
|
* (research/agent-supply-chain-security.md #1): it re-aims the machinery that
|
|
8
8
|
* already exists — `loadPlugin` (surfaces + dangling-ref/MCP/empty-machine
|
|
9
9
|
* warnings), `parseAgentTools` (the declared tool contract), and
|
package/dist/scan.js
CHANGED
|
@@ -4,7 +4,7 @@
|
|
|
4
4
|
* and what's broken, with **no model and no API key**.
|
|
5
5
|
*
|
|
6
6
|
* This is the deterministic substrate under the plugin/skill leaderboard
|
|
7
|
-
* (research/divergent-bets.md #9) and the harness-aware
|
|
7
|
+
* (research/divergent-bets.md #9) and the harness-aware scan
|
|
8
8
|
* (research/agent-supply-chain-security.md #1): it re-aims the machinery that
|
|
9
9
|
* already exists — `loadPlugin` (surfaces + dangling-ref/MCP/empty-machine
|
|
10
10
|
* warnings), `parseAgentTools` (the declared tool contract), and
|
package/dist/setup-plan.d.ts
CHANGED
|
@@ -10,7 +10,7 @@
|
|
|
10
10
|
*/
|
|
11
11
|
/** What `vigiles init` will set up. */
|
|
12
12
|
export interface SetupPlan {
|
|
13
|
-
/** Lint pillar — verify instruction-file references (specs, types, compile, lint
|
|
13
|
+
/** Lint pillar — verify instruction-file references (specs, types, compile, lint, hooks). */
|
|
14
14
|
lint: boolean;
|
|
15
15
|
/** Test pillar — test the harness (scaffold a starter harness test + CI job). */
|
|
16
16
|
test: boolean;
|
|
@@ -20,12 +20,16 @@ export interface SetupPlan {
|
|
|
20
20
|
plugin: boolean;
|
|
21
21
|
/** Strict rule severities in `.vigilesrc.json`. */
|
|
22
22
|
strict: boolean;
|
|
23
|
+
/** Rewrite an existing STALE CI workflow in place (instead of only warning). */
|
|
24
|
+
force: boolean;
|
|
23
25
|
}
|
|
24
26
|
/** The explicit choices a user pinned via flags (undefined = "not specified"). */
|
|
25
27
|
export interface ParsedSetupArgs {
|
|
26
28
|
target?: string;
|
|
27
29
|
strict: boolean;
|
|
28
30
|
yes: boolean;
|
|
31
|
+
/** `--force` — rewrite a stale CI workflow in place. */
|
|
32
|
+
force: boolean;
|
|
29
33
|
/** Lint pillar — `--lint` → true, `--no-lint` → false, absent → undefined. */
|
|
30
34
|
lint?: boolean;
|
|
31
35
|
/** Test pillar — `--test` → true, `--no-test` → false, absent → undefined. */
|
package/dist/setup-plan.js
CHANGED
|
@@ -33,6 +33,7 @@ function parseSetupArgs(args) {
|
|
|
33
33
|
target: flagValue(args, "--target="),
|
|
34
34
|
strict: args.includes("--strict"),
|
|
35
35
|
yes: args.includes("--yes") || args.includes("-y"),
|
|
36
|
+
force: args.includes("--force"),
|
|
36
37
|
lint: boolFlag(args, "lint"),
|
|
37
38
|
test: boolFlag(args, "test"),
|
|
38
39
|
harness: flagValue(args, "--harness="),
|
|
@@ -42,7 +43,14 @@ function parseSetupArgs(args) {
|
|
|
42
43
|
}
|
|
43
44
|
/** The non-interactive defaults: both pillars, CI, and the plugin. */
|
|
44
45
|
function defaultPlan(strict = false) {
|
|
45
|
-
return {
|
|
46
|
+
return {
|
|
47
|
+
lint: true,
|
|
48
|
+
test: true,
|
|
49
|
+
gha: true,
|
|
50
|
+
plugin: true,
|
|
51
|
+
strict,
|
|
52
|
+
force: false,
|
|
53
|
+
};
|
|
46
54
|
}
|
|
47
55
|
/**
|
|
48
56
|
* Whether to drop into interactive prompts: a human at a TTY who passed neither
|
|
@@ -151,6 +159,8 @@ function resolvePlan(parsed, answers) {
|
|
|
151
159
|
plan.gha = false;
|
|
152
160
|
if (parsed.plugin === false)
|
|
153
161
|
plan.plugin = false;
|
|
162
|
+
if (parsed.force)
|
|
163
|
+
plan.force = true;
|
|
154
164
|
if (parsed.target)
|
|
155
165
|
plan.test = false;
|
|
156
166
|
if (answers)
|
package/dist/test-coverage.js
CHANGED
|
@@ -151,7 +151,14 @@ function discoverHooks(basePath) {
|
|
|
151
151
|
}));
|
|
152
152
|
}
|
|
153
153
|
function discoverTests(basePath, globs, ignore) {
|
|
154
|
-
|
|
154
|
+
// `dot: true` so a colocated test under a DOT directory is found — most
|
|
155
|
+
// loose skills live in `.claude/skills/<name>/`, so the eval the warning
|
|
156
|
+
// suggests (`.claude/skills/<name>/<name>.eval.mjs`) is itself dot-pathed.
|
|
157
|
+
// Without this, a globstar (`**/*.eval.mjs`) silently skips it while the
|
|
158
|
+
// skill (matched by the explicit-dot `.claude/skills/*/SKILL.md` pattern) is
|
|
159
|
+
// still discovered — so the surface looks untested even after the user adds
|
|
160
|
+
// exactly the suggested file. DEFAULT_IGNORE still drops .git/node_modules/etc.
|
|
161
|
+
const found = (0, glob_1.globSync)([...globs], { cwd: basePath, ignore, dot: true });
|
|
155
162
|
return found.map((path) => ({ path, content: read((0, node_path_1.join)(basePath, path)) }));
|
|
156
163
|
}
|
|
157
164
|
/** Colocated: a test inside a skill dir, or a name-prefixed sibling of an agent/hook. */
|
package/dist/testing.d.ts
CHANGED
|
@@ -4,9 +4,16 @@
|
|
|
4
4
|
* (eval) — plus the runner-agnostic predicates/assertions. Kept deliberately
|
|
5
5
|
* separate from `vigiles/claude-code` so this surface can stay harness-agnostic as
|
|
6
6
|
* more harnesses are added. Granular paths (`vigiles/run-hook`, etc.) still work.
|
|
7
|
+
*
|
|
8
|
+
* It re-exports the composition-root runner modules (which do the Claude-Code
|
|
9
|
+
* default-wiring), never an adapter directly — the `agnostic-surface` eslint
|
|
10
|
+
* boundary forbids importing `src/adapters/*` from here. See
|
|
11
|
+
* `research/adapter-api-design.md`.
|
|
7
12
|
*/
|
|
8
|
-
export * from "./
|
|
9
|
-
export * from "./
|
|
10
|
-
export * from "./
|
|
13
|
+
export * from "./run-hook.js";
|
|
14
|
+
export * from "./harness-test.js";
|
|
15
|
+
export * from "./eval.js";
|
|
11
16
|
export * from "./harness-assert.js";
|
|
17
|
+
export * from "./check.js";
|
|
18
|
+
export { hookFired } from "./check.js";
|
|
12
19
|
//# sourceMappingURL=testing.d.ts.map
|
package/dist/testing.js
CHANGED
|
@@ -14,15 +14,27 @@ var __exportStar = (this && this.__exportStar) || function(m, exports) {
|
|
|
14
14
|
for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
|
|
15
15
|
};
|
|
16
16
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
17
|
+
exports.hookFired = void 0;
|
|
17
18
|
/**
|
|
18
19
|
* `vigiles/testing` — Pillar 2 entry point: the **harness-testing** API. Re-exports
|
|
19
20
|
* the three tiers — `runHook` (unit), `runHarnessTest` (deterministic), `runEval`
|
|
20
21
|
* (eval) — plus the runner-agnostic predicates/assertions. Kept deliberately
|
|
21
22
|
* separate from `vigiles/claude-code` so this surface can stay harness-agnostic as
|
|
22
23
|
* more harnesses are added. Granular paths (`vigiles/run-hook`, etc.) still work.
|
|
24
|
+
*
|
|
25
|
+
* It re-exports the composition-root runner modules (which do the Claude-Code
|
|
26
|
+
* default-wiring), never an adapter directly — the `agnostic-surface` eslint
|
|
27
|
+
* boundary forbids importing `src/adapters/*` from here. See
|
|
28
|
+
* `research/adapter-api-design.md`.
|
|
23
29
|
*/
|
|
24
|
-
__exportStar(require("./
|
|
25
|
-
__exportStar(require("./
|
|
26
|
-
__exportStar(require("./
|
|
30
|
+
__exportStar(require("./run-hook.js"), exports);
|
|
31
|
+
__exportStar(require("./harness-test.js"), exports);
|
|
32
|
+
__exportStar(require("./eval.js"), exports);
|
|
27
33
|
__exportStar(require("./harness-assert.js"), exports);
|
|
34
|
+
// The declarative check vocabulary is now first-class at the front door. Its
|
|
35
|
+
// `hookFired` (a `Check<Trace>`) supersedes the legacy boolean predicate of the
|
|
36
|
+
// same name — the explicit re-export below wins over the two `export *`s.
|
|
37
|
+
__exportStar(require("./check.js"), exports);
|
|
38
|
+
var check_js_1 = require("./check.js");
|
|
39
|
+
Object.defineProperty(exports, "hookFired", { enumerable: true, get: function () { return check_js_1.hookFired; } });
|
|
28
40
|
//# sourceMappingURL=testing.js.map
|
package/dist/unit.d.ts
CHANGED
|
@@ -12,6 +12,8 @@
|
|
|
12
12
|
* `vigiles/e2e`.
|
|
13
13
|
*/
|
|
14
14
|
export * from "./harness-assert.js";
|
|
15
|
-
export { runHook, parseHookOutput, decideHook, } from "./
|
|
16
|
-
export type { HookInput, HookOutput, HookRunResult, RunHookOptions, } from "./
|
|
15
|
+
export { runHook, parseHookOutput, decideHook, propertyHook, } from "./run-hook.js";
|
|
16
|
+
export type { HookInput, HookOutput, HookRunResult, RunHookOptions, HookPropertyResult, } from "./run-hook.js";
|
|
17
|
+
export * from "./check.js";
|
|
18
|
+
export { hookFired } from "./check.js";
|
|
17
19
|
//# sourceMappingURL=unit.d.ts.map
|
package/dist/unit.js
CHANGED
|
@@ -14,7 +14,7 @@ var __exportStar = (this && this.__exportStar) || function(m, exports) {
|
|
|
14
14
|
for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
|
|
15
15
|
};
|
|
16
16
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
17
|
-
exports.decideHook = exports.parseHookOutput = exports.runHook = void 0;
|
|
17
|
+
exports.hookFired = exports.propertyHook = exports.decideHook = exports.parseHookOutput = exports.runHook = void 0;
|
|
18
18
|
/**
|
|
19
19
|
* `vigiles/unit` — the **no-capability** harness-testing surface.
|
|
20
20
|
*
|
|
@@ -29,8 +29,14 @@ exports.decideHook = exports.parseHookOutput = exports.runHook = void 0;
|
|
|
29
29
|
* `vigiles/e2e`.
|
|
30
30
|
*/
|
|
31
31
|
__exportStar(require("./harness-assert.js"), exports);
|
|
32
|
-
var run_hook_js_1 = require("./
|
|
32
|
+
var run_hook_js_1 = require("./run-hook.js");
|
|
33
33
|
Object.defineProperty(exports, "runHook", { enumerable: true, get: function () { return run_hook_js_1.runHook; } });
|
|
34
34
|
Object.defineProperty(exports, "parseHookOutput", { enumerable: true, get: function () { return run_hook_js_1.parseHookOutput; } });
|
|
35
35
|
Object.defineProperty(exports, "decideHook", { enumerable: true, get: function () { return run_hook_js_1.decideHook; } });
|
|
36
|
+
Object.defineProperty(exports, "propertyHook", { enumerable: true, get: function () { return run_hook_js_1.propertyHook; } });
|
|
37
|
+
// The check vocabulary is part of the base surface (pure, no capability). Its
|
|
38
|
+
// `hookFired` check supersedes the legacy boolean predicate of the same name.
|
|
39
|
+
__exportStar(require("./check.js"), exports);
|
|
40
|
+
var check_js_1 = require("./check.js");
|
|
41
|
+
Object.defineProperty(exports, "hookFired", { enumerable: true, get: function () { return check_js_1.hookFired; } });
|
|
36
42
|
//# sourceMappingURL=unit.js.map
|
package/dist/vitest.d.mts
CHANGED
|
@@ -1,9 +1,11 @@
|
|
|
1
|
+
import type { Check } from "./check.js";
|
|
1
2
|
declare module "@vitest/expect" {
|
|
2
3
|
interface Matchers<T = any> {
|
|
3
4
|
toHaveCreated(path: string): T;
|
|
4
5
|
toBlock(): T;
|
|
5
6
|
toBeatBaseline(baseline: string, arm: string, metric: string, by?: number): T;
|
|
7
|
+
toPass(check: Check<any>): T;
|
|
8
|
+
toPassAll(checks: readonly Check<any>[]): T;
|
|
6
9
|
}
|
|
7
10
|
}
|
|
8
|
-
export {};
|
|
9
11
|
//# sourceMappingURL=vitest.d.mts.map
|
package/hooks/refs-nudge.sh
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
#!/usr/bin/env bash
|
|
2
2
|
# PostToolUse hook — nudge the agent to express references in instruction files
|
|
3
|
-
# (CLAUDE.md / AGENTS.md / SKILL.md) as vigiles marks, so `vigiles
|
|
3
|
+
# (CLAUDE.md / AGENTS.md / SKILL.md) as vigiles marks, so `vigiles lint` can
|
|
4
4
|
# actually verify them. Non-blocking by default; set the `unmarked-refs` rule to
|
|
5
5
|
# "error" in .vigilesrc.json to turn the nudge into a hard block, or to false to
|
|
6
6
|
# disable it. Runs as its OWN PostToolUse entry so its stdout stays clean JSON.
|
package/hooks/session-start.sh
CHANGED
|
@@ -1,8 +1,8 @@
|
|
|
1
1
|
#!/usr/bin/env bash
|
|
2
|
-
# SessionStart hook — inject a one-line vigiles
|
|
2
|
+
# SessionStart hook — inject a one-line vigiles lint summary into
|
|
3
3
|
# Claude's context at session start and after compaction.
|
|
4
4
|
#
|
|
5
|
-
# Uses `vigiles
|
|
5
|
+
# Uses `vigiles lint --summary` which is designed for this: it
|
|
6
6
|
# silences all per-stage output and prints a single line like
|
|
7
7
|
# vigiles: 3 stale / 2 validation errors / 1 duplicate
|
|
8
8
|
# (or "vigiles: clean") so the injection costs a handful of tokens.
|
|
@@ -24,7 +24,7 @@ fi
|
|
|
24
24
|
|
|
25
25
|
# Prefer local install, fall back to npx.
|
|
26
26
|
if command -v npx &>/dev/null; then
|
|
27
|
-
SUMMARY=$(npx --no-install vigiles
|
|
27
|
+
SUMMARY=$(npx --no-install vigiles lint --summary 2>/dev/null || true)
|
|
28
28
|
else
|
|
29
29
|
exit 0
|
|
30
30
|
fi
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "vigiles",
|
|
3
|
-
"version": "4.0.
|
|
3
|
+
"version": "4.0.2",
|
|
4
4
|
"description": "Compile .spec.ts files to instruction files (CLAUDE.md, AGENTS.md) with linter cross-referencing",
|
|
5
5
|
"bin": {
|
|
6
6
|
"vigiles": "dist/cli.js"
|
|
@@ -20,19 +20,20 @@
|
|
|
20
20
|
"./spec": "./dist/core/spec.js",
|
|
21
21
|
"./compile": "./dist/core/compile.js",
|
|
22
22
|
"./linters": "./dist/core/linters.js",
|
|
23
|
-
"./eval": "./dist/
|
|
24
|
-
"./harness-test": "./dist/
|
|
23
|
+
"./eval": "./dist/eval.js",
|
|
24
|
+
"./harness-test": "./dist/harness-test.js",
|
|
25
25
|
"./harness-assert": "./dist/harness-assert.js",
|
|
26
|
-
"./run-hook": "./dist/
|
|
26
|
+
"./run-hook": "./dist/run-hook.js",
|
|
27
27
|
"./plugin-loader": "./dist/adapters/claude-code/plugin-loader.js",
|
|
28
28
|
"./mcp": "./dist/core/mcp.js",
|
|
29
|
-
"./judge": "./dist/
|
|
30
|
-
"./mock-model": "./dist/
|
|
29
|
+
"./judge": "./dist/judge.js",
|
|
30
|
+
"./mock-model": "./dist/mock-model.js",
|
|
31
31
|
"./vitest": {
|
|
32
32
|
"types": "./dist/vitest.d.mts",
|
|
33
33
|
"default": "./dist/vitest.mjs"
|
|
34
34
|
},
|
|
35
|
-
"./jest": "./dist/jest.js"
|
|
35
|
+
"./jest": "./dist/jest.js",
|
|
36
|
+
"./check": "./dist/check.js"
|
|
36
37
|
},
|
|
37
38
|
"files": [
|
|
38
39
|
"dist/**/*.js",
|