vigiles 4.0.2 → 5.0.0

package/dist/tool-intercept.js

@@ -0,0 +1,165 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.DEFAULT_INTERCEPT_REASON = exports.INTERCEPT_TOOLS_ENV = void 0;
+exports.decideIntercept = decideIntercept;
+exports.interceptHookDecision = interceptHookDecision;
+exports.buildInterceptSettings = buildInterceptSettings;
+exports.serializeIntercepts = serializeIntercepts;
+exports.parseIntercepts = parseIntercepts;
+/**
+ * Tool interception — the eval-tier half of the tool-call spy.
+ *
+ * The eval tier drives the REAL model, so the agent's tool decisions are genuine.
+ * But letting a side-effecting tool actually run — a paid image API, `git push`,
+ * spawning a paid subagent — makes the eval expensive and dangerous. An
+ * **intercept** lets the model emit the `tool_use` (so its arguments are captured
+ * in the `Trace`, where `toolWith` / `notTool` assert on them) while a PreToolUse
+ * hook **denies the real execution** with a block message. The run stays cheap and
+ * side-effect-free, but the agent's *decision* — the thing a completion grader
+ * can't see — is fully observable.
+ *
+ * This module is the PURE decision + wiring core, mirroring the agent-contract
+ * rail (`src/adapters/claude-code/agent-runtime.ts`):
+ *
+ * - `decideIntercept` — does this call get intercepted, and with what deny reason;
+ * - `buildInterceptSettings` — the PreToolUse hook fragment routing matched tools
+ *   through `vigiles intercept-tool-hook`;
+ * - `serializeIntercepts` / `parseIntercepts` — the env round-trip (incl. RegExp
+ *   matchers) the hook subprocess reads back.
+ *
+ * The hook denies via exit 2 + a stderr message (the same block mechanism the
+ * agent rail uses). IMPORTANT — Claude Code surfaces that to the model as a
+ * *blocked* call: the tool is intercepted (prevented), NOT executed, and the model
+ * is NOT handed a faked successful return. So this is **intercept-and-prevent +
+ * observe the attempt**, not a faithful tool mock: it's sound for "did the agent
+ * ATTEMPT X / call it with these args / push to the wrong branch" (first-attempt
+ * questions, where what happens after doesn't matter), and unsound for "stub the
+ * tool and let the trajectory continue as if it succeeded" (the model is told it
+ * was blocked, so a multi-step flow that needs the real result will derail). CC
+ * exposes no "skip execution but return this as success" primitive for arbitrary
+ * tools, so deny+reason is the closest available — with that ceiling.
+ */
+const arg_match_js_1 = require("./arg-match.js");
+/** Env var the spawned `vigiles intercept-tool-hook` reads its intercept list from. */
+exports.INTERCEPT_TOOLS_ENV = "VIGILES_INTERCEPT_TOOLS";
+/** The default denial reason — honest that the call was intercepted (prevented), NOT executed. */
+exports.DEFAULT_INTERCEPT_REASON = "vigiles intercepted this tool call for testing — it was NOT executed. " +
+    "Do not retry it; treat the tool as unavailable and continue.";
+/**
+ * Decide whether a tool call should be intercepted. Returns the first matching
+ * intercept's denial reason (preventing real execution), or `{ intercept: false }`
+ * to let the call run for real. Pure — the same logic `vigiles intercept-tool-hook`
+ * runs.
+ */
+function decideIntercept(toolName, input, intercepts) {
+    for (const i of intercepts) {
+        if (i.tool !== toolName)
+            continue;
+        if (i.when && !(0, arg_match_js_1.matchesArgs)(input, i.when))
+            continue;
+        return {
+            intercept: true,
+            denyReason: i.denyReason ?? exports.DEFAULT_INTERCEPT_REASON,
+        };
+    }
+    return { intercept: false };
+}
+/**
+ * Decide from a raw PreToolUse event JSON (the hook's stdin). Parses `tool_name`
+ * + `tool_input`, then defers to {@link decideIntercept}. Malformed input or a
+ * missing tool name is a no-op (let it run) — never fail closed on a parse error.
+ */
+function interceptHookDecision(rawEvent, intercepts) {
+    let parsed;
+    try {
+        parsed = JSON.parse(rawEvent);
+    }
+    catch {
+        return { intercept: false };
+    }
+    const tool = parsed.tool_name ?? "";
+    if (!tool)
+        return { intercept: false };
+    return decideIntercept(tool, parsed.tool_input ?? {}, intercepts);
+}
+function escapeRegex(s) {
+    return s.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
+}
+function uniqueToolNames(intercepts) {
+    return [...new Set(intercepts.map((i) => i.tool))];
+}
+/**
+ * Build the PreToolUse hook fragment that routes every intercepted tool through
+ * `vigiles intercept-tool-hook`. The `matcher` is a CC tool-name regex over the
+ * union of intercepted tool names (each escaped), so unrelated tools are never
+ * intercepted. Merge the result into an arm's `settings`; the intercept list
+ * itself travels in the {@link INTERCEPT_TOOLS_ENV} env var (see
+ * {@link serializeIntercepts}).
+ */
+function buildInterceptSettings(intercepts, opts = {}) {
+    const command = opts.command ?? "npx vigiles intercept-tool-hook";
+    const matcher = uniqueToolNames(intercepts).map(escapeRegex).join("|");
+    return {
+        hooks: {
+            PreToolUse: [{ matcher, hooks: [{ type: "command", command }] }],
+        },
+    };
+}
+function isWireRegex(v) {
+    return typeof v === "object" && v !== null && "re" in v;
+}
+function encodeMatcher(m) {
+    const out = {};
+    for (const [k, v] of Object.entries(m)) {
+        out[k] = v instanceof RegExp ? { re: v.source, flags: v.flags } : v;
+    }
+    return out;
+}
+function decodeMatcher(w) {
+    const out = {};
+    for (const [k, v] of Object.entries(w)) {
+        out[k] = isWireRegex(v) ? new RegExp(v.re, v.flags) : v;
+    }
+    return out;
+}
+/**
+ * Serialize an intercept list to a JSON string for {@link INTERCEPT_TOOLS_ENV}.
+ * RegExp matchers are encoded as `{ re, flags }` so they round-trip exactly (a
+ * plain `JSON.stringify` would drop them to `{}`).
+ */
+function serializeIntercepts(intercepts) {
+    return JSON.stringify(intercepts.map((i) => ({
+        tool: i.tool,
+        denyReason: i.denyReason,
+        when: i.when ? encodeMatcher(i.when) : undefined,
+    })));
+}
+/** Parse an intercept list from the env JSON (tolerant — a bad entry is skipped). */
+function parseIntercepts(json) {
+    let data;
+    try {
+        data = JSON.parse(json);
+    }
+    catch {
+        return [];
+    }
+    if (!Array.isArray(data))
+        return [];
+    const out = [];
+    for (const item of data) {
+        if (item === null || typeof item !== "object")
+            continue;
+        const o = item;
+        if (typeof o.tool !== "string")
+            continue;
+        out.push({
+            tool: o.tool,
+            denyReason: typeof o.denyReason === "string" ? o.denyReason : undefined,
+            when: o.when !== null && typeof o.when === "object"
+                ? decodeMatcher(o.when)
+                : undefined,
+        });
+    }
+    return out;
+}
+//# sourceMappingURL=tool-intercept.js.map

package/dist/tool-stub.d.ts

@@ -0,0 +1,35 @@
+/**
+ * A single fake binary: shadow the tool named `name` on PATH so that any
+ * invocation prints `stdout` (default `""`), writes `stderr` (default none) to
+ * fd 2, and exits with `exitCode` (default 0). Argv is ignored in the MVP.
+ */
+export interface ToolStub {
+    /** The binary name to shadow on PATH (e.g. `"gh"`, `"psql"`). */
+    readonly name: string;
+    /** Canned stdout the fake prints. Default `""`. */
+    readonly stdout?: string;
+    /** Canned stderr the fake writes to fd 2. Default: none. */
+    readonly stderr?: string;
+    /** Exit code the fake returns. Default `0`. */
+    readonly exitCode?: number;
+}
+/**
+ * Render the POSIX shell script for one stub. `printf '%s'` (not `echo`) prints
+ * the decoded bytes with no added trailing newline and no backslash/`-n`
+ * surprises, so the fixture round-trips byte-for-byte.
+ */
+export declare function renderToolStub(stub: ToolStub): string;
+/**
+ * Write one executable POSIX shell stub per entry into `binDir` (which must
+ * already exist). Each file is named exactly `stub.name` and `chmod 0o755` so it
+ * is directly executable once `binDir` is on PATH. Pure-ish — fs only, no spawn.
+ */
+export declare function writeToolStubs(binDir: string, stubs: readonly ToolStub[]): void;
+/**
+ * Convenience: mkdtemp a fresh bin dir under `parentDir`, write `stubs` into it,
+ * and return its absolute path. Caller PREPENDS this dir to PATH so the fakes win
+ * over the real binaries, and removes it when done (it lives under `parentDir`,
+ * so a `parentDir` cleanup also clears it).
+ */
+export declare function stubBinDir(stubs: readonly ToolStub[], parentDir: string): string;
+//# sourceMappingURL=tool-stub.d.ts.map

package/dist/tool-stub.js

@@ -0,0 +1,92 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.renderToolStub = renderToolStub;
+exports.writeToolStubs = writeToolStubs;
+exports.stubBinDir = stubBinDir;
+/**
+ * vigiles — **tool stubs on PATH** (rung R2 of the eval coverage model).
+ *
+ * A skill/hook/agent often calls a CLI tool (`psql`, `redis-cli`, `gh`, `git`,
+ * `z3`, …) and then works with the RESULT. To test that downstream logic without
+ * a live service, you SHADOW the real binary on PATH with a fake that emits a
+ * **recorded / author-provided canned result** — a standard PATH-shim +
+ * VCR-style record/replay technique, not a novel invention. See
+ * `research/eval-coverage-and-isolation.md` (the three-rung model).
+ *
+ * This module is the REPLAY half: write a fake executable per tool that prints a
+ * canned stdout/stderr and exits a canned code. A record-from-real-tool half (run
+ * the real binary once at a known version, capture its output as the fixture) is
+ * a documented follow-on — NOT implemented here.
+ *
+ * CRITICAL: the canned outputs are author/recorded fixtures, **never**
+ * model-synthesized — a synthesized `gh`/`git` output looks plausible but
+ * diverges from the real tool/version, producing false confidence (a green test
+ * against a fiction).
+ *
+ * MVP scope: one canned result per binary; argv is IGNORED (every invocation of
+ * the stub returns the same result). Argv-matching (different output per
+ * sub-command / flags) is a follow-on.
+ */
+const node_fs_1 = require("node:fs");
+const node_path_1 = require("node:path");
+/**
+ * Encode a string as a single-line base64 literal safe to embed verbatim inside a
+ * POSIX shell script.
+ *
+ * We base64-encode-and-decode the canned content (rather than interpolating it
+ * raw, or `cat`ing a sibling data file) because base64's alphabet is a strict
+ * subset of `[A-Za-z0-9+/=]` — it can never contain a quote, `$`, backtick,
+ * newline, `;`, or any other shell metacharacter, so the embedded literal is
+ * injection-proof regardless of the fixture's bytes. The script decodes it back
+ * with `base64 -d` at run time, so the original bytes round-trip exactly. This is
+ * simpler than a sibling data file (one self-contained script, nothing else to
+ * write/clean up) and strictly safer than quoting.
+ */
+function b64(s) {
+    return Buffer.from(s, "utf-8").toString("base64");
+}
+/**
+ * Render the POSIX shell script for one stub. `printf '%s'` (not `echo`) prints
+ * the decoded bytes with no added trailing newline and no backslash/`-n`
+ * surprises, so the fixture round-trips byte-for-byte.
+ */
+function renderToolStub(stub) {
+    const lines = ["#!/bin/sh"];
+    // stdout: decode the base64 literal straight to stdout. We pipe `base64 -d`'s
+    // output directly (NOT through `printf '%s' "$(...)"`) because command
+    // substitution strips trailing newlines — piping preserves the bytes exactly.
+    if (stub.stdout !== undefined && stub.stdout !== "") {
+        lines.push(`printf '%s' '${b64(stub.stdout)}' | base64 -d`);
+    }
+    // stderr: same, redirected to fd 2.
+    if (stub.stderr !== undefined && stub.stderr !== "") {
+        lines.push(`printf '%s' '${b64(stub.stderr)}' | base64 -d >&2`);
+    }
+    lines.push(`exit ${String(stub.exitCode ?? 0)}`);
+    return lines.join("\n") + "\n";
+}
+/**
+ * Write one executable POSIX shell stub per entry into `binDir` (which must
+ * already exist). Each file is named exactly `stub.name` and `chmod 0o755` so it
+ * is directly executable once `binDir` is on PATH. Pure-ish — fs only, no spawn.
+ */
+function writeToolStubs(binDir, stubs) {
+    for (const stub of stubs) {
+        const file = (0, node_path_1.join)(binDir, stub.name);
+        (0, node_fs_1.writeFileSync)(file, renderToolStub(stub), "utf-8");
+        (0, node_fs_1.chmodSync)(file, 0o755);
+    }
+}
+/**
+ * Convenience: mkdtemp a fresh bin dir under `parentDir`, write `stubs` into it,
+ * and return its absolute path. Caller PREPENDS this dir to PATH so the fakes win
+ * over the real binaries, and removes it when done (it lives under `parentDir`,
+ * so a `parentDir` cleanup also clears it).
+ */
+function stubBinDir(stubs, parentDir) {
+    (0, node_fs_1.mkdirSync)(parentDir, { recursive: true });
+    const binDir = (0, node_fs_1.mkdtempSync)((0, node_path_1.join)(parentDir, "vigiles-stub-bin-"));
+    writeToolStubs(binDir, stubs);
+    return binDir;
+}
+//# sourceMappingURL=tool-stub.js.map

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "vigiles",
-  "version": "4.0.2",
+  "version": "5.0.0",
   "description": "Compile .spec.ts files to instruction files (CLAUDE.md, AGENTS.md) with linter cross-referencing",
   "bin": {
     "vigiles": "dist/cli.js"
@@ -9,31 +9,20 @@
   "types": "./dist/core/spec.d.ts",
   "exports": {
     ".": "./dist/core/spec.js",
+    "./spec": "./dist/core/spec.js",
+    "./linting": "./dist/linting.js",
+    "./testing": "./dist/testing.js",
     "./unit": "./dist/unit.js",
     "./integration": "./dist/integration.js",
     "./e2e": "./dist/e2e.js",
-    "./linting": "./dist/linting.js",
-    "./testing": "./dist/testing.js",
     "./claude-code": "./dist/claude-code.js",
     "./codex": "./dist/codex.js",
     "./adapter": "./dist/adapter.js",
-    "./spec": "./dist/core/spec.js",
-    "./compile": "./dist/core/compile.js",
-    "./linters": "./dist/core/linters.js",
-    "./eval": "./dist/eval.js",
-    "./harness-test": "./dist/harness-test.js",
-    "./harness-assert": "./dist/harness-assert.js",
-    "./run-hook": "./dist/run-hook.js",
-    "./plugin-loader": "./dist/adapters/claude-code/plugin-loader.js",
-    "./mcp": "./dist/core/mcp.js",
-    "./judge": "./dist/judge.js",
-    "./mock-model": "./dist/mock-model.js",
     "./vitest": {
       "types": "./dist/vitest.d.mts",
       "default": "./dist/vitest.mjs"
     },
-    "./jest": "./dist/jest.js",
-    "./check": "./dist/check.js"
+    "./jest": "./dist/jest.js"
   },
   "files": [
     "dist/**/*.js",
@@ -67,10 +56,15 @@
     "test:vitest": "npm run build && vitest run --project runners",
     "test:jest": "npm run build && jest",
     "test:types": "npm run build && tsc --noEmit -p test/types/tsconfig.json",
-    "demo:plugin": "npm run build && node examples/plugin-test-demo.mjs"
+    "demo:plugin": "npm run build && node examples/plugin-test-demo.mjs",
+    "api:report": "npm run build && node scripts/api-extractor.mjs --local",
+    "api:check": "npm run build && node scripts/api-extractor.mjs",
+    "docs:api": "npm run api:report && api-documenter markdown -i temp -o api-reference"
   },
   "devDependencies": {
     "@eslint/js": "^10.0.1",
+    "@microsoft/api-documenter": "^7.30.7",
+    "@microsoft/api-extractor": "^7.58.9",
     "@types/js-yaml": "^4.0.9",
     "@types/minimatch": "^5.1.2",
     "@types/node": "^20.19.39",

package/skills/test-harness/SKILL.md

@@ -22,7 +22,8 @@ Match what you're testing to the cheapest tier that can answer it:
 | "Is the hook actually **wired into** the assembled plugin and does it fire in a real session?"                                         | **Deterministic** | free, no API key (real `claude` + scripted mock) | `runHarnessTest` + `scriptModel`                                                              |
 | "Did the injected context (a SessionStart hook, a `/command`) actually **reach the model**?"                                           | **Deterministic** | free, no API key                                 | `runHarnessTest` → `trace.modelRequests` / `assertRequestContains`                            |
 | "Does this skill's **description trigger** when it should (recall) **and stay quiet** when it shouldn't (precision)?"                  | **Eval**          | **paid** (real model)                            | `measureTriggerRate` (+ `irrelevantPrompts`) → `assertTriggerRate({ min, maxFalsePositive })` |
-| "Does this harness change **move what the agent does**?" (A/B, signal vs noise)                                                        | **Eval**          | **paid** (real model)                            | `runEval` + `assertSignificant`                                                               |
+| "Is this exact skill's **output any good**?" — absolute quality, no on/off baseline (the default for testing one skill)                | **Eval**          | **paid** (real model)                            | `measure({ checks: [judged(rubric)] })` → `assertRates({ min })`                              |
+| "Does this harness change **move what the agent does**, _relative_ to off?" — A/B lift, regression, signal vs noise                    | **Eval**          | **paid** (real model)                            | `runEval` (arms) + `assertSignificant`                                                        |
 
 Most harness questions — block/allow, wired-in, context-landed — never need a
 model. Only "does the model trigger / behave differently" needs the eval tier.
@@ -30,6 +31,37 @@ model. Only "does the model trigger / behave differently" needs the eval tier.
 If the unit and deterministic tiers can both answer it, **prefer unit**: it's
 faster and reaches events the deterministic mock can't drive.
 
+## Step 0.5 — Set honest expectations (what's testable, and at what cost)
+
+Be explicit with the user about which bucket each surface falls into — never let
+"we'll test it" hide whether that's free, sub-priced, or needs a container. Every
+surface sorts into one of three buckets:
+
+- **A — Free & deterministic** (no model, runs in CI on every commit): a hook's
+  block/allow decision (`runHook`), a tool-contract / "did NOT call the forbidden
+  tool" check, structural facts (`vigiles scan`), and **record-replay** of any tool
+  a skill shells out to (record the real result once, replay it via a PATH stub).
+- **B — Model-gated, on your subscription** (real model, **no metered API**): does a
+  skill's description **fire** (`measureTriggerRate`, recall + precision) **and**
+  does its guidance actually **produce good output** (score it directly:
+  `measure({ checks: [judged(rubric)] })` + `assertRates` — the absolute oracle;
+  use a `runEval` A/B on-vs-off only when you need the _relative_ lift). This is
+  the half a **prose / guidance skill** lives in —
+  its worth is behavioral, so only a model can judge it. That is **not** "uncovered"
+  and **not** free: it's fully testable on the sub. State it that way.
+- **C — Needs a real service** (a real browser / DB / redis / a11y runtime): vigiles
+  **composes with a container** here; it does not fake real semantics. Name the
+  service and hand off — don't pretend a cheap tier substitutes for it.
+
+So a prose-skill library is roughly **~100% testable (some free, most on your sub),
+~0% needs-a-container** — not "poorly covered." An accessibility/browser plugin is
+the worst case, with a large bucket C. When you report coverage, give **two
+numbers**: "% testable at all (free + sub)" vs "% that needs a container", and say
+which surfaces are free vs sub-priced. The model-gated half is the **point** of the
+eval pillar (affordable on the sub), not a gap — and testing a prose skill's
+_behavior_ requires a real model for **everyone** (promptfoo, the SDKs, all of it);
+vigiles just does it on your subscription instead of metered API.
+
 ## Step 1 — Ensure vigiles is installed
 
 Check whether `vigiles` is a dependency (`package.json`), and install it as a
@@ -103,8 +135,29 @@ assertHookFired(r, "SessionStart");
 assertRequestContains(r, "expected injected text"); // did it actually land?
 ```
 
-**Eval (`runEval`)** — A/B the change on vs off across real-model trials, then
-gate on significance, not eyeballing:
+**Eval — absolute (`measure` + `judged`)** — testing _one_ skill, the usual case:
+score its output directly against a rubric. No on/off baseline — this is the
+"is it any good?" oracle (what promptfoo/DeepEval lead with), and the right
+default when there's nothing to compare against:
+
+```ts
+import { measure, judged, skill, assertRates } from "vigiles/testing";
+
+const report = await measure({
+  pluginDir: "./",
+  task: "…a task the skill should handle…",
+  checks: [
+    skill("my-plugin:my-skill"), // it fired
+    judged("the answer correctly does X and avoids Y"), // …and the output is good
+  ],
+  trials: 6,
+});
+assertRates(report, { min: 0.8 }); // each check passes ≥ 80% of trials
+```
+
+**Eval — relative (`runEval` + `assertSignificant`)** — when the question is
+_lift over no-skill_ (regression, or proving a change isn't noise): A/B the
+change on vs off and gate on significance, not eyeballing:
 
 ```ts
 import { runEval, assertSignificant } from "vigiles/testing";