vigiles 35.0.0 → 37.0.0

package/dist/tool-stub.d.ts

@@ -1,35 +1,30 @@
-/**
- * A single fake binary: shadow the tool named `name` on PATH so that any
- * invocation prints `stdout` (default `""`), writes `stderr` (default none) to
- * fd 2, and exits with `exitCode` (default 0). Argv is ignored in the MVP.
- */
-export interface ToolStub {
-    /** The binary name to shadow on PATH (e.g. `"gh"`, `"psql"`). */
-    readonly name: string;
-    /** Canned stdout the fake prints. Default `""`. */
-    readonly stdout?: string;
-    /** Canned stderr the fake writes to fd 2. Default: none. */
-    readonly stderr?: string;
-    /** Exit code the fake returns. Default `0`. */
-    readonly exitCode?: number;
+import { type StubCall, type ToolStub } from "./core/stub-rules.js";
+/** Where a run's stubs live. `binDir` goes first on the run's PATH. */
+export interface StubDir {
+    readonly root: string;
+    readonly binDir: string;
 }
+/** How a shim reaches the stub runtime: this node, and this vigiles' `cli.js`. */
+export interface StubLauncher {
+    readonly node: string;
+    readonly cli: string;
+}
+/** The shim for one stub. Pure. */
+export declare function stubShim(launcher: StubLauncher, root: string, name: string): string;
 /**
- * Render the POSIX shell script for one stub. `printf '%s'` (not `echo`) prints
- * the decoded bytes with no added trailing newline and no backslash/`-n`
- * surprises, so the fixture round-trips byte-for-byte.
- */
-export declare function renderToolStub(stub: ToolStub): string;
-/**
- * Write one executable POSIX shell stub per entry into `binDir` (which must
- * already exist). Each file is named exactly `stub.name` and `chmod 0o755` so it
- * is directly executable once `binDir` is on PATH. Pure-ish — fs only, no spawn.
+ * Write a fresh stub root for `stubs`: a shim and a rules file per stub. The
+ * stubs must already be parsed (the spec boundary did it). Remove it with
+ * {@link removeStubDir}.
  */
-export declare function writeToolStubs(binDir: string, stubs: readonly ToolStub[]): void;
+export declare function writeStubDir(stubs: readonly ToolStub[], launcher: StubLauncher): StubDir;
+/** Every invocation the run's stubs recorded, in order. Empty when none was called. */
+export declare function readStubCalls(dir: StubDir): readonly StubCall[];
 /**
- * Convenience: mkdtemp a fresh bin dir under `parentDir`, write `stubs` into it,
- * and return its absolute path. Caller PREPENDS this dir to PATH so the fakes win
- * over the real binaries, and removes it when done (it lives under `parentDir`,
- * so a `parentDir` cleanup also clears it).
+ * Forget every call recorded so far. The log is the stubs' only state (an
+ * `inOrder` rule counts its prior answers in it), so a cleared log is a stub
+ * that has never been called. Safe when no call was logged yet.
  */
-export declare function stubBinDir(stubs: readonly ToolStub[], parentDir: string): string;
+export declare function resetStubCalls(dir: StubDir): void;
+/** Remove a stub root. Safe on one already gone. */
+export declare function removeStubDir(dir: StubDir): void;
 //# sourceMappingURL=tool-stub.d.ts.map

package/dist/tool-stub.js

@@ -1,92 +1,81 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.renderToolStub = renderToolStub;
-exports.writeToolStubs = writeToolStubs;
-exports.stubBinDir = stubBinDir;
+exports.stubShim = stubShim;
+exports.writeStubDir = writeStubDir;
+exports.readStubCalls = readStubCalls;
+exports.resetStubCalls = resetStubCalls;
+exports.removeStubDir = removeStubDir;
 /**
- * vigiles — **tool stubs on PATH** (rung R2 of the eval coverage model).
+ * vigiles — tool stubs on PATH: the EFFECTS half (rung R2 of the eval coverage
+ * model). The rules, the decision and the log format are pure, in
+ * `core/stub-rules.ts`; this file writes them to disk and reads the log back.
  *
- * A skill/hook/agent often calls a CLI tool (`psql`, `redis-cli`, `gh`, `git`,
- * `z3`, …) and then works with the RESULT. To test that downstream logic without
- * a live service, you SHADOW the real binary on PATH with a fake that emits a
- * **recorded / author-provided canned result** — a standard PATH-shim +
- * VCR-style record/replay technique, not a novel invention. See
- * `research/eval-coverage-and-isolation.md` (the three-rung model).
+ * Layout of one stub root — a temp dir BESIDE the run's working dir, never in
+ * it, so the agent does not find vigiles plumbing in its project (models did
+ * `ls`/`cat` a stub dir that lived inside the work dir):
  *
- * This module is the REPLAY half: write a fake executable per tool that prints a
- * canned stdout/stderr and exits a canned code. A record-from-real-tool half (run
- * the real binary once at a known version, capture its output as the fixture) is
- * a documented follow-on — NOT implemented here.
+ *   <root>/bin/<name>          a two-line sh shim, first on the run's PATH
+ *   <root>/rules/<name>.json   the encoded rules (`encodeStub`)
+ *   <root>/calls.jsonl         one line per invocation, every stub of the run
  *
- * CRITICAL: the canned outputs are author/recorded fixtures, **never**
- * model-synthesized — a synthesized `gh`/`git` output looks plausible but
- * diverges from the real tool/version, producing false confidence (a green test
- * against a fiction).
+ * The shim execs `node <vigiles cli.js> hook-runtime stub <root> <name> "$@"`,
+ * a fast path in the dispatcher that loads only the pure decision — the same
+ * shape as the intercept hook, which also calls back into the CLI by absolute
+ * path because `npx vigiles` does not resolve from a throwaway dir.
  *
- * MVP scope: one canned result per binary; argv is IGNORED (every invocation of
- * the stub returns the same result). Argv-matching (different output per
- * sub-command / flags) is a follow-on.
+ * The canned answers are author/recorded fixtures, NEVER model-synthesized: a
+ * synthesized `gh` output looks plausible and diverges from the real tool.
  */
 const node_fs_1 = require("node:fs");
-const node_path_1 = require("node:path");
-/**
- * Encode a string as a single-line base64 literal safe to embed verbatim inside a
- * POSIX shell script.
- *
- * We base64-encode-and-decode the canned content (rather than interpolating it
- * raw, or `cat`ing a sibling data file) because base64's alphabet is a strict
- * subset of `[A-Za-z0-9+/=]` — it can never contain a quote, `$`, backtick,
- * newline, `;`, or any other shell metacharacter, so the embedded literal is
- * injection-proof regardless of the fixture's bytes. The script decodes it back
- * with `base64 -d` at run time, so the original bytes round-trip exactly. This is
- * simpler than a sibling data file (one self-contained script, nothing else to
- * write/clean up) and strictly safer than quoting.
- */
-function b64(s) {
-    return Buffer.from(s, "utf-8").toString("base64");
+const stub_rules_js_1 = require("./core/stub-rules.js");
+const tmp_root_js_1 = require("./core/tmp-root.js");
+/** A POSIX single-quoted word: the only quoting `sh` never re-reads. */
+const shQuote = (s) => `'${s.replaceAll("'", `'\\''`)}'`;
+/** The shim for one stub. Pure. */
+function stubShim(launcher, root, name) {
+    const words = [
+        launcher.node,
+        launcher.cli,
+        "hook-runtime",
+        "stub",
+        root,
+        name,
+    ];
+    return `#!/bin/sh\nexec ${words.map(shQuote).join(" ")} "$@"\n`;
 }
 /**
- * Render the POSIX shell script for one stub. `printf '%s'` (not `echo`) prints
- * the decoded bytes with no added trailing newline and no backslash/`-n`
- * surprises, so the fixture round-trips byte-for-byte.
+ * Write a fresh stub root for `stubs`: a shim and a rules file per stub. The
+ * stubs must already be parsed (the spec boundary did it). Remove it with
+ * {@link removeStubDir}.
  */
-function renderToolStub(stub) {
-    const lines = ["#!/bin/sh"];
-    // stdout: decode the base64 literal straight to stdout. We pipe `base64 -d`'s
-    // output directly (NOT through `printf '%s' "$(...)"`) because command
-    // substitution strips trailing newlines — piping preserves the bytes exactly.
-    if (stub.stdout !== undefined && stub.stdout !== "") {
-        lines.push(`printf '%s' '${b64(stub.stdout)}' | base64 -d`);
-    }
-    // stderr: same, redirected to fd 2.
-    if (stub.stderr !== undefined && stub.stderr !== "") {
-        lines.push(`printf '%s' '${b64(stub.stderr)}' | base64 -d >&2`);
-    }
-    lines.push(`exit ${String(stub.exitCode ?? 0)}`);
-    return lines.join("\n") + "\n";
+function writeStubDir(stubs, launcher) {
+    const root = (0, tmp_root_js_1.makeTmpDir)("stubs");
+    const { binDir, rulesDir } = (0, stub_rules_js_1.stubPaths)(root);
+    (0, node_fs_1.mkdirSync)(binDir, { recursive: true });
+    (0, node_fs_1.mkdirSync)(rulesDir, { recursive: true });
+    stubs.forEach((stub) => {
+        const paths = (0, stub_rules_js_1.stubPaths)(root, stub.name);
+        (0, node_fs_1.writeFileSync)(paths.rules, (0, stub_rules_js_1.encodeStub)(stub), "utf8");
+        (0, node_fs_1.writeFileSync)(paths.bin, stubShim(launcher, root, stub.name), "utf8");
+        (0, node_fs_1.chmodSync)(paths.bin, 0o755);
+    });
+    return { root, binDir };
 }
-/**
- * Write one executable POSIX shell stub per entry into `binDir` (which must
- * already exist). Each file is named exactly `stub.name` and `chmod 0o755` so it
- * is directly executable once `binDir` is on PATH. Pure-ish — fs only, no spawn.
- */
-function writeToolStubs(binDir, stubs) {
-    for (const stub of stubs) {
-        const file = (0, node_path_1.join)(binDir, stub.name);
-        (0, node_fs_1.writeFileSync)(file, renderToolStub(stub), "utf-8");
-        (0, node_fs_1.chmodSync)(file, 0o755);
-    }
+/** Every invocation the run's stubs recorded, in order. Empty when none was called. */
+function readStubCalls(dir) {
+    const { log } = (0, stub_rules_js_1.stubPaths)(dir.root);
+    return (0, node_fs_1.existsSync)(log) ? (0, stub_rules_js_1.parseStubLog)((0, node_fs_1.readFileSync)(log, "utf8")) : [];
 }
 /**
- * Convenience: mkdtemp a fresh bin dir under `parentDir`, write `stubs` into it,
- * and return its absolute path. Caller PREPENDS this dir to PATH so the fakes win
- * over the real binaries, and removes it when done (it lives under `parentDir`,
- * so a `parentDir` cleanup also clears it).
+ * Forget every call recorded so far. The log is the stubs' only state (an
+ * `inOrder` rule counts its prior answers in it), so a cleared log is a stub
+ * that has never been called. Safe when no call was logged yet.
  */
-function stubBinDir(stubs, parentDir) {
-    (0, node_fs_1.mkdirSync)(parentDir, { recursive: true });
-    const binDir = (0, node_fs_1.mkdtempSync)((0, node_path_1.join)(parentDir, "vigiles-stub-bin-"));
-    writeToolStubs(binDir, stubs);
-    return binDir;
+function resetStubCalls(dir) {
+    (0, node_fs_1.rmSync)((0, stub_rules_js_1.stubPaths)(dir.root).log, { force: true });
+}
+/** Remove a stub root. Safe on one already gone. */
+function removeStubDir(dir) {
+    (0, node_fs_1.rmSync)(dir.root, { recursive: true, force: true });
 }
 //# sourceMappingURL=tool-stub.js.map

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "vigiles",
-  "version": "35.0.0",
+  "version": "37.0.0",
   "description": "Audit, test and measure the harness your AI agent runs on — grade your CLAUDE.md / AGENTS.md, skills, subagents and hooks, run them against a scripted model, and measure whether they actually fire.",
   "keywords": [
     "claude-code",

package/skills/test-harness/references/observing-a-run.md

@@ -10,18 +10,19 @@ questions are keyed on the **observation** instead — "what did this skill
 actually do?" — and they have answers already. Reach for these before building
 anything; every one of them ships today.
 
-| The question you're actually asking                                     | Use                                                                                    |
-| ----------------------------------------------------------------------- | -------------------------------------------------------------------------------------- |
-| Which tools did it call, and with what arguments?                       | `trace.toolCalls` · `tool` / `toolWith` checks · `parseToolCalls` (`vigiles`)          |
-| Did it call a tool it must not?                                         | `notTool(name)`                                                                        |
-| Did it call **only** tools from a known set?                            | `onlyTools([...])` — the white-list, symmetric to `assertWroteOnly`                    |
-| Did it stay inside the `allowed-tools` its own frontmatter declares?    | `skillContract(dir).surface` — builds that check FROM the declaration                  |
-| What files did the run write?                                           | `filesWritten` · `wrote(path)` / `didNotWrite(path)` · `r.file(path)`                  |
-| Did it write **only** where it was supposed to?                         | `assertWroteOnly([...])` / `assertNoWrite()` — needs `{ sandbox: "auto" }`             |
-| Run a tool call but **don't let it execute** — capture the args instead | the `interceptTools` option on `measure` / `runEval` (a `ToolIntercept[]`)             |
-| Did a subagent do it, and which one?                                    | `subagent(name, [...])` · `SubagentTrace`                                              |
-| Was it an MCP tool?                                                     | `mcp(server, toolName)`                                                                |
-| Assert the whole effect boundary deterministically                      | `assertChecks` + the checks above (see `examples/harness/effect-boundary.harness.mjs`) |
+| The question you're actually asking                                     | Use                                                                                                                                                               |
+| ----------------------------------------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------- |
+| Which tools did it call, and with what arguments?                       | `trace.toolCalls` · `tool` / `toolWith` checks · `parseToolCalls` (`vigiles`)                                                                                     |
+| Did it call a tool it must not?                                         | `notTool(name)`                                                                                                                                                   |
+| Did it call **only** tools from a known set?                            | `onlyTools([...])` — the white-list, symmetric to `assertWroteOnly`                                                                                               |
+| Did it stay inside the `allowed-tools` its own frontmatter declares?    | `skillContract(dir).surface` — builds that check FROM the declaration                                                                                             |
+| What files did the run write?                                           | `filesWritten` · `wrote(path)` / `didNotWrite(path)` · `r.file(path)`                                                                                             |
+| Did it write **only** where it was supposed to?                         | `assertWroteOnly([...])` / `assertNoWrite()` — needs `{ sandbox: "auto" }`                                                                                        |
+| Run a tool call but **don't let it execute** — capture the args instead | the `interceptTools` option on `measure` / `runEval` (a `ToolIntercept[]`)                                                                                        |
+| Let a CLI the agent (or its script) calls **answer** — `gh`, `psql`     | `stubs: [experimental_stub(name, rules)]` on `runEval`; read `ctx.stubCalls`, check with `experimental_stub.called(...)` — an argv no rule answers fails the eval |
+| Did a subagent do it, and which one?                                    | `subagent(name, [...])` · `SubagentTrace`                                                                                                                         |
+| Was it an MCP tool?                                                     | `mcp(server, toolName)`                                                                                                                                           |
+| Assert the whole effect boundary deterministically                      | `assertChecks` + the checks above (see `examples/harness/effect-boundary.harness.mjs`)                                                                            |
 
 `interceptTools` is the one worth knowing about, because it is not obvious it
 exists: it denies a tool its **real execution** via an auto-wired `PreToolUse`

package/skills/test-harness/references/writing-tests.md

@@ -90,6 +90,7 @@ import { defineEval, assertSignificant } from "vigiles";
 
 export default defineEval({
   runEval: {
+    env: { kind: "ephemeral" }, // required: a throwaway HOME, or { kind: "inherit", reason }
     arms: { off: {}, on: { pluginDir: "./" } },
     task: "…a task the harness change should affect…",
     measure: (ctx) => ({ ok: /* a bare predicate over the trace */ true }),