vigiles 35.0.0 → 37.0.0

package/dist/harness-test.d.ts

@@ -1,5 +1,5 @@
 import type { HarnessAdapter } from "./core/adapter.js";
-import type { HarnessTestDriver, ToolCall, HookFire, ParsedRun, ModelTurn } from "./core/harness-driver.js";
+import type { HarnessTestDriver, ToolCall, HookFire, ParsedRun, ModelTurn, ModelRequest } from "./core/harness-driver.js";
 import type { Trace, SubagentTrace } from "./core/eval-driver.js";
 export type { Trace, SubagentTrace } from "./core/eval-driver.js";
 import { parseReplies, reachedModel } from "./adapters/claude-code/replies.js";
@@ -53,16 +53,36 @@ export interface HarnessTestSpec {
     /** The user prompt. Default: "go". */
     readonly prompt?: string;
     /**
-     * The ONLY tools the agent has, when you list them: a tool left out is not
-     * offered (`claude --tools`, Claude Code 2.0.31+), `[]` is refused, and a
-     * scripted call to a tool the list leaves out throws before the run starts.
-     * Omitted, nothing is withheld: every tool is offered and Read, Edit, Write and
-     * Bash are pre-approved. Each listed tool is also pre-approved
-     * (`--allowedTools`), so it does not stop on a permission prompt; a permission
-     * rule such as `Bash(git *)` keeps its specifier for the approval and is
-     * offered as plain `Bash`. An MCP tool is not a built-in: left out, it is still
-     * offered but not approved, so a call to it is refused for permission rather
-     * than as "No such tool".
+     * Which tools EXIST in the session (`claude --tools`, Claude Code 2.0.31+). A
+     * tool left out is not offered at all, so a call to it is "No such tool
+     * available". Omitted, every tool exists. This is the field that withholds a
+     * tool. Names only: a permission rule such as `Bash(git *)` is refused (put the
+     * rule in `allowedTools`). `[]` is refused (an agent with no tools is never
+     * served a scripted turn), and a scripted call to a tool the list leaves out
+     * throws before the run starts. An MCP tool is not a built-in, so `--tools`
+     * does not withhold it. Codex has no equivalent, so a run there refuses it.
+     *
+     * | field          | Claude Code flag  | meaning                      |
+     * | -------------- | ----------------- | ---------------------------- |
+     * | `tools`        | `--tools`         | which tools exist            |
+     * | `allowedTools` | `--allowedTools`  | which are pre-approved       |
+     */
+    readonly tools?: readonly string[];
+    /**
+     * Which tools are PRE-APPROVED (`claude --allowedTools`), so they do not stop
+     * on a permission prompt. Takes permission rules (`Bash(git *)`). Omitted:
+     * Read, Edit, Write and Bash are approved. An explicit list REPLACES that
+     * default, so `tools: ["Read", "Write"]` with `allowedTools: ["Skill"]` leaves
+     * Write unapproved. `[]`: nothing is approved.
+     *
+     * It never removes a tool, and leaving a tool out is not a fence: in headless
+     * mode Claude Code refuses only the calls it would have asked about. For Bash
+     * that is a command that creates, changes or removes a file (`touch`, `>`),
+     * inside or outside the working directory; a command with no file effect
+     * (`ls`, `printf`, `id`) runs unapproved. So a refused `touch` proves the
+     * file-change guard is live, not that Bash is withheld. To withhold a tool use
+     * `tools`; to forbid its use put a deny rule in the fixture's
+     * `settings.permissions.deny`.
      */
     readonly allowedTools?: readonly string[];
     /**
@@ -131,20 +151,34 @@ export declare function parseResultEvent(stdout: string): Record<string, unknown
 export declare function parseOutput(stdout: string): string;
 export declare function parseHooks(stdout: string): HookFire[];
 /**
- * The names `--tools` should offer for an `allowedTools` list. `--allowedTools`
- * takes permission rules (`Bash(git *)`); `--tools` takes tool NAMES, and a rule
- * with a specifier there makes the CLI offer no tools at all. So the specifier is
- * dropped (`Bash(git *)` → `Bash`) and repeats collapse. Pure.
+ * The names `--tools` takes for a `tools` list: trimmed and deduplicated.
+ * `--allowedTools` takes permission rules (`Bash(git *)`); `--tools` takes tool
+ * NAMES, and a rule there makes the CLI offer no tools at all. Reducing it to
+ * `Bash` would offer more than the author wrote, so a rule is refused: it
+ * belongs in `allowedTools`. Pure.
  */
-export declare function toolAvailabilityList(allowed: readonly string[]): readonly string[];
+export declare function toolAvailabilityList(tools: readonly string[]): readonly string[];
 /**
- * A scripted call to a tool an EXPLICIT `allowedTools` does not offer. The CLI
- * answers it with "No such tool available" and the run goes on, so a test that
- * scripts the call and then asserts on something else passes over a step that
- * never happened. Returns the message to throw, or undefined. MCP tools are not
- * built-ins, so `--tools` does not withhold them and they are skipped. Pure.
+ * A scripted call to a tool an EXPLICIT `tools` list leaves out. The CLI answers
+ * it with "No such tool available" and the run goes on, so a test that scripts
+ * the call and then asserts on something else passes over a step that never
+ * happened. A tool that exists but is not in `allowedTools` is NOT this: the CLI
+ * refuses it for permission, which is a result a test can assert on. Returns the
+ * message to throw, or undefined. MCP tools are not built-ins, so `--tools` does
+ * not withhold them and they are skipped. Pure.
  */
-export declare function unofferedScriptedTool(model: readonly ModelTurn[], allowed: readonly string[] | undefined): string | undefined;
+export declare function unofferedScriptedTool(model: readonly ModelTurn[], tools: readonly string[] | undefined): string | undefined;
+/**
+ * The agent's own tool results, read off the requests it sent back to the model,
+ * that say a scripted tool does not exist. The CLI answers an unknown name
+ * (`Bsh` for `Bash`, an `mcp__` server that is not configured) with an error
+ * RESULT and carries on, exit 0, so a test that asserts "the call errored and
+ * left no side effect" passes on a typo. `tools` cannot catch it: the pre-flight
+ * skips `mcp__` names and only runs when `tools` is given. Side-channel requests
+ * are the CLI's own bookkeeping and are not read. Returns the message to throw,
+ * or undefined. Pure.
+ */
+export declare function noSuchToolMessage(model: readonly ModelTurn[], requests: readonly ModelRequest[]): string | undefined;
 /**
  * A run on a `claude` that predates `--tools` exits at argument parsing, and
  * commander's "unknown option" says nothing about WHY vigiles passed it. Returns
@@ -170,7 +204,7 @@ export declare function parseClaudeRun(stdout: string): ParsedRun;
  *   else from the caller: no harness auth (the scripted mock needs none, and a
  *   config-dir variable would point the run back at the real config), no
  *   identity, no secrets. The same scrubbed shape as the eval tier's
- *   `ephemeralEnv` (`scrubbedRunEnv`), without the auth.
+ *   `env: { kind: "ephemeral" }` (`scrubbedRunEnv`), without the auth.
  */
 export type HarnessRunHome = {
     readonly home: "inherit";
@@ -204,6 +238,10 @@ export interface RunHarnessTestOptions {
 }
 /** Say so on stderr when side-channel calls arrived and no script turn was served. */
 export declare function warnUnconsumed(count: number, sideChannelCount: number): void;
+/**
+ * What makes a finished run unsound, most specific first, or undefined. Pure.
+ */
+export declare function runProblem(stderr: string, model: readonly ModelTurn[], modelRequests: readonly ModelRequest[], tools?: readonly string[]): string | undefined;
 /**
  * Run the real agent CLI against a scripted mock model, with the given fixture
  * and settings (hooks). Deterministic — same script, same result. Adapter-driven

package/dist/harness-test.js

@@ -8,12 +8,14 @@ exports.parseOutput = parseOutput;
 exports.parseHooks = parseHooks;
 exports.toolAvailabilityList = toolAvailabilityList;
 exports.unofferedScriptedTool = unofferedScriptedTool;
+exports.noSuchToolMessage = noSuchToolMessage;
 exports.unsupportedToolsFlag = unsupportedToolsFlag;
 exports.buildClaudeArgs = buildClaudeArgs;
 exports.parseClaudeRun = parseClaudeRun;
 exports.harnessSpawnEnv = harnessSpawnEnv;
 exports.claudeAvailable = claudeAvailable;
 exports.warnUnconsumed = warnUnconsumed;
+exports.runProblem = runProblem;
 exports.runHarnessTest = runHarnessTest;
 exports.runHarnessTestIn = runHarnessTestIn;
 exports.runHarness = runHarness;
@@ -287,32 +289,66 @@ function parseHooks(stdout) {
  */
 const TOOLS_FLAG_MIN_CLAUDE = "2.0.31";
 /**
- * The names `--tools` should offer for an `allowedTools` list. `--allowedTools`
- * takes permission rules (`Bash(git *)`); `--tools` takes tool NAMES, and a rule
- * with a specifier there makes the CLI offer no tools at all. So the specifier is
- * dropped (`Bash(git *)` → `Bash`) and repeats collapse. Pure.
+ * The names `--tools` takes for a `tools` list: trimmed and deduplicated.
+ * `--allowedTools` takes permission rules (`Bash(git *)`); `--tools` takes tool
+ * NAMES, and a rule there makes the CLI offer no tools at all. Reducing it to
+ * `Bash` would offer more than the author wrote, so a rule is refused: it
+ * belongs in `allowedTools`. Pure.
  */
-function toolAvailabilityList(allowed) {
-    const names = allowed.map((rule) => rule.replace(/\(.*$/, "").trim());
-    return [...new Set(names.filter((n) => n !== ""))];
+function toolAvailabilityList(tools) {
+    const names = tools.map((entry) => entry.trim()).filter((n) => n !== "");
+    const rule = names.find((name) => name.includes("("));
+    if (rule !== undefined) {
+        throw new Error(`tools takes bare tool names, and "${rule}" is a permission rule — put the rule in allowedTools and the name ("${rule.replace(/\(.*$/, "")}") in tools`);
+    }
+    return [...new Set(names)];
 }
 /**
- * A scripted call to a tool an EXPLICIT `allowedTools` does not offer. The CLI
- * answers it with "No such tool available" and the run goes on, so a test that
- * scripts the call and then asserts on something else passes over a step that
- * never happened. Returns the message to throw, or undefined. MCP tools are not
- * built-ins, so `--tools` does not withhold them and they are skipped. Pure.
+ * A scripted call to a tool an EXPLICIT `tools` list leaves out. The CLI answers
+ * it with "No such tool available" and the run goes on, so a test that scripts
+ * the call and then asserts on something else passes over a step that never
+ * happened. A tool that exists but is not in `allowedTools` is NOT this: the CLI
+ * refuses it for permission, which is a result a test can assert on. Returns the
+ * message to throw, or undefined. MCP tools are not built-ins, so `--tools` does
+ * not withhold them and they are skipped. Pure.
  */
-function unofferedScriptedTool(model, allowed) {
-    if (allowed === undefined)
+function unofferedScriptedTool(model, tools) {
+    if (tools === undefined)
         return undefined;
-    const offered = toolAvailabilityList(allowed);
+    const exist = toolAvailabilityList(tools);
     const at = model.findIndex((t) => t.tool !== undefined &&
         !t.tool.startsWith("mcp__") &&
-        !offered.includes(t.tool));
+        !exist.includes(t.tool));
     return at < 0
         ? undefined
-        : `scripted call to "${model[at]?.tool ?? ""}" on turn ${String(at + 1)}, but this run offers only: ${offered.join(", ")} — add it to allowedTools`;
+        : `scripted call to "${model[at]?.tool ?? ""}" on turn ${String(at + 1)}, but this run's tools are only: ${exist.join(", ")} — add it to tools`;
+}
+/**
+ * The agent's own tool results, read off the requests it sent back to the model,
+ * that say a scripted tool does not exist. The CLI answers an unknown name
+ * (`Bsh` for `Bash`, an `mcp__` server that is not configured) with an error
+ * RESULT and carries on, exit 0, so a test that asserts "the call errored and
+ * left no side effect" passes on a typo. `tools` cannot catch it: the pre-flight
+ * skips `mcp__` names and only runs when `tools` is given. Side-channel requests
+ * are the CLI's own bookkeeping and are not read. Returns the message to throw,
+ * or undefined. Pure.
+ */
+function noSuchToolMessage(model, requests) {
+    // The CLI's own wording, inside its error wrapper, for a name the script
+    // calls: the agent calls tools only from the script, so the same words in the
+    // prompt or in a hook's injected text are not a result of this run.
+    const pattern = /<tool_use_error>Error: No such tool available: ([^\s<]+)<\/tool_use_error>/g;
+    const scripted = new Set(model.flatMap((t) => (t.tool === undefined ? [] : [t.tool])));
+    const hit = requests
+        .filter((r) => r.sideChannel !== true)
+        .flatMap((r) => r.messages)
+        .flatMap((m) => [...m.text.matchAll(pattern)].map((x) => x[1]))
+        .find((name) => name !== undefined && scripted.has(name));
+    if (hit === undefined)
+        return undefined;
+    const at = model.findIndex((t) => t.tool === hit);
+    const turn = at < 0 ? "" : ` on turn ${String(at + 1)}`;
+    return `scripted call to "${hit}"${turn}: the CLI answered "No such tool available: ${hit}", so the call never ran and the test would carry on over a step that did not happen. Check the name for a typo; an mcp__ tool also needs its server configured in the fixture.`;
 }
 /**
  * A run on a `claude` that predates `--tools` exits at argument parsing, and
@@ -321,7 +357,7 @@ function unofferedScriptedTool(model, allowed) {
  */
 function unsupportedToolsFlag(stderr) {
     return /unknown option '--tools'/.test(stderr)
-        ? `this \`claude\` does not know \`--tools\`, which runHarnessTest uses to withhold every tool not in \`allowedTools\` (#252). Update Claude Code to ${TOOLS_FLAG_MIN_CLAUDE} or newer. Running without it would hand the agent every tool while the test reads as if it were fenced.`
+        ? `this \`claude\` does not know \`--tools\`, which runHarnessTest passes for the \`tools\` field (which tools exist). Update Claude Code to ${TOOLS_FLAG_MIN_CLAUDE} or newer. Running without it would hand the agent every tool while the test reads as if it were restricted.`
         : undefined;
 }
 /**
@@ -330,7 +366,7 @@ function unsupportedToolsFlag(stderr) {
  * here. Pure, so the arg shape is unit-tested.
  */
 function buildClaudeArgs(spec, hasSettings) {
-    const tools = spec.allowedTools ?? ["Read", "Edit", "Write", "Bash"];
+    const approved = spec.allowedTools ?? ["Read", "Edit", "Write", "Bash"];
     return [
         "-p",
         spec.prompt ?? "go",
@@ -343,18 +379,15 @@ function buildClaudeArgs(spec, hasSettings) {
             ? ["--plugin-dir", (0, node_path_1.resolve)(spec.pluginDir)]
             : []),
         ...(hasSettings ? ["--settings", "settings.json"] : []),
-        // An EXPLICIT list restricts: `--tools` is the AVAILABILITY list, a tool left
-        // out is not offered at all. `--allowedTools` only PRE-APPROVES (it never
-        // restricts), so on its own it left every unnamed tool runnable (#252). An
-        // empty list is `--tools ""` = no tools. With no list, nothing is withheld:
-        // the default four are pre-approved and every other tool stays available.
-        ...(spec.allowedTools === undefined
+        // `tools` = which tools EXIST (`--tools`, the availability list). Omitted:
+        // nothing is withheld.
+        ...(spec.tools === undefined
             ? []
-            : ["--tools", toolAvailabilityList(spec.allowedTools).join(",")]),
-        // Pre-approval for the offered tools, so none stops on a permission prompt
-        // (headless). An empty one approves nothing, so the flag is left out (a bare
+            : ["--tools", toolAvailabilityList(spec.tools).join(",")]),
+        // `allowedTools` = which are PRE-APPROVED (`--allowedTools`); it never
+        // restricts. An empty list approves nothing, so the flag is left out (a bare
         // `--allowedTools` exits before any model turn).
-        ...(tools.length === 0 ? [] : ["--allowedTools", ...tools]),
+        ...(approved.length === 0 ? [] : ["--allowedTools", ...approved]),
     ];
 }
 /** Build the `claude` argv from the driver context — wraps `buildClaudeArgs`. */
@@ -364,7 +397,8 @@ function buildClaudeArgsFromCtx(ctx) {
         prompt: ctx.prompt,
         transcript: ctx.transcript,
         pluginDir: ctx.pluginDir,
-        allowedTools: ctx.tools,
+        tools: ctx.tools,
+        allowedTools: ctx.allowedTools,
     }, ctx.hasSettings);
 }
 /** Parse the `claude` stdout/stream into the unified trace fields. */
@@ -480,15 +514,41 @@ function warnUnconsumed(count, sideChannelCount) {
     if (unconsumed !== undefined)
         console.error(unconsumed);
 }
+/**
+ * `tools` made only of names the CLI does not know leaves the agent with none:
+ * every request then declares no tools, the mock files it as side-channel
+ * traffic, and the script is never served, so the run decides nothing. Requests
+ * that all came in that way, under an explicit `tools`, mean exactly that. No
+ * request at all (a hook stopped the prompt) is a different run. Pure.
+ */
+function toolsResolvedToNothing(tools, requests) {
+    if (tools === undefined || requests.length === 0)
+        return undefined;
+    if (!requests.every((r) => r.sideChannel === true))
+        return undefined;
+    return `tools ${JSON.stringify(tools)} gave the agent no tools: no name in it is a tool this CLI knows, so no request offered a tool and the script was never served. Check the names for typos.`;
+}
+/**
+ * What makes a finished run unsound, most specific first, or undefined. Pure.
+ */
+function runProblem(stderr, model, modelRequests, tools) {
+    return (unsupportedToolsFlag(stderr) ??
+        toolsResolvedToNothing(tools, modelRequests) ??
+        // A typo'd last turn makes the CLI ask once more, so the overrun follows
+        // from it: report the cause, not its effect.
+        noSuchToolMessage(model, modelRequests) ??
+        (0, script_overrun_js_1.overrunMessageFor)(model.length, modelRequests));
+}
 /**
  * A run that proves nothing about the script fails the test instead of coming
- * back as a result: the `claude` on PATH predates `--tools` (#252), or the agent
+ * back as a result: the `claude` on PATH predates `--tools` (needed for `tools`), or the agent
  * asked for a model turn the script did not have and the mock answered with an
- * error rather than an invented turn (#340). Removes the run's directory, since
+ * error rather than an invented turn (#340), or a scripted call named a tool the
+ * CLI does not have. Removes the run's directory, since
  * no result is handed back to clean it up.
  */
-function assertRunSound(cwd, stderr, scripted, modelRequests) {
-    const problem = unsupportedToolsFlag(stderr) ?? (0, script_overrun_js_1.overrunMessageFor)(scripted, modelRequests);
+function assertRunSound(cwd, stderr, model, modelRequests, tools) {
+    const problem = runProblem(stderr, model, modelRequests, tools);
     if (problem === undefined)
         return;
     (0, node_fs_1.rmSync)(cwd, { recursive: true, force: true });
@@ -539,16 +599,30 @@ async function runHarnessTest(spec, opts = {}) {
     refuseUnrunnableSpec(spec);
     return runHarnessTestIn(spec, opts, "inherit");
 }
+/** The driver's view of one run of `spec`. Pure. */
+function driverContext(spec, cwd, hasSettings, mockArgs) {
+    return {
+        prompt: spec.prompt ?? "go",
+        cwd,
+        hasSettings,
+        tools: spec.tools,
+        allowedTools: spec.allowedTools,
+        transcript: spec.transcript ?? false,
+        pluginDir: spec.pluginDir,
+        mockArgs,
+    };
+}
 /**
  * The scripted model tells an agent turn from the CLI's own bookkeeping calls by
  * the tools the request declares (`isMainLoopRequest`). An agent with none is
  * never served a script turn, and the run would decide on nothing.
  */
 function refuseUnrunnableSpec(spec) {
-    if (spec.allowedTools?.length === 0) {
-        throw new Error("allowedTools: [] leaves the agent with no tools, and the scripted model only serves a turn to a request that declares tools — no script turn would be consumed. Name the tools the agent has.");
+    if (spec.tools !== undefined &&
+        toolAvailabilityList(spec.tools).length === 0) {
+        throw new Error("tools: [] leaves the agent with no tools (a list of blank names counts as empty), and the scripted model only serves a turn to a request that declares tools — no script turn would be consumed. Name the tools the agent has.");
     }
-    const unoffered = unofferedScriptedTool(spec.model, spec.allowedTools);
+    const unoffered = unofferedScriptedTool(spec.model, spec.tools);
     if (unoffered !== undefined)
         throw new Error(unoffered);
 }
@@ -585,15 +659,18 @@ async function runHarnessTestIn(spec, opts, home) {
     const cwd = (0, tmp_root_js_1.makeTmpDir)("harness");
     writeFixture(cwd, files, settings);
     const timeoutMs = spec.timeoutMs ?? 60000;
-    const buildArgs = (mockArgs) => driver.buildArgs({
-        prompt: spec.prompt ?? "go",
-        cwd,
-        hasSettings: settings !== undefined,
-        tools: spec.allowedTools,
-        transcript: spec.transcript ?? false,
-        pluginDir: spec.pluginDir,
-        mockArgs,
-    });
+    // A driver refuses a spec it cannot run (Codex and `tools`) inside buildArgs.
+    // That refusal must not leave the run's temp dir behind; the mock, if started,
+    // is closed by the caller's finally.
+    const buildArgs = (mockArgs) => {
+        try {
+            return driver.buildArgs(driverContext(spec, cwd, settings !== undefined, mockArgs));
+        }
+        catch (error) {
+            (0, node_fs_1.rmSync)(cwd, { recursive: true, force: true });
+            throw error;
+        }
+    };
     // Confined path (Claude Code only): the mock is co-launched in the netns, so
     // the agent reaches it over the loopback URL the sandbox sets — Claude Code is
     // env-only (empty mockArgs), so the argv needs no mock-wiring flags.
@@ -612,7 +689,7 @@ async function runHarnessTestIn(spec, opts, home) {
         // warning, which the sandbox path could not emit at all before.
         const { count, sideChannelCount } = (0, mock_model_js_1.splitRequestCounts)(out.requests);
         warnUnconsumed(count, sideChannelCount);
-        assertRunSound(cwd, "", spec.model.length, out.requests);
+        assertRunSound(cwd, "", spec.model, out.requests, spec.tools);
         return check(makeResult(cwd, out, parseClaudeRun(out.stdout), count, out.requests));
     }
     // Direct path: mock runs in this process; the agent reaches it over localhost.
@@ -631,7 +708,7 @@ async function runHarnessTestIn(spec, opts, home) {
         // it (every request arriving without tool declarations, so nothing looked
         // like an agent turn) instead of leaving it to be rediscovered.
         warnUnconsumed(mock.count, mock.sideChannelCount ?? 0);
-        assertRunSound(cwd, out.stderr, spec.model.length, mock.requests);
+        assertRunSound(cwd, out.stderr, spec.model, mock.requests, spec.tools);
         return check(makeResult(cwd, out, driver.parseRun(out.stdout), mock.count, [
             ...mock.requests,
         ]));

package/dist/services.d.ts

@@ -29,7 +29,7 @@
  *     a dev box with NO production access;
  *   - point the task at the disposable service's connection string ONLY;
  *   - keep real credentials OUT of the run (prod `DATABASE_URL`, cloud keys,
- *     `~/.ssh`) — pair it with the eval tier's `ephemeralEnv` (throwaway HOME +
+ *     `~/.ssh`) — pair it with the eval tier's `env: { kind: "ephemeral" }` (throwaway HOME +
  *     cleared env) to scrub them so the model has no real keys to misuse.
  * Treat it like running an untrusted script. A future increment adds an egress
  * wall (the skill reaches only the model + the service); until then that job is

package/dist/services.js

@@ -30,7 +30,7 @@
  *     a dev box with NO production access;
  *   - point the task at the disposable service's connection string ONLY;
  *   - keep real credentials OUT of the run (prod `DATABASE_URL`, cloud keys,
- *     `~/.ssh`) — pair it with the eval tier's `ephemeralEnv` (throwaway HOME +
+ *     `~/.ssh`) — pair it with the eval tier's `env: { kind: "ephemeral" }` (throwaway HOME +
  *     cleared env) to scrub them so the model has no real keys to misuse.
  * Treat it like running an untrusted script. A future increment adds an egress
  * wall (the skill reaches only the model + the service); until then that job is

package/dist/stub.d.ts

@@ -0,0 +1,57 @@
+/**
+ * `experimental_stub` — a tool stub that answers PER INVOCATION, for the eval
+ * tier's `stubs`. The rules and their matching are in `core/stub-rules.ts`; this
+ * is the typed helper and the check over a run's recorded calls.
+ *
+ * ONE experimental root (STABILITY.md "one root per feature"): `rest` and
+ * `called` are members, so no stable name rests on an unstable one. The rules
+ * themselves are plain data (`ToolStub`): a `.mjs` literal of the same shape is
+ * accepted, because the spec boundary parses `stubs` either way.
+ */
+import type { Check } from "./check.js";
+import type { Trace } from "./harness-test.js";
+import { type ArgvPattern, type ArgvToken, type StubRule, type ToolStub } from "./core/stub-rules.js";
+/** Bounds for {@link experimental_stub.called}. */
+export interface StubCalledBounds {
+    /** At least this many matching calls. Default 1, or 0 when `max` is given. */
+    readonly min?: number;
+    /** At most this many. Default: no limit. */
+    readonly max?: number;
+    /** Order-free tokens after the positional prefix (needs a trailing `rest`). */
+    readonly contains?: readonly ArgvToken[];
+}
+/**
+ * A check over a run's stub log: the ANSWERED invocations of `name` that match
+ * `argv` (and `bounds.contains`) number within `[min, max]`. A call no rule
+ * answered is not counted (it is reported as unanswered, and `vigiles eval`
+ * fails the run on it). A trace with no stub log
+ * FAILS — "called at most 0 times" must not pass because nothing was recorded.
+ */
+declare function called(name: string, argv: ArgvPattern, bounds?: StubCalledBounds): Check<Trace>;
+/** Parse one stub: a throw names the bad field; the value is plain data. */
+declare function stubOf(name: string, rules: readonly [StubRule, ...StubRule[]]): ToolStub;
+/**
+ * A stub for the binary `name`: a non-empty list of rules over its argv, first
+ * match wins. An invocation no rule answers fails the eval — name each command
+ * the agent runs; do not answer everything with one output.
+ *
+ * ```ts
+ * const { rest } = experimental_stub;
+ * const gh = experimental_stub("gh", [
+ *   { argv: ["issue", "create", rest], reply: { kind: "always", stdout: "https://github.com/o/r/issues/9001\n" } },
+ *   { argv: ["api", /^repos\/o\/r\/issues\?/], reply: { kind: "always", stdout: "[]" } },
+ * ]);
+ * ```
+ *
+ * `experimental_stub.rest` — "any further tokens", last position only.
+ * `experimental_stub.called(name, argv, { min, max, contains })` — a check over
+ * the run's recorded calls.
+ *
+ * @experimental
+ */
+export declare const experimental_stub: typeof stubOf & {
+    rest: import("./core/stub-rules.js").ArgvRest;
+    called: typeof called;
+};
+export {};
+//# sourceMappingURL=stub.d.ts.map

package/dist/stub.js

@@ -0,0 +1,103 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.experimental_stub = void 0;
+const eval_spec_parse_js_1 = require("./core/eval-spec-parse.js");
+const stub_rules_js_1 = require("./core/stub-rules.js");
+const result = (pass, message) => ({
+    pass,
+    score: pass ? 1 : 0,
+    message,
+});
+/** The failure for a trace that carries no stub log at all. */
+const noLog = (shown) => result(false, `stub ${shown}: this trace has no stub log — the run declared no stubs, or its tier does not record them`);
+/**
+ * How many of a run's stub calls are `name` with an argv matching `argv` (+
+ * `contains`), split by whether a rule answered them. Only an ANSWERED call
+ * counts toward `called`: "the tool was called" in a measurement means the
+ * agent got its answer, so a create that came back as the miss line did not
+ * file anything. The run fails on the unanswered call separately.
+ */
+function countCalls(calls, name, argv, contains) {
+    const matching = calls.filter((c) => c.tool === name && (0, stub_rules_js_1.matchArgv)({ argv, contains }, c.argv));
+    const answered = matching.filter((c) => !(0, stub_rules_js_1.isUnanswered)(c.outcome)).length;
+    return { answered, unanswered: matching.length - answered };
+}
+/** The [min, max] a `called` check accepts, and how a report prints it. */
+function rangeOf(bounds) {
+    const min = bounds.min ?? (bounds.max === undefined ? 1 : 0);
+    const max = bounds.max ?? Number.POSITIVE_INFINITY;
+    const text = Number.isFinite(max)
+        ? `${String(min)}..${String(max)}`
+        : `at least ${String(min)}`;
+    return { min, max, text };
+}
+/** The verdict of a `called` check over a run's recorded calls. */
+function calledVerdict(calls, shown, expect) {
+    const { name, argv, ...bounds } = expect;
+    const { min, max, text } = rangeOf(bounds);
+    const { answered: n, unanswered } = countCalls(calls, name, argv, bounds.contains);
+    const missed = unanswered === 0
+        ? ""
+        : `; ${String(unanswered)} matching call(s) went unanswered and are not counted`;
+    const msg = `stub ${shown} called ${String(n)} time(s) (expected ${text})${missed}`;
+    return result(n >= min && n <= max, msg);
+}
+/**
+ * A check over a run's stub log: the ANSWERED invocations of `name` that match
+ * `argv` (and `bounds.contains`) number within `[min, max]`. A call no rule
+ * answered is not counted (it is reported as unanswered, and `vigiles eval`
+ * fails the run on it). A trace with no stub log
+ * FAILS — "called at most 0 times" must not pass because nothing was recorded.
+ */
+function called(name, argv, bounds = {}) {
+    // The pattern is parsed the way a rule is (rest position, RegExp flags).
+    const rule = { argv, contains: bounds.contains, reply: { kind: "always" } };
+    (0, eval_spec_parse_js_1.parseToolStubs)([{ name, rules: [rule] }], "experimental_stub.called");
+    const { min, max } = rangeOf(bounds);
+    const shown = `${name} ${(0, stub_rules_js_1.describePattern)(argv)}`;
+    return {
+        kind: "stubCalled",
+        eval(t) {
+            if (t.stubCalls === undefined)
+                return noLog(shown);
+            return calledVerdict(t.stubCalls, shown, { name, argv, ...bounds });
+        },
+        toJSON: () => ({
+            kind: "stubCalled",
+            tool: name,
+            argv: (0, stub_rules_js_1.describePattern)(argv),
+            min,
+            max: Number.isFinite(max) ? max : null,
+        }),
+    };
+}
+/** Parse one stub: a throw names the bad field; the value is plain data. */
+function stubOf(name, rules) {
+    (0, eval_spec_parse_js_1.parseToolStubs)([{ name, rules }], "experimental_stub");
+    return (0, stub_rules_js_1.toToolStub)({ name, rules });
+}
+/**
+ * A stub for the binary `name`: a non-empty list of rules over its argv, first
+ * match wins. An invocation no rule answers fails the eval — name each command
+ * the agent runs; do not answer everything with one output.
+ *
+ * ```ts
+ * const { rest } = experimental_stub;
+ * const gh = experimental_stub("gh", [
+ *   { argv: ["issue", "create", rest], reply: { kind: "always", stdout: "https://github.com/o/r/issues/9001\n" } },
+ *   { argv: ["api", /^repos\/o\/r\/issues\?/], reply: { kind: "always", stdout: "[]" } },
+ * ]);
+ * ```
+ *
+ * `experimental_stub.rest` — "any further tokens", last position only.
+ * `experimental_stub.called(name, argv, { min, max, contains })` — a check over
+ * the run's recorded calls.
+ *
+ * @experimental
+ */
+// eslint-disable-next-line functional/immutable-data -- the one experimental root: a callable with members, built the way experimental_agent is
+exports.experimental_stub = Object.assign(stubOf, {
+    rest: stub_rules_js_1.ARGV_REST,
+    called,
+});
+//# sourceMappingURL=stub.js.map

package/dist/test.d.ts

@@ -68,7 +68,10 @@ export { DISASTER_CATALOG, verifyGuardrail, unblockedDisasters, assertBlocksDisa
 export type { DisasterEvent, DisasterCategory, GuardrailResult, VerifyGuardrailOptions, } from "./guardrail-check.js";
 export { experimental_verifyPluginGuards, experimental_formatPluginGuardReport, } from "./verify-plugin-guards.js";
 export type { PluginGuardReport, SweptHook, SweptHookOutcome, NonCommandHookAction, VerifyPluginGuardsOptions, } from "./verify-plugin-guards.js";
-export * from "./tool-stub.js";
+export { experimental_stub } from "./stub.js";
+export type { StubCalledBounds } from "./stub.js";
+export type { ArgvPattern, ArgvRest, ArgvToken, StubAnswer, StubCall, StubOutcome, StubReply, StubRule, ToolStub, } from "./core/stub-rules.js";
+export type { HomeFiles, HomeSeed, RunEnv } from "./core/run-env.js";
 export { runHarnessTest, runHarness, parseToolCalls, parseSubagents, parseResultEvent, parseOutput, parseHooks, decideSandbox, specTrusted, sandboxAvailable, } from "./harness-test.js";
 export { experimental_outputStyleArms } from "./output-style-arms.js";
 export type { OutputStyleArms } from "./output-style-arms.js";

package/dist/test.js

@@ -71,8 +71,8 @@ var __exportStar = (this && this.__exportStar) || function(m, exports) {
     for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.runHarness = exports.runHarnessTest = exports.experimental_formatPluginGuardReport = exports.experimental_verifyPluginGuards = exports.experimental_alternateSpellings = exports.formatGuardrailReport = exports.assertBlocksDisasters = exports.unblockedDisasters = exports.verifyGuardrail = exports.DISASTER_CATALOG = exports.cacheTokens = exports.outputTokens = exports.inputTokens = exports.tokens = exports.latency = exports.cost = exports.mcp = exports.allowed = exports.blocked = exports.subagent = exports.didNotWrite = exports.wrote = exports.turns = exports.received = exports.hookFired = exports.experimental_replyCount = exports.experimental_eachReply = exports.output = exports.skill = exports.onlyTools = exports.notTool = exports.toolWith = exports.tool = exports.assertChecks = exports.evalChecks = exports.experimental_hookState = exports.loadHook = exports.experimental_assertEmittedOk = exports.experimental_parseEmitted = exports.experimental_emitTool = exports.egressRoutes = exports.fileToolEvents = exports.propertyHook = exports.decideHook = exports.parseHookOutput = exports.runHook = exports.runScript = exports.cleanupTmpDir = exports.makeTmpDir = exports.recordCheck = void 0;
-exports.experimental_makeDockerRuntime = exports.experimental_dockerRuntime = exports.experimental_withServices = exports.experimental_startServices = exports.stubSkillBody = exports.parseClaudeRun = exports.formatTriggerRateReport = exports.formatEvalReport = exports.formatCheckReport = exports.checkReportToJUnit = exports.checkPromptDiversity = exports.assertPromptDiversity = exports.assertRates = exports.defineEval = exports.formatContainment = exports.compareContainment = exports.skillContract = exports.mustNotInclude = exports.mustInclude = exports.commandsIn = exports.experimental_outputStyleArms = exports.sandboxAvailable = exports.specTrusted = exports.decideSandbox = exports.parseHooks = exports.parseOutput = exports.parseResultEvent = exports.parseSubagents = exports.parseToolCalls = void 0;
+exports.runHarnessTest = exports.experimental_stub = exports.experimental_formatPluginGuardReport = exports.experimental_verifyPluginGuards = exports.experimental_alternateSpellings = exports.formatGuardrailReport = exports.assertBlocksDisasters = exports.unblockedDisasters = exports.verifyGuardrail = exports.DISASTER_CATALOG = exports.cacheTokens = exports.outputTokens = exports.inputTokens = exports.tokens = exports.latency = exports.cost = exports.mcp = exports.allowed = exports.blocked = exports.subagent = exports.didNotWrite = exports.wrote = exports.turns = exports.received = exports.hookFired = exports.experimental_replyCount = exports.experimental_eachReply = exports.output = exports.skill = exports.onlyTools = exports.notTool = exports.toolWith = exports.tool = exports.assertChecks = exports.evalChecks = exports.experimental_hookState = exports.loadHook = exports.experimental_assertEmittedOk = exports.experimental_parseEmitted = exports.experimental_emitTool = exports.egressRoutes = exports.fileToolEvents = exports.propertyHook = exports.decideHook = exports.parseHookOutput = exports.runHook = exports.runScript = exports.cleanupTmpDir = exports.makeTmpDir = exports.recordCheck = void 0;
+exports.experimental_makeDockerRuntime = exports.experimental_dockerRuntime = exports.experimental_withServices = exports.experimental_startServices = exports.stubSkillBody = exports.parseClaudeRun = exports.formatTriggerRateReport = exports.formatEvalReport = exports.formatCheckReport = exports.checkReportToJUnit = exports.checkPromptDiversity = exports.assertPromptDiversity = exports.assertRates = exports.defineEval = exports.formatContainment = exports.compareContainment = exports.skillContract = exports.mustNotInclude = exports.mustInclude = exports.commandsIn = exports.experimental_outputStyleArms = exports.sandboxAvailable = exports.specTrusted = exports.decideSandbox = exports.parseHooks = exports.parseOutput = exports.parseResultEvent = exports.parseSubagents = exports.parseToolCalls = exports.runHarness = void 0;
 // --- reporting: how much did this script actually do? ---
 // `vigiles test` can otherwise see only an exit code, so a file that runs NOTHING
 // prints the same `✓` as one that ran and passed (measured 2026-08-08 on a file
@@ -196,8 +196,12 @@ Object.defineProperty(exports, "experimental_alternateSpellings", { enumerable:
 var verify_plugin_guards_js_1 = require("./verify-plugin-guards.js");
 Object.defineProperty(exports, "experimental_verifyPluginGuards", { enumerable: true, get: function () { return verify_plugin_guards_js_1.experimental_verifyPluginGuards; } });
 Object.defineProperty(exports, "experimental_formatPluginGuardReport", { enumerable: true, get: function () { return verify_plugin_guards_js_1.experimental_formatPluginGuardReport; } });
-// Tool stubs on PATH (rung R2): shadow a CLI tool with a recorded canned result.
-__exportStar(require("./tool-stub.js"), exports);
+// Tool stubs on PATH (rung R2): a fake binary that answers PER INVOCATION, by
+// rules over its argv. The helper and its check are experimental; the rule
+// shapes are plain data, because a `.mjs` literal of the same shape is accepted
+// at the spec boundary. Writing the shims is internal (`tool-stub.ts`).
+var stub_js_1 = require("./stub.js");
+Object.defineProperty(exports, "experimental_stub", { enumerable: true, get: function () { return stub_js_1.experimental_stub; } });
 // The assembled machine — AGNOSTIC SURFACE ONLY. The Claude-Code transport
 // (`scriptModel`, `claudeCodeDriver`, `buildClaudeArgs`, `claudeAvailable`,
 // `loadPlugin`, `resolveHarness`) is deliberately NOT re-exported here, so this