vigiles 2.2.0 → 2.4.0

package/dist/eval.js

@@ -1,6 +1,7 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.aggregate = aggregate;
+exports.aggregateStats = aggregateStats;
 exports.runEval = runEval;
 exports.formatEvalReport = formatEvalReport;
 /**
@@ -30,6 +31,7 @@ const node_child_process_1 = require("node:child_process");
 const node_fs_1 = require("node:fs");
 const node_os_1 = require("node:os");
 const node_path_1 = require("node:path");
+const plugin_loader_js_1 = require("./plugin-loader.js");
 function writeFiles(cwd, files) {
     for (const [p, content] of Object.entries(files)) {
         const full = (0, node_path_1.resolve)(cwd, p);
@@ -37,7 +39,7 @@ function writeFiles(cwd, files) {
         (0, node_fs_1.writeFileSync)(full, content);
     }
 }
-function spawnAgent(task, cwd, model, tools, hasSettings, timeoutMs) {
+function spawnAgent(task, cwd, model, tools, hasSettings, pluginDir, timeoutMs) {
     return new Promise((resolvePromise) => {
         const args = [
             "-p",
@@ -48,6 +50,7 @@ function spawnAgent(task, cwd, model, tools, hasSettings, timeoutMs) {
             model,
             "--permission-mode",
             "acceptEdits",
+            ...(pluginDir !== undefined ? ["--plugin-dir", (0, node_path_1.resolve)(pluginDir)] : []),
             ...(hasSettings ? ["--settings", "settings.json"] : []),
             "--allowedTools",
             ...tools,
@@ -101,28 +104,46 @@ function makeContext(cwd, out) {
         },
     };
 }
+/** Coerce a metric value to a number (booleans → 0/1), or null if absent. */
+function numeric(v) {
+    if (typeof v === "number")
+        return v;
+    if (typeof v === "boolean")
+        return v ? 1 : 0;
+    return null;
+}
 /** Aggregate per-run metrics: mean for numbers, fraction-true (0..1) for booleans. */
 function aggregate(rows) {
+    const stats = aggregateStats(rows);
+    const out = {};
+    for (const [k, s] of Object.entries(stats))
+        out[k] = s.mean;
+    return out;
+}
+/**
+ * Aggregate per-run metrics with spread: mean, sample std, standard error, and
+ * n. The se/std let you judge whether an A/B gap between arms is real or noise —
+ * a difference smaller than the combined se is not yet significant.
+ */
+function aggregateStats(rows) {
     const keys = new Set();
     for (const r of rows)
         for (const k of Object.keys(r))
             keys.add(k);
     const out = {};
     for (const k of keys) {
-        let sum = 0;
-        let n = 0;
+        const values = [];
         for (const r of rows) {
-            const v = r[k];
-            if (typeof v === "number") {
-                sum += v;
-                n++;
-            }
-            else if (typeof v === "boolean") {
-                sum += v ? 1 : 0;
-                n++;
-            }
+            const v = numeric(r[k]);
+            if (v !== null)
+                values.push(v);
         }
-        out[k] = n > 0 ? sum / n : 0;
+        const n = values.length;
+        const mean = n > 0 ? values.reduce((a, b) => a + b, 0) / n : 0;
+        const std = n > 1
+            ? Math.sqrt(values.reduce((a, b) => a + (b - mean) ** 2, 0) / (n - 1))
+            : 0;
+        out[k] = { mean, std, se: n > 0 ? std / Math.sqrt(n) : 0, n };
     }
     return out;
 }
@@ -143,12 +164,17 @@ async function runEval(spec) {
         for (let t = 0; t < trials; t++) {
             const cwd = (0, node_fs_1.mkdtempSync)((0, node_path_1.join)((0, node_os_1.tmpdir)(), "vigiles-eval-"));
             try {
-                writeFiles(cwd, { ...spec.fixture, ...arm.files });
-                const hasSettings = arm.settings !== undefined;
+                const { files, settings } = (0, plugin_loader_js_1.resolveHarness)({
+                    plugin: arm.plugin,
+                    settings: arm.settings,
+                    files: { ...spec.fixture, ...arm.files },
+                });
+                writeFiles(cwd, files);
+                const hasSettings = settings !== undefined;
                 if (hasSettings) {
-                    (0, node_fs_1.writeFileSync)((0, node_path_1.join)(cwd, "settings.json"), JSON.stringify(arm.settings, null, 2).replaceAll("{cwd}", cwd));
+                    (0, node_fs_1.writeFileSync)((0, node_path_1.join)(cwd, "settings.json"), JSON.stringify(settings, null, 2).replaceAll("{cwd}", cwd));
                 }
-                const out = await spawnAgent(spec.task, cwd, model, tools, hasSettings, timeoutMs);
+                const out = await spawnAgent(spec.task, cwd, model, tools, hasSettings, arm.pluginDir, timeoutMs);
                 rows.push(spec.measure(makeContext(cwd, out)));
             }
             finally {
@@ -156,16 +182,25 @@ async function runEval(spec) {
                 await sleep(spacing);
             }
         }
-        arms[armName] = { runs: rows.length, metrics: aggregate(rows) };
+        arms[armName] = {
+            runs: rows.length,
+            metrics: aggregate(rows),
+            stats: aggregateStats(rows),
+        };
     }
     return { name: spec.name ?? "eval", trials, arms };
 }
-/** Format an eval report as a compact table for the console. */
+/** Format an eval report as a compact table for the console (mean ± se). */
 function formatEvalReport(report) {
     const lines = [`${report.name} (${String(report.trials)} trials/arm)`];
     for (const [arm, r] of Object.entries(report.arms)) {
         const parts = Object.entries(r.metrics)
-            .map(([k, v]) => `${k}=${v.toFixed(2)}`)
+            .map(([k, v]) => {
+            const se = r.stats[k]?.se ?? 0;
+            return se > 0
+                ? `${k}=${v.toFixed(2)}±${se.toFixed(2)}`
+                : `${k}=${v.toFixed(2)}`;
+        })
             .join("  ");
         lines.push(`  ${arm.padEnd(10)} ${parts}`);
     }

package/dist/harness-assert.d.ts

@@ -0,0 +1,109 @@
+/**
+ * vigiles — runner-agnostic helpers for harness tests / evals.
+ *
+ * `runHarnessTest` and `runEval` are plain async functions that return data, so
+ * they already work inside any runner (node:test, vitest, jest, mocha). These
+ * helpers remove the last bit of boilerplate without coupling to a runner:
+ *
+ *   - `withHarness` — run a harness test and auto-clean the sandbox (try/finally),
+ *     so you don't leak temp dirs in `afterEach`.
+ *   - plain `assert*` helpers that throw — usable in every runner, including
+ *     node:test which has no `expect.extend`.
+ *   - `vigilesMatchers` — register with `expect.extend(vigilesMatchers)` for
+ *     `expect(...).toHaveCreated(...)` sugar. The signature is identical for
+ *     vitest and jest, so the same object supports both.
+ */
+import { type HarnessTestSpec, type HarnessTestResult, type ToolCall } from "./harness-test.js";
+import type { EvalReport } from "./eval.js";
+import type { HookRunResult } from "./run-hook.js";
+/**
+ * Run a harness test, hand the result to `fn`, and always clean up the sandbox.
+ * Returns whatever `fn` returns. Use this instead of calling `cleanup()` by
+ * hand — it survives assertion failures.
+ */
+export declare function withHarness<T>(spec: HarnessTestSpec, fn: (r: HarnessTestResult) => T | Promise<T>): Promise<T>;
+/** Assert the sandbox contains `path` (a hook/agent side-effect file). */
+export declare function assertCreated(r: HarnessTestResult, path: string): void;
+/** Assert the sandbox does NOT contain `path` (e.g. a blocked action's output). */
+export declare function assertNotCreated(r: HarnessTestResult, path: string): void;
+/** Assert the scripted model served at least `n` turns (e.g. a Stop hook forced more). */
+export declare function assertServedTurns(r: HarnessTestResult, n: number): void;
+/** Assert a `runHook` result blocked (exit 2 / decision:block / permission:deny). */
+export declare function assertHookBlocked(r: HookRunResult): void;
+/** Assert a `runHook` result allowed (did not block). */
+export declare function assertHookAllowed(r: HookRunResult): void;
+/**
+ * Assert the agent invoked a tool whose name matches `name` (string = exact,
+ * RegExp = test) — e.g. a skill (`"Skill"`), an MCP tool (`/^mcp__github__/`), or
+ * a subagent (`"Task"`). Needs `transcript: true`. The action invariant the
+ * skill/MCP/command surfaces are really about.
+ */
+export declare function assertToolUsed(r: HarnessTestResult, name: string | RegExp): void;
+/**
+ * Assert the agent did NOT invoke any tool matching `name` — the safety negative
+ * (e.g. a destructive MCP tool was never called). "File unchanged" can pass by
+ * accident; "the tool was never used" is the real invariant. Needs `transcript`.
+ */
+export declare function assertToolNotUsed(r: HarnessTestResult, name: string | RegExp): void;
+/**
+ * Assert the `Skill` tool resolved `skill` (e.g. `"superpowers:test-driven-development"`)
+ * without error — the correct skill-activation invariant, vs. grepping the body.
+ */
+export declare function assertSkillResolved(r: HarnessTestResult, skill: string): void;
+/**
+ * Assert how many tools matching `name` the agent invoked is within bounds — a
+ * budget invariant (e.g. `{ max: 1 }` = "at most one Write", `{ exactly: 0 }` =
+ * "never touched it"). Catches runaway loops and wasted work. Needs `transcript`.
+ */
+export declare function assertToolCount(r: HarnessTestResult, name: string | RegExp, bounds: {
+    min?: number;
+    max?: number;
+    exactly?: number;
+}): void;
+/**
+ * Assert the named tools occurred in this order (as a subsequence — gaps allowed)
+ * — an ordering invariant. e.g. `["Read", "Edit"]` checks a Read came before an
+ * Edit. For a stricter rule (every Edit preceded by a Read), use `assertToolCalls`.
+ * Needs `transcript`.
+ */
+export declare function assertToolSequence(r: HarnessTestResult, names: ReadonlyArray<string | RegExp>): void;
+/**
+ * The escape hatch: assert any custom invariant over the full list of tool calls
+ * the agent made — for rules the helpers above don't express, e.g. "every Edit
+ * was preceded by a Read of that file". Needs `transcript`.
+ */
+export declare function assertToolCalls(r: HarnessTestResult, predicate: (calls: readonly ToolCall[]) => boolean, message?: string): void;
+/** The gap on `metric` between two arms (arm − baseline). */
+export declare function improvement(report: EvalReport, baseline: string, arm: string, metric: string): number;
+/**
+ * Assert `arm` beats `baseline` on `metric` by more than `by`. With `by` left at
+ * 0 this just asserts a positive gap; pass the combined se to demand the gap
+ * clear the noise floor.
+ */
+export declare function assertImproves(report: EvalReport, opts: {
+    baseline: string;
+    arm: string;
+    metric: string;
+    by?: number;
+}): void;
+interface MatcherOutput {
+    pass: boolean;
+    message: () => string;
+}
+/**
+ * Custom matchers compatible with both vitest and jest. Register once:
+ *
+ *   import { expect } from "vitest"; // or "@jest/globals"
+ *   import { vigilesMatchers } from "vigiles/harness-assert";
+ *   expect.extend(vigilesMatchers);
+ *
+ *   expect(result).toHaveCreated("RESULT");
+ *   expect(report).toBeatBaseline("vanilla", "gated", "caught");
+ */
+export declare const vigilesMatchers: {
+    toHaveCreated(received: HarnessTestResult, path: string): MatcherOutput;
+    toBlock(received: HookRunResult): MatcherOutput;
+    toBeatBaseline(received: EvalReport, baseline: string, arm: string, metric: string, by?: number): MatcherOutput;
+};
+export {};
+//# sourceMappingURL=harness-assert.d.ts.map

package/dist/harness-assert.js

@@ -0,0 +1,220 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.vigilesMatchers = void 0;
+exports.withHarness = withHarness;
+exports.assertCreated = assertCreated;
+exports.assertNotCreated = assertNotCreated;
+exports.assertServedTurns = assertServedTurns;
+exports.assertHookBlocked = assertHookBlocked;
+exports.assertHookAllowed = assertHookAllowed;
+exports.assertToolUsed = assertToolUsed;
+exports.assertToolNotUsed = assertToolNotUsed;
+exports.assertSkillResolved = assertSkillResolved;
+exports.assertToolCount = assertToolCount;
+exports.assertToolSequence = assertToolSequence;
+exports.assertToolCalls = assertToolCalls;
+exports.improvement = improvement;
+exports.assertImproves = assertImproves;
+/**
+ * vigiles — runner-agnostic helpers for harness tests / evals.
+ *
+ * `runHarnessTest` and `runEval` are plain async functions that return data, so
+ * they already work inside any runner (node:test, vitest, jest, mocha). These
+ * helpers remove the last bit of boilerplate without coupling to a runner:
+ *
+ *   - `withHarness` — run a harness test and auto-clean the sandbox (try/finally),
+ *     so you don't leak temp dirs in `afterEach`.
+ *   - plain `assert*` helpers that throw — usable in every runner, including
+ *     node:test which has no `expect.extend`.
+ *   - `vigilesMatchers` — register with `expect.extend(vigilesMatchers)` for
+ *     `expect(...).toHaveCreated(...)` sugar. The signature is identical for
+ *     vitest and jest, so the same object supports both.
+ */
+const harness_test_js_1 = require("./harness-test.js");
+/**
+ * Run a harness test, hand the result to `fn`, and always clean up the sandbox.
+ * Returns whatever `fn` returns. Use this instead of calling `cleanup()` by
+ * hand — it survives assertion failures.
+ */
+async function withHarness(spec, fn) {
+    const r = await (0, harness_test_js_1.runHarnessTest)(spec);
+    try {
+        return await fn(r);
+    }
+    finally {
+        r.cleanup();
+    }
+}
+// --- Plain throwing assertions (any runner) --------------------------------
+function fail(message) {
+    throw new Error(message);
+}
+/** Assert the sandbox contains `path` (a hook/agent side-effect file). */
+function assertCreated(r, path) {
+    if (r.file(path) === null)
+        fail(`expected the run to create ${path}`);
+}
+/** Assert the sandbox does NOT contain `path` (e.g. a blocked action's output). */
+function assertNotCreated(r, path) {
+    if (r.file(path) !== null)
+        fail(`expected the run NOT to create ${path}`);
+}
+/** Assert the scripted model served at least `n` turns (e.g. a Stop hook forced more). */
+function assertServedTurns(r, n) {
+    if (r.turns < n) {
+        fail(`expected ≥ ${String(n)} model turns, got ${String(r.turns)}`);
+    }
+}
+/** Assert a `runHook` result blocked (exit 2 / decision:block / permission:deny). */
+function assertHookBlocked(r) {
+    if (!r.blocked) {
+        fail(`expected the hook to block, but it allowed (exit ${String(r.exitCode)})`);
+    }
+}
+/** Assert a `runHook` result allowed (did not block). */
+function assertHookAllowed(r) {
+    if (r.blocked) {
+        fail(`expected the hook to allow, but it blocked (exit ${String(r.exitCode)}, decision ${String(r.decision)})`);
+    }
+}
+function nameMatches(name, pat) {
+    return typeof pat === "string" ? name === pat : pat.test(name);
+}
+function toolNames(r) {
+    return r.toolCalls.map((c) => c.name).join(", ") || "none";
+}
+/**
+ * Assert the agent invoked a tool whose name matches `name` (string = exact,
+ * RegExp = test) — e.g. a skill (`"Skill"`), an MCP tool (`/^mcp__github__/`), or
+ * a subagent (`"Task"`). Needs `transcript: true`. The action invariant the
+ * skill/MCP/command surfaces are really about.
+ */
+function assertToolUsed(r, name) {
+    if (!r.toolCalls.some((c) => nameMatches(c.name, name))) {
+        fail(`expected a tool matching ${String(name)} to be used; tools used: [${toolNames(r)}] (did you set transcript:true?)`);
+    }
+}
+/**
+ * Assert the agent did NOT invoke any tool matching `name` — the safety negative
+ * (e.g. a destructive MCP tool was never called). "File unchanged" can pass by
+ * accident; "the tool was never used" is the real invariant. Needs `transcript`.
+ */
+function assertToolNotUsed(r, name) {
+    const hit = r.toolCalls.find((c) => nameMatches(c.name, name));
+    if (hit) {
+        fail(`expected no tool matching ${String(name)} to be used, but ${hit.name} was`);
+    }
+}
+/**
+ * Assert the `Skill` tool resolved `skill` (e.g. `"superpowers:test-driven-development"`)
+ * without error — the correct skill-activation invariant, vs. grepping the body.
+ */
+function assertSkillResolved(r, skill) {
+    const call = r.toolCalls.find((c) => c.name === "Skill" && c.input?.skill === skill);
+    if (!call) {
+        const seen = r.toolCalls
+            .filter((c) => c.name === "Skill")
+            .map((c) => c.input?.skill ?? "?")
+            .join(", ");
+        fail(`expected the Skill tool to resolve "${skill}"; Skill calls: [${seen || "none"}]`);
+    }
+    if (call.isError) {
+        fail(`the Skill "${skill}" was invoked but errored: ${call.resultText.slice(0, 200)}`);
+    }
+}
+// --- sequence / budget invariants over the agent's actions -----------------
+/**
+ * Assert how many tools matching `name` the agent invoked is within bounds — a
+ * budget invariant (e.g. `{ max: 1 }` = "at most one Write", `{ exactly: 0 }` =
+ * "never touched it"). Catches runaway loops and wasted work. Needs `transcript`.
+ */
+function assertToolCount(r, name, bounds) {
+    const n = r.toolCalls.filter((c) => nameMatches(c.name, name)).length;
+    const ok = (bounds.exactly === undefined || n === bounds.exactly) &&
+        (bounds.min === undefined || n >= bounds.min) &&
+        (bounds.max === undefined || n <= bounds.max);
+    if (!ok) {
+        fail(`expected count of ${String(name)} to satisfy ${JSON.stringify(bounds)}, got ${String(n)} (tools: [${toolNames(r)}])`);
+    }
+}
+/**
+ * Assert the named tools occurred in this order (as a subsequence — gaps allowed)
+ * — an ordering invariant. e.g. `["Read", "Edit"]` checks a Read came before an
+ * Edit. For a stricter rule (every Edit preceded by a Read), use `assertToolCalls`.
+ * Needs `transcript`.
+ */
+function assertToolSequence(r, names) {
+    let i = 0;
+    for (const c of r.toolCalls) {
+        const want = names[i];
+        if (want !== undefined && nameMatches(c.name, want))
+            i++;
+    }
+    if (i < names.length) {
+        fail(`expected tools in order [${names.map((n) => String(n)).join(" → ")}]; got [${toolNames(r)}]`);
+    }
+}
+/**
+ * The escape hatch: assert any custom invariant over the full list of tool calls
+ * the agent made — for rules the helpers above don't express, e.g. "every Edit
+ * was preceded by a Read of that file". Needs `transcript`.
+ */
+function assertToolCalls(r, predicate, message = "tool-call invariant failed") {
+    if (!predicate(r.toolCalls)) {
+        fail(`${message}; tools used: [${toolNames(r)}]`);
+    }
+}
+/** The gap on `metric` between two arms (arm − baseline). */
+function improvement(report, baseline, arm, metric) {
+    const a = report.arms[arm]?.metrics[metric] ?? 0;
+    const b = report.arms[baseline]?.metrics[metric] ?? 0;
+    return a - b;
+}
+/**
+ * Assert `arm` beats `baseline` on `metric` by more than `by`. With `by` left at
+ * 0 this just asserts a positive gap; pass the combined se to demand the gap
+ * clear the noise floor.
+ */
+function assertImproves(report, opts) {
+    const by = opts.by ?? 0;
+    const delta = improvement(report, opts.baseline, opts.arm, opts.metric);
+    if (delta <= by) {
+        fail(`expected ${opts.arm} to beat ${opts.baseline} on ${opts.metric} by > ${String(by)}, got ${delta.toFixed(3)}`);
+    }
+}
+/**
+ * Custom matchers compatible with both vitest and jest. Register once:
+ *
+ *   import { expect } from "vitest"; // or "@jest/globals"
+ *   import { vigilesMatchers } from "vigiles/harness-assert";
+ *   expect.extend(vigilesMatchers);
+ *
+ *   expect(result).toHaveCreated("RESULT");
+ *   expect(report).toBeatBaseline("vanilla", "gated", "caught");
+ */
+exports.vigilesMatchers = {
+    toHaveCreated(received, path) {
+        const pass = received.file(path) !== null;
+        return {
+            pass,
+            message: () => `expected the run ${pass ? "not " : ""}to create ${path}`,
+        };
+    },
+    toBlock(received) {
+        const pass = received.blocked;
+        return {
+            pass,
+            message: () => `expected the hook ${pass ? "not " : ""}to block (exit ${String(received.exitCode)}, decision ${String(received.decision)})`,
+        };
+    },
+    // eslint-disable-next-line max-params -- jest/vitest matchers take positional args
+    toBeatBaseline(received, baseline, arm, metric, by = 0) {
+        const delta = improvement(received, baseline, arm, metric);
+        const pass = delta > by;
+        return {
+            pass,
+            message: () => `expected ${arm} ${pass ? "not " : ""}to beat ${baseline} on ${metric} by > ${String(by)} (got ${delta.toFixed(3)})`,
+        };
+    },
+};
+//# sourceMappingURL=harness-assert.js.map

package/dist/harness-test.d.ts

@@ -1,16 +1,40 @@
 import { type ModelTurn } from "./mock-model.js";
 export { scriptModel, type ModelTurn } from "./mock-model.js";
+export { loadPlugin, resolveHarness } from "./plugin-loader.js";
 export interface HarnessTestSpec {
     /** Fixture files to write in a fresh temp working dir (path → contents). */
     readonly files?: Record<string, string>;
     /** `.claude/settings.json` contents — the hooks/permissions under test. */
     readonly settings?: unknown;
+    /**
+     * Path to a real plugin/repo whose harness (hooks + CLAUDE.md + skills) is
+     * loaded into the sandbox, so you test the assembled machine, not a retyped
+     * subset. Inline `settings`/`files` layer on top. See src/plugin-loader.ts.
+     */
+    readonly plugin?: string;
+    /**
+     * Path to a plugin dir to install NATIVELY via `claude --plugin-dir`, so its
+     * skills / commands / agents / hooks register and ACTIVATE the real way — a
+     * scripted `Skill` tool_use resolves, and the real model can trigger them.
+     * Unlike `plugin` (which materializes a file subset that does NOT register
+     * skills for the `Skill` tool), this is the real install path, so point it at a
+     * COMPLETE plugin (internal references resolve). Inline `settings`/`files` and
+     * `plugin` still layer on top. Resolved to an absolute path.
+     */
+    readonly pluginDir?: string;
     /** The scripted model turns the agent will take. */
     readonly model: readonly ModelTurn[];
     /** The user prompt. Default: "go". */
     readonly prompt?: string;
     /** Tools the agent may use. Default: Read Edit Write Bash. */
     readonly allowedTools?: readonly string[];
+    /**
+     * Capture the full event transcript (`--output-format stream-json`) into
+     * `stdout`, instead of just the final result object, so you can assert on what
+     * the agent's tools returned — e.g. the body a `Skill` tool_use resolved. With
+     * this on, `stdout` is newline-delimited JSON events, not a single object.
+     */
+    readonly transcript?: boolean;
     /** Per-run wall-clock timeout in ms. Default 60000. */
     readonly timeoutMs?: number;
 }
@@ -23,11 +47,33 @@ export interface HarnessTestResult {
     readonly cwd: string;
     /** Number of model turns the agent took (mock turns served). */
     readonly turns: number;
+    /**
+     * The tools the agent invoked, each paired with its result — parsed from the
+     * transcript. Empty unless `transcript: true`. Lets a test assert on the
+     * agent's *actions* (skills, MCP tools, subagents) instead of grepping stdout.
+     */
+    readonly toolCalls: readonly ToolCall[];
     /** Final contents of a file under the working dir, or null if absent. */
     file(path: string): string | null;
     /** Remove the temp working dir. */
     cleanup(): void;
 }
+/** A tool the agent invoked, paired with its result (transcript mode only). */
+export interface ToolCall {
+    readonly name: string;
+    readonly input: unknown;
+    /** The tool_result text ("" if none / not captured). */
+    readonly resultText: string;
+    /** Whether the tool_result came back flagged as an error. */
+    readonly isError: boolean;
+}
+/**
+ * Parse `--output-format stream-json` (the `transcript: true` output) into the
+ * tools the agent invoked, each joined to its result by id. Returns [] for the
+ * non-stream `json` output. The seam that lets a test assert on the agent's
+ * actions, not a brittle stdout substring.
+ */
+export declare function parseToolCalls(streamJson: string): ToolCall[];
 /** Whether the `claude` CLI is available — harness tests need it. */
 export declare function claudeAvailable(): boolean;
 /**