vigiles 2.3.0 → 2.5.0

package/dist/eval.js

@@ -1,9 +1,17 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
+exports.runEval = runEval;
+exports.parseUsage = parseUsage;
 exports.aggregate = aggregate;
 exports.aggregateStats = aggregateStats;
-exports.runEval = runEval;
+exports.aggregateUsage = aggregateUsage;
+exports.isRateLimited = isRateLimited;
+exports.runPool = runPool;
+exports.runEvalWith = runEvalWith;
 exports.formatEvalReport = formatEvalReport;
+exports.measureTriggerRateWith = measureTriggerRateWith;
+exports.measureTriggerRate = measureTriggerRate;
+exports.formatTriggerRateReport = formatTriggerRateReport;
 /**
  * vigiles — Claude Code harness *evals*.
  *
@@ -32,6 +40,8 @@ const node_fs_1 = require("node:fs");
 const node_os_1 = require("node:os");
 const node_path_1 = require("node:path");
 const plugin_loader_js_1 = require("./plugin-loader.js");
+const harness_test_js_1 = require("./harness-test.js");
+const eval_cache_js_1 = require("./eval-cache.js");
 function writeFiles(cwd, files) {
     for (const [p, content] of Object.entries(files)) {
         const full = (0, node_path_1.resolve)(cwd, p);
@@ -39,49 +49,87 @@ function writeFiles(cwd, files) {
         (0, node_fs_1.writeFileSync)(full, content);
     }
 }
-function spawnAgent(task, cwd, model, tools, hasSettings, timeoutMs) {
+/* v8 ignore start -- real claude subprocess; exercised by bench/, not the unit gate */
+function spawnAgent(a) {
     return new Promise((resolvePromise) => {
         const args = [
             "-p",
-            task,
+            a.task,
+            // stream-json (+ --verbose, required with -p) so the per-turn tool_use
+            // events survive into `ctx.toolCalls` — the unified Trace, same as the
+            // harness tier. The terminal `result` event still carries num_turns/output.
             "--output-format",
-            "json",
+            "stream-json",
+            "--verbose",
             "--model",
-            model,
+            a.model,
             "--permission-mode",
             "acceptEdits",
-            ...(hasSettings ? ["--settings", "settings.json"] : []),
+            ...(a.pluginDir !== undefined
+                ? ["--plugin-dir", (0, node_path_1.resolve)(a.pluginDir)]
+                : []),
+            ...(a.hasSettings ? ["--settings", "settings.json"] : []),
             "--allowedTools",
-            ...tools,
+            ...a.tools,
         ];
         const child = (0, node_child_process_1.spawn)("claude", args, {
-            cwd,
+            cwd: a.cwd,
             env: process.env,
             stdio: ["ignore", "pipe", "pipe"],
         });
         let stdout = "";
+        let stderr = "";
         child.stdout.on("data", (d) => (stdout += d.toString()));
-        const timer = setTimeout(() => child.kill("SIGKILL"), timeoutMs);
+        child.stderr.on("data", (d) => (stderr += d.toString()));
+        const timer = setTimeout(() => child.kill("SIGKILL"), a.timeoutMs);
         child.on("close", (code) => {
             clearTimeout(timer);
-            resolvePromise({ code: code ?? 0, stdout });
+            resolvePromise({ code: code ?? 0, stdout, stderr });
         });
     });
 }
+/**
+ * Run the eval: every arm × every trial against the real `claude` CLI, with the
+ * metric computed per run and aggregated per arm. Requires `claude` on PATH and
+ * working model auth (e.g. `ANTHROPIC_API_KEY`). Thin wrapper over
+ * {@link runEvalWith} with the real agent runner.
+ */
+async function runEval(spec) {
+    return runEvalWith(spec, spawnAgent);
+}
+/* v8 ignore stop */
 const sleep = (ms) => new Promise((r) => setTimeout(r, ms));
+/** Pull cost / latency / tokens out of a parsed `result` event (0 when absent). */
+function usageFrom(result) {
+    const num = (v) => (typeof v === "number" ? v : 0);
+    const usage = (result?.usage ?? {});
+    return {
+        costUsd: num(result?.total_cost_usd),
+        durationMs: num(result?.duration_ms),
+        inputTokens: num(usage.input_tokens),
+        outputTokens: num(usage.output_tokens),
+    };
+}
+/** Parse per-run cost/latency/tokens from a stream — pure, model-free. */
+function parseUsage(stdout) {
+    return usageFrom((0, harness_test_js_1.parseResultEvent)(stdout));
+}
 function makeContext(cwd, out) {
-    let turns = 0;
-    try {
-        turns = JSON.parse(out.stdout).num_turns ?? 0;
-    }
-    catch {
-        /* non-JSON output */
-    }
+    const result = (0, harness_test_js_1.parseResultEvent)(out.stdout);
+    const turns = typeof result?.num_turns === "number" ? result.num_turns : 0;
+    const output = typeof result?.result === "string" ? result.result : "";
     return {
         cwd,
         exitCode: out.code,
         stdout: out.stdout,
         turns,
+        toolCalls: (0, harness_test_js_1.parseToolCalls)(out.stdout),
+        hooks: (0, harness_test_js_1.parseHooks)(out.stdout),
+        output,
+        usage: usageFrom(result),
+        // The eval tier drives the real API (no mock between claude and the model),
+        // so the requests can't be captured here — modelRequests is harness-tier only.
+        modelRequests: [],
         file: (p) => {
             const f = (0, node_path_1.resolve)(cwd, p);
             return (0, node_fs_1.existsSync)(f) ? (0, node_fs_1.readFileSync)(f, "utf-8") : null;
@@ -142,66 +190,280 @@ function aggregateStats(rows) {
         const std = n > 1
             ? Math.sqrt(values.reduce((a, b) => a + (b - mean) ** 2, 0) / (n - 1))
             : 0;
-        out[k] = { mean, std, se: n > 0 ? std / Math.sqrt(n) : 0, n };
+        const passK = n > 0 && values.every((v) => v > 0) ? 1 : 0;
+        out[k] = { mean, std, se: n > 0 ? std / Math.sqrt(n) : 0, n, passK };
     }
     return out;
 }
+/** Aggregate per-run usage into an arm's cost / latency / token totals + means. */
+function aggregateUsage(usages) {
+    const n = usages.length;
+    const sum = (f) => usages.reduce((a, u) => a + f(u), 0);
+    const totalCostUsd = sum((u) => u.costUsd);
+    return {
+        totalCostUsd,
+        meanCostUsd: n > 0 ? totalCostUsd / n : 0,
+        meanDurationMs: n > 0 ? sum((u) => u.durationMs) / n : 0,
+        totalInputTokens: sum((u) => u.inputTokens),
+        totalOutputTokens: sum((u) => u.outputTokens),
+    };
+}
 /**
- * Run the eval: every arm × every trial against the real `claude` CLI, with the
- * metric computed per run and aggregated per arm. Requires `claude` on PATH and
- * working model auth (e.g. `ANTHROPIC_API_KEY`).
+ * Run one trial through the cache: on a hit, restore the recorded post-run
+ * filesystem into `cwd` and return the recorded output (no model call); on a
+ * miss, run the agent and (in `readwrite`) record output + cwd snapshot. The
+ * cache key excludes `measure`, so editing the metric still replays.
  */
-async function runEval(spec) {
+async function runWithCache(runArgs, keyParts, runner, cfg) {
+    if (cfg.cache === "off")
+        return runner(runArgs);
+    const key = (0, eval_cache_js_1.cacheKey)({
+        task: runArgs.task,
+        model: runArgs.model,
+        tools: runArgs.tools,
+        files: keyParts.files,
+        settings: keyParts.settings,
+        trialIndex: keyParts.trialIndex,
+    });
+    const hit = (0, eval_cache_js_1.readCache)(cfg.cacheDir, key);
+    if (hit) {
+        (0, eval_cache_js_1.restoreDir)(runArgs.cwd, hit.files);
+        return hit.out;
+    }
+    const out = await runner(runArgs);
+    if (cfg.cache === "readwrite") {
+        (0, eval_cache_js_1.writeCache)(cfg.cacheDir, key, { out, files: (0, eval_cache_js_1.snapshotDir)(runArgs.cwd) });
+    }
+    return out;
+}
+/** Execute one trial in a fresh sandbox; returns its metric row + usage. */
+async function executeTrial(spec, arm, trialIndex, runner, cfg) {
+    const cwd = (0, node_fs_1.mkdtempSync)((0, node_path_1.join)((0, node_os_1.tmpdir)(), "vigiles-eval-"));
+    try {
+        const { files, settings } = (0, plugin_loader_js_1.resolveHarness)({
+            plugin: arm.plugin,
+            settings: arm.settings,
+            files: { ...spec.fixture, ...arm.files },
+        });
+        writeFiles(cwd, files);
+        const hasSettings = settings !== undefined;
+        if (hasSettings) {
+            (0, node_fs_1.writeFileSync)((0, node_path_1.join)(cwd, "settings.json"), JSON.stringify(settings, null, 2).replaceAll("{cwd}", cwd));
+        }
+        const out = await runWithCache({
+            task: spec.task,
+            cwd,
+            model: cfg.model,
+            tools: cfg.tools,
+            hasSettings,
+            pluginDir: arm.pluginDir,
+            timeoutMs: cfg.timeoutMs,
+        }, { files, settings, trialIndex }, runner, cfg);
+        const ctx = makeContext(cwd, out);
+        return { row: spec.measure(ctx), usage: ctx.usage };
+    }
+    finally {
+        (0, node_fs_1.rmSync)(cwd, { recursive: true, force: true });
+    }
+}
+// A signal in the captured streams that the model call was rate-limited /
+// overloaded — worth a backoff + retry rather than counting as a real sample.
+const RATE_LIMIT_RE = /rate.?limit|\b429\b|overloaded|too many requests/i;
+/** Whether a run's captured output looks like a rate-limit / overload. Pure. */
+function isRateLimited(out) {
+    return RATE_LIMIT_RE.test(`${out.stderr ?? ""}\n${out.stdout}`);
+}
+/** Call `runner`, retrying with exponential backoff while it looks rate-limited. */
+async function runWithRetry(runArgs, runner, retries, baseMs) {
+    for (let attempt = 0;; attempt++) {
+        const out = await runner(runArgs);
+        if (!isRateLimited(out) || attempt >= retries)
+            return out;
+        await sleep(baseMs * 2 ** attempt);
+    }
+}
+/** Map `worker` over `items` with at most `concurrency` in flight, order preserved. */
+async function runPool(items, concurrency, worker) {
+    const results = new Array(items.length);
+    let next = 0;
+    const drain = async () => {
+        for (;;) {
+            const i = next++;
+            const item = items[i];
+            if (i >= items.length || item === undefined)
+                return;
+            results[i] = await worker(item);
+        }
+    };
+    const workers = Math.max(1, Math.min(concurrency, items.length || 1));
+    await Promise.all(Array.from({ length: workers }, drain));
+    return results;
+}
+/** Flatten arms × trials into a single work list (so concurrency spans both). */
+function buildUnits(arms, trials) {
+    const units = [];
+    for (const [armName, arm] of Object.entries(arms)) {
+        for (let t = 0; t < trials; t++)
+            units.push({ armName, arm, trialIndex: t });
+    }
+    return units;
+}
+/** Group completed (non-skipped) unit results by arm and aggregate each. */
+function aggregateArms(armNames, results) {
+    const arms = {};
+    let totalCostUsd = 0;
+    for (const armName of armNames) {
+        const done = results.filter((r) => !r.skipped && r.armName === armName);
+        const rows = done.map((d) => d.row);
+        const usage = aggregateUsage(done.map((d) => d.usage));
+        totalCostUsd += usage.totalCostUsd;
+        arms[armName] = {
+            runs: rows.length,
+            metrics: aggregate(rows),
+            stats: aggregateStats(rows),
+            usage,
+        };
+    }
+    return { arms, totalCostUsd };
+}
+/**
+ * The eval orchestration — every arm × trial via `runner`, run through the cache
+ * and a rate-limit retry, with at most `concurrency` in flight and an optional
+ * `maxCostUsd` budget cap; metric + usage computed per run and aggregated per
+ * arm. Exported with an injectable `runner` so the loop, `measure` context,
+ * caching, pooling, and aggregation are unit-testable without spawning a model
+ * (pass a fake returning canned stream-json). `runEval` is this with the real
+ * agent runner.
+ */
+async function runEvalWith(spec, runner) {
     const trials = spec.trials ?? 5;
+    const spacing = (spec.spacingSec ?? 4) * 1000;
+    const concurrency = spec.concurrency ?? 1;
+    const retries = spec.rateLimitRetries ?? 3;
+    const backoffMs = spec.retryBackoffMs ?? 1000;
+    const cfg = {
+        model: spec.model ?? "haiku",
+        tools: spec.allowedTools ?? ["Read", "Edit", "Write", "Bash"],
+        timeoutMs: spec.timeoutMs ?? 240000,
+        cache: spec.cache ?? "off",
+        cacheDir: spec.cacheDir ?? (0, node_path_1.resolve)(process.cwd(), ".vigiles", "eval-cache"),
+    };
+    const retrying = (a) => runWithRetry(a, runner, retries, backoffMs);
+    const units = buildUnits(spec.arms, trials);
+    let spent = 0;
+    let aborted = false;
+    const worker = async (unit) => {
+        if (aborted)
+            return { armName: unit.armName, skipped: true };
+        const { row, usage } = await executeTrial(spec, unit.arm, unit.trialIndex, retrying, cfg);
+        spent += usage.costUsd;
+        if (spec.maxCostUsd !== undefined && spent >= spec.maxCostUsd) {
+            aborted = true;
+        }
+        if (spacing > 0)
+            await sleep(spacing);
+        return { armName: unit.armName, skipped: false, row, usage };
+    };
+    const results = await runPool(units, concurrency, worker);
+    const { arms, totalCostUsd } = aggregateArms(Object.keys(spec.arms), results);
+    return { name: spec.name ?? "eval", trials, arms, totalCostUsd, aborted };
+}
+/** Render one metric: `name=mean±se pass^k=…` (se/pass^k shown when measured). */
+function formatMetric(name, mean, stat) {
+    const base = stat && stat.se > 0
+        ? `${name}=${mean.toFixed(2)}±${stat.se.toFixed(2)}`
+        : `${name}=${mean.toFixed(2)}`;
+    return stat && stat.n > 0 ? `${base} pass^k=${String(stat.passK)}` : base;
+}
+/** Compact tokens like `3.4k`; whole numbers under 1000 stay as-is. */
+function fmtTokens(n) {
+    return n >= 1000 ? `${(n / 1000).toFixed(1)}k` : String(n);
+}
+/** A `($0.0123 · 1.2s/run · 3.4k tok)` suffix, or "" when no usage was reported. */
+function formatUsage(u) {
+    if (u.totalCostUsd === 0 && u.totalInputTokens + u.totalOutputTokens === 0) {
+        return "";
+    }
+    const tok = fmtTokens(u.totalInputTokens + u.totalOutputTokens);
+    return `  ($${u.totalCostUsd.toFixed(4)} · ${(u.meanDurationMs / 1000).toFixed(1)}s/run · ${tok} tok)`;
+}
+/** Format an eval report as a compact table for the console (mean ± se, pass^k). */
+function formatEvalReport(report) {
+    const header = report.totalCostUsd > 0
+        ? `${report.name} (${String(report.trials)} trials/arm) — $${report.totalCostUsd.toFixed(4)} total`
+        : `${report.name} (${String(report.trials)} trials/arm)`;
+    const lines = [header];
+    for (const [arm, r] of Object.entries(report.arms)) {
+        const parts = Object.entries(r.metrics)
+            .map(([k, v]) => formatMetric(k, v, r.stats[k]))
+            .join("  ");
+        lines.push(`  ${arm.padEnd(10)} ${parts}${formatUsage(r.usage)}`);
+    }
+    return lines.join("\n");
+}
+/**
+ * Trigger-rate orchestration — every prompt × trial via `runner`, the `fired`
+ * predicate evaluated per run and aggregated into an overall + per-prompt rate.
+ * Exported with an injectable `runner` so the loop is unit-testable without a
+ * model; `measureTriggerRate` is this with the real agent runner.
+ */
+async function measureTriggerRateWith(spec, runner) {
+    const trials = spec.trials ?? 1;
     const model = spec.model ?? "haiku";
-    const tools = spec.allowedTools ?? ["Read", "Edit", "Write", "Bash"];
+    const tools = spec.allowedTools ?? ["Read", "Edit", "Write", "Bash", "Skill"];
     const timeoutMs = spec.timeoutMs ?? 240000;
     const spacing = (spec.spacingSec ?? 4) * 1000;
-    const arms = {};
-    for (const [armName, arm] of Object.entries(spec.arms)) {
-        const rows = [];
+    const perPrompt = [];
+    let firedTotal = 0;
+    let n = 0;
+    for (const prompt of spec.prompts) {
+        let fired = 0;
         for (let t = 0; t < trials; t++) {
-            const cwd = (0, node_fs_1.mkdtempSync)((0, node_path_1.join)((0, node_os_1.tmpdir)(), "vigiles-eval-"));
+            const cwd = (0, node_fs_1.mkdtempSync)((0, node_path_1.join)((0, node_os_1.tmpdir)(), "vigiles-trigger-"));
             try {
-                const { files, settings } = (0, plugin_loader_js_1.resolveHarness)({
-                    plugin: arm.plugin,
-                    settings: arm.settings,
-                    files: { ...spec.fixture, ...arm.files },
+                const out = await runner({
+                    task: prompt,
+                    cwd,
+                    model,
+                    tools,
+                    hasSettings: false,
+                    pluginDir: spec.pluginDir,
+                    timeoutMs,
                 });
-                writeFiles(cwd, files);
-                const hasSettings = settings !== undefined;
-                if (hasSettings) {
-                    (0, node_fs_1.writeFileSync)((0, node_path_1.join)(cwd, "settings.json"), JSON.stringify(settings, null, 2).replaceAll("{cwd}", cwd));
-                }
-                const out = await spawnAgent(spec.task, cwd, model, tools, hasSettings, timeoutMs);
-                rows.push(spec.measure(makeContext(cwd, out)));
+                if (spec.fired(makeContext(cwd, out)))
+                    fired++;
             }
             finally {
                 (0, node_fs_1.rmSync)(cwd, { recursive: true, force: true });
                 await sleep(spacing);
             }
         }
-        arms[armName] = {
-            runs: rows.length,
-            metrics: aggregate(rows),
-            stats: aggregateStats(rows),
-        };
+        perPrompt.push({
+            prompt,
+            fired,
+            trials,
+            rate: trials > 0 ? fired / trials : 0,
+        });
+        firedTotal += fired;
+        n += trials;
     }
-    return { name: spec.name ?? "eval", trials, arms };
+    return { rate: n > 0 ? firedTotal / n : 0, n, perPrompt };
 }
-/** Format an eval report as a compact table for the console (mean ± se). */
-function formatEvalReport(report) {
-    const lines = [`${report.name} (${String(report.trials)} trials/arm)`];
-    for (const [arm, r] of Object.entries(report.arms)) {
-        const parts = Object.entries(r.metrics)
-            .map(([k, v]) => {
-            const se = r.stats[k]?.se ?? 0;
-            return se > 0
-                ? `${k}=${v.toFixed(2)}±${se.toFixed(2)}`
-                : `${k}=${v.toFixed(2)}`;
-        })
-            .join("  ");
-        lines.push(`  ${arm.padEnd(10)} ${parts}`);
+/* v8 ignore start -- real claude subprocess; thin wrapper over measureTriggerRateWith */
+/**
+ * Measure a skill/behaviour's real trigger rate across prompts × trials against
+ * the real `claude` CLI. Requires `claude` + model auth.
+ */
+async function measureTriggerRate(spec) {
+    return measureTriggerRateWith(spec, spawnAgent);
+}
+/* v8 ignore stop */
+/** Format a trigger-rate report: overall %, then each prompt's rate. */
+function formatTriggerRateReport(report) {
+    const pct = (report.rate * 100).toFixed(0);
+    const lines = [`trigger-rate: ${pct}% (${String(report.n)} runs)`];
+    for (const p of report.perPrompt) {
+        lines.push(`  ${p.rate.toFixed(2)}  ${p.prompt.slice(0, 60)}`);
     }
     return lines.join("\n");
 }

package/dist/harness-assert.d.ts

@@ -13,9 +13,13 @@
  *     `expect(...).toHaveCreated(...)` sugar. The signature is identical for
  *     vitest and jest, so the same object supports both.
  */
-import { type HarnessTestSpec, type HarnessTestResult } from "./harness-test.js";
-import type { EvalReport } from "./eval.js";
+import { type HarnessTestSpec, type HarnessTestResult, type ToolCall, type Trace } from "./harness-test.js";
+import type { EvalReport, TriggerRateReport } from "./eval.js";
 import type { HookRunResult } from "./run-hook.js";
+import type { OutputContract } from "./spec.js";
+import { type ParsedAgentResult } from "./agent-result.js";
+export { compareArms } from "./stats.js";
+export type { Comparison } from "./stats.js";
 /**
  * Run a harness test, hand the result to `fn`, and always clean up the sandbox.
  * Returns whatever `fn` returns. Use this instead of calling `cleanup()` by
@@ -32,18 +36,184 @@ export declare function assertServedTurns(r: HarnessTestResult, n: number): void
 export declare function assertHookBlocked(r: HookRunResult): void;
 /** Assert a `runHook` result allowed (did not block). */
 export declare function assertHookAllowed(r: HookRunResult): void;
+/**
+ * Assert the worker's output is a success result, and return its `value`. With a
+ * `contract`, the value is validated against the success shape (a wrong/missing
+ * field fails the assertion). A malformed or error result throws.
+ */
+export declare function assertAgentOk(output: string, contract?: OutputContract): Record<string, unknown>;
+/**
+ * Assert the worker's output is an error result, and return its `error`. The
+ * railway's error track — proves the worker reported failure with rich detail
+ * (not that it crashed or returned prose). A malformed or success result throws.
+ */
+export declare function assertAgentErr(output: string, contract?: OutputContract): Record<string, unknown>;
+/**
+ * Assert the parsed result satisfies `predicate` — the general form, for
+ * checking rich detail (e.g. `(r) => r.kind === "ok" && r.value.files.length > 0`).
+ */
+export declare function assertAgentResult(output: string, predicate: (r: ParsedAgentResult) => boolean, contract?: OutputContract): void;
+/**
+ * Did the agent invoke a tool whose name matches `name` (string = exact,
+ * RegExp = test)? The predicate behind `assertToolUsed` / `assertToolNotUsed`.
+ */
+export declare function usedTool(trace: Trace, name: string | RegExp): boolean;
+/** How many tools matching `name` the agent invoked. Behind `assertToolCount`. */
+export declare function toolCount(trace: Trace, name: string | RegExp): number;
+/**
+ * Did the `Skill` tool resolve `skill` (e.g. `"superpowers:test-driven-development"`)
+ * without error? The skill-activation predicate behind `assertSkillResolved`.
+ */
+export declare function skillResolved(trace: Trace, skill: string): boolean;
+/**
+ * Did the agent invoke a tool matching `name` whose INPUT satisfies
+ * `inputMatcher` — a tool-ARGUMENT predicate (DeepEval-style), e.g. an `Edit`
+ * that targeted the right file. The predicate behind `assertToolUsedWith`.
+ */
+export declare function toolUsedWith(trace: Trace, name: string | RegExp, inputMatcher: (input: unknown) => boolean): boolean;
+/**
+ * Does the agent's final answer (`trace.output`) contain `needle` (string =
+ * substring, RegExp = test)? The output predicate behind `assertOutputContains`
+ * — the DeepEval-style "what did the agent actually say" check.
+ */
+export declare function outputContains(trace: Trace, needle: string | RegExp): boolean;
+/**
+ * Did ANY request the model received contain `needle` — searching the system
+ * prompt and every message across all requests? The predicate that proves
+ * injected context *reached the model*: a SessionStart hook's `additionalContext`
+ * or a slash command's expansion. Harness tier only — the eval tier drives the
+ * real API, so its `modelRequests` (and this) is empty. Behind `assertRequestContains`.
+ */
+export declare function requestContains(trace: Trace, needle: string | RegExp): boolean;
+/**
+ * Did a hook matching `name` fire? Matches against both the hook label
+ * (`"PreToolUse:Edit"`) and the bare event (`"PreToolUse"`), so `/PreToolUse/`
+ * or `"PreToolUse:Edit"` both work. The predicate behind `assertHookFired`.
+ */
+export declare function hookFired(trace: Trace, name: string | RegExp): boolean;
+/** Did a hook matching `name` fire AND block (exit ≠ 0 / outcome "error")? */
+export declare function hookBlocked(trace: Trace, name: string | RegExp): boolean;
+/**
+ * Assert the agent invoked a tool whose name matches `name` (string = exact,
+ * RegExp = test) — e.g. a skill (`"Skill"`), an MCP tool (`/^mcp__github__/`), or
+ * a subagent (`"Task"`). Needs `transcript: true`. The action invariant the
+ * skill/MCP/command surfaces are really about.
+ */
+export declare function assertToolUsed(trace: Trace, name: string | RegExp): void;
+/**
+ * Assert the agent did NOT invoke any tool matching `name` — the safety negative
+ * (e.g. a destructive MCP tool was never called). "File unchanged" can pass by
+ * accident; "the tool was never used" is the real invariant. Needs `transcript`.
+ */
+export declare function assertToolNotUsed(trace: Trace, name: string | RegExp): void;
+/**
+ * Assert the `Skill` tool resolved `skill` (e.g. `"superpowers:test-driven-development"`)
+ * without error — the correct skill-activation invariant, vs. grepping the body.
+ */
+export declare function assertSkillResolved(trace: Trace, skill: string): void;
+/**
+ * Assert the agent invoked a tool matching `name` whose INPUT satisfies
+ * `inputMatcher` — a tool-ARGUMENT invariant (DeepEval-style). Asserts not just
+ * *that* a tool ran but *with what args*, e.g. an `Edit` that targeted the right
+ * file: `assertToolUsedWith(r, "Edit", (i) => (i as { file_path?: string })
+ * .file_path === "src/x.ts")`. Needs `transcript`.
+ */
+export declare function assertToolUsedWith(trace: Trace, name: string | RegExp, inputMatcher: (input: unknown) => boolean, message?: string): void;
+/** Assert the agent's final answer contains `needle` (string substring / RegExp). */
+export declare function assertOutputContains(trace: Trace, needle: string | RegExp): void;
+/**
+ * Assert some request the model received contained `needle` — the "did the
+ * injected context land" invariant (SessionStart `additionalContext`, slash
+ * command expansion). Harness tier only; a zero-request trace fails with a hint
+ * that the eval tier can't capture requests.
+ */
+export declare function assertRequestContains(trace: Trace, needle: string | RegExp): void;
+/**
+ * Assert a hook matching `name` fired (and, with `{ blocked: true }`, that it
+ * blocked) — the honest hook-firing check, recorded from the run's stream rather
+ * than inferred from a marker file the hook had to write. Needs `transcript`.
+ */
+export declare function assertHookFired(trace: Trace, name: string | RegExp, opts?: {
+    blocked?: boolean;
+}): void;
+/**
+ * Assert how many tools matching `name` the agent invoked is within bounds — a
+ * budget invariant (e.g. `{ max: 1 }` = "at most one Write", `{ exactly: 0 }` =
+ * "never touched it"). Catches runaway loops and wasted work. Needs `transcript`.
+ */
+export declare function assertToolCount(trace: Trace, name: string | RegExp, bounds: {
+    min?: number;
+    max?: number;
+    exactly?: number;
+}): void;
+/**
+ * Assert the named tools occurred in this order (as a subsequence — gaps allowed)
+ * — an ordering invariant. e.g. `["Read", "Edit"]` checks a Read came before an
+ * Edit. For a stricter rule (every Edit preceded by a Read), use `assertToolCalls`.
+ * Needs `transcript`.
+ */
+export declare function assertToolSequence(trace: Trace, names: ReadonlyArray<string | RegExp>): void;
+/**
+ * The escape hatch: assert any custom invariant over the full list of tool calls
+ * the agent made — for rules the helpers above don't express, e.g. "every Edit
+ * was preceded by a Read of that file". Needs `transcript`.
+ */
+export declare function assertToolCalls(trace: Trace, predicate: (calls: readonly ToolCall[]) => boolean, message?: string): void;
+/**
+ * Did `arm` succeed on EVERY trial for `metric` — τ-bench pass^k = 1? The
+ * reliability predicate over an eval report (vs. `improvement`, which reads the
+ * mean gap). Reads `report.arms[arm].stats[metric].passK`.
+ */
+export declare function reliable(report: EvalReport, arm: string, metric: string): boolean;
+/**
+ * Assert `arm` passed `metric` on every trial (pass^k = 1) — the reliability
+ * gate for a non-deterministic harness ("worked every time", not "on average").
+ */
+export declare function assertReliable(report: EvalReport, opts: {
+    arm: string;
+    metric: string;
+}): void;
 /** The gap on `metric` between two arms (arm − baseline). */
 export declare function improvement(report: EvalReport, baseline: string, arm: string, metric: string): number;
 /**
- * Assert `arm` beats `baseline` on `metric` by more than `by`. With `by` left at
- * 0 this just asserts a positive gap; pass the combined se to demand the gap
- * clear the noise floor.
+ * Did `arm` *significantly* beat `baseline` on `metric` — a positive gap whose
+ * two-sided Welch t-test p-value is below `alpha` (default 0.05)? The grounded
+ * upgrade over `improvement`: the noise floor is computed from the arms' spread,
+ * not hand-fed. False when either arm/metric is missing. See `src/stats.ts`.
+ */
+export declare function significantlyBeats(report: EvalReport, baseline: string, arm: string, metric: string, alpha?: number): boolean;
+/**
+ * Assert `arm` significantly beats `baseline` on `metric` (positive gap, p < α).
+ * The statistical gate for a non-deterministic A/B — "the gap clears the noise",
+ * with the noise floor computed, not supplied. The honest version of
+ * `assertImproves(..., { by: se })`.
+ */
+export declare function assertSignificant(report: EvalReport, opts: {
+    baseline: string;
+    arm: string;
+    metric: string;
+    alpha?: number;
+}): void;
+/**
+ * Assert `arm` beats `baseline` on `metric`. By default just a positive gap > `by`
+ * (pass the combined se to clear the noise floor by hand). Pass `{ significant:
+ * true }` to demand a Welch t-test at `alpha` instead — the computed noise floor.
  */
 export declare function assertImproves(report: EvalReport, opts: {
     baseline: string;
     arm: string;
     metric: string;
     by?: number;
+    significant?: boolean;
+    alpha?: number;
+}): void;
+/**
+ * Assert a skill/behaviour triggered on at least `min` (0..1) of its runs — the
+ * reliability gate for a skill's *activation* (does its description fire on the
+ * task), over a {@link TriggerRateReport} from `measureTriggerRate`.
+ */
+export declare function assertTriggerRate(report: TriggerRateReport, opts: {
+    min: number;
 }): void;
 interface MatcherOutput {
     pass: boolean;
@@ -64,5 +234,4 @@ export declare const vigilesMatchers: {
     toBlock(received: HookRunResult): MatcherOutput;
     toBeatBaseline(received: EvalReport, baseline: string, arm: string, metric: string, by?: number): MatcherOutput;
 };
-export {};
 //# sourceMappingURL=harness-assert.d.ts.map