vigiles 2.4.0 → 2.6.0

package/dist/harness-assert.js

@@ -1,20 +1,46 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.vigilesMatchers = void 0;
+exports.vigilesMatchers = exports.diffToJUnit = exports.formatBaselineDiff = exports.writeBaseline = exports.readBaseline = exports.parseBaselineFile = exports.toBaselineFile = exports.diffReports = exports.compareArms = void 0;
 exports.withHarness = withHarness;
 exports.assertCreated = assertCreated;
 exports.assertNotCreated = assertNotCreated;
 exports.assertServedTurns = assertServedTurns;
 exports.assertHookBlocked = assertHookBlocked;
 exports.assertHookAllowed = assertHookAllowed;
+exports.egressHosts = egressHosts;
+exports.assertNoEgress = assertNoEgress;
+exports.assertEgressOnly = assertEgressOnly;
+exports.assertNoWrite = assertNoWrite;
+exports.assertWroteOnly = assertWroteOnly;
+exports.assertAgentOk = assertAgentOk;
+exports.assertAgentErr = assertAgentErr;
+exports.assertAgentResult = assertAgentResult;
+exports.usedTool = usedTool;
+exports.toolCount = toolCount;
+exports.skillResolved = skillResolved;
+exports.toolUsedWith = toolUsedWith;
+exports.outputContains = outputContains;
+exports.requestContains = requestContains;
+exports.hookFired = hookFired;
+exports.hookBlocked = hookBlocked;
 exports.assertToolUsed = assertToolUsed;
 exports.assertToolNotUsed = assertToolNotUsed;
 exports.assertSkillResolved = assertSkillResolved;
+exports.assertToolUsedWith = assertToolUsedWith;
+exports.assertOutputContains = assertOutputContains;
+exports.assertRequestContains = assertRequestContains;
+exports.assertHookFired = assertHookFired;
 exports.assertToolCount = assertToolCount;
 exports.assertToolSequence = assertToolSequence;
 exports.assertToolCalls = assertToolCalls;
+exports.reliable = reliable;
+exports.assertReliable = assertReliable;
 exports.improvement = improvement;
+exports.significantlyBeats = significantlyBeats;
+exports.assertSignificant = assertSignificant;
 exports.assertImproves = assertImproves;
+exports.assertNoRegression = assertNoRegression;
+exports.assertTriggerRate = assertTriggerRate;
 /**
  * vigiles — runner-agnostic helpers for harness tests / evals.
  *
@@ -31,11 +57,27 @@ exports.assertImproves = assertImproves;
  *     vitest and jest, so the same object supports both.
  */
 const harness_test_js_1 = require("./harness-test.js");
+const agent_result_js_1 = require("./agent-result.js");
+const stats_js_1 = require("./stats.js");
+const eval_baseline_js_1 = require("./eval-baseline.js");
+// Re-export the significance primitives so the whole eval-analysis surface lives
+// behind `vigiles/harness-assert` (no separate entry point).
+var stats_js_2 = require("./stats.js");
+Object.defineProperty(exports, "compareArms", { enumerable: true, get: function () { return stats_js_2.compareArms; } });
+var eval_baseline_js_2 = require("./eval-baseline.js");
+Object.defineProperty(exports, "diffReports", { enumerable: true, get: function () { return eval_baseline_js_2.diffReports; } });
+Object.defineProperty(exports, "toBaselineFile", { enumerable: true, get: function () { return eval_baseline_js_2.toBaselineFile; } });
+Object.defineProperty(exports, "parseBaselineFile", { enumerable: true, get: function () { return eval_baseline_js_2.parseBaselineFile; } });
+Object.defineProperty(exports, "readBaseline", { enumerable: true, get: function () { return eval_baseline_js_2.readBaseline; } });
+Object.defineProperty(exports, "writeBaseline", { enumerable: true, get: function () { return eval_baseline_js_2.writeBaseline; } });
+Object.defineProperty(exports, "formatBaselineDiff", { enumerable: true, get: function () { return eval_baseline_js_2.formatBaselineDiff; } });
+Object.defineProperty(exports, "diffToJUnit", { enumerable: true, get: function () { return eval_baseline_js_2.diffToJUnit; } });
 /**
  * Run a harness test, hand the result to `fn`, and always clean up the sandbox.
  * Returns whatever `fn` returns. Use this instead of calling `cleanup()` by
  * hand — it survives assertion failures.
  */
+/* v8 ignore start -- thin wrapper over runHarnessTest (spawns the real CLI) */
 async function withHarness(spec, fn) {
     const r = await (0, harness_test_js_1.runHarnessTest)(spec);
     try {
@@ -45,6 +87,7 @@ async function withHarness(spec, fn) {
         r.cleanup();
     }
 }
+/* v8 ignore stop */
 // --- Plain throwing assertions (any runner) --------------------------------
 function fail(message) {
     throw new Error(message);
@@ -77,11 +120,164 @@ function assertHookAllowed(r) {
         fail(`expected the hook to allow, but it blocked (exit ${String(r.exitCode)}, decision ${String(r.decision)})`);
     }
 }
+const hostPort = (e) => `${e.host}:${String(e.port)}`;
+/** The `host:port` strings a run attempted, e.g. `["registry.npmjs.org:443"]`. */
+function egressHosts(r) {
+    return r.egress.map(hostPort);
+}
+/** Assert the confined run made NO network egress attempt at all. */
+function assertNoEgress(r) {
+    if (r.egress.length > 0) {
+        fail(`expected no egress, but it tried to reach: ${egressHosts(r).join(", ")}`);
+    }
+}
+/**
+ * Assert every egress attempt went to an allowed host. `allowed` matches a host
+ * (exact string or regex), or a specific `host:port`. Any attempt outside the
+ * allowlist fails, naming the offender — exfil / unexpected-registry detection.
+ */
+function assertEgressOnly(r, allowed) {
+    const ok = (e) => allowed.some((a) => typeof a === "string"
+        ? a === e.host || a === hostPort(e)
+        : a.test(e.host) || a.test(hostPort(e)));
+    const bad = r.egress.filter((e) => !ok(e));
+    if (bad.length > 0) {
+        fail(`egress to non-allowlisted host(s): ${bad.map(hostPort).join(", ")}`);
+    }
+}
+const matches = (f, p) => typeof p === "string" ? f.includes(p) : p.test(f);
+/** Assert the run wrote NO file matching `pattern` (substring or regex). */
+function assertNoWrite(r, pattern) {
+    const bad = r.filesWritten.filter((f) => matches(f, pattern));
+    if (bad.length > 0) {
+        fail(`expected no write matching ${String(pattern)}, but wrote: ${bad.join(", ")}`);
+    }
+}
+/** Assert every file the run wrote matches one of `allowed` (substring or regex). */
+function assertWroteOnly(r, allowed) {
+    const bad = r.filesWritten.filter((f) => !allowed.some((a) => matches(f, a)));
+    if (bad.length > 0) {
+        fail(`run wrote file(s) outside the allowlist: ${bad.join(", ")}`);
+    }
+}
+// --- subagent railway outcome (parse the worker's result block) ------------
+//
+// A subagent with a result() contract ends its turn with a vigiles:ok/err block.
+// These wrap parseAgentResult so a test can assert the worker's *outcome* the
+// same way it asserts a hook decision — the testing-framework payoff of the
+// railway contract: `assertAgentOk(r.output)` instead of substring-matching prose.
+/**
+ * Assert the worker's output is a success result, and return its `value`. With a
+ * `contract`, the value is validated against the success shape (a wrong/missing
+ * field fails the assertion). A malformed or error result throws.
+ */
+function assertAgentOk(output, contract) {
+    const r = (0, agent_result_js_1.parseAgentResult)(output, contract);
+    if (r.kind === "ok")
+        return r.value;
+    const why = r.kind === "err" ? "returned an error result" : r.reason;
+    return fail(`expected a success result from the subagent, but ${why}`);
+}
+/**
+ * Assert the worker's output is an error result, and return its `error`. The
+ * railway's error track — proves the worker reported failure with rich detail
+ * (not that it crashed or returned prose). A malformed or success result throws.
+ */
+function assertAgentErr(output, contract) {
+    const r = (0, agent_result_js_1.parseAgentResult)(output, contract);
+    if (r.kind === "err")
+        return r.error;
+    const why = r.kind === "ok" ? "returned a success result" : r.reason;
+    return fail(`expected an error result from the subagent, but ${why}`);
+}
+/**
+ * Assert the parsed result satisfies `predicate` — the general form, for
+ * checking rich detail (e.g. `(r) => r.kind === "ok" && r.value.files.length > 0`).
+ */
+function assertAgentResult(output, predicate, contract) {
+    const r = (0, agent_result_js_1.parseAgentResult)(output, contract);
+    if (!predicate(r)) {
+        const detail = r.kind === "malformed" ? ` (${r.reason})` : "";
+        fail(`subagent result did not satisfy the predicate: ${r.kind}${detail}`);
+    }
+}
 function nameMatches(name, pat) {
     return typeof pat === "string" ? name === pat : pat.test(name);
 }
-function toolNames(r) {
-    return r.toolCalls.map((c) => c.name).join(", ") || "none";
+function toolNames(trace) {
+    return trace.toolCalls.map((c) => c.name).join(", ") || "none";
+}
+// --- bare predicates over a Trace (the shared vocabulary, no throw) ---------
+//
+// Pure fns returning a value, so the SAME vocabulary runs in both consumers:
+// the throwing `assert*` helpers below wrap them for the testing tier, and an
+// eval `measure` reuses them directly as metrics (`measure: (t) => ({ safe:
+// !usedTool(t, /merge|delete/) })`). Never one dual-purpose function.
+/**
+ * Did the agent invoke a tool whose name matches `name` (string = exact,
+ * RegExp = test)? The predicate behind `assertToolUsed` / `assertToolNotUsed`.
+ */
+function usedTool(trace, name) {
+    return trace.toolCalls.some((c) => nameMatches(c.name, name));
+}
+/** How many tools matching `name` the agent invoked. Behind `assertToolCount`. */
+function toolCount(trace, name) {
+    return trace.toolCalls.filter((c) => nameMatches(c.name, name)).length;
+}
+/**
+ * Did the `Skill` tool resolve `skill` (e.g. `"superpowers:test-driven-development"`)
+ * without error? The skill-activation predicate behind `assertSkillResolved`.
+ */
+function skillResolved(trace, skill) {
+    const call = trace.toolCalls.find((c) => c.name === "Skill" && c.input?.skill === skill);
+    return call !== undefined && !call.isError;
+}
+/**
+ * Did the agent invoke a tool matching `name` whose INPUT satisfies
+ * `inputMatcher` — a tool-ARGUMENT predicate (DeepEval-style), e.g. an `Edit`
+ * that targeted the right file. The predicate behind `assertToolUsedWith`.
+ */
+function toolUsedWith(trace, name, inputMatcher) {
+    return trace.toolCalls.some((c) => nameMatches(c.name, name) && inputMatcher(c.input));
+}
+/**
+ * Does the agent's final answer (`trace.output`) contain `needle` (string =
+ * substring, RegExp = test)? The output predicate behind `assertOutputContains`
+ * — the DeepEval-style "what did the agent actually say" check.
+ */
+function outputContains(trace, needle) {
+    return typeof needle === "string"
+        ? trace.output.includes(needle)
+        : needle.test(trace.output);
+}
+/** All text the model received across every request (system + every message). */
+function requestText(trace) {
+    return trace.modelRequests
+        .map((r) => [r.system, ...r.messages.map((m) => m.text)].join("\n"))
+        .join("\n");
+}
+/**
+ * Did ANY request the model received contain `needle` — searching the system
+ * prompt and every message across all requests? The predicate that proves
+ * injected context *reached the model*: a SessionStart hook's `additionalContext`
+ * or a slash command's expansion. Harness tier only — the eval tier drives the
+ * real API, so its `modelRequests` (and this) is empty. Behind `assertRequestContains`.
+ */
+function requestContains(trace, needle) {
+    const text = requestText(trace);
+    return typeof needle === "string" ? text.includes(needle) : needle.test(text);
+}
+/**
+ * Did a hook matching `name` fire? Matches against both the hook label
+ * (`"PreToolUse:Edit"`) and the bare event (`"PreToolUse"`), so `/PreToolUse/`
+ * or `"PreToolUse:Edit"` both work. The predicate behind `assertHookFired`.
+ */
+function hookFired(trace, name) {
+    return trace.hooks.some((h) => nameMatches(h.name, name) || nameMatches(h.event, name));
+}
+/** Did a hook matching `name` fire AND block (exit ≠ 0 / outcome "error")? */
+function hookBlocked(trace, name) {
+    return trace.hooks.some((h) => (nameMatches(h.name, name) || nameMatches(h.event, name)) && h.blocked);
 }
 /**
  * Assert the agent invoked a tool whose name matches `name` (string = exact,
@@ -89,9 +285,9 @@ function toolNames(r) {
  * a subagent (`"Task"`). Needs `transcript: true`. The action invariant the
  * skill/MCP/command surfaces are really about.
  */
-function assertToolUsed(r, name) {
-    if (!r.toolCalls.some((c) => nameMatches(c.name, name))) {
-        fail(`expected a tool matching ${String(name)} to be used; tools used: [${toolNames(r)}] (did you set transcript:true?)`);
+function assertToolUsed(trace, name) {
+    if (!usedTool(trace, name)) {
+        fail(`expected a tool matching ${String(name)} to be used; tools used: [${toolNames(trace)}] (did you set transcript:true?)`);
     }
 }
 /**
@@ -99,8 +295,9 @@ function assertToolUsed(r, name) {
  * (e.g. a destructive MCP tool was never called). "File unchanged" can pass by
  * accident; "the tool was never used" is the real invariant. Needs `transcript`.
  */
-function assertToolNotUsed(r, name) {
-    const hit = r.toolCalls.find((c) => nameMatches(c.name, name));
+function assertToolNotUsed(trace, name) {
+    // `find` is the negative of `usedTool` and narrows the hit for the message.
+    const hit = trace.toolCalls.find((c) => nameMatches(c.name, name));
     if (hit) {
         fail(`expected no tool matching ${String(name)} to be used, but ${hit.name} was`);
     }
@@ -109,17 +306,74 @@ function assertToolNotUsed(r, name) {
  * Assert the `Skill` tool resolved `skill` (e.g. `"superpowers:test-driven-development"`)
  * without error — the correct skill-activation invariant, vs. grepping the body.
  */
-function assertSkillResolved(r, skill) {
-    const call = r.toolCalls.find((c) => c.name === "Skill" && c.input?.skill === skill);
+function assertSkillResolved(trace, skill) {
+    if (skillResolved(trace, skill))
+        return;
+    // skillResolved is false → either no matching Skill call, or it errored.
+    // Reconstruct which, for a useful message.
+    const call = trace.toolCalls.find((c) => c.name === "Skill" && c.input?.skill === skill);
     if (!call) {
-        const seen = r.toolCalls
+        const seen = trace.toolCalls
             .filter((c) => c.name === "Skill")
             .map((c) => c.input?.skill ?? "?")
             .join(", ");
         fail(`expected the Skill tool to resolve "${skill}"; Skill calls: [${seen || "none"}]`);
     }
-    if (call.isError) {
-        fail(`the Skill "${skill}" was invoked but errored: ${call.resultText.slice(0, 200)}`);
+    fail(`the Skill "${skill}" was invoked but errored: ${call.resultText.slice(0, 200)}`);
+}
+/**
+ * Assert the agent invoked a tool matching `name` whose INPUT satisfies
+ * `inputMatcher` — a tool-ARGUMENT invariant (DeepEval-style). Asserts not just
+ * *that* a tool ran but *with what args*, e.g. an `Edit` that targeted the right
+ * file: `assertToolUsedWith(r, "Edit", (i) => (i as { file_path?: string })
+ * .file_path === "src/x.ts")`. Needs `transcript`.
+ */
+function assertToolUsedWith(trace, name, inputMatcher, message) {
+    if (!toolUsedWith(trace, name, inputMatcher)) {
+        const seen = trace.toolCalls
+            .filter((c) => nameMatches(c.name, name))
+            .map((c) => JSON.stringify(c.input))
+            .join(", ");
+        fail(message ??
+            `expected a ${String(name)} call whose input matches; ${String(name)} inputs: [${seen || "none"}]`);
+    }
+}
+/** Assert the agent's final answer contains `needle` (string substring / RegExp). */
+function assertOutputContains(trace, needle) {
+    if (!outputContains(trace, needle)) {
+        const shown = trace.output.slice(0, 200) || "(empty)";
+        fail(`expected the agent's final answer to contain ${String(needle)}; got: ${shown}`);
+    }
+}
+/**
+ * Assert some request the model received contained `needle` — the "did the
+ * injected context land" invariant (SessionStart `additionalContext`, slash
+ * command expansion). Harness tier only; a zero-request trace fails with a hint
+ * that the eval tier can't capture requests.
+ */
+function assertRequestContains(trace, needle) {
+    if (!requestContains(trace, needle)) {
+        const n = trace.modelRequests.length;
+        const hint = n === 0
+            ? " (no requests captured — modelRequests is harness-tier only)"
+            : "";
+        fail(`expected a model request to contain ${String(needle)}; ${String(n)} request(s) captured${hint}`);
+    }
+}
+function hookNames(trace) {
+    return trace.hooks.map((h) => h.name).join(", ") || "none";
+}
+/**
+ * Assert a hook matching `name` fired (and, with `{ blocked: true }`, that it
+ * blocked) — the honest hook-firing check, recorded from the run's stream rather
+ * than inferred from a marker file the hook had to write. Needs `transcript`.
+ */
+function assertHookFired(trace, name, opts = {}) {
+    if (!hookFired(trace, name)) {
+        fail(`expected a hook matching ${String(name)} to fire; hooks fired: [${hookNames(trace)}] (did you set transcript:true?)`);
+    }
+    if (opts.blocked === true && !hookBlocked(trace, name)) {
+        fail(`expected a hook matching ${String(name)} to block, but none did; hooks fired: [${hookNames(trace)}]`);
     }
 }
 // --- sequence / budget invariants over the agent's actions -----------------
@@ -128,13 +382,13 @@ function assertSkillResolved(r, skill) {
  * budget invariant (e.g. `{ max: 1 }` = "at most one Write", `{ exactly: 0 }` =
  * "never touched it"). Catches runaway loops and wasted work. Needs `transcript`.
  */
-function assertToolCount(r, name, bounds) {
-    const n = r.toolCalls.filter((c) => nameMatches(c.name, name)).length;
+function assertToolCount(trace, name, bounds) {
+    const n = toolCount(trace, name);
     const ok = (bounds.exactly === undefined || n === bounds.exactly) &&
         (bounds.min === undefined || n >= bounds.min) &&
         (bounds.max === undefined || n <= bounds.max);
     if (!ok) {
-        fail(`expected count of ${String(name)} to satisfy ${JSON.stringify(bounds)}, got ${String(n)} (tools: [${toolNames(r)}])`);
+        fail(`expected count of ${String(name)} to satisfy ${JSON.stringify(bounds)}, got ${String(n)} (tools: [${toolNames(trace)}])`);
     }
 }
 /**
@@ -143,15 +397,15 @@ function assertToolCount(r, name, bounds) {
  * Edit. For a stricter rule (every Edit preceded by a Read), use `assertToolCalls`.
  * Needs `transcript`.
  */
-function assertToolSequence(r, names) {
+function assertToolSequence(trace, names) {
     let i = 0;
-    for (const c of r.toolCalls) {
+    for (const c of trace.toolCalls) {
         const want = names[i];
         if (want !== undefined && nameMatches(c.name, want))
             i++;
     }
     if (i < names.length) {
-        fail(`expected tools in order [${names.map((n) => String(n)).join(" → ")}]; got [${toolNames(r)}]`);
+        fail(`expected tools in order [${names.map((n) => String(n)).join(" → ")}]; got [${toolNames(trace)}]`);
     }
 }
 /**
@@ -159,9 +413,27 @@ function assertToolSequence(r, names) {
  * the agent made — for rules the helpers above don't express, e.g. "every Edit
  * was preceded by a Read of that file". Needs `transcript`.
  */
-function assertToolCalls(r, predicate, message = "tool-call invariant failed") {
-    if (!predicate(r.toolCalls)) {
-        fail(`${message}; tools used: [${toolNames(r)}]`);
+function assertToolCalls(trace, predicate, message = "tool-call invariant failed") {
+    if (!predicate(trace.toolCalls)) {
+        fail(`${message}; tools used: [${toolNames(trace)}]`);
+    }
+}
+/**
+ * Did `arm` succeed on EVERY trial for `metric` — τ-bench pass^k = 1? The
+ * reliability predicate over an eval report (vs. `improvement`, which reads the
+ * mean gap). Reads `report.arms[arm].stats[metric].passK`.
+ */
+function reliable(report, arm, metric) {
+    return report.arms[arm]?.stats[metric]?.passK === 1;
+}
+/**
+ * Assert `arm` passed `metric` on every trial (pass^k = 1) — the reliability
+ * gate for a non-deterministic harness ("worked every time", not "on average").
+ */
+function assertReliable(report, opts) {
+    if (!reliable(report, opts.arm, opts.metric)) {
+        const pk = report.arms[opts.arm]?.stats[opts.metric]?.passK;
+        fail(`expected ${opts.arm} to pass ${opts.metric} on every trial (pass^k=1), got pass^k=${String(pk ?? "n/a")}`);
     }
 }
 /** The gap on `metric` between two arms (arm − baseline). */
@@ -171,17 +443,78 @@ function improvement(report, baseline, arm, metric) {
     return a - b;
 }
 /**
- * Assert `arm` beats `baseline` on `metric` by more than `by`. With `by` left at
- * 0 this just asserts a positive gap; pass the combined se to demand the gap
- * clear the noise floor.
+ * Did `arm` *significantly* beat `baseline` on `metric` — a positive gap whose
+ * two-sided Welch t-test p-value is below `alpha` (default 0.05)? The grounded
+ * upgrade over `improvement`: the noise floor is computed from the arms' spread,
+ * not hand-fed. False when either arm/metric is missing. See `src/stats.ts`.
+ */
+// eslint-disable-next-line max-params -- positional predicate mirrors `improvement` + alpha
+function significantlyBeats(report, baseline, arm, metric, alpha = 0.05) {
+    const c = (0, stats_js_1.compareArms)(report, baseline, arm, metric, alpha);
+    return c !== null && c.delta > 0 && c.significant;
+}
+/**
+ * Assert `arm` significantly beats `baseline` on `metric` (positive gap, p < α).
+ * The statistical gate for a non-deterministic A/B — "the gap clears the noise",
+ * with the noise floor computed, not supplied. The honest version of
+ * `assertImproves(..., { by: se })`.
+ */
+function assertSignificant(report, opts) {
+    const c = (0, stats_js_1.compareArms)(report, opts.baseline, opts.arm, opts.metric, opts.alpha);
+    if (c === null) {
+        fail(`no data to compare ${opts.arm} vs ${opts.baseline} on ${opts.metric}`);
+    }
+    const alpha = opts.alpha ?? 0.05;
+    if (!(c.delta > 0 && c.significant)) {
+        fail(`expected ${opts.arm} to significantly beat ${opts.baseline} on ${opts.metric} (α=${String(alpha)}); Δ=${c.delta.toFixed(3)}, p=${c.pValue.toFixed(3)}`);
+    }
+}
+/**
+ * Assert `arm` beats `baseline` on `metric`. By default just a positive gap > `by`
+ * (pass the combined se to clear the noise floor by hand). Pass `{ significant:
+ * true }` to demand a Welch t-test at `alpha` instead — the computed noise floor.
  */
 function assertImproves(report, opts) {
+    if (opts.significant === true) {
+        assertSignificant(report, opts);
+        return;
+    }
     const by = opts.by ?? 0;
     const delta = improvement(report, opts.baseline, opts.arm, opts.metric);
     if (delta <= by) {
         fail(`expected ${opts.arm} to beat ${opts.baseline} on ${opts.metric} by > ${String(by)}, got ${delta.toFixed(3)}`);
     }
 }
+/**
+ * Assert the current run has not *regressed* against a committed baseline — the
+ * CI gate (Phase C). A regression is an arm×metric that moved **significantly in
+ * the bad direction** vs. `baseline` (Welch t-test, so sampling noise doesn't
+ * trip it; see `src/eval-baseline.ts`). Higher is better by default; list
+ * `lowerIsBetter` metrics (cost/latency) to flip them. Load the baseline with
+ * `readBaseline(path)` and record a fresh one with `writeBaseline(path, reports)`.
+ */
+function assertNoRegression(current, baseline, opts) {
+    const reports = Array.isArray(current)
+        ? current
+        : [current];
+    const diff = (0, eval_baseline_js_1.diffReports)(baseline, reports, opts);
+    if (!diff.passed) {
+        const detail = diff.regressions
+            .map((r) => `${r.report}/${r.arm}/${r.metric} Δ=${r.comparison.delta.toFixed(3)} p=${r.comparison.pValue.toFixed(3)}`)
+            .join("; ");
+        fail(`regression vs baseline: ${detail}`);
+    }
+}
+/**
+ * Assert a skill/behaviour triggered on at least `min` (0..1) of its runs — the
+ * reliability gate for a skill's *activation* (does its description fire on the
+ * task), over a {@link TriggerRateReport} from `measureTriggerRate`.
+ */
+function assertTriggerRate(report, opts) {
+    if (report.rate < opts.min) {
+        fail(`expected a trigger rate ≥ ${String(opts.min)}, got ${report.rate.toFixed(2)} (${String(report.n)} runs)`);
+    }
+}
 /**
  * Custom matchers compatible with both vitest and jest. Register once:
  *

package/dist/harness-test.d.ts

@@ -1,6 +1,8 @@
-import { type ModelTurn } from "./mock-model.js";
-export { scriptModel, type ModelTurn } from "./mock-model.js";
+import { type ModelTurn, type ModelRequest } from "./mock-model.js";
+import { type SandboxMode } from "./sandbox.js";
+export { scriptModel, type ModelTurn, type ModelRequest, } from "./mock-model.js";
 export { loadPlugin, resolveHarness } from "./plugin-loader.js";
+export { decideSandbox, specTrusted, sandboxAvailable, type SandboxMode, } from "./sandbox.js";
 export interface HarnessTestSpec {
     /** Fixture files to write in a fresh temp working dir (path → contents). */
     readonly files?: Record<string, string>;
@@ -37,8 +39,80 @@ export interface HarnessTestSpec {
     readonly transcript?: boolean;
     /** Per-run wall-clock timeout in ms. Default 60000. */
     readonly timeoutMs?: number;
+    /**
+     * Confinement policy for the code this run executes (`src/sandbox.ts`).
+     * Default `"auto"` is safe-by-default: an inline-only spec (you authored it)
+     * runs directly, but an external `plugin` / `pluginDir` brings in untrusted
+     * third-party hooks and is run under bubblewrap — or, if no sandbox is
+     * available, the run REFUSES rather than executing unconfined. Pass `false` to
+     * opt out and run unconfined (you audited the code, or trust the outer
+     * container); `"strict"` to force confinement even for trusted code.
+     *
+     * NOTE: confined execution is **Linux only** (bubblewrap is a Linux tool). On
+     * macOS / Windows no sandbox is available, so an untrusted run will REFUSE
+     * under `"auto"`/`"strict"` — use `sandbox: false` there if you trust the code.
+     */
+    readonly sandbox?: SandboxMode;
+}
+/**
+ * A hook invocation observed during the run, recorded (not inferred) from the
+ * `hook_response` system events the CLI emits in the stream — so a test can
+ * assert which hook fired and whether it blocked, instead of inferring it from a
+ * marker file the hook had to write.
+ */
+export interface HookFire {
+    /** The hook label, e.g. `"PreToolUse:Edit"` (`Event:Matcher`). */
+    readonly name: string;
+    /** The hook event, e.g. `"PreToolUse"`, `"PostToolUse"`, `"Stop"`. */
+    readonly event: string;
+    /** The hook process exit code (2 = block), or undefined if not reported. */
+    readonly exitCode: number | undefined;
+    /** Whether the hook blocked / errored (exit ≠ 0 or outcome "error"). */
+    readonly blocked: boolean;
+    /** What the hook printed (its block reason / diagnostic), or "". */
+    readonly output: string;
+}
+/**
+ * The observable record of ONE run — the unified shape produced by BOTH testing
+ * tiers: `runHarnessTest`'s result and `runEval`'s `measure` ctx (`eval.ts`)
+ * both satisfy it. That's what lets the bare predicates in `harness-assert.ts`
+ * (`usedTool` / `skillResolved` / `toolCount` / `toolUsedWith` / `hookFired` /
+ * `outputContains`) run over either, with the testing helpers asserting and eval
+ * measuring over the same vocabulary.
+ */
+export interface Trace {
+    /**
+     * The tools the agent invoked, each paired with its result — parsed from the
+     * transcript. Empty unless the run captured the stream (`transcript: true` on
+     * the harness tier; always on the eval tier). Lets a test assert on the
+     * agent's *actions* (skills, MCP tools, subagents) instead of grepping stdout.
+     */
+    readonly toolCalls: readonly ToolCall[];
+    /**
+     * The hooks that fired during the run, each with its decision — parsed from
+     * the CLI's `hook_response` stream events. Same capture requirement as
+     * `toolCalls` (empty without the stream). Lets a test assert hook firing
+     * honestly instead of via a marker file.
+     */
+    readonly hooks: readonly HookFire[];
+    /** The agent's final answer text (the terminal `result` event), or "". */
+    readonly output: string;
+    /**
+     * The requests the model received, captured by the scripted mock — each with
+     * its `system` prompt and `messages`, flattened to text. Lets a test assert
+     * what actually reached the model (a SessionStart hook's injected context, a
+     * slash command's expansion), not just that a hook fired. **Harness tier
+     * only**: the mock sees the requests, so this is populated by `runHarnessTest`
+     * (with or without `transcript`); the eval tier drives the real API, so its
+     * `modelRequests` is always empty.
+     */
+    readonly modelRequests: readonly ModelRequest[];
+    /** Number of model turns. */
+    readonly turns: number;
+    /** Final contents of a file under the working dir, or null if absent. */
+    file(path: string): string | null;
 }
-export interface HarnessTestResult {
+export interface HarnessTestResult extends Trace {
     readonly exitCode: number;
     readonly stdout: string;
     /** Hook block messages and diagnostics land here. */
@@ -47,14 +121,6 @@ export interface HarnessTestResult {
     readonly cwd: string;
     /** Number of model turns the agent took (mock turns served). */
     readonly turns: number;
-    /**
-     * The tools the agent invoked, each paired with its result — parsed from the
-     * transcript. Empty unless `transcript: true`. Lets a test assert on the
-     * agent's *actions* (skills, MCP tools, subagents) instead of grepping stdout.
-     */
-    readonly toolCalls: readonly ToolCall[];
-    /** Final contents of a file under the working dir, or null if absent. */
-    file(path: string): string | null;
     /** Remove the temp working dir. */
     cleanup(): void;
 }
@@ -74,11 +140,31 @@ export interface ToolCall {
  * actions, not a brittle stdout substring.
  */
 export declare function parseToolCalls(streamJson: string): ToolCall[];
+/**
+ * The terminal `result` event — present in BOTH `--output-format` shapes (a
+ * `{type:"result", …}` line in stream-json, the single object in `json`), or
+ * null. The seam for the final answer + turn count without parsing twice.
+ */
+export declare function parseResultEvent(stdout: string): Record<string, unknown> | null;
+/** The agent's final answer text from a transcript / result object, or "". */
+export declare function parseOutput(stdout: string): string;
+export declare function parseHooks(stdout: string): HookFire[];
+/**
+ * The `claude` CLI argv for a harness run (shared by the direct and sandboxed
+ * paths). `ANTHROPIC_BASE_URL` is set by the caller's environment / wrapper, not
+ * here. Pure, so the arg shape is unit-tested.
+ */
+export declare function buildClaudeArgs(spec: HarnessTestSpec, hasSettings: boolean): string[];
 /** Whether the `claude` CLI is available — harness tests need it. */
 export declare function claudeAvailable(): boolean;
 /**
  * Run the real `claude` CLI against a scripted mock model, with the given
  * fixture and settings (hooks). Deterministic — same script, same result.
+ *
+ * Safe by default: an external `plugin` / `pluginDir` brings in untrusted
+ * third-party hooks and is confined under bubblewrap (`spec.sandbox`, default
+ * `"auto"`); if no sandbox is available the run REFUSES rather than executing
+ * unconfined. See `src/sandbox.ts`.
  */
 export declare function runHarnessTest(spec: HarnessTestSpec): Promise<HarnessTestResult>;
 //# sourceMappingURL=harness-test.d.ts.map