vigiles 2.4.0 → 2.6.0

package/dist/harness-assert.d.ts

@@ -13,9 +13,16 @@
  *     `expect(...).toHaveCreated(...)` sugar. The signature is identical for
  *     vitest and jest, so the same object supports both.
  */
-import { type HarnessTestSpec, type HarnessTestResult, type ToolCall } from "./harness-test.js";
-import type { EvalReport } from "./eval.js";
-import type { HookRunResult } from "./run-hook.js";
+import { type HarnessTestSpec, type HarnessTestResult, type ToolCall, type Trace } from "./harness-test.js";
+import type { EvalReport, TriggerRateReport } from "./eval.js";
+import type { HookRunResult, EgressAttempt } from "./run-hook.js";
+import type { OutputContract } from "./spec.js";
+import { type ParsedAgentResult } from "./agent-result.js";
+import { type BaselineFile, type DiffOptions } from "./eval-baseline.js";
+export { compareArms } from "./stats.js";
+export type { Comparison } from "./stats.js";
+export { diffReports, toBaselineFile, parseBaselineFile, readBaseline, writeBaseline, formatBaselineDiff, diffToJUnit, } from "./eval-baseline.js";
+export type { BaselineFile, BaselineDiff, MetricDiff, DiffStatus, DiffOptions, } from "./eval-baseline.js";
 /**
  * Run a harness test, hand the result to `fn`, and always clean up the sandbox.
  * Returns whatever `fn` returns. Use this instead of calling `cleanup()` by
@@ -32,30 +39,134 @@ export declare function assertServedTurns(r: HarnessTestResult, n: number): void
 export declare function assertHookBlocked(r: HookRunResult): void;
 /** Assert a `runHook` result allowed (did not block). */
 export declare function assertHookAllowed(r: HookRunResult): void;
+/** Anything carrying recorded egress attempts (a runHook recordEgress result). */
+interface HasEgress {
+    readonly egress: readonly EgressAttempt[];
+}
+/** The `host:port` strings a run attempted, e.g. `["registry.npmjs.org:443"]`. */
+export declare function egressHosts(r: HasEgress): string[];
+/** Assert the confined run made NO network egress attempt at all. */
+export declare function assertNoEgress(r: HasEgress): void;
+/**
+ * Assert every egress attempt went to an allowed host. `allowed` matches a host
+ * (exact string or regex), or a specific `host:port`. Any attempt outside the
+ * allowlist fails, naming the offender — exfil / unexpected-registry detection.
+ */
+export declare function assertEgressOnly(r: HasEgress, allowed: ReadonlyArray<string | RegExp>): void;
+/** Anything carrying recorded file writes (a confined runHook result). */
+interface HasWrites {
+    readonly filesWritten: readonly string[];
+}
+/** Assert the run wrote NO file matching `pattern` (substring or regex). */
+export declare function assertNoWrite(r: HasWrites, pattern: string | RegExp): void;
+/** Assert every file the run wrote matches one of `allowed` (substring or regex). */
+export declare function assertWroteOnly(r: HasWrites, allowed: ReadonlyArray<string | RegExp>): void;
+/**
+ * Assert the worker's output is a success result, and return its `value`. With a
+ * `contract`, the value is validated against the success shape (a wrong/missing
+ * field fails the assertion). A malformed or error result throws.
+ */
+export declare function assertAgentOk(output: string, contract?: OutputContract): Record<string, unknown>;
+/**
+ * Assert the worker's output is an error result, and return its `error`. The
+ * railway's error track — proves the worker reported failure with rich detail
+ * (not that it crashed or returned prose). A malformed or success result throws.
+ */
+export declare function assertAgentErr(output: string, contract?: OutputContract): Record<string, unknown>;
+/**
+ * Assert the parsed result satisfies `predicate` — the general form, for
+ * checking rich detail (e.g. `(r) => r.kind === "ok" && r.value.files.length > 0`).
+ */
+export declare function assertAgentResult(output: string, predicate: (r: ParsedAgentResult) => boolean, contract?: OutputContract): void;
+/**
+ * Did the agent invoke a tool whose name matches `name` (string = exact,
+ * RegExp = test)? The predicate behind `assertToolUsed` / `assertToolNotUsed`.
+ */
+export declare function usedTool(trace: Trace, name: string | RegExp): boolean;
+/** How many tools matching `name` the agent invoked. Behind `assertToolCount`. */
+export declare function toolCount(trace: Trace, name: string | RegExp): number;
+/**
+ * Did the `Skill` tool resolve `skill` (e.g. `"superpowers:test-driven-development"`)
+ * without error? The skill-activation predicate behind `assertSkillResolved`.
+ */
+export declare function skillResolved(trace: Trace, skill: string): boolean;
+/**
+ * Did the agent invoke a tool matching `name` whose INPUT satisfies
+ * `inputMatcher` — a tool-ARGUMENT predicate (DeepEval-style), e.g. an `Edit`
+ * that targeted the right file. The predicate behind `assertToolUsedWith`.
+ */
+export declare function toolUsedWith(trace: Trace, name: string | RegExp, inputMatcher: (input: unknown) => boolean): boolean;
+/**
+ * Does the agent's final answer (`trace.output`) contain `needle` (string =
+ * substring, RegExp = test)? The output predicate behind `assertOutputContains`
+ * — the DeepEval-style "what did the agent actually say" check.
+ */
+export declare function outputContains(trace: Trace, needle: string | RegExp): boolean;
+/**
+ * Did ANY request the model received contain `needle` — searching the system
+ * prompt and every message across all requests? The predicate that proves
+ * injected context *reached the model*: a SessionStart hook's `additionalContext`
+ * or a slash command's expansion. Harness tier only — the eval tier drives the
+ * real API, so its `modelRequests` (and this) is empty. Behind `assertRequestContains`.
+ */
+export declare function requestContains(trace: Trace, needle: string | RegExp): boolean;
+/**
+ * Did a hook matching `name` fire? Matches against both the hook label
+ * (`"PreToolUse:Edit"`) and the bare event (`"PreToolUse"`), so `/PreToolUse/`
+ * or `"PreToolUse:Edit"` both work. The predicate behind `assertHookFired`.
+ */
+export declare function hookFired(trace: Trace, name: string | RegExp): boolean;
+/** Did a hook matching `name` fire AND block (exit ≠ 0 / outcome "error")? */
+export declare function hookBlocked(trace: Trace, name: string | RegExp): boolean;
 /**
  * Assert the agent invoked a tool whose name matches `name` (string = exact,
  * RegExp = test) — e.g. a skill (`"Skill"`), an MCP tool (`/^mcp__github__/`), or
  * a subagent (`"Task"`). Needs `transcript: true`. The action invariant the
  * skill/MCP/command surfaces are really about.
  */
-export declare function assertToolUsed(r: HarnessTestResult, name: string | RegExp): void;
+export declare function assertToolUsed(trace: Trace, name: string | RegExp): void;
 /**
  * Assert the agent did NOT invoke any tool matching `name` — the safety negative
  * (e.g. a destructive MCP tool was never called). "File unchanged" can pass by
  * accident; "the tool was never used" is the real invariant. Needs `transcript`.
  */
-export declare function assertToolNotUsed(r: HarnessTestResult, name: string | RegExp): void;
+export declare function assertToolNotUsed(trace: Trace, name: string | RegExp): void;
 /**
  * Assert the `Skill` tool resolved `skill` (e.g. `"superpowers:test-driven-development"`)
  * without error — the correct skill-activation invariant, vs. grepping the body.
  */
-export declare function assertSkillResolved(r: HarnessTestResult, skill: string): void;
+export declare function assertSkillResolved(trace: Trace, skill: string): void;
+/**
+ * Assert the agent invoked a tool matching `name` whose INPUT satisfies
+ * `inputMatcher` — a tool-ARGUMENT invariant (DeepEval-style). Asserts not just
+ * *that* a tool ran but *with what args*, e.g. an `Edit` that targeted the right
+ * file: `assertToolUsedWith(r, "Edit", (i) => (i as { file_path?: string })
+ * .file_path === "src/x.ts")`. Needs `transcript`.
+ */
+export declare function assertToolUsedWith(trace: Trace, name: string | RegExp, inputMatcher: (input: unknown) => boolean, message?: string): void;
+/** Assert the agent's final answer contains `needle` (string substring / RegExp). */
+export declare function assertOutputContains(trace: Trace, needle: string | RegExp): void;
+/**
+ * Assert some request the model received contained `needle` — the "did the
+ * injected context land" invariant (SessionStart `additionalContext`, slash
+ * command expansion). Harness tier only; a zero-request trace fails with a hint
+ * that the eval tier can't capture requests.
+ */
+export declare function assertRequestContains(trace: Trace, needle: string | RegExp): void;
+/**
+ * Assert a hook matching `name` fired (and, with `{ blocked: true }`, that it
+ * blocked) — the honest hook-firing check, recorded from the run's stream rather
+ * than inferred from a marker file the hook had to write. Needs `transcript`.
+ */
+export declare function assertHookFired(trace: Trace, name: string | RegExp, opts?: {
+    blocked?: boolean;
+}): void;
 /**
  * Assert how many tools matching `name` the agent invoked is within bounds — a
  * budget invariant (e.g. `{ max: 1 }` = "at most one Write", `{ exactly: 0 }` =
  * "never touched it"). Catches runaway loops and wasted work. Needs `transcript`.
  */
-export declare function assertToolCount(r: HarnessTestResult, name: string | RegExp, bounds: {
+export declare function assertToolCount(trace: Trace, name: string | RegExp, bounds: {
     min?: number;
     max?: number;
     exactly?: number;
@@ -66,25 +177,77 @@ export declare function assertToolCount(r: HarnessTestResult, name: string | Reg
  * Edit. For a stricter rule (every Edit preceded by a Read), use `assertToolCalls`.
  * Needs `transcript`.
  */
-export declare function assertToolSequence(r: HarnessTestResult, names: ReadonlyArray<string | RegExp>): void;
+export declare function assertToolSequence(trace: Trace, names: ReadonlyArray<string | RegExp>): void;
 /**
  * The escape hatch: assert any custom invariant over the full list of tool calls
  * the agent made — for rules the helpers above don't express, e.g. "every Edit
  * was preceded by a Read of that file". Needs `transcript`.
  */
-export declare function assertToolCalls(r: HarnessTestResult, predicate: (calls: readonly ToolCall[]) => boolean, message?: string): void;
+export declare function assertToolCalls(trace: Trace, predicate: (calls: readonly ToolCall[]) => boolean, message?: string): void;
+/**
+ * Did `arm` succeed on EVERY trial for `metric` — τ-bench pass^k = 1? The
+ * reliability predicate over an eval report (vs. `improvement`, which reads the
+ * mean gap). Reads `report.arms[arm].stats[metric].passK`.
+ */
+export declare function reliable(report: EvalReport, arm: string, metric: string): boolean;
+/**
+ * Assert `arm` passed `metric` on every trial (pass^k = 1) — the reliability
+ * gate for a non-deterministic harness ("worked every time", not "on average").
+ */
+export declare function assertReliable(report: EvalReport, opts: {
+    arm: string;
+    metric: string;
+}): void;
 /** The gap on `metric` between two arms (arm − baseline). */
 export declare function improvement(report: EvalReport, baseline: string, arm: string, metric: string): number;
 /**
- * Assert `arm` beats `baseline` on `metric` by more than `by`. With `by` left at
- * 0 this just asserts a positive gap; pass the combined se to demand the gap
- * clear the noise floor.
+ * Did `arm` *significantly* beat `baseline` on `metric` — a positive gap whose
+ * two-sided Welch t-test p-value is below `alpha` (default 0.05)? The grounded
+ * upgrade over `improvement`: the noise floor is computed from the arms' spread,
+ * not hand-fed. False when either arm/metric is missing. See `src/stats.ts`.
+ */
+export declare function significantlyBeats(report: EvalReport, baseline: string, arm: string, metric: string, alpha?: number): boolean;
+/**
+ * Assert `arm` significantly beats `baseline` on `metric` (positive gap, p < α).
+ * The statistical gate for a non-deterministic A/B — "the gap clears the noise",
+ * with the noise floor computed, not supplied. The honest version of
+ * `assertImproves(..., { by: se })`.
+ */
+export declare function assertSignificant(report: EvalReport, opts: {
+    baseline: string;
+    arm: string;
+    metric: string;
+    alpha?: number;
+}): void;
+/**
+ * Assert `arm` beats `baseline` on `metric`. By default just a positive gap > `by`
+ * (pass the combined se to clear the noise floor by hand). Pass `{ significant:
+ * true }` to demand a Welch t-test at `alpha` instead — the computed noise floor.
  */
 export declare function assertImproves(report: EvalReport, opts: {
     baseline: string;
     arm: string;
     metric: string;
     by?: number;
+    significant?: boolean;
+    alpha?: number;
+}): void;
+/**
+ * Assert the current run has not *regressed* against a committed baseline — the
+ * CI gate (Phase C). A regression is an arm×metric that moved **significantly in
+ * the bad direction** vs. `baseline` (Welch t-test, so sampling noise doesn't
+ * trip it; see `src/eval-baseline.ts`). Higher is better by default; list
+ * `lowerIsBetter` metrics (cost/latency) to flip them. Load the baseline with
+ * `readBaseline(path)` and record a fresh one with `writeBaseline(path, reports)`.
+ */
+export declare function assertNoRegression(current: EvalReport | readonly EvalReport[], baseline: BaselineFile, opts?: DiffOptions): void;
+/**
+ * Assert a skill/behaviour triggered on at least `min` (0..1) of its runs — the
+ * reliability gate for a skill's *activation* (does its description fire on the
+ * task), over a {@link TriggerRateReport} from `measureTriggerRate`.
+ */
+export declare function assertTriggerRate(report: TriggerRateReport, opts: {
+    min: number;
 }): void;
 interface MatcherOutput {
     pass: boolean;
@@ -105,5 +268,4 @@ export declare const vigilesMatchers: {
     toBlock(received: HookRunResult): MatcherOutput;
     toBeatBaseline(received: EvalReport, baseline: string, arm: string, metric: string, by?: number): MatcherOutput;
 };
-export {};
 //# sourceMappingURL=harness-assert.d.ts.map