vigiles 2.3.0 → 2.5.0

package/dist/compile.js

@@ -16,6 +16,9 @@ exports.validateCommandRef = validateCommandRef;
 exports.validateSymbolRef = validateSymbolRef;
 exports.compileClaude = compileClaude;
 exports.compileSkill = compileSkill;
+exports.compileAgent = compileAgent;
+exports.validateRailway = validateRailway;
+exports.compileRailway = compileRailway;
 exports.checkFileHash = checkFileHash;
 exports.adoptDiff = adoptDiff;
 const node_fs_1 = require("node:fs");
@@ -641,6 +644,267 @@ function compileSkill(spec, options = {}) {
     const content = renderSkillFrontmatter(spec) + "\n\n" + sections.trim() + "\n";
     return { markdown: addHash(content, specFile), errors };
 }
+// ---------------------------------------------------------------------------
+// Compile a subagent spec → agents/<name>.md
+// ---------------------------------------------------------------------------
+// The tool contract a subagent may declare — the rails it runs on. Anything
+// else must be an MCP tool (mcp__server__tool), else it's a typo / nonexistent
+// tool the dispatched worker could never call.
+const KNOWN_AGENT_TOOLS = [
+    "Read",
+    "Write",
+    "Edit",
+    "Bash",
+    "Grep",
+    "Glob",
+    "WebSearch",
+    "WebFetch",
+    "NotebookEdit",
+    "TodoWrite",
+    "Task",
+    "Skill",
+];
+const MCP_TOOL_RE = /^mcp__[a-z0-9_-]+__[a-z0-9_-]+$/i;
+// Tools the platform never exposes to a subagent, whatever the list says — so a
+// subagent listing one is a guaranteed-dead reference only a compiler catches.
+const NEVER_AVAILABLE_TOOLS = new Set([
+    "Agent",
+    "AskUserQuestion",
+    "EnterPlanMode",
+    "ExitPlanMode",
+    "ScheduleWakeup",
+    "WaitForMcpServers",
+]);
+/** Closest known tool by edit distance (≤ 3), for a "did you mean" hint. */
+function closestTool(tool) {
+    let best = null;
+    let bestDistance = Infinity;
+    for (const known of KNOWN_AGENT_TOOLS) {
+        const d = (0, linters_js_1.editDistance)(tool.toLowerCase(), known.toLowerCase());
+        if (d < bestDistance) {
+            bestDistance = d;
+            best = known;
+        }
+    }
+    return bestDistance <= 3 ? best : null;
+}
+/** Verify a subagent's allowed-tools contract — the rails are real tools. */
+function validateAgentTools(tools) {
+    const errors = [];
+    for (const tool of tools) {
+        if (NEVER_AVAILABLE_TOOLS.has(tool)) {
+            errors.push({
+                type: "unknown-tool",
+                message: `Tool "${tool}" is never available to a subagent — remove it from the tools list.`,
+            });
+            continue;
+        }
+        if (KNOWN_AGENT_TOOLS.includes(tool))
+            continue;
+        if (MCP_TOOL_RE.test(tool))
+            continue;
+        const near = closestTool(tool);
+        const hint = near ? ` Did you mean "${near}"?` : "";
+        errors.push({
+            type: "unknown-tool",
+            message: `Unknown tool "${tool}" in agent tools — use a built-in tool (${KNOWN_AGENT_TOOLS.join(", ")}) or an MCP tool (mcp__server__tool).${hint}`,
+        });
+    }
+    return errors;
+}
+/** Build the subagent YAML frontmatter (name / description / model / tools). */
+function renderAgentFrontmatter(spec) {
+    const fm = [
+        "---",
+        "",
+        `name: ${spec.name}`,
+        `description: ${spec.description}`,
+    ];
+    if (spec.model !== undefined)
+        fm.push(`model: ${spec.model}`);
+    if (spec.tools && spec.tools.length > 0) {
+        fm.push(`tools: ${spec.tools.join(", ")}`);
+    }
+    fm.push("", "---");
+    return fm.join("\n");
+}
+/** Render the subagent's named `##` system-prompt sections (verified like CLAUDE.md). */
+function renderAgentSections(sections, basePath) {
+    const lines = [];
+    const errors = [];
+    for (const [name, content] of Object.entries(sections)) {
+        if (name.toLowerCase() === "rules") {
+            errors.push({
+                type: "reserved-section-key",
+                message: `Section key "${name}" is reserved — use the \`rules\` field instead.`,
+            });
+        }
+        const heading = name.charAt(0).toUpperCase() + name.slice(1);
+        if (typeof content === "string") {
+            errors.push(...validateSectionContent(name, content));
+            lines.push(`## ${heading}\n\n${content.trim()}`);
+        }
+        else {
+            errors.push(...validateRefs(content, basePath));
+            const rendered = content.map(renderFragment).join("");
+            errors.push(...validateSectionContent(name, rendered));
+            lines.push(`## ${heading}\n\n${rendered.trim()}`);
+        }
+    }
+    return { lines, errors };
+}
+/** Render a result-contract track shape as a compact `{ "f": type, … }` line. */
+function renderShape(shape) {
+    const fields = Object.entries(shape)
+        .map(([k, t]) => `"${k}": ${t}`)
+        .join(", ");
+    return fields ? `{ ${fields} }` : "{}";
+}
+/**
+ * Render the subagent's typed result contract — the `## Output contract` section
+ * that turns a flat worker into a railway step: it must end its turn with a
+ * `vigiles:ok` / `vigiles:err` block matching one of these shapes, so its
+ * outcome is parseable (`parseAgentResult`) and testable (`assertAgentOk`).
+ */
+function renderOutputContract(contract) {
+    return [
+        "## Output contract",
+        "",
+        "Finish your turn with exactly one fenced block — success or error — matching one of these shapes.",
+        "",
+        "On success:",
+        "",
+        "```vigiles:ok",
+        renderShape(contract.ok),
+        "```",
+        "",
+        "On error:",
+        "",
+        "```vigiles:err",
+        renderShape(contract.err),
+        "```",
+    ].join("\n");
+}
+/** Render the rules a subagent must follow as a `## Rules` section. */
+function renderAgentRules(rules) {
+    const parts = ["## Rules", ""];
+    for (const [id, rule] of Object.entries(rules)) {
+        parts.push(compileRule(id, rule), "");
+    }
+    return parts.join("\n").trim();
+}
+/**
+ * Compile an AgentSpec into a subagent markdown file with YAML frontmatter.
+ * Verifies the tool contract and the body's references; the marks the body
+ * carries (`vigiles:symbol`, file/cmd refs) are the same ones `audit` re-checks.
+ */
+function compileAgent(spec, options = {}) {
+    const basePath = options.basePath ?? process.cwd();
+    const specFile = options.specFile ?? "agent.md.spec.ts";
+    const errors = [];
+    if (!specFile.endsWith(".spec.ts")) {
+        errors.push({
+            type: "spec-name-mismatch",
+            message: `Spec file "${specFile}" must end with .spec.ts`,
+        });
+    }
+    else if (!/\.md$/i.test((0, node_path_1.basename)(specFile, ".spec.ts"))) {
+        errors.push({
+            type: "spec-name-mismatch",
+            message: `Spec file "${specFile}" should be named <output>.spec.ts (e.g., agents/reviewer.md.spec.ts)`,
+        });
+    }
+    if (spec.tools)
+        errors.push(...validateAgentTools(spec.tools));
+    if (Array.isArray(spec.body)) {
+        errors.push(...validateRefs(spec.body, basePath));
+    }
+    const sections = [];
+    if (spec.body !== undefined)
+        sections.push(renderBody(spec.body).trim());
+    if (spec.sections) {
+        const result = renderAgentSections(spec.sections, basePath);
+        sections.push(...result.lines);
+        errors.push(...result.errors);
+    }
+    if (spec.rules && Object.keys(spec.rules).length > 0) {
+        sections.push(renderAgentRules(spec.rules));
+    }
+    if (spec.output)
+        sections.push(renderOutputContract(spec.output));
+    const body = sections.join("\n\n");
+    errors.push(...checkInlineCode(body, DEFAULT_MAX_INLINE_CODE_LINES));
+    const content = renderAgentFrontmatter(spec) + "\n\n" + body.trim() + "\n";
+    return { markdown: addHash(content, specFile), errors };
+}
+/** Verify a railway: non-empty, bounded recovery, every delegate target real. */
+function validateRailway(rw, knownAgents) {
+    const errors = [];
+    if (rw.steps.length === 0) {
+        errors.push({
+            type: "invalid-railway",
+            message: `Railway "${rw.name}" has no steps.`,
+        });
+    }
+    if (rw.recover && rw.recover.max < 1) {
+        errors.push({
+            type: "invalid-railway",
+            message: `Railway "${rw.name}" recover.max must be ≥ 1 (got ${String(rw.recover.max)}).`,
+        });
+    }
+    if (knownAgents) {
+        const known = new Set(knownAgents);
+        const refs = [...rw.steps];
+        if (rw.onError)
+            refs.push(rw.onError);
+        if (rw.recover)
+            refs.push(rw.recover.step);
+        for (const s of refs) {
+            if (!known.has(s.agent)) {
+                errors.push({
+                    type: "stale-ref",
+                    message: `Railway "${rw.name}" delegates to unknown agent "${s.agent}".`,
+                    path: s.agent,
+                });
+            }
+        }
+    }
+    return errors;
+}
+/** Render the orchestrator command markdown for a railway. */
+function renderRailwayMarkdown(rw) {
+    const lines = [
+        `# Railway: ${rw.name}`,
+        "",
+        "Dispatch these subagents on the **success track**, in order. Each returns a " +
+            "result block (`vigiles:ok` / `vigiles:err`). If a step returns an error, " +
+            "stop the success track and run the error handler with that error payload.",
+        "",
+        "## Success track",
+        "",
+    ];
+    rw.steps.forEach((s, i) => {
+        const task = s.task ? ` — ${s.task}` : "";
+        lines.push(`${String(i + 1)}. **${s.agent}**${task}`);
+    });
+    if (rw.recover) {
+        lines.push("", "## Recovery", "", `If a step errors, retry it via **${rw.recover.step.agent}** up to ${String(rw.recover.max)}× before falling to the error track.`);
+    }
+    if (rw.onError) {
+        lines.push("", "## On error", "", `Run **${rw.onError.agent}** with the failing step's error payload.`);
+    }
+    return lines.join("\n");
+}
+/**
+ * Compile a railway into an orchestrator command markdown (with integrity hash),
+ * resolving every delegate target against `knownAgents` when provided.
+ */
+function compileRailway(rw, options = {}) {
+    const errors = validateRailway(rw, options.knownAgents);
+    const specFile = options.specFile ?? `${rw.name}.railway.spec.ts`;
+    const content = renderRailwayMarkdown(rw) + "\n";
+    return { markdown: addHash(content, specFile), errors };
+}
 /** Check if a generated file's hash is intact. */
 function checkFileHash(filePath) {
     if (!(0, node_fs_1.existsSync)(filePath)) {
@@ -673,6 +937,10 @@ function adoptDiff(filePath, spec, basePath) {
         const { markdown } = compileSkill(spec, { basePath, specFile: filePath });
         compiledContent = markdown;
     }
+    else if (spec._specType === "agent") {
+        const { markdown } = compileAgent(spec, { basePath, specFile: filePath });
+        compiledContent = markdown;
+    }
     // Simple line-based diff
     const currentLines = currentContent.replace(HASH_RE, "").split("\n");
     const compiledLines = (compiledContent ?? "")

package/dist/eval-cache.d.ts

@@ -0,0 +1,33 @@
+import { type SHA256Hash } from "./hash.js";
+import type { RunOut } from "./eval.js";
+/** Cache behaviour: never touch the cache / read-only / read-and-write. */
+export type CacheMode = "off" | "read" | "readwrite";
+/** Everything that determines a trial's model output (the cache key inputs). */
+export interface CacheKeyInput {
+    readonly task: string;
+    readonly model: string;
+    readonly tools: readonly string[];
+    /** The resolved fixture + arm + plugin files written before the run. */
+    readonly files: Record<string, string>;
+    /** The resolved `.claude/settings.json` for the arm (or undefined). */
+    readonly settings: unknown;
+    /** Which trial this is — distinct trials are distinct samples, cached apart. */
+    readonly trialIndex: number;
+}
+/** A recorded trial: its raw output plus the post-run cwd snapshot. */
+export interface CacheRecord {
+    readonly out: RunOut;
+    /** Text files present in the cwd after the run (relative path → contents). */
+    readonly files: Record<string, string>;
+}
+/** Deterministic content hash of the key inputs (order-independent). */
+export declare function cacheKey(input: CacheKeyInput): SHA256Hash;
+/** Read a cached record by key, or null on miss / unreadable / malformed. */
+export declare function readCache(dir: string, key: SHA256Hash): CacheRecord | null;
+/** Write a cached record by key (creating the cache dir as needed). */
+export declare function writeCache(dir: string, key: SHA256Hash, record: CacheRecord): void;
+/** Snapshot the text files under `cwd` as `relativePath → contents` (bounded). */
+export declare function snapshotDir(cwd: string): Record<string, string>;
+/** Restore a snapshot into `cwd`, recreating directories as needed. */
+export declare function restoreDir(cwd: string, files: Record<string, string>): void;
+//# sourceMappingURL=eval-cache.d.ts.map

package/dist/eval-cache.js

@@ -0,0 +1,94 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.cacheKey = cacheKey;
+exports.readCache = readCache;
+exports.writeCache = writeCache;
+exports.snapshotDir = snapshotDir;
+exports.restoreDir = restoreDir;
+/**
+ * vigiles — record/replay cache for the eval tier.
+ *
+ * A real-model eval is slow and costs money, yet most iteration is on the
+ * `measure` function, not the model call. This cache records each trial's raw
+ * output AND its post-run filesystem, keyed on everything that determines the
+ * model's behaviour — `task`, the resolved fixture files + settings, model,
+ * tools, and the trial index — but DELIBERATELY NOT the `measure` function. So
+ * editing your metric and re-running re-scores the captured runs for free; the
+ * model is only re-called when a model-affecting input changes (or `cache:"off"`,
+ * which always re-samples for a fresh statistic).
+ *
+ * Restoring the post-run filesystem is what makes replay *sound*: `measure`
+ * routinely reads agent-produced files via `ctx.file()` / `ctx.sh("grep …")`, so
+ * a stdout-only cache would silently mis-score on replay. We snapshot the cwd's
+ * text files after the run and restore them into a fresh dir before re-scoring.
+ */
+const node_fs_1 = require("node:fs");
+const node_path_1 = require("node:path");
+const hash_js_1 = require("./hash.js");
+const MAX_SNAPSHOT_FILE_BYTES = 1024 * 1024;
+const SKIP_DIRS = new Set(["node_modules", ".git"]);
+/**
+ * Canonicalize a value so the key is stable regardless of object key order —
+ * recursively sorts object keys. Arrays keep order (it's significant for tools).
+ */
+function canonical(value) {
+    if (Array.isArray(value))
+        return value.map(canonical);
+    if (value !== null && typeof value === "object") {
+        const obj = value;
+        const out = {};
+        for (const k of Object.keys(obj).sort())
+            out[k] = canonical(obj[k]);
+        return out;
+    }
+    return value;
+}
+/** Deterministic content hash of the key inputs (order-independent). */
+function cacheKey(input) {
+    return (0, hash_js_1.sha256short)(JSON.stringify(canonical(input)));
+}
+/** Read a cached record by key, or null on miss / unreadable / malformed. */
+function readCache(dir, key) {
+    const path = (0, node_path_1.join)(dir, `${key}.json`);
+    if (!(0, node_fs_1.existsSync)(path))
+        return null;
+    try {
+        return JSON.parse((0, node_fs_1.readFileSync)(path, "utf-8"));
+    }
+    catch {
+        return null;
+    }
+}
+/** Write a cached record by key (creating the cache dir as needed). */
+function writeCache(dir, key, record) {
+    (0, node_fs_1.mkdirSync)(dir, { recursive: true });
+    (0, node_fs_1.writeFileSync)((0, node_path_1.join)(dir, `${key}.json`), JSON.stringify(record));
+}
+/** Snapshot the text files under `cwd` as `relativePath → contents` (bounded). */
+function snapshotDir(cwd) {
+    const out = {};
+    const walk = (dir) => {
+        for (const entry of (0, node_fs_1.readdirSync)(dir)) {
+            if (SKIP_DIRS.has(entry))
+                continue;
+            const full = (0, node_path_1.join)(dir, entry);
+            const st = (0, node_fs_1.statSync)(full);
+            if (st.isDirectory())
+                walk(full);
+            else if (st.isFile() && st.size <= MAX_SNAPSHOT_FILE_BYTES) {
+                out[(0, node_path_1.relative)(cwd, full)] = (0, node_fs_1.readFileSync)(full, "utf-8");
+            }
+        }
+    };
+    walk((0, node_path_1.resolve)(cwd));
+    return out;
+}
+/** Restore a snapshot into `cwd`, recreating directories as needed. */
+function restoreDir(cwd, files) {
+    for (const [rel, content] of Object.entries(files)) {
+        const full = (0, node_path_1.resolve)(cwd, rel);
+        (0, node_fs_1.mkdirSync)((0, node_path_1.dirname)(full), { recursive: true });
+        (0, node_fs_1.writeFileSync)(full, content);
+    }
+}
+//# sourceMappingURL=eval-cache.js.map

package/dist/eval.d.ts

@@ -1,3 +1,5 @@
+import { type ToolCall, type Trace } from "./harness-test.js";
+import { type CacheMode } from "./eval-cache.js";
 /** One arm of the comparison: fixture overrides + settings (hooks) for this arm. */
 export interface EvalArm {
     /** Files written on top of the base fixture for this arm. */
@@ -10,16 +12,40 @@ export interface EvalArm {
      * src/plugin-loader.ts.
      */
     readonly plugin?: string;
+    /**
+     * Path to a plugin dir to install NATIVELY (`claude --plugin-dir`) for this
+     * arm, so its skills/commands/agents activate the real way — the real model
+     * can trigger a skill by its description (vs. `plugin`, which materializes a
+     * file subset that does not register skills). Point at a COMPLETE plugin. Lets
+     * an arm be "skill installed" vs "off" to measure real activation.
+     */
+    readonly pluginDir?: string;
+}
+/** Per-run resource use, parsed from the terminal `result` event (0 when absent). */
+export interface EvalUsage {
+    /** `total_cost_usd` reported by claude. */
+    readonly costUsd: number;
+    /** Wall-clock `duration_ms` of the run. */
+    readonly durationMs: number;
+    readonly inputTokens: number;
+    readonly outputTokens: number;
 }
-/** Context handed to `measure` after a run, to compute that run's metrics. */
-export interface RunContext {
+/**
+ * Context handed to `measure` after a run, to compute that run's metrics. It is
+ * a `Trace` (so the bare predicates `usedTool` / `skillResolved` / `toolCount` /
+ * `toolUsedWith` from `harness-assert.ts` run over it, the same as over a
+ * `runHarnessTest` result) plus the eval-only `sh` end-state probe and `usage`.
+ */
+export interface RunContext extends Trace {
     readonly cwd: string;
     readonly exitCode: number;
     readonly stdout: string;
     /** `num_turns` reported by claude, or 0. */
     readonly turns: number;
-    /** Contents of a file under the working dir, or null if absent. */
-    file(path: string): string | null;
+    /** The tools the agent invoked, each paired with its result (parsed from the stream). */
+    readonly toolCalls: readonly ToolCall[];
+    /** Cost / latency / tokens for this run (use as metrics, e.g. `{ cost: ctx.usage.costUsd }`). */
+    readonly usage: EvalUsage;
     /** Run a shell command in the working dir; returns trimmed stdout ("" on error). */
     sh(command: string): string;
 }
@@ -44,6 +70,32 @@ export interface EvalSpec<M extends Metrics> {
     readonly timeoutMs?: number;
     /** Seconds to wait between runs (avoid rate-limit bursts). Default 4. */
     readonly spacingSec?: number;
+    /**
+     * Record/replay cache mode. Default `"off"` (always re-sample). `"readwrite"`
+     * records each trial (output + post-run files) and replays it on a matching
+     * re-run — so editing `measure` re-scores for free; the model is re-called only
+     * when a model-affecting input changes. `"read"` replays but never records.
+     * The cache key excludes `measure`, so changing your metric still hits.
+     */
+    readonly cache?: CacheMode;
+    /** Where cache records live. Default `.vigiles/eval-cache` under cwd. */
+    readonly cacheDir?: string;
+    /**
+     * How many trials to run at once (across all arms × trials). Default 1 (fully
+     * sequential — the safe, no-surprise default). Raise it to cut wall-clock time;
+     * rate-limit bursts are absorbed by the retry/backoff below.
+     */
+    readonly concurrency?: number;
+    /**
+     * Abort the run once measured cost reaches this many USD. In-flight trials
+     * finish; remaining ones are skipped and `report.aborted` is set. Needs the
+     * model to report `total_cost_usd` (the eval tier does).
+     */
+    readonly maxCostUsd?: number;
+    /** Retries on a detected rate-limit/overload before giving up. Default 3. */
+    readonly rateLimitRetries?: number;
+    /** Base backoff ms (doubled each retry). Default 1000. */
+    readonly retryBackoffMs?: number;
 }
 /** Per-metric summary statistics across an arm's runs. */
 export interface MetricStat {
@@ -55,6 +107,21 @@ export interface MetricStat {
     readonly se: number;
     /** Number of runs the metric was observed in. */
     readonly n: number;
+    /**
+     * pass^k (τ-bench): 1 if the metric succeeded on EVERY trial, else 0. The
+     * reliability question a non-deterministic harness needs — "worked every time"
+     * is not "worked on average". A trial counts as a success when its value is
+     * truthy (booleans true, counts > 0), so model your metric as success/fail.
+     */
+    readonly passK: number;
+}
+/** Aggregated cost / latency / tokens across an arm's runs. */
+export interface ArmUsage {
+    readonly totalCostUsd: number;
+    readonly meanCostUsd: number;
+    readonly meanDurationMs: number;
+    readonly totalInputTokens: number;
+    readonly totalOutputTokens: number;
 }
 export interface ArmReport {
     readonly runs: number;
@@ -62,12 +129,51 @@ export interface ArmReport {
     readonly metrics: Record<string, number>;
     /** Per-metric mean / std / se / n, so an A/B gap can be read for significance. */
     readonly stats: Record<string, MetricStat>;
+    /** Cost / latency / token totals + means for this arm. */
+    readonly usage: ArmUsage;
 }
 export interface EvalReport {
     readonly name: string;
     readonly trials: number;
     readonly arms: Record<string, ArmReport>;
+    /** Total measured cost across every arm × trial (0 when usage wasn't reported). */
+    readonly totalCostUsd: number;
+    /** True if a `maxCostUsd` budget cap stopped the run before all trials ran. */
+    readonly aborted: boolean;
+}
+/** The raw output of one trial: the agent's exit code + captured streams. */
+export interface RunOut {
+    code: number;
+    stdout: string;
+    /** Captured stderr, when the runner provides it (used for rate-limit detection). */
+    stderr?: string;
+}
+/** The per-trial arguments handed to an {@link AgentRunner}. */
+export interface AgentRunArgs {
+    readonly task: string;
+    readonly cwd: string;
+    readonly model: string;
+    readonly tools: readonly string[];
+    readonly hasSettings: boolean;
+    readonly pluginDir: string | undefined;
+    readonly timeoutMs: number;
 }
+/**
+ * Runs one trial and returns its raw output. The default ({@link spawnAgent})
+ * drives the real `claude` CLI; `runEvalWith` takes one explicitly, so the eval
+ * orchestration is testable without a model (pass a fake returning canned
+ * stream-json) and a custom runtime can be plugged in.
+ */
+export type AgentRunner = (args: AgentRunArgs) => Promise<RunOut>;
+/**
+ * Run the eval: every arm × every trial against the real `claude` CLI, with the
+ * metric computed per run and aggregated per arm. Requires `claude` on PATH and
+ * working model auth (e.g. `ANTHROPIC_API_KEY`). Thin wrapper over
+ * {@link runEvalWith} with the real agent runner.
+ */
+export declare function runEval<M extends Metrics>(spec: EvalSpec<M>): Promise<EvalReport>;
+/** Parse per-run cost/latency/tokens from a stream — pure, model-free. */
+export declare function parseUsage(stdout: string): EvalUsage;
 /** Aggregate per-run metrics: mean for numbers, fraction-true (0..1) for booleans. */
 export declare function aggregate(rows: readonly Metrics[]): Record<string, number>;
 /**
@@ -76,12 +182,77 @@ export declare function aggregate(rows: readonly Metrics[]): Record<string, numb
  * a difference smaller than the combined se is not yet significant.
  */
 export declare function aggregateStats(rows: readonly Metrics[]): Record<string, MetricStat>;
+/** Aggregate per-run usage into an arm's cost / latency / token totals + means. */
+export declare function aggregateUsage(usages: readonly EvalUsage[]): ArmUsage;
+/** Whether a run's captured output looks like a rate-limit / overload. Pure. */
+export declare function isRateLimited(out: RunOut): boolean;
+/** Map `worker` over `items` with at most `concurrency` in flight, order preserved. */
+export declare function runPool<T, R>(items: readonly T[], concurrency: number, worker: (item: T) => Promise<R>): Promise<R[]>;
 /**
- * Run the eval: every arm × every trial against the real `claude` CLI, with the
- * metric computed per run and aggregated per arm. Requires `claude` on PATH and
- * working model auth (e.g. `ANTHROPIC_API_KEY`).
+ * The eval orchestration — every arm × trial via `runner`, run through the cache
+ * and a rate-limit retry, with at most `concurrency` in flight and an optional
+ * `maxCostUsd` budget cap; metric + usage computed per run and aggregated per
+ * arm. Exported with an injectable `runner` so the loop, `measure` context,
+ * caching, pooling, and aggregation are unit-testable without spawning a model
+ * (pass a fake returning canned stream-json). `runEval` is this with the real
+ * agent runner.
  */
-export declare function runEval<M extends Metrics>(spec: EvalSpec<M>): Promise<EvalReport>;
-/** Format an eval report as a compact table for the console (mean ± se). */
+export declare function runEvalWith<M extends Metrics>(spec: EvalSpec<M>, runner: AgentRunner): Promise<EvalReport>;
+/** Format an eval report as a compact table for the console (mean ± se, pass^k). */
 export declare function formatEvalReport(report: EvalReport): string;
+/**
+ * Measure how reliably a skill/behaviour *triggers*. A skill's value is its
+ * description firing on the right task — the #1 documented skill-authoring pain —
+ * and that's a property of the real model, not the wiring (which the
+ * deterministic tier already proves). Install the plugin natively (`pluginDir`),
+ * give a set of varied `prompts`, and a `fired` predicate over the run's `Trace`
+ * (reuse the bare predicates, e.g. `(t) => skillResolved(t, "x:y")`).
+ */
+export interface TriggerRateSpec {
+    /** Plugin dir installed natively (`--plugin-dir`) so its skills/commands activate. */
+    readonly pluginDir: string;
+    /** The varied prompts to test the trigger against. */
+    readonly prompts: readonly string[];
+    /** Did the behaviour fire on this run? e.g. `(t) => skillResolved(t, "x:y")`. */
+    readonly fired: (trace: Trace) => boolean;
+    /** Trials per prompt. Default 1. */
+    readonly trials?: number;
+    /** Model alias. Default "haiku". */
+    readonly model?: string;
+    /** Tools the agent may use. Default: Read Edit Write Bash Skill. */
+    readonly allowedTools?: readonly string[];
+    /** Per-run timeout ms. Default 240000. */
+    readonly timeoutMs?: number;
+    /** Seconds to wait between runs (avoid rate-limit bursts). Default 4. */
+    readonly spacingSec?: number;
+}
+/** Per-prompt trigger result: how many of its trials fired. */
+export interface PromptTriggerStat {
+    readonly prompt: string;
+    readonly fired: number;
+    readonly trials: number;
+    /** `fired / trials` (0 when no trials). */
+    readonly rate: number;
+}
+export interface TriggerRateReport {
+    /** Overall fraction of runs in which the behaviour fired (0..1). */
+    readonly rate: number;
+    /** Total runs (prompts × trials). */
+    readonly n: number;
+    readonly perPrompt: readonly PromptTriggerStat[];
+}
+/**
+ * Trigger-rate orchestration — every prompt × trial via `runner`, the `fired`
+ * predicate evaluated per run and aggregated into an overall + per-prompt rate.
+ * Exported with an injectable `runner` so the loop is unit-testable without a
+ * model; `measureTriggerRate` is this with the real agent runner.
+ */
+export declare function measureTriggerRateWith(spec: TriggerRateSpec, runner: AgentRunner): Promise<TriggerRateReport>;
+/**
+ * Measure a skill/behaviour's real trigger rate across prompts × trials against
+ * the real `claude` CLI. Requires `claude` + model auth.
+ */
+export declare function measureTriggerRate(spec: TriggerRateSpec): Promise<TriggerRateReport>;
+/** Format a trigger-rate report: overall %, then each prompt's rate. */
+export declare function formatTriggerRateReport(report: TriggerRateReport): string;
 //# sourceMappingURL=eval.d.ts.map