vigiles 2.4.0 → 2.6.0

package/dist/eval-cache.js

@@ -0,0 +1,94 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.cacheKey = cacheKey;
+exports.readCache = readCache;
+exports.writeCache = writeCache;
+exports.snapshotDir = snapshotDir;
+exports.restoreDir = restoreDir;
+/**
+ * vigiles — record/replay cache for the eval tier.
+ *
+ * A real-model eval is slow and costs money, yet most iteration is on the
+ * `measure` function, not the model call. This cache records each trial's raw
+ * output AND its post-run filesystem, keyed on everything that determines the
+ * model's behaviour — `task`, the resolved fixture files + settings, model,
+ * tools, and the trial index — but DELIBERATELY NOT the `measure` function. So
+ * editing your metric and re-running re-scores the captured runs for free; the
+ * model is only re-called when a model-affecting input changes (or `cache:"off"`,
+ * which always re-samples for a fresh statistic).
+ *
+ * Restoring the post-run filesystem is what makes replay *sound*: `measure`
+ * routinely reads agent-produced files via `ctx.file()` / `ctx.sh("grep …")`, so
+ * a stdout-only cache would silently mis-score on replay. We snapshot the cwd's
+ * text files after the run and restore them into a fresh dir before re-scoring.
+ */
+const node_fs_1 = require("node:fs");
+const node_path_1 = require("node:path");
+const hash_js_1 = require("./hash.js");
+const MAX_SNAPSHOT_FILE_BYTES = 1024 * 1024;
+const SKIP_DIRS = new Set(["node_modules", ".git"]);
+/**
+ * Canonicalize a value so the key is stable regardless of object key order —
+ * recursively sorts object keys. Arrays keep order (it's significant for tools).
+ */
+function canonical(value) {
+    if (Array.isArray(value))
+        return value.map(canonical);
+    if (value !== null && typeof value === "object") {
+        const obj = value;
+        const out = {};
+        for (const k of Object.keys(obj).sort())
+            out[k] = canonical(obj[k]);
+        return out;
+    }
+    return value;
+}
+/** Deterministic content hash of the key inputs (order-independent). */
+function cacheKey(input) {
+    return (0, hash_js_1.sha256short)(JSON.stringify(canonical(input)));
+}
+/** Read a cached record by key, or null on miss / unreadable / malformed. */
+function readCache(dir, key) {
+    const path = (0, node_path_1.join)(dir, `${key}.json`);
+    if (!(0, node_fs_1.existsSync)(path))
+        return null;
+    try {
+        return JSON.parse((0, node_fs_1.readFileSync)(path, "utf-8"));
+    }
+    catch {
+        return null;
+    }
+}
+/** Write a cached record by key (creating the cache dir as needed). */
+function writeCache(dir, key, record) {
+    (0, node_fs_1.mkdirSync)(dir, { recursive: true });
+    (0, node_fs_1.writeFileSync)((0, node_path_1.join)(dir, `${key}.json`), JSON.stringify(record));
+}
+/** Snapshot the text files under `cwd` as `relativePath → contents` (bounded). */
+function snapshotDir(cwd) {
+    const out = {};
+    const walk = (dir) => {
+        for (const entry of (0, node_fs_1.readdirSync)(dir)) {
+            if (SKIP_DIRS.has(entry))
+                continue;
+            const full = (0, node_path_1.join)(dir, entry);
+            const st = (0, node_fs_1.statSync)(full);
+            if (st.isDirectory())
+                walk(full);
+            else if (st.isFile() && st.size <= MAX_SNAPSHOT_FILE_BYTES) {
+                out[(0, node_path_1.relative)(cwd, full)] = (0, node_fs_1.readFileSync)(full, "utf-8");
+            }
+        }
+    };
+    walk((0, node_path_1.resolve)(cwd));
+    return out;
+}
+/** Restore a snapshot into `cwd`, recreating directories as needed. */
+function restoreDir(cwd, files) {
+    for (const [rel, content] of Object.entries(files)) {
+        const full = (0, node_path_1.resolve)(cwd, rel);
+        (0, node_fs_1.mkdirSync)((0, node_path_1.dirname)(full), { recursive: true });
+        (0, node_fs_1.writeFileSync)(full, content);
+    }
+}
+//# sourceMappingURL=eval-cache.js.map

package/dist/eval.d.ts

@@ -1,3 +1,5 @@
+import { type ToolCall, type Trace } from "./harness-test.js";
+import { type CacheMode } from "./eval-cache.js";
 /** One arm of the comparison: fixture overrides + settings (hooks) for this arm. */
 export interface EvalArm {
     /** Files written on top of the base fixture for this arm. */
@@ -19,15 +21,31 @@ export interface EvalArm {
      */
     readonly pluginDir?: string;
 }
-/** Context handed to `measure` after a run, to compute that run's metrics. */
-export interface RunContext {
+/** Per-run resource use, parsed from the terminal `result` event (0 when absent). */
+export interface EvalUsage {
+    /** `total_cost_usd` reported by claude. */
+    readonly costUsd: number;
+    /** Wall-clock `duration_ms` of the run. */
+    readonly durationMs: number;
+    readonly inputTokens: number;
+    readonly outputTokens: number;
+}
+/**
+ * Context handed to `measure` after a run, to compute that run's metrics. It is
+ * a `Trace` (so the bare predicates `usedTool` / `skillResolved` / `toolCount` /
+ * `toolUsedWith` from `harness-assert.ts` run over it, the same as over a
+ * `runHarnessTest` result) plus the eval-only `sh` end-state probe and `usage`.
+ */
+export interface RunContext extends Trace {
     readonly cwd: string;
     readonly exitCode: number;
     readonly stdout: string;
     /** `num_turns` reported by claude, or 0. */
     readonly turns: number;
-    /** Contents of a file under the working dir, or null if absent. */
-    file(path: string): string | null;
+    /** The tools the agent invoked, each paired with its result (parsed from the stream). */
+    readonly toolCalls: readonly ToolCall[];
+    /** Cost / latency / tokens for this run (use as metrics, e.g. `{ cost: ctx.usage.costUsd }`). */
+    readonly usage: EvalUsage;
     /** Run a shell command in the working dir; returns trimmed stdout ("" on error). */
     sh(command: string): string;
 }
@@ -52,6 +70,32 @@ export interface EvalSpec<M extends Metrics> {
     readonly timeoutMs?: number;
     /** Seconds to wait between runs (avoid rate-limit bursts). Default 4. */
     readonly spacingSec?: number;
+    /**
+     * Record/replay cache mode. Default `"off"` (always re-sample). `"readwrite"`
+     * records each trial (output + post-run files) and replays it on a matching
+     * re-run — so editing `measure` re-scores for free; the model is re-called only
+     * when a model-affecting input changes. `"read"` replays but never records.
+     * The cache key excludes `measure`, so changing your metric still hits.
+     */
+    readonly cache?: CacheMode;
+    /** Where cache records live. Default `.vigiles/eval-cache` under cwd. */
+    readonly cacheDir?: string;
+    /**
+     * How many trials to run at once (across all arms × trials). Default 1 (fully
+     * sequential — the safe, no-surprise default). Raise it to cut wall-clock time;
+     * rate-limit bursts are absorbed by the retry/backoff below.
+     */
+    readonly concurrency?: number;
+    /**
+     * Abort the run once measured cost reaches this many USD. In-flight trials
+     * finish; remaining ones are skipped and `report.aborted` is set. Needs the
+     * model to report `total_cost_usd` (the eval tier does).
+     */
+    readonly maxCostUsd?: number;
+    /** Retries on a detected rate-limit/overload before giving up. Default 3. */
+    readonly rateLimitRetries?: number;
+    /** Base backoff ms (doubled each retry). Default 1000. */
+    readonly retryBackoffMs?: number;
 }
 /** Per-metric summary statistics across an arm's runs. */
 export interface MetricStat {
@@ -63,6 +107,21 @@ export interface MetricStat {
     readonly se: number;
     /** Number of runs the metric was observed in. */
     readonly n: number;
+    /**
+     * pass^k (τ-bench): 1 if the metric succeeded on EVERY trial, else 0. The
+     * reliability question a non-deterministic harness needs — "worked every time"
+     * is not "worked on average". A trial counts as a success when its value is
+     * truthy (booleans true, counts > 0), so model your metric as success/fail.
+     */
+    readonly passK: number;
+}
+/** Aggregated cost / latency / tokens across an arm's runs. */
+export interface ArmUsage {
+    readonly totalCostUsd: number;
+    readonly meanCostUsd: number;
+    readonly meanDurationMs: number;
+    readonly totalInputTokens: number;
+    readonly totalOutputTokens: number;
 }
 export interface ArmReport {
     readonly runs: number;
@@ -70,12 +129,51 @@ export interface ArmReport {
     readonly metrics: Record<string, number>;
     /** Per-metric mean / std / se / n, so an A/B gap can be read for significance. */
     readonly stats: Record<string, MetricStat>;
+    /** Cost / latency / token totals + means for this arm. */
+    readonly usage: ArmUsage;
 }
 export interface EvalReport {
     readonly name: string;
     readonly trials: number;
     readonly arms: Record<string, ArmReport>;
+    /** Total measured cost across every arm × trial (0 when usage wasn't reported). */
+    readonly totalCostUsd: number;
+    /** True if a `maxCostUsd` budget cap stopped the run before all trials ran. */
+    readonly aborted: boolean;
 }
+/** The raw output of one trial: the agent's exit code + captured streams. */
+export interface RunOut {
+    code: number;
+    stdout: string;
+    /** Captured stderr, when the runner provides it (used for rate-limit detection). */
+    stderr?: string;
+}
+/** The per-trial arguments handed to an {@link AgentRunner}. */
+export interface AgentRunArgs {
+    readonly task: string;
+    readonly cwd: string;
+    readonly model: string;
+    readonly tools: readonly string[];
+    readonly hasSettings: boolean;
+    readonly pluginDir: string | undefined;
+    readonly timeoutMs: number;
+}
+/**
+ * Runs one trial and returns its raw output. The default ({@link spawnAgent})
+ * drives the real `claude` CLI; `runEvalWith` takes one explicitly, so the eval
+ * orchestration is testable without a model (pass a fake returning canned
+ * stream-json) and a custom runtime can be plugged in.
+ */
+export type AgentRunner = (args: AgentRunArgs) => Promise<RunOut>;
+/**
+ * Run the eval: every arm × every trial against the real `claude` CLI, with the
+ * metric computed per run and aggregated per arm. Requires `claude` on PATH and
+ * working model auth (e.g. `ANTHROPIC_API_KEY`). Thin wrapper over
+ * {@link runEvalWith} with the real agent runner.
+ */
+export declare function runEval<M extends Metrics>(spec: EvalSpec<M>): Promise<EvalReport>;
+/** Parse per-run cost/latency/tokens from a stream — pure, model-free. */
+export declare function parseUsage(stdout: string): EvalUsage;
 /** Aggregate per-run metrics: mean for numbers, fraction-true (0..1) for booleans. */
 export declare function aggregate(rows: readonly Metrics[]): Record<string, number>;
 /**
@@ -84,12 +182,77 @@ export declare function aggregate(rows: readonly Metrics[]): Record<string, numb
  * a difference smaller than the combined se is not yet significant.
  */
 export declare function aggregateStats(rows: readonly Metrics[]): Record<string, MetricStat>;
+/** Aggregate per-run usage into an arm's cost / latency / token totals + means. */
+export declare function aggregateUsage(usages: readonly EvalUsage[]): ArmUsage;
+/** Whether a run's captured output looks like a rate-limit / overload. Pure. */
+export declare function isRateLimited(out: RunOut): boolean;
+/** Map `worker` over `items` with at most `concurrency` in flight, order preserved. */
+export declare function runPool<T, R>(items: readonly T[], concurrency: number, worker: (item: T) => Promise<R>): Promise<R[]>;
 /**
- * Run the eval: every arm × every trial against the real `claude` CLI, with the
- * metric computed per run and aggregated per arm. Requires `claude` on PATH and
- * working model auth (e.g. `ANTHROPIC_API_KEY`).
+ * The eval orchestration — every arm × trial via `runner`, run through the cache
+ * and a rate-limit retry, with at most `concurrency` in flight and an optional
+ * `maxCostUsd` budget cap; metric + usage computed per run and aggregated per
+ * arm. Exported with an injectable `runner` so the loop, `measure` context,
+ * caching, pooling, and aggregation are unit-testable without spawning a model
+ * (pass a fake returning canned stream-json). `runEval` is this with the real
+ * agent runner.
  */
-export declare function runEval<M extends Metrics>(spec: EvalSpec<M>): Promise<EvalReport>;
-/** Format an eval report as a compact table for the console (mean ± se). */
+export declare function runEvalWith<M extends Metrics>(spec: EvalSpec<M>, runner: AgentRunner): Promise<EvalReport>;
+/** Format an eval report as a compact table for the console (mean ± se, pass^k). */
 export declare function formatEvalReport(report: EvalReport): string;
+/**
+ * Measure how reliably a skill/behaviour *triggers*. A skill's value is its
+ * description firing on the right task — the #1 documented skill-authoring pain —
+ * and that's a property of the real model, not the wiring (which the
+ * deterministic tier already proves). Install the plugin natively (`pluginDir`),
+ * give a set of varied `prompts`, and a `fired` predicate over the run's `Trace`
+ * (reuse the bare predicates, e.g. `(t) => skillResolved(t, "x:y")`).
+ */
+export interface TriggerRateSpec {
+    /** Plugin dir installed natively (`--plugin-dir`) so its skills/commands activate. */
+    readonly pluginDir: string;
+    /** The varied prompts to test the trigger against. */
+    readonly prompts: readonly string[];
+    /** Did the behaviour fire on this run? e.g. `(t) => skillResolved(t, "x:y")`. */
+    readonly fired: (trace: Trace) => boolean;
+    /** Trials per prompt. Default 1. */
+    readonly trials?: number;
+    /** Model alias. Default "haiku". */
+    readonly model?: string;
+    /** Tools the agent may use. Default: Read Edit Write Bash Skill. */
+    readonly allowedTools?: readonly string[];
+    /** Per-run timeout ms. Default 240000. */
+    readonly timeoutMs?: number;
+    /** Seconds to wait between runs (avoid rate-limit bursts). Default 4. */
+    readonly spacingSec?: number;
+}
+/** Per-prompt trigger result: how many of its trials fired. */
+export interface PromptTriggerStat {
+    readonly prompt: string;
+    readonly fired: number;
+    readonly trials: number;
+    /** `fired / trials` (0 when no trials). */
+    readonly rate: number;
+}
+export interface TriggerRateReport {
+    /** Overall fraction of runs in which the behaviour fired (0..1). */
+    readonly rate: number;
+    /** Total runs (prompts × trials). */
+    readonly n: number;
+    readonly perPrompt: readonly PromptTriggerStat[];
+}
+/**
+ * Trigger-rate orchestration — every prompt × trial via `runner`, the `fired`
+ * predicate evaluated per run and aggregated into an overall + per-prompt rate.
+ * Exported with an injectable `runner` so the loop is unit-testable without a
+ * model; `measureTriggerRate` is this with the real agent runner.
+ */
+export declare function measureTriggerRateWith(spec: TriggerRateSpec, runner: AgentRunner): Promise<TriggerRateReport>;
+/**
+ * Measure a skill/behaviour's real trigger rate across prompts × trials against
+ * the real `claude` CLI. Requires `claude` + model auth.
+ */
+export declare function measureTriggerRate(spec: TriggerRateSpec): Promise<TriggerRateReport>;
+/** Format a trigger-rate report: overall %, then each prompt's rate. */
+export declare function formatTriggerRateReport(report: TriggerRateReport): string;
 //# sourceMappingURL=eval.d.ts.map