vigiles 2.4.0 → 2.6.0

package/dist/eval.js

@@ -1,9 +1,17 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
+exports.runEval = runEval;
+exports.parseUsage = parseUsage;
 exports.aggregate = aggregate;
 exports.aggregateStats = aggregateStats;
-exports.runEval = runEval;
+exports.aggregateUsage = aggregateUsage;
+exports.isRateLimited = isRateLimited;
+exports.runPool = runPool;
+exports.runEvalWith = runEvalWith;
 exports.formatEvalReport = formatEvalReport;
+exports.measureTriggerRateWith = measureTriggerRateWith;
+exports.measureTriggerRate = measureTriggerRate;
+exports.formatTriggerRateReport = formatTriggerRateReport;
 /**
  * vigiles — Claude Code harness *evals*.
  *
@@ -32,6 +40,8 @@ const node_fs_1 = require("node:fs");
 const node_os_1 = require("node:os");
 const node_path_1 = require("node:path");
 const plugin_loader_js_1 = require("./plugin-loader.js");
+const harness_test_js_1 = require("./harness-test.js");
+const eval_cache_js_1 = require("./eval-cache.js");
 function writeFiles(cwd, files) {
     for (const [p, content] of Object.entries(files)) {
         const full = (0, node_path_1.resolve)(cwd, p);
@@ -39,50 +49,87 @@ function writeFiles(cwd, files) {
         (0, node_fs_1.writeFileSync)(full, content);
     }
 }
-function spawnAgent(task, cwd, model, tools, hasSettings, pluginDir, timeoutMs) {
+/* v8 ignore start -- real claude subprocess; exercised by bench/, not the unit gate */
+function spawnAgent(a) {
     return new Promise((resolvePromise) => {
         const args = [
             "-p",
-            task,
+            a.task,
+            // stream-json (+ --verbose, required with -p) so the per-turn tool_use
+            // events survive into `ctx.toolCalls` — the unified Trace, same as the
+            // harness tier. The terminal `result` event still carries num_turns/output.
             "--output-format",
-            "json",
+            "stream-json",
+            "--verbose",
             "--model",
-            model,
+            a.model,
             "--permission-mode",
             "acceptEdits",
-            ...(pluginDir !== undefined ? ["--plugin-dir", (0, node_path_1.resolve)(pluginDir)] : []),
-            ...(hasSettings ? ["--settings", "settings.json"] : []),
+            ...(a.pluginDir !== undefined
+                ? ["--plugin-dir", (0, node_path_1.resolve)(a.pluginDir)]
+                : []),
+            ...(a.hasSettings ? ["--settings", "settings.json"] : []),
             "--allowedTools",
-            ...tools,
+            ...a.tools,
         ];
         const child = (0, node_child_process_1.spawn)("claude", args, {
-            cwd,
+            cwd: a.cwd,
             env: process.env,
             stdio: ["ignore", "pipe", "pipe"],
         });
         let stdout = "";
+        let stderr = "";
         child.stdout.on("data", (d) => (stdout += d.toString()));
-        const timer = setTimeout(() => child.kill("SIGKILL"), timeoutMs);
+        child.stderr.on("data", (d) => (stderr += d.toString()));
+        const timer = setTimeout(() => child.kill("SIGKILL"), a.timeoutMs);
         child.on("close", (code) => {
             clearTimeout(timer);
-            resolvePromise({ code: code ?? 0, stdout });
+            resolvePromise({ code: code ?? 0, stdout, stderr });
         });
     });
 }
+/**
+ * Run the eval: every arm × every trial against the real `claude` CLI, with the
+ * metric computed per run and aggregated per arm. Requires `claude` on PATH and
+ * working model auth (e.g. `ANTHROPIC_API_KEY`). Thin wrapper over
+ * {@link runEvalWith} with the real agent runner.
+ */
+async function runEval(spec) {
+    return runEvalWith(spec, spawnAgent);
+}
+/* v8 ignore stop */
 const sleep = (ms) => new Promise((r) => setTimeout(r, ms));
+/** Pull cost / latency / tokens out of a parsed `result` event (0 when absent). */
+function usageFrom(result) {
+    const num = (v) => (typeof v === "number" ? v : 0);
+    const usage = (result?.usage ?? {});
+    return {
+        costUsd: num(result?.total_cost_usd),
+        durationMs: num(result?.duration_ms),
+        inputTokens: num(usage.input_tokens),
+        outputTokens: num(usage.output_tokens),
+    };
+}
+/** Parse per-run cost/latency/tokens from a stream — pure, model-free. */
+function parseUsage(stdout) {
+    return usageFrom((0, harness_test_js_1.parseResultEvent)(stdout));
+}
 function makeContext(cwd, out) {
-    let turns = 0;
-    try {
-        turns = JSON.parse(out.stdout).num_turns ?? 0;
-    }
-    catch {
-        /* non-JSON output */
-    }
+    const result = (0, harness_test_js_1.parseResultEvent)(out.stdout);
+    const turns = typeof result?.num_turns === "number" ? result.num_turns : 0;
+    const output = typeof result?.result === "string" ? result.result : "";
     return {
         cwd,
         exitCode: out.code,
         stdout: out.stdout,
         turns,
+        toolCalls: (0, harness_test_js_1.parseToolCalls)(out.stdout),
+        hooks: (0, harness_test_js_1.parseHooks)(out.stdout),
+        output,
+        usage: usageFrom(result),
+        // The eval tier drives the real API (no mock between claude and the model),
+        // so the requests can't be captured here — modelRequests is harness-tier only.
+        modelRequests: [],
         file: (p) => {
             const f = (0, node_path_1.resolve)(cwd, p);
             return (0, node_fs_1.existsSync)(f) ? (0, node_fs_1.readFileSync)(f, "utf-8") : null;
@@ -143,66 +190,280 @@ function aggregateStats(rows) {
         const std = n > 1
             ? Math.sqrt(values.reduce((a, b) => a + (b - mean) ** 2, 0) / (n - 1))
             : 0;
-        out[k] = { mean, std, se: n > 0 ? std / Math.sqrt(n) : 0, n };
+        const passK = n > 0 && values.every((v) => v > 0) ? 1 : 0;
+        out[k] = { mean, std, se: n > 0 ? std / Math.sqrt(n) : 0, n, passK };
     }
     return out;
 }
+/** Aggregate per-run usage into an arm's cost / latency / token totals + means. */
+function aggregateUsage(usages) {
+    const n = usages.length;
+    const sum = (f) => usages.reduce((a, u) => a + f(u), 0);
+    const totalCostUsd = sum((u) => u.costUsd);
+    return {
+        totalCostUsd,
+        meanCostUsd: n > 0 ? totalCostUsd / n : 0,
+        meanDurationMs: n > 0 ? sum((u) => u.durationMs) / n : 0,
+        totalInputTokens: sum((u) => u.inputTokens),
+        totalOutputTokens: sum((u) => u.outputTokens),
+    };
+}
 /**
- * Run the eval: every arm × every trial against the real `claude` CLI, with the
- * metric computed per run and aggregated per arm. Requires `claude` on PATH and
- * working model auth (e.g. `ANTHROPIC_API_KEY`).
+ * Run one trial through the cache: on a hit, restore the recorded post-run
+ * filesystem into `cwd` and return the recorded output (no model call); on a
+ * miss, run the agent and (in `readwrite`) record output + cwd snapshot. The
+ * cache key excludes `measure`, so editing the metric still replays.
  */
-async function runEval(spec) {
+async function runWithCache(runArgs, keyParts, runner, cfg) {
+    if (cfg.cache === "off")
+        return runner(runArgs);
+    const key = (0, eval_cache_js_1.cacheKey)({
+        task: runArgs.task,
+        model: runArgs.model,
+        tools: runArgs.tools,
+        files: keyParts.files,
+        settings: keyParts.settings,
+        trialIndex: keyParts.trialIndex,
+    });
+    const hit = (0, eval_cache_js_1.readCache)(cfg.cacheDir, key);
+    if (hit) {
+        (0, eval_cache_js_1.restoreDir)(runArgs.cwd, hit.files);
+        return hit.out;
+    }
+    const out = await runner(runArgs);
+    if (cfg.cache === "readwrite") {
+        (0, eval_cache_js_1.writeCache)(cfg.cacheDir, key, { out, files: (0, eval_cache_js_1.snapshotDir)(runArgs.cwd) });
+    }
+    return out;
+}
+/** Execute one trial in a fresh sandbox; returns its metric row + usage. */
+async function executeTrial(spec, arm, trialIndex, runner, cfg) {
+    const cwd = (0, node_fs_1.mkdtempSync)((0, node_path_1.join)((0, node_os_1.tmpdir)(), "vigiles-eval-"));
+    try {
+        const { files, settings } = (0, plugin_loader_js_1.resolveHarness)({
+            plugin: arm.plugin,
+            settings: arm.settings,
+            files: { ...spec.fixture, ...arm.files },
+        });
+        writeFiles(cwd, files);
+        const hasSettings = settings !== undefined;
+        if (hasSettings) {
+            (0, node_fs_1.writeFileSync)((0, node_path_1.join)(cwd, "settings.json"), JSON.stringify(settings, null, 2).replaceAll("{cwd}", cwd));
+        }
+        const out = await runWithCache({
+            task: spec.task,
+            cwd,
+            model: cfg.model,
+            tools: cfg.tools,
+            hasSettings,
+            pluginDir: arm.pluginDir,
+            timeoutMs: cfg.timeoutMs,
+        }, { files, settings, trialIndex }, runner, cfg);
+        const ctx = makeContext(cwd, out);
+        return { row: spec.measure(ctx), usage: ctx.usage };
+    }
+    finally {
+        (0, node_fs_1.rmSync)(cwd, { recursive: true, force: true });
+    }
+}
+// A signal in the captured streams that the model call was rate-limited /
+// overloaded — worth a backoff + retry rather than counting as a real sample.
+const RATE_LIMIT_RE = /rate.?limit|\b429\b|overloaded|too many requests/i;
+/** Whether a run's captured output looks like a rate-limit / overload. Pure. */
+function isRateLimited(out) {
+    return RATE_LIMIT_RE.test(`${out.stderr ?? ""}\n${out.stdout}`);
+}
+/** Call `runner`, retrying with exponential backoff while it looks rate-limited. */
+async function runWithRetry(runArgs, runner, retries, baseMs) {
+    for (let attempt = 0;; attempt++) {
+        const out = await runner(runArgs);
+        if (!isRateLimited(out) || attempt >= retries)
+            return out;
+        await sleep(baseMs * 2 ** attempt);
+    }
+}
+/** Map `worker` over `items` with at most `concurrency` in flight, order preserved. */
+async function runPool(items, concurrency, worker) {
+    const results = new Array(items.length);
+    let next = 0;
+    const drain = async () => {
+        for (;;) {
+            const i = next++;
+            const item = items[i];
+            if (i >= items.length || item === undefined)
+                return;
+            results[i] = await worker(item);
+        }
+    };
+    const workers = Math.max(1, Math.min(concurrency, items.length || 1));
+    await Promise.all(Array.from({ length: workers }, drain));
+    return results;
+}
+/** Flatten arms × trials into a single work list (so concurrency spans both). */
+function buildUnits(arms, trials) {
+    const units = [];
+    for (const [armName, arm] of Object.entries(arms)) {
+        for (let t = 0; t < trials; t++)
+            units.push({ armName, arm, trialIndex: t });
+    }
+    return units;
+}
+/** Group completed (non-skipped) unit results by arm and aggregate each. */
+function aggregateArms(armNames, results) {
+    const arms = {};
+    let totalCostUsd = 0;
+    for (const armName of armNames) {
+        const done = results.filter((r) => !r.skipped && r.armName === armName);
+        const rows = done.map((d) => d.row);
+        const usage = aggregateUsage(done.map((d) => d.usage));
+        totalCostUsd += usage.totalCostUsd;
+        arms[armName] = {
+            runs: rows.length,
+            metrics: aggregate(rows),
+            stats: aggregateStats(rows),
+            usage,
+        };
+    }
+    return { arms, totalCostUsd };
+}
+/**
+ * The eval orchestration — every arm × trial via `runner`, run through the cache
+ * and a rate-limit retry, with at most `concurrency` in flight and an optional
+ * `maxCostUsd` budget cap; metric + usage computed per run and aggregated per
+ * arm. Exported with an injectable `runner` so the loop, `measure` context,
+ * caching, pooling, and aggregation are unit-testable without spawning a model
+ * (pass a fake returning canned stream-json). `runEval` is this with the real
+ * agent runner.
+ */
+async function runEvalWith(spec, runner) {
     const trials = spec.trials ?? 5;
+    const spacing = (spec.spacingSec ?? 4) * 1000;
+    const concurrency = spec.concurrency ?? 1;
+    const retries = spec.rateLimitRetries ?? 3;
+    const backoffMs = spec.retryBackoffMs ?? 1000;
+    const cfg = {
+        model: spec.model ?? "haiku",
+        tools: spec.allowedTools ?? ["Read", "Edit", "Write", "Bash"],
+        timeoutMs: spec.timeoutMs ?? 240000,
+        cache: spec.cache ?? "off",
+        cacheDir: spec.cacheDir ?? (0, node_path_1.resolve)(process.cwd(), ".vigiles", "eval-cache"),
+    };
+    const retrying = (a) => runWithRetry(a, runner, retries, backoffMs);
+    const units = buildUnits(spec.arms, trials);
+    let spent = 0;
+    let aborted = false;
+    const worker = async (unit) => {
+        if (aborted)
+            return { armName: unit.armName, skipped: true };
+        const { row, usage } = await executeTrial(spec, unit.arm, unit.trialIndex, retrying, cfg);
+        spent += usage.costUsd;
+        if (spec.maxCostUsd !== undefined && spent >= spec.maxCostUsd) {
+            aborted = true;
+        }
+        if (spacing > 0)
+            await sleep(spacing);
+        return { armName: unit.armName, skipped: false, row, usage };
+    };
+    const results = await runPool(units, concurrency, worker);
+    const { arms, totalCostUsd } = aggregateArms(Object.keys(spec.arms), results);
+    return { name: spec.name ?? "eval", trials, arms, totalCostUsd, aborted };
+}
+/** Render one metric: `name=mean±se pass^k=…` (se/pass^k shown when measured). */
+function formatMetric(name, mean, stat) {
+    const base = stat && stat.se > 0
+        ? `${name}=${mean.toFixed(2)}±${stat.se.toFixed(2)}`
+        : `${name}=${mean.toFixed(2)}`;
+    return stat && stat.n > 0 ? `${base} pass^k=${String(stat.passK)}` : base;
+}
+/** Compact tokens like `3.4k`; whole numbers under 1000 stay as-is. */
+function fmtTokens(n) {
+    return n >= 1000 ? `${(n / 1000).toFixed(1)}k` : String(n);
+}
+/** A `($0.0123 · 1.2s/run · 3.4k tok)` suffix, or "" when no usage was reported. */
+function formatUsage(u) {
+    if (u.totalCostUsd === 0 && u.totalInputTokens + u.totalOutputTokens === 0) {
+        return "";
+    }
+    const tok = fmtTokens(u.totalInputTokens + u.totalOutputTokens);
+    return `  ($${u.totalCostUsd.toFixed(4)} · ${(u.meanDurationMs / 1000).toFixed(1)}s/run · ${tok} tok)`;
+}
+/** Format an eval report as a compact table for the console (mean ± se, pass^k). */
+function formatEvalReport(report) {
+    const header = report.totalCostUsd > 0
+        ? `${report.name} (${String(report.trials)} trials/arm) — $${report.totalCostUsd.toFixed(4)} total`
+        : `${report.name} (${String(report.trials)} trials/arm)`;
+    const lines = [header];
+    for (const [arm, r] of Object.entries(report.arms)) {
+        const parts = Object.entries(r.metrics)
+            .map(([k, v]) => formatMetric(k, v, r.stats[k]))
+            .join("  ");
+        lines.push(`  ${arm.padEnd(10)} ${parts}${formatUsage(r.usage)}`);
+    }
+    return lines.join("\n");
+}
+/**
+ * Trigger-rate orchestration — every prompt × trial via `runner`, the `fired`
+ * predicate evaluated per run and aggregated into an overall + per-prompt rate.
+ * Exported with an injectable `runner` so the loop is unit-testable without a
+ * model; `measureTriggerRate` is this with the real agent runner.
+ */
+async function measureTriggerRateWith(spec, runner) {
+    const trials = spec.trials ?? 1;
     const model = spec.model ?? "haiku";
-    const tools = spec.allowedTools ?? ["Read", "Edit", "Write", "Bash"];
+    const tools = spec.allowedTools ?? ["Read", "Edit", "Write", "Bash", "Skill"];
     const timeoutMs = spec.timeoutMs ?? 240000;
     const spacing = (spec.spacingSec ?? 4) * 1000;
-    const arms = {};
-    for (const [armName, arm] of Object.entries(spec.arms)) {
-        const rows = [];
+    const perPrompt = [];
+    let firedTotal = 0;
+    let n = 0;
+    for (const prompt of spec.prompts) {
+        let fired = 0;
         for (let t = 0; t < trials; t++) {
-            const cwd = (0, node_fs_1.mkdtempSync)((0, node_path_1.join)((0, node_os_1.tmpdir)(), "vigiles-eval-"));
+            const cwd = (0, node_fs_1.mkdtempSync)((0, node_path_1.join)((0, node_os_1.tmpdir)(), "vigiles-trigger-"));
             try {
-                const { files, settings } = (0, plugin_loader_js_1.resolveHarness)({
-                    plugin: arm.plugin,
-                    settings: arm.settings,
-                    files: { ...spec.fixture, ...arm.files },
+                const out = await runner({
+                    task: prompt,
+                    cwd,
+                    model,
+                    tools,
+                    hasSettings: false,
+                    pluginDir: spec.pluginDir,
+                    timeoutMs,
                 });
-                writeFiles(cwd, files);
-                const hasSettings = settings !== undefined;
-                if (hasSettings) {
-                    (0, node_fs_1.writeFileSync)((0, node_path_1.join)(cwd, "settings.json"), JSON.stringify(settings, null, 2).replaceAll("{cwd}", cwd));
-                }
-                const out = await spawnAgent(spec.task, cwd, model, tools, hasSettings, arm.pluginDir, timeoutMs);
-                rows.push(spec.measure(makeContext(cwd, out)));
+                if (spec.fired(makeContext(cwd, out)))
+                    fired++;
             }
             finally {
                 (0, node_fs_1.rmSync)(cwd, { recursive: true, force: true });
                 await sleep(spacing);
             }
         }
-        arms[armName] = {
-            runs: rows.length,
-            metrics: aggregate(rows),
-            stats: aggregateStats(rows),
-        };
+        perPrompt.push({
+            prompt,
+            fired,
+            trials,
+            rate: trials > 0 ? fired / trials : 0,
+        });
+        firedTotal += fired;
+        n += trials;
     }
-    return { name: spec.name ?? "eval", trials, arms };
+    return { rate: n > 0 ? firedTotal / n : 0, n, perPrompt };
 }
-/** Format an eval report as a compact table for the console (mean ± se). */
-function formatEvalReport(report) {
-    const lines = [`${report.name} (${String(report.trials)} trials/arm)`];
-    for (const [arm, r] of Object.entries(report.arms)) {
-        const parts = Object.entries(r.metrics)
-            .map(([k, v]) => {
-            const se = r.stats[k]?.se ?? 0;
-            return se > 0
-                ? `${k}=${v.toFixed(2)}±${se.toFixed(2)}`
-                : `${k}=${v.toFixed(2)}`;
-        })
-            .join("  ");
-        lines.push(`  ${arm.padEnd(10)} ${parts}`);
+/* v8 ignore start -- real claude subprocess; thin wrapper over measureTriggerRateWith */
+/**
+ * Measure a skill/behaviour's real trigger rate across prompts × trials against
+ * the real `claude` CLI. Requires `claude` + model auth.
+ */
+async function measureTriggerRate(spec) {
+    return measureTriggerRateWith(spec, spawnAgent);
+}
+/* v8 ignore stop */
+/** Format a trigger-rate report: overall %, then each prompt's rate. */
+function formatTriggerRateReport(report) {
+    const pct = (report.rate * 100).toFixed(0);
+    const lines = [`trigger-rate: ${pct}% (${String(report.n)} runs)`];
+    for (const p of report.perPrompt) {
+        lines.push(`  ${p.rate.toFixed(2)}  ${p.prompt.slice(0, 60)}`);
     }
     return lines.join("\n");
 }