vigiles 2.5.0 → 3.0.0

package/dist/adapters/claude-code/eval-baseline.js

@@ -0,0 +1,173 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.BASELINE_VERSION = void 0;
+exports.toBaselineFile = toBaselineFile;
+exports.parseBaselineFile = parseBaselineFile;
+exports.diffReports = diffReports;
+exports.formatBaselineDiff = formatBaselineDiff;
+exports.diffToJUnit = diffToJUnit;
+exports.readBaseline = readBaseline;
+exports.writeBaseline = writeBaseline;
+/**
+ * vigiles — eval regression gating (Phase C).
+ *
+ * The eval tier reports mean ± se per arm; `src/stats.ts` turns a gap into a
+ * significance verdict. This module points that machinery at a *committed
+ * baseline*: record one run's `EvalReport`s to `.vigiles/eval-baseline.json`,
+ * then on a later run flag any arm×metric that moved **significantly in the bad
+ * direction** vs. that baseline. "jest snapshots for agent behaviour, with a real
+ * noise floor" — a bare pass-rate can't tell a true regression from sampling
+ * noise, but a Welch t-test over the two runs' summary stats can.
+ *
+ * Pure + model-free (the diff/serialize/JUnit are fully unit-tested); the only
+ * side effects are the two small fs helpers (`readBaseline` / `writeBaseline`).
+ * Reuses `welchTTest` from `src/stats.ts` — the current run is the "arm", the
+ * baseline is the "baseline", so `delta = current − baseline`.
+ */
+const node_fs_1 = require("node:fs");
+const node_path_1 = require("node:path");
+const stats_js_1 = require("./stats.js");
+/** Bumped only on a breaking change to the on-disk shape. */
+exports.BASELINE_VERSION = 1;
+/** Build a `BaselineFile` envelope from a run's reports (keyed by name). */
+function toBaselineFile(reports, recordedAt = new Date().toISOString()) {
+    const byName = {};
+    for (const r of reports)
+        byName[r.name] = r;
+    return { version: exports.BASELINE_VERSION, recordedAt, reports: byName };
+}
+/** Parse + validate a baseline JSON string (throws on a bad version/shape). */
+function parseBaselineFile(json) {
+    const data = JSON.parse(json);
+    if (typeof data !== "object" || data === null) {
+        throw new Error("baseline: expected a JSON object");
+    }
+    const obj = data;
+    if (obj.version !== exports.BASELINE_VERSION) {
+        throw new Error(`baseline: unsupported version ${String(obj.version)} (expected ${String(exports.BASELINE_VERSION)})`);
+    }
+    if (typeof obj.reports !== "object" || obj.reports === null) {
+        throw new Error("baseline: missing `reports`");
+    }
+    return {
+        version: exports.BASELINE_VERSION,
+        recordedAt: typeof obj.recordedAt === "string" ? obj.recordedAt : "",
+        reports: obj.reports,
+    };
+}
+/** Classify one comparison given the metric's direction. */
+function classify(cmp, lowerIsBetter) {
+    if (!cmp.significant || cmp.delta === 0)
+        return "unchanged";
+    const improved = lowerIsBetter ? cmp.delta < 0 : cmp.delta > 0;
+    return improved ? "improved" : "regressed";
+}
+/** Append a diff entry for every arm×metric common to both reports. */
+function collectReportDiffs(baseline, current, cfg, out) {
+    for (const [arm, curArm] of Object.entries(current.arms)) {
+        const baseArm = baseline.arms[arm];
+        if (!baseArm)
+            continue;
+        for (const [metric, curStat] of Object.entries(curArm.stats)) {
+            const baseStat = baseArm.stats[metric];
+            if (!baseStat)
+                continue;
+            const comparison = (0, stats_js_1.welchTTest)(curStat, baseStat, cfg.alpha);
+            out.push({
+                report: current.name,
+                arm,
+                metric,
+                status: classify(comparison, cfg.lower.has(metric)),
+                comparison,
+            });
+        }
+    }
+}
+/**
+ * Diff a current run against a committed baseline. Compares every arm×metric
+ * present in both (by report name), flagging a *significant* move in the
+ * undesired direction as a regression. Metrics absent from one side are skipped
+ * (a new arm/metric is not a regression).
+ */
+function diffReports(baseline, current, opts = {}) {
+    const cfg = {
+        alpha: opts.alpha ?? 0.05,
+        lower: new Set(opts.lowerIsBetter ?? []),
+    };
+    const entries = [];
+    for (const cur of current) {
+        const base = baseline.reports[cur.name];
+        if (base)
+            collectReportDiffs(base, cur, cfg, entries);
+    }
+    const regressions = entries.filter((e) => e.status === "regressed");
+    const improvements = entries.filter((e) => e.status === "improved");
+    return {
+        entries,
+        regressions,
+        improvements,
+        passed: regressions.length === 0,
+    };
+}
+const STATUS_MARK = {
+    regressed: "✗",
+    improved: "✓",
+    unchanged: "·",
+};
+function formatDelta(c) {
+    const sign = c.delta >= 0 ? "+" : "";
+    return `Δ=${sign}${c.delta.toFixed(3)} p=${c.pValue.toFixed(3)}`;
+}
+/** Format a baseline diff as a compact console report. */
+function formatBaselineDiff(diff) {
+    const head = diff.passed
+        ? "baseline OK — no significant regressions"
+        : `baseline FAIL — ${String(diff.regressions.length)} regression(s)`;
+    const lines = [head];
+    for (const e of diff.entries) {
+        lines.push(`  ${STATUS_MARK[e.status]} ${e.report}/${e.arm}/${e.metric}  ${formatDelta(e.comparison)}`);
+    }
+    return lines.join("\n");
+}
+function xmlEscape(s) {
+    return s
+        .replaceAll("&", "&amp;")
+        .replaceAll("<", "&lt;")
+        .replaceAll(">", "&gt;")
+        .replaceAll('"', "&quot;");
+}
+function junitCase(e) {
+    const name = xmlEscape(`${e.report}.${e.arm}.${e.metric}`);
+    const open = `  <testcase classname="${xmlEscape(e.report)}" name="${name}">`;
+    if (e.status !== "regressed")
+        return `${open}</testcase>`;
+    const msg = xmlEscape(`regression: ${formatDelta(e.comparison)}`);
+    return `${open}\n    <failure message="${msg}"/>\n  </testcase>`;
+}
+/**
+ * Render a baseline diff as JUnit XML — one `<testcase>` per arm×metric, a
+ * `<failure>` for each regression. Lets a CI provider show eval regressions in
+ * the same place as unit-test failures.
+ */
+function diffToJUnit(diff) {
+    const cases = diff.entries.map(junitCase).join("\n");
+    return [
+        '<?xml version="1.0" encoding="UTF-8"?>',
+        `<testsuite name="vigiles-eval" tests="${String(diff.entries.length)}" failures="${String(diff.regressions.length)}">`,
+        cases,
+        "</testsuite>",
+        "",
+    ].join("\n");
+}
+/** Read + parse a baseline file, or null if it doesn't exist yet. */
+function readBaseline(path) {
+    if (!(0, node_fs_1.existsSync)(path))
+        return null;
+    return parseBaselineFile((0, node_fs_1.readFileSync)(path, "utf-8"));
+}
+/** Write reports as the committed baseline (pretty JSON, parent dirs created). */
+function writeBaseline(path, reports) {
+    (0, node_fs_1.mkdirSync)((0, node_path_1.dirname)(path), { recursive: true });
+    (0, node_fs_1.writeFileSync)(path, JSON.stringify(toBaselineFile(reports), null, 2) + "\n");
+}
+//# sourceMappingURL=eval-baseline.js.map

package/dist/{eval-cache.d.ts → adapters/claude-code/eval-cache.d.ts}

@@ -1,4 +1,4 @@
-import { type SHA256Hash } from "./hash.js";
+import { type SHA256Hash } from "../../core/hash.js";
 import type { RunOut } from "./eval.js";
 /** Cache behaviour: never touch the cache / read-only / read-and-write. */
 export type CacheMode = "off" | "read" | "readwrite";

package/dist/{eval-cache.js → adapters/claude-code/eval-cache.js}

@@ -24,7 +24,7 @@ exports.restoreDir = restoreDir;
  */
 const node_fs_1 = require("node:fs");
 const node_path_1 = require("node:path");
-const hash_js_1 = require("./hash.js");
+const hash_js_1 = require("../../core/hash.js");
 const MAX_SNAPSHOT_FILE_BYTES = 1024 * 1024;
 const SKIP_DIRS = new Set(["node_modules", ".git"]);
 /**

package/dist/{eval.d.ts → adapters/claude-code/eval.d.ts}

@@ -213,6 +213,14 @@ export interface TriggerRateSpec {
     readonly pluginDir: string;
     /** The varied prompts to test the trigger against. */
     readonly prompts: readonly string[];
+    /**
+     * Optional *irrelevant* prompts the skill should **not** fire on — the
+     * precision side of triggering. Firing on these is a false positive (a skill
+     * whose description is too broad and hijacks unrelated work). When given, the
+     * report adds {@link TriggerRateReport.falsePositiveRate} and
+     * {@link TriggerRateReport.precision}; `prompts` alone measures recall only.
+     */
+    readonly irrelevantPrompts?: readonly string[];
     /** Did the behaviour fire on this run? e.g. `(t) => skillResolved(t, "x:y")`. */
     readonly fired: (trace: Trace) => boolean;
     /** Trials per prompt. Default 1. */
@@ -235,17 +243,33 @@ export interface PromptTriggerStat {
     readonly rate: number;
 }
 export interface TriggerRateReport {
-    /** Overall fraction of runs in which the behaviour fired (0..1). */
+    /** Overall fraction of relevant runs in which the behaviour fired (recall, 0..1). */
     readonly rate: number;
-    /** Total runs (prompts × trials). */
+    /** Total relevant runs (prompts × trials). */
     readonly n: number;
     readonly perPrompt: readonly PromptTriggerStat[];
+    /**
+     * Fraction of *irrelevant* runs that wrongly fired (lower is better). Present
+     * only when {@link TriggerRateSpec.irrelevantPrompts} was given.
+     */
+    readonly falsePositiveRate?: number;
+    /**
+     * `relevantFired / (relevantFired + irrelevantFired)` — of all firings, the
+     * share on the right prompts. Present only when irrelevant prompts were given
+     * AND something fired (undefined when nothing fired at all). Pairs with `rate`
+     * (recall) to catch a skill that fires on everything _or_ nothing.
+     */
+    readonly precision?: number;
+    /** Per-prompt stats for the irrelevant set. Present with irrelevant prompts. */
+    readonly perIrrelevant?: readonly PromptTriggerStat[];
 }
 /**
  * Trigger-rate orchestration — every prompt × trial via `runner`, the `fired`
  * predicate evaluated per run and aggregated into an overall + per-prompt rate.
- * Exported with an injectable `runner` so the loop is unit-testable without a
- * model; `measureTriggerRate` is this with the real agent runner.
+ * With `irrelevantPrompts`, also runs the precision side (firing there is a false
+ * positive) and adds `falsePositiveRate` + `precision`. Exported with an
+ * injectable `runner` so the loop is unit-testable without a model;
+ * `measureTriggerRate` is this with the real agent runner.
  */
 export declare function measureTriggerRateWith(spec: TriggerRateSpec, runner: AgentRunner): Promise<TriggerRateReport>;
 /**

package/dist/{eval.js → adapters/claude-code/eval.js}

@@ -40,6 +40,7 @@ const node_fs_1 = require("node:fs");
 const node_os_1 = require("node:os");
 const node_path_1 = require("node:path");
 const plugin_loader_js_1 = require("./plugin-loader.js");
+const runtime_js_1 = require("./runtime.js");
 const harness_test_js_1 = require("./harness-test.js");
 const eval_cache_js_1 = require("./eval-cache.js");
 function writeFiles(cwd, files) {
@@ -72,7 +73,7 @@ function spawnAgent(a) {
             "--allowedTools",
             ...a.tools,
         ];
-        const child = (0, node_child_process_1.spawn)("claude", args, {
+        const child = (0, node_child_process_1.spawn)(runtime_js_1.claudeCodeRuntime.agentBinary, args, {
             cwd: a.cwd,
             env: process.env,
             stdio: ["ignore", "pipe", "pipe"],
@@ -401,53 +402,78 @@ function formatEvalReport(report) {
     }
     return lines.join("\n");
 }
-/**
- * Trigger-rate orchestration — every prompt × trial via `runner`, the `fired`
- * predicate evaluated per run and aggregated into an overall + per-prompt rate.
- * Exported with an injectable `runner` so the loop is unit-testable without a
- * model; `measureTriggerRate` is this with the real agent runner.
- */
-async function measureTriggerRateWith(spec, runner) {
-    const trials = spec.trials ?? 1;
-    const model = spec.model ?? "haiku";
-    const tools = spec.allowedTools ?? ["Read", "Edit", "Write", "Bash", "Skill"];
-    const timeoutMs = spec.timeoutMs ?? 240000;
-    const spacing = (spec.spacingSec ?? 4) * 1000;
+/** Run one prompt set × trials through `runner`, aggregating fired counts. */
+async function runTriggerSet(prompts, cfg, runner) {
     const perPrompt = [];
     let firedTotal = 0;
     let n = 0;
-    for (const prompt of spec.prompts) {
+    for (const prompt of prompts) {
         let fired = 0;
-        for (let t = 0; t < trials; t++) {
+        for (let t = 0; t < cfg.trials; t++) {
             const cwd = (0, node_fs_1.mkdtempSync)((0, node_path_1.join)((0, node_os_1.tmpdir)(), "vigiles-trigger-"));
             try {
                 const out = await runner({
                     task: prompt,
                     cwd,
-                    model,
-                    tools,
+                    model: cfg.model,
+                    tools: cfg.tools,
                     hasSettings: false,
-                    pluginDir: spec.pluginDir,
-                    timeoutMs,
+                    pluginDir: cfg.pluginDir,
+                    timeoutMs: cfg.timeoutMs,
                 });
-                if (spec.fired(makeContext(cwd, out)))
+                if (cfg.fired(makeContext(cwd, out)))
                     fired++;
             }
             finally {
                 (0, node_fs_1.rmSync)(cwd, { recursive: true, force: true });
-                await sleep(spacing);
+                await sleep(cfg.spacing);
             }
         }
         perPrompt.push({
             prompt,
             fired,
-            trials,
-            rate: trials > 0 ? fired / trials : 0,
+            trials: cfg.trials,
+            rate: cfg.trials > 0 ? fired / cfg.trials : 0,
         });
         firedTotal += fired;
-        n += trials;
+        n += cfg.trials;
     }
-    return { rate: n > 0 ? firedTotal / n : 0, n, perPrompt };
+    return { perPrompt, fired: firedTotal, n };
+}
+/**
+ * Trigger-rate orchestration — every prompt × trial via `runner`, the `fired`
+ * predicate evaluated per run and aggregated into an overall + per-prompt rate.
+ * With `irrelevantPrompts`, also runs the precision side (firing there is a false
+ * positive) and adds `falsePositiveRate` + `precision`. Exported with an
+ * injectable `runner` so the loop is unit-testable without a model;
+ * `measureTriggerRate` is this with the real agent runner.
+ */
+async function measureTriggerRateWith(spec, runner) {
+    const cfg = {
+        trials: spec.trials ?? 1,
+        model: spec.model ?? "haiku",
+        tools: spec.allowedTools ?? ["Read", "Edit", "Write", "Bash", "Skill"],
+        timeoutMs: spec.timeoutMs ?? 240000,
+        spacing: (spec.spacingSec ?? 4) * 1000,
+        pluginDir: spec.pluginDir,
+        fired: spec.fired,
+    };
+    const relevant = await runTriggerSet(spec.prompts, cfg, runner);
+    const base = {
+        rate: relevant.n > 0 ? relevant.fired / relevant.n : 0,
+        n: relevant.n,
+        perPrompt: relevant.perPrompt,
+    };
+    if ((spec.irrelevantPrompts?.length ?? 0) === 0)
+        return base;
+    const irrelevant = await runTriggerSet(spec.irrelevantPrompts ?? [], cfg, runner);
+    const fires = relevant.fired + irrelevant.fired;
+    return {
+        ...base,
+        falsePositiveRate: irrelevant.n > 0 ? irrelevant.fired / irrelevant.n : 0,
+        precision: fires > 0 ? relevant.fired / fires : undefined,
+        perIrrelevant: irrelevant.perPrompt,
+    };
 }
 /* v8 ignore start -- real claude subprocess; thin wrapper over measureTriggerRateWith */
 /**
@@ -465,6 +491,16 @@ function formatTriggerRateReport(report) {
     for (const p of report.perPrompt) {
         lines.push(`  ${p.rate.toFixed(2)}  ${p.prompt.slice(0, 60)}`);
     }
+    if (report.falsePositiveRate !== undefined) {
+        const fpr = (report.falsePositiveRate * 100).toFixed(0);
+        const prec = report.precision === undefined
+            ? "n/a"
+            : `${(report.precision * 100).toFixed(0)}%`;
+        lines.push(`false-positive: ${fpr}%  precision: ${prec}`);
+        for (const p of report.perIrrelevant ?? []) {
+            lines.push(`  ${p.rate.toFixed(2)}  [irrelevant] ${p.prompt.slice(0, 48)}`);
+        }
+    }
     return lines.join("\n");
 }
 //# sourceMappingURL=eval.js.map

package/dist/{harness-test.d.ts → adapters/claude-code/harness-test.d.ts}

@@ -1,6 +1,8 @@
-import { type ModelTurn, type ModelRequest } from "./mock-model.js";
+import type { HarnessAdapter } from "../../core/adapter.js";
+import type { HarnessTestDriver, ToolCall, HookFire, ParsedRun, ModelTurn, ModelRequest } from "../../core/harness-driver.js";
 import { type SandboxMode } from "./sandbox.js";
-export { scriptModel, type ModelTurn, type ModelRequest, } from "./mock-model.js";
+export { scriptModel } from "./mock-model.js";
+export type { ModelTurn, ModelRequest, ToolCall, HookFire, HarnessTestDriver, } from "../../core/harness-driver.js";
 export { loadPlugin, resolveHarness } from "./plugin-loader.js";
 export { decideSandbox, specTrusted, sandboxAvailable, type SandboxMode, } from "./sandbox.js";
 export interface HarnessTestSpec {
@@ -54,24 +56,6 @@ export interface HarnessTestSpec {
      */
     readonly sandbox?: SandboxMode;
 }
-/**
- * A hook invocation observed during the run, recorded (not inferred) from the
- * `hook_response` system events the CLI emits in the stream — so a test can
- * assert which hook fired and whether it blocked, instead of inferring it from a
- * marker file the hook had to write.
- */
-export interface HookFire {
-    /** The hook label, e.g. `"PreToolUse:Edit"` (`Event:Matcher`). */
-    readonly name: string;
-    /** The hook event, e.g. `"PreToolUse"`, `"PostToolUse"`, `"Stop"`. */
-    readonly event: string;
-    /** The hook process exit code (2 = block), or undefined if not reported. */
-    readonly exitCode: number | undefined;
-    /** Whether the hook blocked / errored (exit ≠ 0 or outcome "error"). */
-    readonly blocked: boolean;
-    /** What the hook printed (its block reason / diagnostic), or "". */
-    readonly output: string;
-}
 /**
  * The observable record of ONE run — the unified shape produced by BOTH testing
  * tiers: `runHarnessTest`'s result and `runEval`'s `measure` ctx (`eval.ts`)
@@ -124,15 +108,6 @@ export interface HarnessTestResult extends Trace {
     /** Remove the temp working dir. */
     cleanup(): void;
 }
-/** A tool the agent invoked, paired with its result (transcript mode only). */
-export interface ToolCall {
-    readonly name: string;
-    readonly input: unknown;
-    /** The tool_result text ("" if none / not captured). */
-    readonly resultText: string;
-    /** Whether the tool_result came back flagged as an error. */
-    readonly isError: boolean;
-}
 /**
  * Parse `--output-format stream-json` (the `transcript: true` output) into the
  * tools the agent invoked, each joined to its result by id. Returns [] for the
@@ -155,16 +130,37 @@ export declare function parseHooks(stdout: string): HookFire[];
  * here. Pure, so the arg shape is unit-tested.
  */
 export declare function buildClaudeArgs(spec: HarnessTestSpec, hasSettings: boolean): string[];
-/** Whether the `claude` CLI is available — harness tests need it. */
+/** Parse the `claude` stdout/stream into the unified trace fields. */
+export declare function parseClaudeRun(stdout: string): ParsedRun;
+/** Whether the agent CLI is available — harness tests need it. */
 export declare function claudeAvailable(): boolean;
 /**
- * Run the real `claude` CLI against a scripted mock model, with the given
- * fixture and settings (hooks). Deterministic — same script, same result.
+ * The Claude Code `HarnessTestDriver`: the existing argv/mock/parse seams bundled
+ * behind the port the adapter-driven runner dispatches through. Behaviourally
+ * identical to the previous hard-wired path.
+ */
+export declare const claudeCodeDriver: HarnessTestDriver;
+/** Options for {@link runHarnessTest}. */
+export interface RunHarnessTestOptions {
+    /**
+     * Which harness to drive. Defaults to Claude Code. Pass `codexAdapter`
+     * (`vigiles/codex`) to drive real `codex exec` against its Responses mock.
+     * The adapter must support pillar 2 (`capabilities.harnessTesting`) and carry
+     * a `harnessTestDriver`.
+     */
+    readonly adapter?: HarnessAdapter;
+}
+/**
+ * Run the real agent CLI against a scripted mock model, with the given fixture
+ * and settings (hooks). Deterministic — same script, same result. Adapter-driven
+ * (`opts.adapter`, default Claude Code): the Claude Code path is unchanged
+ * (incl. the safe-by-default sandbox); pass `codexAdapter` to drive real codex.
  *
- * Safe by default: an external `plugin` / `pluginDir` brings in untrusted
- * third-party hooks and is confined under bubblewrap (`spec.sandbox`, default
- * `"auto"`); if no sandbox is available the run REFUSES rather than executing
- * unconfined. See `src/sandbox.ts`.
+ * Safe by default (Claude Code): an external `plugin` / `pluginDir` brings in
+ * untrusted third-party hooks and is confined under bubblewrap (`spec.sandbox`,
+ * default `"auto"`); if no sandbox is available the run REFUSES rather than
+ * executing unconfined. See `src/sandbox.ts`. The sandbox path is Claude Code
+ * only — requesting confinement for another harness throws.
  */
-export declare function runHarnessTest(spec: HarnessTestSpec): Promise<HarnessTestResult>;
+export declare function runHarnessTest(spec: HarnessTestSpec, opts?: RunHarnessTestOptions): Promise<HarnessTestResult>;
 //# sourceMappingURL=harness-test.d.ts.map