vigiles 2.4.0 → 2.5.0

package/dist/sandbox.js

@@ -0,0 +1,241 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.sandboxAvailable = sandboxAvailable;
+exports.specTrusted = specTrusted;
+exports.decideSandbox = decideSandbox;
+exports.bwrapArgs = bwrapArgs;
+exports.parseRequestLog = parseRequestLog;
+exports.runSandboxed = runSandboxed;
+/**
+ * vigiles — safe-by-default confinement for executing untrusted harness code.
+ *
+ * `runHarnessTest` runs the real `claude` CLI, which runs the real hooks of
+ * whatever plugin you load. For code YOU authored (inline `settings`/`files`)
+ * that's fine — trust is implicit. But pointing it at someone else's `plugin` /
+ * `pluginDir` executes THEIR hooks with your privileges. This module makes that
+ * safe by default: untrusted code is confined under bubblewrap, or — if no
+ * sandbox is available — the run refuses rather than executing unconfined.
+ *
+ * Confinement (proven on bwrap 0.9): `--unshare-all` gives a fresh network
+ * namespace whose loopback is auto-up but has NO external route — so the
+ * scripted mock, co-launched INSIDE the namespace, is reachable over 127.0.0.1
+ * while a malicious hook cannot phone home. The filesystem is `--ro-bind`
+ * read-only except the throwaway work dir, a fresh empty `$HOME`, and an IO dir
+ * used to hand the script in and stream captured requests back out.
+ *
+ * The policy (`decideSandbox`), trust test (`specTrusted`), and bwrap argv
+ * (`bwrapArgs`) are pure and unit-tested; the executor (`runSandboxed`) needs a
+ * real bwrap and is covered by the integration test, which skips where bwrap is
+ * absent — the same pattern as the real-`claude` paths.
+ */
+const node_child_process_1 = require("node:child_process");
+const node_fs_1 = require("node:fs");
+const node_os_1 = require("node:os");
+const node_path_1 = require("node:path");
+/**
+ * Whether bubblewrap is available to confine untrusted code. **Linux only** —
+ * bubblewrap is a Linux tool, so this is always `false` on macOS / Windows,
+ * where confined execution isn't supported and untrusted code must instead be
+ * run via `sandbox: false` (trusting the outer container) or skipped.
+ */
+function sandboxAvailable() {
+    /* v8 ignore next -- non-Linux has no bwrap; CI/coverage runs on Linux */
+    if (process.platform !== "linux")
+        return false;
+    try {
+        return (0, node_child_process_1.spawnSync)("bwrap", ["--version"], { stdio: "ignore" }).status === 0;
+    }
+    catch {
+        /* v8 ignore next -- defensive: spawnSync only throws on a fork failure */
+        return false;
+    }
+}
+/**
+ * Is this spec's executed code trusted? Inline `settings`/`files` you authored
+ * are trusted; any external `plugin` / `pluginDir` brings in third-party hooks
+ * and is NOT — committing it to your repo is the same trust decision as a
+ * dependency, so the trust boundary follows provenance: foreign = confined.
+ */
+function specTrusted(spec) {
+    return spec.plugin === undefined && spec.pluginDir === undefined;
+}
+/**
+ * The pure safe-by-default policy. Untrusted code NEVER runs unconfined unless
+ * the caller explicitly opted out (`mode: false`). This is the whole security
+ * contract, isolated as a pure function so it is exhaustively unit-tested.
+ */
+function decideSandbox(opts) {
+    // Explicit dangerous opt-out: run unconfined, trusted or not.
+    if (opts.mode === false)
+        return { action: "direct" };
+    // Force confinement regardless of trust; refuse if we can't.
+    if (opts.mode === "strict") {
+        return opts.available
+            ? { action: "sandbox" }
+            : {
+                action: "throw",
+                reason: "sandbox: 'strict' requires Linux + bubblewrap (bwrap), which was not available",
+            };
+    }
+    // auto: trusted code runs directly; untrusted must be confined or refused.
+    if (opts.trusted)
+        return { action: "direct" };
+    return opts.available
+        ? { action: "sandbox" }
+        : {
+            action: "throw",
+            reason: "refusing to execute an untrusted plugin's hooks without a sandbox: " +
+                "the sandbox needs Linux + bubblewrap (bwrap) — install it to run " +
+                "confined, or pass sandbox: false to run unconfined if you trust this " +
+                "code / the outer container",
+        };
+}
+/**
+ * The bubblewrap confinement argv (everything before the command): a fresh
+ * network namespace (`--unshare-all`, loopback-only, no egress), a read-only
+ * system, writable mounts limited to the work dir, the IO dir, and a fresh empty
+ * HOME (inside the IO dir so it needs no mountpoint on the read-only root, and so
+ * no host credentials/config leak in), and a **cleared environment** —
+ * `--clearenv` drops every host variable (API keys, cloud creds) and only PATH /
+ * HOME / TMPDIR are set back, so untrusted code can't even read your secrets.
+ * Pure, so the confinement shape is asserted in a unit test.
+ */
+function bwrapArgs(opts) {
+    return [
+        // New user/net/pid/ipc/uts/cgroup namespaces. The net namespace has only a
+        // loopback route, so the in-sandbox mock is reachable but egress is blocked.
+        "--unshare-all",
+        // Drop ALL inherited env (host secrets); only the essentials are set back.
+        "--clearenv",
+        "--ro-bind",
+        "/",
+        "/",
+        "--dev",
+        "/dev",
+        "--proc",
+        "/proc",
+        // Writable: the work dir and the IO dir (later binds override the ro-bind).
+        "--bind",
+        opts.cwd,
+        opts.cwd,
+        "--bind",
+        opts.ioDir,
+        opts.ioDir,
+        // A fresh empty HOME so no host credentials/config are visible.
+        "--setenv",
+        "HOME",
+        opts.home,
+        "--setenv",
+        "TMPDIR",
+        opts.ioDir,
+        // PATH must be set back explicitly (cleared above) so node/claude resolve.
+        "--setenv",
+        "PATH",
+        opts.path,
+        "--chdir",
+        opts.cwd,
+        "--die-with-parent",
+        "--new-session",
+    ];
+}
+/** Parse the in-sandbox mock's ndjson request log into {@link ModelRequest}s. */
+function parseRequestLog(ndjson) {
+    const out = [];
+    for (const line of ndjson.split("\n")) {
+        if (!line.trim())
+            continue;
+        try {
+            out.push(JSON.parse(line));
+        }
+        catch {
+            /* a partially-written final line — skip */
+        }
+    }
+    return out;
+}
+/**
+ * Co-launch the scripted mock and `claude` inside ONE bubblewrap network
+ * namespace: the mock serves on the sandbox's loopback (reachable), egress is
+ * blocked, and captured requests stream out through the bound IO dir. Paths come
+ * in via env so the wrapper needs no escaping; `claude`'s args are the wrapper's
+ * positional params (`"$@"`).
+ */
+const WRAPPER = [
+    // start the in-sandbox mock; it writes its port to $VIG_PORT when ready
+    'node "$VIG_MOCKENTRY" "$VIG_SCRIPT" "$VIG_REQS" "$VIG_PORT" &',
+    "MOCKPID=$!",
+    "i=0",
+    'while [ ! -s "$VIG_PORT" ] && [ "$i" -lt 200 ]; do sleep 0.05; i=$((i+1)); done',
+    'export ANTHROPIC_BASE_URL="http://127.0.0.1:$(cat "$VIG_PORT")"',
+    "export ANTHROPIC_API_KEY=sk-vigiles-mock",
+    'claude "$@"',
+    "code=$?",
+    'kill "$MOCKPID" 2>/dev/null',
+    'exit "$code"',
+].join("\n");
+/* v8 ignore start -- spawns bwrap + the real claude CLI; exercised by the
+   bwrap-backed integration test (skipped without bwrap), not the unit gate —
+   the pure policy/args/parse helpers above carry the testable logic. */
+function runSandboxed(opts) {
+    const ioDir = (0, node_fs_1.mkdtempSync)((0, node_path_1.join)((0, node_os_1.tmpdir)(), "vigiles-sbx-"));
+    const home = (0, node_path_1.join)(ioDir, "home");
+    (0, node_fs_1.mkdirSync)(home);
+    const scriptF = (0, node_path_1.join)(ioDir, "script.json");
+    const reqsF = (0, node_path_1.join)(ioDir, "requests.ndjson");
+    const portF = (0, node_path_1.join)(ioDir, "port");
+    (0, node_fs_1.writeFileSync)(scriptF, JSON.stringify(opts.script));
+    (0, node_fs_1.writeFileSync)(reqsF, "");
+    // The mock entry is only runnable as built JS. In production __dirname is
+    // dist/ (sibling); under vitest the source runs from src/, so fall back to
+    // the built dist/ copy.
+    const mockEntry = [
+        (0, node_path_1.join)(__dirname, "mock-entry.js"),
+        (0, node_path_1.join)(__dirname, "..", "dist", "mock-entry.js"),
+    ].find((p) => (0, node_fs_1.existsSync)(p)) ?? (0, node_path_1.join)(__dirname, "mock-entry.js");
+    const args = [
+        ...bwrapArgs({
+            cwd: opts.cwd,
+            ioDir,
+            home,
+            path: process.env.PATH ?? "/usr/local/bin:/usr/bin:/bin",
+        }),
+        "--setenv",
+        "VIG_MOCKENTRY",
+        mockEntry,
+        "--setenv",
+        "VIG_SCRIPT",
+        scriptF,
+        "--setenv",
+        "VIG_REQS",
+        reqsF,
+        "--setenv",
+        "VIG_PORT",
+        portF,
+        "sh",
+        "-c",
+        WRAPPER,
+        "sh",
+        ...opts.claudeArgs,
+    ];
+    return new Promise((resolvePromise) => {
+        const child = (0, node_child_process_1.spawn)("bwrap", args, {
+            cwd: opts.cwd,
+            env: process.env,
+            stdio: ["ignore", "pipe", "pipe"],
+        });
+        let stdout = "";
+        child.stdout.on("data", (d) => (stdout += d.toString()));
+        child.stderr.on("data", () => {
+            /* hook diagnostics — not needed for the captured result */
+        });
+        const timer = setTimeout(() => child.kill("SIGKILL"), opts.timeoutMs);
+        child.on("close", (code) => {
+            clearTimeout(timer);
+            const requests = parseRequestLog((0, node_fs_1.existsSync)(reqsF) ? (0, node_fs_1.readFileSync)(reqsF, "utf-8") : "");
+            (0, node_fs_1.rmSync)(ioDir, { recursive: true, force: true });
+            resolvePromise({ code: code ?? 0, stdout, requests });
+        });
+    });
+}
+/* v8 ignore stop */
+//# sourceMappingURL=sandbox.js.map

package/dist/spec.d.ts

@@ -320,6 +320,136 @@ export interface SkillSpec {
  *   export default skill({ name: "my-skill", description: "...", body: "..." });
  */
 export declare function skill(spec: Omit<SkillSpec, "_specType">): SkillSpec;
+/**
+ * A subagent definition (compiles to `agents/<name>.md`). Unlike a skill —
+ * reference material the model reads on activation — a subagent is a *delegated
+ * worker with a contract*: a dispatch `description`, an allowed-`tools` rail, an
+ * optional `model`, a system-prompt `body`, and the `rules` it must follow. That
+ * tool contract + those rules are the "railway" a subagent runs on, and they're
+ * exactly the compile-time-verifiable surface vigiles owns: the body's
+ * `file()`/`cmd()`/`symbol()` marks are checked like any instruction file, and
+ * the tools list is verified against the real tool set.
+ */
+export interface AgentSpec {
+    readonly _specType: "agent";
+    /** Subagent name (frontmatter + dispatch handle). */
+    readonly name: string;
+    /** When to dispatch this subagent — the trigger (frontmatter). */
+    readonly description: string;
+    /** Model alias (e.g. "sonnet", "opus", "haiku", "inherit"). Optional. */
+    readonly model?: string;
+    /**
+     * The allowed-tools contract — the rails the worker runs on. Each entry must be
+     * a known built-in tool (Read/Write/Edit/Bash/Grep/Glob/WebSearch/WebFetch/
+     * NotebookEdit/TodoWrite/Task/Skill) or an MCP tool (`mcp__server__tool`).
+     * Omit to inherit all tools. Verified at compile time.
+     */
+    readonly tools?: readonly string[];
+    /**
+     * The lead/intro prose of the system prompt (the "You are…" opener), before any
+     * sections. Carries verified `file()`/`cmd()`/`symbol()`/`ref()` marks. No
+     * markdown headers — use `sections` for those.
+     */
+    readonly body?: string | InstructionFragment[];
+    /**
+     * Named `##` sections of the system prompt (e.g. Purpose, Core Principles,
+     * Capabilities) — the shape real subagents actually take. Same verified-ref +
+     * no-nested-`##` rules as a CLAUDE.md spec's sections. Use `body` for the intro
+     * and `sections` for the structured rest.
+     */
+    readonly sections?: Record<string, string | InstructionFragment[]>;
+    /** Rules the worker must follow — rendered as a `## Rules` section. */
+    readonly rules?: Record<string, Rule>;
+    /**
+     * The typed result contract — what this worker returns on success/error. When
+     * set, compiles to an `## Output contract` section instructing the worker to
+     * end with a `vigiles:ok` / `vigiles:err` block, so its outcome is parseable
+     * and testable (see `result()`, `parseAgentResult`, `assertAgentOk`).
+     */
+    readonly output?: OutputContract;
+}
+/**
+ * Define a subagent specification (compiles to `agents/<name>.md`).
+ *
+ *   // agents/reviewer.md.spec.ts
+ *   export default agent({
+ *     name: "reviewer",
+ *     description: "Review a diff for correctness. Dispatch PROACTIVELY after edits.",
+ *     model: "sonnet",
+ *     tools: ["Read", "Grep", "Bash"],
+ *     body: instructions`Review the diff. Run ${cmd("npm test")} first.`,
+ *     rules: {
+ *       "no-floating": enforce("@typescript-eslint/no-floating-promises", "Await promises."),
+ *     },
+ *   });
+ */
+export declare function agent(spec: Omit<AgentSpec, "_specType">): AgentSpec;
+/** The field types a result contract can declare (kept tiny + dependency-free). */
+export type OutputFieldType = "string" | "number" | "boolean" | "string[]";
+/**
+ * A subagent's typed result contract: the shape it must return on success
+ * (`ok`) and on failure (`err`). Rich on both tracks — an error is structured
+ * detail, not a bare pass/fail bit. Compiles into the worker's system prompt
+ * (the `vigiles:ok` / `vigiles:err` block it must emit) and is the schema the
+ * `parseAgentResult` parser + the `assertAgentOk/Err` test helpers validate.
+ */
+export interface OutputContract {
+    readonly _ref: "output";
+    readonly ok: Readonly<Record<string, OutputFieldType>>;
+    readonly err: Readonly<Record<string, OutputFieldType>>;
+}
+/**
+ * Declare a subagent's success/error result contract.
+ *
+ *   result(
+ *     { files: "string[]", summary: "string" },          // rich success
+ *     { reason: "string", retryable: "boolean" },         // rich error
+ *   )
+ *
+ * (Distinct from a skill's `result:` postcondition gate — this types a
+ * subagent's *return value*, the success/error tracks of the railway.)
+ */
+export declare function result(ok: Record<string, OutputFieldType>, err: Record<string, OutputFieldType>): OutputContract;
+/** One step on a railway: dispatch a flat subagent (the "activity"). */
+export interface RailwayStep {
+    readonly _step: "delegate";
+    /** The subagent to dispatch — resolved against compiled agent names. */
+    readonly agent: string;
+    /** Optional task hint passed to the worker. */
+    readonly task?: string;
+}
+/** Build a railway step that dispatches `agent` (optionally with a task hint). */
+export declare function delegate(agent: string, task?: string): RailwayStep;
+/**
+ * A railway over flat subagents. `steps` run in order on the success track; the
+ * first step that returns an error short-circuits to `onError`. `recover`
+ * optionally retries the failing step a *bounded* number of times before the
+ * error track. There is intentionally no loop combinator — the value is a finite
+ * tree, so it always terminates and is fully verifiable at compile time.
+ */
+export interface Railway {
+    readonly _specType: "railway";
+    readonly name: string;
+    readonly steps: readonly RailwayStep[];
+    /** Error track — runs with the failing step's error payload. */
+    readonly onError?: RailwayStep;
+    /** Bounded recovery: retry the failing step up to `max` times (finite). */
+    readonly recover?: {
+        readonly step: RailwayStep;
+        readonly max: number;
+    };
+}
+/**
+ * Compose flat subagents into a railway (compiles to an orchestrator command).
+ *
+ *   railway({
+ *     name: "ship",
+ *     steps: [delegate("planner"), delegate("coder"), delegate("reviewer")],
+ *     onError: delegate("reporter"),
+ *     recover: { step: delegate("fixer"), max: 2 },
+ *   })
+ */
+export declare function railway(spec: Omit<Railway, "_specType">): Railway;
 /** Derive the spec filename from an output filename. */
 export type SpecPath<Output extends `${string}.md`> = `${Output}.spec.ts`;
 /** Extract the output filename from a spec filename. */

package/dist/spec.js

@@ -23,6 +23,10 @@ exports.project = project;
 exports.input = input;
 exports.step = step;
 exports.skill = skill;
+exports.agent = agent;
+exports.result = result;
+exports.delegate = delegate;
+exports.railway = railway;
 exports.defineConfig = defineConfig;
 // ---------------------------------------------------------------------------
 // Builder functions
@@ -154,6 +158,57 @@ function step(instr, opts = {}) {
 function skill(spec) {
     return { _specType: "skill", ...spec };
 }
+/**
+ * Define a subagent specification (compiles to `agents/<name>.md`).
+ *
+ *   // agents/reviewer.md.spec.ts
+ *   export default agent({
+ *     name: "reviewer",
+ *     description: "Review a diff for correctness. Dispatch PROACTIVELY after edits.",
+ *     model: "sonnet",
+ *     tools: ["Read", "Grep", "Bash"],
+ *     body: instructions`Review the diff. Run ${cmd("npm test")} first.`,
+ *     rules: {
+ *       "no-floating": enforce("@typescript-eslint/no-floating-promises", "Await promises."),
+ *     },
+ *   });
+ */
+function agent(spec) {
+    return { _specType: "agent", ...spec };
+}
+/**
+ * Declare a subagent's success/error result contract.
+ *
+ *   result(
+ *     { files: "string[]", summary: "string" },          // rich success
+ *     { reason: "string", retryable: "boolean" },         // rich error
+ *   )
+ *
+ * (Distinct from a skill's `result:` postcondition gate — this types a
+ * subagent's *return value*, the success/error tracks of the railway.)
+ */
+function result(ok, err) {
+    return { _ref: "output", ok, err };
+}
+/** Build a railway step that dispatches `agent` (optionally with a task hint). */
+function delegate(agent, task) {
+    return task === undefined
+        ? { _step: "delegate", agent }
+        : { _step: "delegate", agent, task };
+}
+/**
+ * Compose flat subagents into a railway (compiles to an orchestrator command).
+ *
+ *   railway({
+ *     name: "ship",
+ *     steps: [delegate("planner"), delegate("coder"), delegate("reviewer")],
+ *     onError: delegate("reporter"),
+ *     recover: { step: delegate("fixer"), max: 2 },
+ *   })
+ */
+function railway(spec) {
+    return { _specType: "railway", ...spec };
+}
 function defineConfig(config) {
     return config;
 }

package/dist/stats.d.ts

@@ -0,0 +1,49 @@
+/**
+ * vigiles — significance testing for eval A/B arms.
+ *
+ * The eval tier already reports mean ± se per arm; this answers the question that
+ * `assertImproves(..., { by: se })` punted to the user: is the gap between two
+ * arms real, or noise? A Welch's t-test over the per-arm summary stats (mean, se,
+ * n) — no raw rows needed — yields a two-sided p-value and a significance verdict.
+ * Pure + model-free, so it's fully unit-tested against known t-table values.
+ *
+ * For 0/1 (proportion) metrics this is the t approximation to the two-proportion
+ * test — close at the trial counts evals use, and one code path for any metric.
+ * The numerics (log-gamma, incomplete beta) are specialized to the argument range
+ * these tests produce (a, b ≥ 0.5; x ∈ (0,1)); they are not a general library.
+ */
+import type { EvalReport } from "./eval.js";
+/** Regularized incomplete beta I_x(a, b) ∈ [0, 1]. */
+export declare function regularizedIncompleteBeta(a: number, b: number, x: number): number;
+/** Two-sided p-value for Student's t with `df` degrees of freedom. */
+export declare function tPValueTwoSided(t: number, df: number): number;
+/** The verdict on one arm-vs-baseline comparison for a single metric. */
+export interface Comparison {
+    /** mean(arm) − mean(baseline). */
+    readonly delta: number;
+    /** Combined standard error of the difference. */
+    readonly seDelta: number;
+    /** Welch t statistic (delta / seDelta). */
+    readonly t: number;
+    /** Welch–Satterthwaite degrees of freedom. */
+    readonly df: number;
+    /** Two-sided p-value for the difference. */
+    readonly pValue: number;
+    /** p < alpha — the difference is unlikely to be noise. */
+    readonly significant: boolean;
+}
+type Summary = {
+    readonly mean: number;
+    readonly se: number;
+    readonly n: number;
+};
+/** Welch's unequal-variance t-test between two arms' summary stats. */
+export declare function welchTTest(arm: Summary, baseline: Summary, alpha?: number): Comparison;
+/**
+ * Compare two arms on a metric using their reported summary stats, or null if
+ * either arm/metric is absent. The grounded form of `assertImproves`'s `by`: it
+ * computes the noise floor instead of asking the caller to supply it.
+ */
+export declare function compareArms(report: EvalReport, baseline: string, arm: string, metric: string, alpha?: number): Comparison | null;
+export {};
+//# sourceMappingURL=stats.d.ts.map

package/dist/stats.js

@@ -0,0 +1,109 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.regularizedIncompleteBeta = regularizedIncompleteBeta;
+exports.tPValueTwoSided = tPValueTwoSided;
+exports.welchTTest = welchTTest;
+exports.compareArms = compareArms;
+// Lanczos coefficients (g = 7) for log-gamma; sufficient for the beta args here.
+const LANCZOS = [
+    676.5203681218851, -1259.1392167224028, 771.32342877765313,
+    -176.61502916214059, 12.507343278686905, -0.13857109526572012,
+    9.9843695780195716e-6, 1.5056327351493116e-7,
+];
+/** Log-gamma via Lanczos. Valid for x ≥ 0.5 (all args used below satisfy this). */
+function lgamma(x) {
+    const g = 7;
+    const xm1 = x - 1;
+    const base = LANCZOS.reduce((acc, c, i) => acc + c / (xm1 + i + 1), 0.99999999999980993);
+    const tt = xm1 + g + 0.5;
+    return (0.5 * Math.log(2 * Math.PI) +
+        (xm1 + 0.5) * Math.log(tt) -
+        tt +
+        Math.log(base));
+}
+/** Continued fraction for the incomplete beta (Numerical Recipes betacf). */
+function betacf(a, b, x) {
+    const MAXIT = 200;
+    const EPS = 1e-12;
+    const qab = a + b;
+    const qap = a + 1;
+    const qam = a - 1;
+    let c = 1;
+    let d = 1 / (1 - (qab * x) / qap);
+    let h = d;
+    for (let m = 1; m <= MAXIT; m++) {
+        const m2 = 2 * m;
+        let aa = (m * (b - m) * x) / ((qam + m2) * (a + m2));
+        d = 1 / (1 + aa * d);
+        c = 1 + aa / c;
+        h *= d * c;
+        aa = (-(a + m) * (qab + m) * x) / ((a + m2) * (qap + m2));
+        d = 1 / (1 + aa * d);
+        c = 1 + aa / c;
+        const del = d * c;
+        h *= del;
+        if (Math.abs(del - 1) < EPS)
+            break;
+    }
+    return h;
+}
+/** Regularized incomplete beta I_x(a, b) ∈ [0, 1]. */
+function regularizedIncompleteBeta(a, b, x) {
+    if (x <= 0)
+        return 0;
+    if (x >= 1)
+        return 1;
+    const front = Math.exp(lgamma(a + b) -
+        lgamma(a) -
+        lgamma(b) +
+        a * Math.log(x) +
+        b * Math.log(1 - x));
+    return x < (a + 1) / (a + b + 2)
+        ? (front * betacf(a, b, x)) / a
+        : 1 - (front * betacf(b, a, 1 - x)) / b;
+}
+/** Two-sided p-value for Student's t with `df` degrees of freedom. */
+function tPValueTwoSided(t, df) {
+    if (df <= 0)
+        return 1;
+    return regularizedIncompleteBeta(df / 2, 0.5, df / (df + t * t));
+}
+// Variance contribution of one arm to the Welch df denominator. Guarded by v > 0
+// (se > 0 ⇒ n ≥ 2, so n − 1 ≥ 1); a deterministic arm (se = 0) contributes 0.
+const dfTerm = (v, n) => v > 0 ? (v * v) / (n - 1) : 0;
+/** Welch's unequal-variance t-test between two arms' summary stats. */
+function welchTTest(arm, baseline, alpha = 0.05) {
+    const delta = arm.mean - baseline.mean;
+    const va = arm.se ** 2;
+    const vb = baseline.se ** 2;
+    const seDelta = Math.sqrt(va + vb);
+    if (seDelta === 0) {
+        // Both arms are deterministic: significant iff they differ at all.
+        const significant = delta !== 0;
+        return {
+            delta,
+            seDelta,
+            t: 0,
+            df: 0,
+            pValue: significant ? 0 : 1,
+            significant,
+        };
+    }
+    const t = delta / seDelta;
+    const df = (va + vb) ** 2 / (dfTerm(va, arm.n) + dfTerm(vb, baseline.n));
+    const pValue = tPValueTwoSided(t, df);
+    return { delta, seDelta, t, df, pValue, significant: pValue < alpha };
+}
+/**
+ * Compare two arms on a metric using their reported summary stats, or null if
+ * either arm/metric is absent. The grounded form of `assertImproves`'s `by`: it
+ * computes the noise floor instead of asking the caller to supply it.
+ */
+function compareArms(report, baseline, arm, metric, alpha = 0.05) {
+    const a = report.arms[arm]?.stats[metric];
+    const b = report.arms[baseline]?.stats[metric];
+    if (!a || !b)
+        return null;
+    return welchTTest(a, b, alpha);
+}
+//# sourceMappingURL=stats.js.map

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "vigiles",
-  "version": "2.4.0",
+  "version": "2.5.0",
   "description": "Compile .spec.ts files to instruction files (CLAUDE.md, AGENTS.md) with linter cross-referencing",
   "bin": {
     "vigiles": "dist/cli.js"
@@ -40,7 +40,8 @@
   ],
   "scripts": {
     "build": "tsc",
-    "test": "npm run build && node --test dist/spec.test.js dist/validate.test.js dist/cli.test.js dist/proofs.test.js dist/inline.test.js dist/sidecar.test.js dist/coverage.test.js dist/session.test.js dist/orphans.test.js dist/cedar.test.js dist/doc-refs.test.js dist/frontmatter.test.js dist/skill-pipeline.test.js dist/skill-runtime.test.js dist/skill-driver.test.js dist/skill-test.test.js dist/compile-generator.test.js dist/community-skills.test.js dist/action-gate.test.js dist/symbols.test.js dist/refs.test.js dist/harness-test.test.js dist/eval.test.js dist/run-scripts.test.js dist/plugin-loader.test.js dist/harness-assert.test.js dist/judge.test.js dist/run-hook.test.js dist/mcp.test.js",
+    "test": "npm run build && vitest run",
+    "coverage": "npm run build && vitest run --coverage",
     "lint": "eslint src/",
     "fmt": "prettier --write .",
     "fmt:check": "prettier --check .",
@@ -48,7 +49,7 @@
     "test:e2e": "bash test/e2e/run.sh",
     "test:harness": "npm run build && node dist/cli.js test",
     "test:eval": "npm run build && node dist/cli.js eval",
-    "test:vitest": "npm run build && vitest run",
+    "test:vitest": "npm run build && vitest run --project runners",
     "test:jest": "npm run build && jest",
     "test:types": "npm run build && tsc --noEmit -p test/types/tsconfig.json"
   },
@@ -59,6 +60,7 @@
     "@types/node": "^20.19.39",
     "@typescript-eslint/eslint-plugin": "^8.58.0",
     "@typescript-eslint/parser": "^8.58.0",
+    "@vitest/coverage-v8": "^4.1.8",
     "eslint": "^10.1.0",
     "eslint-plugin-sonarjs": "^4.0.2",
     "globals": "^17.4.0",