vigiles 2.3.0 → 2.5.0

package/dist/spec.js

@@ -23,6 +23,10 @@ exports.project = project;
 exports.input = input;
 exports.step = step;
 exports.skill = skill;
+exports.agent = agent;
+exports.result = result;
+exports.delegate = delegate;
+exports.railway = railway;
 exports.defineConfig = defineConfig;
 // ---------------------------------------------------------------------------
 // Builder functions
@@ -154,6 +158,57 @@ function step(instr, opts = {}) {
 function skill(spec) {
     return { _specType: "skill", ...spec };
 }
+/**
+ * Define a subagent specification (compiles to `agents/<name>.md`).
+ *
+ *   // agents/reviewer.md.spec.ts
+ *   export default agent({
+ *     name: "reviewer",
+ *     description: "Review a diff for correctness. Dispatch PROACTIVELY after edits.",
+ *     model: "sonnet",
+ *     tools: ["Read", "Grep", "Bash"],
+ *     body: instructions`Review the diff. Run ${cmd("npm test")} first.`,
+ *     rules: {
+ *       "no-floating": enforce("@typescript-eslint/no-floating-promises", "Await promises."),
+ *     },
+ *   });
+ */
+function agent(spec) {
+    return { _specType: "agent", ...spec };
+}
+/**
+ * Declare a subagent's success/error result contract.
+ *
+ *   result(
+ *     { files: "string[]", summary: "string" },          // rich success
+ *     { reason: "string", retryable: "boolean" },         // rich error
+ *   )
+ *
+ * (Distinct from a skill's `result:` postcondition gate — this types a
+ * subagent's *return value*, the success/error tracks of the railway.)
+ */
+function result(ok, err) {
+    return { _ref: "output", ok, err };
+}
+/** Build a railway step that dispatches `agent` (optionally with a task hint). */
+function delegate(agent, task) {
+    return task === undefined
+        ? { _step: "delegate", agent }
+        : { _step: "delegate", agent, task };
+}
+/**
+ * Compose flat subagents into a railway (compiles to an orchestrator command).
+ *
+ *   railway({
+ *     name: "ship",
+ *     steps: [delegate("planner"), delegate("coder"), delegate("reviewer")],
+ *     onError: delegate("reporter"),
+ *     recover: { step: delegate("fixer"), max: 2 },
+ *   })
+ */
+function railway(spec) {
+    return { _specType: "railway", ...spec };
+}
 function defineConfig(config) {
     return config;
 }

package/dist/stats.d.ts

@@ -0,0 +1,49 @@
+/**
+ * vigiles — significance testing for eval A/B arms.
+ *
+ * The eval tier already reports mean ± se per arm; this answers the question that
+ * `assertImproves(..., { by: se })` punted to the user: is the gap between two
+ * arms real, or noise? A Welch's t-test over the per-arm summary stats (mean, se,
+ * n) — no raw rows needed — yields a two-sided p-value and a significance verdict.
+ * Pure + model-free, so it's fully unit-tested against known t-table values.
+ *
+ * For 0/1 (proportion) metrics this is the t approximation to the two-proportion
+ * test — close at the trial counts evals use, and one code path for any metric.
+ * The numerics (log-gamma, incomplete beta) are specialized to the argument range
+ * these tests produce (a, b ≥ 0.5; x ∈ (0,1)); they are not a general library.
+ */
+import type { EvalReport } from "./eval.js";
+/** Regularized incomplete beta I_x(a, b) ∈ [0, 1]. */
+export declare function regularizedIncompleteBeta(a: number, b: number, x: number): number;
+/** Two-sided p-value for Student's t with `df` degrees of freedom. */
+export declare function tPValueTwoSided(t: number, df: number): number;
+/** The verdict on one arm-vs-baseline comparison for a single metric. */
+export interface Comparison {
+    /** mean(arm) − mean(baseline). */
+    readonly delta: number;
+    /** Combined standard error of the difference. */
+    readonly seDelta: number;
+    /** Welch t statistic (delta / seDelta). */
+    readonly t: number;
+    /** Welch–Satterthwaite degrees of freedom. */
+    readonly df: number;
+    /** Two-sided p-value for the difference. */
+    readonly pValue: number;
+    /** p < alpha — the difference is unlikely to be noise. */
+    readonly significant: boolean;
+}
+type Summary = {
+    readonly mean: number;
+    readonly se: number;
+    readonly n: number;
+};
+/** Welch's unequal-variance t-test between two arms' summary stats. */
+export declare function welchTTest(arm: Summary, baseline: Summary, alpha?: number): Comparison;
+/**
+ * Compare two arms on a metric using their reported summary stats, or null if
+ * either arm/metric is absent. The grounded form of `assertImproves`'s `by`: it
+ * computes the noise floor instead of asking the caller to supply it.
+ */
+export declare function compareArms(report: EvalReport, baseline: string, arm: string, metric: string, alpha?: number): Comparison | null;
+export {};
+//# sourceMappingURL=stats.d.ts.map

package/dist/stats.js

@@ -0,0 +1,109 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.regularizedIncompleteBeta = regularizedIncompleteBeta;
+exports.tPValueTwoSided = tPValueTwoSided;
+exports.welchTTest = welchTTest;
+exports.compareArms = compareArms;
+// Lanczos coefficients (g = 7) for log-gamma; sufficient for the beta args here.
+const LANCZOS = [
+    676.5203681218851, -1259.1392167224028, 771.32342877765313,
+    -176.61502916214059, 12.507343278686905, -0.13857109526572012,
+    9.9843695780195716e-6, 1.5056327351493116e-7,
+];
+/** Log-gamma via Lanczos. Valid for x ≥ 0.5 (all args used below satisfy this). */
+function lgamma(x) {
+    const g = 7;
+    const xm1 = x - 1;
+    const base = LANCZOS.reduce((acc, c, i) => acc + c / (xm1 + i + 1), 0.99999999999980993);
+    const tt = xm1 + g + 0.5;
+    return (0.5 * Math.log(2 * Math.PI) +
+        (xm1 + 0.5) * Math.log(tt) -
+        tt +
+        Math.log(base));
+}
+/** Continued fraction for the incomplete beta (Numerical Recipes betacf). */
+function betacf(a, b, x) {
+    const MAXIT = 200;
+    const EPS = 1e-12;
+    const qab = a + b;
+    const qap = a + 1;
+    const qam = a - 1;
+    let c = 1;
+    let d = 1 / (1 - (qab * x) / qap);
+    let h = d;
+    for (let m = 1; m <= MAXIT; m++) {
+        const m2 = 2 * m;
+        let aa = (m * (b - m) * x) / ((qam + m2) * (a + m2));
+        d = 1 / (1 + aa * d);
+        c = 1 + aa / c;
+        h *= d * c;
+        aa = (-(a + m) * (qab + m) * x) / ((a + m2) * (qap + m2));
+        d = 1 / (1 + aa * d);
+        c = 1 + aa / c;
+        const del = d * c;
+        h *= del;
+        if (Math.abs(del - 1) < EPS)
+            break;
+    }
+    return h;
+}
+/** Regularized incomplete beta I_x(a, b) ∈ [0, 1]. */
+function regularizedIncompleteBeta(a, b, x) {
+    if (x <= 0)
+        return 0;
+    if (x >= 1)
+        return 1;
+    const front = Math.exp(lgamma(a + b) -
+        lgamma(a) -
+        lgamma(b) +
+        a * Math.log(x) +
+        b * Math.log(1 - x));
+    return x < (a + 1) / (a + b + 2)
+        ? (front * betacf(a, b, x)) / a
+        : 1 - (front * betacf(b, a, 1 - x)) / b;
+}
+/** Two-sided p-value for Student's t with `df` degrees of freedom. */
+function tPValueTwoSided(t, df) {
+    if (df <= 0)
+        return 1;
+    return regularizedIncompleteBeta(df / 2, 0.5, df / (df + t * t));
+}
+// Variance contribution of one arm to the Welch df denominator. Guarded by v > 0
+// (se > 0 ⇒ n ≥ 2, so n − 1 ≥ 1); a deterministic arm (se = 0) contributes 0.
+const dfTerm = (v, n) => v > 0 ? (v * v) / (n - 1) : 0;
+/** Welch's unequal-variance t-test between two arms' summary stats. */
+function welchTTest(arm, baseline, alpha = 0.05) {
+    const delta = arm.mean - baseline.mean;
+    const va = arm.se ** 2;
+    const vb = baseline.se ** 2;
+    const seDelta = Math.sqrt(va + vb);
+    if (seDelta === 0) {
+        // Both arms are deterministic: significant iff they differ at all.
+        const significant = delta !== 0;
+        return {
+            delta,
+            seDelta,
+            t: 0,
+            df: 0,
+            pValue: significant ? 0 : 1,
+            significant,
+        };
+    }
+    const t = delta / seDelta;
+    const df = (va + vb) ** 2 / (dfTerm(va, arm.n) + dfTerm(vb, baseline.n));
+    const pValue = tPValueTwoSided(t, df);
+    return { delta, seDelta, t, df, pValue, significant: pValue < alpha };
+}
+/**
+ * Compare two arms on a metric using their reported summary stats, or null if
+ * either arm/metric is absent. The grounded form of `assertImproves`'s `by`: it
+ * computes the noise floor instead of asking the caller to supply it.
+ */
+function compareArms(report, baseline, arm, metric, alpha = 0.05) {
+    const a = report.arms[arm]?.stats[metric];
+    const b = report.arms[baseline]?.stats[metric];
+    if (!a || !b)
+        return null;
+    return welchTTest(a, b, alpha);
+}
+//# sourceMappingURL=stats.js.map

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "vigiles",
-  "version": "2.3.0",
+  "version": "2.5.0",
   "description": "Compile .spec.ts files to instruction files (CLAUDE.md, AGENTS.md) with linter cross-referencing",
   "bin": {
     "vigiles": "dist/cli.js"
@@ -17,6 +17,7 @@
     "./harness-assert": "./dist/harness-assert.js",
     "./run-hook": "./dist/run-hook.js",
     "./plugin-loader": "./dist/plugin-loader.js",
+    "./mcp": "./dist/mcp.js",
     "./judge": "./dist/judge.js",
     "./mock-model": "./dist/mock-model.js",
     "./vitest": {
@@ -39,14 +40,16 @@
   ],
   "scripts": {
     "build": "tsc",
-    "test": "npm run build && node --test dist/spec.test.js dist/validate.test.js dist/cli.test.js dist/proofs.test.js dist/inline.test.js dist/sidecar.test.js dist/coverage.test.js dist/session.test.js dist/orphans.test.js dist/cedar.test.js dist/doc-refs.test.js dist/frontmatter.test.js dist/skill-pipeline.test.js dist/skill-runtime.test.js dist/skill-driver.test.js dist/skill-test.test.js dist/compile-generator.test.js dist/community-skills.test.js dist/action-gate.test.js dist/symbols.test.js dist/refs.test.js dist/harness-test.test.js dist/eval.test.js dist/run-scripts.test.js dist/plugin-loader.test.js dist/harness-assert.test.js dist/judge.test.js dist/run-hook.test.js",
+    "test": "npm run build && vitest run",
+    "coverage": "npm run build && vitest run --coverage",
     "lint": "eslint src/",
     "fmt": "prettier --write .",
     "fmt:check": "prettier --check .",
+    "demo": "npm run build && bash examples/demo/run.sh",
     "test:e2e": "bash test/e2e/run.sh",
     "test:harness": "npm run build && node dist/cli.js test",
     "test:eval": "npm run build && node dist/cli.js eval",
-    "test:vitest": "npm run build && vitest run",
+    "test:vitest": "npm run build && vitest run --project runners",
     "test:jest": "npm run build && jest",
     "test:types": "npm run build && tsc --noEmit -p test/types/tsconfig.json"
   },
@@ -57,6 +60,7 @@
     "@types/node": "^20.19.39",
     "@typescript-eslint/eslint-plugin": "^8.58.0",
     "@typescript-eslint/parser": "^8.58.0",
+    "@vitest/coverage-v8": "^4.1.8",
     "eslint": "^10.1.0",
     "eslint-plugin-sonarjs": "^4.0.2",
     "globals": "^17.4.0",