vigiles 2.0.0 → 2.2.0

package/dist/doc-refs.js

@@ -0,0 +1,246 @@
+"use strict";
+/**
+ * vigiles — Validate vigiles-builder calls in markdown code blocks.
+ *
+ * Mirror of inline mode but inverted: inline mode skips fenced code blocks
+ * (so `<!-- vigiles:enforce -->` in prose doesn't accidentally match an
+ * example). This module enters fenced code blocks (ts/typescript/js/
+ * javascript) and validates the vigiles builder calls inside —
+ * `enforce("...")`, `file("...")`, `cmd("...")`, `ref("...")` — using the
+ * same engines that validate them in spec.ts.
+ *
+ * Default: validate every ref. Illustrative blocks opt out via
+ * `<!-- vigiles:ignore -->` immediately before the fence. Whole files
+ * opt out via `<!-- vigiles:ignore-file -->` anywhere in the file
+ * (intended for research/design docs that quote hypothetical refs).
+ *
+ * Scope: ONLY vigiles builder calls. Generic TS syntax / type checking
+ * in markdown is explicitly out of scope — use eslint-plugin-markdown or
+ * twoslash for that.
+ */
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.extractDocRefs = extractDocRefs;
+exports.findDocRefs = findDocRefs;
+exports.formatDocRefReport = formatDocRefReport;
+const node_fs_1 = require("node:fs");
+const node_path_1 = require("node:path");
+const glob_1 = require("glob");
+const linters_js_1 = require("./linters.js");
+const compile_js_1 = require("./compile.js");
+// ---------------------------------------------------------------------------
+// Constants
+// ---------------------------------------------------------------------------
+const DEFAULT_IGNORE = [
+    "node_modules/**",
+    "dist/**",
+    ".vigiles/**",
+    ".git/**",
+];
+const FENCE_RE = /^(\s{0,3})(`{3,}|~{3,})(.*)$/;
+const TS_LANGS = new Set(["ts", "typescript", "js", "javascript"]);
+// Ignore markers must appear as standalone lines (whole line is the
+// comment, modulo whitespace) so an inline-code mention like
+// `` `<!-- vigiles:ignore-file -->` `` in prose documenting the syntax
+// doesn't accidentally disable validation.
+const IGNORE_BLOCK_RE = /^\s{0,3}<!--\s*vigiles:ignore\s*-->\s*$/;
+const IGNORE_FILE_RE = /^\s{0,3}<!--\s*vigiles:ignore-file\s*-->\s*$/m;
+const CALL_RE = /\b(enforce|file|cmd|ref)\(\s*["']([^"'\n]+)["']/g;
+const PLACEHOLDER_RE = /[<>]/;
+/**
+ * Error-message patterns that mean "tool not available in this env" rather
+ * than "ref is actually broken." We can't decide between valid and invalid
+ * when the underlying linter or CLI isn't installed, so count these
+ * separately from real errors.
+ */
+const UNVERIFIABLE_PATTERNS = [
+    /Unknown linter:/i,
+    /not found on PATH/i,
+    /No Cedar policies found/i,
+];
+/** @internal */ function extractDocRefs(content, file) {
+    const lines = content.split("\n");
+    const refs = [];
+    let blocksIgnored = 0;
+    let fenceChar = null;
+    let fenceLen = 0;
+    let fenceLang = "";
+    let blockLines = [];
+    let nextBlockIgnored = false;
+    for (let i = 0; i < lines.length; i++) {
+        const line = lines[i];
+        const fm = FENCE_RE.exec(line);
+        if (fm) {
+            const marker = fm[2];
+            const ch = marker[0];
+            const len = marker.length;
+            const info = fm[3].trim();
+            if (fenceChar === null) {
+                fenceChar = ch;
+                fenceLen = len;
+                fenceLang = info.split(/\s+/)[0].toLowerCase();
+                blockLines = [];
+                continue;
+            }
+            else if (ch === fenceChar && len >= fenceLen && info === "") {
+                // Closing fence
+                if (TS_LANGS.has(fenceLang)) {
+                    if (nextBlockIgnored) {
+                        blocksIgnored++;
+                    }
+                    else {
+                        for (const { lineNo, text } of blockLines) {
+                            for (const m of text.matchAll(CALL_RE)) {
+                                refs.push({
+                                    file,
+                                    line: lineNo,
+                                    kind: m[1],
+                                    value: m[2],
+                                });
+                            }
+                        }
+                    }
+                }
+                fenceChar = null;
+                fenceLen = 0;
+                fenceLang = "";
+                nextBlockIgnored = false;
+                blockLines = [];
+                continue;
+            }
+        }
+        if (fenceChar !== null) {
+            blockLines.push({ lineNo: i + 1, text: line });
+            continue;
+        }
+        if (IGNORE_BLOCK_RE.test(line)) {
+            nextBlockIgnored = true;
+        }
+    }
+    return { refs, blocksIgnored };
+}
+function isUnverifiable(message) {
+    return UNVERIFIABLE_PATTERNS.some((p) => p.test(message));
+}
+function validateRefs(refs, basePath) {
+    const errors = [];
+    let unverified = 0;
+    let placeholders = 0;
+    const scripts = (0, compile_js_1.readPackageScripts)(basePath) ?? {};
+    for (const r of refs) {
+        // Skip obvious placeholders like enforce("<linter>/<rule>") that
+        // appear in skill format documentation. They aren't typos — they're
+        // syntax templates. Real refs don't contain < or >.
+        if (PLACEHOLDER_RE.test(r.value)) {
+            placeholders++;
+            continue;
+        }
+        switch (r.kind) {
+            case "enforce": {
+                const result = (0, linters_js_1.checkLinterRule)(r.value, basePath, {
+                    catalogOnly: true,
+                });
+                if (!result.exists) {
+                    const msg = result.error ?? `Rule "${r.value}" not found`;
+                    if (isUnverifiable(msg)) {
+                        unverified++;
+                    }
+                    else {
+                        errors.push({ ...r, message: msg });
+                    }
+                }
+                break;
+            }
+            case "file":
+            case "ref": {
+                if (!(0, node_fs_1.existsSync)((0, node_path_1.resolve)(basePath, r.value))) {
+                    errors.push({ ...r, message: `File not found: "${r.value}"` });
+                }
+                break;
+            }
+            case "cmd": {
+                const npmRun = r.value.match(/^npm\s+run\s+(\S+)/);
+                const npmDirect = r.value.match(/^npm\s+(test|start|build|pretest)\b/);
+                const scriptName = npmRun?.[1] ?? npmDirect?.[1];
+                if (scriptName && !scripts[scriptName]) {
+                    errors.push({
+                        ...r,
+                        message: `Script "${scriptName}" not found in package.json`,
+                    });
+                }
+                break;
+            }
+        }
+    }
+    return { errors, unverified, placeholders };
+}
+// ---------------------------------------------------------------------------
+// Public API
+// ---------------------------------------------------------------------------
+/**
+ * Walk every `.md` under `basePath`, extract vigiles builder calls from
+ * fenced TS/JS code blocks, validate against the same engines used for
+ * spec.ts. Honors `<!-- vigiles:ignore-file -->` (skip the whole file)
+ * and `<!-- vigiles:ignore -->` (skip the next code block).
+ */
+function findDocRefs(options = {}) {
+    const basePath = options.basePath ?? process.cwd();
+    const ignore = [...DEFAULT_IGNORE, ...(options.ignore ?? [])];
+    const files = (0, glob_1.globSync)("**/*.md", { cwd: basePath, ignore });
+    const allRefs = [];
+    let filesIgnored = 0;
+    let blocksIgnored = 0;
+    for (const f of files) {
+        let content;
+        try {
+            content = (0, node_fs_1.readFileSync)((0, node_path_1.resolve)(basePath, f), "utf-8");
+        }
+        catch {
+            continue;
+        }
+        if (IGNORE_FILE_RE.test(content)) {
+            filesIgnored++;
+            continue;
+        }
+        const r = extractDocRefs(content, f);
+        allRefs.push(...r.refs);
+        blocksIgnored += r.blocksIgnored;
+    }
+    const outcome = validateRefs(allRefs, basePath);
+    return {
+        filesScanned: files.length,
+        filesIgnored,
+        blocksIgnored,
+        refs: allRefs,
+        errors: outcome.errors,
+        unverified: outcome.unverified,
+        placeholders: outcome.placeholders,
+    };
+}
+/** Format a DocRefReport as human-readable text. */
+function formatDocRefReport(report) {
+    const lines = [];
+    const meta = [];
+    if (report.filesIgnored > 0)
+        meta.push(`${String(report.filesIgnored)} via vigiles:ignore-file`);
+    if (report.blocksIgnored > 0)
+        meta.push(`${String(report.blocksIgnored)} blocks via vigiles:ignore`);
+    if (report.placeholders > 0)
+        meta.push(`${String(report.placeholders)} placeholders skipped`);
+    if (report.unverified > 0)
+        meta.push(`${String(report.unverified)} unverified (tool unavailable)`);
+    const metaStr = meta.length > 0 ? ` (${meta.join(", ")})` : "";
+    lines.push(`scanned ${String(report.filesScanned)} files${metaStr}`);
+    lines.push(`${String(report.refs.length)} vigiles refs in code blocks${report.errors.length === 0 ? " — all valid" : ""}`);
+    if (report.errors.length === 0)
+        return lines.join("\n");
+    lines.push(`✗ ${String(report.errors.length)} broken ref(s):`);
+    for (const e of report.errors.slice(0, 12)) {
+        const trunc = e.message.length > 80 ? `${e.message.slice(0, 80)}…` : e.message;
+        lines.push(`    ${e.file}:${String(e.line)} ${e.kind}("${e.value}") — ${trunc}`);
+    }
+    if (report.errors.length > 12) {
+        lines.push(`    ... +${String(report.errors.length - 12)} more`);
+    }
+    return lines.join("\n");
+}
+//# sourceMappingURL=doc-refs.js.map

package/dist/eval.d.ts

@@ -0,0 +1,62 @@
+/** One arm of the comparison: fixture overrides + settings (hooks) for this arm. */
+export interface EvalArm {
+    /** Files written on top of the base fixture for this arm. */
+    readonly files?: Record<string, string>;
+    /** `.claude/settings.json` (hooks/permissions) for this arm; omit for none. */
+    readonly settings?: unknown;
+}
+/** Context handed to `measure` after a run, to compute that run's metrics. */
+export interface RunContext {
+    readonly cwd: string;
+    readonly exitCode: number;
+    readonly stdout: string;
+    /** `num_turns` reported by claude, or 0. */
+    readonly turns: number;
+    /** Contents of a file under the working dir, or null if absent. */
+    file(path: string): string | null;
+    /** Run a shell command in the working dir; returns trimmed stdout ("" on error). */
+    sh(command: string): string;
+}
+export type Metrics = Record<string, number | boolean>;
+export interface EvalSpec<M extends Metrics> {
+    readonly name?: string;
+    /** Base fixture files (path → contents), written fresh for every run. */
+    readonly fixture?: Record<string, string>;
+    /** The arms to compare, by name. */
+    readonly arms: Record<string, EvalArm>;
+    /** The task prompt given to the agent. */
+    readonly task: string;
+    /** Compute this run's metrics from its outcome. */
+    readonly measure: (ctx: RunContext) => M;
+    /** Trials per arm. Default 5. */
+    readonly trials?: number;
+    /** Model alias. Default "haiku". */
+    readonly model?: string;
+    /** Tools the agent may use. Default: Read Edit Write Bash. */
+    readonly allowedTools?: readonly string[];
+    /** Per-run timeout ms. Default 240000. */
+    readonly timeoutMs?: number;
+    /** Seconds to wait between runs (avoid rate-limit bursts). Default 4. */
+    readonly spacingSec?: number;
+}
+export interface ArmReport {
+    readonly runs: number;
+    /** Aggregated metrics: mean for numbers, fraction-true (0..1) for booleans. */
+    readonly metrics: Record<string, number>;
+}
+export interface EvalReport {
+    readonly name: string;
+    readonly trials: number;
+    readonly arms: Record<string, ArmReport>;
+}
+/** Aggregate per-run metrics: mean for numbers, fraction-true (0..1) for booleans. */
+export declare function aggregate(rows: readonly Metrics[]): Record<string, number>;
+/**
+ * Run the eval: every arm × every trial against the real `claude` CLI, with the
+ * metric computed per run and aggregated per arm. Requires `claude` on PATH and
+ * working model auth (e.g. `ANTHROPIC_API_KEY`).
+ */
+export declare function runEval<M extends Metrics>(spec: EvalSpec<M>): Promise<EvalReport>;
+/** Format an eval report as a compact table for the console. */
+export declare function formatEvalReport(report: EvalReport): string;
+//# sourceMappingURL=eval.d.ts.map

package/dist/eval.js

@@ -0,0 +1,174 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.aggregate = aggregate;
+exports.runEval = runEval;
+exports.formatEvalReport = formatEvalReport;
+/**
+ * vigiles — Claude Code harness *evals*.
+ *
+ * Measure whether a harness change actually changes agent behaviour. Define a
+ * fixture, a set of **arms** (e.g. a hook on vs off, with/without a CLAUDE.md
+ * rule), a task prompt, and a **metric**; `runEval` drives the real `claude` CLI
+ * N trials per arm and aggregates. This is the generalized form of the
+ * benchmark harness under `bench/` — the empirical half of testing your harness.
+ *
+ *   const report = await runEval({
+ *     fixture: { "src/billing.ts": "export function chargeCard(){}" },
+ *     arms: {
+ *       vanilla: {},
+ *       gated:  { settings: { hooks: { PostToolUse: [refsHook] } } },
+ *     },
+ *     task: "document chargeCard in SKILL.md, referencing it by name",
+ *     measure: (ctx) => ({ marked: ctx.sh("grep -c vigiles:symbol SKILL.md") > 0 }),
+ *     trials: 6,
+ *   });
+ *
+ * Real model → real cost + statistical, not deterministic. For fast, free,
+ * deterministic checks of hook *logic*, see `harness-test.ts`.
+ */
+const node_child_process_1 = require("node:child_process");
+const node_fs_1 = require("node:fs");
+const node_os_1 = require("node:os");
+const node_path_1 = require("node:path");
+function writeFiles(cwd, files) {
+    for (const [p, content] of Object.entries(files)) {
+        const full = (0, node_path_1.resolve)(cwd, p);
+        (0, node_fs_1.mkdirSync)((0, node_path_1.dirname)(full), { recursive: true });
+        (0, node_fs_1.writeFileSync)(full, content);
+    }
+}
+function spawnAgent(task, cwd, model, tools, hasSettings, timeoutMs) {
+    return new Promise((resolvePromise) => {
+        const args = [
+            "-p",
+            task,
+            "--output-format",
+            "json",
+            "--model",
+            model,
+            "--permission-mode",
+            "acceptEdits",
+            ...(hasSettings ? ["--settings", "settings.json"] : []),
+            "--allowedTools",
+            ...tools,
+        ];
+        const child = (0, node_child_process_1.spawn)("claude", args, {
+            cwd,
+            env: process.env,
+            stdio: ["ignore", "pipe", "pipe"],
+        });
+        let stdout = "";
+        child.stdout.on("data", (d) => (stdout += d.toString()));
+        const timer = setTimeout(() => child.kill("SIGKILL"), timeoutMs);
+        child.on("close", (code) => {
+            clearTimeout(timer);
+            resolvePromise({ code: code ?? 0, stdout });
+        });
+    });
+}
+const sleep = (ms) => new Promise((r) => setTimeout(r, ms));
+function makeContext(cwd, out) {
+    let turns = 0;
+    try {
+        turns = JSON.parse(out.stdout).num_turns ?? 0;
+    }
+    catch {
+        /* non-JSON output */
+    }
+    return {
+        cwd,
+        exitCode: out.code,
+        stdout: out.stdout,
+        turns,
+        file: (p) => {
+            const f = (0, node_path_1.resolve)(cwd, p);
+            return (0, node_fs_1.existsSync)(f) ? (0, node_fs_1.readFileSync)(f, "utf-8") : null;
+        },
+        sh: (command) => {
+            try {
+                return (0, node_child_process_1.execSync)(command, {
+                    cwd,
+                    encoding: "utf-8",
+                    stdio: ["ignore", "pipe", "ignore"],
+                }).trim();
+            }
+            catch (e) {
+                // Return captured stdout even on a non-zero exit (e.g. `audit` exits 2
+                // but still prints its findings), rather than swallowing it.
+                const out = e.stdout;
+                return typeof out === "string" ? out.trim() : "";
+            }
+        },
+    };
+}
+/** Aggregate per-run metrics: mean for numbers, fraction-true (0..1) for booleans. */
+function aggregate(rows) {
+    const keys = new Set();
+    for (const r of rows)
+        for (const k of Object.keys(r))
+            keys.add(k);
+    const out = {};
+    for (const k of keys) {
+        let sum = 0;
+        let n = 0;
+        for (const r of rows) {
+            const v = r[k];
+            if (typeof v === "number") {
+                sum += v;
+                n++;
+            }
+            else if (typeof v === "boolean") {
+                sum += v ? 1 : 0;
+                n++;
+            }
+        }
+        out[k] = n > 0 ? sum / n : 0;
+    }
+    return out;
+}
+/**
+ * Run the eval: every arm × every trial against the real `claude` CLI, with the
+ * metric computed per run and aggregated per arm. Requires `claude` on PATH and
+ * working model auth (e.g. `ANTHROPIC_API_KEY`).
+ */
+async function runEval(spec) {
+    const trials = spec.trials ?? 5;
+    const model = spec.model ?? "haiku";
+    const tools = spec.allowedTools ?? ["Read", "Edit", "Write", "Bash"];
+    const timeoutMs = spec.timeoutMs ?? 240000;
+    const spacing = (spec.spacingSec ?? 4) * 1000;
+    const arms = {};
+    for (const [armName, arm] of Object.entries(spec.arms)) {
+        const rows = [];
+        for (let t = 0; t < trials; t++) {
+            const cwd = (0, node_fs_1.mkdtempSync)((0, node_path_1.join)((0, node_os_1.tmpdir)(), "vigiles-eval-"));
+            try {
+                writeFiles(cwd, { ...spec.fixture, ...arm.files });
+                const hasSettings = arm.settings !== undefined;
+                if (hasSettings) {
+                    (0, node_fs_1.writeFileSync)((0, node_path_1.join)(cwd, "settings.json"), JSON.stringify(arm.settings, null, 2).replaceAll("{cwd}", cwd));
+                }
+                const out = await spawnAgent(spec.task, cwd, model, tools, hasSettings, timeoutMs);
+                rows.push(spec.measure(makeContext(cwd, out)));
+            }
+            finally {
+                (0, node_fs_1.rmSync)(cwd, { recursive: true, force: true });
+                await sleep(spacing);
+            }
+        }
+        arms[armName] = { runs: rows.length, metrics: aggregate(rows) };
+    }
+    return { name: spec.name ?? "eval", trials, arms };
+}
+/** Format an eval report as a compact table for the console. */
+function formatEvalReport(report) {
+    const lines = [`${report.name} (${String(report.trials)} trials/arm)`];
+    for (const [arm, r] of Object.entries(report.arms)) {
+        const parts = Object.entries(r.metrics)
+            .map(([k, v]) => `${k}=${v.toFixed(2)}`)
+            .join("  ");
+        lines.push(`  ${arm.padEnd(10)} ${parts}`);
+    }
+    return lines.join("\n");
+}
+//# sourceMappingURL=eval.js.map