vigiles 2.1.1 → 2.3.0

package/dist/inline.js

@@ -29,6 +29,17 @@ exports.hasInlineRules = hasInlineRules;
  * embedded quotes, they can move to spec mode.
  */
 const ENFORCE_RE = /<!--\s*vigiles:enforce\s+([@A-Za-z0-9_/:.-]+)\s+"([^"\n]*)"\s*-->/;
+/**
+ * Match `<!-- vigiles:file <path> -->`. The path is a single whitespace-free
+ * token (project-relative); spaces in paths are vanishingly rare and would be
+ * ambiguous against the closing `-->`.
+ */
+const FILE_RE = /<!--\s*vigiles:file\s+(\S+)\s*-->/;
+/**
+ * Match `<!-- vigiles:cmd "<command>" -->`. The command is quoted because
+ * commands contain spaces (e.g. `npm run build`).
+ */
+const CMD_RE = /<!--\s*vigiles:cmd\s+"([^"\n]*)"\s*-->/;
 /**
  * Detects any `<!-- vigiles:<kind> -->` comment (valid or not) so we can
  * surface errors for typos and reserved-but-unrecognized kinds. Uses a
@@ -36,6 +47,16 @@ const ENFORCE_RE = /<!--\s*vigiles:enforce\s+([@A-Za-z0-9_/:.-]+)\s+"([^"\n]*)"\
  * circuit the pattern.
  */
 const MARKER_RE = /<!--\s*vigiles:([A-Za-z_-]+)[^]*?-->/;
+// vigiles markers handled by other subsystems (skill runtime gates, opt-outs).
+// The inline *rule* parser skips them rather than flagging them as unknown.
+const KNOWN_NON_RULE_MARKERS = new Set([
+    "disable",
+    "ignore",
+    "ignore-file",
+    "gate", // skill step gate (src/skill-runtime.ts)
+    "result", // skill result gate
+    "symbol", // symbol reference mark (src/refs.ts)
+]);
 /**
  * Parse inline vigiles rules out of a markdown file's contents.
  * Does not touch the filesystem and does not verify the rules against
@@ -48,6 +69,8 @@ const MARKER_RE = /<!--\s*vigiles:([A-Za-z_-]+)[^]*?-->/;
  */
 function parseInlineRules(content) {
     const rules = [];
+    const files = [];
+    const commands = [];
     const errors = [];
     const lines = content.split("\n");
     let fenceChar = null;
@@ -97,15 +120,24 @@ function parseInlineRules(content) {
             });
             continue;
         }
+        const fileMatch = FILE_RE.exec(scannable);
+        if (fileMatch) {
+            files.push({ path: fileMatch[1], line: i + 1 });
+            continue;
+        }
+        const cmdMatch = CMD_RE.exec(scannable);
+        if (cmdMatch) {
+            commands.push({ command: cmdMatch[1], line: i + 1 });
+            continue;
+        }
         // Skip the compiled-file hash header (`<!-- vigiles:sha256:... -->`)
         // entirely — it's not a rule marker and should not be reported.
         if (/<!--\s*vigiles:sha\d+:/.test(scannable))
             continue;
         const markerMatch = MARKER_RE.exec(scannable);
         if (markerMatch) {
-            // Looks like a vigiles marker but didn't parse as enforce —
-            // surface it so users catch typos like "vigile:enforce" or
-            // unquoted why.
+            // Looks like a vigiles marker but didn't parse — surface it so users
+            // catch typos like "vigile:enforce" or an unquoted why/command.
             const kind = markerMatch[1];
             if (kind === "enforce") {
                 errors.push({
@@ -114,29 +146,44 @@ function parseInlineRules(content) {
                     raw: line.trim(),
                 });
             }
-            else if (kind !== "disable" && kind !== "ignore") {
-                // `vigiles:disable ...` / `vigiles:ignore ...` are reserved for
-                // future disable-comment support; don't complain about them.
+            else if (kind === "file") {
+                errors.push({
+                    line: i + 1,
+                    message: "Malformed vigiles:file — expected `<!-- vigiles:file <path> -->`",
+                    raw: line.trim(),
+                });
+            }
+            else if (kind === "cmd") {
+                errors.push({
+                    line: i + 1,
+                    message: 'Malformed vigiles:cmd — expected `<!-- vigiles:cmd "<command>" -->`',
+                    raw: line.trim(),
+                });
+            }
+            else if (!KNOWN_NON_RULE_MARKERS.has(kind)) {
                 errors.push({
                     line: i + 1,
-                    message: `Unknown vigiles marker "${kind}". Only \`vigiles:enforce\` is supported.`,
+                    message: `Unknown vigiles marker "${kind}". Only \`vigiles:enforce\`, \`vigiles:file\`, and \`vigiles:cmd\` are supported.`,
                     raw: line.trim(),
                 });
             }
         }
     }
-    return { rules, errors };
+    return { rules, files, commands, errors };
 }
 /**
- * True if the content contains at least one parseable vigiles:enforce
- * rule (ignoring fenced code blocks and malformed markers). Used by
- * `require-spec` validation to treat inline mode as spec-equivalent.
+ * True if the content contains at least one parseable vigiles inline marker —
+ * an `enforce` rule, a `file` reference, or a `cmd` reference (ignoring fenced
+ * code blocks and malformed markers). Used by `require-spec` validation to
+ * treat inline mode as spec-equivalent: a file that pins even a single path is
+ * meaningfully managed.
  *
  * Deliberately delegates to `parseInlineRules` so a loose prefix regex
  * can't satisfy require-spec with a malformed marker that produces no
- * real enforceable rule.
+ * real reference.
  */
 function hasInlineRules(content) {
-    return parseInlineRules(content).rules.length > 0;
+    const r = parseInlineRules(content);
+    return r.rules.length + r.files.length + r.commands.length > 0;
 }
 //# sourceMappingURL=inline.js.map

package/dist/jest.d.ts

@@ -0,0 +1,9 @@
+declare module "@jest/expect" {
+    interface Matchers<R> {
+        toHaveCreated(path: string): R;
+        toBlock(): R;
+        toBeatBaseline(baseline: string, arm: string, metric: string, by?: number): R;
+    }
+}
+export {};
+//# sourceMappingURL=jest.d.ts.map

package/dist/jest.js

@@ -0,0 +1,23 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+/* eslint-disable max-params --
+   The matcher signatures mirror the runtime vigilesMatchers (positional args). */
+/**
+ * vigiles — jest integration (opt-in).
+ *
+ * Importing this entry registers the vigiles matchers AND augments jest's types
+ * so `toHaveCreated` / `toBeatBaseline` type-check.
+ *
+ *   // jest.config.js →  setupFilesAfterEnv: ["vigiles/jest"]
+ *   // …or at the top of a test file:
+ *   import "vigiles/jest";
+ *
+ *   expect(result).toHaveCreated("DONE");
+ *   expect(report).toBeatBaseline("vanilla", "gated", "caught");
+ *
+ * jest is an optional peer dependency — only jest users load this entry.
+ */
+const globals_1 = require("@jest/globals");
+const harness_assert_js_1 = require("./harness-assert.js");
+globals_1.expect.extend(harness_assert_js_1.vigilesMatchers);
+//# sourceMappingURL=jest.js.map

package/dist/judge.d.ts

@@ -0,0 +1,29 @@
+export interface JudgeResult {
+    /** Score in [0, 1] (clamped). 0 on any failure to obtain a verdict. */
+    readonly score: number;
+    /** score ≥ threshold (default 0.5). */
+    readonly pass: boolean;
+    /** The model's one-line rationale, or an error string. */
+    readonly reason: string;
+}
+export interface JudgeOptions {
+    /** The text to grade. */
+    readonly output: string;
+    /** The grading rubric — describe what earns a high vs low score. */
+    readonly rubric: string;
+    /** Model alias. Default "haiku" (cheap; judging is a simple call). */
+    readonly model?: string;
+    /** pass = score ≥ threshold. Default 0.5. */
+    readonly threshold?: number;
+    /** Per-call timeout ms. Default 60000. */
+    readonly timeoutMs?: number;
+}
+/** Grade `output` against `rubric` with a model. Synchronous (for `measure`). */
+export declare function judge(opts: JudgeOptions): JudgeResult;
+/**
+ * Parse a verdict out of the grader's stdout — pure, so the parsing is testable
+ * without a model. Handles `claude --output-format json` (text wrapped in a
+ * `result` field), bare/prose-wrapped JSON, clamping, and the pass threshold.
+ */
+export declare function parseJudgeOutput(stdout: string, threshold?: number): JudgeResult;
+//# sourceMappingURL=judge.d.ts.map

package/dist/judge.js

@@ -0,0 +1,88 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.judge = judge;
+exports.parseJudgeOutput = parseJudgeOutput;
+/**
+ * vigiles — a thin LLM-as-judge for the eval tier.
+ *
+ * Some outcomes aren't a regex: "is this commit message clear?", "did the SKILL
+ * produce a sensible plan?". `judge` grades an output against a rubric with a
+ * model and returns a numeric score + pass/fail, for use *inside* an eval's
+ * `measure` (which is synchronous — so this shells out via the `claude` CLI
+ * synchronously, no extra deps):
+ *
+ *   measure: (ctx) => {
+ *     const v = judge({ output: ctx.file("PLAN.md") ?? "", rubric:
+ *       "1 if the plan lists concrete, ordered steps; else 0." });
+ *     return { quality: v.score, ok: v.pass };
+ *   }
+ *
+ * This is deliberately minimal — for datasets, tracing, and dashboards use a
+ * dedicated eval platform (Braintrust, DeepEval). vigiles owns the harness A/B,
+ * not the judging platform. Needs the `claude` CLI + model auth.
+ */
+const node_child_process_1 = require("node:child_process");
+const clamp01 = (n) => Math.max(0, Math.min(1, n));
+/** Extract the first JSON object from a string (models often wrap it in prose). */
+function firstJsonObject(s) {
+    const start = s.indexOf("{");
+    const end = s.lastIndexOf("}");
+    if (start === -1 || end <= start)
+        return null;
+    try {
+        return JSON.parse(s.slice(start, end + 1));
+    }
+    catch {
+        return null;
+    }
+}
+/** Grade `output` against `rubric` with a model. Synchronous (for `measure`). */
+function judge(opts) {
+    const threshold = opts.threshold ?? 0.5;
+    const prompt = "You are a strict grader. Score the OUTPUT against the RUBRIC. " +
+        'Respond with ONLY a JSON object: {"score": <number 0..1>, "reason": "<one line>"}.\n\n' +
+        `RUBRIC:\n${opts.rubric}\n\nOUTPUT:\n${opts.output}`;
+    let res;
+    try {
+        res = (0, node_child_process_1.spawnSync)("claude", [
+            "-p",
+            prompt,
+            "--model",
+            opts.model ?? "haiku",
+            "--output-format",
+            "json",
+        ], { encoding: "utf-8", timeout: opts.timeoutMs ?? 60000 });
+    }
+    catch (e) {
+        return {
+            score: 0,
+            pass: false,
+            reason: `judge spawn failed: ${String(e)}`,
+        };
+    }
+    if (res.status !== 0) {
+        return { score: 0, pass: false, reason: "judge: no model output" };
+    }
+    return parseJudgeOutput(res.stdout ?? "", threshold);
+}
+/**
+ * Parse a verdict out of the grader's stdout — pure, so the parsing is testable
+ * without a model. Handles `claude --output-format json` (text wrapped in a
+ * `result` field), bare/prose-wrapped JSON, clamping, and the pass threshold.
+ */
+function parseJudgeOutput(stdout, threshold = 0.5) {
+    if (!stdout)
+        return { score: 0, pass: false, reason: "judge: no model output" };
+    // claude --output-format json wraps the model text in a `result` field.
+    let text = stdout;
+    const wrapper = firstJsonObject(stdout);
+    if (wrapper && typeof wrapper.result === "string")
+        text = wrapper.result;
+    const verdict = firstJsonObject(text);
+    if (!verdict || typeof verdict.score !== "number") {
+        return { score: 0, pass: false, reason: "judge: unparseable verdict" };
+    }
+    const score = clamp01(verdict.score);
+    return { score, pass: score >= threshold, reason: verdict.reason ?? "" };
+}
+//# sourceMappingURL=judge.js.map

package/dist/linters.js

@@ -20,6 +20,34 @@ const node_path_1 = require("node:path");
 const node_child_process_1 = require("node:child_process");
 const node_module_1 = require("node:module");
 const glob_1 = require("glob");
+/**
+ * Prepend version-manager shim directories (rbenv / asdf / rvm) to PATH so
+ * gem/pip-installed linters are found in non-login shells and CI, where the
+ * shims dir is often missing from PATH even though the tool is installed.
+ * Runs once; only adds directories that exist and aren't already present.
+ */
+function augmentToolPath() {
+    const home = process.env.HOME ?? "";
+    // rbenv/asdf shims don't resolve without a selected version, so add the
+    // concrete per-version `bin` dirs (where the gem executables actually live).
+    const candidates = [
+        ...(0, glob_1.globSync)("/opt/rbenv/versions/*/bin", { nodir: false }),
+        ...(home
+            ? (0, glob_1.globSync)(`${home}/.rbenv/versions/*/bin`, { nodir: false })
+            : []),
+        ...(home
+            ? (0, glob_1.globSync)(`${home}/.asdf/installs/*/*/bin`, { nodir: false })
+            : []),
+        `${home}/.rvm/bin`,
+        `${home}/.local/bin`,
+    ];
+    const current = (process.env.PATH ?? "").split(":");
+    const additions = candidates.filter((d) => d && (0, node_fs_1.existsSync)(d) && !current.includes(d));
+    if (additions.length > 0) {
+        process.env.PATH = [...current, ...additions].join(":");
+    }
+}
+augmentToolPath();
 // ---------------------------------------------------------------------------
 // Parsing enforcement references
 // ---------------------------------------------------------------------------

package/dist/mock-model.d.ts

@@ -0,0 +1,31 @@
+/** One scripted assistant turn: a final text answer, or a tool call. */
+export interface ModelTurn {
+    /** Final text answer (stops the turn). */
+    readonly text?: string;
+    /** A tool to invoke, e.g. "Bash" | "Write" | "Edit". */
+    readonly tool?: string;
+    /** The tool input, e.g. `{ file_path, content }` or `{ command }`. */
+    readonly input?: Record<string, unknown>;
+}
+/** Build a scripted model from an ordered list of turns. */
+export declare function scriptModel(turns: readonly ModelTurn[]): ModelTurn[];
+export interface TurnInfo {
+    readonly n: number;
+    readonly stream: boolean;
+    readonly hasToolResult: boolean;
+}
+export interface MockHandle {
+    readonly url: string;
+    close(): void;
+    /** Number of model turns served so far. */
+    readonly count: number;
+}
+/**
+ * Start the scripted mock on a free port. Each `/v1/messages` POST consumes the
+ * next turn (the last turn repeats if the client asks for more). Resolves to a
+ * handle with the base `url` and a `close()`.
+ */
+export declare function startMock(script: readonly ModelTurn[], opts?: {
+    onTurn?: (info: TurnInfo) => void;
+}): Promise<MockHandle>;
+//# sourceMappingURL=mock-model.d.ts.map

package/dist/mock-model.js

@@ -0,0 +1,189 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.scriptModel = scriptModel;
+exports.startMock = startMock;
+/**
+ * vigiles — a scripted, deterministic Anthropic Messages API mock.
+ *
+ * Point a Claude Code client at it with `ANTHROPIC_BASE_URL` (and any dummy
+ * `ANTHROPIC_API_KEY`) and it serves a fixed *script* of model turns in order —
+ * each `POST /v1/messages` returns the next. This is the seam that makes harness
+ * testing deterministic: the real `claude` CLI runs your real hooks/settings,
+ * but the model's turns are scripted, so the outcome is reproducible and free.
+ *
+ *   scriptModel([
+ *     { tool: "Write", input: { file_path: "SKILL.md", content: "..." } },
+ *     { text: "done" },
+ *   ])
+ *
+ * Implements the parts a real client needs: SSE streaming (flushed per event),
+ * `/v1/messages/count_tokens` (else Claude Code hangs), HEAD/health tolerance,
+ * and echoing the requested model.
+ */
+const node_http_1 = __importDefault(require("node:http"));
+/** Build a scripted model from an ordered list of turns. */
+function scriptModel(turns) {
+    return [...turns];
+}
+function writeEvent(res, event, data) {
+    res.write(`event: ${event}\n`);
+    res.write(`data: ${JSON.stringify(data)}\n\n`);
+}
+const rid = (p) => `${p}${Math.random().toString(36).slice(2, 10)}`;
+function streamTurn(res, turn, model) {
+    res.writeHead(200, {
+        "content-type": "text/event-stream",
+        "cache-control": "no-cache",
+        connection: "keep-alive",
+    });
+    writeEvent(res, "message_start", {
+        type: "message_start",
+        message: {
+            id: rid("msg_"),
+            type: "message",
+            role: "assistant",
+            model,
+            content: [],
+            stop_reason: null,
+            usage: { input_tokens: 10, output_tokens: 1 },
+        },
+    });
+    if (turn.tool) {
+        writeEvent(res, "content_block_start", {
+            type: "content_block_start",
+            index: 0,
+            content_block: {
+                type: "tool_use",
+                id: rid("toolu_"),
+                name: turn.tool,
+                input: {},
+            },
+        });
+        writeEvent(res, "content_block_delta", {
+            type: "content_block_delta",
+            index: 0,
+            delta: {
+                type: "input_json_delta",
+                partial_json: JSON.stringify(turn.input ?? {}),
+            },
+        });
+        writeEvent(res, "content_block_stop", {
+            type: "content_block_stop",
+            index: 0,
+        });
+        writeEvent(res, "message_delta", {
+            type: "message_delta",
+            delta: { stop_reason: "tool_use", stop_sequence: null },
+            usage: { output_tokens: 5 },
+        });
+    }
+    else {
+        writeEvent(res, "content_block_start", {
+            type: "content_block_start",
+            index: 0,
+            content_block: { type: "text", text: "" },
+        });
+        writeEvent(res, "content_block_delta", {
+            type: "content_block_delta",
+            index: 0,
+            delta: { type: "text_delta", text: turn.text ?? "" },
+        });
+        writeEvent(res, "content_block_stop", {
+            type: "content_block_stop",
+            index: 0,
+        });
+        writeEvent(res, "message_delta", {
+            type: "message_delta",
+            delta: { stop_reason: "end_turn", stop_sequence: null },
+            usage: { output_tokens: 5 },
+        });
+    }
+    writeEvent(res, "message_stop", { type: "message_stop" });
+    res.end();
+}
+function jsonTurn(res, turn, model) {
+    const content = turn.tool
+        ? [
+            {
+                type: "tool_use",
+                id: rid("toolu_"),
+                name: turn.tool,
+                input: turn.input ?? {},
+            },
+        ]
+        : [{ type: "text", text: turn.text ?? "" }];
+    res.writeHead(200, { "content-type": "application/json" });
+    res.end(JSON.stringify({
+        id: rid("msg_"),
+        type: "message",
+        role: "assistant",
+        model,
+        stop_reason: turn.tool ? "tool_use" : "end_turn",
+        stop_sequence: null,
+        content,
+        usage: { input_tokens: 10, output_tokens: 5 },
+    }));
+}
+/**
+ * Start the scripted mock on a free port. Each `/v1/messages` POST consumes the
+ * next turn (the last turn repeats if the client asks for more). Resolves to a
+ * handle with the base `url` and a `close()`.
+ */
+function startMock(script, opts = {}) {
+    let i = 0;
+    const server = node_http_1.default.createServer((req, res) => {
+        let body = "";
+        req.on("data", (c) => (body += c));
+        req.on("end", () => {
+            const url = req.url ?? "";
+            const isCount = url.includes("count_tokens");
+            const isMessages = url.includes("/v1/messages") && !isCount;
+            let reqBody = {};
+            try {
+                reqBody = JSON.parse(body);
+            }
+            catch {
+                /* HEAD / health checks have no JSON body */
+            }
+            if (req.method === "HEAD" || (!isMessages && !isCount)) {
+                res.writeHead(200, { "content-type": "application/json" });
+                res.end("{}");
+                return;
+            }
+            if (isCount) {
+                res.writeHead(200, { "content-type": "application/json" });
+                res.end(JSON.stringify({ input_tokens: 10 }));
+                return;
+            }
+            const last = JSON.stringify(reqBody.messages?.at(-1)?.content ?? "");
+            opts.onTurn?.({
+                n: i,
+                stream: reqBody.stream === true,
+                hasToolResult: last.includes('"tool_result"'),
+            });
+            const turn = script[Math.min(i, script.length - 1)] ?? { text: "" };
+            i++;
+            const model = reqBody.model ?? "claude-mock";
+            if (reqBody.stream === true)
+                streamTurn(res, turn, model);
+            else
+                jsonTurn(res, turn, model);
+        });
+    });
+    return new Promise((resolve) => {
+        server.listen(0, "127.0.0.1", () => {
+            const { port } = server.address();
+            resolve({
+                url: `http://127.0.0.1:${String(port)}`,
+                close: () => server.close(),
+                get count() {
+                    return i;
+                },
+            });
+        });
+    });
+}
+//# sourceMappingURL=mock-model.js.map

package/dist/plugin-loader.d.ts

@@ -0,0 +1,37 @@
+export interface LoadedPlugin {
+    /** A `.claude/settings.json`-shaped object with hooks resolved. */
+    readonly settings: {
+        hooks?: unknown;
+    };
+    /** Files to materialize in the sandbox (CLAUDE.md, skills, agents, commands). */
+    readonly files: Record<string, string>;
+    /**
+     * Surfaces that are present in the plugin but cannot be exercised at the
+     * deterministic tier (subagents and slash commands need a real model; MCP
+     * servers aren't wired by the loader). Empty when the plugin is fully
+     * covered. Surfaced so "load the whole plugin" never silently tests nothing —
+     * read it in a test, or just to know what the deterministic run won't reach.
+     */
+    readonly warnings: readonly string[];
+}
+/**
+ * Load the real harness at `pluginPath`. Returns the resolved settings (hooks),
+ * the files (CLAUDE.md + skills + agents + commands) to write into the test
+ * sandbox, and `warnings` for surfaces the deterministic tier can't drive. Merge
+ * `settings` with any inline settings and spread `files` into the fixture.
+ */
+export declare function loadPlugin(pluginPath: string): LoadedPlugin;
+/**
+ * Resolve the effective harness for a test/eval (arm): load the plugin if given,
+ * then layer inline settings + files on top. Shared by `runHarnessTest` and
+ * `runEval` so both test the assembled machine the same way.
+ */
+export declare function resolveHarness(opts: {
+    plugin?: string;
+    settings?: unknown;
+    files?: Record<string, string>;
+}): {
+    settings: unknown;
+    files: Record<string, string>;
+};
+//# sourceMappingURL=plugin-loader.d.ts.map