vigiles 36.0.0 → 37.0.0

package/dist/core/stub-rules.js

@@ -0,0 +1,397 @@
+"use strict";
+/**
+ * A tool stub that answers PER INVOCATION — the pure core.
+ *
+ * A stub shadows a binary on PATH (`gh`, `git`, `psql`) so a run that calls it
+ * gets a recorded answer instead of the real service. The earlier stub printed
+ * ONE answer for every argv. That is a test double answering by default, and a
+ * default answer is a fabricated observation: a stub set up to answer
+ * `gh issue create` with an issue URL also answered a script's
+ * `gh api repos/o/r/issues?…` with that URL, the script failed to parse it as
+ * JSON, and a model that read the stub's source stopped trusting it.
+ *
+ * So a stub is a non-empty list of RULES over the argv TOKEN ARRAY. Each rule has
+ * a positional pattern (an exact string or a RegExp per token, and an explicit
+ * trailing `rest` for "any further tokens"), an optional order-free `contains`
+ * for flags, and a reply. The first matching rule answers. An argv no rule
+ * matches is UNANSWERED: the stub exits non-zero, the call is logged, and the
+ * run fails — the same choice `script-overrun.ts` made for a scripted model that
+ * runs past its script.
+ *
+ * Argv is a list, not a line: joining it loses token boundaries (`--title "a b"`
+ * is one token), and a RegExp over the joined line matches across them.
+ *
+ * Everything here is pure and JSON-serialisable, because the same rules are
+ * read by the stub process (one per invocation), folded into the eval cache key
+ * and the lock, and printed in failure messages. No zod here on purpose: this
+ * module is on the stub process's path, which runs once per `gh` call.
+ */
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.UNANSWERED_EXIT_CODE = exports.ARGV_REST = void 0;
+exports.matchArgv = matchArgv;
+exports.priorAnswers = priorAnswers;
+exports.decideInvocation = decideInvocation;
+exports.isUnanswered = isUnanswered;
+exports.answerFor = answerFor;
+exports.encodeStub = encodeStub;
+exports.toToolStub = toToolStub;
+exports.decodeStub = decodeStub;
+exports.stubLogLine = stubLogLine;
+exports.parseStubLog = parseStubLog;
+exports.describePattern = describePattern;
+exports.unansweredMessage = unansweredMessage;
+exports.unsupportedLine = unsupportedLine;
+exports.unloggedMisses = unloggedMisses;
+exports.stubPaths = stubPaths;
+exports.runStub = runStub;
+const node_path_1 = require("node:path");
+/** The `rest` marker. One frozen value, so a pattern carries no free-form object. */
+exports.ARGV_REST = Object.freeze({ kind: "rest" });
+const isRest = (t) => typeof t === "object" && !(t instanceof RegExp);
+const isPlainToken = (t) => !isRest(t);
+/** Does one pattern token match one argv token? A RegExp is stateless here: `g`/`y` are refused at parse. */
+const tokenMatches = (pattern, token) => typeof pattern === "string" ? pattern === token : pattern.test(token);
+/**
+ * The positional tokens of a pattern, and whether it ends in `rest`. `rest` can
+ * only be last (the type says so; the parse refuses it elsewhere).
+ */
+function splitPattern(p) {
+    const tokens = p;
+    const fixed = tokens.filter(isPlainToken);
+    return { fixed, open: fixed.length !== tokens.length };
+}
+/** Does `rule` match `argv`? Positional prefix, then `contains` over the tail. Pure, total. */
+function matchArgv(rule, argv) {
+    const { fixed, open } = splitPattern(rule.argv);
+    const lengthOk = open
+        ? argv.length >= fixed.length
+        : argv.length === fixed.length;
+    if (!lengthOk)
+        return false;
+    if (!fixed.every((t, i) => tokenMatches(t, argv[i] ?? "")))
+        return false;
+    const tail = argv.slice(fixed.length);
+    return (rule.contains ?? []).every((c) => tail.some((t) => tokenMatches(c, t)));
+}
+/** How many times each rule of `tool` has already answered, from the log so far. */
+function priorAnswers(calls, tool) {
+    const rules = calls.flatMap((c) => c.tool === tool && c.outcome.kind === "answered" ? [c.outcome.rule] : []);
+    return rules.reduce((m, r) => new Map([...m, [r, (m.get(r) ?? 0) + 1]]), new Map());
+}
+/** The decision for one invocation: (rules, answers so far, argv) → outcome. Pure, total. */
+function decideInvocation(stub, prior, argv) {
+    const rule = stub.rules.findIndex((r) => matchArgv(r, argv));
+    if (rule < 0)
+        return { kind: "no-rule" };
+    const reply = stub.rules[rule]?.reply;
+    if (reply === undefined || reply.kind === "always")
+        return { kind: "answered", rule, answer: 0 };
+    const k = prior.get(rule) ?? 0;
+    return k < reply.answers.length
+        ? { kind: "answered", rule, answer: k }
+        : { kind: "exhausted", rule };
+}
+/** Whether an outcome left the call without an answer. */
+function isUnanswered(o) {
+    return o.kind !== "answered";
+}
+/** The answer an `answered` outcome refers to; `undefined` for an unanswered one. */
+function answerFor(stub, o) {
+    if (o.kind !== "answered")
+        return undefined;
+    const reply = stub.rules[o.rule]?.reply;
+    if (reply === undefined)
+        return undefined;
+    return reply.kind === "always" ? reply : reply.answers[o.answer];
+}
+function encodeToken(t) {
+    if (typeof t === "string")
+        return t;
+    if (t instanceof RegExp)
+        return { kind: "re", source: t.source, flags: t.flags };
+    return { kind: "rest" };
+}
+/**
+ * Serialise a stub to JSON. A RegExp becomes `{kind:"re",source,flags}` —
+ * `JSON.stringify(/x/)` is `{}`, which would make two different rules hash the
+ * same in the cache key and the lock.
+ */
+function encodeStub(stub) {
+    return JSON.stringify({
+        name: stub.name,
+        rules: stub.rules.map((r) => ({
+            argv: r.argv.map(encodeToken),
+            ...(r.contains === undefined
+                ? {}
+                : { contains: r.contains.map(encodeToken) }),
+            reply: r.reply,
+        })),
+    });
+}
+// --- reading data back: narrowing, never a cast ----------------------------------
+//
+// Two readers: the stub process (the rules file and the log — vigiles' own
+// output, so a bad shape is an internal error) and the spec boundary (after its
+// zod parse, to get the typed value without a cast). Neither pays for zod.
+function internal(what) {
+    throw new Error(`vigiles stub: ${what} — an internal error, please report it.`);
+}
+const isObj = (v) => typeof v === "object" &&
+    v !== null &&
+    !Array.isArray(v) &&
+    !(v instanceof RegExp);
+const listOf = (v, what) => Array.isArray(v) ? v : internal(`${what} is not a list`);
+function nonEmpty(xs, what) {
+    const [first, ...more] = xs;
+    return first === undefined ? internal(`${what} is empty`) : [first, ...more];
+}
+function toToken(v) {
+    if (typeof v === "string" || v instanceof RegExp)
+        return v;
+    if (isObj(v) && v.kind === "rest")
+        return exports.ARGV_REST;
+    if (isObj(v) && typeof v.source === "string" && typeof v.flags === "string")
+        return new RegExp(v.source, v.flags);
+    return internal("a rule token is not a string, a RegExp or rest");
+}
+function toPattern(v) {
+    const tokens = listOf(v, "argv").map(toToken);
+    const fixed = tokens.filter(isPlainToken);
+    const last = tokens.at(-1);
+    const open = last !== undefined && isRest(last);
+    if (fixed.length !== tokens.length - (open ? 1 : 0))
+        return internal("rest is not the last token");
+    return open ? [...fixed, exports.ARGV_REST] : fixed;
+}
+function toAnswer(v) {
+    if (!isObj(v))
+        return internal("an answer is not an object");
+    const { stdout, stderr, exitCode } = v;
+    return {
+        ...(typeof stdout === "string" ? { stdout } : {}),
+        ...(typeof stderr === "string" ? { stderr } : {}),
+        ...(typeof exitCode === "number" ? { exitCode } : {}),
+    };
+}
+function toReply(v) {
+    if (!isObj(v))
+        return internal("a reply is not an object");
+    if (v.kind === "always")
+        return { kind: "always", ...toAnswer(v) };
+    if (v.kind === "inOrder")
+        return {
+            kind: "inOrder",
+            answers: nonEmpty(listOf(v.answers, "answers").map(toAnswer), "answers"),
+        };
+    return internal("a reply has an unknown kind");
+}
+function toRule(v) {
+    if (!isObj(v))
+        return internal("a rule is not an object");
+    return {
+        argv: toPattern(v.argv),
+        ...(v.contains === undefined
+            ? {}
+            : {
+                contains: listOf(v.contains, "contains")
+                    .map(toToken)
+                    .filter(isPlainToken),
+            }),
+        reply: toReply(v.reply),
+    };
+}
+/** A typed {@link ToolStub} from a value already known to have its shape. */
+function toToolStub(v) {
+    if (!isObj(v) || typeof v.name !== "string")
+        return internal("a stub has no name");
+    return {
+        name: v.name,
+        rules: nonEmpty(listOf(v.rules, "rules").map(toRule), "rules"),
+    };
+}
+/**
+ * Read back what {@link encodeStub} wrote. The input is vigiles' own output,
+ * written after the spec was parsed, so a bad shape is an internal error.
+ */
+function decodeStub(json) {
+    return toToolStub(JSON.parse(json));
+}
+// --- the invocation log --------------------------------------------------------
+/** One log line for one call (no trailing newline). */
+function stubLogLine(call) {
+    return JSON.stringify(call);
+}
+function toOutcome(v) {
+    if (!isObj(v))
+        return internal("a log outcome is not an object");
+    const { kind, rule, answer } = v;
+    if (kind === "no-rule")
+        return { kind };
+    if (kind === "exhausted" && typeof rule === "number")
+        return { kind, rule };
+    if (kind === "answered" &&
+        typeof rule === "number" &&
+        typeof answer === "number")
+        return { kind, rule, answer };
+    return internal("a log outcome has an unknown kind");
+}
+function toStubCall(v) {
+    if (!isObj(v) || typeof v.tool !== "string")
+        return internal("a log line has no tool");
+    return {
+        tool: v.tool,
+        argv: listOf(v.argv, "argv").map((t) => typeof t === "string" ? t : internal("an argv token is not a string")),
+        outcome: toOutcome(v.outcome),
+    };
+}
+/** Read the JSONL log the stub processes appended. Blank lines are skipped. */
+function parseStubLog(jsonl) {
+    return jsonl.split("\n").flatMap((line, i) => {
+        if (line.trim() === "")
+            return [];
+        try {
+            return [toStubCall(JSON.parse(line))];
+        }
+        catch (e) {
+            const why = e instanceof Error ? e.message : String(e);
+            throw new Error(`vigiles: stub log line ${String(i + 1)} is unreadable (${why}) — an internal error, please report it.`, { cause: e });
+        }
+    });
+}
+// --- the author-facing diagnostic ---------------------------------------------
+function showToken(t) {
+    if (typeof t === "string")
+        return JSON.stringify(t);
+    if (t instanceof RegExp)
+        return `/${t.source}/${t.flags}`;
+    return "experimental_stub.rest";
+}
+/** A pattern the way a rule is written: `["issue", "create", experimental_stub.rest]`. */
+function describePattern(p) {
+    return `[${p.map(showToken).join(", ")}]`;
+}
+const whyUnanswered = (o) => o.kind === "exhausted"
+    ? `rule #${String(o.rule + 1)} has no answer left (its inOrder answers ran out)`
+    : "no rule matches";
+/** The miss whose argv nobody recorded: what happened, and why it cannot name the command. */
+function describeUnlogged(c, n) {
+    const times = n > 1 ? ` ×${String(n)}` : "";
+    return (`  ${c.tool}${times}: the stub printed its unsupported-invocation line but could not log the call — ` +
+        `is the stub directory writable from the agent's shell? The argv is unknown, so no rule is suggested; ` +
+        `run once outside the confinement to see which command it was.`);
+}
+/** One unanswered argv, `n` times: why, the rules it was tried against, the rule to add. */
+function describeMiss(c, n, stubs) {
+    if (c.outcome.kind === "unlogged")
+        return describeUnlogged(c, n);
+    const times = n > 1 ? ` ×${String(n)}` : "";
+    const rules = (stubs.find((s) => s.name === c.tool)?.rules ?? []).map((r, i) => `\n      #${String(i + 1)} ${describePattern(r.argv)}`);
+    const tried = rules.length === 0 ? "" : `; the rules for ${c.tool} are:${rules.join("")}`;
+    return (`  ${c.tool} ${JSON.stringify(c.argv)}${times} — ${whyUnanswered(c.outcome)}${tried}\n` +
+        `    add: { argv: ${describePattern(c.argv)}, reply: { kind: "always", stdout: "…" } }`);
+}
+/**
+ * The failure message for a run with unanswered calls, addressed to the AUTHOR
+ * (the model only ever sees a neutral one-liner). Each distinct argv once, with
+ * a count; the rules it was tried against; and a rule for that exact command to
+ * add. It never suggests a catch-all: answering everything is the defect this
+ * exists to stop. `undefined` when every call was answered.
+ */
+function unansweredMessage(calls, stubs) {
+    const missed = calls.filter((c) => isUnanswered(c.outcome));
+    if (missed.length === 0)
+        return undefined;
+    const keyOf = (c) => `${c.tool}\u0000${JSON.stringify(c.argv)}\u0000${c.outcome.kind}`;
+    const distinct = [...new Map(missed.map((c) => [keyOf(c), c])).values()];
+    const lines = distinct.map((c) => describeMiss(c, missed.filter((m) => keyOf(m) === keyOf(c)).length, stubs));
+    return (`${String(missed.length)} stub call(s) went unanswered — the agent ran a command no rule scripts, ` +
+        `so the trial measured an answer nobody wrote:\n${lines.join("\n")}\n` +
+        `  Add a rule for each command the agent really runs, with the answer the real tool gives ` +
+        `(record it once; do not invent it). A rule that answers every argv would hide this again.`);
+}
+// --- one invocation, end to end (the stub process runs this) ------------------
+/**
+ * The exit code of an unanswered call. Not 127: that tells the caller "not
+ * installed", and a model changes plans for the wrong reason.
+ */
+exports.UNANSWERED_EXIT_CODE = 97;
+/**
+ * What an unanswered call prints to stderr — the only thing the MODEL sees. One
+ * fixed line: no argv, no rules, no "add a rule". Instructions here would tell
+ * the model it is in a test and what the author will do, and the rest of the
+ * trial would measure a reaction to that. The diagnostic goes to the author,
+ * from the log, through {@link unansweredMessage}.
+ */
+function unsupportedLine(name) {
+    return `${name}: unsupported invocation (vigiles stub)\n`;
+}
+/**
+ * The misses a stub could not log, found from the agent's side: every
+ * occurrence of a stubbed tool's neutral miss line ({@link unsupportedLine}) in
+ * `texts` (the agent's tool results) beyond the unanswered calls the log already
+ * holds for that tool. The line is vigiles' own marker, not a guess about what
+ * the agent ran, so this is a count of a thing we printed. A miss the log has
+ * is counted once — by the log. Pure.
+ */
+function unloggedMisses(texts, logged, names) {
+    return names.flatMap((name) => {
+        const line = unsupportedLine(name).trimEnd();
+        const seen = texts.reduce((n, t) => n + (t.split(line).length - 1), 0);
+        const known = logged.filter((c) => c.tool === name && isUnanswered(c.outcome)).length;
+        const unlogged = {
+            tool: name,
+            argv: [],
+            outcome: { kind: "unlogged" },
+        };
+        return Array.from({ length: Math.max(0, seen - known) }, () => unlogged);
+    });
+}
+/** Where a stub root keeps its files (`name` picks the per-stub ones). Pure path arithmetic. */
+function stubPaths(root, name = "") {
+    return {
+        binDir: (0, node_path_1.join)(root, "bin"),
+        rulesDir: (0, node_path_1.join)(root, "rules"),
+        rules: (0, node_path_1.join)(root, "rules", `${name}.json`),
+        log: (0, node_path_1.join)(root, "calls.jsonl"),
+        bin: (0, node_path_1.join)(root, "bin", name),
+    };
+}
+/**
+ * One invocation: read the rules and the log so far, decide, append one log
+ * line, return what to print. It never reads stdin: a caller that leaves the
+ * pipe open (Node's `execFile` does) would block until its own timeout.
+ *
+ * Concurrency, said plainly: two simultaneous calls of the same `inOrder` rule
+ * can both read "answered k times" and both get answer k. One line per write
+ * keeps the log itself intact.
+ */
+function runStub(root, name, argv, io) {
+    const paths = stubPaths(root, name);
+    const json = io.readFile(paths.rules);
+    if (json === null)
+        return {
+            stdout: "",
+            stderr: `vigiles stub: no rules file for ${name} under ${root} — an internal error, please report it.\n`,
+            exitCode: exports.UNANSWERED_EXIT_CODE,
+        };
+    const stub = decodeStub(json);
+    const prior = priorAnswers(parseStubLog(io.readFile(paths.log) ?? ""), name);
+    const outcome = decideInvocation(stub, prior, argv);
+    io.appendFile(paths.log, stubLogLine({ tool: name, argv, outcome }) + "\n");
+    return printed(name, answerFor(stub, outcome));
+}
+/** What the process prints for an answer — or the neutral miss line for none. */
+function printed(name, answer) {
+    if (answer === undefined)
+        return {
+            stdout: "",
+            stderr: unsupportedLine(name),
+            exitCode: exports.UNANSWERED_EXIT_CODE,
+        };
+    return {
+        stdout: answer.stdout ?? "",
+        stderr: answer.stderr ?? "",
+        exitCode: answer.exitCode ?? 0,
+    };
+}
+//# sourceMappingURL=stub-rules.js.map

package/dist/eval-cache.d.ts

@@ -1,5 +1,6 @@
 import { type SHA256Hash } from "./core/hash.js";
 import type { RunOut } from "./eval.js";
+import type { StubCall } from "./core/stub-rules.js";
 /** Cache behaviour: never touch the cache / read-only / read-and-write. */
 export type CacheMode = "off" | "read" | "readwrite";
 /** Everything that determines a trial's model output (the cache key inputs). */
@@ -33,6 +34,17 @@ export interface CacheKeyInput {
      * arm, not a native install). See {@link hashDir}.
      */
     readonly pluginDirHash?: string;
+    /**
+     * The run's tool stubs, each ENCODED (`encodeStub` — a RegExp token as its
+     * source and flags, since `JSON.stringify(/x/)` is `{}`). A changed answer is
+     * a changed input. Omit when there are none.
+     */
+    readonly stubs?: readonly string[];
+    /**
+     * Where the run executes: the env kind and an ephemeral run's seed — never the
+     * per-trial HOME path. A seed file is part of what the agent sees.
+     */
+    readonly runEnv?: unknown;
     /**
      * The harness BINARY version (e.g. `claude --version`). The harness evolves
      * fast — a CLI upgrade changes the system prompt + tool definitions, which steer
@@ -49,6 +61,12 @@ export interface CacheRecord {
     readonly out: RunOut;
     /** Text files present in the cwd after the run (relative path → contents). */
     readonly files: Record<string, string>;
+    /**
+     * The run's stub calls. The stub log lives beside the work dir, not in it, so
+     * the cwd snapshot does not hold it; a replay must report the same calls —
+     * unanswered ones included. Absent when the run had no stubs.
+     */
+    readonly stubCalls?: readonly StubCall[];
 }
 /**
  * Canonicalize a value so the key is stable regardless of object key order —
@@ -64,7 +82,7 @@ export declare function canonical(value: unknown): unknown;
  * brittle read-time version gate needed). A major bump means orphaned files on
  * disk; reclaim them by deleting the cache dir.
  */
-export declare const CACHE_FORMAT_VERSION = 2;
+export declare const CACHE_FORMAT_VERSION = 3;
 /** Deterministic content hash of the key inputs (order-independent). */
 export declare function cacheKey(input: CacheKeyInput): SHA256Hash;
 /**

package/dist/eval-cache.js

@@ -56,7 +56,7 @@ function canonical(value) {
  * brittle read-time version gate needed). A major bump means orphaned files on
  * disk; reclaim them by deleting the cache dir.
  */
-exports.CACHE_FORMAT_VERSION = 2;
+exports.CACHE_FORMAT_VERSION = 3;
 /**
  * Per-run env keys that are PURE NOISE for the cache key — a fresh random path
  * every run, never model-affecting. The opt-in ephemeral run env

package/dist/eval-entry.d.ts

@@ -19,6 +19,34 @@ export declare function trialsOverride(raw: string | undefined): number | undefi
  * hand-write its own `report.n === 0` check (and why several forgot to).
  */
 export declare function runsIn(report: AnyReport): number | undefined;
+/**
+ * The exit code of an eval file whose report has an unanswered stub call. Its
+ * own code — not 2 (the CLI's usage error), 1 (a bad declaration, zero runs) or
+ * 77 (skip) — so the runner can say WHICH cause it is from a code only this path
+ * produces. `vigiles eval` itself exits {@link UNANSWERED_STUB_VERB_EXIT} for it.
+ */
+export declare const UNANSWERED_STUB_EXIT_CODE = 78;
+/**
+ * What `vigiles eval` exits with when a file exited {@link UNANSWERED_STUB_EXIT_CODE}:
+ * 2, the CLI's "error" — the run did not fail an assertion, it produced something
+ * that is not a measurement. Documented in docs/cli.md.
+ */
+export declare const UNANSWERED_STUB_VERB_EXIT = 2;
+/**
+ * The author-facing failure when a `runEval` report carries a stub call no rule
+ * answered, or undefined. Only `runEval` has `stubs`. Pure — the rule itself is
+ * `unansweredInReport`, which the lock's refusal reads too.
+ */
+export declare function unansweredFailure(kind: EvalKind, report: AnyReport, spec: unknown): string | undefined;
+/**
+ * Why a finished run fails before its `assert` runs — with the exit code — or
+ * undefined. Two reasons: it ran ZERO trials (exit 1; the check every file used
+ * to hand-write), or a stub call went unanswered (exit 2). Pure.
+ */
+export declare function runFailure(file: string, kind: EvalKind, report: AnyReport, spec: unknown): {
+    readonly code: number;
+    readonly message: string;
+} | undefined;
 /**
  * `evalDriver` is only wired on `measureTriggerRate` — the only measurement with
  * a public driver seam. Naming it beside any other measurement is a mistake the

package/dist/eval-entry.js

@@ -1,7 +1,10 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
+exports.UNANSWERED_STUB_VERB_EXIT = exports.UNANSWERED_STUB_EXIT_CODE = void 0;
 exports.trialsOverride = trialsOverride;
 exports.runsIn = runsIn;
+exports.unansweredFailure = unansweredFailure;
+exports.runFailure = runFailure;
 exports.driverMisplaced = driverMisplaced;
 exports.notADescriptionMessage = notADescriptionMessage;
 exports.declarationProblem = declarationProblem;
@@ -27,7 +30,10 @@ exports.declarationProblem = declarationProblem;
  *   5. prints the report with the formatter that matches the measurement;
  *   6. fails on a run that executed ZERO trials — the check every file used to
  *      hand-write as `if (report.n === 0) throw`;
- *   7. calls `assert(report)`.
+ *   7. fails — exit 2, before `assert` — on a `runEval` report with a stub call
+ *      no rule answered: such a trial measured an answer nobody wrote, and the
+ *      file's own `assert` may not look (most call only `compareArms`);
+ *   8. calls `assert(report)`.
  *
  * ## Not a CLI verb
  *
@@ -40,6 +46,7 @@ const node_path_1 = require("node:path");
 const eval_load_phase_js_1 = require("./core/eval-load-phase.js");
 const eval_define_js_1 = require("./eval-define.js");
 const eval_js_1 = require("./eval.js");
+const eval_spec_parse_js_1 = require("./core/eval-spec-parse.js");
 const scan_behavioral_js_1 = require("./scan-behavioral.js");
 /**
  * How many trials this run should use, or `undefined` to leave the spec alone.
@@ -70,6 +77,55 @@ function runsIn(report) {
     }
     return "n" in report ? report.n : undefined;
 }
+/**
+ * The exit code of an eval file whose report has an unanswered stub call. Its
+ * own code — not 2 (the CLI's usage error), 1 (a bad declaration, zero runs) or
+ * 77 (skip) — so the runner can say WHICH cause it is from a code only this path
+ * produces. `vigiles eval` itself exits {@link UNANSWERED_STUB_VERB_EXIT} for it.
+ */
+exports.UNANSWERED_STUB_EXIT_CODE = 78;
+/**
+ * What `vigiles eval` exits with when a file exited {@link UNANSWERED_STUB_EXIT_CODE}:
+ * 2, the CLI's "error" — the run did not fail an assertion, it produced something
+ * that is not a measurement. Documented in docs/cli.md.
+ */
+exports.UNANSWERED_STUB_VERB_EXIT = 2;
+/** A `runEval` report: every arm carries its unanswered stub calls. */
+const isEvalReport = (r) => "arms" in r &&
+    Object.values(r.arms).every((a) => "unansweredStubCalls" in a);
+/**
+ * The author-facing failure when a `runEval` report carries a stub call no rule
+ * answered, or undefined. Only `runEval` has `stubs`. Pure — the rule itself is
+ * `unansweredInReport`, which the lock's refusal reads too.
+ */
+function unansweredFailure(kind, report, spec) {
+    if (kind !== "runEval" || !isEvalReport(report))
+        return undefined;
+    const raw = typeof spec === "object" && spec !== null && "stubs" in spec
+        ? spec.stubs
+        : undefined;
+    return (0, eval_js_1.unansweredInReport)(report, (0, eval_spec_parse_js_1.parseToolStubs)(raw, "runEval"));
+}
+/**
+ * Why a finished run fails before its `assert` runs — with the exit code — or
+ * undefined. Two reasons: it ran ZERO trials (exit 1; the check every file used
+ * to hand-write), or a stub call went unanswered (exit 2). Pure.
+ */
+function runFailure(file, kind, report, spec) {
+    if (runsIn(report) === 0)
+        return {
+            code: 1,
+            message: `✗ ${file}: no runs executed (0 trials completed).`,
+        };
+    // `runEval` has already printed the calls in full (to stderr, before its
+    // report): this line is the file's verdict, not a second copy of them.
+    return unansweredFailure(kind, report, spec) === undefined
+        ? undefined
+        : {
+            code: exports.UNANSWERED_STUB_EXIT_CODE,
+            message: `✗ ${file}: stub call(s) went unanswered (listed above) — the report is not a measurement.`,
+        };
+}
 /**
  * `evalDriver` is only wired on `measureTriggerRate` — the only measurement with
  * a public driver seam. Naming it beside any other measurement is a mistake the
@@ -111,10 +167,7 @@ function printReport(kind, report) {
             console.log((0, eval_js_1.formatCheckReport)(report));
             return;
         case "measureArms":
-            for (const [name, arm] of Object.entries(report.arms)) {
-                console.log(`\n[arm: ${name}]`);
-                console.log((0, eval_js_1.formatCheckReport)(arm));
-            }
+            console.log((0, eval_js_1.formatArmsCheckReport)(report));
             return;
         case "measureTriggerRate":
             console.log((0, eval_js_1.formatTriggerRateReport)(report));
@@ -156,11 +209,10 @@ async function main() {
         console.error("vigiles: eval-entry expects one eval file. Run `vigiles eval <file>`.");
         process.exit(2);
     }
-    const url = (0, node_url_1.pathToFileURL)((0, node_path_1.resolve)(file)).href;
     let mod;
     (0, eval_load_phase_js_1.beginEvalLoad)();
     try {
-        mod = await import(url);
+        mod = await import((0, node_url_1.pathToFileURL)((0, node_path_1.resolve)(file)).href);
     }
     finally {
         (0, eval_load_phase_js_1.endEvalLoad)();
@@ -187,9 +239,10 @@ async function main() {
     }
     const report = await runDeclared(declared.kind, declared.spec, trialsOverride(process.env["VIGILES_TRIALS"]), def.evalDriver);
     printReport(declared.kind, report);
-    if (runsIn(report) === 0) {
-        console.error(`✗ ${file}: no runs executed (0 trials completed).`);
-        process.exit(1);
+    const failure = runFailure(file, declared.kind, report, declared.spec);
+    if (failure !== undefined) {
+        console.error(failure.message);
+        process.exit(failure.code);
     }
     await def.assert?.(report);
 }

package/dist/eval-surface.d.ts

@@ -123,5 +123,7 @@ export { judged as paid_judged } from "./check.js";
 export { claudeEvalDriver as paid_claudeEvalDriver } from "./eval.js";
 export type { Check, CheckResult, JudgeFn } from "./check.js";
 export type { Trace } from "./harness-test.js";
+export type { HomeFiles, HomeSeed, RunEnv } from "./core/run-env.js";
+export type { StubCall, StubOutcome, ToolStub } from "./core/stub-rules.js";
 export type { EvalArm, EvalDriver, EvalSpec, EvalReport, EvalUsage, MeasureSpec, ArmsMeasureSpec, ArmReport, ArmUsage, ArmsCheckReport, CheckRate, CheckReport, MetricStat, Metrics, ModelOutputParser, ParsedModelRun, PromptDiversityIssue, PromptTriggerStat, RunContext, RunOut, SelectionTrialResult, TriggerRateReport, TriggerRateSpec, AgentRunArgs, AgentRunner, } from "./eval.js";
 //# sourceMappingURL=eval-surface.d.ts.map