vigiles 11.0.0 → 12.1.0

package/dist/eval-lock.js

@@ -0,0 +1,286 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.DEFAULT_LOCK_DIR = exports.LOCK_VERSION = void 0;
+exports.evalInputsHash = evalInputsHash;
+exports.lockSlug = lockSlug;
+exports.lockPath = lockPath;
+exports.readLock = readLock;
+exports.anyLocksCommitted = anyLocksCommitted;
+exports.isEvalInputFile = isEvalInputFile;
+exports.evalLockNudge = evalLockNudge;
+exports.writeLock = writeLock;
+exports.buildLock = buildLock;
+exports.decideLock = decideLock;
+exports.diffReportNumbers = diffReportNumbers;
+exports.formatLockUpdate = formatLockUpdate;
+exports.lockModeFromEnv = lockModeFromEnv;
+exports.evalApiVersionFromEnv = evalApiVersionFromEnv;
+/**
+ * vigiles — the eval LOCK (the CI staleness gate for evals run on a subscription).
+ *
+ * Real-model evals authenticate as your own `claude` CLI on your Claude
+ * subscription, so they only run **locally** — never in CI (no metered key, and
+ * the subscription can't be driven from a headless runner). That leaves a hole:
+ * how does CI know the committed eval numbers still match the current inputs?
+ * Someone edits a skill, forgets to re-eval, and ships stale results.
+ *
+ * The lock closes it with an **integrity hash**, NOT a cache. The two are
+ * different mechanisms and must not be confused:
+ *
+ *   - the eval CACHE ({@link ./eval-cache}) is a LOCAL speed optimization
+ *     (gitignored, keyed store, skips the model call when re-scoring `measure`);
+ *   - the eval LOCK is a COMMITTED staleness stamp (`.vigiles/eval-locks/<slug>.lock.json`),
+ *     reviewed in the git diff, checked in CI without ever touching the model.
+ *
+ * It is the snapshot/lockfile pattern (`Cargo.lock` + `npm ci`; `jest --ci` /
+ * `cargo-insta`): you produce numbers locally with `--update`, commit the lock,
+ * and CI runs `--check` — recompute the input hash, compare, and fail "stale,
+ * re-run `--update`" on a mismatch. The committed diff of `recall: 0.90 → 0.65`
+ * IS the quality gate a human reviews. The same integrity-hash-of-inputs idea
+ * vigiles already ships in `core/integrity.ts` (compiled markdown) and
+ * `core/sidecar.ts` (spec inputs), applied a third time to eval results.
+ *
+ * Honest scope (no fiction): the lock promises "your committed results match your
+ * current inputs," NOT "your results reflect current model behavior." Model /
+ * harness drift is only caught when YOU re-run `--update` locally — there is no
+ * automated live run, by design. The clean split that makes replay sound: the
+ * lock stores only the model's OBSERVED BEHAVIOR (the report); the script's own
+ * assertions (`assertTriggerRate` / `assertSignificant`) re-run live against the
+ * replayed report, so a threshold-only edit is a valid replay (no model call)
+ * while an input change is stale. See `research/cache-invalidation.md`.
+ *
+ * Pure + model-free (the only side effects are the two small fs helpers); the
+ * inputs hash reuses `canonical` from `eval-cache.ts` so the lock and the cache
+ * canonicalize identically.
+ */
+const node_fs_1 = require("node:fs");
+const node_path_1 = require("node:path");
+const hash_js_1 = require("./core/hash.js");
+const eval_cache_js_1 = require("./eval-cache.js");
+/**
+ * On-disk lock-format version, salted into nothing (the lock is keyed by name,
+ * not by hash) but VALIDATED on read so an incompatible shape fails loud rather
+ * than deserializing into a stale structure. Bump on a breaking shape change.
+ */
+exports.LOCK_VERSION = 1;
+/** Default directory for committed eval locks (tracked, NOT gitignored). */
+exports.DEFAULT_LOCK_DIR = ".vigiles/eval-locks";
+/**
+ * Why the harness binary version is **NOT** hashed (only recorded as provenance):
+ * `--check` runs in CI where `claude` is PINNED to a fixed version, while a dev's
+ * local `claude` is whatever they have — folding the version into the hash would
+ * false-trip `--check` on every PR where those differ. It is also the lock's
+ * honest scope: the gate verifies your committed results match your current
+ * *author-controlled inputs*, not current model/harness behavior (there is no
+ * automated live run). Harness/model drift is caught when YOU re-run `--update`
+ * locally and review the moved numbers in the git diff. Keeping the version out
+ * of the hash is what lets `--check` stay binary-free + deterministic in CI.
+ * (The eval CACHE still keys on it — that's local replay soundness, a different
+ * axis.) See research/cache-invalidation.md.
+ */
+/** Deterministic content hash of a lock's model-affecting inputs. */
+function evalInputsHash(input) {
+    return (0, hash_js_1.sha256short)(JSON.stringify((0, eval_cache_js_1.canonical)(input)));
+}
+/** Filesystem-safe slug for a report name (the lock filename). */
+function lockSlug(name) {
+    const slug = name
+        .toLowerCase()
+        .replace(/[^a-z0-9]+/g, "-")
+        .replace(/^-+|-+$/g, "");
+    return slug || "eval";
+}
+/** Path to a named eval's lock file under `dir`. */
+function lockPath(dir, name) {
+    return (0, node_path_1.join)(dir, `${lockSlug(name)}.lock.json`);
+}
+/**
+ * Read a named eval's lock. A MISS (no file) returns `null`. A CORRUPT or
+ * wrong-version file **throws** — a broken lock is a real failure the CI gate
+ * must surface, not silently treat as "no lock" (which would let a stale eval
+ * pass). The message says how to recover.
+ */
+function readLock(dir, name) {
+    const path = lockPath(dir, name);
+    if (!(0, node_fs_1.existsSync)(path))
+        return null;
+    const raw = (0, node_fs_1.readFileSync)(path, "utf-8");
+    let data;
+    try {
+        data = JSON.parse(raw);
+    }
+    catch {
+        throw new Error(`eval lock: corrupt lock ${path} (invalid JSON) — delete it and re-run \`vigiles eval --update\``);
+    }
+    if (typeof data !== "object" || data === null)
+        throw new Error(`eval lock: ${path} is not a JSON object`);
+    const obj = data;
+    if (obj.version !== exports.LOCK_VERSION)
+        throw new Error(`eval lock: ${path} has unsupported version ${String(obj.version)} ` +
+            `(expected ${String(exports.LOCK_VERSION)}) — re-run \`vigiles eval --update\``);
+    // Slug collision guard: two distinct names can normalize to the same file
+    // (`foo/bar` and `foo bar` → `foo-bar.lock.json`). A lock whose stored `name`
+    // differs from the one requested belongs to the OTHER eval — treat it as a
+    // MISS (not this eval's lock) so `--check` degrades to "stale → re-run" and
+    // NEVER replays the wrong eval's report. The stored name is the source of truth.
+    if (obj.name !== name)
+        return null;
+    return obj;
+}
+/**
+ * Whether ANY lock has been committed under `dir`. The CI staleness gate
+ * (`eval --check`) uses this to stay a NO-OP until the feature is in use: a repo
+ * that has never run `eval --update` has no locks, so there is nothing to verify
+ * and CI passes green. Once the first lock is committed, every named eval is held
+ * to having a fresh one (a new unlocked eval then reads as stale). The graduated,
+ * opt-in-by-committing behavior that keeps a fresh `init` from going red.
+ */
+function anyLocksCommitted(dir) {
+    if (!(0, node_fs_1.existsSync)(dir))
+        return false;
+    return (0, node_fs_1.readdirSync)(dir).some((f) => f.endsWith(".lock.json"));
+}
+/**
+ * Does an edited path plausibly change an eval's INPUTS — so a committed lock may
+ * now be stale? Two surfaces feed the hash: a skill's trigger surface (`SKILL.md`)
+ * and the eval script that holds the prompts/spec (`*.eval.{mjs,cjs,js,mts,cts,ts}`).
+ * Pure (string-only) so the nudge hook stays cheap and never runs an eval script.
+ */
+function isEvalInputFile(path) {
+    const p = path.replace(/\\/g, "/");
+    if (/(^|\/)SKILL\.md$/.test(p))
+        return true;
+    return /\.eval\.(mjs|cjs|js|mts|cts|ts)$/.test(p);
+}
+/**
+ * The NON-BLOCKING nudge to emit after an eval-input edit when committed locks
+ * exist, or `null` for no nudge. Self-gating: it stays silent until you've opted
+ * into the lock (committed one), so it can't annoy a repo that doesn't use evals.
+ * It deliberately does NOT recompute staleness (that needs the eval script + is
+ * the job of `eval --check`) — a reminder, not a gate. The honest harness-neutral
+ * reminder; how it reaches the agent (both CC and Codex inject `additionalContext`
+ * on `PostToolUse`) is the caller's concern. See docs/harness-testing-*.md.
+ */
+function evalLockNudge(filePath, lockDir) {
+    if (!isEvalInputFile(filePath))
+        return null;
+    if (!anyLocksCommitted(lockDir))
+        return null;
+    return (`vigiles: you edited ${filePath}, which can change an eval's inputs — a ` +
+        `committed eval lock may now be stale. When you're done, run ` +
+        `\`vigiles eval --update\` (local, on your subscription) and commit the ` +
+        `updated lock; CI's \`vigiles eval --check\` will otherwise flag it stale. ` +
+        `This is a reminder, not a block.`);
+}
+/** Write a named eval's lock (pretty JSON for a reviewable git diff). */
+function writeLock(dir, lock) {
+    (0, node_fs_1.mkdirSync)(dir, { recursive: true });
+    (0, node_fs_1.writeFileSync)(lockPath(dir, lock.name), JSON.stringify(lock, null, 2) + "\n");
+}
+/**
+ * Build a fresh lock envelope from a just-recorded report (the `--update` write).
+ * `builtAt` is passed in (never read from the clock here) so the module stays
+ * pure + deterministically testable; the CLI stamps the real timestamp.
+ */
+function buildLock(args) {
+    return { version: exports.LOCK_VERSION, ...args };
+}
+/**
+ * Decide what `check` mode should do given the current input hash and the
+ * committed lock. `off`/`update` always `run` (update records afterwards). `check`
+ * replays a matching lock (no model) and is `stale` on a missing lock or a hash
+ * mismatch — the deterministic CI gate.
+ */
+function decideLock(mode, name, currentHash, existing) {
+    if (mode !== "check")
+        return { kind: "run" };
+    if (!existing)
+        return {
+            kind: "stale",
+            reason: `eval lock missing for "${name}" — no committed results to verify against. ` +
+                `Run \`vigiles eval --update\` locally (on your subscription) and commit the lock.`,
+        };
+    if (existing.inputsHash !== currentHash)
+        return {
+            kind: "stale",
+            reason: `eval lock STALE for "${name}" — the inputs changed since the committed results ` +
+                `were recorded (skill/prompts/model/harness/apiVersion). Re-run ` +
+                `\`vigiles eval --update\` locally and commit the updated lock.`,
+        };
+    return { kind: "replay", report: existing.report };
+}
+/**
+ * Collect the numeric leaves that changed between two recorded reports — the
+ * human-facing delta printed at `--update` time (e.g. `rate: 0.900 → 0.650`).
+ * Generic over any report shape (walks numbers by dotted path), so it works for
+ * `EvalReport`, `TriggerRateReport`, and `CheckReport` without per-type code. The
+ * committed git diff is the primary review surface; this is the at-a-glance echo.
+ */
+function diffReportNumbers(before, after) {
+    const out = [];
+    walkNumberLeaves(before, after, "", out);
+    return out;
+}
+function walkNumberLeaves(a, b, path, out) {
+    if (typeof a === "number" && typeof b === "number") {
+        if (a !== b)
+            out.push({ path, before: a, after: b });
+    }
+    else if (Array.isArray(a) && Array.isArray(b)) {
+        walkArrayLeaves(a, b, path, out);
+    }
+    else if (isRecord(a) && isRecord(b)) {
+        walkRecordLeaves(a, b, path, out);
+    }
+}
+function walkArrayLeaves(a, b, path, out) {
+    const len = Math.min(a.length, b.length);
+    for (let i = 0; i < len; i++)
+        walkNumberLeaves(a[i], b[i], `${path}[${String(i)}]`, out);
+}
+function walkRecordLeaves(a, b, path, out) {
+    for (const k of Object.keys(a))
+        if (k in b)
+            walkNumberLeaves(a[k], b[k], path ? `${path}.${k}` : k, out);
+}
+function isRecord(v) {
+    return v !== null && typeof v === "object";
+}
+/** Render the `--update` result for a human: NEW lock, or the per-number deltas. */
+function formatLockUpdate(name, deltas, isNew) {
+    if (isNew)
+        return `eval lock: recorded NEW lock for "${name}"`;
+    if (deltas.length === 0)
+        return `eval lock: "${name}" updated — no numeric change vs the prior lock`;
+    const lines = [
+        `eval lock: "${name}" updated — ${String(deltas.length)} value(s) moved:`,
+    ];
+    for (const d of deltas) {
+        const dir = d.after > d.before ? "▲" : "▼";
+        lines.push(`  ${dir} ${d.path}: ${d.before.toFixed(3)} → ${d.after.toFixed(3)}`);
+    }
+    lines.push("  review the committed lock diff — this is the eval quality gate.");
+    return lines.join("\n");
+}
+/**
+ * Read the lock mode from the environment (`VIGILES_EVAL_LOCK`), set by the CLI's
+ * `eval --check` / `--update` flags. A run knob (like `VIGILES_TRIALS`): the CLI
+ * is the only place that should set it. Anything unrecognized → `off`.
+ */
+function lockModeFromEnv(env = process.env) {
+    const v = env.VIGILES_EVAL_LOCK;
+    return v === "check" || v === "update" ? v : "off";
+}
+/**
+ * The behavior epoch (`evalApiVersion`) for this run, read from the env the CLI
+ * populates from `.vigilesrc.json` `eval.apiVersion`. Default 1. A malformed
+ * value falls back to 1 (never throws) — the lock stays usable.
+ */
+function evalApiVersionFromEnv(env = process.env) {
+    const raw = env.VIGILES_EVAL_API_VERSION;
+    if (raw === undefined)
+        return 1;
+    const n = Number.parseInt(raw, 10);
+    return Number.isFinite(n) && n >= 0 ? n : 1;
+}
+//# sourceMappingURL=eval-lock.js.map

package/dist/eval.d.ts

@@ -1,5 +1,6 @@
 import { parseToolCalls, parseHooks, parseSubagents, type ToolCall, type Trace } from "./harness-test.js";
 import { type CacheMode } from "./eval-cache.js";
+import { type EvalLockOptions } from "./eval-lock.js";
 import type { Check, CheckJSON } from "./check.js";
 import { type Comparison } from "./stats.js";
 import { type ToolIntercept } from "./tool-intercept.js";
@@ -159,6 +160,16 @@ export interface EvalSpec<M extends Metrics> {
      * to today). See {@link ToolStub} and `research/eval-coverage-and-isolation.md`.
      */
     readonly stubs?: readonly ToolStub[];
+    /**
+     * **The eval LOCK** — a committed staleness gate for CI (see `src/eval-lock.ts`).
+     * With a `name` set, `vigiles eval --update` (local, on your subscription)
+     * records the report to `.vigiles/eval-locks/<name>.lock.json`; `--check` (CI)
+     * verifies the committed result against the current inputs WITHOUT a model call,
+     * failing "stale" when they diverge. Mode normally comes from the CLI; set
+     * `lock` to drive it programmatically or point a test at a throwaway dir. The
+     * lock engages only when `name` is set (it keys the lock file).
+     */
+    readonly lock?: EvalLockOptions;
 }
 /** Per-metric summary statistics across an arm's runs. */
 export interface MetricStat {
@@ -316,6 +327,8 @@ export interface CheckRate {
 export interface CheckReport {
     readonly n: number;
     readonly perCheck: readonly CheckRate[];
+    /** Cost / latency / token totals for the run (the same source as `runEval`). */
+    readonly usage: ArmUsage;
 }
 /**
  * Score a check vocabulary across trials — the scored counterpart to
@@ -479,16 +492,6 @@ export declare function modelTier(id: string): number | null;
  * {@link modelTier}); an unrankable model/floor is never "below" (fail-open).
  */
 export declare function belowModelFloor(model: string, floor: string): boolean;
-/**
- * Reduce a raw `--version` string to the **major.minor** cache-key token. We key
- * the cache on major.minor, NOT the patch: a patch release rarely changes agent
- * behaviour, so keying patches would churn the cache on every release for no
- * signal; a minor/major bump is where the system prompt / tool defs actually move.
- * (If a specific patch is known to matter, clear the cache or bump
- * `CACHE_FORMAT_VERSION`.) Falls back to the trimmed raw string when no semver is
- * found. Pure + tested.
- */
-export declare function harnessVersionKey(raw: string): string;
 /**
  * Build an **ephemeral run environment** for a model-driven run: a NEW env object
  * with a *fresh* `HOME` (and `TMPDIR`) pointed at the throwaway `opts.home`, only
@@ -542,15 +545,6 @@ export declare function seedEphemeralHome(throwawayHome: string, realHome: strin
 export declare function isRateLimited(out: RunOut): boolean;
 /** Map `worker` over `items` with at most `concurrency` in flight, order preserved. */
 export declare function runPool<T, R>(items: readonly T[], concurrency: number, worker: (item: T) => Promise<R>): Promise<R[]>;
-/**
- * The eval orchestration — every arm × trial via `runner`, run through the cache
- * and a rate-limit retry, with at most `concurrency` in flight and an optional
- * `maxCostUsd` budget cap; metric + usage computed per run and aggregated per
- * arm. Exported with an injectable `runner` so the loop, `measure` context,
- * caching, pooling, and aggregation are unit-testable without spawning a model
- * (pass a fake returning canned stream-json). `runEval` is this with the real
- * agent runner.
- */
 export declare function runEvalWith<M extends Metrics>(spec: EvalSpec<M>, runner: AgentRunner): Promise<EvalReport>;
 /** Format an eval report as a compact table for the console (mean ± se, pass^k). */
 export declare function formatEvalReport(report: EvalReport): string;
@@ -563,6 +557,19 @@ export declare function formatEvalReport(report: EvalReport): string;
  * (reuse the bare predicates, e.g. `(t) => skillResolved(t, "x:y")`).
  */
 export interface TriggerRateSpec {
+    /**
+     * A stable name for this trigger eval — required to engage the {@link lock}
+     * (it keys the committed `.vigiles/eval-locks/<name>.lock.json`). Two evals over
+     * the same skill with different prompts get distinct names. Optional otherwise.
+     */
+    readonly name?: string;
+    /**
+     * **The eval LOCK** — the CI staleness gate (see `src/eval-lock.ts`). With
+     * `name` set, `vigiles eval --update` records this trigger-rate report locally
+     * and `--check` verifies it against the current inputs (skill contents, prompts,
+     * model) with NO model call. Mode normally comes from the CLI flags.
+     */
+    readonly lock?: EvalLockOptions;
     /**
      * Plugin dir installed natively (`--plugin-dir`) so its skills/commands
      * activate. Provide this OR {@link skillsDir}, not both.
@@ -709,6 +716,8 @@ export interface TriggerRateReport {
      * `n` means the measurement is thin (e.g. a Codex usage limit was hit); re-run.
      */
     readonly errored?: number;
+    /** Cost / tokens SPENT across all runs (relevant + irrelevant) — feeds the cost summary. */
+    readonly usage: ArmUsage;
 }
 /**
  * An eval-tier transport: how to RUN a real harness turn and PARSE its output.
@@ -722,6 +731,14 @@ export interface EvalDriver {
     readonly runner: AgentRunner;
     readonly parse: ModelOutputParser;
     readonly runError?: (out: RunOut) => string | null;
+    /**
+     * The harness this driver runs (e.g. `"claude-code"`, `"codex"`). Folded into a
+     * trigger-rate eval's LOCK hash so a report recorded on one harness is marked
+     * STALE if the eval is later switched to another (a different harness can fire a
+     * skill differently). Optional for back-compat — absent defaults to
+     * `"claude-code"`, so an existing single-harness lock is unaffected.
+     */
+    readonly harness?: string;
 }
 /** The default (Claude Code) eval driver: real `claude` + stream-json parsing. */
 export declare const claudeEvalDriver: EvalDriver;
@@ -801,7 +818,7 @@ export declare function packageInstallSet(opts: {
     dir: string;
     added: number;
 };
-export declare function measureTriggerRateWith(spec: TriggerRateSpec, runner: AgentRunner, parse?: ModelOutputParser, runError?: (out: RunOut) => string | null): Promise<TriggerRateReport>;
+export declare function measureTriggerRateWith(spec: TriggerRateSpec, runner: AgentRunner, parse?: ModelOutputParser, runError?: (out: RunOut) => string | null, harness?: string): Promise<TriggerRateReport>;
 /**
  * Measure a skill/behaviour's real trigger rate across prompts × trials. Defaults
  * to the real `claude` CLI (`claudeEvalDriver`); pass `{ evalDriver }` to drive a