vigiles 11.0.0 → 12.1.0

package/dist/doc-command-coverage.js

@@ -0,0 +1,60 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.COVERAGE_EXEMPT = void 0;
+exports.verbMentioned = verbMentioned;
+exports.findUndocumentedVerbs = findUndocumentedVerbs;
+/**
+ * Doc-command coverage — the INVERSE of self-command-refs, and the deterministic
+ * FLOOR under the `document-the-why` rule. self-command-refs checks that every
+ * `vigiles <cmd>` reference in the docs resolves to a REAL command (docs → code).
+ * This checks the other direction (code → docs): every public CLI VERB must be
+ * MENTIONED somewhere under `docs/`, so a verb shipped without a doc home is a
+ * failing test, not a thing a reader discovers is missing.
+ *
+ * HIGH-PRECISION, and biased toward NOT crying wolf: the risk here is a FALSE
+ * "undocumented" alarm on a verb that IS documented, so "mentioned" is matched
+ * GENEROUSLY — a verb counts as documented if it appears in a COMMAND context:
+ * `vigiles <verb>` (covers `npx vigiles <verb>`) OR a backtick immediately
+ * followed by the verb (`` `<verb>` ``, `` `<verb> ./x` ``). A bare English word
+ * ("test", "audit", "eval", "compile" all double as prose) is NOT enough — it
+ * must sit in a command context — so the check still fires on a genuinely
+ * undocumented verb while never flagging a documented one.
+ *
+ * `hook-runtime` is excluded by default: it is the HIDDEN runtime-entrypoint
+ * umbrella (cohesive-cli-surface keeps it OUT of the human verb surface), not a
+ * verb a user is expected to read about beside `audit`/`lint`. Source of truth
+ * for the verb set: {@link VERBS}.
+ */
+const cli_commands_js_1 = require("./cli-commands.js");
+/** The hidden runtime umbrella — not a human-facing verb to document. */
+exports.COVERAGE_EXEMPT = ["hook-runtime"];
+function escapeRegExp(s) {
+    return s.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
+}
+/**
+ * Whether `verb` appears in a COMMAND context anywhere in `content`:
+ * `vigiles <verb>` or a backtick-prefixed `` `<verb> ``. Generous on purpose
+ * (see the file header) — over-counting a verb as documented is the SAFE
+ * direction; under-counting would cry wolf.
+ */
+function verbMentioned(verb, content) {
+    const esc = escapeRegExp(verb);
+    return new RegExp(String.raw `(\bvigiles\s+|\x60)${esc}\b`).test(content);
+}
+/**
+ * Find public verbs not MENTIONED in any of the given doc files. Pure — the
+ * caller supplies file contents (so it runs over the repo's `docs/` in a test,
+ * or any file set). `verbs` defaults to the canonical {@link VERBS}; `exempt`
+ * drops the hidden umbrella.
+ */
+function findUndocumentedVerbs(docs, verbs = cli_commands_js_1.VERBS, exempt = exports.COVERAGE_EXEMPT) {
+    const mentioned = new Set();
+    for (const { content } of docs) {
+        for (const verb of verbs) {
+            if (!mentioned.has(verb) && verbMentioned(verb, content))
+                mentioned.add(verb);
+        }
+    }
+    return verbs.filter((v) => !exempt.includes(v) && !mentioned.has(v));
+}
+//# sourceMappingURL=doc-command-coverage.js.map

package/dist/eval-cache.d.ts

@@ -43,6 +43,12 @@ export interface CacheRecord {
     /** Text files present in the cwd after the run (relative path → contents). */
     readonly files: Record<string, string>;
 }
+/**
+ * Canonicalize a value so the key is stable regardless of object key order —
+ * recursively sorts object keys. Arrays keep order (it's significant for tools).
+ * Exported so the eval LOCK ({@link ./eval-lock}) hashes its inputs the same way.
+ */
+export declare function canonical(value: unknown): unknown;
 /**
  * Cache record-format version, SALTED into every key (Jest `CACHE_VERSION` /
  * webpack `cache.version` pattern). Bump when the `CacheRecord` shape — or how a

package/dist/eval-cache.js

@@ -1,6 +1,7 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.CACHE_FORMAT_VERSION = void 0;
+exports.canonical = canonical;
 exports.cacheKey = cacheKey;
 exports.readCache = readCache;
 exports.writeCache = writeCache;
@@ -32,6 +33,7 @@ const SKIP_DIRS = new Set(["node_modules", ".git"]);
 /**
  * Canonicalize a value so the key is stable regardless of object key order —
  * recursively sorts object keys. Arrays keep order (it's significant for tools).
+ * Exported so the eval LOCK ({@link ./eval-lock}) hashes its inputs the same way.
  */
 function canonical(value) {
     if (Array.isArray(value))

package/dist/eval-cost.d.ts

@@ -0,0 +1,75 @@
+/**
+ * Eval cost transparency — make what a real-model run SPENT impossible to miss.
+ * vigiles's whole affordability pitch is "runs on your Claude subscription, not a
+ * metered API," so every real-model run should say — out loud — how many tokens it
+ * spent, the API-equivalent dollar cost, and (loudly) if it was billed to a
+ * METERED API key instead of your subscription.
+ *
+ * HONEST SCOPE: we surface tokens + the API-equivalent `$` (`total_cost_usd` from
+ * the `claude` CLI) + a running session tally. We deliberately do NOT show a
+ * "% of your subscription" — Anthropic does not expose a subscription's quota or
+ * limit programmatically (and the real limits are rolling rate windows, not a
+ * dollar bucket), so any percentage would be fiction. See docs/eval-architecture.md.
+ *
+ * Pure + injectable (env + an output sink), so the whole thing is unit-tested
+ * without a model or a real key.
+ */
+import type { EvalUsage, ArmUsage, EvalReport } from "./eval.js";
+/** A normalized cost/token snapshot — the common shape a report renders from. */
+export interface CostSummary {
+    /** API-equivalent cost (`total_cost_usd`) — the number that matters. */
+    readonly costUsd: number;
+    readonly inputTokens: number;
+    readonly outputTokens: number;
+    readonly cacheCreationTokens: number;
+    readonly cacheReadTokens: number;
+}
+/** Total tokens across all four billing buckets. */
+export declare function totalTokens(c: CostSummary): number;
+/** A per-run {@link EvalUsage} → the common snapshot. */
+export declare function costFromRun(u: EvalUsage): CostSummary;
+/** An aggregated per-arm {@link ArmUsage} → the common snapshot. */
+export declare function costFromArm(u: ArmUsage): CostSummary;
+/** Sum any number of snapshots (e.g. every arm of an A/B). */
+export declare function sumCosts(costs: readonly CostSummary[]): CostSummary;
+/** The whole-{@link EvalReport} cost — every arm summed. */
+export declare function costFromEvalReport(report: EvalReport): CostSummary;
+/**
+ * How the run was billed. `metered` is true when a real Anthropic API key is in
+ * the environment — the `claude` CLI bills those PER TOKEN, whereas a
+ * subscription run (auth via `~/.claude`, no key var) costs $0 beyond the sub.
+ * The mock/deterministic tier never reaches here (it has no real cost), so a
+ * present key means a real metered run.
+ */
+export interface Billing {
+    readonly metered: boolean;
+    /** Which env var carried the key (for the actionable "unset X" message). */
+    readonly keyVar: string | null;
+}
+export declare function detectBilling(env?: NodeJS.ProcessEnv): Billing;
+/** Add a run to the running session total and return the new total. */
+export declare function recordSessionCost(c: CostSummary): CostSummary;
+/** The session total so far. */
+export declare function sessionCost(): CostSummary;
+/** Reset the session tally (test seam). */
+export declare function resetSessionCost(): void;
+/**
+ * The human-readable cost block for a run. Shows tokens + API-equivalent `$`, the
+ * billed-to line (a LOUD warning + an actionable fix when metered, a green ✅ when
+ * on the subscription), and the session tally when it exceeds this run.
+ */
+export declare function formatCostSummary(c: CostSummary, opts: {
+    billing: Billing;
+    session?: CostSummary | null;
+}): string;
+/**
+ * Record `c` into the session tally and emit its cost block. The default sink is
+ * stderr (so a run's cost never pollutes `--json` stdout). Injectable env + sink
+ * keep it fully testable. Returns the emitted text (also handy for a skill to
+ * relay to the user).
+ */
+export declare function emitCostSummary(c: CostSummary, opts?: {
+    env?: NodeJS.ProcessEnv;
+    out?: (s: string) => void;
+}): string;
+//# sourceMappingURL=eval-cost.d.ts.map

package/dist/eval-cost.js

@@ -0,0 +1,134 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.totalTokens = totalTokens;
+exports.costFromRun = costFromRun;
+exports.costFromArm = costFromArm;
+exports.sumCosts = sumCosts;
+exports.costFromEvalReport = costFromEvalReport;
+exports.detectBilling = detectBilling;
+exports.recordSessionCost = recordSessionCost;
+exports.sessionCost = sessionCost;
+exports.resetSessionCost = resetSessionCost;
+exports.formatCostSummary = formatCostSummary;
+exports.emitCostSummary = emitCostSummary;
+const ZERO = {
+    costUsd: 0,
+    inputTokens: 0,
+    outputTokens: 0,
+    cacheCreationTokens: 0,
+    cacheReadTokens: 0,
+};
+/** Total tokens across all four billing buckets. */
+function totalTokens(c) {
+    return (c.inputTokens + c.outputTokens + c.cacheCreationTokens + c.cacheReadTokens);
+}
+/** A per-run {@link EvalUsage} → the common snapshot. */
+function costFromRun(u) {
+    return {
+        costUsd: u.costUsd,
+        inputTokens: u.inputTokens,
+        outputTokens: u.outputTokens,
+        cacheCreationTokens: u.cacheCreationTokens,
+        cacheReadTokens: u.cacheReadTokens,
+    };
+}
+/** An aggregated per-arm {@link ArmUsage} → the common snapshot. */
+function costFromArm(u) {
+    return {
+        costUsd: u.totalCostUsd,
+        inputTokens: u.totalInputTokens,
+        outputTokens: u.totalOutputTokens,
+        cacheCreationTokens: u.totalCacheCreationTokens,
+        cacheReadTokens: u.totalCacheReadTokens,
+    };
+}
+/** Sum any number of snapshots (e.g. every arm of an A/B). */
+function sumCosts(costs) {
+    return costs.reduce((a, c) => ({
+        costUsd: a.costUsd + c.costUsd,
+        inputTokens: a.inputTokens + c.inputTokens,
+        outputTokens: a.outputTokens + c.outputTokens,
+        cacheCreationTokens: a.cacheCreationTokens + c.cacheCreationTokens,
+        cacheReadTokens: a.cacheReadTokens + c.cacheReadTokens,
+    }), ZERO);
+}
+/** The whole-{@link EvalReport} cost — every arm summed. */
+function costFromEvalReport(report) {
+    return sumCosts(Object.values(report.arms).map((a) => costFromArm(a.usage)));
+}
+const KEY_VARS = ["ANTHROPIC_API_KEY", "ANTHROPIC_AUTH_TOKEN"];
+function detectBilling(env = process.env) {
+    for (const v of KEY_VARS) {
+        const val = env[v];
+        if (val !== undefined && val.trim() !== "")
+            return { metered: true, keyVar: v };
+    }
+    return { metered: false, keyVar: null };
+}
+// --- Session tally (within one process) ------------------------------------
+let SESSION = ZERO;
+/** Add a run to the running session total and return the new total. */
+function recordSessionCost(c) {
+    SESSION = sumCosts([SESSION, c]);
+    return SESSION;
+}
+/** The session total so far. */
+function sessionCost() {
+    return SESSION;
+}
+/** Reset the session tally (test seam). */
+function resetSessionCost() {
+    SESSION = ZERO;
+}
+// --- Formatting ------------------------------------------------------------
+function fmtUsd(n) {
+    return `$${n < 0.01 && n > 0 ? n.toFixed(4) : n.toFixed(2)}`;
+}
+function fmtInt(n) {
+    return Math.round(n).toLocaleString("en-US");
+}
+function fmtK(n) {
+    return n >= 1000 ? `${(n / 1000).toFixed(1)}k` : String(Math.round(n));
+}
+/**
+ * The human-readable cost block for a run. Shows tokens + API-equivalent `$`, the
+ * billed-to line (a LOUD warning + an actionable fix when metered, a green ✅ when
+ * on the subscription), and the session tally when it exceeds this run.
+ */
+function formatCostSummary(c, opts) {
+    const lines = [];
+    lines.push(`  Spent: ${fmtInt(totalTokens(c))} tokens ` +
+        `(${fmtK(c.inputTokens)} in · ${fmtK(c.outputTokens)} out · ${fmtK(c.cacheReadTokens)} cache) ` +
+        `· ~${fmtUsd(c.costUsd)} API-equivalent`);
+    if (opts.billing.metered) {
+        const v = opts.billing.keyVar ?? "ANTHROPIC_API_KEY";
+        lines.push(`  ⚠ Billed to: METERED API (${v} is set) — you paid ~${fmtUsd(c.costUsd)} this run.`, `     Run it free on your Claude subscription: unset ${v}, then \`claude login\`.`);
+    }
+    else {
+        lines.push(`  Billed to: your Claude subscription — $0 metered ✅`);
+    }
+    if (opts.session && opts.session.costUsd > c.costUsd) {
+        lines.push(`  Session so far: ${fmtInt(totalTokens(opts.session))} tokens · ~${fmtUsd(opts.session.costUsd)} API-equivalent`);
+    }
+    return lines.join("\n");
+}
+/**
+ * Record `c` into the session tally and emit its cost block. The default sink is
+ * stderr (so a run's cost never pollutes `--json` stdout). Injectable env + sink
+ * keep it fully testable. Returns the emitted text (also handy for a skill to
+ * relay to the user).
+ */
+function emitCostSummary(c, opts = {}) {
+    // A no-cost run (a replay, a zero-trial eval) has nothing to report.
+    if (totalTokens(c) === 0 && c.costUsd === 0)
+        return "";
+    const billing = detectBilling(opts.env ?? process.env);
+    const session = recordSessionCost(c);
+    const text = formatCostSummary(c, { billing, session });
+    (opts.out ??
+        ((s) => {
+            console.error(s);
+        }))(text);
+    return text;
+}
+//# sourceMappingURL=eval-cost.js.map

package/dist/eval-lock.d.ts

@@ -0,0 +1,192 @@
+import { type SHA256Hash } from "./core/hash.js";
+/** Lock mode: never touch the lock / verify-only (CI) / record-and-write (local). */
+export type LockMode = "off" | "check" | "update";
+/**
+ * Per-spec lock overrides (additive on `EvalSpec`/`TriggerRateSpec`). Normally the
+ * mode comes from the CLI (`eval --check`/`--update` → `VIGILES_EVAL_LOCK`) and
+ * the dir/epoch from defaults/config; set these to drive the lock programmatically
+ * (or to point a test at a throwaway dir). Each field falls back to its env/default.
+ */
+export interface EvalLockOptions {
+    /** Override the lock mode (else `VIGILES_EVAL_LOCK`, else `off`). */
+    readonly mode?: LockMode;
+    /** Override the lock directory (else `<cwd>/.vigiles/eval-locks`). */
+    readonly dir?: string;
+    /** Override the behavior epoch (else `VIGILES_EVAL_API_VERSION`, else 1). */
+    readonly evalApiVersion?: number;
+}
+/**
+ * On-disk lock-format version, salted into nothing (the lock is keyed by name,
+ * not by hash) but VALIDATED on read so an incompatible shape fails loud rather
+ * than deserializing into a stale structure. Bump on a breaking shape change.
+ */
+export declare const LOCK_VERSION = 1;
+/** Default directory for committed eval locks (tracked, NOT gitignored). */
+export declare const DEFAULT_LOCK_DIR = ".vigiles/eval-locks";
+/**
+ * The model-affecting inputs hashed into a lock's `inputsHash`. Everything here
+ * is something that, if it changes, means the recorded model behavior is stale
+ * and you MUST re-drive the model (→ subscription → local). Deliberately EXCLUDED:
+ * the scoring `measure`/assertions (re-run live against the replayed report), the
+ * trial count (a sample-size knob, not a behavior input), and per-run env noise.
+ */
+export interface EvalLockInputs {
+    /** Model id used (folded in; a floating alias can't detect weight drift — warned). */
+    readonly model: string;
+    /**
+     * A hand-bumped behavior epoch the project owns (`.vigilesrc.json`
+     * `eval.apiVersion`), bumped when a harness-side change YOU made (a CLAUDE.md
+     * edit, a global hook) would shift eval outputs but isn't otherwise in the
+     * inputs. The escape hatch for "force a re-eval."
+     */
+    readonly evalApiVersion: number;
+    /**
+     * The seam-specific canonical input object — the tasks/prompts/files/settings/
+     * sorted-tools/pluginDirHash/serialized-checks that steer the model. Assembled
+     * by each entry point (it knows its own shape) and hashed opaquely here.
+     */
+    readonly inputs: unknown;
+}
+/**
+ * Why the harness binary version is **NOT** hashed (only recorded as provenance):
+ * `--check` runs in CI where `claude` is PINNED to a fixed version, while a dev's
+ * local `claude` is whatever they have — folding the version into the hash would
+ * false-trip `--check` on every PR where those differ. It is also the lock's
+ * honest scope: the gate verifies your committed results match your current
+ * *author-controlled inputs*, not current model/harness behavior (there is no
+ * automated live run). Harness/model drift is caught when YOU re-run `--update`
+ * locally and review the moved numbers in the git diff. Keeping the version out
+ * of the hash is what lets `--check` stay binary-free + deterministic in CI.
+ * (The eval CACHE still keys on it — that's local replay soundness, a different
+ * axis.) See research/cache-invalidation.md.
+ */
+/** Deterministic content hash of a lock's model-affecting inputs. */
+export declare function evalInputsHash(input: EvalLockInputs): SHA256Hash;
+/** A committed eval lock: the integrity stamp + the replayable recorded report. */
+export interface EvalLock {
+    readonly version: number;
+    /** The eval's report name (human-facing; also the lock filename slug source). */
+    readonly name: string;
+    /** Hash of the model-affecting inputs ({@link evalInputsHash}). */
+    readonly inputsHash: string;
+    /** The model id the report was produced against (for the drift warning). */
+    readonly model: string;
+    /** The harness version token at record time (provenance; already in the hash). */
+    readonly harnessVersionKey: string;
+    /** The behavior epoch at record time (provenance; already in the hash). */
+    readonly evalApiVersion: number;
+    /** ISO-8601 timestamp the lock was recorded (provenance; NOT in the hash). */
+    readonly builtAt: string;
+    /**
+     * The entry point's recorded report — the model's observed behavior, REPLAYED
+     * verbatim on `--check` so the script's own assertions judge it. Stored as the
+     * exact return type of the entry point (`EvalReport` / `TriggerRateReport` /
+     * `CheckReport`) so replay is transparent to the caller.
+     */
+    readonly report: unknown;
+}
+/** Filesystem-safe slug for a report name (the lock filename). */
+export declare function lockSlug(name: string): string;
+/** Path to a named eval's lock file under `dir`. */
+export declare function lockPath(dir: string, name: string): string;
+/**
+ * Read a named eval's lock. A MISS (no file) returns `null`. A CORRUPT or
+ * wrong-version file **throws** — a broken lock is a real failure the CI gate
+ * must surface, not silently treat as "no lock" (which would let a stale eval
+ * pass). The message says how to recover.
+ */
+export declare function readLock(dir: string, name: string): EvalLock | null;
+/**
+ * Whether ANY lock has been committed under `dir`. The CI staleness gate
+ * (`eval --check`) uses this to stay a NO-OP until the feature is in use: a repo
+ * that has never run `eval --update` has no locks, so there is nothing to verify
+ * and CI passes green. Once the first lock is committed, every named eval is held
+ * to having a fresh one (a new unlocked eval then reads as stale). The graduated,
+ * opt-in-by-committing behavior that keeps a fresh `init` from going red.
+ */
+export declare function anyLocksCommitted(dir: string): boolean;
+/**
+ * Does an edited path plausibly change an eval's INPUTS — so a committed lock may
+ * now be stale? Two surfaces feed the hash: a skill's trigger surface (`SKILL.md`)
+ * and the eval script that holds the prompts/spec (`*.eval.{mjs,cjs,js,mts,cts,ts}`).
+ * Pure (string-only) so the nudge hook stays cheap and never runs an eval script.
+ */
+export declare function isEvalInputFile(path: string): boolean;
+/**
+ * The NON-BLOCKING nudge to emit after an eval-input edit when committed locks
+ * exist, or `null` for no nudge. Self-gating: it stays silent until you've opted
+ * into the lock (committed one), so it can't annoy a repo that doesn't use evals.
+ * It deliberately does NOT recompute staleness (that needs the eval script + is
+ * the job of `eval --check`) — a reminder, not a gate. The honest harness-neutral
+ * reminder; how it reaches the agent (both CC and Codex inject `additionalContext`
+ * on `PostToolUse`) is the caller's concern. See docs/harness-testing-*.md.
+ */
+export declare function evalLockNudge(filePath: string, lockDir: string): string | null;
+/** Write a named eval's lock (pretty JSON for a reviewable git diff). */
+export declare function writeLock(dir: string, lock: EvalLock): void;
+/**
+ * Build a fresh lock envelope from a just-recorded report (the `--update` write).
+ * `builtAt` is passed in (never read from the clock here) so the module stays
+ * pure + deterministically testable; the CLI stamps the real timestamp.
+ */
+export declare function buildLock(args: {
+    readonly name: string;
+    readonly inputsHash: string;
+    readonly model: string;
+    readonly harnessVersionKey: string;
+    readonly evalApiVersion: number;
+    readonly builtAt: string;
+    readonly report: unknown;
+}): EvalLock;
+/** What the lock layer decides an entry point should do for this run. */
+export type LockDecision = 
+/** Drive the model normally (mode `off`, or `update`, or `check` with no lock-skip). */
+{
+    readonly kind: "run";
+}
+/** `check` + a matching fresh lock → return the recorded report, NO model call. */
+ | {
+    readonly kind: "replay";
+    readonly report: unknown;
+}
+/** `check` + a missing/stale lock → fail; the caller throws `reason`. */
+ | {
+    readonly kind: "stale";
+    readonly reason: string;
+};
+/**
+ * Decide what `check` mode should do given the current input hash and the
+ * committed lock. `off`/`update` always `run` (update records afterwards). `check`
+ * replays a matching lock (no model) and is `stale` on a missing lock or a hash
+ * mismatch — the deterministic CI gate.
+ */
+export declare function decideLock(mode: LockMode, name: string, currentHash: string, existing: EvalLock | null): LockDecision;
+/** A single numeric leaf that moved between the prior lock and a fresh `--update`. */
+export interface NumberDelta {
+    readonly path: string;
+    readonly before: number;
+    readonly after: number;
+}
+/**
+ * Collect the numeric leaves that changed between two recorded reports — the
+ * human-facing delta printed at `--update` time (e.g. `rate: 0.900 → 0.650`).
+ * Generic over any report shape (walks numbers by dotted path), so it works for
+ * `EvalReport`, `TriggerRateReport`, and `CheckReport` without per-type code. The
+ * committed git diff is the primary review surface; this is the at-a-glance echo.
+ */
+export declare function diffReportNumbers(before: unknown, after: unknown): NumberDelta[];
+/** Render the `--update` result for a human: NEW lock, or the per-number deltas. */
+export declare function formatLockUpdate(name: string, deltas: readonly NumberDelta[], isNew: boolean): string;
+/**
+ * Read the lock mode from the environment (`VIGILES_EVAL_LOCK`), set by the CLI's
+ * `eval --check` / `--update` flags. A run knob (like `VIGILES_TRIALS`): the CLI
+ * is the only place that should set it. Anything unrecognized → `off`.
+ */
+export declare function lockModeFromEnv(env?: NodeJS.ProcessEnv): LockMode;
+/**
+ * The behavior epoch (`evalApiVersion`) for this run, read from the env the CLI
+ * populates from `.vigilesrc.json` `eval.apiVersion`. Default 1. A malformed
+ * value falls back to 1 (never throws) — the lock stays usable.
+ */
+export declare function evalApiVersionFromEnv(env?: NodeJS.ProcessEnv): number;
+//# sourceMappingURL=eval-lock.d.ts.map