vigiles 11.0.0 → 12.1.0

package/dist/eval.js

@@ -22,7 +22,6 @@ exports.aggregateUsage = aggregateUsage;
 exports.isDatedModel = isDatedModel;
 exports.modelTier = modelTier;
 exports.belowModelFloor = belowModelFloor;
-exports.harnessVersionKey = harnessVersionKey;
 exports.ephemeralRunEnv = ephemeralRunEnv;
 exports.seedEphemeralHome = seedEphemeralHome;
 exports.isRateLimited = isRateLimited;
@@ -68,9 +67,11 @@ const node_os_1 = require("node:os");
 const node_path_1 = require("node:path");
 const plugin_loader_js_1 = require("./adapters/claude-code/plugin-loader.js");
 const runtime_js_1 = require("./adapters/claude-code/runtime.js");
+const eval_cost_js_1 = require("./eval-cost.js");
 const proofs_js_1 = require("./core/proofs.js");
 const harness_test_js_1 = require("./harness-test.js");
 const eval_cache_js_1 = require("./eval-cache.js");
+const eval_lock_js_1 = require("./eval-lock.js");
 const stats_js_1 = require("./stats.js");
 const tool_intercept_js_1 = require("./tool-intercept.js");
 const tool_stub_js_1 = require("./tool-stub.js");
@@ -144,7 +145,11 @@ function spawnAgent(a) {
  * `runEvalWith` with the real agent runner.
  */
 async function runEval(spec) {
-    return runEvalWith(spec, spawnAgent);
+    const report = await runEvalWith(spec, spawnAgent);
+    // Surface what the run spent — tokens + API-equivalent $, and a LOUD warning if
+    // it was billed to a metered API key instead of the subscription. See eval-cost.ts.
+    (0, eval_cost_js_1.emitCostSummary)((0, eval_cost_js_1.costFromEvalReport)(report));
+    return report;
 }
 /**
  * Score a check vocabulary across trials — the scored counterpart to
@@ -201,6 +206,7 @@ async function measureWith(spec, runner) {
                     n: s?.n ?? 0,
                 };
             }),
+            usage: arm?.usage ?? aggregateUsage([]),
         };
     }
     finally {
@@ -211,7 +217,9 @@ async function measureWith(spec, runner) {
 /* v8 ignore start -- real claude subprocess; thin wrapper over measureWith */
 /** Score a check vocabulary across trials against the real `claude` CLI. */
 async function measure(spec) {
-    return measureWith(spec, spawnAgent);
+    const report = await measureWith(spec, spawnAgent);
+    (0, eval_cost_js_1.emitCostSummary)((0, eval_cost_js_1.costFromArm)(report.usage));
+    return report;
 }
 /** Score checks across arms (injectable runner). Reuses `runEvalWith`. */
 async function measureArmsWith(spec, runner) {
@@ -245,6 +253,7 @@ async function measureArmsWith(spec, runner) {
                         n: s?.n ?? 0,
                     };
                 }),
+                usage: arm.usage,
             };
         }
         return { arms };
@@ -278,7 +287,10 @@ function stubArmPluginDirs(arms) {
 /* v8 ignore start -- real claude subprocess; thin wrapper over measureArmsWith */
 /** Score checks across arms against the real `claude` CLI. */
 async function measureArms(spec) {
-    return measureArmsWith(spec, spawnAgent);
+    const report = await measureArmsWith(spec, spawnAgent);
+    // Sum every arm's spend — an A/B run pays for both arms.
+    (0, eval_cost_js_1.emitCostSummary)((0, eval_cost_js_1.sumCosts)(Object.values(report.arms).map((a) => (0, eval_cost_js_1.costFromArm)(a.usage))));
+    return report;
 }
 /* v8 ignore stop */
 /**
@@ -640,32 +652,22 @@ function belowModelFloor(model, floor) {
     const f = modelTier(floor);
     return m !== null && f !== null && m < f;
 }
-/**
- * Reduce a raw `--version` string to the **major.minor** cache-key token. We key
- * the cache on major.minor, NOT the patch: a patch release rarely changes agent
- * behaviour, so keying patches would churn the cache on every release for no
- * signal; a minor/major bump is where the system prompt / tool defs actually move.
- * (If a specific patch is known to matter, clear the cache or bump
- * `CACHE_FORMAT_VERSION`.) Falls back to the trimmed raw string when no semver is
- * found. Pure + tested.
- */
-function harnessVersionKey(raw) {
-    const m = /(\d+)\.(\d+)\.\d+/.exec(raw);
-    return m ? `${m[1]}.${m[2]}` : raw.trim();
-}
 /* v8 ignore start -- spawns the real harness binary; memoized, cache-path only */
 let cachedHarnessVersion;
 /**
- * The harness binary version (`claude --version`) reduced to major.minor, for the
- * cache key — so a CLI **minor/major** upgrade (new system prompt / tool defs)
- * invalidates a stale replay, while patches don't churn it. Memoized (one spawn
- * per process), resolved only on the cache path, "unknown" if the binary isn't
- * found (then it doesn't partition the key).
+ * The harness binary version (`claude --version`), reduced to its
+ * behaviorally-significant token by the runtime port's
+ * {@link HarnessRuntime.versionKey} — so a Claude Code **minor/major** upgrade
+ * (new system prompt / tool defs) invalidates a stale replay while patches don't
+ * churn it. The reduction is per-harness (CC → `major.minor`, Codex → `""`) and
+ * lives on the adapter, not here. Memoized (one spawn per process), resolved only
+ * on the cache path, "unknown" if the binary isn't found (then it doesn't
+ * partition the key).
  */
 function harnessVersion() {
     if (cachedHarnessVersion === undefined) {
         try {
-            cachedHarnessVersion = harnessVersionKey((0, node_child_process_1.execSync)(`${runtime_js_1.claudeCodeRuntime.agentBinary} --version`, {
+            cachedHarnessVersion = runtime_js_1.claudeCodeRuntime.versionKey((0, node_child_process_1.execSync)(`${runtime_js_1.claudeCodeRuntime.agentBinary} --version`, {
                 encoding: "utf-8",
                 stdio: ["ignore", "pipe", "ignore"],
             }));
@@ -983,15 +985,141 @@ function aggregateArms(armNames, results) {
     }
     return { arms, totalCostUsd };
 }
+function resolveLock(over) {
+    return {
+        mode: over?.mode ?? (0, eval_lock_js_1.lockModeFromEnv)(),
+        dir: over?.dir ?? (0, node_path_1.resolve)(process.cwd(), eval_lock_js_1.DEFAULT_LOCK_DIR),
+        evalApiVersion: over?.evalApiVersion ?? (0, eval_lock_js_1.evalApiVersionFromEnv)(),
+    };
+}
+/** Emit a vigiles message (GitHub annotation under Actions, else stderr/stdout). */
+function emitLockMessage(msg, warn) {
+    if (process.env.GITHUB_ACTIONS)
+        console.log(`::${warn ? "warning" : "notice"}::${msg}`);
+    else if (warn)
+        console.warn(msg);
+    else
+        console.log(msg);
+}
+/**
+ * Run a named eval through the lock. `off` → just `produce()`. `check` → replay a
+ * matching committed lock (NO model call) or throw "stale". `update` → `produce()`,
+ * write the lock, print the human-facing delta. The model is driven ONLY on the
+ * run path — never on a clean `check`, which is what keeps the CI gate binary-free.
+ * A lock-on run with no `name` is a LOUD skip (the lock needs a name to key the file).
+ */
+async function withEvalLock(args, produce) {
+    const { lock } = args;
+    if (lock.mode === "off")
+        return produce();
+    if (!args.name) {
+        // An unnamed eval can't be keyed to a lock. In `check` (the CI gate) running
+        // it would call the model — violating the no-model-in-CI contract and failing
+        // for missing auth — so FAIL LOUDLY instead of silently hitting the model.
+        // `update` (local, model available) keeps producing: the eval just isn't gated.
+        if (lock.mode === "check") {
+            throw new Error(`vigiles eval --check: an unnamed eval cannot run in CI — it has no ` +
+                `committed lock to replay, and running it would call the model. Add a ` +
+                `\`name\` to the spec to gate it, or exclude it from the --check run.`);
+        }
+        emitLockMessage(`vigiles eval --${lock.mode}: skipped the lock for an unnamed eval — set ` +
+            `\`name\` on the spec to enable the staleness gate for it.`, true);
+        return produce();
+    }
+    if (!isDatedModel(args.model))
+        warnFloatingModel(args.model);
+    const inputsHash = (0, eval_lock_js_1.evalInputsHash)({
+        model: args.model,
+        evalApiVersion: lock.evalApiVersion,
+        inputs: args.inputs,
+    });
+    const existing = (0, eval_lock_js_1.readLock)(lock.dir, args.name);
+    const decision = (0, eval_lock_js_1.decideLock)(lock.mode, args.name, inputsHash, existing);
+    if (decision.kind === "stale")
+        throw new Error(decision.reason);
+    if (decision.kind === "replay")
+        return decision.report;
+    const report = await produce();
+    const builtLock = (0, eval_lock_js_1.buildLock)({
+        name: args.name,
+        inputsHash,
+        model: args.model,
+        harnessVersionKey: harnessVersion(),
+        evalApiVersion: lock.evalApiVersion,
+        builtAt: new Date().toISOString(),
+        report,
+    });
+    const deltas = existing ? (0, eval_lock_js_1.diffReportNumbers)(existing.report, report) : [];
+    (0, eval_lock_js_1.writeLock)(lock.dir, builtLock);
+    emitLockMessage((0, eval_lock_js_1.formatLockUpdate)(args.name, deltas, existing === null), false);
+    return report;
+}
+/**
+ * Strip the machine-specific plugin-root prefix from a resolved value before it
+ * enters the lock hash. `resolveHarness` expands `${PLUGIN_ROOT}` in a plugin's
+ * hook commands to the checkout's ABSOLUTE path, so a lock recorded at
+ * `/home/dev/...` would be falsely STALE when `--check` recomputes it at
+ * `/home/runner/...` in CI (or any other machine). Normalizing the prefix back to
+ * a token makes the hash location-independent. No-op when there's no plugin root.
+ */
+function stripPluginRoot(value, absRoot) {
+    if (absRoot === "")
+        return value;
+    const json = JSON.stringify(value);
+    // A hookless plugin resolves to `settings: undefined`, which `JSON.stringify`
+    // returns as `undefined` (not a string) — pass it through rather than `.split`
+    // a non-string (which would throw before the eval can run).
+    if (json === undefined)
+        return value;
+    return JSON.parse(json.split(absRoot).join("${PLUGIN_ROOT}"));
+}
 /**
- * The eval orchestration — every arm × trial via `runner`, run through the cache
- * and a rate-limit retry, with at most `concurrency` in flight and an optional
- * `maxCostUsd` budget cap; metric + usage computed per run and aggregated per
- * arm. Exported with an injectable `runner` so the loop, `measure` context,
- * caching, pooling, and aggregation are unit-testable without spawning a model
- * (pass a fake returning canned stream-json). `runEval` is this with the real
- * agent runner.
+ * Resolve each arm's model-affecting inputs into a canonical object for the lock
+ * hash — WITHOUT running the model (it reads files + hashes plugin dirs only). The
+ * trial count is excluded (a sample-size knob, not a behavior input); `measure` is
+ * excluded by design (the script re-asserts against the replayed report).
  */
+function evalArmsInputs(spec, cfg) {
+    const arms = {};
+    for (const [name, arm] of Object.entries(spec.arms)) {
+        const resolved = (0, plugin_loader_js_1.resolveHarness)({
+            plugin: arm.plugin,
+            settings: arm.settings,
+            files: { ...spec.fixture, ...arm.files },
+        });
+        // The plugin root `resolveHarness` expanded into the resolved files/settings
+        // is this checkout's absolute path — normalize it out so the hash is the same
+        // on the dev's machine and in CI (else every plugin-with-root-hooks eval is
+        // falsely stale across machines).
+        const absRoot = arm.plugin ? (0, node_path_1.resolve)(process.cwd(), arm.plugin) : "";
+        arms[name] = {
+            model: arm.model ?? cfg.model,
+            tools: [...cfg.tools].sort(),
+            files: stripPluginRoot(resolved.files, absRoot),
+            settings: stripPluginRoot(resolved.settings, absRoot),
+            pluginDirHash: arm.pluginDir ? (0, eval_cache_js_1.hashDir)(arm.pluginDir) : undefined,
+            interceptTools: arm.interceptTools
+                ? (0, tool_intercept_js_1.serializeIntercepts)(arm.interceptTools)
+                : undefined,
+        };
+    }
+    // Tool stubs (`spec.stubs`) are written onto PATH before each trial, so a
+    // change to a canned CLI output IS a model-facing input change — fold a
+    // canonical (name-sorted) view into the hash so `--check` catches it. Sorted
+    // for a stable key regardless of declaration order; an empty list is the
+    // byte-identical-to-before default. Each ToolStub is plain serializable data.
+    const stubs = [...(spec.stubs ?? [])].sort((a, b) => a.name.localeCompare(b.name));
+    // `ephemeralEnv` swaps the trial's environment (scrubbed env + throwaway HOME
+    // vs the inherited process env), which can move tool/hook/agent behavior — a
+    // model-facing input, so it belongs in the hash. Normalize to a bool so a
+    // record under one mode can't be replayed under the other with the same hash.
+    return {
+        task: spec.task,
+        arms,
+        stubs,
+        ephemeralEnv: spec.ephemeralEnv === true,
+    };
+}
 async function runEvalWith(spec, runner) {
     const trials = spec.trials ?? 5;
     const spacing = (spec.spacingSec ?? 4) * 1000;
@@ -1024,9 +1152,16 @@ async function runEvalWith(spec, runner) {
             await sleep(spacing);
         return { armName: unit.armName, skipped: false, row, usage };
     };
-    const results = await runPool(units, concurrency, worker);
-    const { arms, totalCostUsd } = aggregateArms(Object.keys(spec.arms), results);
-    return { name: spec.name ?? "eval", trials, arms, totalCostUsd, aborted };
+    const lock = resolveLock(spec.lock);
+    // Resolve the lock inputs only when the lock is active (off skips the
+    // resolveHarness/hashDir work). `check` replays the committed report below
+    // without ever entering the run pool — so no model is driven in CI.
+    const inputs = lock.mode === "off" ? undefined : evalArmsInputs(spec, cfg);
+    return withEvalLock({ name: spec.name, inputs, model: cfg.model, lock }, async () => {
+        const results = await runPool(units, concurrency, worker);
+        const { arms, totalCostUsd } = aggregateArms(Object.keys(spec.arms), results);
+        return { name: spec.name ?? "eval", trials, arms, totalCostUsd, aborted };
+    });
 }
 /** Render one metric: `name=mean±se pass^k=…` (se/pass^k shown when measured). */
 function formatMetric(name, mean, stat) {
@@ -1065,6 +1200,7 @@ function formatEvalReport(report) {
 exports.claudeEvalDriver = {
     runner: spawnAgent,
     parse: parseClaudeRun,
+    harness: "claude-code",
 };
 /**
  * Package loose `<skillsDir>/<name>/SKILL.md` skills into a throwaway plugin dir
@@ -1372,13 +1508,17 @@ async function runTriggerTrial(prompt, cfg, runner) {
             pluginDir: cfg.pluginDir,
             timeoutMs: cfg.timeoutMs,
         });
+        // Usage comes from the parser (harness-neutral: Claude + Codex both fill it),
+        // and a run costs tokens even when it errors — so accumulate it either way.
+        const ctx = makeContext(cwd, out, cfg.parse);
         // An errored/rate-limited turn is NOT a "skill didn't fire" miss — it's
         // excluded from the rate, so e.g. a Codex usage limit can't read as recall 0.
         if (cfg.runError?.(out))
-            return { fired: 0, errored: true };
+            return { fired: 0, errored: true, usage: ctx.usage };
         return {
-            fired: cfg.fired(makeContext(cwd, out, cfg.parse)) ? 1 : 0,
+            fired: cfg.fired(ctx) ? 1 : 0,
             errored: false,
+            usage: ctx.usage,
         };
     }
     finally {
@@ -1415,6 +1555,7 @@ async function runTriggerSet(prompts, cfg, runner) {
         fired: firedBy.reduce((a, b) => a + b, 0),
         n: trialsBy.reduce((a, b) => a + b, 0),
         errored,
+        usages: outcomes.map((o) => o.usage),
     };
 }
 /**
@@ -1439,7 +1580,7 @@ function assertTriggerDiversity(spec) {
         });
     }
 }
-async function measureTriggerRateWith(spec, runner, parse = parseClaudeRun, runError) {
+async function measureTriggerRateWith(spec, runner, parse = parseClaudeRun, runError, harness = "claude-code") {
     // Deterministic gate FIRST — before spending a token (or packaging a skillsDir).
     assertTriggerDiversity(spec);
     // Model floor (default Sonnet): trigger-rate under-measures selection on a
@@ -1469,26 +1610,53 @@ async function measureTriggerRateWith(spec, runner, parse = parseClaudeRun, runE
         parse,
         runError,
     };
+    const lock = resolveLock(spec.lock);
     try {
-        const relevant = await runTriggerSet(spec.prompts, cfg, runner);
-        const base = {
-            rate: relevant.n > 0 ? relevant.fired / relevant.n : 0,
-            n: relevant.n,
-            perPrompt: relevant.perPrompt,
-            competitors,
-            errored: positiveOrUndefined(relevant.errored),
-        };
-        if ((spec.irrelevantPrompts?.length ?? 0) === 0)
-            return base;
-        const irrelevant = await runTriggerSet(spec.irrelevantPrompts ?? [], cfg, runner);
-        const fires = relevant.fired + irrelevant.fired;
-        return {
-            ...base,
-            errored: positiveOrUndefined(relevant.errored + irrelevant.errored),
-            falsePositiveRate: irrelevant.n > 0 ? irrelevant.fired / irrelevant.n : 0,
-            precision: fires > 0 ? relevant.fired / fires : undefined,
-            perIrrelevant: irrelevant.perPrompt,
-        };
+        // The lock hashes the skill UNDER TEST (its dir contents) + the prompts +
+        // model + tools — the inputs that steer whether it fires. `--check` replays
+        // the committed report with no model; `--update` records it. Resolved only
+        // when the lock is active. `trials` is excluded (a sample-size knob).
+        const triggerInputs = lock.mode === "off"
+            ? undefined
+            : {
+                pluginDirHash: (0, eval_cache_js_1.hashDir)(pluginDir),
+                prompts: [...spec.prompts],
+                irrelevantPrompts: spec.irrelevantPrompts
+                    ? [...spec.irrelevantPrompts]
+                    : undefined,
+                model: cfg.model,
+                tools: [...cfg.tools].sort(),
+                fixture: spec.fixture,
+                competitors,
+                // The harness the driver runs is a model-facing input — a Claude vs
+                // Codex run can fire a skill differently — so a recorded report is
+                // STALE if the eval is switched to another harness.
+                harness,
+            };
+        return await withEvalLock({ name: spec.name, inputs: triggerInputs, model: cfg.model, lock }, async () => {
+            const relevant = await runTriggerSet(spec.prompts, cfg, runner);
+            const base = {
+                rate: relevant.n > 0 ? relevant.fired / relevant.n : 0,
+                n: relevant.n,
+                perPrompt: relevant.perPrompt,
+                competitors,
+                errored: positiveOrUndefined(relevant.errored),
+                usage: aggregateUsage(relevant.usages),
+            };
+            if ((spec.irrelevantPrompts?.length ?? 0) === 0)
+                return base;
+            const irrelevant = await runTriggerSet(spec.irrelevantPrompts ?? [], cfg, runner);
+            const fires = relevant.fired + irrelevant.fired;
+            return {
+                ...base,
+                errored: positiveOrUndefined(relevant.errored + irrelevant.errored),
+                falsePositiveRate: irrelevant.n > 0 ? irrelevant.fired / irrelevant.n : 0,
+                precision: fires > 0 ? relevant.fired / fires : undefined,
+                perIrrelevant: irrelevant.perPrompt,
+                // Total cost across BOTH sets (the precision runs cost tokens too).
+                usage: aggregateUsage([...relevant.usages, ...irrelevant.usages]),
+            };
+        });
     }
     finally {
         // Remove the throwaway plugin dir we built from a loose `skillsDir`.
@@ -1506,7 +1674,10 @@ async function measureTriggerRateWith(spec, runner, parse = parseClaudeRun, runE
  */
 async function measureTriggerRate(spec, opts = {}) {
     const d = opts.evalDriver ?? exports.claudeEvalDriver;
-    return measureTriggerRateWith(spec, d.runner, d.parse, d.runError);
+    const report = await measureTriggerRateWith(spec, d.runner, d.parse, d.runError, d.harness ?? "claude-code");
+    // Surface what the run spent (tokens + API-equivalent $ + metered warning).
+    (0, eval_cost_js_1.emitCostSummary)((0, eval_cost_js_1.costFromArm)(report.usage));
+    return report;
 }
 /* v8 ignore stop */
 /** Format a trigger-rate report: overall %, then each prompt's rate. */

package/dist/research-index.d.ts

@@ -0,0 +1,31 @@
+/**
+ * Research-index completeness — the deterministic FLOOR keeping the `research/`
+ * corpus and its index (`research/CLAUDE.md.spec.ts`) in sync. The spec's
+ * `keyFiles` map is the AGENT-FACING index of every research doc; the compiler
+ * already verifies the OTHER direction (every indexed path EXISTS, else
+ * `vigiles compile` fails), so the only open gap is a doc that was ADDED but
+ * never indexed. This check closes it: every `research/*.md` (except the
+ * human-facing `README.md`) must appear in the index, or the dogfood test fails.
+ *
+ * Pure — the caller supplies the doc filenames and the index content (the spec
+ * source, where an entry is AUTHORED), so it runs over the real `research/` dir
+ * in a test or over any file set. Bidirectional sync = compiler (index ⊆ docs)
+ * + this check (docs ⊆ index).
+ */
+/** Docs that are the index itself / human front-door, not indexed entries. */
+export declare const INDEX_EXEMPT: readonly ["README.md"];
+/**
+ * Research doc basenames (e.g. `roadmap.md`) NOT referenced anywhere in
+ * `indexContent`. A doc counts as indexed if its repo-relative path
+ * (`research/<name>.md`) appears in the index — the exact form the spec's
+ * `keyFiles` keys use. Exempt docs (the README) are never flagged.
+ */
+export declare function unindexedResearchDocs(docFilenames: readonly string[], indexContent: string, exempt?: readonly string[]): string[];
+/**
+ * Index entries pointing at a `research/<name>.md` that no longer exists on
+ * disk. The compiler catches this at compile time (a missing `keyFiles` path is
+ * a compile error), so this is a belt-and-suspenders reader for a test that
+ * wants to assert it directly without invoking the compiler.
+ */
+export declare function deadIndexEntries(docFilenames: readonly string[], indexContent: string): string[];
+//# sourceMappingURL=research-index.d.ts.map

package/dist/research-index.js

@@ -0,0 +1,48 @@
+"use strict";
+/**
+ * Research-index completeness — the deterministic FLOOR keeping the `research/`
+ * corpus and its index (`research/CLAUDE.md.spec.ts`) in sync. The spec's
+ * `keyFiles` map is the AGENT-FACING index of every research doc; the compiler
+ * already verifies the OTHER direction (every indexed path EXISTS, else
+ * `vigiles compile` fails), so the only open gap is a doc that was ADDED but
+ * never indexed. This check closes it: every `research/*.md` (except the
+ * human-facing `README.md`) must appear in the index, or the dogfood test fails.
+ *
+ * Pure — the caller supplies the doc filenames and the index content (the spec
+ * source, where an entry is AUTHORED), so it runs over the real `research/` dir
+ * in a test or over any file set. Bidirectional sync = compiler (index ⊆ docs)
+ * + this check (docs ⊆ index).
+ */
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.INDEX_EXEMPT = void 0;
+exports.unindexedResearchDocs = unindexedResearchDocs;
+exports.deadIndexEntries = deadIndexEntries;
+/** Docs that are the index itself / human front-door, not indexed entries. */
+exports.INDEX_EXEMPT = ["README.md"];
+/**
+ * Research doc basenames (e.g. `roadmap.md`) NOT referenced anywhere in
+ * `indexContent`. A doc counts as indexed if its repo-relative path
+ * (`research/<name>.md`) appears in the index — the exact form the spec's
+ * `keyFiles` keys use. Exempt docs (the README) are never flagged.
+ */
+function unindexedResearchDocs(docFilenames, indexContent, exempt = exports.INDEX_EXEMPT) {
+    return docFilenames.filter((name) => !exempt.includes(name) && !indexContent.includes(`research/${name}`));
+}
+/**
+ * Index entries pointing at a `research/<name>.md` that no longer exists on
+ * disk. The compiler catches this at compile time (a missing `keyFiles` path is
+ * a compile error), so this is a belt-and-suspenders reader for a test that
+ * wants to assert it directly without invoking the compiler.
+ */
+function deadIndexEntries(docFilenames, indexContent) {
+    const present = new Set(docFilenames);
+    const refs = indexContent.matchAll(/research\/([\w.-]+\.md)/g);
+    const dead = new Set();
+    for (const m of refs) {
+        const name = m[1];
+        if (!present.has(name))
+            dead.add(name);
+    }
+    return [...dead];
+}
+//# sourceMappingURL=research-index.js.map

package/dist/scan-behavioral.d.ts

@@ -135,6 +135,48 @@ export declare function measurePluginSelectionWith(dir: string, promptSet: Trigg
 export declare function measurePluginSelection(dir: string, promptSet: TriggerPromptSet, opts?: SelectionOptions): Promise<SelectionReport>;
 /** Format the selection-collision matrix as a scan-report section. */
 export declare function formatSelectionReport(r: SelectionReport): string;
+/** Options for {@link measureSelectionMatrix}: {@link SelectionOptions} plus an
+ * optional explicit prompt set (auto-derived from descriptions when omitted). */
+export interface SelectionMatrixOptions extends SelectionOptions {
+    /**
+     * Per-skill recall prompts. Omit for ZERO-SETUP — prompts are auto-derived
+     * from each skill's description (the same generator the audit trigger tier
+     * uses). Supply your own for a curated collision benchmark; only the `prompts`
+     * array per skill is read (any `irrelevant` bank is ignored here).
+     */
+    readonly prompts?: TriggerPromptSet;
+}
+/**
+ * Measure a plugin's skill-SELECTION collision matrix — "when I ask for skill i's
+ * job, does ONLY skill i fire?" The first-class, assertable form of the cross-skill
+ * collision measurement (pair with {@link assertNoCollision}). The matrix diagonal
+ * is recall; off-diagonal mass is collision — skill j hijacking skill i's prompt,
+ * the failure that breaks a multi-skill plugin and that per-skill trigger-rate
+ * (each skill in ISOLATION) structurally can't see.
+ *
+ * ZERO-SETUP: with no `prompts`, they're derived from each model-invocable skill's
+ * description. Claude Code only (Codex has no skill-selection event to read); needs
+ * the `claude` CLI + model auth, else `available: false`. Thin promotion of
+ * {@link measurePluginSelection}. See research/plugin-selection-collision.md.
+ */
+export declare function measureSelectionMatrix(dir: string, opts?: SelectionMatrixOptions): Promise<SelectionReport>;
+/**
+ * Injectable core of {@link measureSelectionMatrix} (for tests): auto-derive the
+ * prompts (unless supplied) and drive the matrix via a fake/real probe.
+ */
+export declare function measureSelectionMatrixWith(dir: string, probe: HarnessProbe, opts?: SelectionMatrixOptions): Promise<SelectionReport>;
+/**
+ * Assert a plugin's skills don't hijack each other — the gate over a
+ * {@link SelectionReport} from {@link measureSelectionMatrix}. `maxOffDiagonal`
+ * caps EACH skill's collision rate (fraction of its own prompts on which a SIBLING
+ * fired); `maxPluginCollision` caps the plugin-wide rate. With neither set it
+ * demands ZERO collision. THROWS (never a silent green) when nothing was measured
+ * — an unavailable harness or a zero-run report is a gap, not a pass.
+ */
+export declare function assertNoCollision(report: SelectionReport, opts?: {
+    maxOffDiagonal?: number;
+    maxPluginCollision?: number;
+}): void;
 /** Does a skill description assert a hard constraint (→ an adversarial-gate candidate)? */
 export declare function isGateDescription(description: string): boolean;
 /** A skill considered for gate detection — name + its (model-visible) description. */

package/dist/scan-behavioral.js

@@ -22,6 +22,9 @@ exports.buildSelectionReport = buildSelectionReport;
 exports.measurePluginSelectionWith = measurePluginSelectionWith;
 exports.measurePluginSelection = measurePluginSelection;
 exports.formatSelectionReport = formatSelectionReport;
+exports.measureSelectionMatrix = measureSelectionMatrix;
+exports.measureSelectionMatrixWith = measureSelectionMatrixWith;
+exports.assertNoCollision = assertNoCollision;
 exports.isGateDescription = isGateDescription;
 exports.detectGateSkills = detectGateSkills;
 exports.gateRubric = gateRubric;
@@ -33,6 +36,7 @@ const node_os_1 = require("node:os");
 const node_path_1 = require("node:path");
 const node_child_process_1 = require("node:child_process");
 const scan_js_1 = require("./scan.js");
+const audit_prompts_js_1 = require("./audit-prompts.js");
 const judge_js_1 = require("./judge.js");
 const eval_js_1 = require("./eval.js");
 const harness_assert_js_1 = require("./harness-assert.js");
@@ -397,6 +401,69 @@ function formatSelectionReport(r) {
     }
     return lines.join("\n");
 }
+/**
+ * Measure a plugin's skill-SELECTION collision matrix — "when I ask for skill i's
+ * job, does ONLY skill i fire?" The first-class, assertable form of the cross-skill
+ * collision measurement (pair with {@link assertNoCollision}). The matrix diagonal
+ * is recall; off-diagonal mass is collision — skill j hijacking skill i's prompt,
+ * the failure that breaks a multi-skill plugin and that per-skill trigger-rate
+ * (each skill in ISOLATION) structurally can't see.
+ *
+ * ZERO-SETUP: with no `prompts`, they're derived from each model-invocable skill's
+ * description. Claude Code only (Codex has no skill-selection event to read); needs
+ * the `claude` CLI + model auth, else `available: false`. Thin promotion of
+ * {@link measurePluginSelection}. See research/plugin-selection-collision.md.
+ */
+async function measureSelectionMatrix(dir, opts = {}) {
+    return measurePluginSelection(dir, resolveSelectionPrompts(dir, opts), opts);
+}
+/**
+ * Injectable core of {@link measureSelectionMatrix} (for tests): auto-derive the
+ * prompts (unless supplied) and drive the matrix via a fake/real probe.
+ */
+async function measureSelectionMatrixWith(dir, probe, opts = {}) {
+    return measurePluginSelectionWith(dir, resolveSelectionPrompts(dir, opts), probe, opts);
+}
+/** The prompts for a selection run: explicit if given, else auto-derived from the
+ * model-invocable skills' descriptions (the zero-setup path). */
+function resolveSelectionPrompts(dir, opts) {
+    return (opts.prompts ??
+        (0, audit_prompts_js_1.autoTriggerPrompts)((0, scan_js_1.scanPlugin)(dir)
+            .skills.filter((s) => !s.userInvoked && s.hasDescription)
+            .map((s) => ({ name: s.name, description: s.description ?? "" }))));
+}
+/**
+ * Assert a plugin's skills don't hijack each other — the gate over a
+ * {@link SelectionReport} from {@link measureSelectionMatrix}. `maxOffDiagonal`
+ * caps EACH skill's collision rate (fraction of its own prompts on which a SIBLING
+ * fired); `maxPluginCollision` caps the plugin-wide rate. With neither set it
+ * demands ZERO collision. THROWS (never a silent green) when nothing was measured
+ * — an unavailable harness or a zero-run report is a gap, not a pass.
+ */
+function assertNoCollision(report, opts = {}) {
+    if (!report.available)
+        throw new Error(`selection matrix unavailable — ${report.note ?? "n/a"}`);
+    if (report.n === 0)
+        throw new Error(`selection matrix measured nothing — ${report.note ?? "no runs"}`);
+    // Enforce the per-skill ceiling when asked, OR by default (name = NoCollision);
+    // if only the plugin-wide cap is given, don't also silently demand zero per-skill.
+    const maxOff = opts.maxOffDiagonal ??
+        (opts.maxPluginCollision === undefined ? 0 : undefined);
+    if (maxOff !== undefined) {
+        const worst = report.perSkill
+            .filter((s) => s.n > 0)
+            .reduce((w, s) => (w && w.collisionRate >= s.collisionRate ? w : s), undefined);
+        if (worst && worst.collisionRate > maxOff) {
+            const top = worst.collidesWith[0];
+            const tail = top ? ` (top collider: ${top.skill} ${pct(top.rate)})` : "";
+            throw new Error(`expected each skill's collision rate ≤ ${String(maxOff)}, but ${worst.skill} = ${worst.collisionRate.toFixed(2)}${tail}`);
+        }
+    }
+    if (opts.maxPluginCollision !== undefined &&
+        report.collisionRate > opts.maxPluginCollision) {
+        throw new Error(`expected plugin collision rate ≤ ${String(opts.maxPluginCollision)}, got ${report.collisionRate.toFixed(2)}`);
+    }
+}
 // ─── Enforcement-gate detection (for the adversarial-gate eval) ───────────────
 //
 // A skill whose description states a HARD CONSTRAINT ("always write tests first",