vigiles 14.3.0 → 14.4.0

package/dist/adapters/codex/eval.d.ts

@@ -68,6 +68,17 @@ export declare function installCodexSkills(pluginDir: string, cwd: string): numb
  * codexEvalDriver })` dispatches through.
  */
 export declare function codexEvalAgentRunner(args: AgentRunArgs): Promise<RunOut>;
+/**
+ * Why Codex trigger-rate is EXPERIMENTAL (not a supported measurement). Codex
+ * has NO skill-selection event, so `codexSkillFired` infers firing from whether
+ * the model READ `skills/<name>/SKILL.md` — which is wrong in BOTH directions: a
+ * cached/already-in-context skill isn't re-read (false NEGATIVE), and an
+ * exploratory read while listing isn't a real fire (false POSITIVE). So the
+ * number can be off either way. Deterministic `vigiles audit` is fully supported
+ * on Codex; only this behavioral tier is experimental until a live run measures
+ * the oracle's accuracy vs ground truth. See docs/harness-testing-codex.md.
+ */
+export declare const CODEX_TRIGGER_RATE_EXPERIMENTAL: string;
 /**
  * The Codex eval driver — pass to `measureTriggerRate(spec, { evalDriver:
  * codexEvalDriver })` to run a trigger-rate eval natively on `codex exec`. Pair

package/dist/adapters/codex/eval.js

@@ -27,7 +27,7 @@
  * not be re-read); pair with a behavioral/judged check for certainty.
  */
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.codexEvalDriver = void 0;
+exports.codexEvalDriver = exports.CODEX_TRIGGER_RATE_EXPERIMENTAL = void 0;
 exports.parseCodexEvalRun = parseCodexEvalRun;
 exports.codexRunError = codexRunError;
 exports.codexSkillFired = codexSkillFired;
@@ -186,6 +186,20 @@ function codexEvalAgentRunner(args) {
         timeoutMs: args.timeoutMs,
     }));
 }
+/**
+ * Why Codex trigger-rate is EXPERIMENTAL (not a supported measurement). Codex
+ * has NO skill-selection event, so `codexSkillFired` infers firing from whether
+ * the model READ `skills/<name>/SKILL.md` — which is wrong in BOTH directions: a
+ * cached/already-in-context skill isn't re-read (false NEGATIVE), and an
+ * exploratory read while listing isn't a real fire (false POSITIVE). So the
+ * number can be off either way. Deterministic `vigiles audit` is fully supported
+ * on Codex; only this behavioral tier is experimental until a live run measures
+ * the oracle's accuracy vs ground truth. See docs/harness-testing-codex.md.
+ */
+exports.CODEX_TRIGGER_RATE_EXPERIMENTAL = "Codex trigger-rate is experimental — Codex has no skill-selection event, so " +
+    "firing is inferred from a SKILL.md read (cache → false negative, exploratory " +
+    "read → false positive). Treat as directional, not a measurement. Deterministic " +
+    "`vigiles audit` is fully supported on Codex.";
 /**
  * The Codex eval driver — pass to `measureTriggerRate(spec, { evalDriver:
  * codexEvalDriver })` to run a trigger-rate eval natively on `codex exec`. Pair
@@ -198,6 +212,8 @@ exports.codexEvalDriver = {
     // The harness identity → folded into the trigger-rate lock hash, so a report
     // recorded on Claude Code is STALE if the eval is switched to Codex (and v.v.).
     harness: "codex",
+    // Codex-only: the trigger-rate number is not validated (see the constant above).
+    experimental: exports.CODEX_TRIGGER_RATE_EXPERIMENTAL,
 };
 /**
  * Spawn real `codex exec --json` for the eval tier (real model, the user's codex

package/dist/eval.d.ts

@@ -733,6 +733,13 @@ export interface TriggerRateReport {
     readonly errored?: number;
     /** Cost / tokens SPENT across all runs (relevant + irrelevant) — feeds the cost summary. */
     readonly usage: ArmUsage;
+    /**
+     * Present when the driver's harness measures trigger-rate on an EXPERIMENTAL
+     * basis (copied from {@link EvalDriver.experimental}) — the number is not
+     * validated and can be wrong. `formatTriggerRateReport` prints it as a loud
+     * caveat. Absent = a supported, trustworthy measurement (Claude Code).
+     */
+    readonly experimental?: string;
 }
 /**
  * An eval-tier transport: how to RUN a real harness turn and PARSE its output.
@@ -754,6 +761,16 @@ export interface EvalDriver {
      * `"claude-code"`, so an existing single-harness lock is unaffected.
      */
     readonly harness?: string;
+    /**
+     * When set, this driver's trigger-rate number is EXPERIMENTAL and not
+     * validated — the string is the human caveat explaining why (e.g. Codex has no
+     * skill-selection event, so firing is inferred from a SKILL.md read, which can
+     * be wrong in both directions). Absent = supported/trustworthy (the default,
+     * Claude Code). `measureTriggerRate` copies it onto the report and warns; the
+     * formatter prints it. Precision-first: never let a possibly-wrong number read
+     * as a measurement.
+     */
+    readonly experimental?: string;
 }
 /**
  * The default (Claude Code) eval driver: real `claude` + stream-json parsing.

package/dist/eval.js

@@ -1730,7 +1730,16 @@ async function measureTriggerRateWith(spec, runner, parse = parseClaudeRun, runE
  */
 async function measureTriggerRate(spec, opts = {}) {
     const d = opts.evalDriver ?? exports.claudeEvalDriver;
-    const report = await measureTriggerRateWith(spec, d.runner, d.parse, d.runError, d.harness ?? "claude-code");
+    const measured = await measureTriggerRateWith(spec, d.runner, d.parse, d.runError, d.harness ?? "claude-code");
+    // Precision-first: if the driver flags its trigger-rate EXPERIMENTAL (Codex),
+    // carry the caveat onto the report and warn loudly, so the number is never
+    // mistaken for a validated measurement.
+    const report = d.experimental
+        ? { ...measured, experimental: d.experimental }
+        : measured;
+    if (d.experimental) {
+        process.stderr.write(`⚠ EXPERIMENTAL trigger-rate on ${d.harness ?? "?"}: ${d.experimental}\n`);
+    }
     // Surface what the run spent (tokens + API-equivalent $ + metered warning).
     (0, eval_cost_js_1.emitCostSummary)((0, eval_cost_js_1.costFromArm)(report.usage));
     // Feed the flight recorder: recall (+ precision when measured) for this skill.
@@ -1755,7 +1764,11 @@ async function measureTriggerRate(spec, opts = {}) {
 /** Format a trigger-rate report: overall %, then each prompt's rate. */
 function formatTriggerRateReport(report) {
     const pct = (report.rate * 100).toFixed(0);
-    const lines = [`trigger-rate: ${pct}% (${String(report.n)} runs)`];
+    const lines = [];
+    if (report.experimental) {
+        lines.push(`⚠ EXPERIMENTAL — ${report.experimental}`);
+    }
+    lines.push(`trigger-rate: ${pct}% (${String(report.n)} runs)`);
     for (const p of report.perPrompt) {
         lines.push(`  ${p.rate.toFixed(2)}  ${p.prompt.slice(0, 60)}`);
     }

package/dist/scan-behavioral.d.ts

@@ -40,6 +40,13 @@ export interface BehavioralReport {
     /** False when the `claude` CLI / auth is absent — the column couldn't run. */
     readonly available: boolean;
     readonly results: readonly SkillTriggerResult[];
+    /**
+     * Set when the driving harness measures trigger-rate on an EXPERIMENTAL basis
+     * (copied from {@link EvalDriver.experimental}) — Codex, whose firing is
+     * inferred from a SKILL.md read and can be wrong. {@link formatBehavioralReport}
+     * prints it as a loud caveat above the numbers. Absent = supported (Claude Code).
+     */
+    readonly experimental?: string;
 }
 export interface ProbeOptions {
     readonly concurrency?: number;

package/dist/scan-behavioral.js

@@ -165,6 +165,7 @@ async function probePluginTriggersWith(dir, promptSet, probe, opts = {}) {
     return {
         available: true,
         results: relabelTriggerArtifact(dir, probe, results),
+        experimental: probe.evalDriver.experimental,
     };
 }
 /** Relabel an all-zero-recall stubbed run on a hooked plugin as unmeasured (Layer 1). */
@@ -197,6 +198,11 @@ function formatBehavioralReport(b) {
         return "Behavioral (trigger-rate): no model-invocable skills to probe";
     }
     const lines = ["Behavioral (trigger-rate):"];
+    // Codex-only: the trigger-rate number is not validated (no skill-selection
+    // event → firing inferred from a SKILL.md read). Say so loudly, above the numbers.
+    if (b.experimental) {
+        lines.push(`  ⚠ EXPERIMENTAL — ${b.experimental}`);
+    }
     for (const r of b.results) {
         if (!r.measured) {
             lines.push(`  · ${r.skill} — unmeasured (${r.note ?? "skipped"})`);

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "vigiles",
-  "version": "14.3.0",
+  "version": "14.4.0",
   "description": "Lint & test the harness your AI agent runs on — verify the references in your CLAUDE.md / AGENTS.md and test that your hooks and skills actually work.",
   "keywords": [
     "claude-code",