vigiles 5.0.1 → 5.2.0

package/dist/core/types.d.ts

@@ -71,7 +71,7 @@ export interface OrphansConfig {
 }
 /**
  * Shared options for the per-kind untested-* rules (`untested-skill` /
- * `untested-agent` / `untested-hook`). Which kinds are scanned is controlled by
+ * `untested-subagent` / `untested-hook`). Which kinds are scanned is controlled by
  * each rule's severity (set a rule to `false` to skip that kind), so only the
  * test-discovery knobs live here.
  */
@@ -97,7 +97,7 @@ export interface RulesConfig {
     /** Flag a skill (SKILL.md) that ships with no test or eval. Default: "warn". */
     "untested-skill"?: RuleWithOptions<TestCoverageConfig>;
     /** Flag a subagent (agents/*.md) that ships with no test or eval. Default: "warn". */
-    "untested-agent"?: RuleWithOptions<TestCoverageConfig>;
+    "untested-subagent"?: RuleWithOptions<TestCoverageConfig>;
     /** Flag a hook script that ships with no test or eval. Default: "warn". */
     "untested-hook"?: RuleWithOptions<TestCoverageConfig>;
     /**
@@ -108,6 +108,95 @@ export interface RulesConfig {
      * block the edit, false → off.
      */
     "unmarked-refs"?: RuleSeverity;
+    /**
+     * Cross-reference each subagent's `tools:` rail against the harness tool
+     * catalog — flag a never-available tool or a close typo (the moat). Only
+     * high-confidence issues are reported (a bare unrecognized tool is likely
+     * plugin/MCP-provided, never flagged). Off unless set; "warn" surfaces,
+     * "error" gates CI. Same detector as `scan` + `compileAgent`.
+     */
+    "subagent-tool-contract"?: RuleSeverity;
+    /**
+     * Flag a hook registered under an event name the harness doesn't define (a
+     * typo → the hook never fires). High-precision: close typos only, never a
+     * framework/custom event. Default "warn"; "error" gates CI. Same detector as
+     * `scan`.
+     */
+    "hook-events"?: RuleSeverity;
+    /**
+     * Flag a skill/agent missing a required frontmatter field — a skill needs
+     * `name` (to load), an agent needs `name` + `description`. A broken surface
+     * that won't register. Default "warn"; "error" gates CI. Same detector as `scan`.
+     */
+    "subagent-frontmatter"?: RuleSeverity;
+    /**
+     * Flag a declared MCP server that can't start — neither a `command` (stdio)
+     * nor a `url` (http/sse). Default "warn"; "error" gates CI. Same detector as
+     * `scan`. (JSON `.mcp.json`/manifest `mcpServers`; Codex TOML not yet parsed.)
+     */
+    "mcp-config"?: RuleSeverity;
+    /**
+     * RECOMMEND (not require) that a SKILL.md declares an explicit `name` +
+     * `description` rather than relying on the dir-name / first-paragraph
+     * fallbacks — a more reliable trigger surface. The skill still loads without
+     * them, so this is a best-practice nudge: default "warn"; set "error" to
+     * enforce on your own skills. Same detector as `scan` (skillMetaIssues).
+     */
+    "skill-frontmatter"?: RuleSeverity;
+    /**
+     * Cross-reference an `mcp__server__tool` in a subagent's contract against the
+     * plugin's declared `mcpServers` — flag a server the plugin doesn't declare
+     * (the MCP half of the tool moat; `subagent-tool-contract` checks the built-in
+     * half). High-precision: only flags when the plugin SHIPS a declared set,
+     * allowlists harness built-ins (`ide`), and skips the plugin-namespaced
+     * `mcp__plugin_…` form. Default "warn"; "error" gates CI. Same detector as
+     * `scan` (mcpToolIssues).
+     */
+    "mcp-tool-resolves"?: RuleSeverity;
+    /**
+     * Flag a hook command that references a script file which doesn't exist on
+     * disk (with `${CLAUDE_PLUGIN_ROOT}` resolved) — the hook silently never runs.
+     * FP-safe: skips unresolved `$VAR` paths, existence-guarded one-liners, and
+     * inline commands. Matches Anthropic's own `claude plugin validate`. Default
+     * "warn"; "error" gates CI. Same detector as `scan` (hooks status "missing").
+     */
+    "hook-script-exists"?: RuleSeverity;
+    /**
+     * Cross-reference a subagent's `disallowedTools:` block-list against the
+     * catalog — the deny-side mirror of `subagent-tool-contract`. A close typo there
+     * blocks NOTHING (you meant to deny `Bash`, wrote `Bsh`), leaving the tool
+     * available. High-precision: close-typo only (a never-available tool is
+     * harmless to list, a bare unknown is likely a plugin tool). Default "warn";
+     * "error" gates CI. Same detector as `scan` (disallowedToolIssues).
+     */
+    "disallowed-tools-contract"?: RuleSeverity;
+    /**
+     * Flag two model-invocable skills whose descriptions are near-identical — the
+     * selector can't tell them apart, so the wrong one fires (a precision
+     * collision). A DETERMINISTIC NCD proxy for a `--trigger`-class behavioral bug;
+     * calibrated FP-safe (only basically-identical text, below the sweep's
+     * most-similar distinct pair). Default "warn"; "error" gates CI. Same detector
+     * as `scan` (descriptionOverlaps).
+     */
+    "description-overlap"?: RuleSeverity;
+    /**
+     * Flag a skill/agent whose `---` frontmatter block EXISTS but isn't valid YAML
+     * — fields may not parse as intended. CAVEAT: a real YAML parser (js-yaml) is
+     * stricter than some loaders, so a one-line `description:` containing a `: `
+     * colon or an `<example>` block is flagged even though it may still load.
+     * Hence default "warn" (a nudge), not "error" — verify before enforcing. Same
+     * detector as `scan` (malformedFrontmatter).
+     */
+    "frontmatter-valid"?: RuleSeverity;
+    /**
+     * Flag a `type: "mcp_tool"` hook action that's incomplete (missing `server` /
+     * `tool`) or targets a server the plugin doesn't declare in `mcpServers` — the
+     * hook silently never dispatches. High-precision: the undeclared-server half is
+     * gated on the plugin shipping a declared set and allowlists built-ins (`ide`),
+     * mirroring `mcp-tool-resolves`. Default "warn"; "error" gates CI. Same detector
+     * as `scan` (mcpHookIssues).
+     */
+    "mcp-hook-target-resolves"?: RuleSeverity;
 }
 /** Extract severity from a rule value (handles both simple and tuple forms). */
 export declare function ruleSeverity<T>(rule: RuleWithOptions<T> | undefined): RuleSeverity;

package/dist/core/validate.js

@@ -43,9 +43,31 @@ const DEFAULT_RULES = {
     coverage: false,
     // Per-kind surface-coverage: a skill/agent/hook must ship with a test or eval.
     "untested-skill": "warn",
-    "untested-agent": "warn",
+    "untested-subagent": "warn",
     "untested-hook": "warn",
     "unmarked-refs": "warn",
+    // High-precision (never-available + close typos only), so on by default at warn.
+    "subagent-tool-contract": "warn",
+    // High-precision (close typos only), on by default at warn.
+    "hook-events": "warn",
+    // Missing required frontmatter (name/description) — on by default at warn.
+    "subagent-frontmatter": "warn",
+    // A declared MCP server with no command/url can't start — on by default at warn.
+    "mcp-config": "warn",
+    // Best-practice nudge (skills load without frontmatter) — warn, not error.
+    "skill-frontmatter": "warn",
+    // High-precision (gated on a declared MCP set; built-ins allowlisted) — warn.
+    "mcp-tool-resolves": "warn",
+    // A hook script referenced but missing never runs — on by default at warn.
+    "hook-script-exists": "warn",
+    // High-precision (close-typo only) deny-list mirror of subagent-tool-contract.
+    "disallowed-tools-contract": "warn",
+    // Deterministic NCD precision proxy (near-identical skill descriptions) — warn.
+    "description-overlap": "warn",
+    // Malformed-YAML frontmatter — WARN only (js-yaml is stricter than some loaders).
+    "frontmatter-valid": "warn",
+    // A mcp_tool hook incomplete / targeting an undeclared server — on by default at warn.
+    "mcp-hook-target-resolves": "warn",
 };
 const DEFAULT_CONFIG = {
     ruleMarkers: ["headings", "checkboxes"],

package/dist/eval.d.ts

@@ -1,4 +1,4 @@
-import { type ToolCall, type Trace } from "./harness-test.js";
+import { parseToolCalls, parseHooks, parseSubagents, type ToolCall, type Trace } from "./harness-test.js";
 import { type CacheMode } from "./eval-cache.js";
 import type { Check, CheckJSON } from "./check.js";
 import { type Comparison } from "./stats.js";
@@ -250,6 +250,9 @@ export type AgentRunner = (args: AgentRunArgs) => Promise<RunOut>;
  * leak the host environment into an untrusted, model-driven run.
  */
 export declare function resolveSpawnEnv(a: Pick<AgentRunArgs, "env" | "replaceEnv">, base?: NodeJS.ProcessEnv): NodeJS.ProcessEnv;
+/** The real `claude`-spawning runner (composition root). Exported so other
+ *  real-model entries (e.g. `scan --trigger`) bind the same runner. */
+export declare function spawnAgent(a: AgentRunArgs): Promise<RunOut>;
 /**
  * Run the eval: every arm × every trial against the real `claude` CLI, with the
  * metric computed per run and aggregated per arm. Requires `claude` on PATH and
@@ -392,6 +395,24 @@ export declare function checkReportToJUnit(report: CheckReport, opts?: {
 }): string;
 /** Parse per-run cost/latency/tokens from a stream — pure, model-free. */
 export declare function parseUsage(stdout: string): EvalUsage;
+/**
+ * The harness-specific half of a run trace: how a real model's raw stdout maps
+ * to the common fields. Claude Code's `parseClaudeRun` reads its stream-json; a
+ * second harness (Codex) supplies its own parser of `codex exec --json` JSONL, so
+ * the eval tier (`measureTriggerRate`/`runEval`) isn't bound to Claude's format.
+ * The non-harness fields (cwd/exitCode/stdout/file/sh) stay in `makeContext`.
+ */
+export interface ParsedModelRun {
+    readonly turns: number;
+    readonly output: string;
+    readonly toolCalls: ReturnType<typeof parseToolCalls>;
+    readonly hooks: ReturnType<typeof parseHooks>;
+    readonly subagents: ReturnType<typeof parseSubagents>;
+    readonly usage: EvalUsage;
+}
+export type ModelOutputParser = (out: RunOut) => ParsedModelRun;
+/** Parse Claude Code's stream-json stdout into the common trace fields. */
+export declare function parseClaudeRun(out: RunOut): ParsedModelRun;
 /** Aggregate per-run metrics: mean for numbers, fraction-true (0..1) for booleans. */
 export declare function aggregate(rows: readonly Metrics[]): Record<string, number>;
 /**
@@ -593,6 +614,22 @@ export interface TriggerRateSpec {
     readonly timeoutMs?: number;
     /** Seconds to wait between runs (avoid rate-limit bursts). Default 4. */
     readonly spacingSec?: number;
+    /**
+     * Files (path → contents) seeded into every run's cwd before the prompt — the
+     * filesystem CONTEXT the skill is measured in. The default empty cwd is faithful
+     * for opening-move skills ("describe a feature", "debug this") but biased-low for
+     * skills whose trigger is a repo STATE ("in a git repo", "dirty tree"); seed that
+     * state here so recall is honest instead of an artifact of the cold start. Mirrors
+     * `MeasureSpec.fixture`. See `research/plugin-behavioral-findings.md`.
+     */
+    readonly fixture?: Record<string, string>;
+    /**
+     * How many runs to execute in parallel across the whole prompts × trials grid.
+     * Default 1 (serial, the politest to rate limits). Raise it to cut wall-clock on
+     * a large prompt set or roster sweep — the `spacingSec` pause still applies per
+     * run, so it stays best-effort polite. Mirrors `EvalSpec.concurrency`.
+     */
+    readonly concurrency?: number;
 }
 /** Per-prompt trigger result: how many of its trials fired. */
 export interface PromptTriggerStat {
@@ -630,7 +667,29 @@ export interface TriggerRateReport {
      * description). A non-zero count is the whole-harness measurement.
      */
     readonly competitors: number;
+    /**
+     * Runs EXCLUDED because the turn errored / was rate-limited (detected by the
+     * driver's `runError`), present only when > 0. These are NOT counted in `n` or
+     * as misses — so `rate` reflects only valid runs. A large `errored` relative to
+     * `n` means the measurement is thin (e.g. a Codex usage limit was hit); re-run.
+     */
+    readonly errored?: number;
 }
+/**
+ * An eval-tier transport: how to RUN a real harness turn and PARSE its output.
+ * The default is Claude Code (`claudeEvalDriver`); a second harness supplies its
+ * own (e.g. `codexEvalDriver` from `vigiles/codex`) and passes it as
+ * `measureTriggerRate(spec, { evalDriver })` — the eval-tier analog of
+ * `runHarnessTest`'s `{ adapter }`. `runError` lets the loop drop an
+ * errored/rate-limited turn instead of scoring it as a miss.
+ */
+export interface EvalDriver {
+    readonly runner: AgentRunner;
+    readonly parse: ModelOutputParser;
+    readonly runError?: (out: RunOut) => string | null;
+}
+/** The default (Claude Code) eval driver: real `claude` + stream-json parsing. */
+export declare const claudeEvalDriver: EvalDriver;
 /**
  * Package loose `<skillsDir>/<name>/SKILL.md` skills into a throwaway plugin dir
  * that `claude --plugin-dir` accepts — so repo-local skills (e.g. `.claude/skills`)
@@ -707,20 +766,17 @@ export declare function packageInstallSet(opts: {
     dir: string;
     added: number;
 };
+export declare function measureTriggerRateWith(spec: TriggerRateSpec, runner: AgentRunner, parse?: ModelOutputParser, runError?: (out: RunOut) => string | null): Promise<TriggerRateReport>;
 /**
- * Trigger-rate orchestration — every prompt × trial via `runner`, the `fired`
- * predicate evaluated per run and aggregated into an overall + per-prompt rate.
- * With `irrelevantPrompts`, also runs the precision side (firing there is a false
- * positive) and adds `falsePositiveRate` + `precision`. Exported with an
- * injectable `runner` so the loop is unit-testable without a model;
- * `measureTriggerRate` is this with the real agent runner.
- */
-export declare function measureTriggerRateWith(spec: TriggerRateSpec, runner: AgentRunner): Promise<TriggerRateReport>;
-/**
- * Measure a skill/behaviour's real trigger rate across prompts × trials against
- * the real `claude` CLI. Requires `claude` + model auth.
+ * Measure a skill/behaviour's real trigger rate across prompts × trials. Defaults
+ * to the real `claude` CLI (`claudeEvalDriver`); pass `{ evalDriver }` to drive a
+ * second harness — e.g. `measureTriggerRate(spec, { evalDriver: codexEvalDriver })`
+ * from `vigiles/codex` (the eval-tier analog of `runHarnessTest`'s `{ adapter }`).
+ * Requires that harness's binary + auth.
  */
-export declare function measureTriggerRate(spec: TriggerRateSpec): Promise<TriggerRateReport>;
+export declare function measureTriggerRate(spec: TriggerRateSpec, opts?: {
+    evalDriver?: EvalDriver;
+}): Promise<TriggerRateReport>;
 /** Format a trigger-rate report: overall %, then each prompt's rate. */
 export declare function formatTriggerRateReport(report: TriggerRateReport): string;
 //# sourceMappingURL=eval.d.ts.map

package/dist/eval.js

@@ -1,7 +1,8 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.EPHEMERAL_HOME_KEEP = void 0;
+exports.claudeEvalDriver = exports.EPHEMERAL_HOME_KEEP = void 0;
 exports.resolveSpawnEnv = resolveSpawnEnv;
+exports.spawnAgent = spawnAgent;
 exports.runEval = runEval;
 exports.measureWith = measureWith;
 exports.measure = measure;
@@ -12,6 +13,7 @@ exports.formatCheckReport = formatCheckReport;
 exports.assertRates = assertRates;
 exports.checkReportToJUnit = checkReportToJUnit;
 exports.parseUsage = parseUsage;
+exports.parseClaudeRun = parseClaudeRun;
 exports.aggregate = aggregate;
 exports.aggregateStats = aggregateStats;
 exports.aggregateUsage = aggregateUsage;
@@ -91,6 +93,8 @@ function resolveSpawnEnv(a, base = process.env) {
     return a.replaceEnv ? (a.env ?? {}) : { ...base, ...a.env };
 }
 /* v8 ignore start -- real claude subprocess; exercised by bench/, not the unit gate */
+/** The real `claude`-spawning runner (composition root). Exported so other
+ *  real-model entries (e.g. `scan --trigger`) bind the same runner. */
 function spawnAgent(a) {
     return new Promise((resolvePromise) => {
         const args = [
@@ -392,21 +396,31 @@ function usageFrom(result) {
 function parseUsage(stdout) {
     return usageFrom((0, harness_test_js_1.parseResultEvent)(stdout));
 }
-function makeContext(cwd, out) {
+/** Parse Claude Code's stream-json stdout into the common trace fields. */
+function parseClaudeRun(out) {
     const result = (0, harness_test_js_1.parseResultEvent)(out.stdout);
-    const turns = typeof result?.num_turns === "number" ? result.num_turns : 0;
-    const output = typeof result?.result === "string" ? result.result : "";
     return {
-        cwd,
-        exitCode: out.code,
-        stdout: out.stdout,
-        turns,
+        turns: typeof result?.num_turns === "number" ? result.num_turns : 0,
+        output: typeof result?.result === "string" ? result.result : "",
         toolCalls: (0, harness_test_js_1.parseToolCalls)(out.stdout),
         hooks: (0, harness_test_js_1.parseHooks)(out.stdout),
-        output,
         subagents: (0, harness_test_js_1.parseSubagents)(out.stdout),
         usage: usageFrom(result),
-        // The eval tier drives the real API (no mock between claude and the model),
+    };
+}
+function makeContext(cwd, out, parse = parseClaudeRun) {
+    const p = parse(out);
+    return {
+        cwd,
+        exitCode: out.code,
+        stdout: out.stdout,
+        turns: p.turns,
+        toolCalls: p.toolCalls,
+        hooks: p.hooks,
+        output: p.output,
+        subagents: p.subagents,
+        usage: p.usage,
+        // The eval tier drives the real API (no mock between the agent and the model),
         // so the requests can't be captured here — modelRequests is harness-tier only.
         modelRequests: [],
         file: (p) => {
@@ -993,6 +1007,11 @@ function formatEvalReport(report) {
     }
     return lines.join("\n");
 }
+/** The default (Claude Code) eval driver: real `claude` + stream-json parsing. */
+exports.claudeEvalDriver = {
+    runner: spawnAgent,
+    parse: parseClaudeRun,
+};
 /**
  * Package loose `<skillsDir>/<name>/SKILL.md` skills into a throwaway plugin dir
  * that `claude --plugin-dir` accepts — so repo-local skills (e.g. `.claude/skills`)
@@ -1284,42 +1303,65 @@ function resolveTriggerPluginDir(spec) {
     };
 }
 /** Run one prompt set × trials through `runner`, aggregating fired counts. */
-async function runTriggerSet(prompts, cfg, runner) {
-    const perPrompt = [];
-    let firedTotal = 0;
-    let n = 0;
-    for (const prompt of prompts) {
-        let fired = 0;
-        for (let t = 0; t < cfg.trials; t++) {
-            const cwd = (0, node_fs_1.mkdtempSync)((0, node_path_1.join)((0, node_os_1.tmpdir)(), "vigiles-trigger-"));
-            try {
-                const out = await runner({
-                    task: prompt,
-                    cwd,
-                    model: cfg.model,
-                    tools: cfg.tools,
-                    hasSettings: false,
-                    pluginDir: cfg.pluginDir,
-                    timeoutMs: cfg.timeoutMs,
-                });
-                if (cfg.fired(makeContext(cwd, out)))
-                    fired++;
-            }
-            finally {
-                (0, node_fs_1.rmSync)(cwd, { recursive: true, force: true });
-                await sleep(cfg.spacing);
-            }
-        }
-        perPrompt.push({
-            prompt,
-            fired,
-            trials: cfg.trials,
-            rate: cfg.trials > 0 ? fired / cfg.trials : 0,
+/** Run one trigger trial in a throwaway cwd (fixture seeded) → fired 0/1. */
+async function runTriggerTrial(prompt, cfg, runner) {
+    const cwd = (0, node_fs_1.mkdtempSync)((0, node_path_1.join)((0, node_os_1.tmpdir)(), "vigiles-trigger-"));
+    try {
+        if (cfg.fixture)
+            writeFiles(cwd, cfg.fixture);
+        const out = await runner({
+            task: prompt,
+            cwd,
+            model: cfg.model,
+            tools: cfg.tools,
+            hasSettings: false,
+            pluginDir: cfg.pluginDir,
+            timeoutMs: cfg.timeoutMs,
         });
-        firedTotal += fired;
-        n += cfg.trials;
+        // An errored/rate-limited turn is NOT a "skill didn't fire" miss — it's
+        // excluded from the rate, so e.g. a Codex usage limit can't read as recall 0.
+        if (cfg.runError?.(out))
+            return { fired: 0, errored: true };
+        return {
+            fired: cfg.fired(makeContext(cwd, out, cfg.parse)) ? 1 : 0,
+            errored: false,
+        };
+    }
+    finally {
+        (0, node_fs_1.rmSync)(cwd, { recursive: true, force: true });
+        await sleep(cfg.spacing);
     }
-    return { perPrompt, fired: firedTotal, n };
+}
+/** A count for reporting: the number if positive, else undefined (omit zero). */
+const positiveOrUndefined = (n) => n > 0 ? n : undefined;
+async function runTriggerSet(prompts, cfg, runner) {
+    // Flatten prompts × trials into one work list so concurrency spans both.
+    const jobs = prompts.flatMap((prompt, promptIndex) => Array.from({ length: cfg.trials }, () => ({ prompt, promptIndex })));
+    const outcomes = await runPool(jobs, cfg.concurrency, (job) => runTriggerTrial(job.prompt, cfg, runner));
+    // Re-aggregate per prompt, preserving input order; errored runs don't count.
+    const firedBy = new Array(prompts.length).fill(0);
+    const trialsBy = new Array(prompts.length).fill(0);
+    let errored = 0;
+    jobs.forEach((job, i) => {
+        if (outcomes[i].errored) {
+            errored += 1;
+            return;
+        }
+        firedBy[job.promptIndex] += outcomes[i].fired;
+        trialsBy[job.promptIndex] += 1;
+    });
+    const perPrompt = prompts.map((prompt, i) => ({
+        prompt,
+        fired: firedBy[i],
+        trials: trialsBy[i],
+        rate: trialsBy[i] > 0 ? firedBy[i] / trialsBy[i] : 0,
+    }));
+    return {
+        perPrompt,
+        fired: firedBy.reduce((a, b) => a + b, 0),
+        n: trialsBy.reduce((a, b) => a + b, 0),
+        errored,
+    };
 }
 /**
  * Trigger-rate orchestration — every prompt × trial via `runner`, the `fired`
@@ -1329,9 +1371,8 @@ async function runTriggerSet(prompts, cfg, runner) {
  * injectable `runner` so the loop is unit-testable without a model;
  * `measureTriggerRate` is this with the real agent runner.
  */
-async function measureTriggerRateWith(spec, runner) {
-    // Deterministic gate FIRST — reject a too-small / near-duplicate prompt set
-    // before spending a token (and before packaging a skillsDir).
+/** Diversity pre-flight: reject a too-small / near-duplicate prompt set (both sides). */
+function assertTriggerDiversity(spec) {
     const diversity = {
         minPrompts: spec.minPrompts,
         minDistance: spec.minDistance,
@@ -1343,6 +1384,10 @@ async function measureTriggerRateWith(spec, runner) {
             label: "irrelevantPrompts",
         });
     }
+}
+async function measureTriggerRateWith(spec, runner, parse = parseClaudeRun, runError) {
+    // Deterministic gate FIRST — before spending a token (or packaging a skillsDir).
+    assertTriggerDiversity(spec);
     // Model floor (default Sonnet): trigger-rate under-measures selection on a
     // weaker model, so FAIL before spending a token rather than report a
     // false-negative recall. The floor lives in the spec (`minModel`), not an env
@@ -1365,6 +1410,10 @@ async function measureTriggerRateWith(spec, runner) {
         spacing: (spec.spacingSec ?? 4) * 1000,
         pluginDir,
         fired: spec.fired,
+        fixture: spec.fixture,
+        concurrency: Math.max(1, spec.concurrency ?? 1),
+        parse,
+        runError,
     };
     try {
         const relevant = await runTriggerSet(spec.prompts, cfg, runner);
@@ -1373,6 +1422,7 @@ async function measureTriggerRateWith(spec, runner) {
             n: relevant.n,
             perPrompt: relevant.perPrompt,
             competitors,
+            errored: positiveOrUndefined(relevant.errored),
         };
         if ((spec.irrelevantPrompts?.length ?? 0) === 0)
             return base;
@@ -1380,6 +1430,7 @@ async function measureTriggerRateWith(spec, runner) {
         const fires = relevant.fired + irrelevant.fired;
         return {
             ...base,
+            errored: positiveOrUndefined(relevant.errored + irrelevant.errored),
             falsePositiveRate: irrelevant.n > 0 ? irrelevant.fired / irrelevant.n : 0,
             precision: fires > 0 ? relevant.fired / fires : undefined,
             perIrrelevant: irrelevant.perPrompt,
@@ -1393,11 +1444,15 @@ async function measureTriggerRateWith(spec, runner) {
 }
 /* v8 ignore start -- real claude subprocess; thin wrapper over measureTriggerRateWith */
 /**
- * Measure a skill/behaviour's real trigger rate across prompts × trials against
- * the real `claude` CLI. Requires `claude` + model auth.
+ * Measure a skill/behaviour's real trigger rate across prompts × trials. Defaults
+ * to the real `claude` CLI (`claudeEvalDriver`); pass `{ evalDriver }` to drive a
+ * second harness — e.g. `measureTriggerRate(spec, { evalDriver: codexEvalDriver })`
+ * from `vigiles/codex` (the eval-tier analog of `runHarnessTest`'s `{ adapter }`).
+ * Requires that harness's binary + auth.
  */
-async function measureTriggerRate(spec) {
-    return measureTriggerRateWith(spec, spawnAgent);
+async function measureTriggerRate(spec, opts = {}) {
+    const d = opts.evalDriver ?? exports.claudeEvalDriver;
+    return measureTriggerRateWith(spec, d.runner, d.parse, d.runError);
 }
 /* v8 ignore stop */
 /** Format a trigger-rate report: overall %, then each prompt's rate. */

package/dist/harness-test.d.ts

@@ -110,6 +110,13 @@ export interface SubagentTrace {
     readonly name: string;
     /** The tools the subagent invoked (events tagged with the Task's id). */
     readonly toolCalls: readonly ToolCall[];
+    /**
+     * The subagent's RETURNED text — the dispatch tool_result the orchestrator
+     * receives back. This is where a `result()` contract's `vigiles:ok`/`vigiles:err`
+     * block lands, so `subagent(name, [output(/vigiles:ok/)])` can assert the typed
+     * outcome. "" if not captured.
+     */
+    readonly output: string;
 }
 export interface HarnessTestResult extends Trace {
     readonly exitCode: number;

package/dist/harness-test.js

@@ -130,6 +130,7 @@ function parseToolCalls(streamJson) {
  */
 function parseSubagents(streamJson) {
     const tasks = new Map(); // dispatch id → subagent name
+    const dispatchOutput = new Map(); // dispatch id → returned text
     const byParent = new Map();
     const groupFor = (parent) => {
         let g = byParent.get(parent);
@@ -162,7 +163,10 @@ function parseSubagents(streamJson) {
                     // A subagent dispatch is any top-level tool_use whose input carries a
                     // `subagent_type` — the dispatch tool is named "Agent" on the live CLI
                     // (older docs say "Task"), so match the input field, NOT the tool name,
-                    // to survive the rename. Confirmed against real claude output.
+                    // to survive the rename. Confirmed against real claude output. CC NOTE:
+                    // under `--plugin-dir` the value is NAMESPACED `plugin:agent` (captured
+                    // "reviewer-spec:code-reviewer"); the bare agent name is matched in the
+                    // `subagent()` check (src/check.ts), so the full id is preserved here.
                     const sub = b.input?.subagent_type;
                     if (typeof sub === "string")
                         tasks.set(id, sub);
@@ -170,12 +174,20 @@ function parseSubagents(streamJson) {
                 if (parent)
                     groupFor(parent).uses.push({ id, name: b.name, input: b.input });
             }
-            else if (b.type === "tool_result" && parent) {
+            else if (b.type === "tool_result") {
                 const id = typeof b.tool_use_id === "string" ? b.tool_use_id : "";
-                groupFor(parent).results.set(id, {
-                    text: contentText(b.content),
-                    isError: b.is_error === true,
-                });
+                if (parent) {
+                    groupFor(parent).results.set(id, {
+                        text: contentText(b.content),
+                        isError: b.is_error === true,
+                    });
+                }
+                else if (id) {
+                    // A top-level tool_result whose id is a subagent dispatch is the SUB's
+                    // RETURN to the orchestrator (where a result() vigiles:ok/err block
+                    // lands). Record it; matched to its dispatch by id below.
+                    dispatchOutput.set(id, contentText(b.content));
+                }
             }
         }
     }
@@ -188,7 +200,7 @@ function parseSubagents(streamJson) {
             resultText: g?.results.get(u.id)?.text ?? "",
             isError: g?.results.get(u.id)?.isError ?? false,
         }));
-        out.push({ name, toolCalls });
+        out.push({ name, toolCalls, output: dispatchOutput.get(taskId) ?? "" });
     }
     return out;
 }

package/dist/leaderboard.d.ts

@@ -21,6 +21,8 @@ export interface PluginScore {
     readonly issues: readonly string[];
     readonly report: ScanReport;
 }
+/** Map a 0–100 structural-health score to its letter grade (A ≥90 … F <60). */
+export declare function gradeFor(score: number): PluginScore["grade"];
 /** Deterministic structural-health score for one scanned plugin. */
 export declare function scoreReport(r: ScanReport): {
     score: number;