vigiles 5.0.0 → 5.1.0

package/dist/core/tool-contract.d.ts

@@ -0,0 +1,68 @@
+/**
+ * Tool-contract verification — the cross-referencing moat ("valid is not true")
+ * applied to a subagent's declared `tools:` rail. A subagent may only run
+ * built-in tools from the harness dialect's catalog or an MCP tool; anything else
+ * is a typo or a nonexistent / never-available tool — a guaranteed-dead reference
+ * a compiler catches, not a runtime surprise.
+ *
+ * ONE pure detector (`one-detector-no-drift`), reused by THREE callers so they
+ * can't disagree: `compileAgent` (spec authoring), `scan` (read-only audit of a
+ * shipped plugin), and the `agent-tool-contract` lint rule (the severity-gated
+ * commit gate). The dialect is injected (core ⊄ adapter) — the composition root
+ * passes `claudeCodeDialect` / `codexDialect`.
+ *
+ * Scope note: this validates a SUBAGENT contract against the SUBAGENT catalog
+ * (`builtinAgentTools` / `neverAvailableTools`). A skill's `allowed-tools` is a
+ * DIFFERENT namespace (skills legitimately use `AskUserQuestion`, `TaskCreate`,
+ * … which are never-available to a subagent), so it is deliberately NOT validated
+ * here — doing so against the agent catalog would be a false-positive factory.
+ */
+import type { HarnessDialect } from "./dialect.js";
+export type ToolIssueKind = "never-available" | "unknown";
+export interface ToolIssue {
+    readonly tool: string;
+    readonly kind: ToolIssueKind;
+    /** Closest known built-in tool (did-you-mean), or null. */
+    readonly suggestion: string | null;
+    /** A ready-to-show, actionable message. */
+    readonly message: string;
+}
+/**
+ * Closest known built-in tool by edit distance (≤ 2), for a "did you mean" hint.
+ * The ≤ 2 bound is deliberately tight: a suggestion is a CONFIDENCE signal (this
+ * `unknown` is really a typo of a real tool), and a loose bound mis-suggests —
+ * `TaskGet → Task?` (distance 3) is a real tool set, not a typo of `Task`.
+ */
+export declare function closestTool(tool: string, dialect: HarnessDialect): string | null;
+/**
+ * The HIGH-CONFIDENCE subset of a contract's issues — the ones safe to flag when
+ * AUDITING a third-party plugin (scan / lint), where the catalog can't know
+ * every tool (plugin-/MCP-provided, newer platform tools). Only two are confident:
+ * a `never-available` tool (a curated denylist) and an `unknown` with a close
+ * typo suggestion (`Edt → Edit`). A bare `unknown` with no near match is NOT
+ * flagged here — it is more likely a tool vigiles doesn't know than a defect
+ * (sweeping real plugins surfaced a 280★ plugin using `TaskCreate/TaskGet/…`
+ * consistently; flagging those would be crying wolf). `compileAgent` stays strict
+ * — when you author your OWN spec, every unrecognized tool is worth an error.
+ */
+export declare function confidentToolIssues(issues: readonly ToolIssue[]): ToolIssue[];
+/**
+ * Verify a subagent's `disallowedTools:` BLOCK-list — the mirror of the allow
+ * contract. A typo here is dangerous: you meant to block `Bash` but wrote `Bsh`,
+ * so nothing is blocked and the dangerous tool stays available, silently. Returns
+ * one {@link ToolIssue} per entry that's a CLOSE TYPO of a real built-in (the
+ * high-confidence signal). Deliberately NOT flagged: a real built-in (it IS being
+ * blocked — correct), a never-available tool (harmless to block), an MCP tool (a
+ * legitimate plugin tool to block), or a bare unknown with no near match (likely
+ * a plugin/MCP tool, not a typo — the cry-wolf trap). The block-list inverts the
+ * allow check: never-available is fine to list, a typo is the actual defect.
+ */
+export declare function disallowedToolIssues(tools: readonly string[], dialect: HarnessDialect): ToolIssue[];
+/**
+ * Verify a subagent's `tools:` contract against the dialect catalog. Returns one
+ * {@link ToolIssue} per offending entry (empty when every tool is a real built-in
+ * or a well-formed MCP tool). A `Tool(restriction)` suffix (e.g. `Bash(git:*)`)
+ * is stripped to its base tool before checking.
+ */
+export declare function verifyToolContract(tools: readonly string[], dialect: HarnessDialect): ToolIssue[];
+//# sourceMappingURL=tool-contract.d.ts.map

package/dist/core/tool-contract.js

@@ -0,0 +1,113 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.closestTool = closestTool;
+exports.confidentToolIssues = confidentToolIssues;
+exports.disallowedToolIssues = disallowedToolIssues;
+exports.verifyToolContract = verifyToolContract;
+const linters_js_1 = require("./linters.js");
+/**
+ * Closest known built-in tool by edit distance (≤ 2), for a "did you mean" hint.
+ * The ≤ 2 bound is deliberately tight: a suggestion is a CONFIDENCE signal (this
+ * `unknown` is really a typo of a real tool), and a loose bound mis-suggests —
+ * `TaskGet → Task?` (distance 3) is a real tool set, not a typo of `Task`.
+ */
+function closestTool(tool, dialect) {
+    let best = null;
+    let bestDistance = Infinity;
+    for (const known of dialect.builtinAgentTools) {
+        const d = (0, linters_js_1.editDistance)(tool.toLowerCase(), known.toLowerCase());
+        if (d < bestDistance) {
+            bestDistance = d;
+            best = known;
+        }
+    }
+    return bestDistance <= 2 ? best : null;
+}
+/**
+ * The HIGH-CONFIDENCE subset of a contract's issues — the ones safe to flag when
+ * AUDITING a third-party plugin (scan / lint), where the catalog can't know
+ * every tool (plugin-/MCP-provided, newer platform tools). Only two are confident:
+ * a `never-available` tool (a curated denylist) and an `unknown` with a close
+ * typo suggestion (`Edt → Edit`). A bare `unknown` with no near match is NOT
+ * flagged here — it is more likely a tool vigiles doesn't know than a defect
+ * (sweeping real plugins surfaced a 280★ plugin using `TaskCreate/TaskGet/…`
+ * consistently; flagging those would be crying wolf). `compileAgent` stays strict
+ * — when you author your OWN spec, every unrecognized tool is worth an error.
+ */
+function confidentToolIssues(issues) {
+    return issues.filter((i) => i.kind === "never-available" || i.suggestion !== null);
+}
+/**
+ * Verify a subagent's `disallowedTools:` BLOCK-list — the mirror of the allow
+ * contract. A typo here is dangerous: you meant to block `Bash` but wrote `Bsh`,
+ * so nothing is blocked and the dangerous tool stays available, silently. Returns
+ * one {@link ToolIssue} per entry that's a CLOSE TYPO of a real built-in (the
+ * high-confidence signal). Deliberately NOT flagged: a real built-in (it IS being
+ * blocked — correct), a never-available tool (harmless to block), an MCP tool (a
+ * legitimate plugin tool to block), or a bare unknown with no near match (likely
+ * a plugin/MCP tool, not a typo — the cry-wolf trap). The block-list inverts the
+ * allow check: never-available is fine to list, a typo is the actual defect.
+ */
+function disallowedToolIssues(tools, dialect) {
+    const never = new Set(dialect.neverAvailableTools);
+    const issues = [];
+    for (const raw of tools) {
+        const tool = raw.split("(")[0].trim();
+        if (tool === "" || tool === "*")
+            continue;
+        if (dialect.builtinAgentTools.includes(tool))
+            continue; // legitimately blocked
+        if (never.has(tool))
+            continue; // harmless to list (already unavailable)
+        if (dialect.mcpToolPattern.test(tool))
+            continue; // a real plugin/MCP tool to block
+        const near = closestTool(tool, dialect);
+        if (near === null)
+            continue; // bare unknown → likely a plugin tool, not a typo
+        issues.push({
+            tool,
+            kind: "unknown",
+            suggestion: near,
+            message: `disallowedTools entry "${tool}" matches no real tool — it blocks nothing. Did you mean "${near}"?`,
+        });
+    }
+    return issues;
+}
+/**
+ * Verify a subagent's `tools:` contract against the dialect catalog. Returns one
+ * {@link ToolIssue} per offending entry (empty when every tool is a real built-in
+ * or a well-formed MCP tool). A `Tool(restriction)` suffix (e.g. `Bash(git:*)`)
+ * is stripped to its base tool before checking.
+ */
+function verifyToolContract(tools, dialect) {
+    const never = new Set(dialect.neverAvailableTools);
+    const issues = [];
+    for (const raw of tools) {
+        const tool = raw.split("(")[0].trim(); // strip a Tool(restriction) suffix
+        if (tool === "" || tool === "*")
+            continue; // "" / "*" = wildcard, inherits all
+        if (never.has(tool)) {
+            issues.push({
+                tool,
+                kind: "never-available",
+                suggestion: null,
+                message: `Tool "${tool}" is never available to a subagent — remove it from the tools list.`,
+            });
+            continue;
+        }
+        if (dialect.builtinAgentTools.includes(tool))
+            continue;
+        if (dialect.mcpToolPattern.test(tool))
+            continue;
+        const near = closestTool(tool, dialect);
+        const hint = near ? ` Did you mean "${near}"?` : "";
+        issues.push({
+            tool,
+            kind: "unknown",
+            suggestion: near,
+            message: `Unknown tool "${tool}" — use a built-in tool (${dialect.builtinAgentTools.join(", ")}) or an MCP tool (mcp__server__tool).${hint}`,
+        });
+    }
+    return issues;
+}
+//# sourceMappingURL=tool-contract.js.map

package/dist/core/types.d.ts

@@ -108,6 +108,95 @@ export interface RulesConfig {
      * block the edit, false → off.
      */
     "unmarked-refs"?: RuleSeverity;
+    /**
+     * Cross-reference each subagent's `tools:` rail against the harness tool
+     * catalog — flag a never-available tool or a close typo (the moat). Only
+     * high-confidence issues are reported (a bare unrecognized tool is likely
+     * plugin/MCP-provided, never flagged). Off unless set; "warn" surfaces,
+     * "error" gates CI. Same detector as `scan` + `compileAgent`.
+     */
+    "agent-tool-contract"?: RuleSeverity;
+    /**
+     * Flag a hook registered under an event name the harness doesn't define (a
+     * typo → the hook never fires). High-precision: close typos only, never a
+     * framework/custom event. Default "warn"; "error" gates CI. Same detector as
+     * `scan`.
+     */
+    "hook-events"?: RuleSeverity;
+    /**
+     * Flag a skill/agent missing a required frontmatter field — a skill needs
+     * `name` (to load), an agent needs `name` + `description`. A broken surface
+     * that won't register. Default "warn"; "error" gates CI. Same detector as `scan`.
+     */
+    "agent-frontmatter"?: RuleSeverity;
+    /**
+     * Flag a declared MCP server that can't start — neither a `command` (stdio)
+     * nor a `url` (http/sse). Default "warn"; "error" gates CI. Same detector as
+     * `scan`. (JSON `.mcp.json`/manifest `mcpServers`; Codex TOML not yet parsed.)
+     */
+    "mcp-config"?: RuleSeverity;
+    /**
+     * RECOMMEND (not require) that a SKILL.md declares an explicit `name` +
+     * `description` rather than relying on the dir-name / first-paragraph
+     * fallbacks — a more reliable trigger surface. The skill still loads without
+     * them, so this is a best-practice nudge: default "warn"; set "error" to
+     * enforce on your own skills. Same detector as `scan` (skillMetaIssues).
+     */
+    "skill-frontmatter"?: RuleSeverity;
+    /**
+     * Cross-reference an `mcp__server__tool` in a subagent's contract against the
+     * plugin's declared `mcpServers` — flag a server the plugin doesn't declare
+     * (the MCP half of the tool moat; `agent-tool-contract` checks the built-in
+     * half). High-precision: only flags when the plugin SHIPS a declared set,
+     * allowlists harness built-ins (`ide`), and skips the plugin-namespaced
+     * `mcp__plugin_…` form. Default "warn"; "error" gates CI. Same detector as
+     * `scan` (mcpToolIssues).
+     */
+    "mcp-tool-resolves"?: RuleSeverity;
+    /**
+     * Flag a hook command that references a script file which doesn't exist on
+     * disk (with `${CLAUDE_PLUGIN_ROOT}` resolved) — the hook silently never runs.
+     * FP-safe: skips unresolved `$VAR` paths, existence-guarded one-liners, and
+     * inline commands. Matches Anthropic's own `claude plugin validate`. Default
+     * "warn"; "error" gates CI. Same detector as `scan` (hooks status "missing").
+     */
+    "hook-script-exists"?: RuleSeverity;
+    /**
+     * Cross-reference a subagent's `disallowedTools:` block-list against the
+     * catalog — the deny-side mirror of `agent-tool-contract`. A close typo there
+     * blocks NOTHING (you meant to deny `Bash`, wrote `Bsh`), leaving the tool
+     * available. High-precision: close-typo only (a never-available tool is
+     * harmless to list, a bare unknown is likely a plugin tool). Default "warn";
+     * "error" gates CI. Same detector as `scan` (disallowedToolIssues).
+     */
+    "disallowed-tools-contract"?: RuleSeverity;
+    /**
+     * Flag two model-invocable skills whose descriptions are near-identical — the
+     * selector can't tell them apart, so the wrong one fires (a precision
+     * collision). A DETERMINISTIC NCD proxy for a `--trigger`-class behavioral bug;
+     * calibrated FP-safe (only basically-identical text, below the sweep's
+     * most-similar distinct pair). Default "warn"; "error" gates CI. Same detector
+     * as `scan` (descriptionOverlaps).
+     */
+    "description-overlap"?: RuleSeverity;
+    /**
+     * Flag a skill/agent whose `---` frontmatter block EXISTS but isn't valid YAML
+     * — fields may not parse as intended. CAVEAT: a real YAML parser (js-yaml) is
+     * stricter than some loaders, so a one-line `description:` containing a `: `
+     * colon or an `<example>` block is flagged even though it may still load.
+     * Hence default "warn" (a nudge), not "error" — verify before enforcing. Same
+     * detector as `scan` (malformedFrontmatter).
+     */
+    "frontmatter-valid"?: RuleSeverity;
+    /**
+     * Flag a `type: "mcp_tool"` hook action that's incomplete (missing `server` /
+     * `tool`) or targets a server the plugin doesn't declare in `mcpServers` — the
+     * hook silently never dispatches. High-precision: the undeclared-server half is
+     * gated on the plugin shipping a declared set and allowlists built-ins (`ide`),
+     * mirroring `mcp-tool-resolves`. Default "warn"; "error" gates CI. Same detector
+     * as `scan` (mcpHookIssues).
+     */
+    "mcp-hook-target-resolves"?: RuleSeverity;
 }
 /** Extract severity from a rule value (handles both simple and tuple forms). */
 export declare function ruleSeverity<T>(rule: RuleWithOptions<T> | undefined): RuleSeverity;

package/dist/core/validate.js

@@ -46,6 +46,28 @@ const DEFAULT_RULES = {
     "untested-agent": "warn",
     "untested-hook": "warn",
     "unmarked-refs": "warn",
+    // High-precision (never-available + close typos only), so on by default at warn.
+    "agent-tool-contract": "warn",
+    // High-precision (close typos only), on by default at warn.
+    "hook-events": "warn",
+    // Missing required frontmatter (name/description) — on by default at warn.
+    "agent-frontmatter": "warn",
+    // A declared MCP server with no command/url can't start — on by default at warn.
+    "mcp-config": "warn",
+    // Best-practice nudge (skills load without frontmatter) — warn, not error.
+    "skill-frontmatter": "warn",
+    // High-precision (gated on a declared MCP set; built-ins allowlisted) — warn.
+    "mcp-tool-resolves": "warn",
+    // A hook script referenced but missing never runs — on by default at warn.
+    "hook-script-exists": "warn",
+    // High-precision (close-typo only) deny-list mirror of agent-tool-contract.
+    "disallowed-tools-contract": "warn",
+    // Deterministic NCD precision proxy (near-identical skill descriptions) — warn.
+    "description-overlap": "warn",
+    // Malformed-YAML frontmatter — WARN only (js-yaml is stricter than some loaders).
+    "frontmatter-valid": "warn",
+    // A mcp_tool hook incomplete / targeting an undeclared server — on by default at warn.
+    "mcp-hook-target-resolves": "warn",
 };
 const DEFAULT_CONFIG = {
     ruleMarkers: ["headings", "checkboxes"],

package/dist/eval.d.ts

@@ -1,4 +1,4 @@
-import { type ToolCall, type Trace } from "./harness-test.js";
+import { parseToolCalls, parseHooks, parseSubagents, type ToolCall, type Trace } from "./harness-test.js";
 import { type CacheMode } from "./eval-cache.js";
 import type { Check, CheckJSON } from "./check.js";
 import { type Comparison } from "./stats.js";
@@ -250,6 +250,9 @@ export type AgentRunner = (args: AgentRunArgs) => Promise<RunOut>;
  * leak the host environment into an untrusted, model-driven run.
  */
 export declare function resolveSpawnEnv(a: Pick<AgentRunArgs, "env" | "replaceEnv">, base?: NodeJS.ProcessEnv): NodeJS.ProcessEnv;
+/** The real `claude`-spawning runner (composition root). Exported so other
+ *  real-model entries (e.g. `scan --trigger`) bind the same runner. */
+export declare function spawnAgent(a: AgentRunArgs): Promise<RunOut>;
 /**
  * Run the eval: every arm × every trial against the real `claude` CLI, with the
  * metric computed per run and aggregated per arm. Requires `claude` on PATH and
@@ -392,6 +395,24 @@ export declare function checkReportToJUnit(report: CheckReport, opts?: {
 }): string;
 /** Parse per-run cost/latency/tokens from a stream — pure, model-free. */
 export declare function parseUsage(stdout: string): EvalUsage;
+/**
+ * The harness-specific half of a run trace: how a real model's raw stdout maps
+ * to the common fields. Claude Code's `parseClaudeRun` reads its stream-json; a
+ * second harness (Codex) supplies its own parser of `codex exec --json` JSONL, so
+ * the eval tier (`measureTriggerRate`/`runEval`) isn't bound to Claude's format.
+ * The non-harness fields (cwd/exitCode/stdout/file/sh) stay in `makeContext`.
+ */
+export interface ParsedModelRun {
+    readonly turns: number;
+    readonly output: string;
+    readonly toolCalls: ReturnType<typeof parseToolCalls>;
+    readonly hooks: ReturnType<typeof parseHooks>;
+    readonly subagents: ReturnType<typeof parseSubagents>;
+    readonly usage: EvalUsage;
+}
+export type ModelOutputParser = (out: RunOut) => ParsedModelRun;
+/** Parse Claude Code's stream-json stdout into the common trace fields. */
+export declare function parseClaudeRun(out: RunOut): ParsedModelRun;
 /** Aggregate per-run metrics: mean for numbers, fraction-true (0..1) for booleans. */
 export declare function aggregate(rows: readonly Metrics[]): Record<string, number>;
 /**
@@ -593,6 +614,22 @@ export interface TriggerRateSpec {
     readonly timeoutMs?: number;
     /** Seconds to wait between runs (avoid rate-limit bursts). Default 4. */
     readonly spacingSec?: number;
+    /**
+     * Files (path → contents) seeded into every run's cwd before the prompt — the
+     * filesystem CONTEXT the skill is measured in. The default empty cwd is faithful
+     * for opening-move skills ("describe a feature", "debug this") but biased-low for
+     * skills whose trigger is a repo STATE ("in a git repo", "dirty tree"); seed that
+     * state here so recall is honest instead of an artifact of the cold start. Mirrors
+     * `MeasureSpec.fixture`. See `research/plugin-behavioral-findings.md`.
+     */
+    readonly fixture?: Record<string, string>;
+    /**
+     * How many runs to execute in parallel across the whole prompts × trials grid.
+     * Default 1 (serial, the politest to rate limits). Raise it to cut wall-clock on
+     * a large prompt set or roster sweep — the `spacingSec` pause still applies per
+     * run, so it stays best-effort polite. Mirrors `EvalSpec.concurrency`.
+     */
+    readonly concurrency?: number;
 }
 /** Per-prompt trigger result: how many of its trials fired. */
 export interface PromptTriggerStat {
@@ -630,7 +667,29 @@ export interface TriggerRateReport {
      * description). A non-zero count is the whole-harness measurement.
      */
     readonly competitors: number;
+    /**
+     * Runs EXCLUDED because the turn errored / was rate-limited (detected by the
+     * driver's `runError`), present only when > 0. These are NOT counted in `n` or
+     * as misses — so `rate` reflects only valid runs. A large `errored` relative to
+     * `n` means the measurement is thin (e.g. a Codex usage limit was hit); re-run.
+     */
+    readonly errored?: number;
 }
+/**
+ * An eval-tier transport: how to RUN a real harness turn and PARSE its output.
+ * The default is Claude Code (`claudeEvalDriver`); a second harness supplies its
+ * own (e.g. `codexEvalDriver` from `vigiles/codex`) and passes it as
+ * `measureTriggerRate(spec, { evalDriver })` — the eval-tier analog of
+ * `runHarnessTest`'s `{ adapter }`. `runError` lets the loop drop an
+ * errored/rate-limited turn instead of scoring it as a miss.
+ */
+export interface EvalDriver {
+    readonly runner: AgentRunner;
+    readonly parse: ModelOutputParser;
+    readonly runError?: (out: RunOut) => string | null;
+}
+/** The default (Claude Code) eval driver: real `claude` + stream-json parsing. */
+export declare const claudeEvalDriver: EvalDriver;
 /**
  * Package loose `<skillsDir>/<name>/SKILL.md` skills into a throwaway plugin dir
  * that `claude --plugin-dir` accepts — so repo-local skills (e.g. `.claude/skills`)
@@ -707,20 +766,17 @@ export declare function packageInstallSet(opts: {
     dir: string;
     added: number;
 };
+export declare function measureTriggerRateWith(spec: TriggerRateSpec, runner: AgentRunner, parse?: ModelOutputParser, runError?: (out: RunOut) => string | null): Promise<TriggerRateReport>;
 /**
- * Trigger-rate orchestration — every prompt × trial via `runner`, the `fired`
- * predicate evaluated per run and aggregated into an overall + per-prompt rate.
- * With `irrelevantPrompts`, also runs the precision side (firing there is a false
- * positive) and adds `falsePositiveRate` + `precision`. Exported with an
- * injectable `runner` so the loop is unit-testable without a model;
- * `measureTriggerRate` is this with the real agent runner.
- */
-export declare function measureTriggerRateWith(spec: TriggerRateSpec, runner: AgentRunner): Promise<TriggerRateReport>;
-/**
- * Measure a skill/behaviour's real trigger rate across prompts × trials against
- * the real `claude` CLI. Requires `claude` + model auth.
+ * Measure a skill/behaviour's real trigger rate across prompts × trials. Defaults
+ * to the real `claude` CLI (`claudeEvalDriver`); pass `{ evalDriver }` to drive a
+ * second harness — e.g. `measureTriggerRate(spec, { evalDriver: codexEvalDriver })`
+ * from `vigiles/codex` (the eval-tier analog of `runHarnessTest`'s `{ adapter }`).
+ * Requires that harness's binary + auth.
  */
-export declare function measureTriggerRate(spec: TriggerRateSpec): Promise<TriggerRateReport>;
+export declare function measureTriggerRate(spec: TriggerRateSpec, opts?: {
+    evalDriver?: EvalDriver;
+}): Promise<TriggerRateReport>;
 /** Format a trigger-rate report: overall %, then each prompt's rate. */
 export declare function formatTriggerRateReport(report: TriggerRateReport): string;
 //# sourceMappingURL=eval.d.ts.map