vigiles 5.0.1 → 5.2.0

package/dist/scan-behavioral.d.ts

@@ -0,0 +1,73 @@
+/**
+ * `vigiles scan --trigger` — the BEHAVIORAL column of the scan report.
+ *
+ * Structural `scan`/`scanPlugin` is deterministic, no-model, CI-free — and stays
+ * that way. This is the opt-in, model-gated column that stacks on top: for each
+ * model-invocable skill in a plugin, it measures how reliably the description
+ * actually FIRES (recall, + precision when irrelevant prompts are supplied),
+ * reusing `measureTriggerRate`. It degrades honestly when the `claude` CLI / auth
+ * is absent rather than faking a pass — exactly like the egress column.
+ *
+ * Prompts are AUTHOR-SUPPLIED (a per-skill JSON map), not model-generated — a
+ * path in prose is undecidable, and the deterministic-input discipline is what
+ * makes the column trustworthy. See `research/plugin-behavioral-findings.md`.
+ */
+import { type EvalDriver } from "./eval.js";
+import { type Trace } from "./harness-test.js";
+/** Which harness drives the behavioral column (default Claude Code). */
+export type ProbeHarness = "claude-code" | "codex";
+/** Author-supplied prompt sets for one skill (bare skill name → these). */
+export interface SkillPrompts {
+    readonly prompts: readonly string[];
+    readonly irrelevant?: readonly string[];
+}
+/** The `--prompts <file>` shape: bare skill name → its prompt sets. */
+export type TriggerPromptSet = Record<string, SkillPrompts>;
+export interface SkillTriggerResult {
+    readonly skill: string;
+    /** Whether a model probe actually ran (false = skipped, see `note`). */
+    readonly measured: boolean;
+    readonly recall?: number;
+    readonly precision?: number;
+    readonly falsePositiveRate?: number;
+    readonly n?: number;
+    /** Why it was skipped, or a measurement error. */
+    readonly note?: string;
+}
+export interface BehavioralReport {
+    /** False when the `claude` CLI / auth is absent — the column couldn't run. */
+    readonly available: boolean;
+    readonly results: readonly SkillTriggerResult[];
+}
+export interface ProbeOptions {
+    readonly concurrency?: number;
+    readonly model?: string;
+    readonly minPrompts?: number;
+    readonly minDistance?: number;
+    /** Which harness to drive (default `"claude-code"`). */
+    readonly harness?: ProbeHarness;
+}
+/**
+ * Per-harness probe wiring: the eval driver (runner+parse), how to build the
+ * `fired` predicate for a skill, whether to stub bodies, and an availability
+ * gate. Claude detects firing via the `Skill` tool_use (namespaced by the
+ * plugin name); Codex has no skill event, so firing is the SKILL.md read
+ * (`codexSkillFired`, bare name) — see `research/codex-prototype-findings.md`.
+ */
+export interface HarnessProbe {
+    readonly evalDriver: EvalDriver;
+    readonly firedFor: (name: string) => (t: Trace) => boolean;
+    readonly stub: boolean;
+    readonly available: () => boolean;
+}
+/** The injectable core (for tests): probe every model-invocable skill that has prompts. */
+export declare function probePluginTriggersWith(dir: string, promptSet: TriggerPromptSet, probe: HarnessProbe, opts?: ProbeOptions): Promise<BehavioralReport>;
+/**
+ * Probe a plugin's skills against the real harness (default Claude Code; Codex via
+ * `opts.harness`). Needs that harness's binary + auth; degrades to
+ * `available: false` otherwise.
+ */
+export declare function probePluginTriggers(dir: string, promptSet: TriggerPromptSet, opts?: ProbeOptions): Promise<BehavioralReport>;
+/** Format the behavioral column as a scan-report section. */
+export declare function formatBehavioralReport(b: BehavioralReport): string;
+//# sourceMappingURL=scan-behavioral.d.ts.map

package/dist/scan-behavioral.js

@@ -0,0 +1,150 @@
+"use strict";
+/**
+ * `vigiles scan --trigger` — the BEHAVIORAL column of the scan report.
+ *
+ * Structural `scan`/`scanPlugin` is deterministic, no-model, CI-free — and stays
+ * that way. This is the opt-in, model-gated column that stacks on top: for each
+ * model-invocable skill in a plugin, it measures how reliably the description
+ * actually FIRES (recall, + precision when irrelevant prompts are supplied),
+ * reusing `measureTriggerRate`. It degrades honestly when the `claude` CLI / auth
+ * is absent rather than faking a pass — exactly like the egress column.
+ *
+ * Prompts are AUTHOR-SUPPLIED (a per-skill JSON map), not model-generated — a
+ * path in prose is undecidable, and the deterministic-input discipline is what
+ * makes the column trustworthy. See `research/plugin-behavioral-findings.md`.
+ */
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.probePluginTriggersWith = probePluginTriggersWith;
+exports.probePluginTriggers = probePluginTriggers;
+exports.formatBehavioralReport = formatBehavioralReport;
+const node_fs_1 = require("node:fs");
+const node_path_1 = require("node:path");
+const scan_js_1 = require("./scan.js");
+const eval_js_1 = require("./eval.js");
+const harness_assert_js_1 = require("./harness-assert.js");
+const harness_test_js_1 = require("./harness-test.js");
+const eval_js_2 = require("./adapters/codex/eval.js");
+const driver_js_1 = require("./adapters/codex/driver.js");
+function buildProbe(dir, harness) {
+    if (harness === "codex") {
+        return {
+            evalDriver: eval_js_2.codexEvalDriver,
+            firedFor: (name) => (t) => (0, eval_js_2.codexSkillFired)(t, name),
+            // Codex stubbing of a non-Claude plugin isn't validated; install the real
+            // skills (firing is the SKILL.md read, detected regardless of body).
+            stub: false,
+            available: () => driver_js_1.codexDriver.available(),
+        };
+    }
+    const ns = pluginName(dir);
+    return {
+        evalDriver: eval_js_1.claudeEvalDriver,
+        firedFor: (name) => (t) => (0, harness_assert_js_1.skillResolved)(t, ns ? `${ns}:${name}` : name),
+        stub: true,
+        available: harness_test_js_1.claudeAvailable,
+    };
+}
+/** Plugin name from `.claude-plugin/plugin.json` (the skill id's namespace). */
+function pluginName(dir) {
+    const p = (0, node_path_1.join)(dir, ".claude-plugin", "plugin.json");
+    if (!(0, node_fs_1.existsSync)(p))
+        return null;
+    try {
+        return (JSON.parse((0, node_fs_1.readFileSync)(p, "utf-8")).name ?? null);
+    }
+    catch {
+        return null;
+    }
+}
+/** Probe one skill via the harness probe's eval driver → result, never throwing. */
+async function probeSkill(ctx, name, ps) {
+    try {
+        const r = await (0, eval_js_1.measureTriggerRate)({
+            pluginDir: ctx.dir,
+            stubSkillBodies: ctx.probe.stub,
+            prompts: ps.prompts,
+            irrelevantPrompts: ps.irrelevant,
+            minPrompts: ctx.opts.minPrompts,
+            minDistance: ctx.opts.minDistance,
+            model: ctx.opts.model,
+            concurrency: ctx.opts.concurrency,
+            fired: ctx.probe.firedFor(name),
+        }, { evalDriver: ctx.probe.evalDriver });
+        return {
+            skill: name,
+            measured: true,
+            recall: r.rate,
+            precision: r.precision,
+            falsePositiveRate: r.falsePositiveRate,
+            n: r.n,
+        };
+    }
+    catch (e) {
+        // A thin/near-duplicate prompt set (diversity gate) or a model-floor reject
+        // shouldn't crash the whole scan — surface it per skill.
+        return {
+            skill: name,
+            measured: false,
+            note: e instanceof Error ? e.message : String(e),
+        };
+    }
+}
+/** The injectable core (for tests): probe every model-invocable skill that has prompts. */
+async function probePluginTriggersWith(dir, promptSet, probe, opts = {}) {
+    const ctx = { dir, opts, probe };
+    // Only model-invocable, describable skills can auto-trigger; user-invoked and
+    // description-less ones can't, so they're not behavioral candidates.
+    const candidates = (0, scan_js_1.scanPlugin)(dir).skills.filter((s) => !s.userInvoked && s.hasDescription);
+    const results = [];
+    for (const s of candidates) {
+        const ps = promptSet[s.name];
+        if (!ps || ps.prompts.length === 0) {
+            results.push({
+                skill: s.name,
+                measured: false,
+                note: "no prompts supplied",
+            });
+            continue;
+        }
+        results.push(await probeSkill(ctx, s.name, ps));
+    }
+    return { available: true, results };
+}
+/**
+ * Probe a plugin's skills against the real harness (default Claude Code; Codex via
+ * `opts.harness`). Needs that harness's binary + auth; degrades to
+ * `available: false` otherwise.
+ */
+async function probePluginTriggers(dir, promptSet, opts = {}) {
+    const probe = buildProbe(dir, opts.harness ?? "claude-code");
+    if (!probe.available())
+        return { available: false, results: [] };
+    return probePluginTriggersWith(dir, promptSet, probe, opts);
+}
+const pct = (x) => `${(x * 100).toFixed(0)}%`;
+/** Format the behavioral column as a scan-report section. */
+function formatBehavioralReport(b) {
+    if (!b.available) {
+        return "Behavioral (trigger-rate): unavailable — needs the harness CLI + model auth";
+    }
+    if (b.results.length === 0) {
+        return "Behavioral (trigger-rate): no model-invocable skills to probe";
+    }
+    const lines = ["Behavioral (trigger-rate):"];
+    for (const r of b.results) {
+        if (!r.measured) {
+            lines.push(`  · ${r.skill} — unmeasured (${r.note ?? "skipped"})`);
+            continue;
+        }
+        const extra = [];
+        if (r.precision !== undefined)
+            extra.push(`precision ${pct(r.precision)}`);
+        if (r.falsePositiveRate !== undefined)
+            extra.push(`fp ${pct(r.falsePositiveRate)}`);
+        extra.push(`${String(r.n ?? 0)} runs`);
+        const mark = (r.recall ?? 0) >= 0.6 ? "✓" : "⚠";
+        lines.push(`  ${mark} ${r.skill} — recall ${pct(r.recall ?? 0)} (${extra.join(", ")})`);
+    }
+    return lines.join("\n");
+}
+//# sourceMappingURL=scan-behavioral.js.map

package/dist/scan.d.ts

@@ -12,17 +12,75 @@
  * stack on top later; this core stays pure so it runs anywhere in CI for free.
  */
 import type { PluginLayout } from "./core/layout.js";
+import type { HarnessDialect } from "./core/dialect.js";
+import { type ToolIssue } from "./core/tool-contract.js";
+import { type HookEventIssue } from "./core/hook-events.js";
+import { type McpIssue } from "./core/mcp-config.js";
+import { type DescriptionOverlap } from "./core/description-overlap.js";
+import { type McpToolIssue } from "./core/mcp-tool.js";
+import { type McpHookIssue } from "./core/mcp-hook.js";
+import { type PurityLevel, type EffectSurface } from "./core/effects.js";
+/** A named writing system. The label `unexpectedScript` reports + the config's expectation parse into this. */
+export type Script = "Latin" | "Cyrillic" | "Han" | "Japanese" | "Korean" | "Arabic" | "Hebrew" | "Greek" | "Devanagari" | "Thai";
 export interface ScanSkill {
     readonly name: string;
     readonly path: string;
     readonly hasDescription: boolean;
     readonly userInvoked: boolean;
+    /**
+     * The description's dominant script when it DIFFERS from the expected one
+     * (default `"Latin"`), else null. The model's skill-selection context is
+     * English-centric, so a description in another script carries a cross-language
+     * trigger risk — it may under-fire on English prompts. A RISK flag, not a
+     * defect (a language-matched audience is fine); measure the real gap with
+     * `scan --trigger`.
+     */
+    readonly descriptionScript: Script | null;
 }
 export interface ScanAgent {
     readonly name: string;
     readonly path: string;
     /** Declared tool contract, or null when the agent ships no `tools:` (inherits all). */
     readonly tools: readonly string[] | null;
+    /** Contract entries that don't resolve to a real built-in / MCP tool (typo, never-available). */
+    readonly toolIssues: readonly ToolIssue[];
+    /** MCP tool entries naming a server the plugin doesn't declare (can't resolve). */
+    readonly mcpToolIssues: readonly McpToolIssue[];
+    /** `disallowedTools:` block-list entries that are typos of a real tool (block nothing). */
+    readonly disallowedToolIssues: readonly ToolIssue[];
+    /**
+     * Static effect-surface purity of the agent's declared tool contract.
+     * - `"pure"` — no side-effecting tools (read-only, deterministically testable).
+     * - `"bounded"` — has side-effecting tools (Edit/Write/…) but no Bash or unknown.
+     * - `"unrestricted"` — has Bash, any MCP/unknown tool, or inherits-all (no contract).
+     * Computed by `effectSurface()` from `src/core/effects.ts` — one detector, no drift.
+     */
+    readonly purity: PurityLevel;
+    /**
+     * The three tool buckets from `effectSurface()`: read-only, side-effecting, and
+     * unknown-effect (MCP or unrecognized) tool names in the declared contract.
+     */
+    readonly effectBuckets: Pick<EffectSurface, "readOnly" | "sideEffecting" | "unknown">;
+}
+/** A skill/agent whose frontmatter is missing a required field (name / description). */
+export interface FrontmatterIssue {
+    readonly path: string;
+    readonly kind: "skill" | "agent";
+    readonly missing: readonly ("name" | "description")[];
+    readonly message: string;
+}
+/** A skill/agent whose `---` block exists but isn't valid YAML (may not parse as intended). */
+export interface FrontmatterParseIssue {
+    readonly path: string;
+    readonly message: string;
+}
+/** An agent frontmatter field whose VALUE is invalid (a typo of a real model/color). */
+export interface FrontmatterValueIssue {
+    readonly path: string;
+    readonly field: "model" | "color";
+    readonly value: string;
+    readonly suggestion: string;
+    readonly message: string;
 }
 /** ok = file present; missing = referenced but absent; unresolved = path still has an unexpanded var, can't check. */
 export type HookStatus = "ok" | "missing" | "unresolved";
@@ -30,8 +88,21 @@ export interface ScanHook {
     readonly script: string;
     readonly status: HookStatus;
 }
+/**
+ * The repo's top-level instruction file (`CLAUDE.md` / `AGENTS.md`), if present.
+ * Every cc/codex repo has one even when it ships no plugin surface, so `scan`
+ * reports it — otherwise a plain instruction-only repo looks empty. `hasSpec` is
+ * the deterministic fact that a `<file>.spec.ts` sits beside it (spec-managed vs
+ * hand-written); it is informational, NOT the `require-spec` gate (that's lint).
+ */
+export interface ScanInstructions {
+    readonly file: string;
+    readonly hasSpec: boolean;
+}
 export interface ScanReport {
     readonly dir: string;
+    /** The detected instruction file (CLAUDE.md/AGENTS.md), or null if none. */
+    readonly instructions: ScanInstructions | null;
     readonly skills: readonly ScanSkill[];
     readonly agents: readonly ScanAgent[];
     readonly hooks: readonly ScanHook[];
@@ -39,11 +110,105 @@ export interface ScanReport {
     readonly inlineHooks: number;
     readonly commands: number;
     readonly mcp: boolean;
+    /**
+     * Intra-plugin file references (hook scripts, skill bodies) pointing at files
+     * that don't exist on disk — the broken-path / partial-vendor class. A
+     * first-class structural finding, not just a free-text warning, so the verdict
+     * and the leaderboard can count it.
+     */
+    readonly danglingRefs: readonly string[];
+    /** Hooks registered under an event name the harness doesn't define (typo / dead). */
+    readonly hookEventIssues: readonly HookEventIssue[];
+    /** Skills/agents missing a required frontmatter field (name; agents also description). */
+    readonly frontmatterIssues: readonly FrontmatterIssue[];
+    /** Agent frontmatter fields with an invalid value (a typo of a real model/color). */
+    readonly frontmatterValueIssues: readonly FrontmatterValueIssue[];
+    /** Skills lacking an EXPLICIT name/description — a best-practice recommendation, not a defect. */
+    readonly skillMetaIssues: readonly FrontmatterIssue[];
+    /** Declared MCP servers that can't start (no command/url). */
+    readonly mcpIssues: readonly McpIssue[];
+    /** `type: mcp_tool` hook actions that are incomplete or target an undeclared server. */
+    readonly mcpHookIssues: readonly McpHookIssue[];
+    /** Pairs of model-invocable skills whose descriptions are near-identical (precision collision). */
+    readonly descriptionOverlaps: readonly DescriptionOverlap[];
+    /** Skills/agents whose `---` block isn't valid YAML — informational (may still load via salvage). */
+    readonly malformedFrontmatter: readonly FrontmatterParseIssue[];
     readonly warnings: readonly string[];
     readonly untested: number;
+    /**
+     * Harness-level purity summary: how many scanned agents fall into each purity
+     * rung. A high `pure` count means more of the harness is statically testable
+     * (deterministic, no mocks); `unrestricted` is the blind-spot count.
+     * Computed by `effectSurface()` (one detector, no drift).
+     */
+    readonly puritySummary: {
+        pure: number;
+        bounded: number;
+        unrestricted: number;
+    };
+}
+/**
+ * Per-kind surface classifiers, built from the harness `PluginLayout`'s
+ * `skillDir`/`agentDir`/`commandDir` — so adding a harness whose subagents live
+ * somewhere other than `agents/` (OpenCode's `.opencode/agent`) needs no change
+ * here. Each anchors on a real path boundary (start-of-path or a `/`), so a
+ * directory whose NAME merely ends in the keyword isn't misclassified — e.g. the
+ * skill `skills/dispatching-parallel-agents/SKILL.md` must NOT register as an
+ * agent named "SKILL" (the `-agents/` substring), which real plugins like
+ * obra/superpowers ship. See scan.test.ts for the regression cases.
+ */
+export interface SurfaceClassifier {
+    readonly isSkill: (f: string) => boolean;
+    readonly isAgent: (f: string) => boolean;
+    readonly isCommand: (f: string) => boolean;
 }
+/**
+ * The description's dominant alphabetic script when it DIFFERS from `expected`
+ * (default `"Latin"`) — the cross-language trigger-risk signal. The model's
+ * skill-selection context is English-centric, so a description written mostly in
+ * another script may under-fire on English prompts. `expected` is a configurable
+ * default, not a value judgement: a Russian-targeted pack sets it to `"Cyrillic"`
+ * so its Cyrillic descriptions pass and an English one is flagged instead.
+ * Returns null when the dominant script IS the expected one (or there's no
+ * alphabetic content). Shared by `scan` and the future lint rule (one detector,
+ * no drift). The ≥20% guard avoids a near-empty string tripping on one letter.
+ */
+export declare function unexpectedScript(text: string, expected?: Script): Script | null;
 /** Scan a plugin/repo directory and report its surfaces + structural issues. */
-export declare function scanPlugin(dir: string, layout?: PluginLayout): ScanReport;
+export declare function scanPlugin(dir: string, layout?: PluginLayout, dialect?: HarnessDialect): ScanReport;
+/**
+ * A plugin MARKETPLACE (`.claude-plugin/marketplace.json`) decomposed into its
+ * members. A marketplace either VENDORS its plugins in-tree (string `source`
+ * paths, e.g. wshobson/agents — `onDisk` populated) or CURATES external ones
+ * (object `source` with a git/url, e.g. obra/superpowers-marketplace,
+ * anthropics/claude-plugins-community — `external` populated, nothing on disk).
+ * Distinguishing the two lets `scan` report a curated marketplace honestly
+ * instead of mistaking it for an empty repo.
+ */
+export interface MarketplaceInfo {
+    readonly name: string;
+    /** Member plugin dirs that exist on disk (string `source` paths). */
+    readonly onDisk: readonly string[];
+    /** Members referencing an off-disk source (url/git/github) — can't be scanned here. */
+    readonly external: number;
+    readonly total: number;
+}
+/**
+ * Read a `marketplace.json` beside the layout's plugin manifest and classify its
+ * members into on-disk vs external. Returns `null` when `dir` is not a
+ * marketplace. The source of truth behind {@link expandMarketplace} and the
+ * curated-marketplace report in `vigiles scan`.
+ */
+export declare function inspectMarketplace(dir: string, layout?: PluginLayout): MarketplaceInfo | null;
+/**
+ * If `dir` is a plugin MARKETPLACE (a `marketplace.json` beside the layout's
+ * plugin manifest, e.g. `.claude-plugin/marketplace.json`), expand it into the
+ * absolute dirs of its member plugins. Returns `null` when there's no
+ * marketplace, `[]` when it's a marketplace whose members are all external (not
+ * on disk). Used by `vigiles scan` to rank a whole marketplace — wshobson/agents
+ * alone ships 80+ plugins under one `marketplace.json`. See {@link inspectMarketplace}.
+ */
+export declare function expandMarketplace(dir: string, layout?: PluginLayout): string[] | null;
 /** Format a scan report as human-readable text. */
 export declare function formatScanReport(r: ScanReport): string;
 //# sourceMappingURL=scan.d.ts.map