vigiles 5.0.1 → 5.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (38) hide show
  1. package/README.md +14 -8
  2. package/dist/adapters/claude-code/agent-runtime.d.ts +10 -0
  3. package/dist/adapters/claude-code/agent-runtime.js +15 -29
  4. package/dist/adapters/claude-code/dialect.js +18 -2
  5. package/dist/adapters/codex/eval.d.ts +94 -0
  6. package/dist/adapters/codex/eval.js +227 -0
  7. package/dist/cli.js +464 -8
  8. package/dist/codex.d.ts +1 -0
  9. package/dist/codex.js +3 -0
  10. package/dist/core/compile.js +8 -36
  11. package/dist/core/description-overlap.d.ts +27 -0
  12. package/dist/core/description-overlap.js +53 -0
  13. package/dist/core/dialect.d.ts +8 -0
  14. package/dist/core/frontmatter-read.d.ts +25 -0
  15. package/dist/core/frontmatter-read.js +138 -0
  16. package/dist/core/hook-events.d.ts +34 -0
  17. package/dist/core/hook-events.js +48 -0
  18. package/dist/core/mcp-config.d.ts +20 -0
  19. package/dist/core/mcp-config.js +40 -0
  20. package/dist/core/mcp-hook.d.ts +35 -0
  21. package/dist/core/mcp-hook.js +70 -0
  22. package/dist/core/mcp-tool.d.ts +50 -0
  23. package/dist/core/mcp-tool.js +61 -0
  24. package/dist/core/tool-contract.d.ts +68 -0
  25. package/dist/core/tool-contract.js +113 -0
  26. package/dist/core/types.d.ts +89 -0
  27. package/dist/core/validate.js +22 -0
  28. package/dist/eval.d.ts +69 -13
  29. package/dist/eval.js +106 -51
  30. package/dist/leaderboard.js +61 -3
  31. package/dist/plugin-loader.d.ts +1 -0
  32. package/dist/plugin-loader.js +71 -18
  33. package/dist/scan-behavioral.d.ts +73 -0
  34. package/dist/scan-behavioral.js +150 -0
  35. package/dist/scan.d.ts +126 -1
  36. package/dist/scan.js +559 -40
  37. package/package.json +1 -1
  38. package/skills/migrate-to-spec/SKILL.md +0 -2
package/dist/cli.js CHANGED
@@ -20,6 +20,10 @@ const setup_plan_js_1 = require("./setup-plan.js");
20
20
  const types_js_1 = require("./core/types.js");
21
21
  const test_coverage_js_1 = require("./test-coverage.js");
22
22
  const scan_js_1 = require("./scan.js");
23
+ const tool_contract_js_1 = require("./core/tool-contract.js");
24
+ const agent_runtime_js_1 = require("./adapters/claude-code/agent-runtime.js");
25
+ const dialect_js_1 = require("./adapters/claude-code/dialect.js");
26
+ const scan_behavioral_js_1 = require("./scan-behavioral.js");
23
27
  const adapter_registry_js_1 = require("./adapter-registry.js");
24
28
  const skill_harness_js_1 = require("./skill-harness.js");
25
29
  const leaderboard_js_1 = require("./leaderboard.js");
@@ -31,7 +35,7 @@ const generate_schema_js_1 = require("./core/generate-schema.js");
31
35
  const compose_js_1 = require("./core/compose.js");
32
36
  const compile_generator_js_1 = require("./core/compile-generator.js");
33
37
  const action_gate_js_1 = require("./action-gate.js");
34
- const agent_runtime_js_1 = require("./adapters/claude-code/agent-runtime.js");
38
+ const agent_runtime_js_2 = require("./adapters/claude-code/agent-runtime.js");
35
39
  const tool_intercept_js_1 = require("./tool-intercept.js");
36
40
  const refs_js_1 = require("./core/refs.js");
37
41
  const mcp_js_1 = require("./core/mcp.js");
@@ -633,6 +637,17 @@ function lintExitCode(report) {
633
637
  report.integrityErrors > 0 ||
634
638
  report.coverageErrors > 0 ||
635
639
  report.untestedErrors > 0 ||
640
+ report.toolContractErrors > 0 ||
641
+ report.hookEventErrors > 0 ||
642
+ report.frontmatterSchemaErrors > 0 ||
643
+ report.mcpConfigErrors > 0 ||
644
+ report.skillFrontmatterErrors > 0 ||
645
+ report.mcpToolErrors > 0 ||
646
+ report.hookScriptErrors > 0 ||
647
+ report.disallowedToolErrors > 0 ||
648
+ report.descriptionOverlapErrors > 0 ||
649
+ report.frontmatterValidErrors > 0 ||
650
+ report.mcpHookErrors > 0 ||
636
651
  report.symbolRefErrors > 0 ||
637
652
  report.mcpRefErrors > 0)
638
653
  return 2;
@@ -913,6 +928,39 @@ async function runLint(restArgs, flags, config) {
913
928
  // eval. Warning by default (a nudge, exit 0); set rules.untested-{skill,agent,
914
929
  // hook} to "error" to gate CI. See src/test-coverage.ts and docs/rules/.
915
930
  const untested = checkUntestedSurfaces(config, silent);
931
+ // 7c. Agent tool-contract check — cross-reference each subagent's `tools:` rail
932
+ // against the harness catalog (the moat). Off by default unless a severity is
933
+ // configured; warning surfaces a typo/never-available tool, error gates CI.
934
+ const toolContract = checkAgentToolContracts(config, silent);
935
+ // 7d. Hook-event check — a hook registered under an event the harness doesn't
936
+ // define never fires. High-precision (close typos only). Off unless configured.
937
+ const hookEvents = checkHookEvents(config, silent);
938
+ // 7e. Frontmatter-schema check — a skill/agent missing required frontmatter
939
+ // (name; agents also description) won't load/register. High-confidence.
940
+ const frontmatter = checkFrontmatterSchema(config, silent);
941
+ // 7f. MCP-config check — a declared MCP server with no command/url can't start.
942
+ const mcpConfig = checkMcpConfig(config, silent);
943
+ // 7g. Skill-frontmatter — RECOMMEND explicit name/description on skills (a
944
+ // reliable trigger surface). Best-practice nudge; skills load without it.
945
+ const skillFm = checkSkillFrontmatter(config, silent);
946
+ // 7h. MCP tool-resolution — an `mcp__server__tool` in a contract whose server
947
+ // the plugin doesn't declare can't resolve (the MCP half of the tool moat).
948
+ const mcpToolResolves = checkMcpToolResolves(config, silent);
949
+ // 7i. Hook-script existence — a hook command referencing a missing script file
950
+ // never runs (matches Anthropic's own `claude plugin validate`).
951
+ const hookScripts = checkHookScriptExists(config, silent);
952
+ // 7j. Disallowed-tools — a `disallowedTools:` block-list typo blocks nothing
953
+ // (the deny-side mirror of agent-tool-contract; close-typo only).
954
+ const disallowedTools = checkDisallowedTools(config, silent);
955
+ // 7k. Description-overlap — two model-invocable skills with near-identical
956
+ // descriptions collide in the selector (deterministic NCD precision proxy).
957
+ const descriptionOverlap = checkDescriptionOverlap(config, silent);
958
+ // 7l. Frontmatter-valid — a `---` block that isn't valid YAML (warn; js-yaml is
959
+ // stricter than some loaders, so verify before enforcing).
960
+ const frontmatterValid = checkFrontmatterValid(config, silent);
961
+ // 7m. MCP hook-target — a `type: mcp_tool` hook action that's incomplete or
962
+ // targets an undeclared server (the moat applied to the hook surface).
963
+ const mcpHookTargets = checkMcpHookTargets(config, silent);
916
964
  // 8. Validate vigiles builder calls inside markdown code blocks. Default
917
965
  // is to validate every ref; illustrative blocks opt out via
918
966
  // `<!-- vigiles:ignore -->` (single block) or
@@ -956,6 +1004,28 @@ async function runLint(restArgs, flags, config) {
956
1004
  orphanCount: orphanReport.orphans.length,
957
1005
  untestedSurfaces: untested.untested,
958
1006
  untestedErrors: untested.errors,
1007
+ toolContractIssues: toolContract.issues,
1008
+ toolContractErrors: toolContract.errors,
1009
+ hookEventIssues: hookEvents.issues,
1010
+ hookEventErrors: hookEvents.errors,
1011
+ frontmatterSchemaIssues: frontmatter.issues,
1012
+ frontmatterSchemaErrors: frontmatter.errors,
1013
+ mcpConfigIssues: mcpConfig.issues,
1014
+ mcpConfigErrors: mcpConfig.errors,
1015
+ skillFrontmatterIssues: skillFm.issues,
1016
+ skillFrontmatterErrors: skillFm.errors,
1017
+ mcpToolIssues: mcpToolResolves.issues,
1018
+ mcpToolErrors: mcpToolResolves.errors,
1019
+ hookScriptIssues: hookScripts.issues,
1020
+ hookScriptErrors: hookScripts.errors,
1021
+ disallowedToolIssues: disallowedTools.issues,
1022
+ disallowedToolErrors: disallowedTools.errors,
1023
+ descriptionOverlapIssues: descriptionOverlap.issues,
1024
+ descriptionOverlapErrors: descriptionOverlap.errors,
1025
+ frontmatterValidIssues: frontmatterValid.issues,
1026
+ frontmatterValidErrors: frontmatterValid.errors,
1027
+ mcpHookIssues: mcpHookTargets.issues,
1028
+ mcpHookErrors: mcpHookTargets.errors,
959
1029
  docRefErrors: docRefReport.errors.length,
960
1030
  symbolRefErrors,
961
1031
  mcpRefErrors,
@@ -1931,6 +2001,328 @@ function checkUntestedSurfaces(config, silent) {
1931
2001
  errors: report.untested.filter((s) => sevFor(s.kind) === "error").length,
1932
2002
  };
1933
2003
  }
2004
+ /**
2005
+ * Apply the `agent-tool-contract` rule: cross-reference every subagent's `tools:`
2006
+ * rail against the harness tool catalog (the moat — "valid is not true"). Flags
2007
+ * only the HIGH-CONFIDENCE issues (a never-available tool, or a close typo) via
2008
+ * the shared `confidentToolIssues` detector — the same code `scan` and
2009
+ * `compileAgent` use (one-detector-no-drift), so a bare unrecognized tool
2010
+ * (plugin/MCP-provided) is never a false alarm. Warning by default; set
2011
+ * `agent-tool-contract: "error"` to gate CI. Returns the issue + error counts.
2012
+ */
2013
+ function checkAgentToolContracts(config, silent) {
2014
+ const sev = (0, types_js_1.ruleSeverity)(config?.rules?.["agent-tool-contract"]);
2015
+ if (!sev)
2016
+ return { issues: 0, errors: 0 };
2017
+ const files = (0, glob_1.globSync)(["agents/*.md", ".claude/agents/*.md"], {
2018
+ cwd: process.cwd(),
2019
+ ignore: ["**/*.spec.ts"],
2020
+ });
2021
+ let issues = 0;
2022
+ let printedHeader = false;
2023
+ for (const rel of files.sort()) {
2024
+ let md;
2025
+ try {
2026
+ md = (0, node_fs_1.readFileSync)((0, node_path_1.resolve)(process.cwd(), rel), "utf-8");
2027
+ }
2028
+ catch {
2029
+ continue;
2030
+ }
2031
+ const tools = (0, agent_runtime_js_1.parseAgentTools)(md);
2032
+ if (tools === null)
2033
+ continue; // no contract → inherits all (a different rule)
2034
+ const found = (0, tool_contract_js_1.confidentToolIssues)((0, tool_contract_js_1.verifyToolContract)(tools, dialect_js_1.claudeCodeDialect));
2035
+ if (found.length === 0)
2036
+ continue;
2037
+ issues += found.length;
2038
+ if (!silent) {
2039
+ if (!printedHeader) {
2040
+ console.log("\nAgent tool-contract check:\n");
2041
+ printedHeader = true;
2042
+ }
2043
+ for (const issue of found) {
2044
+ console.log(` ${sev === "error" ? "✗" : "⚠"} ${rel}: ${issue.message}`);
2045
+ ghAnnotate(sev === "error" ? "error" : "warning", issue.message, rel);
2046
+ }
2047
+ }
2048
+ }
2049
+ return { issues, errors: sev === "error" ? issues : 0 };
2050
+ }
2051
+ /**
2052
+ * Apply the `hook-events` rule: flag a hook registered under an event name the
2053
+ * harness doesn't define (a typo → the hook never fires). Reuses `scanPlugin`'s
2054
+ * `hookEventIssues` (the shared detector, high-precision: close typos only, never
2055
+ * a framework/custom event). Warning by default; "error" gates CI.
2056
+ */
2057
+ function checkHookEvents(config, silent) {
2058
+ const sev = (0, types_js_1.ruleSeverity)(config?.rules?.["hook-events"]);
2059
+ if (!sev)
2060
+ return { issues: 0, errors: 0 };
2061
+ let found;
2062
+ try {
2063
+ found = (0, scan_js_1.scanPlugin)(process.cwd()).hookEventIssues;
2064
+ }
2065
+ catch {
2066
+ return { issues: 0, errors: 0 };
2067
+ }
2068
+ if (found.length > 0 && !silent) {
2069
+ console.log("\nHook-event check:\n");
2070
+ for (const issue of found) {
2071
+ console.log(` ${sev === "error" ? "✗" : "⚠"} ${issue.message}`);
2072
+ ghAnnotate(sev === "error" ? "error" : "warning", issue.message);
2073
+ }
2074
+ }
2075
+ return { issues: found.length, errors: sev === "error" ? found.length : 0 };
2076
+ }
2077
+ /**
2078
+ * Apply the `agent-frontmatter` rule. Two kinds of agent-frontmatter defect, one
2079
+ * rule: (1) a subagent MISSING a required field (`name`/`description`) — it won't
2080
+ * register; (2) a subagent with an INVALID `model:`/`color:` value (a close typo
2081
+ * of a real one) — it silently falls back / is ignored. Reuses `scanPlugin`'s
2082
+ * `frontmatterIssues` + `frontmatterValueIssues`. Warning by default; "error" gates CI.
2083
+ */
2084
+ function checkFrontmatterSchema(config, silent) {
2085
+ const sev = (0, types_js_1.ruleSeverity)(config?.rules?.["agent-frontmatter"]);
2086
+ if (!sev)
2087
+ return { issues: 0, errors: 0 };
2088
+ let found;
2089
+ try {
2090
+ const r = (0, scan_js_1.scanPlugin)(process.cwd());
2091
+ found = [...r.frontmatterIssues, ...r.frontmatterValueIssues];
2092
+ }
2093
+ catch {
2094
+ return { issues: 0, errors: 0 };
2095
+ }
2096
+ if (found.length > 0 && !silent) {
2097
+ console.log("\nFrontmatter-schema check:\n");
2098
+ for (const issue of found) {
2099
+ console.log(` ${sev === "error" ? "✗" : "⚠"} ${issue.message}`);
2100
+ ghAnnotate(sev === "error" ? "error" : "warning", issue.message, issue.path);
2101
+ }
2102
+ }
2103
+ return { issues: found.length, errors: sev === "error" ? found.length : 0 };
2104
+ }
2105
+ /**
2106
+ * Apply the `skill-frontmatter` rule: RECOMMEND (not require) that a SKILL.md
2107
+ * declares an explicit `name` + `description` rather than relying on the
2108
+ * dir-name / first-paragraph fallbacks — a more reliable trigger surface. The
2109
+ * skill still LOADS without them, so this is a best-practice nudge: warn by
2110
+ * default; set "error" to enforce it on your own skills. Reuses `scanPlugin`'s
2111
+ * `skillMetaIssues`.
2112
+ */
2113
+ function checkSkillFrontmatter(config, silent) {
2114
+ const sev = (0, types_js_1.ruleSeverity)(config?.rules?.["skill-frontmatter"]);
2115
+ if (!sev)
2116
+ return { issues: 0, errors: 0 };
2117
+ let found;
2118
+ try {
2119
+ found = (0, scan_js_1.scanPlugin)(process.cwd()).skillMetaIssues;
2120
+ }
2121
+ catch {
2122
+ return { issues: 0, errors: 0 };
2123
+ }
2124
+ if (found.length > 0 && !silent) {
2125
+ console.log("\nSkill-frontmatter check:\n");
2126
+ for (const issue of found) {
2127
+ console.log(` ${sev === "error" ? "✗" : "⚠"} ${issue.message}`);
2128
+ ghAnnotate(sev === "error" ? "error" : "warning", issue.message, issue.path);
2129
+ }
2130
+ }
2131
+ return { issues: found.length, errors: sev === "error" ? found.length : 0 };
2132
+ }
2133
+ /**
2134
+ * Apply the `mcp-config` rule: a declared MCP server with neither a `command`
2135
+ * (stdio) nor a `url` (http/sse) can't start. Reuses `scanPlugin`'s `mcpIssues`.
2136
+ * Warning by default; "error" gates CI.
2137
+ */
2138
+ function checkMcpConfig(config, silent) {
2139
+ const sev = (0, types_js_1.ruleSeverity)(config?.rules?.["mcp-config"]);
2140
+ if (!sev)
2141
+ return { issues: 0, errors: 0 };
2142
+ let found;
2143
+ try {
2144
+ found = (0, scan_js_1.scanPlugin)(process.cwd()).mcpIssues;
2145
+ }
2146
+ catch {
2147
+ return { issues: 0, errors: 0 };
2148
+ }
2149
+ if (found.length > 0 && !silent) {
2150
+ console.log("\nMCP-config check:\n");
2151
+ for (const issue of found) {
2152
+ console.log(` ${sev === "error" ? "✗" : "⚠"} ${issue.message}`);
2153
+ ghAnnotate(sev === "error" ? "error" : "warning", issue.message);
2154
+ }
2155
+ }
2156
+ return { issues: found.length, errors: sev === "error" ? found.length : 0 };
2157
+ }
2158
+ /**
2159
+ * Apply the `disallowed-tools-contract` rule: a subagent's `disallowedTools:`
2160
+ * block-list entry that's a close typo of a real tool blocks NOTHING — the tool
2161
+ * it was meant to deny stays available, silently. Reuses `scanPlugin`'s per-agent
2162
+ * `disallowedToolIssues` (close-typo only — high-precision). Warning by default;
2163
+ * "error" gates CI.
2164
+ */
2165
+ function checkDisallowedTools(config, silent) {
2166
+ const sev = (0, types_js_1.ruleSeverity)(config?.rules?.["disallowed-tools-contract"]);
2167
+ if (!sev)
2168
+ return { issues: 0, errors: 0 };
2169
+ let found;
2170
+ try {
2171
+ found = (0, scan_js_1.scanPlugin)(process.cwd()).agents.flatMap((a) => a.disallowedToolIssues.map((i) => ({ message: i.message, path: a.path })));
2172
+ }
2173
+ catch {
2174
+ return { issues: 0, errors: 0 };
2175
+ }
2176
+ if (found.length > 0 && !silent) {
2177
+ console.log("\nDisallowed-tools check:\n");
2178
+ for (const issue of found) {
2179
+ console.log(` ${sev === "error" ? "✗" : "⚠"} ${issue.path}: ${issue.message}`);
2180
+ ghAnnotate(sev === "error" ? "error" : "warning", issue.message, issue.path);
2181
+ }
2182
+ }
2183
+ return { issues: found.length, errors: sev === "error" ? found.length : 0 };
2184
+ }
2185
+ /**
2186
+ * Apply the `frontmatter-valid` rule: a skill/agent `---` block that EXISTS but
2187
+ * isn't valid YAML — fields may not parse as intended. Reuses `scanPlugin`'s
2188
+ * `malformedFrontmatter`. HONEST caveat (see docs/rules/frontmatter-valid.md):
2189
+ * js-yaml is stricter than some loaders, so a one-line `description:` with a
2190
+ * colon / `<example>` is flagged though it may still load — hence WARN by default
2191
+ * (verify before setting "error").
2192
+ */
2193
+ function checkFrontmatterValid(config, silent) {
2194
+ const sev = (0, types_js_1.ruleSeverity)(config?.rules?.["frontmatter-valid"]);
2195
+ if (!sev)
2196
+ return { issues: 0, errors: 0 };
2197
+ let found;
2198
+ try {
2199
+ found = (0, scan_js_1.scanPlugin)(process.cwd()).malformedFrontmatter;
2200
+ }
2201
+ catch {
2202
+ return { issues: 0, errors: 0 };
2203
+ }
2204
+ if (found.length > 0 && !silent) {
2205
+ console.log("\nFrontmatter-validity check:\n");
2206
+ for (const issue of found) {
2207
+ console.log(` ${sev === "error" ? "✗" : "⚠"} ${issue.message}`);
2208
+ ghAnnotate(sev === "error" ? "error" : "warning", issue.message, issue.path);
2209
+ }
2210
+ }
2211
+ return { issues: found.length, errors: sev === "error" ? found.length : 0 };
2212
+ }
2213
+ /**
2214
+ * Apply the `description-overlap` rule: two model-invocable skills with
2215
+ * near-identical descriptions collide in the selector — the wrong one fires. A
2216
+ * deterministic NCD proxy for a `--trigger`-class precision bug. Reuses
2217
+ * `scanPlugin`'s `descriptionOverlaps` (calibrated FP-safe: only basically
2218
+ * identical text). Warning by default; "error" gates CI.
2219
+ */
2220
+ function checkDescriptionOverlap(config, silent) {
2221
+ const sev = (0, types_js_1.ruleSeverity)(config?.rules?.["description-overlap"]);
2222
+ if (!sev)
2223
+ return { issues: 0, errors: 0 };
2224
+ let found;
2225
+ try {
2226
+ found = (0, scan_js_1.scanPlugin)(process.cwd()).descriptionOverlaps;
2227
+ }
2228
+ catch {
2229
+ return { issues: 0, errors: 0 };
2230
+ }
2231
+ if (found.length > 0 && !silent) {
2232
+ console.log("\nDescription-overlap check:\n");
2233
+ for (const issue of found) {
2234
+ console.log(` ${sev === "error" ? "✗" : "⚠"} ${issue.message}`);
2235
+ ghAnnotate(sev === "error" ? "error" : "warning", issue.message);
2236
+ }
2237
+ }
2238
+ return { issues: found.length, errors: sev === "error" ? found.length : 0 };
2239
+ }
2240
+ /**
2241
+ * Apply the `mcp-hook-target-resolves` rule: a `type: "mcp_tool"` hook action
2242
+ * that's incomplete (no `server`/`tool`) or targets a server the plugin doesn't
2243
+ * declare — the hook silently never dispatches. Reuses `scanPlugin`'s
2244
+ * `mcpHookIssues` (high-precision: declared-set gated, built-ins allowlisted).
2245
+ * Warning by default; "error" gates CI.
2246
+ */
2247
+ function checkMcpHookTargets(config, silent) {
2248
+ const sev = (0, types_js_1.ruleSeverity)(config?.rules?.["mcp-hook-target-resolves"]);
2249
+ if (!sev)
2250
+ return { issues: 0, errors: 0 };
2251
+ let found;
2252
+ try {
2253
+ found = (0, scan_js_1.scanPlugin)(process.cwd()).mcpHookIssues;
2254
+ }
2255
+ catch {
2256
+ return { issues: 0, errors: 0 };
2257
+ }
2258
+ if (found.length > 0 && !silent) {
2259
+ console.log("\nMCP hook-target check:\n");
2260
+ for (const issue of found) {
2261
+ console.log(` ${sev === "error" ? "✗" : "⚠"} ${issue.message}`);
2262
+ ghAnnotate(sev === "error" ? "error" : "warning", issue.message);
2263
+ }
2264
+ }
2265
+ return { issues: found.length, errors: sev === "error" ? found.length : 0 };
2266
+ }
2267
+ /**
2268
+ * Apply the `hook-script-exists` rule: a hook command references a script file
2269
+ * that doesn't exist on disk (with `${CLAUDE_PLUGIN_ROOT}` resolved) → the hook
2270
+ * silently never runs. Reuses `scanPlugin`'s `hooks` (status "missing"); the
2271
+ * shared resolver already excludes the FP-prone cases (unresolved vars,
2272
+ * existence-guarded one-liners, inline commands). Matches Anthropic's own
2273
+ * `claude plugin validate`. Warning by default; "error" gates CI.
2274
+ */
2275
+ function checkHookScriptExists(config, silent) {
2276
+ const sev = (0, types_js_1.ruleSeverity)(config?.rules?.["hook-script-exists"]);
2277
+ if (!sev)
2278
+ return { issues: 0, errors: 0 };
2279
+ let missing;
2280
+ try {
2281
+ missing = (0, scan_js_1.scanPlugin)(process.cwd()).hooks.filter((h) => h.status === "missing");
2282
+ }
2283
+ catch {
2284
+ return { issues: 0, errors: 0 };
2285
+ }
2286
+ if (missing.length > 0 && !silent) {
2287
+ console.log("\nHook-script existence check:\n");
2288
+ for (const h of missing) {
2289
+ const msg = `hook script "${h.script}" is referenced but missing — the hook never runs.`;
2290
+ console.log(` ${sev === "error" ? "✗" : "⚠"} ${msg}`);
2291
+ ghAnnotate(sev === "error" ? "error" : "warning", msg);
2292
+ }
2293
+ }
2294
+ return {
2295
+ issues: missing.length,
2296
+ errors: sev === "error" ? missing.length : 0,
2297
+ };
2298
+ }
2299
+ /**
2300
+ * Apply the `mcp-tool-resolves` rule: an `mcp__server__tool` in a subagent's
2301
+ * contract whose server isn't in the plugin's declared `mcpServers` can't resolve
2302
+ * (the MCP half of the tool moat). Reuses `scanPlugin`'s per-agent `mcpToolIssues`
2303
+ * — high-precision (gated on a declared set, built-ins allowlisted, the
2304
+ * plugin-namespaced form skipped). Warning by default; "error" gates CI.
2305
+ */
2306
+ function checkMcpToolResolves(config, silent) {
2307
+ const sev = (0, types_js_1.ruleSeverity)(config?.rules?.["mcp-tool-resolves"]);
2308
+ if (!sev)
2309
+ return { issues: 0, errors: 0 };
2310
+ let found;
2311
+ try {
2312
+ found = (0, scan_js_1.scanPlugin)(process.cwd()).agents.flatMap((a) => a.mcpToolIssues.map((i) => ({ message: i.message, path: a.path })));
2313
+ }
2314
+ catch {
2315
+ return { issues: 0, errors: 0 };
2316
+ }
2317
+ if (found.length > 0 && !silent) {
2318
+ console.log("\nMCP tool-resolution check:\n");
2319
+ for (const issue of found) {
2320
+ console.log(` ${sev === "error" ? "✗" : "⚠"} ${issue.path}: ${issue.message}`);
2321
+ ghAnnotate(sev === "error" ? "error" : "warning", issue.message, issue.path);
2322
+ }
2323
+ }
2324
+ return { issues: found.length, errors: sev === "error" ? found.length : 0 };
2325
+ }
1934
2326
  /**
1935
2327
  * Apply the configured coverage thresholds. Returns the number of failing
1936
2328
  * thresholds (so the lint can fail CI when severity is "error").
@@ -2008,6 +2400,43 @@ function findInstructionFiles(restArgs) {
2008
2400
  }
2009
2401
  return files;
2010
2402
  }
2403
+ /** Value of a `--flag=value` arg (the `=` form, so it never collides with a positional). */
2404
+ function flagValue(args, name) {
2405
+ return args.find((a) => a.startsWith(`${name}=`))?.slice(name.length + 1);
2406
+ }
2407
+ /**
2408
+ * The `scan --trigger` behavioral column: load the author-supplied per-skill
2409
+ * prompt sets, probe the plugin's model-invocable skills, print the column.
2410
+ * Model-gated and opt-in — the structural scan above stays deterministic.
2411
+ */
2412
+ async function handleScanTrigger(root, args, json, harness) {
2413
+ const promptsPath = flagValue(args, "--prompts");
2414
+ if (!promptsPath) {
2415
+ console.error("scan --trigger needs --prompts=<file.json> (a map of skill name → { prompts, irrelevant }).");
2416
+ process.exitCode = 2;
2417
+ return;
2418
+ }
2419
+ let promptSet;
2420
+ try {
2421
+ promptSet = JSON.parse((0, node_fs_1.readFileSync)((0, node_path_1.resolve)(promptsPath), "utf-8"));
2422
+ }
2423
+ catch (e) {
2424
+ console.error(`scan --trigger: could not read --prompts file "${promptsPath}": ${e instanceof Error ? e.message : String(e)}`);
2425
+ process.exitCode = 2;
2426
+ return;
2427
+ }
2428
+ const concurrencyRaw = flagValue(args, "--concurrency");
2429
+ const minPromptsRaw = flagValue(args, "--min-prompts");
2430
+ const report = await (0, scan_behavioral_js_1.probePluginTriggers)(root, promptSet, {
2431
+ concurrency: concurrencyRaw ? Number(concurrencyRaw) : undefined,
2432
+ minPrompts: minPromptsRaw ? Number(minPromptsRaw) : undefined,
2433
+ model: flagValue(args, "--model"),
2434
+ harness,
2435
+ });
2436
+ console.log(json
2437
+ ? JSON.stringify(report, null, 2)
2438
+ : `\n${(0, scan_behavioral_js_1.formatBehavioralReport)(report)}`);
2439
+ }
2011
2440
  function handleGenerateTypes(args, restArgs) {
2012
2441
  const checkOnly = args.includes("--check");
2013
2442
  const outPath = restArgs[0] ?? ".vigiles/generated.d.ts";
@@ -2295,7 +2724,7 @@ function agentHookCommand() {
2295
2724
  }
2296
2725
  if (!tool)
2297
2726
  return;
2298
- const decision = (0, agent_runtime_js_1.evaluatePreToolUse)(process.cwd(), tool);
2727
+ const decision = (0, agent_runtime_js_2.evaluatePreToolUse)(process.cwd(), tool);
2299
2728
  if (!decision.allow) {
2300
2729
  console.error(decision.message);
2301
2730
  process.exit(2);
@@ -2331,7 +2760,7 @@ function agentStartCommand(target) {
2331
2760
  console.error("Usage: vigiles agent-start <agents/<name>.md>");
2332
2761
  process.exit(2);
2333
2762
  }
2334
- (0, agent_runtime_js_1.setActiveAgent)(process.cwd(), target);
2763
+ (0, agent_runtime_js_2.setActiveAgent)(process.cwd(), target);
2335
2764
  console.log(`Active agent: ${target}`);
2336
2765
  }
2337
2766
  /** Dispatch the skill-runtime subcommands. Returns false if unrecognized. */
@@ -2353,7 +2782,7 @@ function handleSkillCommand(command, restArgs) {
2353
2782
  agentStartCommand(restArgs[0]);
2354
2783
  return true;
2355
2784
  case "agent-done":
2356
- (0, agent_runtime_js_1.clearActiveAgent)(process.cwd());
2785
+ (0, agent_runtime_js_2.clearActiveAgent)(process.cwd());
2357
2786
  return true;
2358
2787
  case "agent-hook":
2359
2788
  agentHookCommand();
@@ -2574,13 +3003,36 @@ async function main() {
2574
3003
  case "scan": {
2575
3004
  const dirs = restArgs.length > 0 ? restArgs : ["."];
2576
3005
  const json = args.includes("--json");
2577
- if (dirs.length > 1) {
3006
+ // A single dir that's a marketplace (e.g. wshobson/agents' 80+ plugins
3007
+ // under one marketplace.json) expands into its members and ranks them.
3008
+ const market = dirs.length === 1 ? (0, scan_js_1.inspectMarketplace)((0, node_path_1.resolve)(dirs[0])) : null;
3009
+ const targets = market && market.onDisk.length > 0 ? [...market.onDisk] : dirs;
3010
+ const wantTrigger = args.includes("--trigger");
3011
+ if (market && market.onDisk.length === 0 && market.total > 0) {
3012
+ // A CURATED marketplace — every member is an external git/url plugin, so
3013
+ // there's nothing on disk to scan. Say so honestly instead of falling
3014
+ // through to a misleading "empty machine / no structural issues" report
3015
+ // (obra/superpowers-marketplace, anthropics/claude-plugins-community).
3016
+ if (json) {
3017
+ console.log(JSON.stringify(market, null, 2));
3018
+ }
3019
+ else {
3020
+ console.log(`Marketplace "${market.name}": ${String(market.total)} plugin(s), all external ` +
3021
+ `(url/git sources, not on disk).\n` +
3022
+ `Nothing to scan here — clone a member plugin and scan that, or scan a ` +
3023
+ `marketplace that vendors its plugins in-tree.`);
3024
+ }
3025
+ }
3026
+ else if (targets.length > 1) {
2578
3027
  // Multiple targets → rank them (the leaderboard engine).
2579
- const scores = (0, leaderboard_js_1.rankPlugins)(dirs);
3028
+ const scores = (0, leaderboard_js_1.rankPlugins)(targets);
2580
3029
  console.log(json ? JSON.stringify(scores, null, 2) : (0, leaderboard_js_1.formatLeaderboard)(scores));
3030
+ if (wantTrigger) {
3031
+ console.log("\n⚠ --trigger (behavioral column) runs per single plugin; not yet wired into the leaderboard. Scan one plugin dir to probe it.");
3032
+ }
2581
3033
  }
2582
3034
  else {
2583
- const root = (0, node_path_1.resolve)(dirs[0]);
3035
+ const root = (0, node_path_1.resolve)(targets[0]);
2584
3036
  const harnessFlag = args
2585
3037
  .find((a) => a.startsWith("--harness="))
2586
3038
  ?.slice("--harness=".length);
@@ -2588,7 +3040,7 @@ async function main() {
2588
3040
  const adapter = harnessFlag
2589
3041
  ? (0, adapter_registry_js_1.resolveAdapter)(root, harnessFlag)
2590
3042
  : det.adapter;
2591
- const report = (0, scan_js_1.scanPlugin)(dirs[0], adapter.layout);
3043
+ const report = (0, scan_js_1.scanPlugin)(targets[0], adapter.layout, adapter.dialect);
2592
3044
  if (!json) {
2593
3045
  console.log(`Detected harness: ${adapter.name}`);
2594
3046
  if (!harnessFlag && det.ambiguousWith.length > 0) {
@@ -2597,6 +3049,10 @@ async function main() {
2597
3049
  console.log("");
2598
3050
  }
2599
3051
  console.log(json ? JSON.stringify(report, null, 2) : (0, scan_js_1.formatScanReport)(report));
3052
+ if (wantTrigger) {
3053
+ const harness = adapter.name === "codex" ? "codex" : "claude-code";
3054
+ await handleScanTrigger(root, args, json, harness);
3055
+ }
2600
3056
  }
2601
3057
  break;
2602
3058
  }
package/dist/codex.d.ts CHANGED
@@ -17,4 +17,5 @@ export * from "./adapters/codex/model-mock.js";
17
17
  export * from "./adapters/codex/mock-model.js";
18
18
  export * from "./adapters/codex/driver.js";
19
19
  export * from "./adapters/codex/adapter.js";
20
+ export * from "./adapters/codex/eval.js";
20
21
  //# sourceMappingURL=codex.d.ts.map
package/dist/codex.js CHANGED
@@ -33,4 +33,7 @@ __exportStar(require("./adapters/codex/model-mock.js"), exports);
33
33
  __exportStar(require("./adapters/codex/mock-model.js"), exports);
34
34
  __exportStar(require("./adapters/codex/driver.js"), exports);
35
35
  __exportStar(require("./adapters/codex/adapter.js"), exports);
36
+ // Eval-tier transport (increment 2 — scaffold, pending live-binary validation):
37
+ // parseCodexEvalRun (the ModelOutputParser for `codex exec --json`) + codexEvalRunner.
38
+ __exportStar(require("./adapters/codex/eval.js"), exports);
36
39
  //# sourceMappingURL=codex.js.map
@@ -26,6 +26,7 @@ const node_path_1 = require("node:path");
26
26
  const hash_js_1 = require("./hash.js");
27
27
  const symbols_js_1 = require("./symbols.js");
28
28
  const linters_js_1 = require("./linters.js");
29
+ const tool_contract_js_1 = require("./tool-contract.js");
29
30
  // vigiles's default compile target when a spec names none and no dialect is
30
31
  // injected — a product convention (vigiles emits CLAUDE.md by default), not a
31
32
  // harness dialect. When a dialect IS injected its instructionTargets win.
@@ -680,43 +681,14 @@ function compileSkill(spec, options = {}) {
680
681
  // vigiles does not emit a TOML [agents] block. The Codex dialect still verifies
681
682
  // an agent()'s tool contract (its built-in catalog) — only the OUTPUT renderer
682
683
  // is CC-only here. See research/codex-prototype-findings.md (gaps).
683
- /** Closest known tool by edit distance (≤ 3), for a "did you mean" hint. */
684
- function closestTool(tool, dialect) {
685
- let best = null;
686
- let bestDistance = Infinity;
687
- for (const known of dialect.builtinAgentTools) {
688
- const d = (0, linters_js_1.editDistance)(tool.toLowerCase(), known.toLowerCase());
689
- if (d < bestDistance) {
690
- bestDistance = d;
691
- best = known;
692
- }
693
- }
694
- return bestDistance <= 3 ? best : null;
695
- }
696
- /** Verify a subagent's allowed-tools contract — the rails are real tools. */
684
+ /** Verify a subagent's allowed-tools contract the rails are real tools. The
685
+ * detection lives in the shared `verifyToolContract` detector (one-detector-no-
686
+ * drift: compile + scan + the agent-tool-contract lint rule call the same code). */
697
687
  function validateAgentTools(tools, dialect) {
698
- const never = new Set(dialect.neverAvailableTools);
699
- const errors = [];
700
- for (const tool of tools) {
701
- if (never.has(tool)) {
702
- errors.push({
703
- type: "unknown-tool",
704
- message: `Tool "${tool}" is never available to a subagent — remove it from the tools list.`,
705
- });
706
- continue;
707
- }
708
- if (dialect.builtinAgentTools.includes(tool))
709
- continue;
710
- if (dialect.mcpToolPattern.test(tool))
711
- continue;
712
- const near = closestTool(tool, dialect);
713
- const hint = near ? ` Did you mean "${near}"?` : "";
714
- errors.push({
715
- type: "unknown-tool",
716
- message: `Unknown tool "${tool}" in agent tools — use a built-in tool (${dialect.builtinAgentTools.join(", ")}) or an MCP tool (mcp__server__tool).${hint}`,
717
- });
718
- }
719
- return errors;
688
+ return (0, tool_contract_js_1.verifyToolContract)(tools, dialect).map((issue) => ({
689
+ type: "unknown-tool",
690
+ message: issue.message,
691
+ }));
720
692
  }
721
693
  /** Build the subagent YAML frontmatter (name / description / model / tools). */
722
694
  function renderAgentFrontmatter(spec) {
@@ -0,0 +1,27 @@
1
+ /** A skill identified by name + its trigger-surface description. */
2
+ export interface DescribedSurface {
3
+ readonly name: string;
4
+ readonly description: string;
5
+ }
6
+ export interface DescriptionOverlap {
7
+ readonly a: string;
8
+ readonly b: string;
9
+ /** 0–1, higher = more alike (1 − NCD), rounded to 2 dp. */
10
+ readonly similarity: number;
11
+ readonly message: string;
12
+ }
13
+ /**
14
+ * The NCD cutoff below which two descriptions count as a near-duplicate. 0.2 sits
15
+ * safely under the sweep's most-similar legitimately-distinct pair (0.25), so
16
+ * only basically-identical text is flagged. Exported so a caller / test can see
17
+ * the calibrated value.
18
+ */
19
+ export declare const OVERLAP_NCD_CUTOFF = 0.2;
20
+ /**
21
+ * Find near-duplicate description pairs among `surfaces`. Returns one
22
+ * {@link DescriptionOverlap} per pair whose NCD is below `cutoff`, most-similar
23
+ * first. Pure; pass only the surfaces that actually compete for auto-selection
24
+ * (model-invocable, described) so a user-invoked pair isn't a false alarm.
25
+ */
26
+ export declare function findDescriptionOverlaps(surfaces: readonly DescribedSurface[], cutoff?: number): DescriptionOverlap[];
27
+ //# sourceMappingURL=description-overlap.d.ts.map