vigiles 5.0.1 → 5.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +14 -8
- package/dist/adapters/claude-code/agent-runtime.d.ts +10 -0
- package/dist/adapters/claude-code/agent-runtime.js +15 -29
- package/dist/adapters/claude-code/dialect.js +18 -2
- package/dist/adapters/codex/eval.d.ts +94 -0
- package/dist/adapters/codex/eval.js +227 -0
- package/dist/cli.js +464 -8
- package/dist/codex.d.ts +1 -0
- package/dist/codex.js +3 -0
- package/dist/core/compile.js +8 -36
- package/dist/core/description-overlap.d.ts +27 -0
- package/dist/core/description-overlap.js +53 -0
- package/dist/core/dialect.d.ts +8 -0
- package/dist/core/frontmatter-read.d.ts +25 -0
- package/dist/core/frontmatter-read.js +138 -0
- package/dist/core/hook-events.d.ts +34 -0
- package/dist/core/hook-events.js +48 -0
- package/dist/core/mcp-config.d.ts +20 -0
- package/dist/core/mcp-config.js +40 -0
- package/dist/core/mcp-hook.d.ts +35 -0
- package/dist/core/mcp-hook.js +70 -0
- package/dist/core/mcp-tool.d.ts +50 -0
- package/dist/core/mcp-tool.js +61 -0
- package/dist/core/tool-contract.d.ts +68 -0
- package/dist/core/tool-contract.js +113 -0
- package/dist/core/types.d.ts +89 -0
- package/dist/core/validate.js +22 -0
- package/dist/eval.d.ts +69 -13
- package/dist/eval.js +106 -51
- package/dist/leaderboard.js +61 -3
- package/dist/plugin-loader.d.ts +1 -0
- package/dist/plugin-loader.js +71 -18
- package/dist/scan-behavioral.d.ts +73 -0
- package/dist/scan-behavioral.js +150 -0
- package/dist/scan.d.ts +126 -1
- package/dist/scan.js +559 -40
- package/package.json +1 -1
- package/skills/migrate-to-spec/SKILL.md +0 -2
package/dist/cli.js
CHANGED
|
@@ -20,6 +20,10 @@ const setup_plan_js_1 = require("./setup-plan.js");
|
|
|
20
20
|
const types_js_1 = require("./core/types.js");
|
|
21
21
|
const test_coverage_js_1 = require("./test-coverage.js");
|
|
22
22
|
const scan_js_1 = require("./scan.js");
|
|
23
|
+
const tool_contract_js_1 = require("./core/tool-contract.js");
|
|
24
|
+
const agent_runtime_js_1 = require("./adapters/claude-code/agent-runtime.js");
|
|
25
|
+
const dialect_js_1 = require("./adapters/claude-code/dialect.js");
|
|
26
|
+
const scan_behavioral_js_1 = require("./scan-behavioral.js");
|
|
23
27
|
const adapter_registry_js_1 = require("./adapter-registry.js");
|
|
24
28
|
const skill_harness_js_1 = require("./skill-harness.js");
|
|
25
29
|
const leaderboard_js_1 = require("./leaderboard.js");
|
|
@@ -31,7 +35,7 @@ const generate_schema_js_1 = require("./core/generate-schema.js");
|
|
|
31
35
|
const compose_js_1 = require("./core/compose.js");
|
|
32
36
|
const compile_generator_js_1 = require("./core/compile-generator.js");
|
|
33
37
|
const action_gate_js_1 = require("./action-gate.js");
|
|
34
|
-
const
|
|
38
|
+
const agent_runtime_js_2 = require("./adapters/claude-code/agent-runtime.js");
|
|
35
39
|
const tool_intercept_js_1 = require("./tool-intercept.js");
|
|
36
40
|
const refs_js_1 = require("./core/refs.js");
|
|
37
41
|
const mcp_js_1 = require("./core/mcp.js");
|
|
@@ -633,6 +637,17 @@ function lintExitCode(report) {
|
|
|
633
637
|
report.integrityErrors > 0 ||
|
|
634
638
|
report.coverageErrors > 0 ||
|
|
635
639
|
report.untestedErrors > 0 ||
|
|
640
|
+
report.toolContractErrors > 0 ||
|
|
641
|
+
report.hookEventErrors > 0 ||
|
|
642
|
+
report.frontmatterSchemaErrors > 0 ||
|
|
643
|
+
report.mcpConfigErrors > 0 ||
|
|
644
|
+
report.skillFrontmatterErrors > 0 ||
|
|
645
|
+
report.mcpToolErrors > 0 ||
|
|
646
|
+
report.hookScriptErrors > 0 ||
|
|
647
|
+
report.disallowedToolErrors > 0 ||
|
|
648
|
+
report.descriptionOverlapErrors > 0 ||
|
|
649
|
+
report.frontmatterValidErrors > 0 ||
|
|
650
|
+
report.mcpHookErrors > 0 ||
|
|
636
651
|
report.symbolRefErrors > 0 ||
|
|
637
652
|
report.mcpRefErrors > 0)
|
|
638
653
|
return 2;
|
|
@@ -913,6 +928,39 @@ async function runLint(restArgs, flags, config) {
|
|
|
913
928
|
// eval. Warning by default (a nudge, exit 0); set rules.untested-{skill,agent,
|
|
914
929
|
// hook} to "error" to gate CI. See src/test-coverage.ts and docs/rules/.
|
|
915
930
|
const untested = checkUntestedSurfaces(config, silent);
|
|
931
|
+
// 7c. Agent tool-contract check — cross-reference each subagent's `tools:` rail
|
|
932
|
+
// against the harness catalog (the moat). Off by default unless a severity is
|
|
933
|
+
// configured; warning surfaces a typo/never-available tool, error gates CI.
|
|
934
|
+
const toolContract = checkAgentToolContracts(config, silent);
|
|
935
|
+
// 7d. Hook-event check — a hook registered under an event the harness doesn't
|
|
936
|
+
// define never fires. High-precision (close typos only). Off unless configured.
|
|
937
|
+
const hookEvents = checkHookEvents(config, silent);
|
|
938
|
+
// 7e. Frontmatter-schema check — a skill/agent missing required frontmatter
|
|
939
|
+
// (name; agents also description) won't load/register. High-confidence.
|
|
940
|
+
const frontmatter = checkFrontmatterSchema(config, silent);
|
|
941
|
+
// 7f. MCP-config check — a declared MCP server with no command/url can't start.
|
|
942
|
+
const mcpConfig = checkMcpConfig(config, silent);
|
|
943
|
+
// 7g. Skill-frontmatter — RECOMMEND explicit name/description on skills (a
|
|
944
|
+
// reliable trigger surface). Best-practice nudge; skills load without it.
|
|
945
|
+
const skillFm = checkSkillFrontmatter(config, silent);
|
|
946
|
+
// 7h. MCP tool-resolution — an `mcp__server__tool` in a contract whose server
|
|
947
|
+
// the plugin doesn't declare can't resolve (the MCP half of the tool moat).
|
|
948
|
+
const mcpToolResolves = checkMcpToolResolves(config, silent);
|
|
949
|
+
// 7i. Hook-script existence — a hook command referencing a missing script file
|
|
950
|
+
// never runs (matches Anthropic's own `claude plugin validate`).
|
|
951
|
+
const hookScripts = checkHookScriptExists(config, silent);
|
|
952
|
+
// 7j. Disallowed-tools — a `disallowedTools:` block-list typo blocks nothing
|
|
953
|
+
// (the deny-side mirror of agent-tool-contract; close-typo only).
|
|
954
|
+
const disallowedTools = checkDisallowedTools(config, silent);
|
|
955
|
+
// 7k. Description-overlap — two model-invocable skills with near-identical
|
|
956
|
+
// descriptions collide in the selector (deterministic NCD precision proxy).
|
|
957
|
+
const descriptionOverlap = checkDescriptionOverlap(config, silent);
|
|
958
|
+
// 7l. Frontmatter-valid — a `---` block that isn't valid YAML (warn; js-yaml is
|
|
959
|
+
// stricter than some loaders, so verify before enforcing).
|
|
960
|
+
const frontmatterValid = checkFrontmatterValid(config, silent);
|
|
961
|
+
// 7m. MCP hook-target — a `type: mcp_tool` hook action that's incomplete or
|
|
962
|
+
// targets an undeclared server (the moat applied to the hook surface).
|
|
963
|
+
const mcpHookTargets = checkMcpHookTargets(config, silent);
|
|
916
964
|
// 8. Validate vigiles builder calls inside markdown code blocks. Default
|
|
917
965
|
// is to validate every ref; illustrative blocks opt out via
|
|
918
966
|
// `<!-- vigiles:ignore -->` (single block) or
|
|
@@ -956,6 +1004,28 @@ async function runLint(restArgs, flags, config) {
|
|
|
956
1004
|
orphanCount: orphanReport.orphans.length,
|
|
957
1005
|
untestedSurfaces: untested.untested,
|
|
958
1006
|
untestedErrors: untested.errors,
|
|
1007
|
+
toolContractIssues: toolContract.issues,
|
|
1008
|
+
toolContractErrors: toolContract.errors,
|
|
1009
|
+
hookEventIssues: hookEvents.issues,
|
|
1010
|
+
hookEventErrors: hookEvents.errors,
|
|
1011
|
+
frontmatterSchemaIssues: frontmatter.issues,
|
|
1012
|
+
frontmatterSchemaErrors: frontmatter.errors,
|
|
1013
|
+
mcpConfigIssues: mcpConfig.issues,
|
|
1014
|
+
mcpConfigErrors: mcpConfig.errors,
|
|
1015
|
+
skillFrontmatterIssues: skillFm.issues,
|
|
1016
|
+
skillFrontmatterErrors: skillFm.errors,
|
|
1017
|
+
mcpToolIssues: mcpToolResolves.issues,
|
|
1018
|
+
mcpToolErrors: mcpToolResolves.errors,
|
|
1019
|
+
hookScriptIssues: hookScripts.issues,
|
|
1020
|
+
hookScriptErrors: hookScripts.errors,
|
|
1021
|
+
disallowedToolIssues: disallowedTools.issues,
|
|
1022
|
+
disallowedToolErrors: disallowedTools.errors,
|
|
1023
|
+
descriptionOverlapIssues: descriptionOverlap.issues,
|
|
1024
|
+
descriptionOverlapErrors: descriptionOverlap.errors,
|
|
1025
|
+
frontmatterValidIssues: frontmatterValid.issues,
|
|
1026
|
+
frontmatterValidErrors: frontmatterValid.errors,
|
|
1027
|
+
mcpHookIssues: mcpHookTargets.issues,
|
|
1028
|
+
mcpHookErrors: mcpHookTargets.errors,
|
|
959
1029
|
docRefErrors: docRefReport.errors.length,
|
|
960
1030
|
symbolRefErrors,
|
|
961
1031
|
mcpRefErrors,
|
|
@@ -1931,6 +2001,328 @@ function checkUntestedSurfaces(config, silent) {
|
|
|
1931
2001
|
errors: report.untested.filter((s) => sevFor(s.kind) === "error").length,
|
|
1932
2002
|
};
|
|
1933
2003
|
}
|
|
2004
|
+
/**
|
|
2005
|
+
* Apply the `agent-tool-contract` rule: cross-reference every subagent's `tools:`
|
|
2006
|
+
* rail against the harness tool catalog (the moat — "valid is not true"). Flags
|
|
2007
|
+
* only the HIGH-CONFIDENCE issues (a never-available tool, or a close typo) via
|
|
2008
|
+
* the shared `confidentToolIssues` detector — the same code `scan` and
|
|
2009
|
+
* `compileAgent` use (one-detector-no-drift), so a bare unrecognized tool
|
|
2010
|
+
* (plugin/MCP-provided) is never a false alarm. Warning by default; set
|
|
2011
|
+
* `agent-tool-contract: "error"` to gate CI. Returns the issue + error counts.
|
|
2012
|
+
*/
|
|
2013
|
+
function checkAgentToolContracts(config, silent) {
|
|
2014
|
+
const sev = (0, types_js_1.ruleSeverity)(config?.rules?.["agent-tool-contract"]);
|
|
2015
|
+
if (!sev)
|
|
2016
|
+
return { issues: 0, errors: 0 };
|
|
2017
|
+
const files = (0, glob_1.globSync)(["agents/*.md", ".claude/agents/*.md"], {
|
|
2018
|
+
cwd: process.cwd(),
|
|
2019
|
+
ignore: ["**/*.spec.ts"],
|
|
2020
|
+
});
|
|
2021
|
+
let issues = 0;
|
|
2022
|
+
let printedHeader = false;
|
|
2023
|
+
for (const rel of files.sort()) {
|
|
2024
|
+
let md;
|
|
2025
|
+
try {
|
|
2026
|
+
md = (0, node_fs_1.readFileSync)((0, node_path_1.resolve)(process.cwd(), rel), "utf-8");
|
|
2027
|
+
}
|
|
2028
|
+
catch {
|
|
2029
|
+
continue;
|
|
2030
|
+
}
|
|
2031
|
+
const tools = (0, agent_runtime_js_1.parseAgentTools)(md);
|
|
2032
|
+
if (tools === null)
|
|
2033
|
+
continue; // no contract → inherits all (a different rule)
|
|
2034
|
+
const found = (0, tool_contract_js_1.confidentToolIssues)((0, tool_contract_js_1.verifyToolContract)(tools, dialect_js_1.claudeCodeDialect));
|
|
2035
|
+
if (found.length === 0)
|
|
2036
|
+
continue;
|
|
2037
|
+
issues += found.length;
|
|
2038
|
+
if (!silent) {
|
|
2039
|
+
if (!printedHeader) {
|
|
2040
|
+
console.log("\nAgent tool-contract check:\n");
|
|
2041
|
+
printedHeader = true;
|
|
2042
|
+
}
|
|
2043
|
+
for (const issue of found) {
|
|
2044
|
+
console.log(` ${sev === "error" ? "✗" : "⚠"} ${rel}: ${issue.message}`);
|
|
2045
|
+
ghAnnotate(sev === "error" ? "error" : "warning", issue.message, rel);
|
|
2046
|
+
}
|
|
2047
|
+
}
|
|
2048
|
+
}
|
|
2049
|
+
return { issues, errors: sev === "error" ? issues : 0 };
|
|
2050
|
+
}
|
|
2051
|
+
/**
|
|
2052
|
+
* Apply the `hook-events` rule: flag a hook registered under an event name the
|
|
2053
|
+
* harness doesn't define (a typo → the hook never fires). Reuses `scanPlugin`'s
|
|
2054
|
+
* `hookEventIssues` (the shared detector, high-precision: close typos only, never
|
|
2055
|
+
* a framework/custom event). Warning by default; "error" gates CI.
|
|
2056
|
+
*/
|
|
2057
|
+
function checkHookEvents(config, silent) {
|
|
2058
|
+
const sev = (0, types_js_1.ruleSeverity)(config?.rules?.["hook-events"]);
|
|
2059
|
+
if (!sev)
|
|
2060
|
+
return { issues: 0, errors: 0 };
|
|
2061
|
+
let found;
|
|
2062
|
+
try {
|
|
2063
|
+
found = (0, scan_js_1.scanPlugin)(process.cwd()).hookEventIssues;
|
|
2064
|
+
}
|
|
2065
|
+
catch {
|
|
2066
|
+
return { issues: 0, errors: 0 };
|
|
2067
|
+
}
|
|
2068
|
+
if (found.length > 0 && !silent) {
|
|
2069
|
+
console.log("\nHook-event check:\n");
|
|
2070
|
+
for (const issue of found) {
|
|
2071
|
+
console.log(` ${sev === "error" ? "✗" : "⚠"} ${issue.message}`);
|
|
2072
|
+
ghAnnotate(sev === "error" ? "error" : "warning", issue.message);
|
|
2073
|
+
}
|
|
2074
|
+
}
|
|
2075
|
+
return { issues: found.length, errors: sev === "error" ? found.length : 0 };
|
|
2076
|
+
}
|
|
2077
|
+
/**
|
|
2078
|
+
* Apply the `agent-frontmatter` rule. Two kinds of agent-frontmatter defect, one
|
|
2079
|
+
* rule: (1) a subagent MISSING a required field (`name`/`description`) — it won't
|
|
2080
|
+
* register; (2) a subagent with an INVALID `model:`/`color:` value (a close typo
|
|
2081
|
+
* of a real one) — it silently falls back / is ignored. Reuses `scanPlugin`'s
|
|
2082
|
+
* `frontmatterIssues` + `frontmatterValueIssues`. Warning by default; "error" gates CI.
|
|
2083
|
+
*/
|
|
2084
|
+
function checkFrontmatterSchema(config, silent) {
|
|
2085
|
+
const sev = (0, types_js_1.ruleSeverity)(config?.rules?.["agent-frontmatter"]);
|
|
2086
|
+
if (!sev)
|
|
2087
|
+
return { issues: 0, errors: 0 };
|
|
2088
|
+
let found;
|
|
2089
|
+
try {
|
|
2090
|
+
const r = (0, scan_js_1.scanPlugin)(process.cwd());
|
|
2091
|
+
found = [...r.frontmatterIssues, ...r.frontmatterValueIssues];
|
|
2092
|
+
}
|
|
2093
|
+
catch {
|
|
2094
|
+
return { issues: 0, errors: 0 };
|
|
2095
|
+
}
|
|
2096
|
+
if (found.length > 0 && !silent) {
|
|
2097
|
+
console.log("\nFrontmatter-schema check:\n");
|
|
2098
|
+
for (const issue of found) {
|
|
2099
|
+
console.log(` ${sev === "error" ? "✗" : "⚠"} ${issue.message}`);
|
|
2100
|
+
ghAnnotate(sev === "error" ? "error" : "warning", issue.message, issue.path);
|
|
2101
|
+
}
|
|
2102
|
+
}
|
|
2103
|
+
return { issues: found.length, errors: sev === "error" ? found.length : 0 };
|
|
2104
|
+
}
|
|
2105
|
+
/**
|
|
2106
|
+
* Apply the `skill-frontmatter` rule: RECOMMEND (not require) that a SKILL.md
|
|
2107
|
+
* declares an explicit `name` + `description` rather than relying on the
|
|
2108
|
+
* dir-name / first-paragraph fallbacks — a more reliable trigger surface. The
|
|
2109
|
+
* skill still LOADS without them, so this is a best-practice nudge: warn by
|
|
2110
|
+
* default; set "error" to enforce it on your own skills. Reuses `scanPlugin`'s
|
|
2111
|
+
* `skillMetaIssues`.
|
|
2112
|
+
*/
|
|
2113
|
+
function checkSkillFrontmatter(config, silent) {
|
|
2114
|
+
const sev = (0, types_js_1.ruleSeverity)(config?.rules?.["skill-frontmatter"]);
|
|
2115
|
+
if (!sev)
|
|
2116
|
+
return { issues: 0, errors: 0 };
|
|
2117
|
+
let found;
|
|
2118
|
+
try {
|
|
2119
|
+
found = (0, scan_js_1.scanPlugin)(process.cwd()).skillMetaIssues;
|
|
2120
|
+
}
|
|
2121
|
+
catch {
|
|
2122
|
+
return { issues: 0, errors: 0 };
|
|
2123
|
+
}
|
|
2124
|
+
if (found.length > 0 && !silent) {
|
|
2125
|
+
console.log("\nSkill-frontmatter check:\n");
|
|
2126
|
+
for (const issue of found) {
|
|
2127
|
+
console.log(` ${sev === "error" ? "✗" : "⚠"} ${issue.message}`);
|
|
2128
|
+
ghAnnotate(sev === "error" ? "error" : "warning", issue.message, issue.path);
|
|
2129
|
+
}
|
|
2130
|
+
}
|
|
2131
|
+
return { issues: found.length, errors: sev === "error" ? found.length : 0 };
|
|
2132
|
+
}
|
|
2133
|
+
/**
|
|
2134
|
+
* Apply the `mcp-config` rule: a declared MCP server with neither a `command`
|
|
2135
|
+
* (stdio) nor a `url` (http/sse) can't start. Reuses `scanPlugin`'s `mcpIssues`.
|
|
2136
|
+
* Warning by default; "error" gates CI.
|
|
2137
|
+
*/
|
|
2138
|
+
function checkMcpConfig(config, silent) {
|
|
2139
|
+
const sev = (0, types_js_1.ruleSeverity)(config?.rules?.["mcp-config"]);
|
|
2140
|
+
if (!sev)
|
|
2141
|
+
return { issues: 0, errors: 0 };
|
|
2142
|
+
let found;
|
|
2143
|
+
try {
|
|
2144
|
+
found = (0, scan_js_1.scanPlugin)(process.cwd()).mcpIssues;
|
|
2145
|
+
}
|
|
2146
|
+
catch {
|
|
2147
|
+
return { issues: 0, errors: 0 };
|
|
2148
|
+
}
|
|
2149
|
+
if (found.length > 0 && !silent) {
|
|
2150
|
+
console.log("\nMCP-config check:\n");
|
|
2151
|
+
for (const issue of found) {
|
|
2152
|
+
console.log(` ${sev === "error" ? "✗" : "⚠"} ${issue.message}`);
|
|
2153
|
+
ghAnnotate(sev === "error" ? "error" : "warning", issue.message);
|
|
2154
|
+
}
|
|
2155
|
+
}
|
|
2156
|
+
return { issues: found.length, errors: sev === "error" ? found.length : 0 };
|
|
2157
|
+
}
|
|
2158
|
+
/**
|
|
2159
|
+
* Apply the `disallowed-tools-contract` rule: a subagent's `disallowedTools:`
|
|
2160
|
+
* block-list entry that's a close typo of a real tool blocks NOTHING — the tool
|
|
2161
|
+
* it was meant to deny stays available, silently. Reuses `scanPlugin`'s per-agent
|
|
2162
|
+
* `disallowedToolIssues` (close-typo only — high-precision). Warning by default;
|
|
2163
|
+
* "error" gates CI.
|
|
2164
|
+
*/
|
|
2165
|
+
function checkDisallowedTools(config, silent) {
|
|
2166
|
+
const sev = (0, types_js_1.ruleSeverity)(config?.rules?.["disallowed-tools-contract"]);
|
|
2167
|
+
if (!sev)
|
|
2168
|
+
return { issues: 0, errors: 0 };
|
|
2169
|
+
let found;
|
|
2170
|
+
try {
|
|
2171
|
+
found = (0, scan_js_1.scanPlugin)(process.cwd()).agents.flatMap((a) => a.disallowedToolIssues.map((i) => ({ message: i.message, path: a.path })));
|
|
2172
|
+
}
|
|
2173
|
+
catch {
|
|
2174
|
+
return { issues: 0, errors: 0 };
|
|
2175
|
+
}
|
|
2176
|
+
if (found.length > 0 && !silent) {
|
|
2177
|
+
console.log("\nDisallowed-tools check:\n");
|
|
2178
|
+
for (const issue of found) {
|
|
2179
|
+
console.log(` ${sev === "error" ? "✗" : "⚠"} ${issue.path}: ${issue.message}`);
|
|
2180
|
+
ghAnnotate(sev === "error" ? "error" : "warning", issue.message, issue.path);
|
|
2181
|
+
}
|
|
2182
|
+
}
|
|
2183
|
+
return { issues: found.length, errors: sev === "error" ? found.length : 0 };
|
|
2184
|
+
}
|
|
2185
|
+
/**
|
|
2186
|
+
* Apply the `frontmatter-valid` rule: a skill/agent `---` block that EXISTS but
|
|
2187
|
+
* isn't valid YAML — fields may not parse as intended. Reuses `scanPlugin`'s
|
|
2188
|
+
* `malformedFrontmatter`. HONEST caveat (see docs/rules/frontmatter-valid.md):
|
|
2189
|
+
* js-yaml is stricter than some loaders, so a one-line `description:` with a
|
|
2190
|
+
* colon / `<example>` is flagged though it may still load — hence WARN by default
|
|
2191
|
+
* (verify before setting "error").
|
|
2192
|
+
*/
|
|
2193
|
+
function checkFrontmatterValid(config, silent) {
|
|
2194
|
+
const sev = (0, types_js_1.ruleSeverity)(config?.rules?.["frontmatter-valid"]);
|
|
2195
|
+
if (!sev)
|
|
2196
|
+
return { issues: 0, errors: 0 };
|
|
2197
|
+
let found;
|
|
2198
|
+
try {
|
|
2199
|
+
found = (0, scan_js_1.scanPlugin)(process.cwd()).malformedFrontmatter;
|
|
2200
|
+
}
|
|
2201
|
+
catch {
|
|
2202
|
+
return { issues: 0, errors: 0 };
|
|
2203
|
+
}
|
|
2204
|
+
if (found.length > 0 && !silent) {
|
|
2205
|
+
console.log("\nFrontmatter-validity check:\n");
|
|
2206
|
+
for (const issue of found) {
|
|
2207
|
+
console.log(` ${sev === "error" ? "✗" : "⚠"} ${issue.message}`);
|
|
2208
|
+
ghAnnotate(sev === "error" ? "error" : "warning", issue.message, issue.path);
|
|
2209
|
+
}
|
|
2210
|
+
}
|
|
2211
|
+
return { issues: found.length, errors: sev === "error" ? found.length : 0 };
|
|
2212
|
+
}
|
|
2213
|
+
/**
|
|
2214
|
+
* Apply the `description-overlap` rule: two model-invocable skills with
|
|
2215
|
+
* near-identical descriptions collide in the selector — the wrong one fires. A
|
|
2216
|
+
* deterministic NCD proxy for a `--trigger`-class precision bug. Reuses
|
|
2217
|
+
* `scanPlugin`'s `descriptionOverlaps` (calibrated FP-safe: only basically
|
|
2218
|
+
* identical text). Warning by default; "error" gates CI.
|
|
2219
|
+
*/
|
|
2220
|
+
function checkDescriptionOverlap(config, silent) {
|
|
2221
|
+
const sev = (0, types_js_1.ruleSeverity)(config?.rules?.["description-overlap"]);
|
|
2222
|
+
if (!sev)
|
|
2223
|
+
return { issues: 0, errors: 0 };
|
|
2224
|
+
let found;
|
|
2225
|
+
try {
|
|
2226
|
+
found = (0, scan_js_1.scanPlugin)(process.cwd()).descriptionOverlaps;
|
|
2227
|
+
}
|
|
2228
|
+
catch {
|
|
2229
|
+
return { issues: 0, errors: 0 };
|
|
2230
|
+
}
|
|
2231
|
+
if (found.length > 0 && !silent) {
|
|
2232
|
+
console.log("\nDescription-overlap check:\n");
|
|
2233
|
+
for (const issue of found) {
|
|
2234
|
+
console.log(` ${sev === "error" ? "✗" : "⚠"} ${issue.message}`);
|
|
2235
|
+
ghAnnotate(sev === "error" ? "error" : "warning", issue.message);
|
|
2236
|
+
}
|
|
2237
|
+
}
|
|
2238
|
+
return { issues: found.length, errors: sev === "error" ? found.length : 0 };
|
|
2239
|
+
}
|
|
2240
|
+
/**
|
|
2241
|
+
* Apply the `mcp-hook-target-resolves` rule: a `type: "mcp_tool"` hook action
|
|
2242
|
+
* that's incomplete (no `server`/`tool`) or targets a server the plugin doesn't
|
|
2243
|
+
* declare — the hook silently never dispatches. Reuses `scanPlugin`'s
|
|
2244
|
+
* `mcpHookIssues` (high-precision: declared-set gated, built-ins allowlisted).
|
|
2245
|
+
* Warning by default; "error" gates CI.
|
|
2246
|
+
*/
|
|
2247
|
+
function checkMcpHookTargets(config, silent) {
|
|
2248
|
+
const sev = (0, types_js_1.ruleSeverity)(config?.rules?.["mcp-hook-target-resolves"]);
|
|
2249
|
+
if (!sev)
|
|
2250
|
+
return { issues: 0, errors: 0 };
|
|
2251
|
+
let found;
|
|
2252
|
+
try {
|
|
2253
|
+
found = (0, scan_js_1.scanPlugin)(process.cwd()).mcpHookIssues;
|
|
2254
|
+
}
|
|
2255
|
+
catch {
|
|
2256
|
+
return { issues: 0, errors: 0 };
|
|
2257
|
+
}
|
|
2258
|
+
if (found.length > 0 && !silent) {
|
|
2259
|
+
console.log("\nMCP hook-target check:\n");
|
|
2260
|
+
for (const issue of found) {
|
|
2261
|
+
console.log(` ${sev === "error" ? "✗" : "⚠"} ${issue.message}`);
|
|
2262
|
+
ghAnnotate(sev === "error" ? "error" : "warning", issue.message);
|
|
2263
|
+
}
|
|
2264
|
+
}
|
|
2265
|
+
return { issues: found.length, errors: sev === "error" ? found.length : 0 };
|
|
2266
|
+
}
|
|
2267
|
+
/**
|
|
2268
|
+
* Apply the `hook-script-exists` rule: a hook command references a script file
|
|
2269
|
+
* that doesn't exist on disk (with `${CLAUDE_PLUGIN_ROOT}` resolved) → the hook
|
|
2270
|
+
* silently never runs. Reuses `scanPlugin`'s `hooks` (status "missing"); the
|
|
2271
|
+
* shared resolver already excludes the FP-prone cases (unresolved vars,
|
|
2272
|
+
* existence-guarded one-liners, inline commands). Matches Anthropic's own
|
|
2273
|
+
* `claude plugin validate`. Warning by default; "error" gates CI.
|
|
2274
|
+
*/
|
|
2275
|
+
function checkHookScriptExists(config, silent) {
|
|
2276
|
+
const sev = (0, types_js_1.ruleSeverity)(config?.rules?.["hook-script-exists"]);
|
|
2277
|
+
if (!sev)
|
|
2278
|
+
return { issues: 0, errors: 0 };
|
|
2279
|
+
let missing;
|
|
2280
|
+
try {
|
|
2281
|
+
missing = (0, scan_js_1.scanPlugin)(process.cwd()).hooks.filter((h) => h.status === "missing");
|
|
2282
|
+
}
|
|
2283
|
+
catch {
|
|
2284
|
+
return { issues: 0, errors: 0 };
|
|
2285
|
+
}
|
|
2286
|
+
if (missing.length > 0 && !silent) {
|
|
2287
|
+
console.log("\nHook-script existence check:\n");
|
|
2288
|
+
for (const h of missing) {
|
|
2289
|
+
const msg = `hook script "${h.script}" is referenced but missing — the hook never runs.`;
|
|
2290
|
+
console.log(` ${sev === "error" ? "✗" : "⚠"} ${msg}`);
|
|
2291
|
+
ghAnnotate(sev === "error" ? "error" : "warning", msg);
|
|
2292
|
+
}
|
|
2293
|
+
}
|
|
2294
|
+
return {
|
|
2295
|
+
issues: missing.length,
|
|
2296
|
+
errors: sev === "error" ? missing.length : 0,
|
|
2297
|
+
};
|
|
2298
|
+
}
|
|
2299
|
+
/**
|
|
2300
|
+
* Apply the `mcp-tool-resolves` rule: an `mcp__server__tool` in a subagent's
|
|
2301
|
+
* contract whose server isn't in the plugin's declared `mcpServers` can't resolve
|
|
2302
|
+
* (the MCP half of the tool moat). Reuses `scanPlugin`'s per-agent `mcpToolIssues`
|
|
2303
|
+
* — high-precision (gated on a declared set, built-ins allowlisted, the
|
|
2304
|
+
* plugin-namespaced form skipped). Warning by default; "error" gates CI.
|
|
2305
|
+
*/
|
|
2306
|
+
function checkMcpToolResolves(config, silent) {
|
|
2307
|
+
const sev = (0, types_js_1.ruleSeverity)(config?.rules?.["mcp-tool-resolves"]);
|
|
2308
|
+
if (!sev)
|
|
2309
|
+
return { issues: 0, errors: 0 };
|
|
2310
|
+
let found;
|
|
2311
|
+
try {
|
|
2312
|
+
found = (0, scan_js_1.scanPlugin)(process.cwd()).agents.flatMap((a) => a.mcpToolIssues.map((i) => ({ message: i.message, path: a.path })));
|
|
2313
|
+
}
|
|
2314
|
+
catch {
|
|
2315
|
+
return { issues: 0, errors: 0 };
|
|
2316
|
+
}
|
|
2317
|
+
if (found.length > 0 && !silent) {
|
|
2318
|
+
console.log("\nMCP tool-resolution check:\n");
|
|
2319
|
+
for (const issue of found) {
|
|
2320
|
+
console.log(` ${sev === "error" ? "✗" : "⚠"} ${issue.path}: ${issue.message}`);
|
|
2321
|
+
ghAnnotate(sev === "error" ? "error" : "warning", issue.message, issue.path);
|
|
2322
|
+
}
|
|
2323
|
+
}
|
|
2324
|
+
return { issues: found.length, errors: sev === "error" ? found.length : 0 };
|
|
2325
|
+
}
|
|
1934
2326
|
/**
|
|
1935
2327
|
* Apply the configured coverage thresholds. Returns the number of failing
|
|
1936
2328
|
* thresholds (so the lint can fail CI when severity is "error").
|
|
@@ -2008,6 +2400,43 @@ function findInstructionFiles(restArgs) {
|
|
|
2008
2400
|
}
|
|
2009
2401
|
return files;
|
|
2010
2402
|
}
|
|
2403
|
+
/** Value of a `--flag=value` arg (the `=` form, so it never collides with a positional). */
|
|
2404
|
+
function flagValue(args, name) {
|
|
2405
|
+
return args.find((a) => a.startsWith(`${name}=`))?.slice(name.length + 1);
|
|
2406
|
+
}
|
|
2407
|
+
/**
|
|
2408
|
+
* The `scan --trigger` behavioral column: load the author-supplied per-skill
|
|
2409
|
+
* prompt sets, probe the plugin's model-invocable skills, print the column.
|
|
2410
|
+
* Model-gated and opt-in — the structural scan above stays deterministic.
|
|
2411
|
+
*/
|
|
2412
|
+
async function handleScanTrigger(root, args, json, harness) {
|
|
2413
|
+
const promptsPath = flagValue(args, "--prompts");
|
|
2414
|
+
if (!promptsPath) {
|
|
2415
|
+
console.error("scan --trigger needs --prompts=<file.json> (a map of skill name → { prompts, irrelevant }).");
|
|
2416
|
+
process.exitCode = 2;
|
|
2417
|
+
return;
|
|
2418
|
+
}
|
|
2419
|
+
let promptSet;
|
|
2420
|
+
try {
|
|
2421
|
+
promptSet = JSON.parse((0, node_fs_1.readFileSync)((0, node_path_1.resolve)(promptsPath), "utf-8"));
|
|
2422
|
+
}
|
|
2423
|
+
catch (e) {
|
|
2424
|
+
console.error(`scan --trigger: could not read --prompts file "${promptsPath}": ${e instanceof Error ? e.message : String(e)}`);
|
|
2425
|
+
process.exitCode = 2;
|
|
2426
|
+
return;
|
|
2427
|
+
}
|
|
2428
|
+
const concurrencyRaw = flagValue(args, "--concurrency");
|
|
2429
|
+
const minPromptsRaw = flagValue(args, "--min-prompts");
|
|
2430
|
+
const report = await (0, scan_behavioral_js_1.probePluginTriggers)(root, promptSet, {
|
|
2431
|
+
concurrency: concurrencyRaw ? Number(concurrencyRaw) : undefined,
|
|
2432
|
+
minPrompts: minPromptsRaw ? Number(minPromptsRaw) : undefined,
|
|
2433
|
+
model: flagValue(args, "--model"),
|
|
2434
|
+
harness,
|
|
2435
|
+
});
|
|
2436
|
+
console.log(json
|
|
2437
|
+
? JSON.stringify(report, null, 2)
|
|
2438
|
+
: `\n${(0, scan_behavioral_js_1.formatBehavioralReport)(report)}`);
|
|
2439
|
+
}
|
|
2011
2440
|
function handleGenerateTypes(args, restArgs) {
|
|
2012
2441
|
const checkOnly = args.includes("--check");
|
|
2013
2442
|
const outPath = restArgs[0] ?? ".vigiles/generated.d.ts";
|
|
@@ -2295,7 +2724,7 @@ function agentHookCommand() {
|
|
|
2295
2724
|
}
|
|
2296
2725
|
if (!tool)
|
|
2297
2726
|
return;
|
|
2298
|
-
const decision = (0,
|
|
2727
|
+
const decision = (0, agent_runtime_js_2.evaluatePreToolUse)(process.cwd(), tool);
|
|
2299
2728
|
if (!decision.allow) {
|
|
2300
2729
|
console.error(decision.message);
|
|
2301
2730
|
process.exit(2);
|
|
@@ -2331,7 +2760,7 @@ function agentStartCommand(target) {
|
|
|
2331
2760
|
console.error("Usage: vigiles agent-start <agents/<name>.md>");
|
|
2332
2761
|
process.exit(2);
|
|
2333
2762
|
}
|
|
2334
|
-
(0,
|
|
2763
|
+
(0, agent_runtime_js_2.setActiveAgent)(process.cwd(), target);
|
|
2335
2764
|
console.log(`Active agent: ${target}`);
|
|
2336
2765
|
}
|
|
2337
2766
|
/** Dispatch the skill-runtime subcommands. Returns false if unrecognized. */
|
|
@@ -2353,7 +2782,7 @@ function handleSkillCommand(command, restArgs) {
|
|
|
2353
2782
|
agentStartCommand(restArgs[0]);
|
|
2354
2783
|
return true;
|
|
2355
2784
|
case "agent-done":
|
|
2356
|
-
(0,
|
|
2785
|
+
(0, agent_runtime_js_2.clearActiveAgent)(process.cwd());
|
|
2357
2786
|
return true;
|
|
2358
2787
|
case "agent-hook":
|
|
2359
2788
|
agentHookCommand();
|
|
@@ -2574,13 +3003,36 @@ async function main() {
|
|
|
2574
3003
|
case "scan": {
|
|
2575
3004
|
const dirs = restArgs.length > 0 ? restArgs : ["."];
|
|
2576
3005
|
const json = args.includes("--json");
|
|
2577
|
-
|
|
3006
|
+
// A single dir that's a marketplace (e.g. wshobson/agents' 80+ plugins
|
|
3007
|
+
// under one marketplace.json) expands into its members and ranks them.
|
|
3008
|
+
const market = dirs.length === 1 ? (0, scan_js_1.inspectMarketplace)((0, node_path_1.resolve)(dirs[0])) : null;
|
|
3009
|
+
const targets = market && market.onDisk.length > 0 ? [...market.onDisk] : dirs;
|
|
3010
|
+
const wantTrigger = args.includes("--trigger");
|
|
3011
|
+
if (market && market.onDisk.length === 0 && market.total > 0) {
|
|
3012
|
+
// A CURATED marketplace — every member is an external git/url plugin, so
|
|
3013
|
+
// there's nothing on disk to scan. Say so honestly instead of falling
|
|
3014
|
+
// through to a misleading "empty machine / no structural issues" report
|
|
3015
|
+
// (obra/superpowers-marketplace, anthropics/claude-plugins-community).
|
|
3016
|
+
if (json) {
|
|
3017
|
+
console.log(JSON.stringify(market, null, 2));
|
|
3018
|
+
}
|
|
3019
|
+
else {
|
|
3020
|
+
console.log(`Marketplace "${market.name}": ${String(market.total)} plugin(s), all external ` +
|
|
3021
|
+
`(url/git sources, not on disk).\n` +
|
|
3022
|
+
`Nothing to scan here — clone a member plugin and scan that, or scan a ` +
|
|
3023
|
+
`marketplace that vendors its plugins in-tree.`);
|
|
3024
|
+
}
|
|
3025
|
+
}
|
|
3026
|
+
else if (targets.length > 1) {
|
|
2578
3027
|
// Multiple targets → rank them (the leaderboard engine).
|
|
2579
|
-
const scores = (0, leaderboard_js_1.rankPlugins)(
|
|
3028
|
+
const scores = (0, leaderboard_js_1.rankPlugins)(targets);
|
|
2580
3029
|
console.log(json ? JSON.stringify(scores, null, 2) : (0, leaderboard_js_1.formatLeaderboard)(scores));
|
|
3030
|
+
if (wantTrigger) {
|
|
3031
|
+
console.log("\n⚠ --trigger (behavioral column) runs per single plugin; not yet wired into the leaderboard. Scan one plugin dir to probe it.");
|
|
3032
|
+
}
|
|
2581
3033
|
}
|
|
2582
3034
|
else {
|
|
2583
|
-
const root = (0, node_path_1.resolve)(
|
|
3035
|
+
const root = (0, node_path_1.resolve)(targets[0]);
|
|
2584
3036
|
const harnessFlag = args
|
|
2585
3037
|
.find((a) => a.startsWith("--harness="))
|
|
2586
3038
|
?.slice("--harness=".length);
|
|
@@ -2588,7 +3040,7 @@ async function main() {
|
|
|
2588
3040
|
const adapter = harnessFlag
|
|
2589
3041
|
? (0, adapter_registry_js_1.resolveAdapter)(root, harnessFlag)
|
|
2590
3042
|
: det.adapter;
|
|
2591
|
-
const report = (0, scan_js_1.scanPlugin)(
|
|
3043
|
+
const report = (0, scan_js_1.scanPlugin)(targets[0], adapter.layout, adapter.dialect);
|
|
2592
3044
|
if (!json) {
|
|
2593
3045
|
console.log(`Detected harness: ${adapter.name}`);
|
|
2594
3046
|
if (!harnessFlag && det.ambiguousWith.length > 0) {
|
|
@@ -2597,6 +3049,10 @@ async function main() {
|
|
|
2597
3049
|
console.log("");
|
|
2598
3050
|
}
|
|
2599
3051
|
console.log(json ? JSON.stringify(report, null, 2) : (0, scan_js_1.formatScanReport)(report));
|
|
3052
|
+
if (wantTrigger) {
|
|
3053
|
+
const harness = adapter.name === "codex" ? "codex" : "claude-code";
|
|
3054
|
+
await handleScanTrigger(root, args, json, harness);
|
|
3055
|
+
}
|
|
2600
3056
|
}
|
|
2601
3057
|
break;
|
|
2602
3058
|
}
|
package/dist/codex.d.ts
CHANGED
|
@@ -17,4 +17,5 @@ export * from "./adapters/codex/model-mock.js";
|
|
|
17
17
|
export * from "./adapters/codex/mock-model.js";
|
|
18
18
|
export * from "./adapters/codex/driver.js";
|
|
19
19
|
export * from "./adapters/codex/adapter.js";
|
|
20
|
+
export * from "./adapters/codex/eval.js";
|
|
20
21
|
//# sourceMappingURL=codex.d.ts.map
|
package/dist/codex.js
CHANGED
|
@@ -33,4 +33,7 @@ __exportStar(require("./adapters/codex/model-mock.js"), exports);
|
|
|
33
33
|
__exportStar(require("./adapters/codex/mock-model.js"), exports);
|
|
34
34
|
__exportStar(require("./adapters/codex/driver.js"), exports);
|
|
35
35
|
__exportStar(require("./adapters/codex/adapter.js"), exports);
|
|
36
|
+
// Eval-tier transport (increment 2 — scaffold, pending live-binary validation):
|
|
37
|
+
// parseCodexEvalRun (the ModelOutputParser for `codex exec --json`) + codexEvalRunner.
|
|
38
|
+
__exportStar(require("./adapters/codex/eval.js"), exports);
|
|
36
39
|
//# sourceMappingURL=codex.js.map
|
package/dist/core/compile.js
CHANGED
|
@@ -26,6 +26,7 @@ const node_path_1 = require("node:path");
|
|
|
26
26
|
const hash_js_1 = require("./hash.js");
|
|
27
27
|
const symbols_js_1 = require("./symbols.js");
|
|
28
28
|
const linters_js_1 = require("./linters.js");
|
|
29
|
+
const tool_contract_js_1 = require("./tool-contract.js");
|
|
29
30
|
// vigiles's default compile target when a spec names none and no dialect is
|
|
30
31
|
// injected — a product convention (vigiles emits CLAUDE.md by default), not a
|
|
31
32
|
// harness dialect. When a dialect IS injected its instructionTargets win.
|
|
@@ -680,43 +681,14 @@ function compileSkill(spec, options = {}) {
|
|
|
680
681
|
// vigiles does not emit a TOML [agents] block. The Codex dialect still verifies
|
|
681
682
|
// an agent()'s tool contract (its built-in catalog) — only the OUTPUT renderer
|
|
682
683
|
// is CC-only here. See research/codex-prototype-findings.md (gaps).
|
|
683
|
-
/**
|
|
684
|
-
|
|
685
|
-
|
|
686
|
-
let bestDistance = Infinity;
|
|
687
|
-
for (const known of dialect.builtinAgentTools) {
|
|
688
|
-
const d = (0, linters_js_1.editDistance)(tool.toLowerCase(), known.toLowerCase());
|
|
689
|
-
if (d < bestDistance) {
|
|
690
|
-
bestDistance = d;
|
|
691
|
-
best = known;
|
|
692
|
-
}
|
|
693
|
-
}
|
|
694
|
-
return bestDistance <= 3 ? best : null;
|
|
695
|
-
}
|
|
696
|
-
/** Verify a subagent's allowed-tools contract — the rails are real tools. */
|
|
684
|
+
/** Verify a subagent's allowed-tools contract — the rails are real tools. The
|
|
685
|
+
* detection lives in the shared `verifyToolContract` detector (one-detector-no-
|
|
686
|
+
* drift: compile + scan + the agent-tool-contract lint rule call the same code). */
|
|
697
687
|
function validateAgentTools(tools, dialect) {
|
|
698
|
-
|
|
699
|
-
|
|
700
|
-
|
|
701
|
-
|
|
702
|
-
errors.push({
|
|
703
|
-
type: "unknown-tool",
|
|
704
|
-
message: `Tool "${tool}" is never available to a subagent — remove it from the tools list.`,
|
|
705
|
-
});
|
|
706
|
-
continue;
|
|
707
|
-
}
|
|
708
|
-
if (dialect.builtinAgentTools.includes(tool))
|
|
709
|
-
continue;
|
|
710
|
-
if (dialect.mcpToolPattern.test(tool))
|
|
711
|
-
continue;
|
|
712
|
-
const near = closestTool(tool, dialect);
|
|
713
|
-
const hint = near ? ` Did you mean "${near}"?` : "";
|
|
714
|
-
errors.push({
|
|
715
|
-
type: "unknown-tool",
|
|
716
|
-
message: `Unknown tool "${tool}" in agent tools — use a built-in tool (${dialect.builtinAgentTools.join(", ")}) or an MCP tool (mcp__server__tool).${hint}`,
|
|
717
|
-
});
|
|
718
|
-
}
|
|
719
|
-
return errors;
|
|
688
|
+
return (0, tool_contract_js_1.verifyToolContract)(tools, dialect).map((issue) => ({
|
|
689
|
+
type: "unknown-tool",
|
|
690
|
+
message: issue.message,
|
|
691
|
+
}));
|
|
720
692
|
}
|
|
721
693
|
/** Build the subagent YAML frontmatter (name / description / model / tools). */
|
|
722
694
|
function renderAgentFrontmatter(spec) {
|
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
/** A skill identified by name + its trigger-surface description. */
|
|
2
|
+
export interface DescribedSurface {
|
|
3
|
+
readonly name: string;
|
|
4
|
+
readonly description: string;
|
|
5
|
+
}
|
|
6
|
+
export interface DescriptionOverlap {
|
|
7
|
+
readonly a: string;
|
|
8
|
+
readonly b: string;
|
|
9
|
+
/** 0–1, higher = more alike (1 − NCD), rounded to 2 dp. */
|
|
10
|
+
readonly similarity: number;
|
|
11
|
+
readonly message: string;
|
|
12
|
+
}
|
|
13
|
+
/**
|
|
14
|
+
* The NCD cutoff below which two descriptions count as a near-duplicate. 0.2 sits
|
|
15
|
+
* safely under the sweep's most-similar legitimately-distinct pair (0.25), so
|
|
16
|
+
* only basically-identical text is flagged. Exported so a caller / test can see
|
|
17
|
+
* the calibrated value.
|
|
18
|
+
*/
|
|
19
|
+
export declare const OVERLAP_NCD_CUTOFF = 0.2;
|
|
20
|
+
/**
|
|
21
|
+
* Find near-duplicate description pairs among `surfaces`. Returns one
|
|
22
|
+
* {@link DescriptionOverlap} per pair whose NCD is below `cutoff`, most-similar
|
|
23
|
+
* first. Pure; pass only the surfaces that actually compete for auto-selection
|
|
24
|
+
* (model-invocable, described) so a user-invoked pair isn't a false alarm.
|
|
25
|
+
*/
|
|
26
|
+
export declare function findDescriptionOverlaps(surfaces: readonly DescribedSurface[], cutoff?: number): DescriptionOverlap[];
|
|
27
|
+
//# sourceMappingURL=description-overlap.d.ts.map
|