vigiles 2.6.0 → 3.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude-plugin/plugin.json +9 -0
- package/README.md +74 -129
- package/action.yml +143 -7
- package/dist/action-gate.d.ts +1 -1
- package/dist/action-gate.js +1 -1
- package/dist/adapter-conformance.d.ts +30 -0
- package/dist/adapter-conformance.js +153 -0
- package/dist/adapter-registry.d.ts +42 -0
- package/dist/adapter-registry.js +55 -0
- package/dist/adapter.d.ts +26 -0
- package/dist/adapter.js +16 -0
- package/dist/adapters/claude-code/adapter.d.ts +3 -0
- package/dist/adapters/claude-code/adapter.js +46 -0
- package/dist/{agent-result.d.ts → adapters/claude-code/agent-result.d.ts} +1 -1
- package/dist/adapters/claude-code/dialect.d.ts +13 -0
- package/dist/adapters/claude-code/dialect.js +51 -0
- package/dist/adapters/claude-code/egress-entry.d.ts +2 -0
- package/dist/adapters/claude-code/egress-entry.js +115 -0
- package/dist/adapters/claude-code/egress.d.ts +114 -0
- package/dist/adapters/claude-code/egress.js +276 -0
- package/dist/{eval-cache.d.ts → adapters/claude-code/eval-cache.d.ts} +1 -1
- package/dist/{eval-cache.js → adapters/claude-code/eval-cache.js} +1 -1
- package/dist/{eval.d.ts → adapters/claude-code/eval.d.ts} +28 -4
- package/dist/{eval.js → adapters/claude-code/eval.js} +61 -25
- package/dist/{harness-test.d.ts → adapters/claude-code/harness-test.d.ts} +33 -37
- package/dist/{harness-test.js → adapters/claude-code/harness-test.js} +124 -43
- package/dist/adapters/claude-code/hook-protocol.d.ts +10 -0
- package/dist/adapters/claude-code/hook-protocol.js +10 -0
- package/dist/adapters/claude-code/layout.d.ts +8 -0
- package/dist/adapters/claude-code/layout.js +18 -0
- package/dist/{mock-model.d.ts → adapters/claude-code/mock-model.d.ts} +2 -24
- package/dist/adapters/claude-code/model-mock.d.ts +11 -0
- package/dist/adapters/claude-code/model-mock.js +10 -0
- package/dist/adapters/claude-code/plugin-loader.d.ts +25 -0
- package/dist/adapters/claude-code/plugin-loader.js +19 -0
- package/dist/{run-hook.d.ts → adapters/claude-code/run-hook.d.ts} +50 -3
- package/dist/{run-hook.js → adapters/claude-code/run-hook.js} +178 -18
- package/dist/adapters/claude-code/run-scripts.d.ts +52 -0
- package/dist/adapters/claude-code/run-scripts.js +150 -0
- package/dist/adapters/claude-code/runtime.d.ts +16 -0
- package/dist/adapters/claude-code/runtime.js +39 -0
- package/dist/{sandbox.d.ts → adapters/claude-code/sandbox.d.ts} +10 -0
- package/dist/{sandbox.js → adapters/claude-code/sandbox.js} +5 -3
- package/dist/{skill-driver.d.ts → adapters/claude-code/skill-driver.d.ts} +1 -1
- package/dist/adapters/codex/adapter.d.ts +3 -0
- package/dist/adapters/codex/adapter.js +49 -0
- package/dist/adapters/codex/dialect.d.ts +10 -0
- package/dist/adapters/codex/dialect.js +30 -0
- package/dist/adapters/codex/driver.d.ts +20 -0
- package/dist/adapters/codex/driver.js +89 -0
- package/dist/adapters/codex/hook-protocol.d.ts +10 -0
- package/dist/adapters/codex/hook-protocol.js +18 -0
- package/dist/adapters/codex/layout.d.ts +16 -0
- package/dist/adapters/codex/layout.js +18 -0
- package/dist/adapters/codex/mock-model.d.ts +52 -0
- package/dist/adapters/codex/mock-model.js +210 -0
- package/dist/adapters/codex/model-mock.d.ts +11 -0
- package/dist/adapters/codex/model-mock.js +10 -0
- package/dist/adapters/codex/runtime.d.ts +38 -0
- package/dist/adapters/codex/runtime.js +62 -0
- package/dist/adapters/opencode/adapter.d.ts +3 -0
- package/dist/adapters/opencode/adapter.js +48 -0
- package/dist/adapters/opencode/dialect.d.ts +10 -0
- package/dist/adapters/opencode/dialect.js +36 -0
- package/dist/adapters/opencode/layout.d.ts +9 -0
- package/dist/adapters/opencode/layout.js +25 -0
- package/dist/adapters/opencode/model-mock.d.ts +10 -0
- package/dist/adapters/opencode/model-mock.js +10 -0
- package/dist/adapters/opencode/runtime.d.ts +9 -0
- package/dist/adapters/opencode/runtime.js +21 -0
- package/dist/claude-code.d.ts +8 -2
- package/dist/claude-code.js +8 -2
- package/dist/cli-flags.d.ts +22 -0
- package/dist/cli-flags.js +38 -0
- package/dist/cli.js +406 -206
- package/dist/codex.d.ts +20 -0
- package/dist/codex.js +36 -0
- package/dist/community-skills.d.ts +1 -1
- package/dist/community-skills.js +2 -2
- package/dist/core/adapter.d.ts +88 -0
- package/dist/core/adapter.js +3 -0
- package/dist/{compile.d.ts → core/compile.d.ts} +11 -2
- package/dist/{compile.js → core/compile.js} +68 -53
- package/dist/core/compose.d.ts +79 -0
- package/dist/core/compose.js +145 -0
- package/dist/core/dialect.d.ts +51 -0
- package/dist/core/dialect.js +3 -0
- package/dist/core/harness-driver.d.ts +134 -0
- package/dist/core/harness-driver.js +3 -0
- package/dist/core/hook-protocol.d.ts +28 -0
- package/dist/core/hook-protocol.js +3 -0
- package/dist/core/layout.d.ts +43 -0
- package/dist/core/layout.js +3 -0
- package/dist/core/model-mock.d.ts +27 -0
- package/dist/core/model-mock.js +3 -0
- package/dist/core/refs.d.ts +62 -0
- package/dist/{refs.js → core/refs.js} +49 -23
- package/dist/core/runtime.d.ts +38 -0
- package/dist/core/runtime.js +3 -0
- package/dist/{types.d.ts → core/types.d.ts} +29 -0
- package/dist/{validate.d.ts → core/validate.d.ts} +1 -1
- package/dist/{validate.js → core/validate.js} +41 -4
- package/dist/e2e.d.ts +19 -0
- package/dist/e2e.js +39 -0
- package/dist/harness-assert.d.ts +29 -14
- package/dist/harness-assert.js +37 -10
- package/dist/integration.d.ts +16 -0
- package/dist/integration.js +32 -0
- package/dist/leaderboard.d.ts +33 -0
- package/dist/leaderboard.js +107 -0
- package/dist/linting.d.ts +2 -2
- package/dist/linting.js +2 -2
- package/dist/plugin-loader.d.ts +6 -3
- package/dist/plugin-loader.js +101 -49
- package/dist/scan.d.ts +49 -0
- package/dist/scan.js +177 -0
- package/dist/setup-plan.d.ts +51 -0
- package/dist/setup-plan.js +85 -0
- package/dist/skill-test.d.ts +1 -1
- package/dist/skill-test.js +1 -1
- package/dist/test-coverage.d.ts +71 -0
- package/dist/test-coverage.js +228 -0
- package/dist/testing.d.ts +3 -3
- package/dist/testing.js +3 -3
- package/dist/unit.d.ts +17 -0
- package/dist/unit.js +36 -0
- package/hooks/refs-nudge.sh +24 -0
- package/package.json +30 -15
- package/skills/audit-feedback-loop/SKILL.md +76 -0
- package/skills/edit-spec/SKILL.md +131 -0
- package/skills/enforce-rules-format/SKILL.md +71 -0
- package/skills/generate-logo/SKILL.md +103 -0
- package/skills/generate-rule/SKILL.md +64 -0
- package/skills/linter-docs/clippy.md +241 -0
- package/skills/linter-docs/eslint.md +384 -0
- package/skills/linter-docs/pylint.md +288 -0
- package/skills/linter-docs/rubocop.md +277 -0
- package/skills/linter-docs/ruff.md +187 -0
- package/skills/linter-docs/stylelint.md +247 -0
- package/skills/migrate-to-spec/SKILL.md +126 -0
- package/skills/pr-to-lint-rule/SKILL.md +97 -0
- package/skills/strengthen/SKILL.md +168 -0
- package/skills/test-harness/SKILL.md +157 -0
- package/dist/action.d.ts +0 -7
- package/dist/action.js +0 -180
- package/dist/refs.d.ts +0 -44
- package/dist/run-scripts.d.ts +0 -20
- package/dist/run-scripts.js +0 -70
- /package/dist/{agent-result.js → adapters/claude-code/agent-result.js} +0 -0
- /package/dist/{agent-runtime.d.ts → adapters/claude-code/agent-runtime.d.ts} +0 -0
- /package/dist/{agent-runtime.js → adapters/claude-code/agent-runtime.js} +0 -0
- /package/dist/{egress-proxy.d.ts → adapters/claude-code/egress-proxy.d.ts} +0 -0
- /package/dist/{egress-proxy.js → adapters/claude-code/egress-proxy.js} +0 -0
- /package/dist/{eval-baseline.d.ts → adapters/claude-code/eval-baseline.d.ts} +0 -0
- /package/dist/{eval-baseline.js → adapters/claude-code/eval-baseline.js} +0 -0
- /package/dist/{judge.d.ts → adapters/claude-code/judge.d.ts} +0 -0
- /package/dist/{judge.js → adapters/claude-code/judge.js} +0 -0
- /package/dist/{mock-entry.d.ts → adapters/claude-code/mock-entry.d.ts} +0 -0
- /package/dist/{mock-entry.js → adapters/claude-code/mock-entry.js} +0 -0
- /package/dist/{mock-model.js → adapters/claude-code/mock-model.js} +0 -0
- /package/dist/{skill-driver.js → adapters/claude-code/skill-driver.js} +0 -0
- /package/dist/{skill-runtime.d.ts → adapters/claude-code/skill-runtime.d.ts} +0 -0
- /package/dist/{skill-runtime.js → adapters/claude-code/skill-runtime.js} +0 -0
- /package/dist/{stats.d.ts → adapters/claude-code/stats.d.ts} +0 -0
- /package/dist/{stats.js → adapters/claude-code/stats.js} +0 -0
- /package/dist/{compile-generator.d.ts → core/compile-generator.d.ts} +0 -0
- /package/dist/{compile-generator.js → core/compile-generator.js} +0 -0
- /package/dist/{coverage.d.ts → core/coverage.d.ts} +0 -0
- /package/dist/{coverage.js → core/coverage.js} +0 -0
- /package/dist/{doc-refs.d.ts → core/doc-refs.d.ts} +0 -0
- /package/dist/{doc-refs.js → core/doc-refs.js} +0 -0
- /package/dist/{evolve.d.ts → core/evolve.d.ts} +0 -0
- /package/dist/{evolve.js → core/evolve.js} +0 -0
- /package/dist/{frontmatter.d.ts → core/frontmatter.d.ts} +0 -0
- /package/dist/{frontmatter.js → core/frontmatter.js} +0 -0
- /package/dist/{generate-schema.d.ts → core/generate-schema.d.ts} +0 -0
- /package/dist/{generate-schema.js → core/generate-schema.js} +0 -0
- /package/dist/{generate-types.d.ts → core/generate-types.d.ts} +0 -0
- /package/dist/{generate-types.js → core/generate-types.js} +0 -0
- /package/dist/{hash.d.ts → core/hash.d.ts} +0 -0
- /package/dist/{hash.js → core/hash.js} +0 -0
- /package/dist/{inline.d.ts → core/inline.d.ts} +0 -0
- /package/dist/{inline.js → core/inline.js} +0 -0
- /package/dist/{integrity.d.ts → core/integrity.d.ts} +0 -0
- /package/dist/{integrity.js → core/integrity.js} +0 -0
- /package/dist/{linters.d.ts → core/linters.d.ts} +0 -0
- /package/dist/{linters.js → core/linters.js} +0 -0
- /package/dist/{mcp.d.ts → core/mcp.d.ts} +0 -0
- /package/dist/{mcp.js → core/mcp.js} +0 -0
- /package/dist/{orphans.d.ts → core/orphans.d.ts} +0 -0
- /package/dist/{orphans.js → core/orphans.js} +0 -0
- /package/dist/{proofs.d.ts → core/proofs.d.ts} +0 -0
- /package/dist/{proofs.js → core/proofs.js} +0 -0
- /package/dist/{session.d.ts → core/session.d.ts} +0 -0
- /package/dist/{session.js → core/session.js} +0 -0
- /package/dist/{sidecar.d.ts → core/sidecar.d.ts} +0 -0
- /package/dist/{sidecar.js → core/sidecar.js} +0 -0
- /package/dist/{spec.d.ts → core/spec.d.ts} +0 -0
- /package/dist/{spec.js → core/spec.js} +0 -0
- /package/dist/{symbols.d.ts → core/symbols.d.ts} +0 -0
- /package/dist/{symbols.js → core/symbols.js} +0 -0
- /package/dist/{test-utils.d.ts → core/test-utils.d.ts} +0 -0
- /package/dist/{test-utils.js → core/test-utils.js} +0 -0
- /package/dist/{types.js → core/types.js} +0 -0
- /package/{.claude-plugin/hooks → hooks}/post-edit.sh +0 -0
- /package/{.claude-plugin/hooks → hooks}/pre-edit.sh +0 -0
- /package/{.claude-plugin/hooks → hooks}/session-start.sh +0 -0
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* HarnessRuntime — the runtime/transport PORT (hexagonal transport axis). The
|
|
3
|
+
* facts the test tiers need to actually drive a harness: the agent binary to
|
|
4
|
+
* spawn, and the env a no-key mock model is reached through (the base-URL var,
|
|
5
|
+
* the API-key var, and a dummy key the mock ignores). These were hard-coded
|
|
6
|
+
* `"claude"` / `ANTHROPIC_*` literals in `harness-test.ts` and `eval.ts`; behind
|
|
7
|
+
* this interface a second harness (Codex) supplies its own `HarnessRuntime`
|
|
8
|
+
* (a different binary + its model's env) and the runners spawn it the same way.
|
|
9
|
+
*
|
|
10
|
+
* The Claude Code implementation is `claudeCodeRuntime` in
|
|
11
|
+
* `src/adapters/claude-code/runtime.ts`.
|
|
12
|
+
*/
|
|
13
|
+
export interface HarnessRuntime {
|
|
14
|
+
/** Stable identifier, e.g. "claude-code". */
|
|
15
|
+
readonly name: string;
|
|
16
|
+
/** The CLI binary that runs the agent, e.g. "claude". */
|
|
17
|
+
readonly agentBinary: string;
|
|
18
|
+
/** Env var pointing the client at the mock model, e.g. "ANTHROPIC_BASE_URL". */
|
|
19
|
+
readonly modelBaseUrlEnv: string;
|
|
20
|
+
/** Env var carrying the (dummy) API key, e.g. "ANTHROPIC_API_KEY". */
|
|
21
|
+
readonly modelApiKeyEnv: string;
|
|
22
|
+
/** A dummy key value the mock ignores — avoids needing real auth. */
|
|
23
|
+
readonly mockApiKey: string;
|
|
24
|
+
/**
|
|
25
|
+
* How to point the spawned binary at a mock model served at `baseUrl` — the
|
|
26
|
+
* args to add to the binary's argv and the env to spawn it with. For Claude
|
|
27
|
+
* Code this is env-only (`ANTHROPIC_BASE_URL` + a dummy key, no args); for
|
|
28
|
+
* Codex it is the keyless `-c model_providers.mock.*` flag recipe plus a
|
|
29
|
+
* dummy-key env. Behind one method so the runner wires either harness the
|
|
30
|
+
* same way, without knowing which transport axis (env var vs config flags) a
|
|
31
|
+
* given harness uses.
|
|
32
|
+
*/
|
|
33
|
+
wireMock(baseUrl: string): {
|
|
34
|
+
readonly args: readonly string[];
|
|
35
|
+
readonly env: Record<string, string>;
|
|
36
|
+
};
|
|
37
|
+
}
|
|
38
|
+
//# sourceMappingURL=runtime.d.ts.map
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import type { HarnessDialect } from "./dialect.js";
|
|
1
2
|
/** A parsed rule from a markdown instruction file. */
|
|
2
3
|
export interface ParsedRule {
|
|
3
4
|
title: string;
|
|
@@ -68,6 +69,21 @@ export interface OrphansConfig {
|
|
|
68
69
|
*/
|
|
69
70
|
exclude?: readonly string[];
|
|
70
71
|
}
|
|
72
|
+
/** Options for the untested-surface check. */
|
|
73
|
+
export interface TestCoverageConfig {
|
|
74
|
+
/** Scan skills. Default true. */
|
|
75
|
+
skills?: boolean;
|
|
76
|
+
/** Scan subagents. Default true. */
|
|
77
|
+
agents?: boolean;
|
|
78
|
+
/** Scan hook scripts referenced from plugin.json / settings.json. Default true. */
|
|
79
|
+
hooks?: boolean;
|
|
80
|
+
/** Require a test for user-invoked (disable-model-invocation) skills. Default false. */
|
|
81
|
+
includeUserInvokedSkills?: boolean;
|
|
82
|
+
/** Globs of test files that count as coverage. */
|
|
83
|
+
testGlobs?: readonly string[];
|
|
84
|
+
/** Extra ignore globs. */
|
|
85
|
+
exclude?: readonly string[];
|
|
86
|
+
}
|
|
71
87
|
export interface RulesConfig {
|
|
72
88
|
/** Require .spec.ts for CLAUDE.md / AGENTS.md. Default: "warn". */
|
|
73
89
|
"require-spec"?: RuleSeverity;
|
|
@@ -77,6 +93,16 @@ export interface RulesConfig {
|
|
|
77
93
|
integrity?: RuleSeverity;
|
|
78
94
|
/** Enforce minimum spec coverage thresholds. Default: false. ESLint-style: ["warn", { scripts: 50 }]. */
|
|
79
95
|
coverage?: RuleWithOptions<CoverageThresholds>;
|
|
96
|
+
/** Flag skills/agents/hooks with no test or eval. Default: "warn". */
|
|
97
|
+
"untested-surface"?: RuleWithOptions<TestCoverageConfig>;
|
|
98
|
+
/**
|
|
99
|
+
* Nudge (or block) when an instruction file has code-shaped references that
|
|
100
|
+
* aren't expressed as vigiles marks (so the audit can't verify them), or a
|
|
101
|
+
* `vigiles:symbol` mark that points at a missing symbol. Drives the
|
|
102
|
+
* PostToolUse refs-hook: "warn" (default) → a non-blocking nudge, "error" →
|
|
103
|
+
* block the edit, false → off.
|
|
104
|
+
*/
|
|
105
|
+
"unmarked-refs"?: RuleSeverity;
|
|
80
106
|
}
|
|
81
107
|
/** Extract severity from a rule value (handles both simple and tuple forms). */
|
|
82
108
|
export declare function ruleSeverity<T>(rule: RuleWithOptions<T> | undefined): RuleSeverity;
|
|
@@ -113,6 +139,9 @@ export interface ValidateOptions {
|
|
|
113
139
|
ruleMarkers?: MarkerType[];
|
|
114
140
|
rules?: RulesConfig;
|
|
115
141
|
filePath?: string;
|
|
142
|
+
/** Injected harness dialect; its instructionTargets define recognized
|
|
143
|
+
* instruction filenames. Omitted → the validator's built-in default set. */
|
|
144
|
+
dialect?: HarnessDialect;
|
|
116
145
|
}
|
|
117
146
|
/** Options for validatePaths(). */
|
|
118
147
|
export interface ValidatePathsOptions {
|
|
@@ -3,7 +3,7 @@ export type { ParsedRule, ValidationError, ValidationResult, ReadResult, FileRes
|
|
|
3
3
|
export declare function findInstructionFiles(cwd?: string, configFiles?: string[]): string[];
|
|
4
4
|
export declare function loadConfig(): VigilesConfig;
|
|
5
5
|
export declare function parseRules(content: string, { ruleMarkers }?: ParseOptions): ParsedRule[];
|
|
6
|
-
export declare function validate(content: string, { ruleMarkers, rules: rulesConfig, filePath }?: ValidateOptions): ValidationResult;
|
|
6
|
+
export declare function validate(content: string, { ruleMarkers, rules: rulesConfig, filePath, dialect }?: ValidateOptions): ValidationResult;
|
|
7
7
|
export declare function readInstructionFile(filePath: string, options?: ReadOptions): ReadResult;
|
|
8
8
|
export declare function expandGlobs(patterns: string[]): string[];
|
|
9
9
|
export declare function validatePaths(paths: string[], { followSymlinks, ruleMarkers, rules: rulesConfig, }?: ValidatePathsOptions): ValidatePathsResult;
|
|
@@ -24,12 +24,19 @@ const VALID_MARKERS = ["headings", "checkboxes"];
|
|
|
24
24
|
// ---------------------------------------------------------------------------
|
|
25
25
|
// Default config
|
|
26
26
|
// ---------------------------------------------------------------------------
|
|
27
|
-
|
|
27
|
+
// The instruction filenames vigiles recognizes when no dialect is injected — a
|
|
28
|
+
// validator-level default, not a harness dialect (the concrete dialects live in
|
|
29
|
+
// the adapters; an injected ValidateOptions.dialect overrides this).
|
|
30
|
+
const INSTRUCTION_FILES = ["CLAUDE.md", "AGENTS.md"];
|
|
31
|
+
// The default instruction file to validate when no config names one.
|
|
32
|
+
const DEFAULT_FILES = [INSTRUCTION_FILES[0]];
|
|
28
33
|
const DEFAULT_RULES = {
|
|
29
34
|
"require-spec": "warn",
|
|
30
35
|
"require-skill-spec": "warn",
|
|
31
36
|
integrity: "warn",
|
|
32
37
|
coverage: false,
|
|
38
|
+
"untested-surface": "warn",
|
|
39
|
+
"unmarked-refs": "warn",
|
|
33
40
|
};
|
|
34
41
|
const DEFAULT_CONFIG = {
|
|
35
42
|
ruleMarkers: ["headings", "checkboxes"],
|
|
@@ -129,7 +136,7 @@ function parseRules(content, { ruleMarkers } = {}) {
|
|
|
129
136
|
// ---------------------------------------------------------------------------
|
|
130
137
|
// Core validation
|
|
131
138
|
// ---------------------------------------------------------------------------
|
|
132
|
-
function validate(content, { ruleMarkers, rules: rulesConfig, filePath } = {}) {
|
|
139
|
+
function validate(content, { ruleMarkers, rules: rulesConfig, filePath, dialect } = {}) {
|
|
133
140
|
const activeRules = rulesConfig ?? DEFAULT_RULES;
|
|
134
141
|
const parsedRules = parseRules(content, { ruleMarkers });
|
|
135
142
|
const enforced = parsedRules.filter((r) => r.enforcement === "enforced").length;
|
|
@@ -141,7 +148,8 @@ function validate(content, { ruleMarkers, rules: rulesConfig, filePath } = {}) {
|
|
|
141
148
|
const disableComment = /<!--\s*vigiles-disable\s+require-spec\s*-->/;
|
|
142
149
|
if (filePath) {
|
|
143
150
|
const basename = (0, node_path_1.basename)(filePath);
|
|
144
|
-
const
|
|
151
|
+
const recognized = dialect?.instructionTargets ?? INSTRUCTION_FILES;
|
|
152
|
+
const isInstruction = recognized.includes(basename);
|
|
145
153
|
const isSkill = basename === "SKILL.md";
|
|
146
154
|
// --- require-spec (CLAUDE.md / AGENTS.md) ---
|
|
147
155
|
const specSeverity = activeRules["require-spec"];
|
|
@@ -259,7 +267,30 @@ function expandGlobs(patterns) {
|
|
|
259
267
|
function validatePaths(paths, { followSymlinks = false, ruleMarkers, rules: rulesConfig, } = {}) {
|
|
260
268
|
const fileResults = [];
|
|
261
269
|
let allValid = true;
|
|
270
|
+
// Maps a real (symlink-resolved) path → the first path validated for it, so a
|
|
271
|
+
// symlinked/synced CLAUDE.md⇄AGENTS.md mirror is validated ONCE on the real
|
|
272
|
+
// file instead of double-firing require-spec on the mirror's name (sync-tool-
|
|
273
|
+
// compatibility.md req 7). Recorded only on a successful validation, so a
|
|
274
|
+
// symlink seen first (and skipped) never shadows its real target.
|
|
275
|
+
const seenReal = new Map();
|
|
262
276
|
for (const filePath of paths) {
|
|
277
|
+
let real;
|
|
278
|
+
try {
|
|
279
|
+
real = (0, node_fs_1.realpathSync)(filePath);
|
|
280
|
+
}
|
|
281
|
+
catch {
|
|
282
|
+
real = (0, node_path_1.resolve)(filePath);
|
|
283
|
+
}
|
|
284
|
+
const prior = seenReal.get(real);
|
|
285
|
+
if (prior !== undefined) {
|
|
286
|
+
fileResults.push({
|
|
287
|
+
path: filePath,
|
|
288
|
+
skipped: true,
|
|
289
|
+
reason: `mirror of ${prior} (same file via symlink/sync) — validated once`,
|
|
290
|
+
result: null,
|
|
291
|
+
});
|
|
292
|
+
continue;
|
|
293
|
+
}
|
|
263
294
|
const { content, skipped, reason } = readInstructionFile(filePath, {
|
|
264
295
|
followSymlinks,
|
|
265
296
|
});
|
|
@@ -274,10 +305,16 @@ function validatePaths(paths, { followSymlinks = false, ruleMarkers, rules: rule
|
|
|
274
305
|
allValid = false;
|
|
275
306
|
continue;
|
|
276
307
|
}
|
|
308
|
+
// Attribute require-spec/integrity to the REAL file when this path is a
|
|
309
|
+
// symlink, so a symlinked AGENTS.md resolves to CLAUDE.md's spec rather than
|
|
310
|
+
// a nonexistent AGENTS.md.spec.ts. Non-symlinks keep the original path
|
|
311
|
+
// verbatim (behaviour-preserving).
|
|
312
|
+
const attributePath = real !== (0, node_path_1.resolve)(filePath) ? real : filePath;
|
|
313
|
+
seenReal.set(real, filePath);
|
|
277
314
|
const result = validate(content, {
|
|
278
315
|
ruleMarkers,
|
|
279
316
|
rules: rulesConfig,
|
|
280
|
-
filePath,
|
|
317
|
+
filePath: attributePath,
|
|
281
318
|
});
|
|
282
319
|
fileResults.push({ path: filePath, skipped: false, reason: null, result });
|
|
283
320
|
if (!result.valid)
|
package/dist/e2e.d.ts
ADDED
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* `vigiles/e2e` — the **deterministic** end-to-end tier (real sandbox + real
|
|
3
|
+
* network, but a definite pass/fail).
|
|
4
|
+
*
|
|
5
|
+
* Re-exports everything in [`vigiles/integration`](./integration.ts) and adds the
|
|
6
|
+
* real-egress capability: `egressRoutes()` (probe whether allowlisted egress can
|
|
7
|
+
* actually route) and — by re-export — `runHook` used with `egress: { allow }`
|
|
8
|
+
* (allowlisted real outbound). Capability contract: needs a **routable rootless
|
|
9
|
+
* sandbox + real network**, and each test self-skips via `egressRoutes()` where
|
|
10
|
+
* that's unavailable. Still a **verification** tier — you assert pass/fail.
|
|
11
|
+
*
|
|
12
|
+
* NOT here: **evals** (`runEval` / `measureTriggerRate` / `judge`). Those are a
|
|
13
|
+
* different axis — **non-deterministic measurement** (real model, mean ± se), run
|
|
14
|
+
* via `vigiles eval` on `*.eval.mjs`. Import them from
|
|
15
|
+
* [`vigiles/eval`](./eval.ts) + [`vigiles/judge`](./judge.ts), not from here.
|
|
16
|
+
*/
|
|
17
|
+
export * from "./integration.js";
|
|
18
|
+
export { egressRoutes } from "./adapters/claude-code/run-hook.js";
|
|
19
|
+
//# sourceMappingURL=e2e.d.ts.map
|
package/dist/e2e.js
ADDED
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
|
|
3
|
+
if (k2 === undefined) k2 = k;
|
|
4
|
+
var desc = Object.getOwnPropertyDescriptor(m, k);
|
|
5
|
+
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
|
|
6
|
+
desc = { enumerable: true, get: function() { return m[k]; } };
|
|
7
|
+
}
|
|
8
|
+
Object.defineProperty(o, k2, desc);
|
|
9
|
+
}) : (function(o, m, k, k2) {
|
|
10
|
+
if (k2 === undefined) k2 = k;
|
|
11
|
+
o[k2] = m[k];
|
|
12
|
+
}));
|
|
13
|
+
var __exportStar = (this && this.__exportStar) || function(m, exports) {
|
|
14
|
+
for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
|
|
15
|
+
};
|
|
16
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
17
|
+
exports.egressRoutes = void 0;
|
|
18
|
+
/**
|
|
19
|
+
* `vigiles/e2e` — the **deterministic** end-to-end tier (real sandbox + real
|
|
20
|
+
* network, but a definite pass/fail).
|
|
21
|
+
*
|
|
22
|
+
* Re-exports everything in [`vigiles/integration`](./integration.ts) and adds the
|
|
23
|
+
* real-egress capability: `egressRoutes()` (probe whether allowlisted egress can
|
|
24
|
+
* actually route) and — by re-export — `runHook` used with `egress: { allow }`
|
|
25
|
+
* (allowlisted real outbound). Capability contract: needs a **routable rootless
|
|
26
|
+
* sandbox + real network**, and each test self-skips via `egressRoutes()` where
|
|
27
|
+
* that's unavailable. Still a **verification** tier — you assert pass/fail.
|
|
28
|
+
*
|
|
29
|
+
* NOT here: **evals** (`runEval` / `measureTriggerRate` / `judge`). Those are a
|
|
30
|
+
* different axis — **non-deterministic measurement** (real model, mean ± se), run
|
|
31
|
+
* via `vigiles eval` on `*.eval.mjs`. Import them from
|
|
32
|
+
* [`vigiles/eval`](./eval.ts) + [`vigiles/judge`](./judge.ts), not from here.
|
|
33
|
+
*/
|
|
34
|
+
__exportStar(require("./integration.js"), exports);
|
|
35
|
+
// The real-egress capability probe (the egress-using runHook is already re-exported
|
|
36
|
+
// via the integration→unit chain).
|
|
37
|
+
var run_hook_js_1 = require("./adapters/claude-code/run-hook.js");
|
|
38
|
+
Object.defineProperty(exports, "egressRoutes", { enumerable: true, get: function () { return run_hook_js_1.egressRoutes; } });
|
|
39
|
+
//# sourceMappingURL=e2e.js.map
|
package/dist/harness-assert.d.ts
CHANGED
|
@@ -13,22 +13,32 @@
|
|
|
13
13
|
* `expect(...).toHaveCreated(...)` sugar. The signature is identical for
|
|
14
14
|
* vitest and jest, so the same object supports both.
|
|
15
15
|
*/
|
|
16
|
-
import { type HarnessTestSpec, type HarnessTestResult, type ToolCall, type Trace } from "./harness-test.js";
|
|
17
|
-
import type { EvalReport, TriggerRateReport } from "./eval.js";
|
|
18
|
-
import type { HookRunResult, EgressAttempt } from "./run-hook.js";
|
|
19
|
-
import type { OutputContract } from "./spec.js";
|
|
20
|
-
import { type ParsedAgentResult } from "./agent-result.js";
|
|
21
|
-
import { type BaselineFile, type DiffOptions } from "./eval-baseline.js";
|
|
22
|
-
export { compareArms } from "./stats.js";
|
|
23
|
-
export type { Comparison } from "./stats.js";
|
|
24
|
-
export { diffReports, toBaselineFile, parseBaselineFile, readBaseline, writeBaseline, formatBaselineDiff, diffToJUnit, } from "./eval-baseline.js";
|
|
25
|
-
export type { BaselineFile, BaselineDiff, MetricDiff, DiffStatus, DiffOptions, } from "./eval-baseline.js";
|
|
16
|
+
import { type HarnessTestSpec, type HarnessTestResult, type ToolCall, type Trace } from "./adapters/claude-code/harness-test.js";
|
|
17
|
+
import type { EvalReport, TriggerRateReport } from "./adapters/claude-code/eval.js";
|
|
18
|
+
import type { HookRunResult, EgressAttempt } from "./adapters/claude-code/run-hook.js";
|
|
19
|
+
import type { OutputContract } from "./core/spec.js";
|
|
20
|
+
import { type ParsedAgentResult } from "./adapters/claude-code/agent-result.js";
|
|
21
|
+
import { type BaselineFile, type DiffOptions } from "./adapters/claude-code/eval-baseline.js";
|
|
22
|
+
export { compareArms } from "./adapters/claude-code/stats.js";
|
|
23
|
+
export type { Comparison } from "./adapters/claude-code/stats.js";
|
|
24
|
+
export { diffReports, toBaselineFile, parseBaselineFile, readBaseline, writeBaseline, formatBaselineDiff, diffToJUnit, } from "./adapters/claude-code/eval-baseline.js";
|
|
25
|
+
export type { BaselineFile, BaselineDiff, MetricDiff, DiffStatus, DiffOptions, } from "./adapters/claude-code/eval-baseline.js";
|
|
26
26
|
/**
|
|
27
27
|
* Run a harness test, hand the result to `fn`, and always clean up the sandbox.
|
|
28
28
|
* Returns whatever `fn` returns. Use this instead of calling `cleanup()` by
|
|
29
29
|
* hand — it survives assertion failures.
|
|
30
30
|
*/
|
|
31
31
|
export declare function withHarness<T>(spec: HarnessTestSpec, fn: (r: HarnessTestResult) => T | Promise<T>): Promise<T>;
|
|
32
|
+
/**
|
|
33
|
+
* Mark the current `*.harness.*` / `*.eval.*` SCRIPT as SKIPPED and exit, so
|
|
34
|
+
* `vigiles test` / `vigiles eval` report a loud `⊘ SKIPPED` instead of a silent
|
|
35
|
+
* `✓` — e.g. the deterministic tier when the `claude` CLI isn't installed. A skip
|
|
36
|
+
* never fails the run. Exit 77 is the runner's `SKIP_EXIT_CODE` (run-scripts.ts).
|
|
37
|
+
*
|
|
38
|
+
* For standalone CLI-fallback scripts only — inside a runner (vitest / jest /
|
|
39
|
+
* node:test) use that runner's own skip, not a process exit.
|
|
40
|
+
*/
|
|
41
|
+
export declare function skip(reason?: string): never;
|
|
32
42
|
/** Assert the sandbox contains `path` (a hook/agent side-effect file). */
|
|
33
43
|
export declare function assertCreated(r: HarnessTestResult, path: string): void;
|
|
34
44
|
/** Assert the sandbox does NOT contain `path` (e.g. a blocked action's output). */
|
|
@@ -242,12 +252,17 @@ export declare function assertImproves(report: EvalReport, opts: {
|
|
|
242
252
|
*/
|
|
243
253
|
export declare function assertNoRegression(current: EvalReport | readonly EvalReport[], baseline: BaselineFile, opts?: DiffOptions): void;
|
|
244
254
|
/**
|
|
245
|
-
* Assert a skill/behaviour triggered
|
|
246
|
-
*
|
|
247
|
-
*
|
|
255
|
+
* Assert a skill/behaviour triggered reliably — the gate for a skill's
|
|
256
|
+
* *activation*, over a {@link TriggerRateReport} from `measureTriggerRate`.
|
|
257
|
+
* `min` gates recall (fires when it should). With irrelevant prompts measured,
|
|
258
|
+
* `maxFalsePositive` gates the precision side (does NOT fire when it shouldn't)
|
|
259
|
+
* and `minPrecision` gates `firedRight / firedTotal` — so a too-broad
|
|
260
|
+
* description that hijacks unrelated work fails, not just a too-narrow one.
|
|
248
261
|
*/
|
|
249
262
|
export declare function assertTriggerRate(report: TriggerRateReport, opts: {
|
|
250
|
-
min
|
|
263
|
+
min?: number;
|
|
264
|
+
maxFalsePositive?: number;
|
|
265
|
+
minPrecision?: number;
|
|
251
266
|
}): void;
|
|
252
267
|
interface MatcherOutput {
|
|
253
268
|
pass: boolean;
|
package/dist/harness-assert.js
CHANGED
|
@@ -2,6 +2,7 @@
|
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
3
|
exports.vigilesMatchers = exports.diffToJUnit = exports.formatBaselineDiff = exports.writeBaseline = exports.readBaseline = exports.parseBaselineFile = exports.toBaselineFile = exports.diffReports = exports.compareArms = void 0;
|
|
4
4
|
exports.withHarness = withHarness;
|
|
5
|
+
exports.skip = skip;
|
|
5
6
|
exports.assertCreated = assertCreated;
|
|
6
7
|
exports.assertNotCreated = assertNotCreated;
|
|
7
8
|
exports.assertServedTurns = assertServedTurns;
|
|
@@ -56,15 +57,15 @@ exports.assertTriggerRate = assertTriggerRate;
|
|
|
56
57
|
* `expect(...).toHaveCreated(...)` sugar. The signature is identical for
|
|
57
58
|
* vitest and jest, so the same object supports both.
|
|
58
59
|
*/
|
|
59
|
-
const harness_test_js_1 = require("./harness-test.js");
|
|
60
|
-
const agent_result_js_1 = require("./agent-result.js");
|
|
61
|
-
const stats_js_1 = require("./stats.js");
|
|
62
|
-
const eval_baseline_js_1 = require("./eval-baseline.js");
|
|
60
|
+
const harness_test_js_1 = require("./adapters/claude-code/harness-test.js");
|
|
61
|
+
const agent_result_js_1 = require("./adapters/claude-code/agent-result.js");
|
|
62
|
+
const stats_js_1 = require("./adapters/claude-code/stats.js");
|
|
63
|
+
const eval_baseline_js_1 = require("./adapters/claude-code/eval-baseline.js");
|
|
63
64
|
// Re-export the significance primitives so the whole eval-analysis surface lives
|
|
64
65
|
// behind `vigiles/harness-assert` (no separate entry point).
|
|
65
|
-
var stats_js_2 = require("./stats.js");
|
|
66
|
+
var stats_js_2 = require("./adapters/claude-code/stats.js");
|
|
66
67
|
Object.defineProperty(exports, "compareArms", { enumerable: true, get: function () { return stats_js_2.compareArms; } });
|
|
67
|
-
var eval_baseline_js_2 = require("./eval-baseline.js");
|
|
68
|
+
var eval_baseline_js_2 = require("./adapters/claude-code/eval-baseline.js");
|
|
68
69
|
Object.defineProperty(exports, "diffReports", { enumerable: true, get: function () { return eval_baseline_js_2.diffReports; } });
|
|
69
70
|
Object.defineProperty(exports, "toBaselineFile", { enumerable: true, get: function () { return eval_baseline_js_2.toBaselineFile; } });
|
|
70
71
|
Object.defineProperty(exports, "parseBaselineFile", { enumerable: true, get: function () { return eval_baseline_js_2.parseBaselineFile; } });
|
|
@@ -92,6 +93,21 @@ async function withHarness(spec, fn) {
|
|
|
92
93
|
function fail(message) {
|
|
93
94
|
throw new Error(message);
|
|
94
95
|
}
|
|
96
|
+
/**
|
|
97
|
+
* Mark the current `*.harness.*` / `*.eval.*` SCRIPT as SKIPPED and exit, so
|
|
98
|
+
* `vigiles test` / `vigiles eval` report a loud `⊘ SKIPPED` instead of a silent
|
|
99
|
+
* `✓` — e.g. the deterministic tier when the `claude` CLI isn't installed. A skip
|
|
100
|
+
* never fails the run. Exit 77 is the runner's `SKIP_EXIT_CODE` (run-scripts.ts).
|
|
101
|
+
*
|
|
102
|
+
* For standalone CLI-fallback scripts only — inside a runner (vitest / jest /
|
|
103
|
+
* node:test) use that runner's own skip, not a process exit.
|
|
104
|
+
*/
|
|
105
|
+
/* v8 ignore start -- process.exit can't be exercised in-process (tested via a child in cli.test.ts / run-scripts.test.ts) */
|
|
106
|
+
function skip(reason) {
|
|
107
|
+
console.log(`SKIPPED${reason ? `: ${reason}` : ""}`);
|
|
108
|
+
process.exit(77);
|
|
109
|
+
}
|
|
110
|
+
/* v8 ignore stop */
|
|
95
111
|
/** Assert the sandbox contains `path` (a hook/agent side-effect file). */
|
|
96
112
|
function assertCreated(r, path) {
|
|
97
113
|
if (r.file(path) === null)
|
|
@@ -506,14 +522,25 @@ function assertNoRegression(current, baseline, opts) {
|
|
|
506
522
|
}
|
|
507
523
|
}
|
|
508
524
|
/**
|
|
509
|
-
* Assert a skill/behaviour triggered
|
|
510
|
-
*
|
|
511
|
-
*
|
|
525
|
+
* Assert a skill/behaviour triggered reliably — the gate for a skill's
|
|
526
|
+
* *activation*, over a {@link TriggerRateReport} from `measureTriggerRate`.
|
|
527
|
+
* `min` gates recall (fires when it should). With irrelevant prompts measured,
|
|
528
|
+
* `maxFalsePositive` gates the precision side (does NOT fire when it shouldn't)
|
|
529
|
+
* and `minPrecision` gates `firedRight / firedTotal` — so a too-broad
|
|
530
|
+
* description that hijacks unrelated work fails, not just a too-narrow one.
|
|
512
531
|
*/
|
|
513
532
|
function assertTriggerRate(report, opts) {
|
|
514
|
-
if (report.rate < opts.min) {
|
|
533
|
+
if (opts.min !== undefined && report.rate < opts.min) {
|
|
515
534
|
fail(`expected a trigger rate ≥ ${String(opts.min)}, got ${report.rate.toFixed(2)} (${String(report.n)} runs)`);
|
|
516
535
|
}
|
|
536
|
+
if (opts.maxFalsePositive !== undefined &&
|
|
537
|
+
(report.falsePositiveRate ?? 0) > opts.maxFalsePositive) {
|
|
538
|
+
fail(`expected a false-positive rate ≤ ${String(opts.maxFalsePositive)}, got ${(report.falsePositiveRate ?? 0).toFixed(2)}`);
|
|
539
|
+
}
|
|
540
|
+
if (opts.minPrecision !== undefined &&
|
|
541
|
+
(report.precision ?? 0) < opts.minPrecision) {
|
|
542
|
+
fail(`expected precision ≥ ${String(opts.minPrecision)}, got ${report.precision === undefined ? "n/a (nothing fired)" : report.precision.toFixed(2)}`);
|
|
543
|
+
}
|
|
517
544
|
}
|
|
518
545
|
/**
|
|
519
546
|
* Custom matchers compatible with both vitest and jest. Register once:
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* `vigiles/integration` — the **deterministic, assembled-machine** tier.
|
|
3
|
+
*
|
|
4
|
+
* Re-exports everything in [`vigiles/unit`](./unit.ts) and adds the runners that
|
|
5
|
+
* drive the real `claude` CLI against a **scripted mock model** plus structural
|
|
6
|
+
* assembly: `runHarnessTest` / `withHarness` (real hooks fire, outcome
|
|
7
|
+
* reproducible), `scriptModel` (the mock), and `loadPlugin` / `resolveHarness`
|
|
8
|
+
* (the plugin loader). Capability contract: needs the **`claude` binary and
|
|
9
|
+
* bubblewrap**, but **no API key and no network**. A `*.integration.test.ts`
|
|
10
|
+
* imports from here.
|
|
11
|
+
*/
|
|
12
|
+
export * from "./unit.js";
|
|
13
|
+
export * from "./adapters/claude-code/harness-test.js";
|
|
14
|
+
export * from "./adapters/claude-code/mock-model.js";
|
|
15
|
+
export * from "./adapters/claude-code/plugin-loader.js";
|
|
16
|
+
//# sourceMappingURL=integration.d.ts.map
|
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
|
|
3
|
+
if (k2 === undefined) k2 = k;
|
|
4
|
+
var desc = Object.getOwnPropertyDescriptor(m, k);
|
|
5
|
+
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
|
|
6
|
+
desc = { enumerable: true, get: function() { return m[k]; } };
|
|
7
|
+
}
|
|
8
|
+
Object.defineProperty(o, k2, desc);
|
|
9
|
+
}) : (function(o, m, k, k2) {
|
|
10
|
+
if (k2 === undefined) k2 = k;
|
|
11
|
+
o[k2] = m[k];
|
|
12
|
+
}));
|
|
13
|
+
var __exportStar = (this && this.__exportStar) || function(m, exports) {
|
|
14
|
+
for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
|
|
15
|
+
};
|
|
16
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
17
|
+
/**
|
|
18
|
+
* `vigiles/integration` — the **deterministic, assembled-machine** tier.
|
|
19
|
+
*
|
|
20
|
+
* Re-exports everything in [`vigiles/unit`](./unit.ts) and adds the runners that
|
|
21
|
+
* drive the real `claude` CLI against a **scripted mock model** plus structural
|
|
22
|
+
* assembly: `runHarnessTest` / `withHarness` (real hooks fire, outcome
|
|
23
|
+
* reproducible), `scriptModel` (the mock), and `loadPlugin` / `resolveHarness`
|
|
24
|
+
* (the plugin loader). Capability contract: needs the **`claude` binary and
|
|
25
|
+
* bubblewrap**, but **no API key and no network**. A `*.integration.test.ts`
|
|
26
|
+
* imports from here.
|
|
27
|
+
*/
|
|
28
|
+
__exportStar(require("./unit.js"), exports);
|
|
29
|
+
__exportStar(require("./adapters/claude-code/harness-test.js"), exports);
|
|
30
|
+
__exportStar(require("./adapters/claude-code/mock-model.js"), exports);
|
|
31
|
+
__exportStar(require("./adapters/claude-code/plugin-loader.js"), exports);
|
|
32
|
+
//# sourceMappingURL=integration.js.map
|
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Plugin health leaderboard — rank many plugins by the deterministic structural
|
|
3
|
+
* signals `scan` already computes (research/divergent-bets.md #9, the engine).
|
|
4
|
+
*
|
|
5
|
+
* This is the no-model half: a defensible health score from concrete facts
|
|
6
|
+
* (missing hook scripts, skills that can't trigger, agents with no tool
|
|
7
|
+
* contract, untested surfaces). It deliberately does NOT score on the loader's
|
|
8
|
+
* free-text warnings — those include doc-mention false positives (see
|
|
9
|
+
* src/scan.ts), and a ranking that penalizes a prose mention would be unfair.
|
|
10
|
+
* The behavioural columns (real trigger-rate, observed egress, safety) need a
|
|
11
|
+
* model and stack on top later; this part runs anywhere in CI for free.
|
|
12
|
+
*/
|
|
13
|
+
import { type ScanReport } from "./scan.js";
|
|
14
|
+
export interface PluginScore {
|
|
15
|
+
readonly dir: string;
|
|
16
|
+
readonly name: string;
|
|
17
|
+
/** 0–100 structural-health score (100 = no structural issues found). */
|
|
18
|
+
readonly score: number;
|
|
19
|
+
readonly grade: "A" | "B" | "C" | "D" | "F";
|
|
20
|
+
/** Human-readable deductions, worst first. */
|
|
21
|
+
readonly issues: readonly string[];
|
|
22
|
+
readonly report: ScanReport;
|
|
23
|
+
}
|
|
24
|
+
/** Deterministic structural-health score for one scanned plugin. */
|
|
25
|
+
export declare function scoreReport(r: ScanReport): {
|
|
26
|
+
score: number;
|
|
27
|
+
issues: string[];
|
|
28
|
+
};
|
|
29
|
+
/** Scan + score each directory, ranked best-first (ties broken by name). */
|
|
30
|
+
export declare function rankPlugins(dirs: readonly string[]): PluginScore[];
|
|
31
|
+
/** Format a ranked leaderboard as human-readable text. */
|
|
32
|
+
export declare function formatLeaderboard(scores: readonly PluginScore[]): string;
|
|
33
|
+
//# sourceMappingURL=leaderboard.d.ts.map
|
|
@@ -0,0 +1,107 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
/**
|
|
3
|
+
* Plugin health leaderboard — rank many plugins by the deterministic structural
|
|
4
|
+
* signals `scan` already computes (research/divergent-bets.md #9, the engine).
|
|
5
|
+
*
|
|
6
|
+
* This is the no-model half: a defensible health score from concrete facts
|
|
7
|
+
* (missing hook scripts, skills that can't trigger, agents with no tool
|
|
8
|
+
* contract, untested surfaces). It deliberately does NOT score on the loader's
|
|
9
|
+
* free-text warnings — those include doc-mention false positives (see
|
|
10
|
+
* src/scan.ts), and a ranking that penalizes a prose mention would be unfair.
|
|
11
|
+
* The behavioural columns (real trigger-rate, observed egress, safety) need a
|
|
12
|
+
* model and stack on top later; this part runs anywhere in CI for free.
|
|
13
|
+
*/
|
|
14
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
15
|
+
exports.scoreReport = scoreReport;
|
|
16
|
+
exports.rankPlugins = rankPlugins;
|
|
17
|
+
exports.formatLeaderboard = formatLeaderboard;
|
|
18
|
+
const node_path_1 = require("node:path");
|
|
19
|
+
const scan_js_1 = require("./scan.js");
|
|
20
|
+
// Penalty weights — broken-at-runtime costs most, footguns less, nudges least.
|
|
21
|
+
const W_MISSING_HOOK = 15; // a hook script that doesn't exist → never runs
|
|
22
|
+
const W_NO_DESCRIPTION = 10; // a skill with no usable description → can't trigger
|
|
23
|
+
const W_NO_CONTRACT = 5; // an agent with no `tools:` line → inherits everything
|
|
24
|
+
const W_UNTESTED = 3; // a surface with no test/eval → warning-tier
|
|
25
|
+
function gradeFor(score) {
|
|
26
|
+
if (score >= 90)
|
|
27
|
+
return "A";
|
|
28
|
+
if (score >= 80)
|
|
29
|
+
return "B";
|
|
30
|
+
if (score >= 70)
|
|
31
|
+
return "C";
|
|
32
|
+
if (score >= 60)
|
|
33
|
+
return "D";
|
|
34
|
+
return "F";
|
|
35
|
+
}
|
|
36
|
+
/** Deterministic structural-health score for one scanned plugin. */
|
|
37
|
+
function scoreReport(r) {
|
|
38
|
+
// An empty/unloadable machine isn't healthy — it's a non-plugin or a broken load.
|
|
39
|
+
if (r.skills.length + r.agents.length + r.hooks.length === 0) {
|
|
40
|
+
return { score: 0, issues: ["no loadable plugin surface"] };
|
|
41
|
+
}
|
|
42
|
+
const missingHooks = r.hooks.filter((h) => h.status === "missing").length;
|
|
43
|
+
const noDesc = r.skills.filter((s) => !s.hasDescription).length;
|
|
44
|
+
const noContract = r.agents.filter((a) => a.tools === null).length;
|
|
45
|
+
const deductions = [
|
|
46
|
+
{
|
|
47
|
+
n: missingHooks,
|
|
48
|
+
weight: W_MISSING_HOOK,
|
|
49
|
+
label: "hook script(s) MISSING",
|
|
50
|
+
},
|
|
51
|
+
{
|
|
52
|
+
n: noDesc,
|
|
53
|
+
weight: W_NO_DESCRIPTION,
|
|
54
|
+
label: "skill(s) with no usable description",
|
|
55
|
+
},
|
|
56
|
+
{
|
|
57
|
+
n: noContract,
|
|
58
|
+
weight: W_NO_CONTRACT,
|
|
59
|
+
label: "agent(s) inherit all tools (no contract)",
|
|
60
|
+
},
|
|
61
|
+
{ n: r.untested, weight: W_UNTESTED, label: "untested surface(s)" },
|
|
62
|
+
];
|
|
63
|
+
let penalty = 0;
|
|
64
|
+
const issues = [];
|
|
65
|
+
for (const d of deductions) {
|
|
66
|
+
if (d.n === 0)
|
|
67
|
+
continue;
|
|
68
|
+
penalty += d.n * d.weight;
|
|
69
|
+
issues.push(`${String(d.n)} ${d.label}`);
|
|
70
|
+
}
|
|
71
|
+
// Sort issues by cost (worst first) so the report leads with what matters.
|
|
72
|
+
issues.sort((a, b) => Number(b.split(" ")[0]) - Number(a.split(" ")[0]));
|
|
73
|
+
return { score: Math.max(0, 100 - penalty), issues };
|
|
74
|
+
}
|
|
75
|
+
/** Scan + score each directory, ranked best-first (ties broken by name). */
|
|
76
|
+
function rankPlugins(dirs) {
|
|
77
|
+
const scored = dirs.map((dir) => {
|
|
78
|
+
const report = (0, scan_js_1.scanPlugin)(dir);
|
|
79
|
+
const { score, issues } = scoreReport(report);
|
|
80
|
+
return {
|
|
81
|
+
dir,
|
|
82
|
+
name: (0, node_path_1.basename)(dir) || dir,
|
|
83
|
+
score,
|
|
84
|
+
grade: gradeFor(score),
|
|
85
|
+
issues,
|
|
86
|
+
report,
|
|
87
|
+
};
|
|
88
|
+
});
|
|
89
|
+
return scored.sort((a, b) => b.score - a.score || a.name.localeCompare(b.name));
|
|
90
|
+
}
|
|
91
|
+
/** Format a ranked leaderboard as human-readable text. */
|
|
92
|
+
function formatLeaderboard(scores) {
|
|
93
|
+
const out = [
|
|
94
|
+
`Plugin health leaderboard (${String(scores.length)} scanned)`,
|
|
95
|
+
"",
|
|
96
|
+
" # score grade plugin",
|
|
97
|
+
];
|
|
98
|
+
scores.forEach((s, i) => {
|
|
99
|
+
const rank = String(i + 1).padStart(2);
|
|
100
|
+
const score = String(s.score).padStart(3);
|
|
101
|
+
const issue = s.issues.length > 0 ? ` — ${s.issues.join("; ")}` : "";
|
|
102
|
+
out.push(` ${rank} ${score} ${s.grade} ${s.name}${issue}`);
|
|
103
|
+
});
|
|
104
|
+
out.push("", "Structural health only (no model). Weights: missing hook -15, no-description", "skill -10, agent-without-tool-contract -5, untested surface -3.");
|
|
105
|
+
return out.join("\n");
|
|
106
|
+
}
|
|
107
|
+
//# sourceMappingURL=leaderboard.js.map
|
package/dist/linting.d.ts
CHANGED
|
@@ -4,6 +4,6 @@
|
|
|
4
4
|
* concern-named import. The granular paths (`vigiles/spec`, `vigiles/compile`)
|
|
5
5
|
* keep working; this just groups them so the import name matches the pillar.
|
|
6
6
|
*/
|
|
7
|
-
export * from "./spec.js";
|
|
8
|
-
export * from "./compile.js";
|
|
7
|
+
export * from "./core/spec.js";
|
|
8
|
+
export * from "./core/compile.js";
|
|
9
9
|
//# sourceMappingURL=linting.d.ts.map
|
package/dist/linting.js
CHANGED
|
@@ -20,6 +20,6 @@ Object.defineProperty(exports, "__esModule", { value: true });
|
|
|
20
20
|
* concern-named import. The granular paths (`vigiles/spec`, `vigiles/compile`)
|
|
21
21
|
* keep working; this just groups them so the import name matches the pillar.
|
|
22
22
|
*/
|
|
23
|
-
__exportStar(require("./spec.js"), exports);
|
|
24
|
-
__exportStar(require("./compile.js"), exports);
|
|
23
|
+
__exportStar(require("./core/spec.js"), exports);
|
|
24
|
+
__exportStar(require("./core/compile.js"), exports);
|
|
25
25
|
//# sourceMappingURL=linting.js.map
|
package/dist/plugin-loader.d.ts
CHANGED
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import type { PluginLayout } from "./core/layout.js";
|
|
1
2
|
export interface LoadedPlugin {
|
|
2
3
|
/** A `.claude/settings.json`-shaped object with hooks resolved. */
|
|
3
4
|
readonly settings: {
|
|
@@ -20,17 +21,19 @@ export interface LoadedPlugin {
|
|
|
20
21
|
* sandbox, and `warnings` for surfaces the deterministic tier can't drive. Merge
|
|
21
22
|
* `settings` with any inline settings and spread `files` into the fixture.
|
|
22
23
|
*/
|
|
23
|
-
export declare function loadPlugin(pluginPath: string): LoadedPlugin;
|
|
24
|
+
export declare function loadPlugin(pluginPath: string, layout: PluginLayout): LoadedPlugin;
|
|
24
25
|
/**
|
|
25
26
|
* Resolve the effective harness for a test/eval (arm): load the plugin if given,
|
|
26
27
|
* then layer inline settings + files on top. Shared by `runHarnessTest` and
|
|
27
|
-
* `runEval` so both test the assembled machine the same way.
|
|
28
|
+
* `runEval` so both test the assembled machine the same way. `layout` is
|
|
29
|
+
* REQUIRED (no harness default at the composition root) — the Claude Code
|
|
30
|
+
* wrapper supplies `claudeCodeLayout` to preserve `resolveHarness(opts)`.
|
|
28
31
|
*/
|
|
29
32
|
export declare function resolveHarness(opts: {
|
|
30
33
|
plugin?: string;
|
|
31
34
|
settings?: unknown;
|
|
32
35
|
files?: Record<string, string>;
|
|
33
|
-
}): {
|
|
36
|
+
}, layout: PluginLayout): {
|
|
34
37
|
settings: unknown;
|
|
35
38
|
files: Record<string, string>;
|
|
36
39
|
};
|