vigiles 21.0.2 → 23.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/dist/adapters/claude-code/hook-protocol.js +3 -0
- package/dist/adapters/claude-code/layout.js +2 -0
- package/dist/cli-flag-check.js +1 -0
- package/dist/cli.d.ts +2 -1
- package/dist/cli.js +229 -45
- package/dist/core/hook-block-ineffective.js +24 -1
- package/dist/core/hook-protocol.d.ts +14 -0
- package/dist/core/layout.d.ts +15 -0
- package/dist/coverage-evidence.d.ts +48 -2
- package/dist/coverage-evidence.js +96 -3
- package/dist/eval.d.ts +10 -0
- package/dist/eval.js +14 -3
- package/dist/plugin-loader.js +35 -0
- package/dist/run-hook.d.ts +23 -2
- package/dist/run-hook.js +10 -3
- package/dist/scan-core.d.ts +5 -0
- package/dist/scan-core.js +10 -1
- package/dist/spec-hooks.d.mts +35 -0
- package/dist/spec-hooks.mjs +68 -0
- package/dist/spec-host.d.mts +2 -0
- package/dist/spec-host.mjs +79 -0
- package/dist/test-coverage-files.js +17 -8
- package/dist/test-coverage.js +29 -16
- package/package.json +1 -1
package/README.md
CHANGED
|
@@ -163,7 +163,7 @@ Every one of these is **valid markdown** — parses fine, does the wrong thing.
|
|
|
163
163
|
| `test` | Does the harness behave? | No — a scripted stand-in | Every commit |
|
|
164
164
|
| `eval` | Does a skill actually help? | Yes — your subscription | On demand |
|
|
165
165
|
|
|
166
|
-
**One engine, two doors.** `audit` is the local report; **`lint` is the CI gate** that fails the build on the same deterministic checks — broken refs, bad tool contracts, dead hooks, skill collisions (Proofs 1–2). `test` and `eval` go further: past _does it exist_ to _does it work_. (`init` / `compile` / `eject` manage the optional typed-spec layer for the structural rules no linter can express — a graduation step you rarely run by hand.) [How the verbs relate →](docs/commands-and-how-they-relate.md)
|
|
166
|
+
**One engine, two doors.** `audit` is the local report; **`lint` is the CI gate** that fails the build on the same deterministic checks — broken refs, bad tool contracts, dead hooks, skill collisions (Proofs 1–2). `test` and `eval` go further: past _does it exist_ to _does it work_. (`init` / `compile` / `eject` manage the optional typed-spec layer for the structural rules no linter can express — a graduation step you rarely run by hand. If you use a spec, run `compile` in CI too: it is what re-derives that spec's refs, while `lint` verifies the compiled file is intact.) [How the verbs relate →](docs/commands-and-how-they-relate.md)
|
|
167
167
|
|
|
168
168
|
### 🔎 Lint — your instructions stop lying
|
|
169
169
|
|
|
@@ -6,6 +6,9 @@ exports.claudeCodeHookProtocol = {
|
|
|
6
6
|
blockExitCode: 2,
|
|
7
7
|
denyDecisionValues: ["block", "deny"],
|
|
8
8
|
eventEnvVars: [],
|
|
9
|
+
// `{"continue": false}` stops the turn outright and returns `stopReason` to
|
|
10
|
+
// the agent — a stronger stop than a per-call deny, and a documented one.
|
|
11
|
+
haltsTurnField: "continue",
|
|
9
12
|
// Events that honor `hookSpecificOutput.additionalContext` (developer-context
|
|
10
13
|
// injection). Covers vigiles's shipped inject hooks: the SessionStart lint
|
|
11
14
|
// summary and the PostToolUse refs / eval-lock nudges.
|
|
@@ -16,6 +16,8 @@ exports.claudeCodeLayout = {
|
|
|
16
16
|
skillDir: "skills",
|
|
17
17
|
agentDir: "agents",
|
|
18
18
|
commandDir: "commands",
|
|
19
|
+
// `.claude/rules/*.md` — path-scoped project instructions (see PluginLayout).
|
|
20
|
+
rulesDir: "rules",
|
|
19
21
|
materializeRoot: ".claude",
|
|
20
22
|
pluginRootToken: "${CLAUDE_PLUGIN_ROOT}",
|
|
21
23
|
// Both names Claude Code uses for the project root (mirrors the
|
package/dist/cli-flag-check.js
CHANGED
package/dist/cli.d.ts
CHANGED
|
@@ -9,5 +9,6 @@
|
|
|
9
9
|
* `self-command-refs.test.ts` did not catch it because it guards against refs to
|
|
10
10
|
* REMOVED commands, not against a list that merely stops growing.
|
|
11
11
|
*/
|
|
12
|
-
|
|
12
|
+
/** Reason the most recent `loadSpec()` returned null, or null if it succeeded. */
|
|
13
|
+
export declare function specLoadFailureReason(): string | null;
|
|
13
14
|
//# sourceMappingURL=cli.d.ts.map
|
package/dist/cli.js
CHANGED
|
@@ -11,8 +11,10 @@
|
|
|
11
11
|
* REMOVED commands, not against a list that merely stops growing.
|
|
12
12
|
*/
|
|
13
13
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
14
|
+
exports.specLoadFailureReason = specLoadFailureReason;
|
|
14
15
|
const node_fs_1 = require("node:fs");
|
|
15
16
|
const node_path_1 = require("node:path");
|
|
17
|
+
const node_child_process_1 = require("node:child_process");
|
|
16
18
|
const glob_1 = require("glob");
|
|
17
19
|
const generate_types_js_1 = require("./core/generate-types.js");
|
|
18
20
|
const generate_harness_js_1 = require("./core/generate-harness.js");
|
|
@@ -96,54 +98,168 @@ function findSpecs(pattern) {
|
|
|
96
98
|
cwd: process.cwd(),
|
|
97
99
|
});
|
|
98
100
|
}
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
101
|
+
/**
|
|
102
|
+
* Why the last `loadSpec()` returned null.
|
|
103
|
+
*
|
|
104
|
+
* Kept as module state rather than a widened return type: `loadSpec` has six
|
|
105
|
+
* call sites and only one of them reports to a human.
|
|
106
|
+
*/
|
|
107
|
+
let lastSpecLoadFailure = null;
|
|
108
|
+
/** Reason the most recent `loadSpec()` returned null, or null if it succeeded. */
|
|
109
|
+
function specLoadFailureReason() {
|
|
110
|
+
return lastSpecLoadFailure;
|
|
111
|
+
}
|
|
112
|
+
/**
|
|
113
|
+
* How long one spec may take to evaluate before the host is killed.
|
|
114
|
+
*
|
|
115
|
+
* Overridable because 15s is a guess that fits the specs we have seen, not a
|
|
116
|
+
* law; a repo with genuinely slow specs should be able to raise it rather than
|
|
117
|
+
* discover the number by hitting it.
|
|
118
|
+
*/
|
|
119
|
+
const SPEC_DEADLINE_MS = Number(process.env.VIGILES_SPEC_TIMEOUT_MS) || 15_000;
|
|
120
|
+
let host = null;
|
|
121
|
+
/** The compiled host entry, beside this file in `dist/`. */
|
|
122
|
+
function hostEntry() {
|
|
123
|
+
return (0, node_path_1.resolve)(__dirname, "spec-host.mjs");
|
|
124
|
+
}
|
|
125
|
+
function startHost() {
|
|
126
|
+
const child = (0, node_child_process_1.spawn)(process.execPath, [hostEntry()], {
|
|
127
|
+
cwd: process.cwd(),
|
|
128
|
+
stdio: ["pipe", "pipe", "pipe"],
|
|
129
|
+
});
|
|
130
|
+
const h = { child, pending: new Map(), started: null, buffered: "" };
|
|
131
|
+
child.stdout.setEncoding("utf-8");
|
|
132
|
+
child.stdout.on("data", (chunk) => {
|
|
133
|
+
h.buffered += chunk;
|
|
134
|
+
let nl;
|
|
135
|
+
while ((nl = h.buffered.indexOf("\n")) >= 0) {
|
|
136
|
+
const line = h.buffered.slice(0, nl).trim();
|
|
137
|
+
h.buffered = h.buffered.slice(nl + 1);
|
|
138
|
+
if (!line)
|
|
139
|
+
continue;
|
|
140
|
+
let reply;
|
|
117
141
|
try {
|
|
118
|
-
|
|
119
|
-
// CJS double-default: `{ default: { default: spec } }`.
|
|
120
|
-
const raw = mod.default;
|
|
121
|
-
if (raw && typeof raw === "object" && "default" in raw) {
|
|
122
|
-
return raw.default;
|
|
123
|
-
}
|
|
124
|
-
return raw;
|
|
142
|
+
reply = JSON.parse(line);
|
|
125
143
|
}
|
|
126
144
|
catch {
|
|
127
|
-
//
|
|
145
|
+
continue; // not ours; a spec writing to stdout cannot corrupt the stream
|
|
146
|
+
}
|
|
147
|
+
if ("phase" in reply) {
|
|
148
|
+
h.started = reply.path;
|
|
149
|
+
continue;
|
|
128
150
|
}
|
|
151
|
+
const done = h.pending.get(reply.path);
|
|
152
|
+
h.pending.delete(reply.path);
|
|
153
|
+
done?.(reply);
|
|
129
154
|
}
|
|
155
|
+
});
|
|
156
|
+
// Anything the child says on stderr is the spec's own noise; keep it out of
|
|
157
|
+
// our stdout so `--json` consumers are not corrupted, but do not lose it.
|
|
158
|
+
child.stderr.setEncoding("utf-8");
|
|
159
|
+
child.stderr.on("data", (chunk) => process.stderr.write(chunk));
|
|
160
|
+
// 🔴 Unreferenced, or the CLI never exits. A piped child and its three
|
|
161
|
+
// streams each hold the event loop open, so `compile` finished its work and
|
|
162
|
+
// then hung forever waiting on a host that had nothing left to say. The
|
|
163
|
+
// in-flight deadline timer keeps the loop alive while a request is pending,
|
|
164
|
+
// which is exactly as long as we need it.
|
|
165
|
+
// ONE exit listener per host, not one per request: with concurrent callers the
|
|
166
|
+
// per-request version added a listener each time and Node warned at eleven.
|
|
167
|
+
// It fails every outstanding request, because a dead host answers none of them.
|
|
168
|
+
child.once("exit", () => {
|
|
169
|
+
const waiting = [...h.pending.values()];
|
|
170
|
+
h.pending.clear();
|
|
171
|
+
for (const settle of waiting)
|
|
172
|
+
settle("died");
|
|
173
|
+
});
|
|
174
|
+
// The stdio types are Readable/Writable, which do not declare `unref` — the
|
|
175
|
+
// objects are pipes and do have it. Optional-called so this stays correct if
|
|
176
|
+
// a platform ever hands back a stream that genuinely lacks it.
|
|
177
|
+
const unref = (s) => s?.unref?.();
|
|
178
|
+
child.unref();
|
|
179
|
+
unref(child.stdin);
|
|
180
|
+
unref(child.stdout);
|
|
181
|
+
unref(child.stderr);
|
|
182
|
+
return h;
|
|
183
|
+
}
|
|
184
|
+
/**
|
|
185
|
+
* Kill the host and forget it; the next request starts a fresh one.
|
|
186
|
+
*
|
|
187
|
+
* Outstanding requests are failed rather than dropped: a killed host will never
|
|
188
|
+
* answer them, and a promise nobody settles is a hang wearing a different hat.
|
|
189
|
+
*/
|
|
190
|
+
function dropHost() {
|
|
191
|
+
if (!host)
|
|
192
|
+
return;
|
|
193
|
+
const dying = host;
|
|
194
|
+
host = null;
|
|
195
|
+
const waiting = [...dying.pending.values()];
|
|
196
|
+
dying.pending.clear();
|
|
197
|
+
dying.child.kill("SIGKILL");
|
|
198
|
+
for (const settle of waiting)
|
|
199
|
+
settle("died");
|
|
200
|
+
}
|
|
201
|
+
process.on("exit", dropHost);
|
|
202
|
+
/**
|
|
203
|
+
* Load one spec in the spec host.
|
|
204
|
+
*
|
|
205
|
+
* 🔴 **Why a child process rather than `import()` here.** A module evaluation
|
|
206
|
+
* cannot be cancelled once started — `Promise.race` hands control back but the
|
|
207
|
+
* evaluation keeps running and holds the event loop — so an in-process loader
|
|
208
|
+
* gives a stalled spec an unbounded hang in `compile`, `test` and `audit`. It
|
|
209
|
+
* also cannot tell whether a failed spec already ran (Node reports
|
|
210
|
+
* `ERR_MODULE_NOT_FOUND` and `SyntaxError` both before and during evaluation),
|
|
211
|
+
* which is what made the previous two-loader arrangement unfixable rather than
|
|
212
|
+
* merely buggy: it had to guess whether re-running was safe.
|
|
213
|
+
*
|
|
214
|
+
* The host is spawned with `process.execPath` — never `npx` — so nothing is
|
|
215
|
+
* fetched and nothing needs installing.
|
|
216
|
+
*/
|
|
217
|
+
async function loadSpec(specPath) {
|
|
218
|
+
const fullPath = (0, node_path_1.resolve)(process.cwd(), specPath);
|
|
219
|
+
lastSpecLoadFailure = null;
|
|
220
|
+
if (!(0, node_fs_1.existsSync)(fullPath)) {
|
|
221
|
+
lastSpecLoadFailure = `no such file: ${specPath}`;
|
|
222
|
+
return null;
|
|
130
223
|
}
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
const
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
224
|
+
host ??= startHost();
|
|
225
|
+
const h = host;
|
|
226
|
+
const reply = await new Promise((done) => {
|
|
227
|
+
let settled = false;
|
|
228
|
+
const finish = (r) => {
|
|
229
|
+
if (settled)
|
|
230
|
+
return;
|
|
231
|
+
settled = true;
|
|
232
|
+
clearTimeout(timer);
|
|
233
|
+
h.pending.delete(fullPath);
|
|
234
|
+
done(r);
|
|
235
|
+
};
|
|
236
|
+
const timer = setTimeout(() => {
|
|
237
|
+
finish("timeout");
|
|
238
|
+
}, SPEC_DEADLINE_MS);
|
|
239
|
+
h.pending.set(fullPath, finish);
|
|
240
|
+
h.child.stdin.write(JSON.stringify({ path: fullPath }) + "\n");
|
|
241
|
+
});
|
|
242
|
+
if (reply === "timeout") {
|
|
243
|
+
// The host's last `start` names the spec that stalled. Without it a hang
|
|
244
|
+
// produced N identical failures and no culprit.
|
|
245
|
+
const culprit = h.started ?? fullPath;
|
|
246
|
+
dropHost();
|
|
247
|
+
lastSpecLoadFailure =
|
|
248
|
+
`evaluating ${(0, node_path_1.relative)(process.cwd(), culprit)} exceeded ` +
|
|
249
|
+
`${SPEC_DEADLINE_MS}ms and was killed. Set VIGILES_SPEC_TIMEOUT_MS to ` +
|
|
250
|
+
`raise the limit, or look for a top-level await that never settles.`;
|
|
251
|
+
return null;
|
|
143
252
|
}
|
|
144
|
-
|
|
253
|
+
if (reply === "died") {
|
|
254
|
+
dropHost();
|
|
255
|
+
lastSpecLoadFailure = "the spec host exited unexpectedly.";
|
|
256
|
+
return null;
|
|
257
|
+
}
|
|
258
|
+
if (!("ok" in reply) || !reply.ok) {
|
|
259
|
+
lastSpecLoadFailure = `the spec did not load. ${"error" in reply ? reply.error : "no reason given"}`;
|
|
145
260
|
return null;
|
|
146
261
|
}
|
|
262
|
+
return reply.value;
|
|
147
263
|
}
|
|
148
264
|
// ---------------------------------------------------------------------------
|
|
149
265
|
// Output helpers
|
|
@@ -200,7 +316,20 @@ function compileClaudeToFile(spec, specPath, config, dialect) {
|
|
|
200
316
|
if (errors.length > 0) {
|
|
201
317
|
console.log(`\n✗ ${specPath} — ${String(errors.length)} error(s)`);
|
|
202
318
|
printErrors(specPath, errors);
|
|
203
|
-
|
|
319
|
+
// 🔴 NOTHING IS WRITTEN ON A FAILED COMPILE (#173).
|
|
320
|
+
//
|
|
321
|
+
// It used to write the artifact anyway, and the result was the exact
|
|
322
|
+
// false-confidence object this tool exists to prevent: a `CLAUDE.md`
|
|
323
|
+
// carrying refs already KNOWN to be dead, stamped with a VALID integrity
|
|
324
|
+
// hash. `lint` then verified the hash, found it intact, and exited 0 — so
|
|
325
|
+
// the command the README calls "the CI gate … broken refs" went green over
|
|
326
|
+
// breakage `compile` had printed minutes earlier. Compile locally, get
|
|
327
|
+
// distracted, commit: CI never mentions it again.
|
|
328
|
+
//
|
|
329
|
+
// Not writing leaves the LAST GOOD artifact in place, which is strictly
|
|
330
|
+
// better than replacing it with a broken one: the error is on screen, the
|
|
331
|
+
// exit code is 1, and no green hash is minted over a known-bad file.
|
|
332
|
+
console.log(` → ${primaryOutput} was NOT written; the previous version is left in place.`);
|
|
204
333
|
return false;
|
|
205
334
|
}
|
|
206
335
|
(0, node_fs_1.writeFileSync)((0, node_path_1.resolve)(basePath, primaryOutput), markdown);
|
|
@@ -357,7 +486,7 @@ async function compile(specPaths, config, opts = {}) {
|
|
|
357
486
|
const spec = await loadSpec(specPath);
|
|
358
487
|
if (!spec) {
|
|
359
488
|
console.log(`\n✗ ${specPath} — failed to load`);
|
|
360
|
-
console.log(`
|
|
489
|
+
console.log(` ${specLoadFailureReason() ?? "reason unavailable"}`);
|
|
361
490
|
allValid = false;
|
|
362
491
|
continue;
|
|
363
492
|
}
|
|
@@ -1821,6 +1950,15 @@ function scaffoldSpec(args) {
|
|
|
1821
1950
|
(0, node_fs_1.writeFileSync)(specAbs, source);
|
|
1822
1951
|
console.log(`Adopted ${target} → ${specPath} (${tier}, ${String(sectionCount)} section${sectionCount === 1 ? "" : "s"}). ` +
|
|
1823
1952
|
`Run \`vigiles compile\` and review the diff; the \`/strengthen\` skill upgrades prose to verified rules.`);
|
|
1953
|
+
// Adoption is faithful by design: it infers NO rules and extracts NO refs,
|
|
1954
|
+
// so a raw adoption verifies nothing on its own. Saying so is the whole
|
|
1955
|
+
// fix — the cost (the file becomes a build artifact, edits move into TS)
|
|
1956
|
+
// lands immediately, and without this line the benefit reads as zero
|
|
1957
|
+
// rather than as not-yet-claimed. Deliberately not a heuristic extractor:
|
|
1958
|
+
// guessing refs out of prose is what got `doc-refs` disabled.
|
|
1959
|
+
if (tier === "raw")
|
|
1960
|
+
console.log(` ℹ 0 refs extracted — this spec verifies nothing yet. Wrap paths in \`file()\` ` +
|
|
1961
|
+
`and commands in \`cmd()\` to make \`compile\` check them.`);
|
|
1824
1962
|
}
|
|
1825
1963
|
return;
|
|
1826
1964
|
}
|
|
@@ -3450,7 +3588,12 @@ function checkSkillResourceResolves(config, silent, adapter, scanRoot) {
|
|
|
3450
3588
|
if (found.length > 0 && !silent) {
|
|
3451
3589
|
console.log("\nSkill-resource check:\n");
|
|
3452
3590
|
for (const s of found) {
|
|
3453
|
-
const msg = `${s.name}: bundled resource "${s.finding.ref}" (line ${String(s.finding.line)}) is referenced but missing — the agent reads the instruction and gets nothing
|
|
3591
|
+
const msg = `${s.name}: bundled resource "${s.finding.ref}" (line ${String(s.finding.line)}) is referenced but missing — the agent reads the instruction and gets nothing.` +
|
|
3592
|
+
// The main false-positive source in a skills monorepo, where a SKILL.md
|
|
3593
|
+
// legitimately names a repo-root path. The fix already exists and works;
|
|
3594
|
+
// it was documented only in docs/skills-monorepo.md, so a CI log gave no
|
|
3595
|
+
// hint and the rule read as broken rather than misconfigured.
|
|
3596
|
+
` If it resolves from the repo root instead, add its directory to \`sharedDirs\` in .vigilesrc.json.`;
|
|
3454
3597
|
console.log(` ${sev === "error" ? "✗" : "⚠"} ${s.path}: ${msg}`);
|
|
3455
3598
|
ghAnnotate(sev === "error" ? "error" : "warning", msg, s.path);
|
|
3456
3599
|
}
|
|
@@ -4471,6 +4614,7 @@ const COMMAND_HELP = {
|
|
|
4471
4614
|
usage: " vigiles audit [dir...] Grade it on your machine. Reports everything, fails nothing.",
|
|
4472
4615
|
detail: [
|
|
4473
4616
|
" 2+ dirs → a leaderboard. Writes vigiles-report.html + .json (auto-gitignored).",
|
|
4617
|
+
" --single audit this dir as ONE harness, even if it holds many bundles",
|
|
4474
4618
|
" The executing checks (run your hooks · live MCP · do skills fire?) run only",
|
|
4475
4619
|
" interactively — audit asks once and remembers; automation uses the testing API.",
|
|
4476
4620
|
],
|
|
@@ -4605,7 +4749,12 @@ function printUsage(command) {
|
|
|
4605
4749
|
console.log("New here? Start with `vigiles audit .`");
|
|
4606
4750
|
if (command && command !== "--help") {
|
|
4607
4751
|
console.log(`\nUnknown command: "${command}"`);
|
|
4608
|
-
|
|
4752
|
+
// 2, not 1 — `docs/cli.md` fixes the contract as 1 = "I ran, and what you
|
|
4753
|
+
// asked about is bad", 2 = "I could not do what you asked". A typo'd verb is
|
|
4754
|
+
// the second, and the unknown-FLAG path (cli-flag-check.ts) already exits 2.
|
|
4755
|
+
// A script telling "found problems" from "could not start" by exit code read
|
|
4756
|
+
// a mistyped command as a finding.
|
|
4757
|
+
process.exit(2);
|
|
4609
4758
|
}
|
|
4610
4759
|
}
|
|
4611
4760
|
// ---------------------------------------------------------------------------
|
|
@@ -5798,6 +5947,14 @@ async function runHookProgramCommand(file) {
|
|
|
5798
5947
|
function ensureReportGitignored(cwd, entries) {
|
|
5799
5948
|
if (entries.length === 0)
|
|
5800
5949
|
return;
|
|
5950
|
+
// An `--out` outside the repo produced entries like
|
|
5951
|
+
// `../../../../private/tmp/x/vigiles-report.json`, which ignore NOTHING —
|
|
5952
|
+
// .gitignore does not reach outside its own tree — and accumulate one dead
|
|
5953
|
+
// block per output path. Worse in principle than in practice: `audit` is
|
|
5954
|
+
// documented as a read-only report, and this made it edit a tracked file for
|
|
5955
|
+
// no benefit at all. Inside the repo the write is expected and documented.
|
|
5956
|
+
if (entries.some((e) => e.startsWith("..") || (0, node_path_1.isAbsolute)(e)))
|
|
5957
|
+
return;
|
|
5801
5958
|
const gi = (0, node_path_1.resolve)(cwd, ".gitignore");
|
|
5802
5959
|
try {
|
|
5803
5960
|
if (!(0, node_fs_1.existsSync)(gi)) {
|
|
@@ -6313,10 +6470,37 @@ async function main() {
|
|
|
6313
6470
|
// carries `market` into its explanation and exits 2 — this repo's own rule
|
|
6314
6471
|
// that 1 is "I measured, and it's bad" and 2 is "I could not do what you
|
|
6315
6472
|
// asked". Nothing was measured here, so it is a 2.
|
|
6316
|
-
|
|
6317
|
-
|
|
6473
|
+
// `--single` pins the SINGLE-harness reading of the given directory, whatever
|
|
6474
|
+
// is nested inside it. Without it, a repo holding many bundles auto-switches
|
|
6475
|
+
// to the leaderboard and there is no way back — so the full ring report for
|
|
6476
|
+
// the ROOT was simply unreachable, and the reported workaround was a CI job
|
|
6477
|
+
// looping `audit` over 29 directories. The mode branch already existed; this
|
|
6478
|
+
// only stops it being decided for you.
|
|
6479
|
+
const single = args.includes("--single");
|
|
6480
|
+
// `--single` names ONE harness, so more than one explicit directory is a
|
|
6481
|
+
// contradiction. Refuse it (exit 2 = "could not do what you asked") rather
|
|
6482
|
+
// than auditing the first and dropping the rest — silently honouring half
|
|
6483
|
+
// an argument list is the same defect class as the ignored `--out` above.
|
|
6484
|
+
if (single && dirs.length > 1) {
|
|
6485
|
+
console.error(`--single audits ONE directory as one harness, but ${String(dirs.length)} were given. ` +
|
|
6486
|
+
`Drop --single for a leaderboard, or pass a single directory.`);
|
|
6487
|
+
process.exit(2);
|
|
6488
|
+
}
|
|
6489
|
+
const targets = !single && market && market.onDisk.length > 0
|
|
6490
|
+
? [...market.onDisk]
|
|
6491
|
+
: dirs;
|
|
6492
|
+
if (!single && targets.length > 1) {
|
|
6318
6493
|
// Multiple targets → rank them (the leaderboard engine). `--md` emits the
|
|
6319
6494
|
// publishable Markdown table (a README / gist / the leaderboard site).
|
|
6495
|
+
//
|
|
6496
|
+
// `--out` writes nothing here: the per-bundle HTML/JSON report is built
|
|
6497
|
+
// in the single-target branch below, and this one only prints a table.
|
|
6498
|
+
// SAY SO. A silent no-op is how a CI job ships an empty artifact and
|
|
6499
|
+
// stays green — which is exactly how this was found, and the same
|
|
6500
|
+
// never-fail-silently shape as the rest of this file.
|
|
6501
|
+
if (args.some((a) => a.startsWith("--out=")) && !json)
|
|
6502
|
+
console.log(`⚠ --out is ignored here: ${String(targets.length)} bundles → leaderboard mode, ` +
|
|
6503
|
+
`which produces no per-bundle report. Run audit per directory to write one.`);
|
|
6320
6504
|
const scores = (0, leaderboard_js_1.rankPlugins)(targets);
|
|
6321
6505
|
const text = args.includes("--md")
|
|
6322
6506
|
? (0, leaderboard_js_1.formatLeaderboardMarkdown)(scores)
|
|
@@ -79,6 +79,24 @@ const DECISION_BLOCK = /"decision"\s*:\s*"(block|deny)"/;
|
|
|
79
79
|
* (Both require a structured response, as opposed to the legacy field.)
|
|
80
80
|
*/
|
|
81
81
|
const PERMISSION_DENY = /"permissionDecision"\s*:\s*"(deny|ask)"/;
|
|
82
|
+
/**
|
|
83
|
+
* A `"continue": false` halt — the OTHER documented way a Claude Code hook stops
|
|
84
|
+
* an action (it ends the turn and returns `stopReason` to the agent).
|
|
85
|
+
*
|
|
86
|
+
* Read ONLY as a SUPPRESSOR of `wrong-field`, never as a block ATTEMPT, and the
|
|
87
|
+
* asymmetry is the whole point. #174 proposed adding it alongside the three
|
|
88
|
+
* mechanisms above; doing that would have made `wrong-event` fire on a hook that
|
|
89
|
+
* works, because a halt is NOT event-scoped — it stops the turn from
|
|
90
|
+
* `SessionStart` just as it does from `PreToolUse`, which is precisely the set
|
|
91
|
+
* `wrong-event` flags. For a rule that can be wired at `error`, a false positive
|
|
92
|
+
* costs more than a miss: it fails a correct build, and a rule that fails
|
|
93
|
+
* correct builds gets switched off rather than fixed.
|
|
94
|
+
*
|
|
95
|
+
* What it legitimately fixes is the reverse: a hook on a permission-gated event
|
|
96
|
+
* that pairs a legacy `"decision":"block"` with a real halt was told "nothing is
|
|
97
|
+
* blocked" while it blocked.
|
|
98
|
+
*/
|
|
99
|
+
const CONTINUE_FALSE = /"continue"\s*:\s*false/;
|
|
82
100
|
// ---------------------------------------------------------------------------
|
|
83
101
|
// Detector
|
|
84
102
|
// ---------------------------------------------------------------------------
|
|
@@ -125,6 +143,8 @@ function hookBlockIssues(entries, opts) {
|
|
|
125
143
|
const hasExit2 = EXIT_2.test(text) || EXIT_2_CODE.test(text);
|
|
126
144
|
const hasDecisionBlock = DECISION_BLOCK.test(text);
|
|
127
145
|
const hasPermissionDeny = PERMISSION_DENY.test(text);
|
|
146
|
+
const hasContinueFalse = CONTINUE_FALSE.test(text);
|
|
147
|
+
// Deliberately NOT `|| hasContinueFalse` — see CONTINUE_FALSE.
|
|
128
148
|
const triesBlock = hasExit2 || hasDecisionBlock || hasPermissionDeny;
|
|
129
149
|
if (!triesBlock)
|
|
130
150
|
continue;
|
|
@@ -143,7 +163,10 @@ function hookBlockIssues(entries, opts) {
|
|
|
143
163
|
}
|
|
144
164
|
else if (permissionDecisionEvents.has(entry.event) &&
|
|
145
165
|
hasDecisionBlock &&
|
|
146
|
-
!hasPermissionDeny
|
|
166
|
+
!hasPermissionDeny &&
|
|
167
|
+
// A halt alongside the legacy field DOES stop the action, so the legacy
|
|
168
|
+
// field being ignored costs nothing. Flagging it would be a false alarm.
|
|
169
|
+
!hasContinueFalse) {
|
|
147
170
|
// wrong-field: on a permission-gated event, uses the legacy field.
|
|
148
171
|
kind = "wrong-field";
|
|
149
172
|
message =
|
|
@@ -46,5 +46,19 @@ export interface HookProtocol {
|
|
|
46
46
|
* shell-hook harness declares a non-empty set.
|
|
47
47
|
*/
|
|
48
48
|
readonly injectableEvents: readonly string[];
|
|
49
|
+
/**
|
|
50
|
+
* The boolean stdout field whose `false` value HALTS THE WHOLE TURN, if the
|
|
51
|
+
* harness has one (Claude Code: `"continue"`). Distinct from a deny: a deny
|
|
52
|
+
* refuses one tool call, this stops the iteration and hands `stopReason` back
|
|
53
|
+
* to the agent as text — so authors reach for it exactly when they want to
|
|
54
|
+
* explain themselves, and a guard written that way still prevents the action.
|
|
55
|
+
*
|
|
56
|
+
* Optional (additive, non-breaking) and per-harness on purpose. It is
|
|
57
|
+
* DOCUMENTED for Claude Code and UNVERIFIED for Codex, whose protocol notes
|
|
58
|
+
* only record the shared exit-2 / `decision` / `permissionDecision` model — so
|
|
59
|
+
* Codex leaves it unset rather than inheriting a claim nobody measured. Read
|
|
60
|
+
* by `decideHook`; absent ⇒ no field halts the turn on this harness.
|
|
61
|
+
*/
|
|
62
|
+
readonly haltsTurnField?: string;
|
|
49
63
|
}
|
|
50
64
|
//# sourceMappingURL=hook-protocol.d.ts.map
|
package/dist/core/layout.d.ts
CHANGED
|
@@ -51,6 +51,21 @@ export interface PluginLayout {
|
|
|
51
51
|
readonly agentDir: string;
|
|
52
52
|
/** Slash-commands dir, holding flat `<dir>/<name>.md`, e.g. `commands`. */
|
|
53
53
|
readonly commandDir: string;
|
|
54
|
+
/**
|
|
55
|
+
* Path-scoped RULES dir, holding flat `<dir>/<name>.md`, e.g. `rules`
|
|
56
|
+
* (`""` or absent = this harness has no such layer).
|
|
57
|
+
*
|
|
58
|
+
* Claude Code loads `.claude/rules/*.md` as project instructions, scoped by a
|
|
59
|
+
* `paths:` frontmatter key. It is an INSTRUCTION surface — often where a
|
|
60
|
+
* team's hardest policies actually live — and until now no layout named it, so
|
|
61
|
+
* `frontmatter-valid` and the rule map simply never saw those files. An
|
|
62
|
+
* adopter reported five such files arriving in a session labelled "project
|
|
63
|
+
* instructions" while `lint` did not mention them at all (#175.3).
|
|
64
|
+
*
|
|
65
|
+
* Optional and additive: a layout that omits it behaves exactly as before, so
|
|
66
|
+
* this adds a directory to the existing checks rather than a new check.
|
|
67
|
+
*/
|
|
68
|
+
readonly rulesDir?: string;
|
|
54
69
|
/** Dir the surfaces are materialized under, e.g. `.claude`. */
|
|
55
70
|
readonly materializeRoot: string;
|
|
56
71
|
/** Env token expanded to the plugin's absolute root in hook commands. */
|
|
@@ -8,7 +8,9 @@
|
|
|
8
8
|
* conventions: one is about a process that ran, the other about a directory
|
|
9
9
|
* listing. See the module header.
|
|
10
10
|
*/
|
|
11
|
-
export type CoverageEvidence = "executed" | "colocated";
|
|
11
|
+
export type CoverageEvidence = "executed" | "colocated" | "configured";
|
|
12
|
+
/** Is `a` stronger evidence than `b`? */
|
|
13
|
+
export declare function strongerEvidence(a: CoverageEvidence, b: CoverageEvidence): boolean;
|
|
12
14
|
/** The minimum a surface must expose to be matched — structural, no import cycle. */
|
|
13
15
|
export interface CoverableSurface {
|
|
14
16
|
/** Repo-relative path of the surface file (SKILL.md / agent .md / hook script). */
|
|
@@ -100,12 +102,56 @@ export declare function hookScriptRefs(manifestText: string | undefined, layout:
|
|
|
100
102
|
* `colocated` is passed in because placement is a path question the two twins
|
|
101
103
|
* answer with their own (disk vs POSIX-string) path helpers.
|
|
102
104
|
*/
|
|
103
|
-
export declare function evidenceFor(_surface: CoverableSurface, _test: PreparedTest, colocated: boolean): CoverageEvidence | null;
|
|
105
|
+
export declare function evidenceFor(_surface: CoverableSurface, _test: PreparedTest, colocated: boolean, configured?: boolean): CoverageEvidence | null;
|
|
106
|
+
/**
|
|
107
|
+
* The `{surface}` placeholder in a user's `testGlobs` — the ONE thing that makes
|
|
108
|
+
* a centralized test layout expressible without weakening what coverage MEANS.
|
|
109
|
+
*
|
|
110
|
+
* The retired `declared` and `name-mentioned` tiers died because they could
|
|
111
|
+
* credit a surface no test touched: a mention is a substring, and a substring
|
|
112
|
+
* matched this file's own fixtures. `{surface}` cannot do that. The user writes
|
|
113
|
+
* `tests/{surface}/evals/promptfooconfig*.yaml`, and the placeholder is replaced
|
|
114
|
+
* with the surface's NAME before matching — so the binding between test and
|
|
115
|
+
* surface is still the name, exactly as under colocation. Only the PLACE moves.
|
|
116
|
+
*
|
|
117
|
+
* What it costs, stated plainly because it is the argument colocation was chosen
|
|
118
|
+
* on: `ls` beside the skill no longer answers "is this tested?" — you have to
|
|
119
|
+
* know where the project keeps its tests. That is a real loss, and it is why
|
|
120
|
+
* this is opt-in per repo rather than a second default. A project that has
|
|
121
|
+
* already centralized its suites has paid that cost anyway.
|
|
122
|
+
*/
|
|
123
|
+
export declare const SURFACE_TOKEN = "{surface}";
|
|
124
|
+
/** Does this glob delegate its surface binding to the placeholder? */
|
|
125
|
+
export declare function hasSurfaceToken(glob: string): boolean;
|
|
126
|
+
/**
|
|
127
|
+
* The pattern to DISCOVER files with: the placeholder widened to `*` so one
|
|
128
|
+
* glob pass finds every candidate. Narrowing back to the right surface happens
|
|
129
|
+
* at match time — discovery must stay surface-agnostic or it would be one glob
|
|
130
|
+
* pass per surface.
|
|
131
|
+
*/
|
|
132
|
+
export declare function discoveryGlob(glob: string): string;
|
|
133
|
+
/**
|
|
134
|
+
* Does this test file sit at a `{surface}` path configured FOR THIS SURFACE?
|
|
135
|
+
*
|
|
136
|
+
* The placeholder is replaced with the surface's own name, so
|
|
137
|
+
* `tests/{surface}/evals/*.yaml` credits `tests/mysql-designer/evals/x.yaml` to
|
|
138
|
+
* `mysql-designer` and to nothing else. A glob WITHOUT the placeholder returns
|
|
139
|
+
* false here on purpose: a plain custom glob widens what counts as a test file,
|
|
140
|
+
* which it always did, but it says nothing about WHICH surface the file is for
|
|
141
|
+
* — and inferring that from a substring is exactly the retired `name-mentioned`
|
|
142
|
+
* tier that credited surfaces nothing had touched.
|
|
143
|
+
*
|
|
144
|
+
* `minimatch` (already a direct dependency, pure JS) so the browser twin can
|
|
145
|
+
* share this instead of growing a second matcher that disagrees.
|
|
146
|
+
*/
|
|
147
|
+
export declare function matchesSurfaceGlob(surface: Pick<CoverableSurface, "name">, testPath: string, globs: readonly string[]): boolean;
|
|
104
148
|
/** Per-evidence tallies — the provenance summary the report prints. */
|
|
105
149
|
export interface EvidenceCounts {
|
|
106
150
|
/** Decided by a recorded run against this version of the surface. */
|
|
107
151
|
readonly executed: number;
|
|
108
152
|
readonly colocated: number;
|
|
153
|
+
/** Decided by a `{surface}` testGlob — the name still binds, the place moved. */
|
|
154
|
+
readonly configured: number;
|
|
109
155
|
}
|
|
110
156
|
/** Tally a list of decisions by evidence kind. */
|
|
111
157
|
export declare function countEvidence(decisions: readonly {
|