vigiles 10.0.0 → 12.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude-plugin/plugin.json +9 -0
- package/README.md +121 -86
- package/action.yml +13 -2
- package/dist/adapter-conformance.js +6 -0
- package/dist/adapter-registry.d.ts +20 -0
- package/dist/adapter-registry.js +27 -0
- package/dist/adapters/claude-code/dialect.js +15 -0
- package/dist/adapters/claude-code/hook-protocol.js +4 -0
- package/dist/adapters/claude-code/runtime.js +12 -0
- package/dist/adapters/codex/eval.js +3 -0
- package/dist/adapters/codex/hook-protocol.d.ts +9 -1
- package/dist/adapters/codex/hook-protocol.js +10 -0
- package/dist/adapters/codex/runtime.js +10 -0
- package/dist/adapters/opencode/runtime.js +4 -0
- package/dist/audit-report.d.ts +1 -1
- package/dist/audit-report.template.html +1 -1
- package/dist/audit-score.d.ts +19 -12
- package/dist/audit-score.js +65 -11
- package/dist/cli-commands.d.ts +1 -1
- package/dist/cli-commands.js +1 -0
- package/dist/cli.js +460 -29
- package/dist/core/CLAUDE.md.spec.d.ts +3 -0
- package/dist/core/CLAUDE.md.spec.js +26 -0
- package/dist/core/delegation-trifecta.d.ts +64 -0
- package/dist/core/delegation-trifecta.js +124 -0
- package/dist/core/dialect.d.ts +18 -0
- package/dist/core/hook-block-ineffective.d.ts +62 -0
- package/dist/core/hook-block-ineffective.js +153 -0
- package/dist/core/hook-matcher.d.ts +66 -0
- package/dist/core/hook-matcher.js +182 -0
- package/dist/core/hook-normalize.d.ts +43 -0
- package/dist/core/hook-normalize.js +78 -0
- package/dist/core/hook-protocol.d.ts +15 -0
- package/dist/core/lethal-trifecta.d.ts +100 -0
- package/dist/core/lethal-trifecta.js +197 -0
- package/dist/core/plugin-dir-layout.d.ts +30 -0
- package/dist/core/plugin-dir-layout.js +73 -0
- package/dist/core/rule-meta.d.ts +82 -0
- package/dist/core/rule-meta.js +266 -0
- package/dist/core/runtime.d.ts +20 -0
- package/dist/core/skill-missing-fence.d.ts +47 -0
- package/dist/core/skill-missing-fence.js +119 -0
- package/dist/core/skill-resources.d.ts +27 -0
- package/dist/core/skill-resources.js +167 -0
- package/dist/core/types.d.ts +83 -0
- package/dist/core/validate.d.ts +1 -0
- package/dist/core/validate.js +26 -4
- package/dist/eval-cache.d.ts +6 -0
- package/dist/eval-cache.js +2 -0
- package/dist/eval-lock.d.ts +192 -0
- package/dist/eval-lock.js +286 -0
- package/dist/eval.d.ts +33 -20
- package/dist/eval.js +199 -51
- package/dist/leaderboard.d.ts +1 -0
- package/dist/leaderboard.js +42 -4
- package/dist/scan.d.ts +106 -0
- package/dist/scan.js +251 -45
- package/dist/setup-plan.d.ts +43 -3
- package/dist/setup-plan.js +78 -6
- package/hooks/eval-lock-nudge.sh +21 -0
- package/package.json +1 -1
- package/skills/test-harness/SKILL.md +27 -0
|
@@ -39,6 +39,15 @@
|
|
|
39
39
|
"command": "bash ${CLAUDE_PLUGIN_ROOT}/hooks/refs-nudge.sh"
|
|
40
40
|
}
|
|
41
41
|
]
|
|
42
|
+
},
|
|
43
|
+
{
|
|
44
|
+
"matcher": "Edit|Write",
|
|
45
|
+
"hooks": [
|
|
46
|
+
{
|
|
47
|
+
"type": "command",
|
|
48
|
+
"command": "bash ${CLAUDE_PLUGIN_ROOT}/hooks/eval-lock-nudge.sh"
|
|
49
|
+
}
|
|
50
|
+
]
|
|
42
51
|
}
|
|
43
52
|
],
|
|
44
53
|
"SessionStart": [
|
package/README.md
CHANGED
|
@@ -6,13 +6,16 @@
|
|
|
6
6
|
SPINE = CONCEPT 5 (proof/demo-led). Lead with REAL, screenshotable catches on
|
|
7
7
|
plugins people actually ship, THEN explain the mechanism. The proofs are not
|
|
8
8
|
illustrative — every block traces to a real dogfood run captured in
|
|
9
|
-
research/dogfood/.
|
|
10
|
-
→
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
9
|
+
research/dogfood/. THREE COMMUNITY catches, anonymized (2026-06-29: Proof 1 is now a
|
|
10
|
+
lethal-trifecta exfil path → Safety 80, from madappgang's `tester` shown as
|
|
11
|
+
"my-plugin" — added to pay off the new Safety-ring hero; Proof 2 a skill-description
|
|
12
|
+
collision → wrong-skill-fires (claude-flow, Triggering F); Proof 3 an
|
|
13
|
+
AskUserQuestion-never-available tool) — all real GRADED/structural defects that
|
|
14
|
+
REPRODUCE on current main. NEVER replace a real catch with a fabricated one. (The
|
|
15
|
+
earlier Proof 1 was a missing-SKILL.md/Truthfulness catch, swapped 2026-06-28: its
|
|
16
|
+
source (superpowers) is clean on current main and NO reproducible dead-file-ref
|
|
17
|
+
exists in popular OSS — those are an adopt+strengthen payoff, see
|
|
18
|
+
research/oss-audit-render-findings.md.)
|
|
16
19
|
|
|
17
20
|
WHY ONLY TWO (decided 2026-06-28): the earlier Proofs 3-4 leaned on
|
|
18
21
|
pr-review-toolkit's "review agents inherit all tools" as an official-plugin
|
|
@@ -45,13 +48,32 @@
|
|
|
45
48
|
6. NO INTERNAL VOCABULARY (moat / measurement-authority / flywheel) and NO
|
|
46
49
|
research/ links — name the user benefit.
|
|
47
50
|
7. ASSETS: the hero vigiles-audit.png is a REAL current report (a community
|
|
48
|
-
plugin rendered as "my-plugin" to anonymize) —
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
51
|
+
plugin rendered as "my-plugin" to anonymize) — C 72 with five rings, the
|
|
52
|
+
SAFETY ring (80) flagging a subagent holding all three lethal-trifecta legs
|
|
53
|
+
(a prompt-injection exfil path) + an inline subagent-tool-contract fix; the
|
|
54
|
+
dramatic Safety catch is the whole point of leading with this report (chose
|
|
55
|
+
the "bite" over a clean A 92 on 2026-06-29). No dialect-drift banner (HTML
|
|
56
|
+
report is terminal-banner-free by design). Re-render via headless Chromium on
|
|
57
|
+
the React report if the UI changes (recipe: copy a trifecta-bearing plugin to
|
|
58
|
+
my-plugin/, `node dist/cli.js audit my-plugin --no-json --no-serve`,
|
|
59
|
+
headless_shell `--window-size=820,1180 --force-device-scale-factor=2
|
|
60
|
+
--screenshot` on vigiles-report.html, then `rm -rf my-plugin
|
|
61
|
+
vigiles-report.html`). (vigiles-demo.gif was removed
|
|
52
62
|
from Proof 1 — it rendered as a frozen half-typed terminal and was redundant
|
|
53
63
|
with the code block; if a lint demo returns, it belongs in the Lint section
|
|
54
64
|
with a non-frozen asset.)
|
|
65
|
+
|
|
66
|
+
READABILITY (the 2026-06-29 pass — why this reads the way it does):
|
|
67
|
+
A. ONE bold per block, on the single phrase the eye should catch. Bold
|
|
68
|
+
everywhere = bold nowhere. Link CTAs may stay bold (they're navigation).
|
|
69
|
+
B. ONE idea per sentence. No em-dash clause-chains, no stacked parentheticals.
|
|
70
|
+
If a clause needs a paren, cut it or give it its own line.
|
|
71
|
+
C. PLAIN words in every LEAD; push jargon (rings, recall/precision,
|
|
72
|
+
interceptTools, selector, deterministic) into the linked docs. A skimmer who
|
|
73
|
+
lives in Claude Code still may not know the vocabulary.
|
|
74
|
+
D. SHOW via the proofs/code blocks; don't stack adjectives ("real, popular,
|
|
75
|
+
free, model-less") on top of what the block already proves.
|
|
76
|
+
E. SELL the outcome before the mechanism; the instruments come AFTER the proofs.
|
|
55
77
|
-->
|
|
56
78
|
|
|
57
79
|
<p align="center">
|
|
@@ -61,7 +83,7 @@
|
|
|
61
83
|
<h1 align="center">vigiles</h1>
|
|
62
84
|
|
|
63
85
|
<p align="center">
|
|
64
|
-
<strong>The tests your
|
|
86
|
+
<strong>The tests your agent's skills and hooks never had.</strong>
|
|
65
87
|
</p>
|
|
66
88
|
|
|
67
89
|
<p align="center">
|
|
@@ -72,137 +94,149 @@
|
|
|
72
94
|
|
|
73
95
|
---
|
|
74
96
|
|
|
75
|
-
**You installed
|
|
76
|
-
|
|
77
|
-
|
|
97
|
+
**You installed some plugins and wrote a few skills. Do they actually work?**
|
|
98
|
+
|
|
99
|
+
Your skills, hooks, and instructions are your agent's **harness** — the half you wrote
|
|
100
|
+
and own, and the half nothing checks. A skill that never fires. Two skills the agent
|
|
101
|
+
confuses. A subagent wired to a tool that doesn't exist. It breaks silently, and you
|
|
102
|
+
find out mid-task.
|
|
78
103
|
|
|
79
|
-
|
|
80
|
-
any repo:
|
|
104
|
+
It's a library with no tests. This runs them:
|
|
81
105
|
|
|
82
106
|
```bash
|
|
83
107
|
npx vigiles audit
|
|
84
108
|
```
|
|
85
109
|
|
|
86
|
-
Here's what it
|
|
110
|
+
No key, no config, safe on any repo. Here's what it caught on plugins people actually
|
|
111
|
+
ship. ↓
|
|
87
112
|
|
|
88
|
-
##
|
|
113
|
+
## What it caught
|
|
89
114
|
|
|
90
115
|
<p align="center">
|
|
91
|
-
<img src="vigiles-audit.png" width="760" alt="vigiles audit report
|
|
116
|
+
<img src="vigiles-audit.png" width="760" alt="vigiles audit report scoring my-plugin C (72/100): five categories scored A–F — Truthfulness, Triggering, Structure, Safety, Tested — with the Safety category flagging a subagent that holds all three lethal-trifecta legs (a prompt-injection exfil path), plus an inline fix card for a subagent declaring a tool that doesn't exist" />
|
|
92
117
|
</p>
|
|
93
118
|
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
119
|
+
**Like Google's Lighthouse, but for your agent harness.** Five categories, each scored
|
|
120
|
+
A–F — Truthfulness, Triggering, Structure, Safety, Tested — with every fix shown inline.
|
|
121
|
+
|
|
122
|
+
It runs locally and only reads, so it's safe on any repo and the same on every OS.
|
|
123
|
+
For CI gating, use `vigiles lint` instead. **[Audit a harness →](docs/for-plugin-authors.md)**
|
|
124
|
+
|
|
125
|
+
## Proof 1 — your agent can read your secrets and ship them out
|
|
126
|
+
|
|
127
|
+
```text
|
|
128
|
+
◑ Safety 80 (80/100)
|
|
129
|
+
└ subagent "tester" holds all three lethal-trifecta legs:
|
|
130
|
+
reads private data (Bash, Read) · takes in untrusted web content (WebFetch)
|
|
131
|
+
· can send data out (Bash, WebFetch)
|
|
132
|
+
```
|
|
133
|
+
|
|
134
|
+
Give one subagent all three powers and it's a **prompt-injection exfil path**: a poisoned
|
|
135
|
+
web page can tell it to read your `.env` and POST it anywhere — no exploit code, just the
|
|
136
|
+
tools it was handed. vigiles flags it from the tool list alone, free, no model.
|
|
137
|
+
**[How the Safety check works →](docs/for-plugin-authors.md)**
|
|
98
138
|
|
|
99
|
-
## Proof
|
|
139
|
+
## Proof 2 — two skills your agent can't tell apart
|
|
100
140
|
|
|
101
141
|
```text
|
|
102
|
-
✗ Triggering 0
|
|
103
|
-
└ 45 near-identical skill descriptions — the
|
|
104
|
-
so the wrong one fires (e.g. "agent-coder" ↔ "agent-tester", 83% alike)
|
|
142
|
+
✗ Triggering 0 (0/100)
|
|
143
|
+
└ 45 pairs of near-identical skill descriptions — the agent can't tell them
|
|
144
|
+
apart, so the wrong one fires (e.g. "agent-coder" ↔ "agent-tester", 83% alike)
|
|
105
145
|
```
|
|
106
146
|
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
the
|
|
147
|
+
One popular plugin ships **45 pairs of skills** with near-identical descriptions. The
|
|
148
|
+
agent picks which skill to run by reading those descriptions, so when two match it
|
|
149
|
+
fires the wrong one. The markdown is perfectly valid.
|
|
110
150
|
**[How triggering works →](docs/measuring-skills.md)**
|
|
111
151
|
|
|
112
|
-
## Proof
|
|
152
|
+
## Proof 3 — a tool your subagent silently can't call
|
|
113
153
|
|
|
114
154
|
```text
|
|
115
155
|
✗ tester — Tool "AskUserQuestion" is never available to a subagent.
|
|
116
156
|
→ remove or correct it — it's silently dropped from the contract.
|
|
117
157
|
```
|
|
118
158
|
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
harness against **reality**, not style — every file path, script, code symbol, and
|
|
123
|
-
linter rule across **7 catalogs** (ESLint, Ruff, Clippy + four more), each verified
|
|
124
|
-
to **exist _and_ be enabled**. **[Full guide →](docs/verifying-instruction-files.md)**
|
|
159
|
+
This subagent — a helper your main agent hands a task to — declares a tool that
|
|
160
|
+
doesn't exist. The harness drops it silently, so the agent loses a capability it
|
|
161
|
+
thinks it has. vigiles catches it and gives you the **one-line fix**.
|
|
125
162
|
|
|
126
|
-
|
|
127
|
-
|
|
163
|
+
That's the whole idea — it checks your harness against reality, not style. Every path,
|
|
164
|
+
script, code symbol, and linter rule, verified to exist _and_ be enabled across 7
|
|
165
|
+
catalogs (ESLint, Ruff, Clippy + four more).
|
|
166
|
+
**[Full guide →](docs/verifying-instruction-files.md)**
|
|
128
167
|
|
|
129
|
-
|
|
130
|
-
|
|
168
|
+
All three catches are free and need no model — and vigiles **prevents** other whole
|
|
169
|
+
classes of bug by construction (a typed spec or compiled hook just won't compile).
|
|
170
|
+
**[Everything it catches and prevents →](docs/what-vigiles-catches.md)** · point `audit`
|
|
171
|
+
at a whole marketplace and it ranks every plugin the same way.
|
|
172
|
+
**[Audit a marketplace →](docs/for-plugin-authors.md)**
|
|
131
173
|
|
|
132
|
-
## How it
|
|
174
|
+
## How it works
|
|
133
175
|
|
|
134
|
-
|
|
135
|
-
|
|
176
|
+
The model isn't yours to fix. Your harness is. `audit` shows you the problems — here's
|
|
177
|
+
what fixes and proves each one, almost all of it with no model and no key.
|
|
136
178
|
|
|
137
179
|
### 🔎 Lint — your CLAUDE.md stops lying
|
|
138
180
|
|
|
139
|
-
Every path, script, symbol
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
181
|
+
Every path, script, symbol, and rule verified against reality — the catches above.
|
|
182
|
+
You don't write the checks: `npx vigiles init` turns your CLAUDE.md, skills, and
|
|
183
|
+
subagents into _specs_ (same content, plus a layer vigiles can verify). Non-destructive,
|
|
184
|
+
edited by your agent in plain English, undone by `eject`.
|
|
185
|
+
**[How →](docs/verifying-instruction-files.md)**
|
|
144
186
|
|
|
145
187
|
### 🧪 Test — does the harness actually do its job?
|
|
146
188
|
|
|
147
|
-
A hook that blocks nothing, a skill that hijacks unrelated prompts,
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
```typescript
|
|
152
|
-
import { runHook } from "vigiles/testing";
|
|
153
|
-
|
|
154
|
-
const r = runHook(guard, {
|
|
155
|
-
hook_event_name: "PreToolUse",
|
|
156
|
-
tool_name: "Bash",
|
|
157
|
-
tool_input: { command: "git commit --no-verify" },
|
|
158
|
-
});
|
|
159
|
-
assert(r.blocked); // a red ✗ means your guard silently lets it through
|
|
160
|
-
```
|
|
161
|
-
|
|
162
|
-
Hooks **block** · skills **trigger** (recall _and_ precision) · subagents **finish**
|
|
163
|
-
in the outcome they promised (no LLM judge) · **safety holds** — `interceptTools`
|
|
164
|
-
catches a push or paid-API call as an _attempt_, so the side effect never happens.
|
|
165
|
-
Almost every tier runs with **no model and no key**, on every commit.
|
|
189
|
+
A hook that blocks nothing, a skill that hijacks unrelated prompts, context that never
|
|
190
|
+
reaches the model — each passes a naive "did it run?" check. vigiles tests the real
|
|
191
|
+
thing: hooks **block**, skills **fire**, subagents **finish what they promised**, and a
|
|
192
|
+
stray `git push` is caught before it happens. No model, no key, on every commit.
|
|
166
193
|
**[How testing works →](docs/harness-testing.md)**
|
|
167
194
|
|
|
168
195
|
### 📊 Eval — does a skill help, or just cost more?
|
|
169
196
|
|
|
170
|
-
_"65% fewer tokens." Says who?_ vigiles A/Bs the claim on real coding tasks and reports
|
|
171
|
-
the
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
**per token, every run**. vigiles answers most questions with **no model at all**, and
|
|
176
|
-
runs the rest on your own **Claude Pro/Max subscription — $0 extra.**
|
|
177
|
-
**[Measure a skill →](docs/measuring-skills.md)**
|
|
197
|
+
_"65% fewer tokens." Says who?_ vigiles[^name] A/Bs the claim on real coding tasks and reports
|
|
198
|
+
the token bill, whether it hit its target, and whether the code still works. promptfoo
|
|
199
|
+
and DeepEval bill **per token, every run**; vigiles runs on your own Claude Pro/Max
|
|
200
|
+
subscription. Evals run locally — a committed lock then lets **CI catch stale results with no
|
|
201
|
+
model call**. **[Measure a skill →](docs/measuring-skills.md)**
|
|
178
202
|
|
|
179
203
|
## Quick start
|
|
180
204
|
|
|
181
|
-
**
|
|
205
|
+
**1. See what's broken** — read-only, no setup:
|
|
206
|
+
|
|
207
|
+
```bash
|
|
208
|
+
npx vigiles audit
|
|
209
|
+
```
|
|
210
|
+
|
|
211
|
+
**2. Set it up** when you like what you see. Paste into Claude Code or Codex:
|
|
182
212
|
|
|
183
213
|
```text
|
|
184
214
|
Set up vigiles in this repo: run `npx vigiles init` and accept the defaults. If I
|
|
185
215
|
already have a CLAUDE.md or AGENTS.md, adopt it into a spec and show me which
|
|
186
|
-
references are stale. Then
|
|
187
|
-
|
|
188
|
-
eval without asking me first.
|
|
216
|
+
references are stale. Then compile and write + run one harness test for a hook or
|
|
217
|
+
skill of mine. Don't run a real-model eval without asking me first.
|
|
189
218
|
```
|
|
190
219
|
|
|
191
|
-
Or
|
|
220
|
+
Or run it yourself:
|
|
192
221
|
|
|
193
222
|
```bash
|
|
194
|
-
npx vigiles init #
|
|
223
|
+
npx vigiles init # adopts your files (non-destructive — eject reverses), adds CI,
|
|
224
|
+
# installs the Claude Code plugin globally
|
|
195
225
|
```
|
|
196
226
|
|
|
197
227
|
Interactive in a terminal, non-interactive for agents/CI (or `--yes`).
|
|
198
228
|
|
|
199
|
-
**
|
|
200
|
-
|
|
229
|
+
**Adoption is smooth: one command, then your agent does the rest.** `init` installs
|
|
230
|
+
the **skills and hooks**, so a plain-English ask does the work — no specs to
|
|
231
|
+
hand-write, no hooks to wire:
|
|
201
232
|
|
|
202
|
-
- _"test my skills"_ → scaffolds **and runs** a trigger/behaviour test (`test-harness`)
|
|
233
|
+
- _"test my skills"_ → scaffolds **and runs** a trigger/behaviour test, then commits its result so CI can check it (`test-harness`)
|
|
203
234
|
- _"harden my rules"_ → upgrades prose guidance into enforced linter rules (`strengthen`)
|
|
204
235
|
- _"add a rule to my CLAUDE.md"_ → edits the source and recompiles (`edit-spec`)
|
|
205
236
|
|
|
237
|
+
The **hooks** keep it honest in-loop — nudging the agent to mark a reference or
|
|
238
|
+
refresh a stale eval — so there are no chores to remember.
|
|
239
|
+
|
|
206
240
|
<details>
|
|
207
241
|
<summary>What <code>init</code> sets up</summary>
|
|
208
242
|
|
|
@@ -227,6 +261,7 @@ JS **or** TS (`*.harness.{mjs,ts}`) — run with `npx vigiles test`.
|
|
|
227
261
|
|
|
228
262
|
## More
|
|
229
263
|
|
|
264
|
+
- **[What vigiles catches and prevents →](docs/what-vigiles-catches.md)** — the full matrix of harness problems it handles, biggest first, marked prevent / catch / measure.
|
|
230
265
|
- **[CLI →](docs/cli.md)** · **[GitHub Action →](docs/github-action.md)** · the full **[lint rules matrix →](docs/verifying-instruction-files.md#the-validation-rules--the-full-matrix)** lives with the linting guide.
|
|
231
266
|
- **[Skills →](docs/skills.md)** — the skills `init` installs, and how the model-invocable ones trigger.
|
|
232
267
|
- **[Ship plugins? The plugin-author guide →](docs/for-plugin-authors.md)** — scan a draft, make your skills fire, rank a whole marketplace — no key.
|
package/action.yml
CHANGED
|
@@ -7,7 +7,11 @@ branding:
|
|
|
7
7
|
|
|
8
8
|
inputs:
|
|
9
9
|
command:
|
|
10
|
-
description:
|
|
10
|
+
description: >
|
|
11
|
+
Which vigiles command to run: 'lint' (verify references + integrity +
|
|
12
|
+
coverage), 'compile' (specs → markdown), or 'eval-check' (verify committed
|
|
13
|
+
eval locks against current inputs — the staleness gate; runs NO model, so
|
|
14
|
+
it is the CI-safe half of evals you produce locally with `vigiles eval --update`).
|
|
11
15
|
required: false
|
|
12
16
|
default: "lint"
|
|
13
17
|
paths:
|
|
@@ -65,7 +69,14 @@ runs:
|
|
|
65
69
|
set -euo pipefail
|
|
66
70
|
|
|
67
71
|
cmd="${VIGILES_COMMAND:-lint}"
|
|
68
|
-
|
|
72
|
+
# 'eval-check' is the CI staleness gate — it maps to the real verb
|
|
73
|
+
# `eval --check` (verify committed locks vs current inputs, NO model).
|
|
74
|
+
# Real evals run locally on a subscription (`eval --update`), never in CI.
|
|
75
|
+
if [[ "$cmd" == "eval-check" ]]; then
|
|
76
|
+
args=("eval" "--check")
|
|
77
|
+
else
|
|
78
|
+
args=("$cmd")
|
|
79
|
+
fi
|
|
69
80
|
|
|
70
81
|
# paths: split on commas and whitespace into positional args.
|
|
71
82
|
paths="${VIGILES_PATHS:-}"
|
|
@@ -67,6 +67,12 @@ function checkAdapterConformance(adapter) {
|
|
|
67
67
|
need(adapter.hookProtocol !== undefined, "capabilities.shellHooks is true but hookProtocol is missing");
|
|
68
68
|
if (adapter.hookProtocol) {
|
|
69
69
|
need(Number.isInteger(adapter.hookProtocol.blockExitCode), "hookProtocol.blockExitCode is not an integer");
|
|
70
|
+
// A shell-hook harness must declare WHICH events can inject developer
|
|
71
|
+
// context (`additionalContext`). Encoding it makes "can this harness
|
|
72
|
+
// deliver an inject hook?" a tested contract — the gap that let Codex's
|
|
73
|
+
// inject support sit unverified in prose. Empty would mean the harness
|
|
74
|
+
// can't inject context from a hook at all; every harness we support can.
|
|
75
|
+
need(adapter.hookProtocol.injectableEvents.length > 0, "hookProtocol.injectableEvents is empty — a shell-hook harness must declare the events that honor additionalContext injection (or it can't deliver an inject/nudge hook)");
|
|
70
76
|
portNames.push(["hookProtocol", adapter.hookProtocol.name]);
|
|
71
77
|
}
|
|
72
78
|
}
|
|
@@ -83,4 +83,24 @@ export declare function resolveHarnessSelection(opts: {
|
|
|
83
83
|
flag?: string;
|
|
84
84
|
configHarness?: string | readonly string[];
|
|
85
85
|
}): HarnessSelection;
|
|
86
|
+
/**
|
|
87
|
+
* The FULL adapter set a compile-time INSTALL should fan out to. Unlike
|
|
88
|
+
* `resolveHarnessSelection` (which picks ONE dialect for a single-output compile,
|
|
89
|
+
* since you emit a markdown file in one harness's format), an install writes the
|
|
90
|
+
* SAME artifact into EVERY enabled harness's native config — so a repo targeting
|
|
91
|
+
* both harnesses gets a compiled hook in `.claude/settings.json` AND
|
|
92
|
+
* `.codex/config.toml`, not just the first. Precedence mirrors the single picker:
|
|
93
|
+
*
|
|
94
|
+
* 1. `--harness=` flag → just that one (an explicit override is singular).
|
|
95
|
+
* 2. config `harness` list → ALL of them (the multi-harness fan-out).
|
|
96
|
+
* 3. no config → auto-detect → the one detected.
|
|
97
|
+
*
|
|
98
|
+
* Returns ≥1 adapter, de-duplicated by name (a config that lists a harness twice,
|
|
99
|
+
* or an alias + its canonical, collapses to one install).
|
|
100
|
+
*/
|
|
101
|
+
export declare function resolveHarnessAdapters(opts: {
|
|
102
|
+
root: string;
|
|
103
|
+
flag?: string;
|
|
104
|
+
configHarness?: string | readonly string[];
|
|
105
|
+
}): HarnessAdapter[];
|
|
86
106
|
//# sourceMappingURL=adapter-registry.d.ts.map
|
package/dist/adapter-registry.js
CHANGED
|
@@ -9,6 +9,7 @@ exports.adapterForInstructionFile = adapterForInstructionFile;
|
|
|
9
9
|
exports.resolveAdapter = resolveAdapter;
|
|
10
10
|
exports.normalizeHarnessList = normalizeHarnessList;
|
|
11
11
|
exports.resolveHarnessSelection = resolveHarnessSelection;
|
|
12
|
+
exports.resolveHarnessAdapters = resolveHarnessAdapters;
|
|
12
13
|
const adapter_js_1 = require("./adapters/claude-code/adapter.js");
|
|
13
14
|
const adapter_js_2 = require("./adapters/codex/adapter.js");
|
|
14
15
|
/** The default adapter when detection finds no harness markers. */
|
|
@@ -127,4 +128,30 @@ function resolveHarnessSelection(opts) {
|
|
|
127
128
|
}
|
|
128
129
|
return { kind: "ok", adapter: det.adapter };
|
|
129
130
|
}
|
|
131
|
+
/**
|
|
132
|
+
* The FULL adapter set a compile-time INSTALL should fan out to. Unlike
|
|
133
|
+
* `resolveHarnessSelection` (which picks ONE dialect for a single-output compile,
|
|
134
|
+
* since you emit a markdown file in one harness's format), an install writes the
|
|
135
|
+
* SAME artifact into EVERY enabled harness's native config — so a repo targeting
|
|
136
|
+
* both harnesses gets a compiled hook in `.claude/settings.json` AND
|
|
137
|
+
* `.codex/config.toml`, not just the first. Precedence mirrors the single picker:
|
|
138
|
+
*
|
|
139
|
+
* 1. `--harness=` flag → just that one (an explicit override is singular).
|
|
140
|
+
* 2. config `harness` list → ALL of them (the multi-harness fan-out).
|
|
141
|
+
* 3. no config → auto-detect → the one detected.
|
|
142
|
+
*
|
|
143
|
+
* Returns ≥1 adapter, de-duplicated by name (a config that lists a harness twice,
|
|
144
|
+
* or an alias + its canonical, collapses to one install).
|
|
145
|
+
*/
|
|
146
|
+
function resolveHarnessAdapters(opts) {
|
|
147
|
+
const { root, flag, configHarness } = opts;
|
|
148
|
+
if (flag !== undefined && flag !== "")
|
|
149
|
+
return [resolveAdapter(root, flag)];
|
|
150
|
+
const list = normalizeHarnessList(configHarness);
|
|
151
|
+
const adapters = list.length > 0
|
|
152
|
+
? list.map((h) => resolveAdapter(root, h))
|
|
153
|
+
: [detectAdapterResult(root).adapter];
|
|
154
|
+
const seen = new Set();
|
|
155
|
+
return adapters.filter((a) => !seen.has(a.name) && seen.add(a.name));
|
|
156
|
+
}
|
|
130
157
|
//# sourceMappingURL=adapter-registry.js.map
|
|
@@ -79,6 +79,21 @@ exports.claudeCodeDialect = {
|
|
|
79
79
|
"SessionStart",
|
|
80
80
|
"SessionEnd",
|
|
81
81
|
],
|
|
82
|
+
// Events where a block decision is silently ignored ENTIRELY — no veto AND no
|
|
83
|
+
// model feedback (exit 2 there writes stderr only to the user). These are the
|
|
84
|
+
// ONLY events hook-block-ineffective flags as wrong-event. PostToolUse is NOT
|
|
85
|
+
// here: its exit 2 feeds stderr back to the model (a legitimate nudge/feedback
|
|
86
|
+
// channel), so flagging it would cry wolf (e.g. vigiles's own refs-nudge.sh).
|
|
87
|
+
noEffectHookEvents: [
|
|
88
|
+
"SessionStart",
|
|
89
|
+
"SessionEnd",
|
|
90
|
+
"Notification",
|
|
91
|
+
"PreCompact",
|
|
92
|
+
],
|
|
93
|
+
// PreToolUse is the one event whose deny needs the structured
|
|
94
|
+
// `hookSpecificOutput.permissionDecision:"deny"`; the legacy top-level
|
|
95
|
+
// `decision` field is ignored there.
|
|
96
|
+
permissionDecisionHookEvents: ["PreToolUse"],
|
|
82
97
|
// Claude Code natively reads CLAUDE.md only — it does NOT auto-load AGENTS.md
|
|
83
98
|
// (anthropics/claude-code#34235 is open; AGENTS.md works solely via an
|
|
84
99
|
// `@AGENTS.md` import inside CLAUDE.md or a symlink). AGENTS.md is the
|
|
@@ -6,5 +6,9 @@ exports.claudeCodeHookProtocol = {
|
|
|
6
6
|
blockExitCode: 2,
|
|
7
7
|
denyDecisionValues: ["block", "deny"],
|
|
8
8
|
eventEnvVars: [],
|
|
9
|
+
// Events that honor `hookSpecificOutput.additionalContext` (developer-context
|
|
10
|
+
// injection). Covers vigiles's shipped inject hooks: the SessionStart lint
|
|
11
|
+
// summary and the PostToolUse refs / eval-lock nudges.
|
|
12
|
+
injectableEvents: ["SessionStart", "UserPromptSubmit", "PostToolUse"],
|
|
9
13
|
};
|
|
10
14
|
//# sourceMappingURL=hook-protocol.js.map
|
|
@@ -22,6 +22,18 @@ exports.claudeCodeRuntime = {
|
|
|
22
22
|
},
|
|
23
23
|
};
|
|
24
24
|
},
|
|
25
|
+
/**
|
|
26
|
+
* Claude Code keys on **major.minor**: a minor/major bump is where the system
|
|
27
|
+
* prompt + tool defs actually move (0.2 → 1.0 → 2.0 → 2.1, ~quarterly), while
|
|
28
|
+
* the daily patch stream rarely changes behavior — so keying patches would
|
|
29
|
+
* churn the cache for no signal. Falls back to the trimmed raw string when no
|
|
30
|
+
* semver is found. (If a specific patch is known to matter, clear the cache or
|
|
31
|
+
* bump `CACHE_FORMAT_VERSION`.)
|
|
32
|
+
*/
|
|
33
|
+
versionKey(raw) {
|
|
34
|
+
const m = /(\d+)\.(\d+)\.\d+/.exec(raw);
|
|
35
|
+
return m ? `${m[1]}.${m[2]}` : raw.trim();
|
|
36
|
+
},
|
|
25
37
|
};
|
|
26
38
|
/**
|
|
27
39
|
* Build the spawn env that points the agent CLI at the mock model: the caller's
|
|
@@ -195,6 +195,9 @@ exports.codexEvalDriver = {
|
|
|
195
195
|
runner: codexEvalAgentRunner,
|
|
196
196
|
parse: parseCodexEvalRun,
|
|
197
197
|
runError: codexRunError,
|
|
198
|
+
// The harness identity → folded into the trigger-rate lock hash, so a report
|
|
199
|
+
// recorded on Claude Code is STALE if the eval is switched to Codex (and v.v.).
|
|
200
|
+
harness: "codex",
|
|
198
201
|
};
|
|
199
202
|
/**
|
|
200
203
|
* Spawn real `codex exec --json` for the eval tier (real model, the user's codex
|
|
@@ -1,9 +1,17 @@
|
|
|
1
1
|
/**
|
|
2
|
-
* codexHookProtocol —
|
|
2
|
+
* codexHookProtocol — Codex's hook wire protocol.
|
|
3
3
|
* Finding: it is essentially IDENTICAL to Claude Code's (exit 2 / `decision:block`
|
|
4
4
|
* / `permissionDecision:deny`) — the thin `HookProtocol` port was the right call.
|
|
5
5
|
* The genuine deltas are the env vars a hook receives + the TOML config format
|
|
6
6
|
* (the latter lives in PluginLayout.settingsFormat, not here).
|
|
7
|
+
*
|
|
8
|
+
* Context injection (`hookSpecificOutput.additionalContext`) is ALSO shared — same
|
|
9
|
+
* shape, confirmed against the official Codex hooks docs
|
|
10
|
+
* (developers.openai.com/codex/hooks): supported on SessionStart, UserPromptSubmit,
|
|
11
|
+
* PreToolUse, PostToolUse, SubagentStart. (Earlier docs called this "deferred" —
|
|
12
|
+
* it is not.) So vigiles's PostToolUse nudges + SessionStart summary deliver on
|
|
13
|
+
* Codex unchanged. Caveats: Stop/SubagentStop/PreCompact carry no context, and
|
|
14
|
+
* Codex marks a hook run failed if it emits an unsupported field for the event.
|
|
7
15
|
*/
|
|
8
16
|
import type { HookProtocol } from "../../core/hook-protocol.js";
|
|
9
17
|
export declare const codexHookProtocol: HookProtocol;
|
|
@@ -8,6 +8,16 @@ exports.codexHookProtocol = {
|
|
|
8
8
|
// Codex matchers are anchored regexes (`matcher = "^Bash$"`), unlike Claude
|
|
9
9
|
// Code's exact tool name / `A|B` alternation.
|
|
10
10
|
matcherStyle: "regex",
|
|
11
|
+
// Events that honor `hookSpecificOutput.additionalContext` on Codex, per the
|
|
12
|
+
// official hooks docs. Includes the events vigiles's shipped hooks use
|
|
13
|
+
// (PostToolUse, SessionStart), so those nudges reach the Codex agent too.
|
|
14
|
+
injectableEvents: [
|
|
15
|
+
"SessionStart",
|
|
16
|
+
"UserPromptSubmit",
|
|
17
|
+
"PreToolUse",
|
|
18
|
+
"PostToolUse",
|
|
19
|
+
"SubagentStart",
|
|
20
|
+
],
|
|
11
21
|
eventEnvVars: [
|
|
12
22
|
"session_id",
|
|
13
23
|
"cwd",
|
|
@@ -22,6 +22,16 @@ exports.codexRuntime = {
|
|
|
22
22
|
wireMock(baseUrl) {
|
|
23
23
|
return { args: codexMockArgs(baseUrl), env: codexMockEnv() };
|
|
24
24
|
},
|
|
25
|
+
/**
|
|
26
|
+
* Codex opts OUT of version partitioning (`""`). It is perpetual `0.x` where
|
|
27
|
+
* the *minor* is the patch cadence (~2 bumps/week, 134 minors in 14 months), so
|
|
28
|
+
* keying `major.minor` like Claude Code would churn the cache/lock weekly. With
|
|
29
|
+
* no stable behavior boundary in the version string, Codex relies on the dated
|
|
30
|
+
* model id + `evalApiVersion` for staleness instead. See research/cache-invalidation.md.
|
|
31
|
+
*/
|
|
32
|
+
versionKey(_raw) {
|
|
33
|
+
return "";
|
|
34
|
+
},
|
|
25
35
|
};
|
|
26
36
|
/**
|
|
27
37
|
* The PROVEN `-c` flag array that points `codex exec` at a mock served at
|
package/dist/audit-report.d.ts
CHANGED
|
@@ -77,7 +77,7 @@ export interface Adoptable {
|
|
|
77
77
|
*/
|
|
78
78
|
export interface AuditReport {
|
|
79
79
|
readonly meta: AuditReportMeta;
|
|
80
|
-
/** The
|
|
80
|
+
/** The five deterministic category rings + the weighted overall + grade. */
|
|
81
81
|
readonly score: AuditScore;
|
|
82
82
|
/** The deterministic, ranked fixes (the inline recommendations). */
|
|
83
83
|
readonly recommendations: readonly Recommendation[];
|