vigiles 12.1.0 → 12.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +128 -154
- package/action.yml +73 -0
- package/dist/adapters/claude-code/run-scripts.d.ts +41 -0
- package/dist/adapters/claude-code/run-scripts.js +26 -0
- package/dist/audit-report.d.ts +11 -0
- package/dist/audit-report.js +1 -0
- package/dist/audit-report.template.html +36 -26
- package/dist/cli-commands.d.ts +1 -1
- package/dist/cli-commands.js +0 -1
- package/dist/cli.js +111 -139
- package/dist/eval.d.ts +15 -0
- package/dist/eval.js +60 -0
- package/dist/observe.d.ts +109 -0
- package/dist/observe.js +164 -0
- package/dist/scaffold-test.js +3 -2
- package/package.json +2 -2
- package/skills/debug-my-harness/SKILL.md +56 -0
- package/dist/core/hook-spec.d.ts +0 -74
- package/dist/core/hook-spec.js +0 -130
package/README.md
CHANGED
|
@@ -1,79 +1,66 @@
|
|
|
1
1
|
<!--
|
|
2
2
|
README DIRECTION — read before editing; keep changes aligned.
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
with a non-frozen asset.)
|
|
65
|
-
|
|
66
|
-
READABILITY (the 2026-06-29 pass — why this reads the way it does):
|
|
67
|
-
A. ONE bold per block, on the single phrase the eye should catch. Bold
|
|
68
|
-
everywhere = bold nowhere. Link CTAs may stay bold (they're navigation).
|
|
69
|
-
B. ONE idea per sentence. No em-dash clause-chains, no stacked parentheticals.
|
|
70
|
-
If a clause needs a paren, cut it or give it its own line.
|
|
71
|
-
C. PLAIN words in every LEAD; push jargon (rings, recall/precision,
|
|
72
|
-
interceptTools, selector, deterministic) into the linked docs. A skimmer who
|
|
73
|
-
lives in Claude Code still may not know the vocabulary.
|
|
74
|
-
D. SHOW via the proofs/code blocks; don't stack adjectives ("real, popular,
|
|
75
|
-
free, model-less") on top of what the block already proves.
|
|
76
|
-
E. SELL the outcome before the mechanism; the instruments come AFTER the proofs.
|
|
3
|
+
Front door + marketing asset for someone who lives in Claude Code / Codex.
|
|
4
|
+
Optimize for a phone-skimmer who must come away knowing WHAT IT IS and wanting
|
|
5
|
+
to run it — never scared off. Validated by a 6-persona cold-read (2026-07):
|
|
6
|
+
newcomer / power-user / plugin-author / skeptical-senior / decision-maker /
|
|
7
|
+
Codex-user. The fixes below trace to that review — don't regress them.
|
|
8
|
+
|
|
9
|
+
HOOK = FELT PAIN, then breadth (founder direction 2026-07). Bold tagline is a
|
|
10
|
+
specific second-person pain: "You have a rule your agent follows half the time
|
|
11
|
+
— and no way to know which one." It's rhetorical (the reader's own uncertainty),
|
|
12
|
+
NOT an ecosystem stat — do NOT reintroduce a bare "%"/number in the tagline (the
|
|
13
|
+
doc mocks uncited stats: "'65% fewer tokens.' Says who?", so a fake stat up top
|
|
14
|
+
reads as hypocrisy). Dev-native CRAFT bait, NOT enterprise fear (OSS dev tool,
|
|
15
|
+
not a security product). The report-card + 5 rings RIGHT BELOW show the breadth
|
|
16
|
+
so the specific hook doesn't read as narrow.
|
|
17
|
+
|
|
18
|
+
DEFINE "HARNESS" on first use (the load-bearing noun, used ~15×) — gloss it as
|
|
19
|
+
"the CLAUDE.md/AGENTS.md rules, skills, subagents, and hooks steering your
|
|
20
|
+
agent." 4 of 6 personas bounced on it being undefined. Keep the gloss.
|
|
21
|
+
|
|
22
|
+
INSTRUCTION-NEUTRAL NOUNS (Codex was the lowest score): body copy says CLAUDE.md
|
|
23
|
+
OR AGENTS.md, never CLAUDE.md alone. The tagline may keep punch, but Codex must
|
|
24
|
+
appear within the first sentence or two (the harness gloss names AGENTS.md), and
|
|
25
|
+
every "adopts your CLAUDE.md"/"verifies your CLAUDE.md" gets "or AGENTS.md".
|
|
26
|
+
|
|
27
|
+
CATEGORY = A TOOL YOU RUN (ESLint/Lighthouse/npm audit class), NOT a framework,
|
|
28
|
+
NOT a lib-collection. The FAQ says this outright — it's the #1 thing that scares
|
|
29
|
+
people off. The library/subpath exports are the automation door for the 5%.
|
|
30
|
+
|
|
31
|
+
AUDIT vs LINT — ONE CONSISTENT, ACCURATE STORY (a persona caught 3 conflicting
|
|
32
|
+
answers): `lint` = the CI gate on the deterministic checks (broken refs, tool
|
|
33
|
+
contracts, dead hooks, skill collisions — Proofs 1 & 2). `audit` = those same
|
|
34
|
+
checks + the Safety ring + two opt-in LIVE checks (MCP connects? skills fire?)
|
|
35
|
+
+ the graded report. Do NOT claim lint gates the lethal-trifecta Safety flag
|
|
36
|
+
(it's an audit ring, not a default gating rule) and do NOT say lint is
|
|
37
|
+
refs-only. The table row, the reconciliation line, and the Lint subsection must
|
|
38
|
+
all agree.
|
|
39
|
+
|
|
40
|
+
SPINE = proof/demo-led. Real, screenshotable catches on shipped plugins, THEN
|
|
41
|
+
mechanism. Every proof traces to a real dogfood run (research/dogfood/) — NEVER
|
|
42
|
+
fabricate one. Order = most-RELATABLE first (broken tool ref → skill collision →
|
|
43
|
+
secrets-exfil gotcha last as the bite). The intro triplet maps 1:1 to the 3
|
|
44
|
+
proofs. Security is ONE dev-native GOTCHA proof, never the brand. Add a repro
|
|
45
|
+
line ("run `npx vigiles audit <any-repo>` for your own") + a one-line note that
|
|
46
|
+
examples use CC subagents but the checks run on Codex too. FALSE CONFIDENCE is
|
|
47
|
+
the coined term (a guard that looks like it works and silently doesn't), defined
|
|
48
|
+
once in the Test section.
|
|
49
|
+
|
|
50
|
+
FUNNEL: "How it works" opens with the audit/lint/test/eval verb-map table — the
|
|
51
|
+
load-bearing "one tool, not four" fix. Frame it vibes → verified. Name that
|
|
52
|
+
init/compile/eject manage the spec layer (personas noticed the verb-count gap).
|
|
53
|
+
|
|
54
|
+
DON'T SHAME OSS: catches are ANONYMIZED (no obra/superpowers, madappgang,
|
|
55
|
+
claude-flow by name) — real names live only in research/dogfood/.
|
|
56
|
+
Guard/compiled-hooks + the 2/7→7/7 battery are PARKED FOR LAUNCH — not the hero.
|
|
57
|
+
|
|
58
|
+
RULES: lead with the reader's CONCRETE PAIN; ≤ ~3-line paragraphs; ONE bold per
|
|
59
|
+
block; ONE idea per sentence; NO internal vocabulary (moat/flywheel) / NO
|
|
60
|
+
research/ links / NO enterprise/national-interest framing — name the user
|
|
61
|
+
benefit; ~220-line body cap; push depth into docs/ and LINK it. Assets: the hero
|
|
62
|
+
vigiles-audit.png is a REAL current report (a community plugin as "my-plugin"),
|
|
63
|
+
C 72, five rings — re-render via headless Chromium if the UI changes.
|
|
77
64
|
-->
|
|
78
65
|
|
|
79
66
|
<p align="center">
|
|
@@ -83,7 +70,11 @@
|
|
|
83
70
|
<h1 align="center">vigiles</h1>
|
|
84
71
|
|
|
85
72
|
<p align="center">
|
|
86
|
-
<strong>
|
|
73
|
+
<strong>You have a rule your agent follows half the time — and no way to know which one.</strong>
|
|
74
|
+
</p>
|
|
75
|
+
|
|
76
|
+
<p align="center">
|
|
77
|
+
Verify your CLAUDE.md or AGENTS.md, skills, and hooks are real — and prove they actually work.
|
|
87
78
|
</p>
|
|
88
79
|
|
|
89
80
|
<p align="center">
|
|
@@ -94,47 +85,44 @@
|
|
|
94
85
|
|
|
95
86
|
---
|
|
96
87
|
|
|
97
|
-
**You
|
|
88
|
+
**You review every PR. Nothing reviews your CLAUDE.md.**
|
|
98
89
|
|
|
99
|
-
Your skills,
|
|
100
|
-
and own, and the half nothing checks. A skill that never fires. Two skills the agent
|
|
101
|
-
confuses. A subagent wired to a tool that doesn't exist. It breaks silently, and you
|
|
102
|
-
find out mid-task.
|
|
90
|
+
Your **harness** — the CLAUDE.md or AGENTS.md rules, skills, subagents, and hooks steering your agent — is the one part nobody checks. Nobody verified it's real. Nobody tested it works. That's not a system. That's vibes.
|
|
103
91
|
|
|
104
|
-
|
|
92
|
+
And vibes break silently mid-task: a subagent wired to a tool that doesn't exist, two skills your agent can't tell apart, one helper quietly able to read your secrets and send them out.
|
|
93
|
+
|
|
94
|
+
vigiles[^name] checks your harness is _real_, not just well-formed — Claude Code and Codex alike. One command, no key, no config, safe on any repo:
|
|
105
95
|
|
|
106
96
|
```bash
|
|
107
97
|
npx vigiles audit
|
|
108
98
|
```
|
|
109
99
|
|
|
110
|
-
|
|
111
|
-
ship. ↓
|
|
100
|
+
It's free and open-source, runs entirely on your machine, and never bills per token. (`eval` is the only step that calls a model — on your own Claude subscription.) Here's what it caught on plugins people actually ship. ↓
|
|
112
101
|
|
|
113
102
|
## What it caught
|
|
114
103
|
|
|
115
104
|
<p align="center">
|
|
116
|
-
<img src="vigiles-audit.png" width="760" alt="vigiles audit report scoring my-plugin C (72/100): five categories scored A–F — Truthfulness, Triggering, Structure, Safety, Tested — with
|
|
105
|
+
<img src="vigiles-audit.png" width="760" alt="vigiles audit report scoring my-plugin C (72/100): five categories scored A–F — Truthfulness, Triggering, Structure, Safety, Tested — with an inline fix card for a subagent declaring a tool that doesn't exist" />
|
|
117
106
|
</p>
|
|
118
107
|
|
|
119
|
-
**Like Google's Lighthouse, but for your agent harness.**
|
|
120
|
-
|
|
108
|
+
**Like Google's Lighthouse, but for your agent harness.** One command grades it A–F across five categories, every fix shown inline:
|
|
109
|
+
|
|
110
|
+
- **Truthfulness** — do the references resolve?
|
|
111
|
+
- **Triggering** — do skills fire, without colliding?
|
|
112
|
+
- **Structure** — are tool contracts and configs valid?
|
|
113
|
+
- **Safety** — any way for the agent to leak your data?
|
|
114
|
+
- **Tested** — does the harness ship tests?
|
|
121
115
|
|
|
122
|
-
|
|
123
|
-
For CI gating, use `vigiles lint` instead. **[Audit a harness →](docs/for-plugin-authors.md)**
|
|
116
|
+
These are real scans of public plugins — run `npx vigiles audit <any-repo>` for your own. The examples below use Claude Code subagents; the same checks run on Codex `AGENTS.md`, skills, and hooks. ↓
|
|
124
117
|
|
|
125
|
-
## Proof 1 — your agent
|
|
118
|
+
## Proof 1 — a tool your agent thinks it has and doesn't
|
|
126
119
|
|
|
127
120
|
```text
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
reads private data (Bash, Read) · takes in untrusted web content (WebFetch)
|
|
131
|
-
· can send data out (Bash, WebFetch)
|
|
121
|
+
✗ tester — Tool "AskUserQuestion" is never available to a subagent.
|
|
122
|
+
→ remove or correct it — it's silently dropped from the contract.
|
|
132
123
|
```
|
|
133
124
|
|
|
134
|
-
|
|
135
|
-
web page can tell it to read your `.env` and POST it anywhere — no exploit code, just the
|
|
136
|
-
tools it was handed. vigiles flags it from the tool list alone, free, no model.
|
|
137
|
-
**[How the Safety check works →](docs/for-plugin-authors.md)**
|
|
125
|
+
This subagent — a helper your main agent hands work to — lists a tool that doesn't exist for it. The harness drops it without a word, so the agent quietly loses a capability it thinks it has. The markdown is perfectly valid. vigiles catches it and hands you the **one-line fix**.
|
|
138
126
|
|
|
139
127
|
## Proof 2 — two skills your agent can't tell apart
|
|
140
128
|
|
|
@@ -144,61 +132,50 @@ tools it was handed. vigiles flags it from the tool list alone, free, no model.
|
|
|
144
132
|
apart, so the wrong one fires (e.g. "agent-coder" ↔ "agent-tester", 83% alike)
|
|
145
133
|
```
|
|
146
134
|
|
|
147
|
-
One popular plugin ships **45 pairs of skills** with near-identical descriptions.
|
|
148
|
-
agent picks which skill to run by reading those descriptions, so when two match it
|
|
149
|
-
fires the wrong one. The markdown is perfectly valid.
|
|
135
|
+
One popular plugin ships **45 pairs of skills** with near-identical descriptions. Your agent picks which skill to run by _reading_ those descriptions, so when two match it fires the wrong one. Still perfectly valid markdown.
|
|
150
136
|
**[How triggering works →](docs/measuring-skills.md)**
|
|
151
137
|
|
|
152
|
-
## Proof 3 —
|
|
138
|
+
## Proof 3 — it can quietly read your secrets and send them out
|
|
153
139
|
|
|
154
140
|
```text
|
|
155
|
-
|
|
156
|
-
|
|
141
|
+
◑ Safety 80 (80/100)
|
|
142
|
+
└ subagent "tester" holds all three lethal-trifecta legs:
|
|
143
|
+
reads private data (Bash, Read) · takes in untrusted web content (WebFetch)
|
|
144
|
+
· can send data out (Bash, WebFetch)
|
|
157
145
|
```
|
|
158
146
|
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
147
|
+
Hand one subagent all three powers and a poisoned web page can tell it to read your `.env` and POST it anywhere — no exploit code, just the tools it was given. The **80 still looks like a B** — that's the point: a healthy-looking grade can hide a single subagent that's a data-leak waiting to happen. vigiles spots it from the tool list alone, free, no model.
|
|
148
|
+
|
|
149
|
+
That's the whole idea: it checks your harness against **reality, not style**. Every tool, hook, file, script, and skill you reference is verified to actually resolve — and where you name a linter rule, it's checked to exist _and_ be enabled (ESLint, Ruff, Clippy, and more).
|
|
150
|
+
**[Everything it catches →](docs/what-vigiles-catches.md)** · point `audit` at a whole marketplace and it ranks every plugin the same way.
|
|
162
151
|
|
|
163
|
-
|
|
164
|
-
script, code symbol, and linter rule, verified to exist _and_ be enabled across 7
|
|
165
|
-
catalogs (ESLint, Ruff, Clippy + four more).
|
|
166
|
-
**[Full guide →](docs/verifying-instruction-files.md)**
|
|
152
|
+
## How it works — vibes → verified
|
|
167
153
|
|
|
168
|
-
|
|
169
|
-
classes of bug by construction (a typed spec or compiled hook just won't compile).
|
|
170
|
-
**[Everything it catches and prevents →](docs/what-vigiles-catches.md)** · point `audit`
|
|
171
|
-
at a whole marketplace and it ranks every plugin the same way.
|
|
172
|
-
**[Audit a marketplace →](docs/for-plugin-authors.md)**
|
|
154
|
+
`audit` shows you where your setup is still vibes. Turning that into _verified_ is four commands over one engine — and almost none of it needs a model or a key.
|
|
173
155
|
|
|
174
|
-
|
|
156
|
+
| Command | Answers | Needs a model? | When to run |
|
|
157
|
+
| ------- | ------------------------------ | ------------------------ | ------------------------ |
|
|
158
|
+
| `audit` | Everything, graded A–F | No — read-only[^audit] | Anytime; it's the report |
|
|
159
|
+
| `lint` | Do the structural checks pass? | No | CI gate, every push |
|
|
160
|
+
| `test` | Does the harness behave? | No — a scripted stand-in | Every commit |
|
|
161
|
+
| `eval` | Does a skill actually help? | Yes — your subscription | On demand |
|
|
175
162
|
|
|
176
|
-
|
|
177
|
-
what fixes and proves each one, almost all of it with no model and no key.
|
|
163
|
+
`audit` and `lint` share one engine. **`lint` is the CI gate** — it fails the build on broken references, bad tool contracts, dead hooks, and skill collisions (Proofs 1 and 2). **`audit`** runs those same checks, adds the Safety ring, renders the graded report, and can also run two opt-in _live_ checks (does your MCP server connect, do your skills fire). `test` and `eval` go past _does it exist_ to _does it work_. (`init` / `compile` / `eject` manage the spec layer underneath; you rarely run them by hand.)
|
|
178
164
|
|
|
179
|
-
### 🔎 Lint — your
|
|
165
|
+
### 🔎 Lint — your instructions stop lying
|
|
180
166
|
|
|
181
|
-
Every path, script, symbol, and rule verified against reality — the catches above.
|
|
182
|
-
You don't write the checks: `npx vigiles init` turns your CLAUDE.md, skills, and
|
|
183
|
-
subagents into _specs_ (same content, plus a layer vigiles can verify). Non-destructive,
|
|
184
|
-
edited by your agent in plain English, undone by `eject`.
|
|
167
|
+
Every path, script, symbol, and rule verified against reality — plus tool contracts, skill collisions, and dead hooks (the catches above). You don't write the checks. `npx vigiles init` writes a `CLAUDE.md.spec.ts` beside your file: the same rules, each reference now wrapped so vigiles can confirm it exists. `compile` turns that back into the `CLAUDE.md` (or `AGENTS.md`) your agent already reads. Your agent edits the spec in plain English; `eject` deletes it and leaves your original untouched.
|
|
185
168
|
**[How →](docs/verifying-instruction-files.md)**
|
|
186
169
|
|
|
187
170
|
### 🧪 Test — does the harness actually do its job?
|
|
188
171
|
|
|
189
|
-
A hook that blocks nothing, a skill that hijacks unrelated prompts, context that never
|
|
190
|
-
reaches the model — each passes a naive "did it run?" check. vigiles tests the real
|
|
191
|
-
thing: hooks **block**, skills **fire**, subagents **finish what they promised**, and a
|
|
192
|
-
stray `git push` is caught before it happens. No model, no key, on every commit.
|
|
172
|
+
A hook that blocks nothing, a skill that hijacks unrelated prompts, context that never reaches the model — each passes a naive "did it run?" check. That gap is **false confidence**: a guard that looks like it works and silently doesn't. vigiles tests the real thing — hooks block, skills fire, subagents finish what they promised, a stray `git push` is caught before it happens. It drives a scripted stand-in for the model, not a live call, so it needs no key and runs on every commit.
|
|
193
173
|
**[How testing works →](docs/harness-testing.md)**
|
|
194
174
|
|
|
195
175
|
### 📊 Eval — does a skill help, or just cost more?
|
|
196
176
|
|
|
197
|
-
_"65% fewer tokens." Says who?_ vigiles
|
|
198
|
-
|
|
199
|
-
and DeepEval bill **per token, every run**; vigiles runs on your own Claude Pro/Max
|
|
200
|
-
subscription. Evals run locally — a committed lock then lets **CI catch stale results with no
|
|
201
|
-
model call**. **[Measure a skill →](docs/measuring-skills.md)**
|
|
177
|
+
_"65% fewer tokens." Says who?_ vigiles A/Bs the claim on real coding tasks and reports the token bill, whether it hit its target, and whether the code still works. promptfoo and DeepEval bill **per token, every run**; vigiles runs on your own Claude Pro/Max subscription. Evals run locally; a committed lock file — like a `package-lock` — records the result, so CI catches stale numbers without calling the model again. (On Claude Code today; Codex eval support is landing.)
|
|
178
|
+
**[Measure a skill →](docs/measuring-skills.md)**
|
|
202
179
|
|
|
203
180
|
## Quick start
|
|
204
181
|
|
|
@@ -221,21 +198,19 @@ Or run it yourself:
|
|
|
221
198
|
|
|
222
199
|
```bash
|
|
223
200
|
npx vigiles init # adopts your files (non-destructive — eject reverses), adds CI,
|
|
224
|
-
# installs
|
|
201
|
+
# installs vigiles's skills + hooks as a Claude Code plugin (in
|
|
202
|
+
# ~/.claude/, not your repo). On Codex, skills install globally too.
|
|
225
203
|
```
|
|
226
204
|
|
|
227
|
-
Interactive in a terminal, non-interactive for agents/CI (or `--yes`).
|
|
205
|
+
Interactive in a terminal, non-interactive for agents/CI (or `--yes`). **Works with Claude Code and Codex** — vigiles verifies `CLAUDE.md` and `AGENTS.md` the same way. **[Codex setup →](docs/harnesses.md)**
|
|
228
206
|
|
|
229
|
-
**Adoption is smooth: one command, then your agent does the rest.** `init` installs
|
|
230
|
-
the **skills and hooks**, so a plain-English ask does the work — no specs to
|
|
231
|
-
hand-write, no hooks to wire:
|
|
207
|
+
**Adoption is smooth: one command, then your agent does the rest.** `init` installs the **skills and hooks**, so a plain-English ask does the work — no specs to hand-write, no hooks to wire:
|
|
232
208
|
|
|
233
209
|
- _"test my skills"_ → scaffolds **and runs** a trigger/behaviour test, then commits its result so CI can check it (`test-harness`)
|
|
234
210
|
- _"harden my rules"_ → upgrades prose guidance into enforced linter rules (`strengthen`)
|
|
235
|
-
- _"add a rule to my CLAUDE.md"_ → edits the source and recompiles (`edit-spec`)
|
|
211
|
+
- _"add a rule to my CLAUDE.md or AGENTS.md"_ → edits the source and recompiles (`edit-spec`)
|
|
236
212
|
|
|
237
|
-
The **hooks** keep it honest in-loop — nudging the agent to
|
|
238
|
-
refresh a stale eval — so there are no chores to remember.
|
|
213
|
+
The **hooks** keep it honest in-loop — nudging the agent to tag a linter-rule mention so vigiles can verify it, or to re-run a test whose result just went stale — so there are no chores to remember.
|
|
239
214
|
|
|
240
215
|
<details>
|
|
241
216
|
<summary>What <code>init</code> sets up</summary>
|
|
@@ -243,35 +218,34 @@ refresh a stale eval — so there are no chores to remember.
|
|
|
243
218
|
- **Both lint and test** by default; scope with `--lint` / `--test`.
|
|
244
219
|
- **Already have a CLAUDE.md / AGENTS.md, skills, or subagents? `init` adopts them all** into specs faithfully and **non-destructively** — untouched until you `compile` (and `eject` undoes it).
|
|
245
220
|
- Adds `vigiles` to `devDependencies`; installs the Claude Code plugin (skills + hooks) via the marketplace — globally, never vendored.
|
|
246
|
-
- Wires CI as a `zernie/vigiles@v1` workflow that posts a sticky PR comment + a `valid` output.
|
|
221
|
+
- Wires CI as a `zernie/vigiles@v1` workflow (needs only read + PR-comment permissions) that posts a sticky PR comment + a `valid` output.
|
|
247
222
|
|
|
248
|
-
|
|
249
|
-
[your own harness](docs/authoring-an-adapter.md). Prefer to write tests yourself?
|
|
250
|
-
JS **or** TS (`*.harness.{mjs,ts}`) — run with `npx vigiles test`.
|
|
223
|
+
Targets Claude Code and Codex out of the box, or [your own harness](docs/authoring-an-adapter.md). Prefer to write tests yourself? JS **or** TS (`*.harness.{mjs,ts}`) — run with `npx vigiles test`.
|
|
251
224
|
|
|
252
225
|
</details>
|
|
253
226
|
|
|
254
227
|
## FAQ
|
|
255
228
|
|
|
229
|
+
- **Is this a framework I have to build around?** No. It's a tool you run — like ESLint, Lighthouse, or `npm audit`. One command, a report, an optional CI gate. There's a library API for automation, but you never touch it to get value.
|
|
256
230
|
- **Isn't this just a markdown linter?** No — it checks whether your instruction file is _true_ (every path/script/symbol/rule exists and is enabled), then tests and measures your harness. A style linter can't do any of that.
|
|
257
|
-
- **Do I have to write TypeScript?** No — your agent writes the spec (`init` adopts your CLAUDE.md into one), or plain markdown lints with zero new files. Compiler-grade guarantees are opt-in, like TS's `strict` ([why?](docs/faq.md#why-are-the-strongest-guarantees-opt-in-not-the-default)).
|
|
258
|
-
- **
|
|
231
|
+
- **Do I have to write TypeScript?** No — your agent writes the spec (`init` adopts your CLAUDE.md or AGENTS.md into one), or plain markdown lints with zero new files. Compiler-grade guarantees are opt-in, like TS's `strict` ([why?](docs/faq.md#why-are-the-strongest-guarantees-opt-in-not-the-default)).
|
|
232
|
+
- **Is it stable enough to adopt?** Yes — the CLI is stable; only the library API is still evolving ([details](STABILITY.md)).
|
|
233
|
+
- **Non-JS repo?** `npx vigiles lint` verifies your CLAUDE.md or AGENTS.md with no install (Ruff/Clippy/Pylint/… too).
|
|
259
234
|
|
|
260
235
|
**[Full FAQ →](docs/faq.md)**
|
|
261
236
|
|
|
237
|
+
**Not for you if** you want a model/capability benchmark or runtime guardrails in the request path — vigiles is build-/CI-time.
|
|
238
|
+
|
|
262
239
|
## More
|
|
263
240
|
|
|
264
|
-
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
- **[Ship plugins? The plugin-author guide →](docs/for-plugin-authors.md)** — scan a draft, make your skills fire, rank a whole marketplace — no key.
|
|
268
|
-
- **[Docs index →](docs/README.md)** · **[API reference →](https://zernie.github.io/vigiles/)** · **[Related tools →](docs/related-tools.md)**.
|
|
269
|
-
- **[Stability →](STABILITY.md)** — 0.x: the CLI is stable; the library API is still evolving.
|
|
270
|
-
- **Not for you if** you want a model/capability benchmark or runtime guardrails in the request path — vigiles is build-/CI-time.
|
|
271
|
-
- Companion to [Feedback Loop Is All You Need](https://zernie.com/blog/feedback-loop-is-all-you-need).
|
|
241
|
+
**Docs** — **[What it catches and prevents →](docs/what-vigiles-catches.md)** · **[Verifying instruction files →](docs/verifying-instruction-files.md)** ([rules matrix](docs/verifying-instruction-files.md#the-validation-rules--the-full-matrix)) · **[Harness testing →](docs/harness-testing.md)** · **[Measuring skills →](docs/measuring-skills.md)** · **[CLI →](docs/cli.md)** · **[GitHub Action →](docs/github-action.md)** · **[Skills →](docs/skills.md)** · **[Plugin-author guide →](docs/for-plugin-authors.md)** · **[Docs index →](docs/README.md)** · **[API reference →](https://zernie.github.io/vigiles/)**
|
|
242
|
+
|
|
243
|
+
**Project** — **[Stability →](STABILITY.md)** · **[Related tools →](docs/related-tools.md)** · companion to [Feedback Loop Is All You Need](https://zernie.com/blog/feedback-loop-is-all-you-need).
|
|
272
244
|
|
|
273
245
|
## License
|
|
274
246
|
|
|
275
247
|
[MIT](LICENSE)
|
|
276
248
|
|
|
277
249
|
[^name]: **vigiles** — the watchmen of ancient Rome, who guarded the city (and fought its fires) by night. _Quis custodiet ipsos custodes?_ — "who watches the watchmen?" (Juvenal, _Satire VI_).
|
|
250
|
+
|
|
251
|
+
[^audit]: `audit` reads only by default. Two deeper checks — live MCP connections and skill-firing — are opt-in and ask before they run.
|
package/action.yml
CHANGED
|
@@ -38,6 +38,18 @@ inputs:
|
|
|
38
38
|
description: "On pull_request events, post/update a sticky PR comment with the result ('true' or 'false'). Requires `pull-requests: write`."
|
|
39
39
|
required: false
|
|
40
40
|
default: "true"
|
|
41
|
+
capability-diff:
|
|
42
|
+
description: >
|
|
43
|
+
On pull_request events, compare the agent's capability surface (the
|
|
44
|
+
subagents' tool/effect blast radius) against the PR base and fold the diff
|
|
45
|
+
into the sticky comment (maps to `audit --capability-diff`). Requires
|
|
46
|
+
`fetch-depth: 0` on checkout so the base commit is present. 'true' or 'false'.
|
|
47
|
+
required: false
|
|
48
|
+
default: "false"
|
|
49
|
+
fail-on-widen:
|
|
50
|
+
description: "With capability-diff, fail the run when the PR WIDENS the blast radius (maps to the CLI's --fail-on-widen). 'true' or 'false'."
|
|
51
|
+
required: false
|
|
52
|
+
default: "false"
|
|
41
53
|
github-token:
|
|
42
54
|
description: "Token used to post the PR comment. Defaults to the workflow token."
|
|
43
55
|
required: false
|
|
@@ -64,6 +76,8 @@ runs:
|
|
|
64
76
|
VIGILES_CATALOG_ONLY: ${{ inputs.catalog-only }}
|
|
65
77
|
VIGILES_ACTION_PATH: ${{ github.action_path }}
|
|
66
78
|
VIGILES_COMMENT: ${{ inputs.comment }}
|
|
79
|
+
VIGILES_CAPABILITY_DIFF: ${{ inputs.capability-diff }}
|
|
80
|
+
VIGILES_FAIL_ON_WIDEN: ${{ inputs.fail-on-widen }}
|
|
67
81
|
GH_TOKEN: ${{ inputs.github-token }}
|
|
68
82
|
run: |
|
|
69
83
|
set -euo pipefail
|
|
@@ -114,6 +128,57 @@ runs:
|
|
|
114
128
|
status="${PIPESTATUS[0]}"
|
|
115
129
|
set -e
|
|
116
130
|
|
|
131
|
+
# Capability diff (moat #2): did this PR WIDEN the agent's blast radius?
|
|
132
|
+
# Runs orthogonally to the main command — it's an additive PR-comment
|
|
133
|
+
# section, so it works whatever `command` gated above. The BEFORE tree is
|
|
134
|
+
# a detached worktree of the PR base; the AFTER tree is this checkout.
|
|
135
|
+
capdiff="" # markdown appended to the sticky comment
|
|
136
|
+
capstatus=0
|
|
137
|
+
if [[ "${VIGILES_CAPABILITY_DIFF:-false}" == "true" \
|
|
138
|
+
&& "${GITHUB_EVENT_NAME:-}" == "pull_request" ]]; then
|
|
139
|
+
base_sha="$(jq -r '.pull_request.base.sha // empty' "${GITHUB_EVENT_PATH:-/dev/null}")"
|
|
140
|
+
if [[ -z "$base_sha" ]]; then
|
|
141
|
+
echo "::warning::capability-diff: no PR base sha in the event payload"
|
|
142
|
+
else
|
|
143
|
+
# A shallow checkout may not carry the base commit — fetch it first.
|
|
144
|
+
git fetch --no-tags --depth=1 origin "$base_sha" >/dev/null 2>&1 \
|
|
145
|
+
|| git fetch --no-tags origin "$base_sha" >/dev/null 2>&1 || true
|
|
146
|
+
base_dir="$(mktemp -d)"
|
|
147
|
+
# Materialize the base tree with `git archive` (NOT a worktree): it
|
|
148
|
+
# exports stored blobs WITHOUT running checkout/smudge filters, so it's
|
|
149
|
+
# light and can't trip a required filter (e.g. git-crypt) the runner
|
|
150
|
+
# lacks. The agent surface (unencrypted) lands as plaintext to diff.
|
|
151
|
+
if git archive "$base_sha" 2>/dev/null | tar -x -C "$base_dir" 2>/dev/null; then
|
|
152
|
+
# Diff the SAME relative dir in both trees (support a subdir working-directory).
|
|
153
|
+
rel="$(git rev-parse --show-prefix 2>/dev/null || echo '')"
|
|
154
|
+
before="${base_dir%/}/${rel}"; before="${before%/}"
|
|
155
|
+
capout="$(mktemp)"
|
|
156
|
+
fail_flag=(); [[ "${VIGILES_FAIL_ON_WIDEN:-false}" == "true" ]] && fail_flag=(--fail-on-widen)
|
|
157
|
+
# --no-html --no-json: this run is for the diff only, so skip the
|
|
158
|
+
# report artifacts. The full audit still streams to the CI log; only
|
|
159
|
+
# the capability-diff paragraph is extracted for the sticky comment.
|
|
160
|
+
set +e
|
|
161
|
+
"${runner[@]}" audit . --capability-diff="$before" --no-html --no-json "${fail_flag[@]}" 2>&1 | tee "$capout"
|
|
162
|
+
capstatus="${PIPESTATUS[0]}"
|
|
163
|
+
set -e
|
|
164
|
+
# Pull out just the "Capability surface …" paragraph (a self-contained
|
|
165
|
+
# block with no internal blank lines), then drop the no-change case
|
|
166
|
+
# so a PR that touches nothing capability-relevant gets no section.
|
|
167
|
+
capdiff="$(awk '/Capability surface/{f=1} f&&NF{print} f&&!NF{exit}' "$capout")"
|
|
168
|
+
case "$capdiff" in *"unchanged"*) capdiff="" ;; esac
|
|
169
|
+
else
|
|
170
|
+
echo "::warning::capability-diff: could not materialize PR base ${base_sha} (need fetch-depth: 0?)"
|
|
171
|
+
fi
|
|
172
|
+
rm -rf "$base_dir" 2>/dev/null || true
|
|
173
|
+
fi
|
|
174
|
+
fi
|
|
175
|
+
|
|
176
|
+
# A capability WIDENING (fail-on-widen → exit 1) flips an otherwise-green
|
|
177
|
+
# run to failed, so the gate is visible in the output + PR comment.
|
|
178
|
+
if [[ "$capstatus" -ne 0 && "$status" -eq 0 ]]; then
|
|
179
|
+
status="$capstatus"
|
|
180
|
+
fi
|
|
181
|
+
|
|
117
182
|
if [[ "$status" -eq 0 ]]; then
|
|
118
183
|
echo "valid=true" >> "$GITHUB_OUTPUT"
|
|
119
184
|
headline="✅ \`vigiles ${cmd}\` passed"
|
|
@@ -130,6 +195,14 @@ runs:
|
|
|
130
195
|
echo
|
|
131
196
|
echo "$headline"
|
|
132
197
|
echo
|
|
198
|
+
if [[ -n "$capdiff" ]]; then
|
|
199
|
+
echo "### Capability diff"
|
|
200
|
+
echo
|
|
201
|
+
echo '```'
|
|
202
|
+
echo "$capdiff"
|
|
203
|
+
echo '```'
|
|
204
|
+
echo
|
|
205
|
+
fi
|
|
133
206
|
echo '<details><summary>Output</summary>'
|
|
134
207
|
echo
|
|
135
208
|
echo '```'
|
|
@@ -46,6 +46,47 @@ export declare function discoverScripts(patterns: readonly string[], defaultGlob
|
|
|
46
46
|
export declare function runScripts(files: readonly string[], cwd: string, env?: NodeJS.ProcessEnv): ScriptRunResult[];
|
|
47
47
|
/** Whether any script FAILED (a skip is not a failure). */
|
|
48
48
|
export declare function anyFailed(results: readonly ScriptRunResult[]): boolean;
|
|
49
|
+
/**
|
|
50
|
+
* What a `test`/`eval` invocation should do about actually RUNNING the discovered
|
|
51
|
+
* scripts:
|
|
52
|
+
* - `run` — proceed.
|
|
53
|
+
* - `confirm` — interactive human, no explicit intent: ask before firing `count`.
|
|
54
|
+
* - `refuse` — headless, no explicit intent: don't silently fire the whole tree.
|
|
55
|
+
*/
|
|
56
|
+
export type RunScriptsDecision = {
|
|
57
|
+
readonly kind: "run";
|
|
58
|
+
} | {
|
|
59
|
+
readonly kind: "confirm";
|
|
60
|
+
readonly count: number;
|
|
61
|
+
} | {
|
|
62
|
+
readonly kind: "refuse";
|
|
63
|
+
readonly count: number;
|
|
64
|
+
};
|
|
65
|
+
export interface RunScriptsEnv {
|
|
66
|
+
/** `test` is free/deterministic → always runs. `eval` spends model quota. */
|
|
67
|
+
readonly kind: "test" | "eval";
|
|
68
|
+
/** The user named explicit target files/globs (positional args) — clear intent. */
|
|
69
|
+
readonly explicitTargets: boolean;
|
|
70
|
+
/** How many script files the discovery matched. */
|
|
71
|
+
readonly matchedCount: number;
|
|
72
|
+
/** A human at a terminal who can answer + wait. */
|
|
73
|
+
readonly isTTY: boolean;
|
|
74
|
+
/** `--all` — opt in to running the whole discovered set without a prompt. */
|
|
75
|
+
readonly all: boolean;
|
|
76
|
+
/** `--yes` / `--no-interactive` — agent/CI mode: never prompt. */
|
|
77
|
+
readonly yes: boolean;
|
|
78
|
+
}
|
|
79
|
+
/**
|
|
80
|
+
* Consent gate for a bare (no-target) `vigiles eval`. `eval` runs the REAL model
|
|
81
|
+
* on your subscription, and a no-target run discovers every `*.eval.*` over the
|
|
82
|
+
* whole tree — so a repo with many evals fires them all and spends quota. Mirrors
|
|
83
|
+
* `audit`'s read-vs-run consent (`decideExecute`): a paid, side-effecting verb
|
|
84
|
+
* never fans out over an unbounded glob without either an explicit target, an
|
|
85
|
+
* `--all` opt-in, or an interactive yes. `test` is free + deterministic, so it
|
|
86
|
+
* always runs. Total + pure, first match wins; the IO (prompt/refuse) lives in the
|
|
87
|
+
* CLI.
|
|
88
|
+
*/
|
|
89
|
+
export declare function decideRunScripts(o: RunScriptsEnv): RunScriptsDecision;
|
|
49
90
|
/** One line per file + an explicit pass/skip/fail tally. Skips are SHOWN, never
|
|
50
91
|
* folded into "passed" — a `⊘ SKIPPED` is loud, not a silent green. */
|
|
51
92
|
export declare function formatScriptSummary(results: readonly ScriptRunResult[]): string;
|
|
@@ -7,6 +7,7 @@ exports.detectNodeCaps = detectNodeCaps;
|
|
|
7
7
|
exports.discoverScripts = discoverScripts;
|
|
8
8
|
exports.runScripts = runScripts;
|
|
9
9
|
exports.anyFailed = anyFailed;
|
|
10
|
+
exports.decideRunScripts = decideRunScripts;
|
|
10
11
|
exports.formatScriptSummary = formatScriptSummary;
|
|
11
12
|
/**
|
|
12
13
|
* vigiles — run harness-test / eval script files via the CLI.
|
|
@@ -123,6 +124,31 @@ function runScripts(files, cwd, env = {}) {
|
|
|
123
124
|
function anyFailed(results) {
|
|
124
125
|
return results.some((r) => r.status === "fail");
|
|
125
126
|
}
|
|
127
|
+
/**
|
|
128
|
+
* Consent gate for a bare (no-target) `vigiles eval`. `eval` runs the REAL model
|
|
129
|
+
* on your subscription, and a no-target run discovers every `*.eval.*` over the
|
|
130
|
+
* whole tree — so a repo with many evals fires them all and spends quota. Mirrors
|
|
131
|
+
* `audit`'s read-vs-run consent (`decideExecute`): a paid, side-effecting verb
|
|
132
|
+
* never fans out over an unbounded glob without either an explicit target, an
|
|
133
|
+
* `--all` opt-in, or an interactive yes. `test` is free + deterministic, so it
|
|
134
|
+
* always runs. Total + pure, first match wins; the IO (prompt/refuse) lives in the
|
|
135
|
+
* CLI.
|
|
136
|
+
*/
|
|
137
|
+
function decideRunScripts(o) {
|
|
138
|
+
if (o.kind === "test")
|
|
139
|
+
return { kind: "run" };
|
|
140
|
+
if (o.explicitTargets)
|
|
141
|
+
return { kind: "run" };
|
|
142
|
+
if (o.all || o.yes)
|
|
143
|
+
return { kind: "run" };
|
|
144
|
+
// A bounded no-target run (0 = no-op, 1 = a single obviously-intended eval) is
|
|
145
|
+
// not the footgun; the footgun is fanning out over the whole tree.
|
|
146
|
+
if (o.matchedCount <= 1)
|
|
147
|
+
return { kind: "run" };
|
|
148
|
+
if (!o.isTTY)
|
|
149
|
+
return { kind: "refuse", count: o.matchedCount };
|
|
150
|
+
return { kind: "confirm", count: o.matchedCount };
|
|
151
|
+
}
|
|
126
152
|
const MARK = {
|
|
127
153
|
pass: "✓",
|
|
128
154
|
skip: "⊘",
|