vigiles 2.5.0 → 3.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (209) hide show
  1. package/.claude-plugin/plugin.json +9 -0
  2. package/README.md +73 -434
  3. package/action.yml +143 -7
  4. package/dist/action-gate.d.ts +1 -1
  5. package/dist/action-gate.js +1 -1
  6. package/dist/adapter-conformance.d.ts +30 -0
  7. package/dist/adapter-conformance.js +153 -0
  8. package/dist/adapter-registry.d.ts +42 -0
  9. package/dist/adapter-registry.js +55 -0
  10. package/dist/adapter.d.ts +26 -0
  11. package/dist/adapter.js +16 -0
  12. package/dist/adapters/claude-code/adapter.d.ts +3 -0
  13. package/dist/adapters/claude-code/adapter.js +46 -0
  14. package/dist/{agent-result.d.ts → adapters/claude-code/agent-result.d.ts} +1 -1
  15. package/dist/adapters/claude-code/dialect.d.ts +13 -0
  16. package/dist/adapters/claude-code/dialect.js +51 -0
  17. package/dist/adapters/claude-code/egress-entry.d.ts +2 -0
  18. package/dist/adapters/claude-code/egress-entry.js +115 -0
  19. package/dist/adapters/claude-code/egress-proxy.d.ts +2 -0
  20. package/dist/adapters/claude-code/egress-proxy.js +60 -0
  21. package/dist/adapters/claude-code/egress.d.ts +114 -0
  22. package/dist/adapters/claude-code/egress.js +276 -0
  23. package/dist/adapters/claude-code/eval-baseline.d.ts +68 -0
  24. package/dist/adapters/claude-code/eval-baseline.js +173 -0
  25. package/dist/{eval-cache.d.ts → adapters/claude-code/eval-cache.d.ts} +1 -1
  26. package/dist/{eval-cache.js → adapters/claude-code/eval-cache.js} +1 -1
  27. package/dist/{eval.d.ts → adapters/claude-code/eval.d.ts} +28 -4
  28. package/dist/{eval.js → adapters/claude-code/eval.js} +61 -25
  29. package/dist/{harness-test.d.ts → adapters/claude-code/harness-test.d.ts} +33 -37
  30. package/dist/{harness-test.js → adapters/claude-code/harness-test.js} +124 -43
  31. package/dist/adapters/claude-code/hook-protocol.d.ts +10 -0
  32. package/dist/adapters/claude-code/hook-protocol.js +10 -0
  33. package/dist/adapters/claude-code/layout.d.ts +8 -0
  34. package/dist/adapters/claude-code/layout.js +18 -0
  35. package/dist/{mock-model.d.ts → adapters/claude-code/mock-model.d.ts} +2 -24
  36. package/dist/adapters/claude-code/model-mock.d.ts +11 -0
  37. package/dist/adapters/claude-code/model-mock.js +10 -0
  38. package/dist/adapters/claude-code/plugin-loader.d.ts +25 -0
  39. package/dist/adapters/claude-code/plugin-loader.js +19 -0
  40. package/dist/adapters/claude-code/run-hook.d.ts +204 -0
  41. package/dist/adapters/claude-code/run-hook.js +418 -0
  42. package/dist/adapters/claude-code/run-scripts.d.ts +52 -0
  43. package/dist/adapters/claude-code/run-scripts.js +150 -0
  44. package/dist/adapters/claude-code/runtime.d.ts +16 -0
  45. package/dist/adapters/claude-code/runtime.js +39 -0
  46. package/dist/{sandbox.d.ts → adapters/claude-code/sandbox.d.ts} +45 -4
  47. package/dist/{sandbox.js → adapters/claude-code/sandbox.js} +76 -8
  48. package/dist/{skill-driver.d.ts → adapters/claude-code/skill-driver.d.ts} +1 -1
  49. package/dist/adapters/codex/adapter.d.ts +3 -0
  50. package/dist/adapters/codex/adapter.js +49 -0
  51. package/dist/adapters/codex/dialect.d.ts +10 -0
  52. package/dist/adapters/codex/dialect.js +30 -0
  53. package/dist/adapters/codex/driver.d.ts +20 -0
  54. package/dist/adapters/codex/driver.js +89 -0
  55. package/dist/adapters/codex/hook-protocol.d.ts +10 -0
  56. package/dist/adapters/codex/hook-protocol.js +18 -0
  57. package/dist/adapters/codex/layout.d.ts +16 -0
  58. package/dist/adapters/codex/layout.js +18 -0
  59. package/dist/adapters/codex/mock-model.d.ts +52 -0
  60. package/dist/adapters/codex/mock-model.js +210 -0
  61. package/dist/adapters/codex/model-mock.d.ts +11 -0
  62. package/dist/adapters/codex/model-mock.js +10 -0
  63. package/dist/adapters/codex/runtime.d.ts +38 -0
  64. package/dist/adapters/codex/runtime.js +62 -0
  65. package/dist/adapters/opencode/adapter.d.ts +3 -0
  66. package/dist/adapters/opencode/adapter.js +48 -0
  67. package/dist/adapters/opencode/dialect.d.ts +10 -0
  68. package/dist/adapters/opencode/dialect.js +36 -0
  69. package/dist/adapters/opencode/layout.d.ts +9 -0
  70. package/dist/adapters/opencode/layout.js +25 -0
  71. package/dist/adapters/opencode/model-mock.d.ts +10 -0
  72. package/dist/adapters/opencode/model-mock.js +10 -0
  73. package/dist/adapters/opencode/runtime.d.ts +9 -0
  74. package/dist/adapters/opencode/runtime.js +21 -0
  75. package/dist/claude-code.d.ts +16 -0
  76. package/dist/claude-code.js +32 -0
  77. package/dist/cli-flags.d.ts +22 -0
  78. package/dist/cli-flags.js +38 -0
  79. package/dist/cli.js +406 -206
  80. package/dist/codex.d.ts +20 -0
  81. package/dist/codex.js +36 -0
  82. package/dist/community-skills.d.ts +1 -1
  83. package/dist/community-skills.js +2 -2
  84. package/dist/core/adapter.d.ts +88 -0
  85. package/dist/core/adapter.js +3 -0
  86. package/dist/{compile.d.ts → core/compile.d.ts} +11 -2
  87. package/dist/{compile.js → core/compile.js} +68 -53
  88. package/dist/core/compose.d.ts +79 -0
  89. package/dist/core/compose.js +145 -0
  90. package/dist/core/dialect.d.ts +51 -0
  91. package/dist/core/dialect.js +3 -0
  92. package/dist/core/harness-driver.d.ts +134 -0
  93. package/dist/core/harness-driver.js +3 -0
  94. package/dist/core/hook-protocol.d.ts +28 -0
  95. package/dist/core/hook-protocol.js +3 -0
  96. package/dist/core/layout.d.ts +43 -0
  97. package/dist/core/layout.js +3 -0
  98. package/dist/core/model-mock.d.ts +27 -0
  99. package/dist/core/model-mock.js +3 -0
  100. package/dist/core/refs.d.ts +62 -0
  101. package/dist/{refs.js → core/refs.js} +49 -23
  102. package/dist/core/runtime.d.ts +38 -0
  103. package/dist/core/runtime.js +3 -0
  104. package/dist/{types.d.ts → core/types.d.ts} +29 -0
  105. package/dist/{validate.d.ts → core/validate.d.ts} +1 -1
  106. package/dist/{validate.js → core/validate.js} +41 -4
  107. package/dist/e2e.d.ts +19 -0
  108. package/dist/e2e.js +39 -0
  109. package/dist/harness-assert.d.ts +60 -11
  110. package/dist/harness-assert.js +111 -9
  111. package/dist/integration.d.ts +16 -0
  112. package/dist/integration.js +32 -0
  113. package/dist/leaderboard.d.ts +33 -0
  114. package/dist/leaderboard.js +107 -0
  115. package/dist/linting.d.ts +9 -0
  116. package/dist/linting.js +25 -0
  117. package/dist/plugin-loader.d.ts +6 -3
  118. package/dist/plugin-loader.js +101 -49
  119. package/dist/scan.d.ts +49 -0
  120. package/dist/scan.js +177 -0
  121. package/dist/setup-plan.d.ts +51 -0
  122. package/dist/setup-plan.js +85 -0
  123. package/dist/skill-test.d.ts +1 -1
  124. package/dist/skill-test.js +1 -1
  125. package/dist/test-coverage.d.ts +71 -0
  126. package/dist/test-coverage.js +228 -0
  127. package/dist/testing.d.ts +12 -0
  128. package/dist/testing.js +28 -0
  129. package/dist/unit.d.ts +17 -0
  130. package/dist/unit.js +36 -0
  131. package/hooks/refs-nudge.sh +24 -0
  132. package/package.json +35 -16
  133. package/skills/audit-feedback-loop/SKILL.md +76 -0
  134. package/skills/edit-spec/SKILL.md +131 -0
  135. package/skills/enforce-rules-format/SKILL.md +71 -0
  136. package/skills/generate-logo/SKILL.md +103 -0
  137. package/skills/generate-rule/SKILL.md +64 -0
  138. package/skills/linter-docs/clippy.md +241 -0
  139. package/skills/linter-docs/eslint.md +384 -0
  140. package/skills/linter-docs/pylint.md +288 -0
  141. package/skills/linter-docs/rubocop.md +277 -0
  142. package/skills/linter-docs/ruff.md +187 -0
  143. package/skills/linter-docs/stylelint.md +247 -0
  144. package/skills/migrate-to-spec/SKILL.md +126 -0
  145. package/skills/pr-to-lint-rule/SKILL.md +97 -0
  146. package/skills/strengthen/SKILL.md +168 -0
  147. package/skills/test-harness/SKILL.md +157 -0
  148. package/dist/action.d.ts +0 -7
  149. package/dist/action.js +0 -180
  150. package/dist/refs.d.ts +0 -44
  151. package/dist/run-hook.d.ts +0 -77
  152. package/dist/run-hook.js +0 -80
  153. package/dist/run-scripts.d.ts +0 -20
  154. package/dist/run-scripts.js +0 -70
  155. /package/dist/{agent-result.js → adapters/claude-code/agent-result.js} +0 -0
  156. /package/dist/{agent-runtime.d.ts → adapters/claude-code/agent-runtime.d.ts} +0 -0
  157. /package/dist/{agent-runtime.js → adapters/claude-code/agent-runtime.js} +0 -0
  158. /package/dist/{judge.d.ts → adapters/claude-code/judge.d.ts} +0 -0
  159. /package/dist/{judge.js → adapters/claude-code/judge.js} +0 -0
  160. /package/dist/{mock-entry.d.ts → adapters/claude-code/mock-entry.d.ts} +0 -0
  161. /package/dist/{mock-entry.js → adapters/claude-code/mock-entry.js} +0 -0
  162. /package/dist/{mock-model.js → adapters/claude-code/mock-model.js} +0 -0
  163. /package/dist/{skill-driver.js → adapters/claude-code/skill-driver.js} +0 -0
  164. /package/dist/{skill-runtime.d.ts → adapters/claude-code/skill-runtime.d.ts} +0 -0
  165. /package/dist/{skill-runtime.js → adapters/claude-code/skill-runtime.js} +0 -0
  166. /package/dist/{stats.d.ts → adapters/claude-code/stats.d.ts} +0 -0
  167. /package/dist/{stats.js → adapters/claude-code/stats.js} +0 -0
  168. /package/dist/{compile-generator.d.ts → core/compile-generator.d.ts} +0 -0
  169. /package/dist/{compile-generator.js → core/compile-generator.js} +0 -0
  170. /package/dist/{coverage.d.ts → core/coverage.d.ts} +0 -0
  171. /package/dist/{coverage.js → core/coverage.js} +0 -0
  172. /package/dist/{doc-refs.d.ts → core/doc-refs.d.ts} +0 -0
  173. /package/dist/{doc-refs.js → core/doc-refs.js} +0 -0
  174. /package/dist/{evolve.d.ts → core/evolve.d.ts} +0 -0
  175. /package/dist/{evolve.js → core/evolve.js} +0 -0
  176. /package/dist/{frontmatter.d.ts → core/frontmatter.d.ts} +0 -0
  177. /package/dist/{frontmatter.js → core/frontmatter.js} +0 -0
  178. /package/dist/{generate-schema.d.ts → core/generate-schema.d.ts} +0 -0
  179. /package/dist/{generate-schema.js → core/generate-schema.js} +0 -0
  180. /package/dist/{generate-types.d.ts → core/generate-types.d.ts} +0 -0
  181. /package/dist/{generate-types.js → core/generate-types.js} +0 -0
  182. /package/dist/{hash.d.ts → core/hash.d.ts} +0 -0
  183. /package/dist/{hash.js → core/hash.js} +0 -0
  184. /package/dist/{inline.d.ts → core/inline.d.ts} +0 -0
  185. /package/dist/{inline.js → core/inline.js} +0 -0
  186. /package/dist/{integrity.d.ts → core/integrity.d.ts} +0 -0
  187. /package/dist/{integrity.js → core/integrity.js} +0 -0
  188. /package/dist/{linters.d.ts → core/linters.d.ts} +0 -0
  189. /package/dist/{linters.js → core/linters.js} +0 -0
  190. /package/dist/{mcp.d.ts → core/mcp.d.ts} +0 -0
  191. /package/dist/{mcp.js → core/mcp.js} +0 -0
  192. /package/dist/{orphans.d.ts → core/orphans.d.ts} +0 -0
  193. /package/dist/{orphans.js → core/orphans.js} +0 -0
  194. /package/dist/{proofs.d.ts → core/proofs.d.ts} +0 -0
  195. /package/dist/{proofs.js → core/proofs.js} +0 -0
  196. /package/dist/{session.d.ts → core/session.d.ts} +0 -0
  197. /package/dist/{session.js → core/session.js} +0 -0
  198. /package/dist/{sidecar.d.ts → core/sidecar.d.ts} +0 -0
  199. /package/dist/{sidecar.js → core/sidecar.js} +0 -0
  200. /package/dist/{spec.d.ts → core/spec.d.ts} +0 -0
  201. /package/dist/{spec.js → core/spec.js} +0 -0
  202. /package/dist/{symbols.d.ts → core/symbols.d.ts} +0 -0
  203. /package/dist/{symbols.js → core/symbols.js} +0 -0
  204. /package/dist/{test-utils.d.ts → core/test-utils.d.ts} +0 -0
  205. /package/dist/{test-utils.js → core/test-utils.js} +0 -0
  206. /package/dist/{types.js → core/types.js} +0 -0
  207. /package/{.claude-plugin/hooks → hooks}/post-edit.sh +0 -0
  208. /package/{.claude-plugin/hooks → hooks}/pre-edit.sh +0 -0
  209. /package/{.claude-plugin/hooks → hooks}/session-start.sh +0 -0
@@ -30,6 +30,15 @@
30
30
  "command": "bash ${CLAUDE_PLUGIN_ROOT}/hooks/post-edit.sh"
31
31
  }
32
32
  ]
33
+ },
34
+ {
35
+ "matcher": "Edit|Write",
36
+ "hooks": [
37
+ {
38
+ "type": "command",
39
+ "command": "bash ${CLAUDE_PLUGIN_ROOT}/hooks/refs-nudge.sh"
40
+ }
41
+ ]
33
42
  }
34
43
  ],
35
44
  "SessionStart": [
package/README.md CHANGED
@@ -9,8 +9,8 @@
9
9
  </p>
10
10
 
11
11
  <p align="center">
12
- <strong>Test &amp; verify your Claude Code harness.</strong><br />
13
- vigiles <strong>verifies the references</strong> your instruction files make — linter rules, file paths, scripts, code symbols — and <strong>evals</strong> whether your hooks, skills, and CLAUDE.md actually change what the agent does.
12
+ <strong>The missing linting + testing layer for agentic coding.</strong><br />
13
+ vigiles <strong>lints</strong> the references your instruction files make — linter rules, file paths, scripts, code symbols — and <strong>tests</strong> whether your hooks, skills, and CLAUDE.md actually change what the agent does.
14
14
  </p>
15
15
 
16
16
  <p align="center">
@@ -21,479 +21,118 @@
21
21
 
22
22
  ---
23
23
 
24
- <details>
25
- <summary><b>Contents</b></summary>
24
+ `Agent = Model + Harness`. You'd never ship an app without a linter and a test
25
+ suite — yet an AI agent steering your repo is trusted on vibes. vigiles is the
26
+ deterministic layer for the harness: it **lints** the references your instruction
27
+ files make and **tests** that your hooks and skills actually fire. Two independent
28
+ pillars — adopt either, or both:
26
29
 
27
- **Pillar 1 verify your instruction files** · references your CLAUDE.md makes that a linter, the filesystem, and package.json can prove
30
+ | | Pillar | What it does |
31
+ | ----- | --------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
32
+ | **①** | **Verify your instruction files** | Every linter rule, file path, script, and code symbol your CLAUDE.md cites is checked against reality, so stale references can't silently mislead the agent. → [guide](docs/verifying-instruction-files.md) |
33
+ | **②** | **Test your harness** | Your hooks and skills are code — vigiles tests they actually fire, **deterministically and free** (no model, no API key) before you pay for an eval. → [guide](docs/harness-testing.md) |
28
34
 
29
- - Three adoption levels: [inline comments](#level-0--inline-comments-30-seconds-no-new-files) [YAML frontmatter](#level-1--yaml-frontmatter-editor-autocomplete-still-no-typescript) [typed spec](#level-2--typed-spec-compiler-grade-guarantees)
30
- - [What changes with vigiles](#what-changes-with-vigiles)
31
- - [Quick start](#quick-start)
32
- - [Three rule types](#three-rule-types) — `enforce` / `guidance` / `guard`
33
- - [Verified references](#verified-references) — `file` / `cmd` / `symbol` / `ref`
35
+ Neither pillar depends on the other pick the one that hurts today. **Works with
36
+ Claude Code and Codex** ([`vigiles/codex`](docs/harnesses.md)) behind a five-port
37
+ adapter; [custom adapters welcome](docs/authoring-an-adapter.md).
34
38
 
35
- **Pillar 2[test your Claude Code harness](#test-your-claude-code-harness)** · eval whether your hooks, skills, and CLAUDE.md actually change what the agent does
39
+ ## Verify — your CLAUDE.md lies to your agent
36
40
 
37
- - [Level 1 unit-test a hook (no AI)](#level-1--test-a-hook-by-itself-no-ai-milliseconds)
38
- - [Level 2 does it fire in a real session?](#level-2--does-it-fire-in-a-real-session-free-scripted-ai)
39
- - [Level 3 does it change behaviour?](#level-3--does-it-change-what-claude-does-real-ai-occasional)
40
- - [Test skills for real + assert on actions](#test-your-skills-for-real--and-assert-on-what-claude-did)
41
- - [Run them in CI](#run-them-in-ci)
41
+ Your CLAUDE.md says _"enforce `eslint/no-console`."_ But it was switched off
42
+ months agoand the agent trusts the claim. (Same story for the file path it
43
+ cites that got renamed, and the script that was deleted.)
42
44
 
43
- **More** [CLI & CI](#cli--ci) · [Skills](#skills) · [Maturity levels](#maturity-levels) · [Related tools](#related-tools)
45
+ **Without vigiles:** nobody checks. The agent acts on fiction.
44
46
 
45
- </details>
47
+ **With vigiles:** `npx vigiles audit` resolves every reference against reality —
46
48
 
47
- Your CLAUDE.md lies to your agent. Here's the fix.
48
-
49
- Hand-written CLAUDE.md files rot silently. Here's what a typical one looks like:
50
-
51
- ```markdown
52
- ## Code Style
53
-
54
- Never use `any` — the `@typescript-eslint/no-explicit-any` rule
55
- catches this. Always use `unknown` and narrow with type guards.
56
- See `src/utils/type-helpers.ts` for project utilities.
57
-
58
- ## Testing
59
-
60
- Run `npm run typecheck` before submitting. Every service in
61
- src/services/ should have a corresponding test file.
49
+ ```text
50
+ CLAUDE.md (inline mode):
51
+ line 1: Rule "eslint/no-console" exists but is disabled in eslint config
52
+ ✓ line 2: eslint/eqeqeq
53
+ ✗ line 3: Rule "no-consoel" not found in eslint. Did you mean: "eslint/no-console"?
62
54
  ```
63
55
 
64
- Reads fine. Four things are wrong:
65
-
66
- 1. `@typescript-eslint/no-explicit-any` disabled to unblock a deadline, never re-enabled
67
- 2. `src/utils/type-helpers.ts` — renamed to `src/utils/narrowing.ts` last quarter
68
- 3. `npm run typecheck` — script removed from package.json
69
- 4. Service/test pairing — no automated check, just a hope
70
-
71
- The agent reads this, trusts it, and writes code based on stale claims nobody verified. vigiles **verifies the references in your instruction files** — that each linter rule exists and is enabled, that every file path and script is real, and that referenced **code symbols** (functions, classes, constants) actually exist in the files that define them — and meets you at whatever commitment level you want.
72
-
73
- > **See it in 60 seconds:** `npm run demo` runs `vigiles audit` against a deliberately-broken instruction file and catches a renamed symbol and a missing MCP tool (_"did you mean `purge`?"_), while the truthful references pass silently. [examples/demo →](examples/demo)
74
-
75
- Three levels. Each is independently useful; adopt as far up as you like.
76
-
77
- ### Level 0 — inline comments (30 seconds, no new files)
78
-
79
- Add a comment to your existing CLAUDE.md and audit it:
56
+ It resolves rule names across **7 linter catalogs** — the rule exists **and is
57
+ enabled** — and checks file paths, scripts, and code symbols the same way. Start
58
+ with one comment, no new files:
80
59
 
81
60
  ```md
82
61
  <!-- vigiles:enforce eslint/no-console "Route output through logger.ts" -->
83
62
  ```
84
63
 
85
- ```bash
86
- npx vigiles audit CLAUDE.md
87
- ```
88
-
89
- Each rule is checked against your real linter config — typos get closest-match suggestions, disabled rules are flagged. Zero install commitment, zero new files.
90
-
91
- ### Level 1 — YAML frontmatter (editor autocomplete, still no TypeScript)
92
-
93
- Promote your rules into a `vigiles:` block at the top of the file:
94
-
95
- ```yaml
96
- ---
97
- # yaml-language-server: $schema=./.vigiles/schema.json
98
- vigiles:
99
- enforce:
100
- - rule: "@typescript-eslint/no-explicit-any"
101
- why: "Use unknown and narrow with type guards."
102
- - rule: eslint/no-console
103
- why: "Route output through logger.ts"
104
- ---
105
- ```
106
-
107
- `npx vigiles generate-schema` emits a JSON Schema from your project's _actual_ enabled rules, so your editor's built-in YAML language server (VS Code, JetBrains, neovim) autocompletes rule names and red-squiggles typos — at edit time, with no TypeScript in the project. `vigiles audit` enforces the same rules in CI. [Markdown mode →](docs/markdown-mode.md)
108
-
109
- ### Level 2 — typed spec (compiler-grade guarantees)
110
-
111
- When you want the strongest guarantees, compile a typed spec. Every linter rule reference is verified against your real config, every file path against the filesystem, every npm script against package.json. Stale references become compile errors — caught at edit time, not when the agent silently ignores you.
112
-
113
- ```typescript
114
- // CLAUDE.md.spec.ts
115
- import { claude, enforce, guidance } from "vigiles/spec";
116
-
117
- export default claude({
118
- commands: {
119
- "npm run build": "Compile TypeScript to dist/",
120
- "npm test": "Build and run all tests",
121
- // ✗ "npm run typecheck" → compile error: script not in package.json
122
- },
123
-
124
- keyFiles: {
125
- "src/utils/narrowing.ts": "Type guard utilities",
126
- // ✗ "src/utils/type-helpers.ts" → compile error: file not found
127
- },
128
-
129
- rules: {
130
- "no-explicit-any": enforce(
131
- "@typescript-eslint/no-explicit-any",
132
- "Use unknown and narrow with type guards.",
133
- ),
134
- // ✗ if rule is disabled in config → compile error
135
-
136
- "research-first": guidance("Google unfamiliar APIs first."),
137
- },
138
- });
139
- ```
140
-
141
- ```bash
142
- $ npx vigiles compile
143
-
144
- ✓ CLAUDE.md.spec.ts → CLAUDE.md
145
- 2 rules (1 linter-verified, 1 guidance)
146
- ~180 tokens
147
- ```
148
-
149
- At this level the spec is the source of truth and CLAUDE.md is a build artifact. The agent edits the spec — hooks auto-compile, types catch typos in the editor, CI catches drift.
150
-
151
- Companion repo for [Feedback Loop Is All You Need](https://zernie.com/blog/feedback-loop-is-all-you-need).
152
-
153
- ## What Changes With vigiles
154
-
155
- ### Claude Code
156
-
157
- | | Without vigiles | With vigiles |
158
- | ----------------------------------- | ---------------------------- | -------------------------------------------------------------- |
159
- | **Instructions** | Hand-written CLAUDE.md | Compiled from `.spec.ts` (build artifact) |
160
- | **Linter rule references** | Trust-based (nobody checks) | Verified at compile time against real config |
161
- | **File paths** | Rot silently when renamed | `file()` references checked against filesystem |
162
- | **Commands** | Stale scripts go unnoticed | `cmd()` references checked against package.json |
163
- | **Direct edits to CLAUDE.md** | Anyone can, nobody knows | PreToolUse hook blocks edits, redirects to spec |
164
- | **Spec edits** | N/A | PostToolUse hook auto-compiles to markdown |
165
- | **Linter config changes** | CLAUDE.md drifts out of sync | PostToolUse hook auto-regenerates types |
166
- | **guidance → enforce upgrades** | Manual guesswork | `/strengthen` reads per-linter docs, suggests upgrades |
167
- | **New lint rules from PR feedback** | Copy-paste from review | `/pr-to-lint-rule` generates rule + tests + spec entry |
168
- | **CI** | Nothing to verify | `vigiles audit` catches hand-edits, disabled rules, stale refs |
169
-
170
- <details>
171
- <summary><b>Codex</b> (same compile-time checks, no hooks)</summary>
172
-
173
- | | Without vigiles | With vigiles |
174
- | ----------------------------- | -------------------------------- | ------------------------------------------------------- |
175
- | **Instructions** | Hand-written AGENTS.md | Compiled from `.spec.ts` |
176
- | **Linter rule references** | Trust-based | Verified at compile time |
177
- | **File paths / commands** | Rot silently | Checked at compile time |
178
- | **Direct edits to AGENTS.md** | Undetected | CI catches hash mismatch |
179
- | **Hooks / auto-compile** | Not available (no plugin system) | Not available — run `vigiles compile` manually or in CI |
180
- | **CI** | Nothing to verify | Same `vigiles audit` pipeline as Claude |
181
-
182
- </details>
183
-
184
- Everything vigiles compiles and audits is **deterministic** — same input, same output, no LLM in the loop. The non-deterministic parts (authoring specs, suggesting upgrades, writing custom rules) are agent skills that run outside the compilation pipeline. [Determinism breakdown and flow diagram →](docs/comparison.md)
185
-
186
- ## Quick Start
187
-
188
- The fastest path is markdown mode — add a marker to your existing CLAUDE.md and audit it, no install or new files (see [Level 0 / Level 1](#level-0--inline-comments-30-seconds-no-new-files) above and [docs/markdown-mode.md](docs/markdown-mode.md)). When you want compiler-grade guarantees, scaffold a typed spec:
189
-
190
- ```bash
191
- npx vigiles init
192
- ```
193
-
194
- The wizard auto-detects your project, creates a spec, scans your linters, compiles to markdown, adds a CI step, and installs Claude Code hooks. After install: the agent edits the spec (hooks block direct CLAUDE.md edits), the spec auto-compiles on save, and `vigiles audit` catches drift in CI.
195
-
196
- Start with `guidance()` rules (zero config). When you're ready, run `/strengthen` to find rules that can be upgraded to compile-verified `enforce()`. Already have a hand-written CLAUDE.md? The wizard detects it and offers migration. Flags (`--strict`, `--target=AGENTS.md`, `--no-gha`) and non-interactive agent usage are in the [CLI reference](docs/cli.md) and [agent setup guide](docs/agent-setup.md).
197
-
198
- ## Three Rule Types
199
-
200
- **`enforce()`** — delegated to a linter. vigiles verifies the rule exists in the catalog AND is enabled in your project config. A disabled rule is a compile error.
201
-
202
- <!-- vigiles:ignore -->
203
-
204
- ```typescript
205
- "no-any": enforce("@typescript-eslint/no-explicit-any", "Use unknown and narrow."),
206
- "no-print": enforce("ruff/T201", "Use logging module."),
207
- "no-unwrap": enforce("clippy/unwrap_used", "Use expect() with context."),
208
- ```
209
-
210
- Supports ESLint, Stylelint, Ruff, Clippy, Pylint, RuboCop, and Cedar policies. [Full linter support details →](docs/linter-support.md)
211
-
212
- **`guidance()`** — prose advice. No mechanical enforcement, but not untracked: guidance rules participate in the monotonicity proof system. Once a rule exists, it can be strengthened ( `guidance` → `enforce` ) but never weakened or removed without an explicit allowlist. This prevents silent erosion of conventions over time.
213
-
214
- ```typescript
215
- "research-first": guidance("Google unfamiliar APIs first."),
216
- ```
217
-
218
- **`guard()`** — reactive: runs a command when watched files change (e.g. `*.spec.ts` → `npx vigiles compile`). One declaration emits hooks for every supported system (Claude Code PostToolUse, husky pre-commit, etc.) — no copy-pasting the same trigger across `.claude/settings.json`, `.husky/`, and CI. Same monotonicity guarantees as `enforce()`. [Full spec format →](docs/spec-format.md)
219
-
220
- ## Verified References
221
-
222
- `file()`, `cmd()`, `symbol()`, and `ref()` catch stale references at compile time:
223
-
224
- ```typescript
225
- import { claude, file, cmd, symbol, ref, instructions } from "vigiles/spec";
226
-
227
- export default claude({
228
- sections: {
229
- architecture: instructions`
230
- Core engine in ${file("src/compile.ts")}.
231
- Compile specs with ${symbol("src/compile.ts", "compileClaude")}.
232
- Run ${cmd("npm test")} to verify.
233
- See ${ref("skills/strengthen/SKILL.md")} for the strengthen skill.
234
- `,
235
- // If any path / script / symbol is stale → compile error
236
- },
237
- // ...
238
- });
239
- ```
240
-
241
- There's a small family of inline **marks** that `audit` checks, each binding a reference to its real source:
242
-
243
- - `` `vigiles:symbol file#name` `` — the named file actually **defines** that symbol (function, class, method, constant), parsed with [ast-grep](https://ast-grep.github.io) across **JS/TS, Python, Ruby, Rust, and CSS**. Rename it and `audit` fails; in markdown mode the `refs-hook` **forces the mark**, blocking edits that leave a code reference bare. [Details →](research/symbol-verification.md)
244
- - `` `vigiles:mcp server#tool` `` — the referenced **MCP tool exists** on its server. `audit` reads `.mcp.json`, starts the server, lists its tools, and flags a renamed/removed one with a "did you mean" — catching e.g. the GitHub MCP server renaming `create_issue` → `issue_write`, which otherwise fails silently.
245
-
246
- **Typo-safe at authoring time, too.** `vigiles generate-types` emits a `.vigiles/generated.d.ts` so `enforce("eslint/no-consolee")` red-squiggles in your editor; `generate-schema` gives Level 1 frontmatter the same via your YAML language server. Both have `--check` CI freshness modes. [How it works →](docs/linter-support.md#generate-types)
64
+ Step up to a typed `.spec.ts` (compiled to CLAUDE.md, compiler-grade) when you
65
+ want it. **[Full guide →](docs/verifying-instruction-files.md)**
247
66
 
248
- ## Test your Claude Code harness
67
+ ## Test — does your harness actually fire?
249
68
 
250
- You wrote hooks, a skill, a CLAUDE.md rule how do you know they work, beyond
251
- running Claude and eyeballing it? vigiles ships a library to **test the harness
252
- itself**, at three levels, cheapest first. It's plain async functions, so it
253
- drops into **node:test / vitest / jest**, or a zero-setup `vigiles test`.
69
+ A hook can be wired wrong, a skill's description can fail to trigger, injected
70
+ context can never reach the model silently, all passing a naive "did it run?"
71
+ check.
254
72
 
255
- ### Level 1 test a hook by itself (no AI, milliseconds)
73
+ **Without vigiles:** you assume your `--no-verify` guard blocks. You don't know.
256
74
 
257
- A hook is just a process that's handed a "Claude is about to do X" event and
258
- answers block/allow. Hand it a fake event and check the answer — no `claude`, no
259
- model, and **every** event type is reachable (incl. Edit/Write, PreCompact,
260
- SessionEnd):
75
+ **With vigiles:** a deterministic test proves it no model, no API key,
76
+ milliseconds:
261
77
 
262
78
  ```typescript
263
- import { runHook } from "vigiles/run-hook";
79
+ import { runHook } from "vigiles/testing";
264
80
 
265
- const r = runHook(guardCommand, {
81
+ const r = runHook(guard, {
266
82
  hook_event_name: "PreToolUse",
267
83
  tool_name: "Bash",
268
84
  tool_input: { command: "git commit --no-verify" },
269
85
  });
270
- assert(r.blocked); // exit 2 / decision:"block" / permissionDecision:"deny"
86
+ assert(r.blocked); // a red here means your hook silently lets it through
271
87
  ```
272
88
 
273
- The same shape governs **MCP tools** — the dominant real MCP test — with no
274
- server running, because the hook only sees the tool _name_:
89
+ ```text
90
+ guard blocks --no-verify and allows a clean commit
275
91
 
276
- ```typescript
277
- // block the destructive github-MCP tool; read-only ones pass
278
- runHook(guard, {
279
- hook_event_name: "PreToolUse",
280
- tool_name: "mcp__github__merge_pull_request",
281
- tool_input: { pull_number: 42 },
282
- }).blocked; // true
92
+ 2 passed.
283
93
  ```
284
94
 
285
- ### Level 2 does it fire in a real session? (free, scripted "AI")
286
-
287
- Right logic wired in correctly. `runHarnessTest` runs the **real** `claude`
288
- against a **scripted mock model** you control your hooks fire for real, the
289
- agent's turns are fixed, no API key, same result every time. Covers the
290
- governance shapes: SessionStart, Stop, UserPromptSubmit, and Bash **and
291
- Edit/Write** Pre/PostToolUse.
292
-
293
- ```typescript
294
- import { runHarnessTest, scriptModel } from "vigiles/harness-test";
295
-
296
- const r = await runHarnessTest({
297
- settings: {
298
- hooks: {
299
- Stop: [
300
- { hooks: [{ type: "command", command: "test -f DONE || exit 2" }] },
301
- ],
302
- },
303
- },
304
- model: scriptModel([
305
- { text: "I'm done" }, // tries to stop → blocked (no DONE)
306
- { tool: "Bash", input: { command: "touch DONE" } },
307
- { text: "now done" },
308
- ]),
309
- });
310
- assert(JSON.parse(r.stdout).num_turns > 1); // the Stop hook forced more work
311
- ```
95
+ Three tiers, cheapest first: **`runHook`** (a hook's logic), **`runHarnessTest`**
96
+ (the real agent CLI against a scripted mock model), **`runEval`** (the real model
97
+ A/B with a significance gate). **Testing a skill?** `measureTriggerRate` checks
98
+ its description actually **fires** across varied prompts (recall) without
99
+ hijacking unrelated ones (precision). **[Full guide →](docs/harness-testing.md)**
312
100
 
313
- ### Level 3 — does it change what Claude does? (real AI, occasional)
101
+ ## Quick start
314
102
 
315
- `runEval` runs the **real** model N times with your change **on vs off** and
316
- reports the gap as **mean ± se** — so you can tell signal from noise instead of
317
- eyeballing two averages:
103
+ > **Paste into Claude Code or Codex:**
104
+ >
105
+ > > Install vigiles in this repo and run it. Verify my CLAUDE.md / AGENTS.md
106
+ > > references and show me what's stale, then write and run a harness test for one
107
+ > > of my hooks or skills. Use good defaults (both pillars, non-interactive), but
108
+ > > **ask me first** whether to gate it in CI, whether to add a real-model eval,
109
+ > > and whether to enforce strictly (`--strict`).
318
110
 
319
- ```typescript
320
- import { runEval, formatEvalReport } from "vigiles/eval";
321
-
322
- const report = await runEval({
323
- arms: { off: {}, on: { settings: { hooks: { PostToolUse: [refsHook] } } } },
324
- task: "Document chargeCard in SKILL.md, referencing it by name.",
325
- measure: (ctx) => ({
326
- marked: ctx.sh("grep -c vigiles:symbol SKILL.md") !== "0",
327
- }),
328
- trials: 6,
329
- cache: "readwrite", // replay past runs — editing `measure` re-scores for free
330
- });
331
- console.log(formatEvalReport(report));
332
- // off marked=0.00 on marked=0.50±0.20 pass^k=0 ($0.07 · 1.2s/run · 4.1k tok)
333
- ```
334
-
335
- `assertSignificant(report, { baseline: "off", arm: "on", metric: "marked" })`
336
- turns the gap into a CI gate — a Welch t-test decides whether it cleared the
337
- noise floor, **computed** from the arms' spread, not hand-fed. Runs go
338
- **concurrently**, track **cost / latency / tokens** (cap them with `maxCostUsd`),
339
- and the **record/replay cache** makes re-scoring after a `measure` edit free.
340
-
341
- Same tier, different question: **`measureTriggerRate`** measures how reliably a
342
- skill's _description fires_ across varied prompts — the #1 skill-authoring pain.
343
-
344
- ### Test your skills for real — and assert on what Claude _did_
345
-
346
- Install a plugin the way Claude actually does (`pluginDir` → `--plugin-dir`) so
347
- its **skills genuinely activate**, then assert on the agent's _actions_, not a
348
- stdout grep:
349
-
350
- ```typescript
351
- import { assertSkillResolved, assertToolNotUsed } from "vigiles/harness-assert";
352
-
353
- const r = await runHarnessTest({
354
- pluginDir: "./my-plugin",
355
- transcript: true, // populate r.toolCalls
356
- allowedTools: ["Read", "Write", "Bash", "Skill"],
357
- model: scriptModel([
358
- { tool: "Skill", input: { skill: "my-plugin:greet" } },
359
- { text: "ok" },
360
- ]),
361
- });
362
- assertSkillResolved(r, "my-plugin:greet"); // the skill fired, no error
363
- assertToolNotUsed(r, /^mcp__github__merge/); // the safety negative: the scary tool was never called
364
- ```
365
-
366
- `assertToolNotUsed` is how you test a safety rule **honestly** — _proving_ the
367
- dangerous tool was never used, which "the file looks unchanged" can't. It works
368
- on **real third-party plugins** too: the suite confirms real `obra/superpowers`
369
- and `wshobson/agents` skills resolve this way, with no markers injected.
370
-
371
- ### Did the injected context actually reach the model?
372
-
373
- A SessionStart hook or a slash command can _fire_ and still inject **nothing** —
374
- wrong output shape, wrong platform. `trace.modelRequests` records what the model
375
- actually received (system + messages), so you assert it landed, not just that the
376
- hook ran — **"fired ≠ landed"**:
377
-
378
- ```typescript
379
- import { assertRequestContains } from "vigiles/harness-assert";
380
-
381
- assertRequestContains(r, "You have superpowers"); // the additionalContext reached the model
382
- ```
383
-
384
- (Dogfood: this is exactly how vigiles found that real `obra/superpowers` emits a
385
- _top-level_ `additionalContext`, which Claude Code — reading the _nested_ form —
386
- never injects. The hook fired; the context never landed.)
387
-
388
- ### Running an untrusted plugin? It's confined by default
389
-
390
- Testing a third-party plugin means executing **its** hooks. `runHarnessTest` is
391
- safe by default: code you wrote (inline `settings`/`files`) runs directly, but an
392
- external `plugin` / `pluginDir` is **confined under bubblewrap** — a network
393
- namespace with **no egress** (a malicious hook can't phone home), a read-only
394
- filesystem, and a **cleared environment** (your `ANTHROPIC_API_KEY` and other
395
- secrets aren't even visible). If no sandbox is available the run **refuses**
396
- rather than executing unconfined:
397
-
398
- ```typescript
399
- runHarnessTest({ pluginDir: "./vendor/some-plugin", model }); // confined, or refuses
400
- runHarnessTest({ pluginDir: "./audited", model, sandbox: false }); // you vouch for it → direct
401
- ```
402
-
403
- Confinement is **Linux-only** (bubblewrap); on macOS / Windows an untrusted run
404
- refuses unless you pass `sandbox: false`. The suite dogfoods it on real
405
- `obra/superpowers` — its `SessionStart` hook runs in a no-egress sandbox, and the
406
- test proves egress is blocked while the scripted mock stays reachable.
407
-
408
- ### Run them in CI
409
-
410
- `vigiles test` runs `*.harness.mjs` files (free, no key); `vigiles eval` runs
411
- `*.eval.mjs` files (real model). Point a test at a whole plugin (or `"./"` for
412
- your repo) to load **what ships** — hooks (with `${CLAUDE_PLUGIN_ROOT}`
413
- resolved), CLAUDE.md, skills, subagents, commands — and `loadPlugin().warnings`
414
- flags anything only a real model can drive, so you never silently test an empty
415
- machine.
111
+ Or do it yourself:
416
112
 
417
113
  ```bash
418
- npx vigiles test examples/harness/policy-gate.harness.mjs
419
- npx vigiles eval --trials=6 examples/harness/skill-outcome.eval.mjs
114
+ npx vigiles init # sets up BOTH pillars: spec + harness test + CI + plugin
420
115
  ```
421
116
 
422
- ### What's covered today surface × tier
423
-
424
- | Surface | Unit / static | Integration (no API key) | Eval (real model) |
425
- | ------------------------------------------------------------- | ---------------------------- | --------------------------- | ----------------- |
426
- | Hooks Bash / SessionStart / Stop / UserPromptSubmit | ✅ logic | ✅ fires | ✅ |
427
- | Hooks Edit / Write | logic | fires | ✅ |
428
- | Hooks — PreCompact / Notification / SessionEnd / SubagentStop | logic | — (mock can't trigger) | 🟡 |
429
- | CLAUDE.md / instructions | ✅ refs | 🟡 present, not behaviour | ✅ behaviour |
430
- | Skills | 🟡 refs | ✅ resolves via `pluginDir` | ✅ activation |
431
- | Subagents (`agents/`) | ✅ tool rail · 🟡 refs | 🟡 rail not live-armed | ✅ via Task |
432
- | Slash commands (`commands/`) | 🟡 refs | 🟡 needs prompt capture | ✅ via `/cmd` |
433
- | MCP servers | ✅ tool refs (`vigiles:mcp`) | 🔴 | 🔴 |
434
- | settings.json | 🟡 assert merged | ✅ applied | ✅ |
435
- | Hook context injection (does it _land_?) | — n/a | ✅ `trace.modelRequests` | ✅ |
436
- | Untrusted plugin execution | — n/a | ✅ confined (bwrap, Linux) | 🟡 outer sandbox |
437
-
438
- ✅ shipped · 🟡 partial · 🔴 gap · — n/a. Full detail + roadmap: [`research/harness-testing-coverage-matrix.md`](research/harness-testing-coverage-matrix.md).
439
-
440
- [Full guide → `docs/harness-testing.md`](docs/harness-testing.md) · [benchmarks](research/benchmarks-runtime-gates.md).
441
-
442
- ## CLI & CI
117
+ It's interactive in a terminal and non-interactive for agents/CI (or with
118
+ `--yes`), so "set up vigiles" from a Claude Code / Codex prompt Just Works — and
119
+ it installs a model-invocable **`test-harness` skill**, so afterward you can just
120
+ tell your agent _"test my skills"_ and it picks the tier and writes the test.
121
+ Scope `init` with `--pillars=verify|test|both`. Or write harness tests yourself in
122
+ JS **or** TS (`*.harness.{mjs,ts}`) and run `npx vigiles test`. `init` wires CI as
123
+ a `zernie/vigiles@v1` workflow a composite over the same CLI:
443
124
 
444
- ```bash
445
- npx vigiles init # Scaffold a spec (full setup wizard)
446
- npx vigiles compile # Compile .spec.ts .md
447
- npx vigiles audit # Verify hashes + inline/frontmatter/spec rules + symbols + coverage
448
- npx vigiles test # Run *.harness.mjs deterministic harness tests (no API key)
449
- npx vigiles eval # Run *.eval.mjs real-model harness evals (--trials=N)
125
+ ```yaml
126
+ - uses: actions/checkout@v4
127
+ - uses: zernie/vigiles@v1 # audits by default; posts a sticky PR comment + a `valid` output
450
128
  ```
451
129
 
452
- `vigiles audit` enforces four rules — `require-spec`, `require-skill-spec`, `integrity`, `coverage` — configurable in `.vigilesrc.json`. The GitHub Action runs `audit` by default; the Claude Code plugin (`npx skills add zernie/vigiles`) adds the Pre/PostToolUse hooks that block direct `.md` edits and auto-compile specs. [Full CLI, Action, plugin & validation reference →](docs/cli.md)
453
-
454
- ## Skills
455
-
456
- Install with [Vercel Skills](https://github.com/vercel-labs/skills): `npx skills add zernie/vigiles`
457
-
458
- <details>
459
- <summary><b>The 7 skills</b></summary>
460
-
461
- | Skill | What it does |
462
- | ---------------------- | ----------------------------------------------------------------------- |
463
- | `strengthen` | Upgrade `guidance()` → `enforce()` using linter-specific reference docs |
464
- | `edit-spec` | Edit a spec file — guided workflow with compile step |
465
- | `migrate-to-spec` | Convert a hand-written CLAUDE.md to a typed `.spec.ts` |
466
- | `generate-rule` | Add a new `enforce()` / `guidance()` rule to a spec |
467
- | `pr-to-lint-rule` | Turn a recurring PR review comment into a lint rule + spec entry |
468
- | `enforce-rules-format` | Validate all rules have enforcement classification |
469
- | `audit-feedback-loop` | Score your repo's feedback loop maturity |
470
-
471
- </details>
472
-
473
- ## Maturity Levels
474
-
475
- From [Feedback Loop Is All You Need](https://zernie.com/blog/feedback-loop-is-all-you-need): **Vibes → Guardrails → Architecture as Code → The Organism**.
476
-
477
- <details>
478
- <summary>What each level means</summary>
479
-
480
- | Level | Name | What it means |
481
- | ----- | -------------------- | ------------------------------------------------------------------- |
482
- | 0 | Vibes | No CI, no linters, no CLAUDE.md |
483
- | 1 | Guardrails | CI + standard linters, no custom rules |
484
- | 2 | Architecture as Code | Custom lint rules + enforced CLAUDE.md |
485
- | 3 | The Organism | CI + custom rules + visual tests + observability + scheduled agents |
486
-
487
- </details>
488
-
489
- ## Related Tools
490
-
491
- vigiles owns one thing: compile-time verification of typed specs against real linter configs, filesystems, and package.json, plus testing the harness those specs describe. Everything else it composes with rather than replaces — architectural linters ([ast-grep](https://ast-grep.github.io/), [Dependency Cruiser](https://github.com/sverweij/dependency-cruiser)) referenced via `enforce()`, file-sync tools ([Ruler](https://github.com/intellectronica/ruler), [rulesync](https://github.com/dyoshikawa/rulesync)) that distribute the compiled output, and markdown/prose linters that check a different layer. [How vigiles composes with each, and why runtime-LLM rule checkers are the opposite paradigm →](docs/related-tools.md)
492
-
493
- ## Documentation
130
+ ## More
494
131
 
495
- - **[docs/](docs/README.md)** — how-to & reference: the adoption ladder, CLI, linter support, the harness-testing guide, skills/agents.
496
- - **[research/](research/README.md)** — the thinking behind it: design docs, the [harness-testing coverage roadmap](research/harness-testing-coverage-matrix.md), benchmark findings, landscape, and parked ideas.
132
+ - **[CLI & GitHub Action →](docs/cli.md)** — every command, the Action (inputs / output / versioning), the Claude Code plugin, and the five `audit` rules.
133
+ - **[Skills →](docs/skills.md)** — 8 skills (`strengthen`, `migrate-to-spec`, `test-harness`, ) via `npx skills add zernie/vigiles`.
134
+ - **[Docs index →](docs/README.md)** · **[Research →](research/README.md)** · **[Related tools →](docs/related-tools.md)** (ast-grep, Dependency Cruiser, Ruler, rulesync).
135
+ - Companion to [Feedback Loop Is All You Need](https://zernie.com/blog/feedback-loop-is-all-you-need).
497
136
 
498
137
  ## License
499
138