vigiles 2.6.0 → 3.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (207) hide show
  1. package/.claude-plugin/plugin.json +9 -0
  2. package/README.md +74 -129
  3. package/action.yml +143 -7
  4. package/dist/action-gate.d.ts +1 -1
  5. package/dist/action-gate.js +1 -1
  6. package/dist/adapter-conformance.d.ts +30 -0
  7. package/dist/adapter-conformance.js +153 -0
  8. package/dist/adapter-registry.d.ts +42 -0
  9. package/dist/adapter-registry.js +55 -0
  10. package/dist/adapter.d.ts +26 -0
  11. package/dist/adapter.js +16 -0
  12. package/dist/adapters/claude-code/adapter.d.ts +3 -0
  13. package/dist/adapters/claude-code/adapter.js +46 -0
  14. package/dist/{agent-result.d.ts → adapters/claude-code/agent-result.d.ts} +1 -1
  15. package/dist/adapters/claude-code/dialect.d.ts +13 -0
  16. package/dist/adapters/claude-code/dialect.js +51 -0
  17. package/dist/adapters/claude-code/egress-entry.d.ts +2 -0
  18. package/dist/adapters/claude-code/egress-entry.js +115 -0
  19. package/dist/adapters/claude-code/egress.d.ts +114 -0
  20. package/dist/adapters/claude-code/egress.js +276 -0
  21. package/dist/{eval-cache.d.ts → adapters/claude-code/eval-cache.d.ts} +1 -1
  22. package/dist/{eval-cache.js → adapters/claude-code/eval-cache.js} +1 -1
  23. package/dist/{eval.d.ts → adapters/claude-code/eval.d.ts} +28 -4
  24. package/dist/{eval.js → adapters/claude-code/eval.js} +61 -25
  25. package/dist/{harness-test.d.ts → adapters/claude-code/harness-test.d.ts} +33 -37
  26. package/dist/{harness-test.js → adapters/claude-code/harness-test.js} +124 -43
  27. package/dist/adapters/claude-code/hook-protocol.d.ts +10 -0
  28. package/dist/adapters/claude-code/hook-protocol.js +10 -0
  29. package/dist/adapters/claude-code/layout.d.ts +8 -0
  30. package/dist/adapters/claude-code/layout.js +18 -0
  31. package/dist/{mock-model.d.ts → adapters/claude-code/mock-model.d.ts} +2 -24
  32. package/dist/adapters/claude-code/model-mock.d.ts +11 -0
  33. package/dist/adapters/claude-code/model-mock.js +10 -0
  34. package/dist/adapters/claude-code/plugin-loader.d.ts +25 -0
  35. package/dist/adapters/claude-code/plugin-loader.js +19 -0
  36. package/dist/{run-hook.d.ts → adapters/claude-code/run-hook.d.ts} +50 -3
  37. package/dist/{run-hook.js → adapters/claude-code/run-hook.js} +178 -18
  38. package/dist/adapters/claude-code/run-scripts.d.ts +52 -0
  39. package/dist/adapters/claude-code/run-scripts.js +150 -0
  40. package/dist/adapters/claude-code/runtime.d.ts +16 -0
  41. package/dist/adapters/claude-code/runtime.js +39 -0
  42. package/dist/{sandbox.d.ts → adapters/claude-code/sandbox.d.ts} +10 -0
  43. package/dist/{sandbox.js → adapters/claude-code/sandbox.js} +5 -3
  44. package/dist/{skill-driver.d.ts → adapters/claude-code/skill-driver.d.ts} +1 -1
  45. package/dist/adapters/codex/adapter.d.ts +3 -0
  46. package/dist/adapters/codex/adapter.js +49 -0
  47. package/dist/adapters/codex/dialect.d.ts +10 -0
  48. package/dist/adapters/codex/dialect.js +30 -0
  49. package/dist/adapters/codex/driver.d.ts +20 -0
  50. package/dist/adapters/codex/driver.js +89 -0
  51. package/dist/adapters/codex/hook-protocol.d.ts +10 -0
  52. package/dist/adapters/codex/hook-protocol.js +18 -0
  53. package/dist/adapters/codex/layout.d.ts +16 -0
  54. package/dist/adapters/codex/layout.js +18 -0
  55. package/dist/adapters/codex/mock-model.d.ts +52 -0
  56. package/dist/adapters/codex/mock-model.js +210 -0
  57. package/dist/adapters/codex/model-mock.d.ts +11 -0
  58. package/dist/adapters/codex/model-mock.js +10 -0
  59. package/dist/adapters/codex/runtime.d.ts +38 -0
  60. package/dist/adapters/codex/runtime.js +62 -0
  61. package/dist/adapters/opencode/adapter.d.ts +3 -0
  62. package/dist/adapters/opencode/adapter.js +48 -0
  63. package/dist/adapters/opencode/dialect.d.ts +10 -0
  64. package/dist/adapters/opencode/dialect.js +36 -0
  65. package/dist/adapters/opencode/layout.d.ts +9 -0
  66. package/dist/adapters/opencode/layout.js +25 -0
  67. package/dist/adapters/opencode/model-mock.d.ts +10 -0
  68. package/dist/adapters/opencode/model-mock.js +10 -0
  69. package/dist/adapters/opencode/runtime.d.ts +9 -0
  70. package/dist/adapters/opencode/runtime.js +21 -0
  71. package/dist/claude-code.d.ts +8 -2
  72. package/dist/claude-code.js +8 -2
  73. package/dist/cli-flags.d.ts +22 -0
  74. package/dist/cli-flags.js +38 -0
  75. package/dist/cli.js +406 -206
  76. package/dist/codex.d.ts +20 -0
  77. package/dist/codex.js +36 -0
  78. package/dist/community-skills.d.ts +1 -1
  79. package/dist/community-skills.js +2 -2
  80. package/dist/core/adapter.d.ts +88 -0
  81. package/dist/core/adapter.js +3 -0
  82. package/dist/{compile.d.ts → core/compile.d.ts} +11 -2
  83. package/dist/{compile.js → core/compile.js} +68 -53
  84. package/dist/core/compose.d.ts +79 -0
  85. package/dist/core/compose.js +145 -0
  86. package/dist/core/dialect.d.ts +51 -0
  87. package/dist/core/dialect.js +3 -0
  88. package/dist/core/harness-driver.d.ts +134 -0
  89. package/dist/core/harness-driver.js +3 -0
  90. package/dist/core/hook-protocol.d.ts +28 -0
  91. package/dist/core/hook-protocol.js +3 -0
  92. package/dist/core/layout.d.ts +43 -0
  93. package/dist/core/layout.js +3 -0
  94. package/dist/core/model-mock.d.ts +27 -0
  95. package/dist/core/model-mock.js +3 -0
  96. package/dist/core/refs.d.ts +62 -0
  97. package/dist/{refs.js → core/refs.js} +49 -23
  98. package/dist/core/runtime.d.ts +38 -0
  99. package/dist/core/runtime.js +3 -0
  100. package/dist/{types.d.ts → core/types.d.ts} +29 -0
  101. package/dist/{validate.d.ts → core/validate.d.ts} +1 -1
  102. package/dist/{validate.js → core/validate.js} +41 -4
  103. package/dist/e2e.d.ts +19 -0
  104. package/dist/e2e.js +39 -0
  105. package/dist/harness-assert.d.ts +29 -14
  106. package/dist/harness-assert.js +37 -10
  107. package/dist/integration.d.ts +16 -0
  108. package/dist/integration.js +32 -0
  109. package/dist/leaderboard.d.ts +33 -0
  110. package/dist/leaderboard.js +107 -0
  111. package/dist/linting.d.ts +2 -2
  112. package/dist/linting.js +2 -2
  113. package/dist/plugin-loader.d.ts +6 -3
  114. package/dist/plugin-loader.js +101 -49
  115. package/dist/scan.d.ts +49 -0
  116. package/dist/scan.js +177 -0
  117. package/dist/setup-plan.d.ts +51 -0
  118. package/dist/setup-plan.js +85 -0
  119. package/dist/skill-test.d.ts +1 -1
  120. package/dist/skill-test.js +1 -1
  121. package/dist/test-coverage.d.ts +71 -0
  122. package/dist/test-coverage.js +228 -0
  123. package/dist/testing.d.ts +3 -3
  124. package/dist/testing.js +3 -3
  125. package/dist/unit.d.ts +17 -0
  126. package/dist/unit.js +36 -0
  127. package/hooks/refs-nudge.sh +24 -0
  128. package/package.json +30 -15
  129. package/skills/audit-feedback-loop/SKILL.md +76 -0
  130. package/skills/edit-spec/SKILL.md +131 -0
  131. package/skills/enforce-rules-format/SKILL.md +71 -0
  132. package/skills/generate-logo/SKILL.md +103 -0
  133. package/skills/generate-rule/SKILL.md +64 -0
  134. package/skills/linter-docs/clippy.md +241 -0
  135. package/skills/linter-docs/eslint.md +384 -0
  136. package/skills/linter-docs/pylint.md +288 -0
  137. package/skills/linter-docs/rubocop.md +277 -0
  138. package/skills/linter-docs/ruff.md +187 -0
  139. package/skills/linter-docs/stylelint.md +247 -0
  140. package/skills/migrate-to-spec/SKILL.md +126 -0
  141. package/skills/pr-to-lint-rule/SKILL.md +97 -0
  142. package/skills/strengthen/SKILL.md +168 -0
  143. package/skills/test-harness/SKILL.md +157 -0
  144. package/dist/action.d.ts +0 -7
  145. package/dist/action.js +0 -180
  146. package/dist/refs.d.ts +0 -44
  147. package/dist/run-scripts.d.ts +0 -20
  148. package/dist/run-scripts.js +0 -70
  149. /package/dist/{agent-result.js → adapters/claude-code/agent-result.js} +0 -0
  150. /package/dist/{agent-runtime.d.ts → adapters/claude-code/agent-runtime.d.ts} +0 -0
  151. /package/dist/{agent-runtime.js → adapters/claude-code/agent-runtime.js} +0 -0
  152. /package/dist/{egress-proxy.d.ts → adapters/claude-code/egress-proxy.d.ts} +0 -0
  153. /package/dist/{egress-proxy.js → adapters/claude-code/egress-proxy.js} +0 -0
  154. /package/dist/{eval-baseline.d.ts → adapters/claude-code/eval-baseline.d.ts} +0 -0
  155. /package/dist/{eval-baseline.js → adapters/claude-code/eval-baseline.js} +0 -0
  156. /package/dist/{judge.d.ts → adapters/claude-code/judge.d.ts} +0 -0
  157. /package/dist/{judge.js → adapters/claude-code/judge.js} +0 -0
  158. /package/dist/{mock-entry.d.ts → adapters/claude-code/mock-entry.d.ts} +0 -0
  159. /package/dist/{mock-entry.js → adapters/claude-code/mock-entry.js} +0 -0
  160. /package/dist/{mock-model.js → adapters/claude-code/mock-model.js} +0 -0
  161. /package/dist/{skill-driver.js → adapters/claude-code/skill-driver.js} +0 -0
  162. /package/dist/{skill-runtime.d.ts → adapters/claude-code/skill-runtime.d.ts} +0 -0
  163. /package/dist/{skill-runtime.js → adapters/claude-code/skill-runtime.js} +0 -0
  164. /package/dist/{stats.d.ts → adapters/claude-code/stats.d.ts} +0 -0
  165. /package/dist/{stats.js → adapters/claude-code/stats.js} +0 -0
  166. /package/dist/{compile-generator.d.ts → core/compile-generator.d.ts} +0 -0
  167. /package/dist/{compile-generator.js → core/compile-generator.js} +0 -0
  168. /package/dist/{coverage.d.ts → core/coverage.d.ts} +0 -0
  169. /package/dist/{coverage.js → core/coverage.js} +0 -0
  170. /package/dist/{doc-refs.d.ts → core/doc-refs.d.ts} +0 -0
  171. /package/dist/{doc-refs.js → core/doc-refs.js} +0 -0
  172. /package/dist/{evolve.d.ts → core/evolve.d.ts} +0 -0
  173. /package/dist/{evolve.js → core/evolve.js} +0 -0
  174. /package/dist/{frontmatter.d.ts → core/frontmatter.d.ts} +0 -0
  175. /package/dist/{frontmatter.js → core/frontmatter.js} +0 -0
  176. /package/dist/{generate-schema.d.ts → core/generate-schema.d.ts} +0 -0
  177. /package/dist/{generate-schema.js → core/generate-schema.js} +0 -0
  178. /package/dist/{generate-types.d.ts → core/generate-types.d.ts} +0 -0
  179. /package/dist/{generate-types.js → core/generate-types.js} +0 -0
  180. /package/dist/{hash.d.ts → core/hash.d.ts} +0 -0
  181. /package/dist/{hash.js → core/hash.js} +0 -0
  182. /package/dist/{inline.d.ts → core/inline.d.ts} +0 -0
  183. /package/dist/{inline.js → core/inline.js} +0 -0
  184. /package/dist/{integrity.d.ts → core/integrity.d.ts} +0 -0
  185. /package/dist/{integrity.js → core/integrity.js} +0 -0
  186. /package/dist/{linters.d.ts → core/linters.d.ts} +0 -0
  187. /package/dist/{linters.js → core/linters.js} +0 -0
  188. /package/dist/{mcp.d.ts → core/mcp.d.ts} +0 -0
  189. /package/dist/{mcp.js → core/mcp.js} +0 -0
  190. /package/dist/{orphans.d.ts → core/orphans.d.ts} +0 -0
  191. /package/dist/{orphans.js → core/orphans.js} +0 -0
  192. /package/dist/{proofs.d.ts → core/proofs.d.ts} +0 -0
  193. /package/dist/{proofs.js → core/proofs.js} +0 -0
  194. /package/dist/{session.d.ts → core/session.d.ts} +0 -0
  195. /package/dist/{session.js → core/session.js} +0 -0
  196. /package/dist/{sidecar.d.ts → core/sidecar.d.ts} +0 -0
  197. /package/dist/{sidecar.js → core/sidecar.js} +0 -0
  198. /package/dist/{spec.d.ts → core/spec.d.ts} +0 -0
  199. /package/dist/{spec.js → core/spec.js} +0 -0
  200. /package/dist/{symbols.d.ts → core/symbols.d.ts} +0 -0
  201. /package/dist/{symbols.js → core/symbols.js} +0 -0
  202. /package/dist/{test-utils.d.ts → core/test-utils.d.ts} +0 -0
  203. /package/dist/{test-utils.js → core/test-utils.js} +0 -0
  204. /package/dist/{types.js → core/types.js} +0 -0
  205. /package/{.claude-plugin/hooks → hooks}/post-edit.sh +0 -0
  206. /package/{.claude-plugin/hooks → hooks}/pre-edit.sh +0 -0
  207. /package/{.claude-plugin/hooks → hooks}/session-start.sh +0 -0
@@ -30,6 +30,15 @@
30
30
  "command": "bash ${CLAUDE_PLUGIN_ROOT}/hooks/post-edit.sh"
31
31
  }
32
32
  ]
33
+ },
34
+ {
35
+ "matcher": "Edit|Write",
36
+ "hooks": [
37
+ {
38
+ "type": "command",
39
+ "command": "bash ${CLAUDE_PLUGIN_ROOT}/hooks/refs-nudge.sh"
40
+ }
41
+ ]
33
42
  }
34
43
  ],
35
44
  "SessionStart": [
package/README.md CHANGED
@@ -19,175 +19,120 @@
19
19
  <a href="https://github.com/zernie/vigiles/blob/main/LICENSE"><img src="https://img.shields.io/github/license/zernie/vigiles" alt="License" /></a>
20
20
  </p>
21
21
 
22
- <p align="center">
23
- <b>Two pillars →</b>
24
- &nbsp;<a href="#verify-your-instruction-files">① Verify your instruction files</a>
25
- &nbsp;·&nbsp;
26
- <a href="#test-your-claude-code-harness">② Test your Claude Code harness</a>
27
- </p>
28
-
29
22
  ---
30
23
 
31
- <details>
32
- <summary><b>Contents</b></summary>
33
-
34
- - [**Two pillars pick one or both**](#two-pillars--pick-one-or-both)
35
- - **Pillar 1** [verify your instruction files](#verify-your-instruction-files) · full guide: [docs/verifying-instruction-files.md](docs/verifying-instruction-files.md)
36
- - **Pillar 2** — [test your Claude Code harness](#test-your-claude-code-harness) · full guide: [docs/harness-testing.md](docs/harness-testing.md)
37
- - [Quick start](#quick-start) · [CLI & CI](#cli--ci) · [Skills](#skills) · [Related tools](#related-tools)
38
-
39
- </details>
40
-
41
- ## Two pillars — pick one or both
42
-
43
- An agent runs real commands in your repo — it can delete the wrong files, leak a secret, or burn tokens looping on a stale instruction nobody checked. You'd never ship an app without a linter and a test suite. An AI agent steering your codebase is no different — so why is its harness trusted on vibes?
44
-
45
- `Agent = Model + Harness`. Your harness is everything that steers a run — the **instructions** you write _and_ the **hooks, skills, and settings** that enforce them. vigiles is the **missing linting + testing layer for agentic coding**: it **lints** your instruction files and **tests** your harness. Two pillars of equal weight — adopt either on its own, or both:
46
-
47
- | | Pillar | What it does |
48
- | ----- | ------------------------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
49
- | **①** | [**Verify your instruction files**](#verify-your-instruction-files) | Every linter rule, file path, script, and code symbol your CLAUDE.md cites is checked against reality, so stale references can't silently mislead the agent. |
50
- | **②** | [**Test your harness**](#test-your-claude-code-harness) | Your hooks and skills are code — vigiles tests that they actually fire, **deterministically and for free** (no model, no API key) before you ever pay for a real-model eval. |
24
+ `Agent = Model + Harness`. You'd never ship an app without a linter and a test
25
+ suite — yet an AI agent steering your repo is trusted on vibes. vigiles is the
26
+ deterministic layer for the harness: it **lints** the references your instruction
27
+ files make and **tests** that your hooks and skills actually fire. Two independent
28
+ pillarsadopt either, or both:
51
29
 
52
- They share the thesis but not a dependency: verify your instructions without ever writing a harness test, or test your harness without a single `.spec.ts`. Pick the pillar that hurts today.
30
+ | | Pillar | What it does |
31
+ | ----- | --------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
32
+ | **①** | **Verify your instruction files** | Every linter rule, file path, script, and code symbol your CLAUDE.md cites is checked against reality, so stale references can't silently mislead the agent. → [guide](docs/verifying-instruction-files.md) |
33
+ | **②** | **Test your harness** | Your hooks and skills are code — vigiles tests they actually fire, **deterministically and free** (no model, no API key) before you pay for an eval. → [guide](docs/harness-testing.md) |
53
34
 
54
- ## Verify your instruction files
35
+ Neither pillar depends on the other — pick the one that hurts today. **Works with
36
+ Claude Code and Codex** ([`vigiles/codex`](docs/harnesses.md)) behind a five-port
37
+ adapter; [custom adapters welcome](docs/authoring-an-adapter.md).
55
38
 
56
- Your CLAUDE.md lies to your agent. Here's the fix.
39
+ ## ① Verify — your CLAUDE.md lies to your agent
57
40
 
58
- Hand-written CLAUDE.md files rot silently. Here's what a typical one looks like:
41
+ Your CLAUDE.md says _"enforce `eslint/no-console`."_ But it was switched off
42
+ months ago — and the agent trusts the claim. (Same story for the file path it
43
+ cites that got renamed, and the script that was deleted.)
59
44
 
60
- ```markdown
61
- ## Code Style
45
+ **Without vigiles:** nobody checks. The agent acts on fiction.
62
46
 
63
- Never use `any` the `@typescript-eslint/no-explicit-any` rule
64
- catches this. Always use `unknown` and narrow with type guards.
65
- See `src/utils/type-helpers.ts` for project utilities.
47
+ **With vigiles:** `npx vigiles audit` resolves every reference against reality —
66
48
 
67
- ## Testing
68
-
69
- Run `npm run typecheck` before submitting. Every service in
70
- src/services/ should have a corresponding test file.
49
+ ```text
50
+ CLAUDE.md (inline mode):
51
+ line 1: Rule "eslint/no-console" exists but is disabled in eslint config
52
+ line 2: eslint/eqeqeq
53
+ ✗ line 3: Rule "no-consoel" not found in eslint. Did you mean: "eslint/no-console"?
71
54
  ```
72
55
 
73
- Reads fine. Four things are wrong:
74
-
75
- 1. `@typescript-eslint/no-explicit-any` disabled to unblock a deadline, never re-enabled
76
- 2. `src/utils/type-helpers.ts` — renamed to `src/utils/narrowing.ts` last quarter
77
- 3. `npm run typecheck` — script removed from package.json
78
- 4. Service/test pairing — no automated check, just a hope
79
-
80
- The agent reads this, trusts it, and writes code based on stale claims nobody verified. vigiles **verifies the references in your instruction files** — that each linter rule exists and is enabled, that every file path and script is real, and that referenced **code symbols** (functions, classes, constants) actually exist in the files that define them — and meets you at whatever commitment level you want.
81
-
82
- > **Set it up — paste this into Claude Code:**
83
- >
84
- > > Install vigiles and verify my instruction files. Scan my CLAUDE.md / AGENTS.md, turn its real claims into checked references (linter rules, file paths, scripts, symbols), run `vigiles audit`, and show me what's already stale. Use sensible defaults, but **ask me first** whether to stay in markdown mode or generate a typed `.spec.ts`, and whether to wire the audit into CI and install the edit-blocking hooks.
85
-
86
- Or do it by hand — add a marker to your existing CLAUDE.md and audit it, no install, no new files:
56
+ It resolves rule names across **7 linter catalogs** — the rule exists **and is
57
+ enabled** — and checks file paths, scripts, and code symbols the same way. Start
58
+ with one comment, no new files:
87
59
 
88
60
  ```md
89
61
  <!-- vigiles:enforce eslint/no-console "Route output through logger.ts" -->
90
62
  ```
91
63
 
92
- ```bash
93
- npx vigiles audit CLAUDE.md
94
- ```
64
+ Step up to a typed `.spec.ts` (compiled to CLAUDE.md, compiler-grade) when you
65
+ want it. **[Full guide →](docs/verifying-instruction-files.md)**
95
66
 
96
- Each reference is checked against reality a typo gets a closest-match suggestion, a disabled rule is flagged. That's **markdown mode**; step up to a **typed spec** (`.spec.ts` → compiled CLAUDE.md, compiler-grade guarantees) when you want it.
67
+ ## Testdoes your harness actually fire?
97
68
 
98
- **Full guide: [docs/verifying-instruction-files.md](docs/verifying-instruction-files.md)** the adoption ladder, the three rule types (`enforce` / `guidance` / `guard`), verified references (`file` / `cmd` / `symbol` / `ref`), and the before/after tables.
69
+ A hook can be wired wrong, a skill's description can fail to trigger, injected
70
+ context can never reach the model — silently, all passing a naive "did it run?"
71
+ check.
99
72
 
100
- ## Quick Start
73
+ **Without vigiles:** you assume your `--no-verify` guard blocks. You don't know.
101
74
 
102
- ```bash
103
- npx vigiles init
104
- ```
105
-
106
- The wizard auto-detects your project, creates a spec, scans your linters, compiles to markdown, adds a CI step, and installs Claude Code hooks. After install: the agent edits the spec (hooks block direct CLAUDE.md edits), the spec auto-compiles on save, and `vigiles audit` catches drift in CI. Prefer no new files? Stay in [markdown mode](docs/markdown-mode.md). Start with `guidance()` rules and `/strengthen` them to `enforce()` later; flags and agent usage are in the [CLI reference](docs/cli.md).
107
-
108
- Companion repo for [Feedback Loop Is All You Need](https://zernie.com/blog/feedback-loop-is-all-you-need).
109
-
110
- ## Test your Claude Code harness
111
-
112
- Verifying references proves your instructions are _true_ — but the hooks and
113
- skills that enforce them still have to **actually fire**. A hook can be wired
114
- wrong, a skill's description can fail to trigger, injected context can never reach
115
- the model — all silently, all passing a naive "did it run?" check. So vigiles's
116
- second pillar **tests the harness itself**, as the assembled machine it ships as.
117
-
118
- > **Set it up — paste this into Claude Code:**
119
- >
120
- > > Install vigiles and use its `test-harness` skill to write and run a harness test for this project. If I didn't say what to test, pick something real from my hooks / skills / settings, choose the cheapest tier (unit / deterministic / eval), write the test, and run it. Use good defaults, but **ask me** whether to gate it in CI and whether to add a real-model eval.
121
-
122
- It's a small library of plain async functions (drops into node:test / vitest /
123
- jest, or a zero-setup `vigiles test`), with three tiers, cheapest first. The
124
- design bet is **deterministic and cheap**: the first two tiers never call a model
125
- or need an API key, so they run on every commit for free — the opposite of
126
- eval-only frameworks like promptfoo, where every run hits a real model **by
127
- design**. You only reach for the paid real-model tier when the question genuinely
128
- needs it.
129
-
130
- - **Unit-test a hook** — `runHook` hands a hook a fake event and checks block/allow. No `claude`, no model, milliseconds, reaches **every** event type.
131
- - **Deterministic harness test** — `runHarnessTest` runs the **real** `claude` against a **scripted mock model**, so your hooks fire for real with no API key and the same result every time.
132
- - **Eval** — `runEval` runs the real model A/B (change on vs off) and reports the gap as **mean ± se**, with a Welch-t-test [significance gate](docs/harness-testing.md#significance--is-the-gap-real), regression baselines, and cost/latency/token tracking.
75
+ **With vigiles:** a deterministic test proves it — no model, no API key,
76
+ milliseconds:
133
77
 
134
78
  ```typescript
135
79
  import { runHook } from "vigiles/testing";
136
80
 
137
- const r = runHook(guardCommand, {
81
+ const r = runHook(guard, {
138
82
  hook_event_name: "PreToolUse",
139
83
  tool_name: "Bash",
140
84
  tool_input: { command: "git commit --no-verify" },
141
85
  });
142
- assert(r.blocked); // exit 2 / decision:"block" / permissionDecision:"deny"
86
+ assert(r.blocked); // a red here means your hook silently lets it through
143
87
  ```
144
88
 
145
- **[Full guide → `docs/harness-testing.md`](docs/harness-testing.md)** — the tier
146
- walkthrough, testing skills for real, "fired landed" (`trace.modelRequests`),
147
- the safe-by-default sandbox for untrusted plugins, the surface × tier coverage
148
- matrix, and how it compares to promptfoo. Also: [benchmarks](research/benchmarks-runtime-gates.md).
89
+ ```text
90
+ guard blocks --no-verify and allows a clean commit
149
91
 
150
- ## CLI & CI
151
-
152
- ```bash
153
- npx vigiles init # Scaffold a spec (full setup wizard)
154
- npx vigiles compile # Compile .spec.ts → .md
155
- npx vigiles audit # Verify hashes + inline/frontmatter/spec rules + symbols + coverage
156
- npx vigiles test # Run *.harness.mjs deterministic harness tests (no API key)
157
- npx vigiles eval # Run *.eval.mjs real-model harness evals (--trials=N)
92
+ 2 passed.
158
93
  ```
159
94
 
160
- `vigiles audit` enforces four rules — `require-spec`, `require-skill-spec`, `integrity`, `coverage` — configurable in `.vigilesrc.json`. The GitHub Action runs `audit` by default; the Claude Code plugin (`npx skills add zernie/vigiles`) adds the Pre/PostToolUse hooks that block direct `.md` edits and auto-compile specs. [Full CLI, Action, plugin & validation reference →](docs/cli.md)
161
-
162
- ## Skills
163
-
164
- Install with [Vercel Skills](https://github.com/vercel-labs/skills): `npx skills add zernie/vigiles`
95
+ Three tiers, cheapest first: **`runHook`** (a hook's logic), **`runHarnessTest`**
96
+ (the real agent CLI against a scripted mock model), **`runEval`** (the real model
97
+ A/B with a significance gate). **Testing a skill?** `measureTriggerRate` checks
98
+ its description actually **fires** across varied prompts (recall) without
99
+ hijacking unrelated ones (precision). **[Full guide →](docs/harness-testing.md)**
165
100
 
166
- <details>
167
- <summary><b>The 8 skills</b></summary>
101
+ ## Quick start
168
102
 
169
- | Skill | What it does |
170
- | ---------------------- | ------------------------------------------------------------------------------------------------- |
171
- | `strengthen` | Upgrade `guidance()` `enforce()` using linter-specific reference docs |
172
- | `edit-spec` | Edit a spec file guided workflow with compile step |
173
- | `migrate-to-spec` | Convert a hand-written CLAUDE.md to a typed `.spec.ts` |
174
- | `generate-rule` | Add a new `enforce()` / `guidance()` rule to a spec |
175
- | `pr-to-lint-rule` | Turn a recurring PR review comment into a lint rule + spec entry |
176
- | `enforce-rules-format` | Validate all rules have enforcement classification |
177
- | `audit-feedback-loop` | Score your repo's feedback loop maturity |
178
- | `test-harness` | Test a Claude Code harness — pick the tier (unit / deterministic / eval) and write a passing test |
103
+ > **Paste into Claude Code or Codex:**
104
+ >
105
+ > > Install vigiles in this repo and run it. Verify my CLAUDE.md / AGENTS.md
106
+ > > references and show me what's stale, then write and run a harness test for one
107
+ > > of my hooks or skills. Use good defaults (both pillars, non-interactive), but
108
+ > > **ask me first** whether to gate it in CI, whether to add a real-model eval,
109
+ > > and whether to enforce strictly (`--strict`).
179
110
 
180
- </details>
111
+ Or do it yourself:
181
112
 
182
- ## Related Tools
113
+ ```bash
114
+ npx vigiles init # sets up BOTH pillars: spec + harness test + CI + plugin
115
+ ```
183
116
 
184
- vigiles composes with other tools rather than replacing them: architectural linters ([ast-grep](https://ast-grep.github.io/), [Dependency Cruiser](https://github.com/sverweij/dependency-cruiser)) referenced via `enforce()`, and file-sync tools ([Ruler](https://github.com/intellectronica/ruler), [rulesync](https://github.com/dyoshikawa/rulesync)) that distribute the compiled output. [How it fits with each, and why runtime-LLM rule checkers are the opposite paradigm →](docs/related-tools.md)
117
+ It's interactive in a terminal and non-interactive for agents/CI (or with
118
+ `--yes`), so "set up vigiles" from a Claude Code / Codex prompt Just Works — and
119
+ it installs a model-invocable **`test-harness` skill**, so afterward you can just
120
+ tell your agent _"test my skills"_ and it picks the tier and writes the test.
121
+ Scope `init` with `--pillars=verify|test|both`. Or write harness tests yourself in
122
+ JS **or** TS (`*.harness.{mjs,ts}`) and run `npx vigiles test`. `init` wires CI as
123
+ a `zernie/vigiles@v1` workflow — a composite over the same CLI:
124
+
125
+ ```yaml
126
+ - uses: actions/checkout@v4
127
+ - uses: zernie/vigiles@v1 # audits by default; posts a sticky PR comment + a `valid` output
128
+ ```
185
129
 
186
- ## Documentation
130
+ ## More
187
131
 
188
- - **The two pillar guides:** [verifying instruction files](docs/verifying-instruction-files.md) (Pillar 1) · [testing your harness](docs/harness-testing.md) (Pillar 2).
189
- - **[docs/](docs/README.md)** — the full how-to & reference index: adoption ladder, CLI, linter support, skills/agents.
190
- - **[research/](research/README.md)** the thinking behind it: design docs, the [harness-testing coverage roadmap](research/harness-testing-coverage-matrix.md), benchmark findings, landscape, and parked ideas.
132
+ - **[CLI & GitHub Action ](docs/cli.md)** every command, the Action (inputs / output / versioning), the Claude Code plugin, and the five `audit` rules.
133
+ - **[Skills →](docs/skills.md)** — 8 skills (`strengthen`, `migrate-to-spec`, `test-harness`, …) via `npx skills add zernie/vigiles`.
134
+ - **[Docs index →](docs/README.md)** · **[Research →](research/README.md)** · **[Related tools ](docs/related-tools.md)** (ast-grep, Dependency Cruiser, Ruler, rulesync).
135
+ - Companion to [Feedback Loop Is All You Need](https://zernie.com/blog/feedback-loop-is-all-you-need).
191
136
 
192
137
  ## License
193
138
 
package/action.yml CHANGED
@@ -1,25 +1,161 @@
1
1
  name: "vigiles"
2
- description: "Compile typed specs to instruction files and verify integrity"
2
+ description: "Verify the references in your agent instruction files (CLAUDE.md/AGENTS.md) and compile typed specs — in CI."
3
+ author: "zernie"
3
4
  branding:
4
5
  icon: "check-circle"
5
6
  color: "orange"
6
7
 
7
8
  inputs:
8
9
  command:
9
- description: "Command to run: 'compile' or 'audit' (default: audit)"
10
+ description: "Which vigiles command to run: 'audit' (verify references + integrity + coverage) or 'compile' (specs → markdown)."
10
11
  required: false
11
12
  default: "audit"
12
13
  paths:
13
- description: "Comma-separated paths to .spec.ts files (compile) or .md files (audit). Auto-discovers by default."
14
+ description: "Comma- or space-separated paths to operate on (.md files for audit, .spec.ts files for compile). Auto-discovers when empty."
14
15
  required: false
16
+ default: ""
17
+ version:
18
+ description: "Version of the vigiles npm package to run (e.g. '1', '1.2.3', 'latest'). Use 'local' to run the action's own checkout — for dogfooding via `uses: ./` in this repo's CI."
19
+ required: false
20
+ default: "latest"
15
21
  max-rules:
16
- description: "Maximum number of rules per spec file"
22
+ description: "Maximum number of rules per spec file (maps to the CLI's --max-rules)."
17
23
  required: false
24
+ default: ""
18
25
  catalog-only:
19
- description: "Only check that linter rules exist, skip config-enabled checks (true/false)"
26
+ description: "Only check that linter rules exist; skip config-enabled checks (maps to the CLI's --catalog-only). 'true' or 'false'."
20
27
  required: false
21
28
  default: "false"
29
+ working-directory:
30
+ description: "Directory to run vigiles in."
31
+ required: false
32
+ default: "."
33
+ comment:
34
+ description: "On pull_request events, post/update a sticky PR comment with the result ('true' or 'false'). Requires `pull-requests: write`."
35
+ required: false
36
+ default: "true"
37
+ github-token:
38
+ description: "Token used to post the PR comment. Defaults to the workflow token."
39
+ required: false
40
+ default: ${{ github.token }}
41
+
42
+ outputs:
43
+ valid:
44
+ description: "'true' if vigiles passed (exit 0), 'false' otherwise."
45
+ value: ${{ steps.vigiles.outputs.valid }}
22
46
 
23
47
  runs:
24
- using: "node20"
25
- main: "dist/action.js"
48
+ using: "composite"
49
+ steps:
50
+ - id: vigiles
51
+ shell: bash
52
+ working-directory: ${{ inputs.working-directory }}
53
+ env:
54
+ # Inputs are passed through the environment (never interpolated into
55
+ # the script body) so a value can't break out of the command line.
56
+ VIGILES_COMMAND: ${{ inputs.command }}
57
+ VIGILES_PATHS: ${{ inputs.paths }}
58
+ VIGILES_VERSION: ${{ inputs.version }}
59
+ VIGILES_MAX_RULES: ${{ inputs.max-rules }}
60
+ VIGILES_CATALOG_ONLY: ${{ inputs.catalog-only }}
61
+ VIGILES_ACTION_PATH: ${{ github.action_path }}
62
+ VIGILES_COMMENT: ${{ inputs.comment }}
63
+ GH_TOKEN: ${{ inputs.github-token }}
64
+ run: |
65
+ set -euo pipefail
66
+
67
+ cmd="${VIGILES_COMMAND:-audit}"
68
+ args=("$cmd")
69
+
70
+ # paths: split on commas and whitespace into positional args.
71
+ paths="${VIGILES_PATHS:-}"
72
+ if [[ -n "${paths// /}" ]]; then
73
+ IFS=', ' read -r -a _paths <<< "$paths"
74
+ args+=("${_paths[@]}")
75
+ fi
76
+
77
+ max_rules="${VIGILES_MAX_RULES:-}"
78
+ if [[ -n "${max_rules// /}" ]]; then
79
+ args+=("--max-rules=${max_rules}")
80
+ fi
81
+
82
+ if [[ "${VIGILES_CATALOG_ONLY:-false}" == "true" ]]; then
83
+ args+=("--catalog-only")
84
+ fi
85
+
86
+ # Resolve the runner. 'local' runs the action's own build (CI
87
+ # dogfood via `uses: ./`); any other value runs the published npm
88
+ # package at that version, so the Action reuses the same tested
89
+ # artifact users get from `npx vigiles`.
90
+ version="${VIGILES_VERSION:-latest}"
91
+ if [[ "$version" == "local" ]]; then
92
+ runner=(node "${VIGILES_ACTION_PATH}/dist/cli.js")
93
+ else
94
+ runner=(npx --yes "vigiles@${version}")
95
+ fi
96
+
97
+ echo "vigiles: ${runner[*]} ${args[*]}"
98
+
99
+ # Run, streaming to the log AND capturing for the summary/comment.
100
+ out="$(mktemp)"
101
+ set +e
102
+ GITHUB_ACTIONS=true "${runner[@]}" "${args[@]}" 2>&1 | tee "$out"
103
+ status="${PIPESTATUS[0]}"
104
+ set -e
105
+
106
+ if [[ "$status" -eq 0 ]]; then
107
+ echo "valid=true" >> "$GITHUB_OUTPUT"
108
+ headline="✅ \`vigiles ${cmd}\` passed"
109
+ else
110
+ echo "valid=false" >> "$GITHUB_OUTPUT"
111
+ headline="❌ \`vigiles ${cmd}\` failed (exit ${status})"
112
+ echo "::error::vigiles ${cmd} failed (exit ${status})"
113
+ fi
114
+
115
+ # Build a markdown report (cap the log tail so a huge run stays sane).
116
+ report="$(mktemp)"
117
+ {
118
+ echo "## vigiles"
119
+ echo
120
+ echo "$headline"
121
+ echo
122
+ echo '<details><summary>Output</summary>'
123
+ echo
124
+ echo '```'
125
+ tail -c 60000 "$out"
126
+ echo '```'
127
+ echo
128
+ echo '</details>'
129
+ } > "$report"
130
+
131
+ # Job summary (always, when available) — the modern run-page output.
132
+ if [[ -n "${GITHUB_STEP_SUMMARY:-}" ]]; then
133
+ cat "$report" >> "$GITHUB_STEP_SUMMARY"
134
+ fi
135
+
136
+ # Sticky PR comment (pull_request events only, best-effort): find the
137
+ # prior vigiles comment by marker and update it in place, else create.
138
+ marker="<!-- vigiles-action -->"
139
+ if [[ "${VIGILES_COMMENT:-true}" == "true" \
140
+ && "${GITHUB_EVENT_NAME:-}" == "pull_request" \
141
+ && -n "${GH_TOKEN:-}" ]] && command -v gh >/dev/null 2>&1; then
142
+ pr="$(jq -r '.pull_request.number // empty' "${GITHUB_EVENT_PATH:-/dev/null}")"
143
+ if [[ -n "$pr" ]]; then
144
+ body="$(printf '%s\n%s\n' "$marker" "$(cat "$report")")"
145
+ repo="${GITHUB_REPOSITORY}"
146
+ cid="$(gh api "repos/${repo}/issues/${pr}/comments" --paginate \
147
+ --jq "map(select(.body | startswith(\"${marker}\"))) | .[0].id // empty" \
148
+ 2>/dev/null || true)"
149
+ if [[ -n "$cid" ]]; then
150
+ gh api -X PATCH "repos/${repo}/issues/comments/${cid}" -f body="$body" >/dev/null \
151
+ && echo "Updated PR comment #${cid}" \
152
+ || echo "::warning::could not update PR comment (need pull-requests: write?)"
153
+ else
154
+ gh api -X POST "repos/${repo}/issues/${pr}/comments" -f body="$body" >/dev/null \
155
+ && echo "Posted PR comment" \
156
+ || echo "::warning::could not post PR comment (need pull-requests: write?)"
157
+ fi
158
+ fi
159
+ fi
160
+
161
+ exit "$status"
@@ -1,4 +1,4 @@
1
- import { type RuntimeGate } from "./skill-runtime.js";
1
+ import { type RuntimeGate } from "./adapters/claude-code/skill-runtime.js";
2
2
  export interface ActionGate {
3
3
  /** Tool name to gate, e.g. "Write" | "Edit" | "Bash". */
4
4
  readonly on: string;
@@ -21,7 +21,7 @@ exports.loadActionGates = loadActionGates;
21
21
  */
22
22
  const node_fs_1 = require("node:fs");
23
23
  const node_path_1 = require("node:path");
24
- const skill_runtime_js_1 = require("./skill-runtime.js");
24
+ const skill_runtime_js_1 = require("./adapters/claude-code/skill-runtime.js");
25
25
  /** The file path an action touched, for `{file}` substitution. */
26
26
  function fileOf(event) {
27
27
  const i = event.input ?? {};
@@ -0,0 +1,30 @@
1
+ import type { HarnessAdapter } from "./core/adapter.js";
2
+ export interface ConformanceResult {
3
+ readonly ok: boolean;
4
+ readonly failures: readonly string[];
5
+ }
6
+ /** Check an adapter against the port contracts; returns the (possibly empty) failure list. */
7
+ export declare function checkAdapterConformance(adapter: HarnessAdapter): ConformanceResult;
8
+ /** Throw if the adapter fails conformance — drop this in an adapter's test suite. */
9
+ export declare function assertAdapterConformance(adapter: HarnessAdapter): void;
10
+ /**
11
+ * Guard for the pillar-2 entry points (runHarnessTest/runEval): a pillar-1-only
12
+ * adapter (Cursor, Devin, Amp, Amazon Q) has no mockable transport, so driving
13
+ * the deterministic/eval tiers against it would hang or spawn nothing. Calling
14
+ * this up front turns that into a clear, immediate error. Returns the narrowed
15
+ * runtime+modelMock so the caller can use them without re-checking for undefined.
16
+ */
17
+ export declare function assertHarnessTestable(adapter: HarnessAdapter): {
18
+ runtime: NonNullable<HarnessAdapter["runtime"]>;
19
+ modelMock: NonNullable<HarnessAdapter["modelMock"]>;
20
+ };
21
+ /**
22
+ * Behavioural conformance the pure checks can't reach: write a minimal settings
23
+ * file in the adapter's declared `settingsFormat` (with a hook), load it through
24
+ * the adapter's `layout`, and assert the hooks actually came back. This is what
25
+ * catches a layout that points at the right file but in the wrong format (the
26
+ * JSON-vs-TOML trap) — the pure checker would pass it, the agent would silently
27
+ * run with zero hooks. Does filesystem IO, so it's a separate opt-in assert.
28
+ */
29
+ export declare function assertAdapterLoadsHooks(adapter: HarnessAdapter): void;
30
+ //# sourceMappingURL=adapter-conformance.d.ts.map