vigiles 2.5.0 → 3.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (209) hide show
  1. package/.claude-plugin/plugin.json +9 -0
  2. package/README.md +73 -434
  3. package/action.yml +143 -7
  4. package/dist/action-gate.d.ts +1 -1
  5. package/dist/action-gate.js +1 -1
  6. package/dist/adapter-conformance.d.ts +30 -0
  7. package/dist/adapter-conformance.js +153 -0
  8. package/dist/adapter-registry.d.ts +42 -0
  9. package/dist/adapter-registry.js +55 -0
  10. package/dist/adapter.d.ts +26 -0
  11. package/dist/adapter.js +16 -0
  12. package/dist/adapters/claude-code/adapter.d.ts +3 -0
  13. package/dist/adapters/claude-code/adapter.js +46 -0
  14. package/dist/{agent-result.d.ts → adapters/claude-code/agent-result.d.ts} +1 -1
  15. package/dist/adapters/claude-code/dialect.d.ts +13 -0
  16. package/dist/adapters/claude-code/dialect.js +51 -0
  17. package/dist/adapters/claude-code/egress-entry.d.ts +2 -0
  18. package/dist/adapters/claude-code/egress-entry.js +115 -0
  19. package/dist/adapters/claude-code/egress-proxy.d.ts +2 -0
  20. package/dist/adapters/claude-code/egress-proxy.js +60 -0
  21. package/dist/adapters/claude-code/egress.d.ts +114 -0
  22. package/dist/adapters/claude-code/egress.js +276 -0
  23. package/dist/adapters/claude-code/eval-baseline.d.ts +68 -0
  24. package/dist/adapters/claude-code/eval-baseline.js +173 -0
  25. package/dist/{eval-cache.d.ts → adapters/claude-code/eval-cache.d.ts} +1 -1
  26. package/dist/{eval-cache.js → adapters/claude-code/eval-cache.js} +1 -1
  27. package/dist/{eval.d.ts → adapters/claude-code/eval.d.ts} +28 -4
  28. package/dist/{eval.js → adapters/claude-code/eval.js} +61 -25
  29. package/dist/{harness-test.d.ts → adapters/claude-code/harness-test.d.ts} +33 -37
  30. package/dist/{harness-test.js → adapters/claude-code/harness-test.js} +124 -43
  31. package/dist/adapters/claude-code/hook-protocol.d.ts +10 -0
  32. package/dist/adapters/claude-code/hook-protocol.js +10 -0
  33. package/dist/adapters/claude-code/layout.d.ts +8 -0
  34. package/dist/adapters/claude-code/layout.js +18 -0
  35. package/dist/{mock-model.d.ts → adapters/claude-code/mock-model.d.ts} +2 -24
  36. package/dist/adapters/claude-code/model-mock.d.ts +11 -0
  37. package/dist/adapters/claude-code/model-mock.js +10 -0
  38. package/dist/adapters/claude-code/plugin-loader.d.ts +25 -0
  39. package/dist/adapters/claude-code/plugin-loader.js +19 -0
  40. package/dist/adapters/claude-code/run-hook.d.ts +204 -0
  41. package/dist/adapters/claude-code/run-hook.js +418 -0
  42. package/dist/adapters/claude-code/run-scripts.d.ts +52 -0
  43. package/dist/adapters/claude-code/run-scripts.js +150 -0
  44. package/dist/adapters/claude-code/runtime.d.ts +16 -0
  45. package/dist/adapters/claude-code/runtime.js +39 -0
  46. package/dist/{sandbox.d.ts → adapters/claude-code/sandbox.d.ts} +45 -4
  47. package/dist/{sandbox.js → adapters/claude-code/sandbox.js} +76 -8
  48. package/dist/{skill-driver.d.ts → adapters/claude-code/skill-driver.d.ts} +1 -1
  49. package/dist/adapters/codex/adapter.d.ts +3 -0
  50. package/dist/adapters/codex/adapter.js +49 -0
  51. package/dist/adapters/codex/dialect.d.ts +10 -0
  52. package/dist/adapters/codex/dialect.js +30 -0
  53. package/dist/adapters/codex/driver.d.ts +20 -0
  54. package/dist/adapters/codex/driver.js +89 -0
  55. package/dist/adapters/codex/hook-protocol.d.ts +10 -0
  56. package/dist/adapters/codex/hook-protocol.js +18 -0
  57. package/dist/adapters/codex/layout.d.ts +16 -0
  58. package/dist/adapters/codex/layout.js +18 -0
  59. package/dist/adapters/codex/mock-model.d.ts +52 -0
  60. package/dist/adapters/codex/mock-model.js +210 -0
  61. package/dist/adapters/codex/model-mock.d.ts +11 -0
  62. package/dist/adapters/codex/model-mock.js +10 -0
  63. package/dist/adapters/codex/runtime.d.ts +38 -0
  64. package/dist/adapters/codex/runtime.js +62 -0
  65. package/dist/adapters/opencode/adapter.d.ts +3 -0
  66. package/dist/adapters/opencode/adapter.js +48 -0
  67. package/dist/adapters/opencode/dialect.d.ts +10 -0
  68. package/dist/adapters/opencode/dialect.js +36 -0
  69. package/dist/adapters/opencode/layout.d.ts +9 -0
  70. package/dist/adapters/opencode/layout.js +25 -0
  71. package/dist/adapters/opencode/model-mock.d.ts +10 -0
  72. package/dist/adapters/opencode/model-mock.js +10 -0
  73. package/dist/adapters/opencode/runtime.d.ts +9 -0
  74. package/dist/adapters/opencode/runtime.js +21 -0
  75. package/dist/claude-code.d.ts +16 -0
  76. package/dist/claude-code.js +32 -0
  77. package/dist/cli-flags.d.ts +22 -0
  78. package/dist/cli-flags.js +38 -0
  79. package/dist/cli.js +406 -206
  80. package/dist/codex.d.ts +20 -0
  81. package/dist/codex.js +36 -0
  82. package/dist/community-skills.d.ts +1 -1
  83. package/dist/community-skills.js +2 -2
  84. package/dist/core/adapter.d.ts +88 -0
  85. package/dist/core/adapter.js +3 -0
  86. package/dist/{compile.d.ts → core/compile.d.ts} +11 -2
  87. package/dist/{compile.js → core/compile.js} +68 -53
  88. package/dist/core/compose.d.ts +79 -0
  89. package/dist/core/compose.js +145 -0
  90. package/dist/core/dialect.d.ts +51 -0
  91. package/dist/core/dialect.js +3 -0
  92. package/dist/core/harness-driver.d.ts +134 -0
  93. package/dist/core/harness-driver.js +3 -0
  94. package/dist/core/hook-protocol.d.ts +28 -0
  95. package/dist/core/hook-protocol.js +3 -0
  96. package/dist/core/layout.d.ts +43 -0
  97. package/dist/core/layout.js +3 -0
  98. package/dist/core/model-mock.d.ts +27 -0
  99. package/dist/core/model-mock.js +3 -0
  100. package/dist/core/refs.d.ts +62 -0
  101. package/dist/{refs.js → core/refs.js} +49 -23
  102. package/dist/core/runtime.d.ts +38 -0
  103. package/dist/core/runtime.js +3 -0
  104. package/dist/{types.d.ts → core/types.d.ts} +29 -0
  105. package/dist/{validate.d.ts → core/validate.d.ts} +1 -1
  106. package/dist/{validate.js → core/validate.js} +41 -4
  107. package/dist/e2e.d.ts +19 -0
  108. package/dist/e2e.js +39 -0
  109. package/dist/harness-assert.d.ts +60 -11
  110. package/dist/harness-assert.js +111 -9
  111. package/dist/integration.d.ts +16 -0
  112. package/dist/integration.js +32 -0
  113. package/dist/leaderboard.d.ts +33 -0
  114. package/dist/leaderboard.js +107 -0
  115. package/dist/linting.d.ts +9 -0
  116. package/dist/linting.js +25 -0
  117. package/dist/plugin-loader.d.ts +6 -3
  118. package/dist/plugin-loader.js +101 -49
  119. package/dist/scan.d.ts +49 -0
  120. package/dist/scan.js +177 -0
  121. package/dist/setup-plan.d.ts +51 -0
  122. package/dist/setup-plan.js +85 -0
  123. package/dist/skill-test.d.ts +1 -1
  124. package/dist/skill-test.js +1 -1
  125. package/dist/test-coverage.d.ts +71 -0
  126. package/dist/test-coverage.js +228 -0
  127. package/dist/testing.d.ts +12 -0
  128. package/dist/testing.js +28 -0
  129. package/dist/unit.d.ts +17 -0
  130. package/dist/unit.js +36 -0
  131. package/hooks/refs-nudge.sh +24 -0
  132. package/package.json +35 -16
  133. package/skills/audit-feedback-loop/SKILL.md +76 -0
  134. package/skills/edit-spec/SKILL.md +131 -0
  135. package/skills/enforce-rules-format/SKILL.md +71 -0
  136. package/skills/generate-logo/SKILL.md +103 -0
  137. package/skills/generate-rule/SKILL.md +64 -0
  138. package/skills/linter-docs/clippy.md +241 -0
  139. package/skills/linter-docs/eslint.md +384 -0
  140. package/skills/linter-docs/pylint.md +288 -0
  141. package/skills/linter-docs/rubocop.md +277 -0
  142. package/skills/linter-docs/ruff.md +187 -0
  143. package/skills/linter-docs/stylelint.md +247 -0
  144. package/skills/migrate-to-spec/SKILL.md +126 -0
  145. package/skills/pr-to-lint-rule/SKILL.md +97 -0
  146. package/skills/strengthen/SKILL.md +168 -0
  147. package/skills/test-harness/SKILL.md +157 -0
  148. package/dist/action.d.ts +0 -7
  149. package/dist/action.js +0 -180
  150. package/dist/refs.d.ts +0 -44
  151. package/dist/run-hook.d.ts +0 -77
  152. package/dist/run-hook.js +0 -80
  153. package/dist/run-scripts.d.ts +0 -20
  154. package/dist/run-scripts.js +0 -70
  155. /package/dist/{agent-result.js → adapters/claude-code/agent-result.js} +0 -0
  156. /package/dist/{agent-runtime.d.ts → adapters/claude-code/agent-runtime.d.ts} +0 -0
  157. /package/dist/{agent-runtime.js → adapters/claude-code/agent-runtime.js} +0 -0
  158. /package/dist/{judge.d.ts → adapters/claude-code/judge.d.ts} +0 -0
  159. /package/dist/{judge.js → adapters/claude-code/judge.js} +0 -0
  160. /package/dist/{mock-entry.d.ts → adapters/claude-code/mock-entry.d.ts} +0 -0
  161. /package/dist/{mock-entry.js → adapters/claude-code/mock-entry.js} +0 -0
  162. /package/dist/{mock-model.js → adapters/claude-code/mock-model.js} +0 -0
  163. /package/dist/{skill-driver.js → adapters/claude-code/skill-driver.js} +0 -0
  164. /package/dist/{skill-runtime.d.ts → adapters/claude-code/skill-runtime.d.ts} +0 -0
  165. /package/dist/{skill-runtime.js → adapters/claude-code/skill-runtime.js} +0 -0
  166. /package/dist/{stats.d.ts → adapters/claude-code/stats.d.ts} +0 -0
  167. /package/dist/{stats.js → adapters/claude-code/stats.js} +0 -0
  168. /package/dist/{compile-generator.d.ts → core/compile-generator.d.ts} +0 -0
  169. /package/dist/{compile-generator.js → core/compile-generator.js} +0 -0
  170. /package/dist/{coverage.d.ts → core/coverage.d.ts} +0 -0
  171. /package/dist/{coverage.js → core/coverage.js} +0 -0
  172. /package/dist/{doc-refs.d.ts → core/doc-refs.d.ts} +0 -0
  173. /package/dist/{doc-refs.js → core/doc-refs.js} +0 -0
  174. /package/dist/{evolve.d.ts → core/evolve.d.ts} +0 -0
  175. /package/dist/{evolve.js → core/evolve.js} +0 -0
  176. /package/dist/{frontmatter.d.ts → core/frontmatter.d.ts} +0 -0
  177. /package/dist/{frontmatter.js → core/frontmatter.js} +0 -0
  178. /package/dist/{generate-schema.d.ts → core/generate-schema.d.ts} +0 -0
  179. /package/dist/{generate-schema.js → core/generate-schema.js} +0 -0
  180. /package/dist/{generate-types.d.ts → core/generate-types.d.ts} +0 -0
  181. /package/dist/{generate-types.js → core/generate-types.js} +0 -0
  182. /package/dist/{hash.d.ts → core/hash.d.ts} +0 -0
  183. /package/dist/{hash.js → core/hash.js} +0 -0
  184. /package/dist/{inline.d.ts → core/inline.d.ts} +0 -0
  185. /package/dist/{inline.js → core/inline.js} +0 -0
  186. /package/dist/{integrity.d.ts → core/integrity.d.ts} +0 -0
  187. /package/dist/{integrity.js → core/integrity.js} +0 -0
  188. /package/dist/{linters.d.ts → core/linters.d.ts} +0 -0
  189. /package/dist/{linters.js → core/linters.js} +0 -0
  190. /package/dist/{mcp.d.ts → core/mcp.d.ts} +0 -0
  191. /package/dist/{mcp.js → core/mcp.js} +0 -0
  192. /package/dist/{orphans.d.ts → core/orphans.d.ts} +0 -0
  193. /package/dist/{orphans.js → core/orphans.js} +0 -0
  194. /package/dist/{proofs.d.ts → core/proofs.d.ts} +0 -0
  195. /package/dist/{proofs.js → core/proofs.js} +0 -0
  196. /package/dist/{session.d.ts → core/session.d.ts} +0 -0
  197. /package/dist/{session.js → core/session.js} +0 -0
  198. /package/dist/{sidecar.d.ts → core/sidecar.d.ts} +0 -0
  199. /package/dist/{sidecar.js → core/sidecar.js} +0 -0
  200. /package/dist/{spec.d.ts → core/spec.d.ts} +0 -0
  201. /package/dist/{spec.js → core/spec.js} +0 -0
  202. /package/dist/{symbols.d.ts → core/symbols.d.ts} +0 -0
  203. /package/dist/{symbols.js → core/symbols.js} +0 -0
  204. /package/dist/{test-utils.d.ts → core/test-utils.d.ts} +0 -0
  205. /package/dist/{test-utils.js → core/test-utils.js} +0 -0
  206. /package/dist/{types.js → core/types.js} +0 -0
  207. /package/{.claude-plugin/hooks → hooks}/post-edit.sh +0 -0
  208. /package/{.claude-plugin/hooks → hooks}/pre-edit.sh +0 -0
  209. /package/{.claude-plugin/hooks → hooks}/session-start.sh +0 -0
@@ -0,0 +1,168 @@
1
+ ---
2
+ name: strengthen
3
+ description: Upgrade guidance() rules to enforce() by finding existing linter rules that match
4
+ disable-model-invocation: true
5
+ ---
6
+
7
+ Scan spec files for `guidance()` rules and suggest `enforce()` replacements backed by real linter rules.
8
+
9
+ ## Instructions
10
+
11
+ ### Step 0: Choose Mode
12
+
13
+ Ask the user:
14
+
15
+ > **Auto or interactive?**
16
+ >
17
+ > - **Auto** — I'll apply all safe changes (direct replacements where the rule is already enabled), commit, and show you the diff. Risky changes (require config edits or plugin installs) go in a summary for you to review.
18
+ > - **Interactive** — I'll present each suggestion and you pick which ones to apply.
19
+
20
+ Default to interactive if the user doesn't specify.
21
+
22
+ ### Step 1: Discover What's Installed
23
+
24
+ Run `npx vigiles generate-types` to get the full list of enabled linter rules in the project. Read `.vigiles/generated.d.ts` to see every rule available across all detected linters.
25
+
26
+ Note which linter prefixes appear in the generated types (e.g., `EslintRule`, `RuffRule`). You'll only need reference docs for detected linters.
27
+
28
+ ### Step 2: Find All Guidance Rules
29
+
30
+ Find all `.spec.ts` files in the project (`**/*.md.spec.ts`). For each file, identify every `guidance()` rule.
31
+
32
+ ### Step 3: Match Against Generated Types (Fast Path)
33
+
34
+ For each guidance rule, check if an enabled rule in `.vigiles/generated.d.ts` directly matches. This is the fast, deterministic path — no doc reading needed.
35
+
36
+ Look for:
37
+
38
+ - **Exact rule name in text** — guidance says "no-console" and `no-console` is in EslintRule
39
+ - **Semantic match** — guidance says "don't use console.log" and `no-console` is available
40
+ - **Rule description match** — guidance says "unused variables" and `no-unused-vars` or `@typescript-eslint/no-unused-vars` is available
41
+
42
+ If a match is found and the rule is in the generated types (meaning it's already enabled), this is a **direct replacement** — no config changes needed.
43
+
44
+ ### Step 4: Read Linter Docs (Slow Path)
45
+
46
+ For guidance rules that didn't match in Step 3, read the linter reference docs for the project's detected linters:
47
+
48
+ - ESLint → `../linter-docs/eslint.md`
49
+ - Stylelint → `../linter-docs/stylelint.md`
50
+ - Ruff → `../linter-docs/ruff.md`
51
+ - Pylint → `../linter-docs/pylint.md`
52
+ - RuboCop → `../linter-docs/rubocop.md`
53
+ - Clippy → `../linter-docs/clippy.md`
54
+
55
+ **Only read docs for linters the project actually uses.** Skip docs for linters with no rules in generated types.
56
+
57
+ Check the plugin tables and decision matrices. The guidance text may describe a pattern covered by:
58
+
59
+ - A plugin rule that's installed but not enabled
60
+ - A plugin that's not installed yet
61
+ - A `no-restricted-*` config pattern (see Step 4b)
62
+
63
+ ### Step 4b: `no-restricted-*` Patterns
64
+
65
+ Many guidance rules can be enforced via built-in linter config without a custom rule. This is the most common strengthen pattern — "don't do X" maps to a restriction config.
66
+
67
+ **ESLint:**
68
+
69
+ ```js
70
+ // "Don't import from internal modules"
71
+ "no-restricted-imports": ["error", {
72
+ patterns: [{ group: ["src/internal/*"], message: "Use the public API." }]
73
+ }],
74
+
75
+ // "Don't call console.log"
76
+ "no-restricted-syntax": ["error", {
77
+ selector: 'CallExpression[callee.object.name="console"]',
78
+ message: "Use the project logger."
79
+ }],
80
+
81
+ // "Don't use moment.js"
82
+ "no-restricted-imports": ["error", {
83
+ paths: [{ name: "moment", message: "Use dayjs instead." }]
84
+ }],
85
+ ```
86
+
87
+ **Ruff:**
88
+
89
+ ```toml
90
+ # "Don't use os.system"
91
+ [tool.ruff.lint.flake8-tidy-imports.banned-api]
92
+ "os.system".msg = "Use subprocess.run instead."
93
+ ```
94
+
95
+ **RuboCop:**
96
+
97
+ ```yaml
98
+ # "Don't use puts in production" — if Rails/Output doesn't fit
99
+ Custom/NoPuts:
100
+ Enabled: true
101
+ ```
102
+
103
+ When suggesting a `no-restricted-*` change:
104
+
105
+ 1. Show the exact config edit needed (which file, which section)
106
+ 2. Show the `enforce()` rule that references it
107
+ 3. Note that this changes linter config, not just the spec
108
+
109
+ ### Step 5: Present Suggestions
110
+
111
+ Group the output into tiers:
112
+
113
+ **Tier 1: Direct replacements** (rule already enabled — zero risk)
114
+
115
+ ```typescript
116
+ // Before
117
+ "no-console": guidance("Use structured logger instead of console.log"),
118
+ // After
119
+ "no-console": enforce("eslint/no-console", "Use structured logger instead of console.log"),
120
+ ```
121
+
122
+ **Tier 2: Config-backed** (rule exists but needs config options)
123
+
124
+ ```typescript
125
+ // Spec change:
126
+ "no-moment": enforce("eslint/no-restricted-imports", "Use dayjs instead of moment."),
127
+
128
+ // Config change needed (eslint.config.mjs):
129
+ "no-restricted-imports": ["error", {
130
+ paths: [{ name: "moment", message: "Use dayjs instead." }]
131
+ }],
132
+ ```
133
+
134
+ **Tier 3: Plugin install needed**
135
+
136
+ ```
137
+ "cognitive-complexity": guidance("Keep functions simple")
138
+ → Install eslint-plugin-sonarjs, enable sonarjs/cognitive-complexity
139
+ → enforce("eslint/sonarjs/cognitive-complexity", "Keep functions simple")
140
+ ```
141
+
142
+ **Tier 4: No match** (stays as guidance, or candidate for `/pr-to-lint-rule`)
143
+
144
+ ```
145
+ "research-first": guidance("Google unfamiliar APIs first.")
146
+ → No linter rule can enforce this. Stays as guidance.
147
+ → Want me to run /pr-to-lint-rule to create a custom rule?
148
+ ```
149
+
150
+ ### Step 6: Apply Changes
151
+
152
+ **In auto mode:**
153
+
154
+ 1. Apply all Tier 1 changes (edit spec files, replace `guidance()` with `enforce()`)
155
+ 2. Run `npm run build && npx vigiles compile` to verify each change compiles
156
+ 3. If any compilation fails, revert that specific change and report the error
157
+ 4. Commit all successful changes
158
+ 5. Present Tier 2-4 as a summary for the user to review
159
+
160
+ **In interactive mode:**
161
+
162
+ 1. Present all tiers
163
+ 2. Ask the user which suggestions to apply
164
+ 3. For approved Tier 2 changes: edit the linter config, then edit the spec
165
+ 4. Run `npm run build && npx vigiles compile` to verify
166
+ 5. If compilation fails, report the error and revert
167
+
168
+ **For Tier 4 (no match):** Ask the user if they want to run `/pr-to-lint-rule` for any of the unmatched rules to create custom rules.
@@ -0,0 +1,157 @@
1
+ ---
2
+ name: test-harness
3
+ description: Install vigiles and test a Claude Code harness — hooks, skills, settings, CLAUDE.md — by picking the right tier (unit / deterministic / eval) and writing a test that passes. Use when the user wants to check that a hook fires or blocks, that a skill triggers, that injected context lands, or that a harness change moves what the agent does.
4
+ ---
5
+
6
+ Test the Claude Code **harness** — the hooks, skills, settings, and CLAUDE.md that
7
+ steer an agent — as the assembled machine it ships as. vigiles gives three tiers,
8
+ cheapest first; this skill picks the right one, writes the test, and runs it.
9
+
10
+ The guiding rule: **start at the cheapest tier that can answer the question, and
11
+ climb only when it genuinely can't.** Two of the three tiers need no model and no
12
+ API key, so they run on every commit for free — reach for the paid real-model
13
+ tier only when the question actually requires a real model.
14
+
15
+ ## Step 0 — Pick the tier (the judgment call)
16
+
17
+ Match what you're testing to the cheapest tier that can answer it:
18
+
19
+ | What you're testing | Tier | Cost | API |
20
+ | -------------------------------------------------------------------------------------------------------------------------------------- | ----------------- | ------------------------------------------------ | --------------------------------------------------------------------------------------------- |
21
+ | "Does this hook block/allow event X?" — pure hook logic, **every** event type (incl. Edit/Write, PreCompact, SessionEnd, SubagentStop) | **Unit** | free, milliseconds, no `claude` | `runHook` |
22
+ | "Is the hook actually **wired into** the assembled plugin and does it fire in a real session?" | **Deterministic** | free, no API key (real `claude` + scripted mock) | `runHarnessTest` + `scriptModel` |
23
+ | "Did the injected context (a SessionStart hook, a `/command`) actually **reach the model**?" | **Deterministic** | free, no API key | `runHarnessTest` → `trace.modelRequests` / `assertRequestContains` |
24
+ | "Does this skill's **description trigger** when it should (recall) **and stay quiet** when it shouldn't (precision)?" | **Eval** | **paid** (real model) | `measureTriggerRate` (+ `irrelevantPrompts`) → `assertTriggerRate({ min, maxFalsePositive })` |
25
+ | "Does this harness change **move what the agent does**?" (A/B, signal vs noise) | **Eval** | **paid** (real model) | `runEval` + `assertSignificant` |
26
+
27
+ Most harness questions — block/allow, wired-in, context-landed — never need a
28
+ model. Only "does the model trigger / behave differently" needs the eval tier.
29
+
30
+ If the unit and deterministic tiers can both answer it, **prefer unit**: it's
31
+ faster and reaches events the deterministic mock can't drive.
32
+
33
+ ## Step 1 — Ensure vigiles is installed
34
+
35
+ Check whether `vigiles` is a dependency (`package.json`), and install it as a
36
+ dev dependency if not:
37
+
38
+ ```bash
39
+ npm i -D vigiles # or: pnpm add -D vigiles / yarn add -D vigiles
40
+ ```
41
+
42
+ The deterministic tier additionally needs the `claude` CLI on PATH (no API key):
43
+ `npm i -g @anthropic-ai/claude-code`. The eval tier needs model auth. If the
44
+ `claude` CLI is missing, you can still write and run **unit**-tier tests.
45
+
46
+ ## Step 2 — Locate the harness surface to test
47
+
48
+ Find what the project actually ships, in this order:
49
+
50
+ 1. `.claude/settings.json` / `.claude/settings.local.json` — inline `hooks`.
51
+ 2. `.claude-plugin/plugin.json` — a plugin manifest (`hooks`, `skills`, `agents`, `mcpServers`).
52
+ 3. `hooks/hooks.json` — the plugin hooks convention (e.g. obra/superpowers).
53
+ 4. `skills/<name>/SKILL.md`, `agents/<name>.md`, `commands/<name>.md`.
54
+
55
+ Pick one concrete thing to pin down — a specific `PreToolUse` hook, a specific
56
+ `SessionStart` injection, a specific skill.
57
+
58
+ ## Step 3 — Write the test for the chosen tier
59
+
60
+ **Unit (`runHook`)** — hand a hook a synthesized event, assert the decision:
61
+
62
+ ```ts
63
+ import { runHook, assertHookBlocked } from "vigiles/testing";
64
+
65
+ const r = runHook(hookCommand, {
66
+ hook_event_name: "PreToolUse",
67
+ tool_name: "Bash",
68
+ tool_input: { command: "git commit --no-verify" },
69
+ });
70
+ assertHookBlocked(r); // exit 2 / decision:"block" / permissionDecision:"deny"
71
+ ```
72
+
73
+ Testing a hook you didn't write (a vendored third-party script)? Mark it
74
+ `{ trusted: false }` and it runs confined under bubblewrap by default (read-only
75
+ host, cleared env, no network egress). Add `{ recordEgress: true }` to also
76
+ **record** what it tries to reach — `r.egress` plus `assertNoEgress(r)` /
77
+ `assertEgressOnly(r, [...])` — the supply-chain check for "what does this skill
78
+ phone home to / install from?". When the hook's setup needs a _real_ install,
79
+ `{ egress: { allow: ["registry.npmjs.org"] } }` lets it reach only that
80
+ allowlist (a packet-layer `nft` wall, so a raw socket off-list is dropped too) →
81
+ `r.egress` (allowed hosts) + `r.egressDropped`. Be precise about the boundaries:
82
+ see
83
+ [`docs/sandboxing.md`](../../docs/sandboxing.md) (it blocks destruction and
84
+ egress, but does NOT isolate reads of host files, and only under bwrap).
85
+
86
+ **Deterministic (`runHarnessTest`)** — load the real plugin, drive a scripted
87
+ mock model, assert the hook fired (or the context landed):
88
+
89
+ ```ts
90
+ import {
91
+ runHarnessTest,
92
+ scriptModel,
93
+ assertHookFired,
94
+ assertRequestContains,
95
+ } from "vigiles/testing";
96
+
97
+ const r = await runHarnessTest({
98
+ pluginDir: "./", // or { settings: { hooks: {...} } }
99
+ transcript: true,
100
+ model: scriptModel([{ text: "ok" }]),
101
+ });
102
+ assertHookFired(r, "SessionStart");
103
+ assertRequestContains(r, "expected injected text"); // did it actually land?
104
+ ```
105
+
106
+ **Eval (`runEval`)** — A/B the change on vs off across real-model trials, then
107
+ gate on significance, not eyeballing:
108
+
109
+ ```ts
110
+ import { runEval, assertSignificant } from "vigiles/testing";
111
+
112
+ const report = await runEval({
113
+ arms: { off: {}, on: { pluginDir: "./" } },
114
+ task: "…a task the harness change should affect…",
115
+ measure: (ctx) => ({ ok: /* a bare predicate over the trace */ true }),
116
+ trials: 6,
117
+ cache: "readwrite",
118
+ });
119
+ assertSignificant(report, { baseline: "off", arm: "on", metric: "ok" });
120
+ ```
121
+
122
+ ## Step 4 — Run it
123
+
124
+ In a runner (node:test / vitest / jest) the tests are plain async functions. Or
125
+ use the zero-setup CLI, which discovers and runs the files:
126
+
127
+ ```bash
128
+ npx vigiles test # *.harness.{mjs,ts} — unit + deterministic, no API key
129
+ npx vigiles eval --trials=6 # *.eval.{mjs,ts} — real model (local / nightly, not CI)
130
+ ```
131
+
132
+ Unit-tier `runHook` tests need no `claude` and **always run** — write and run them
133
+ even with no `claude` installed. A tier that genuinely can't run reports a loud
134
+ `⊘ SKIPPED` (tallied separately, never a fake `✓`); a standalone script emits one
135
+ via `skip(reason)` from `vigiles/testing`. A skip passes by default, but in a CI
136
+ job that asserts the capability is present, run **`vigiles test --no-skip`** so a
137
+ skipped tier fails — a green-with-skips is untested surface. Keep unit +
138
+ deterministic tests in CI (free); run evals locally or on a schedule with auth.
139
+
140
+ ## When the user didn't say what to test
141
+
142
+ Don't ask them to specify — **pick something real and demonstrate.** Scan the
143
+ harness surface (Step 2), choose the cheapest meaningful test, write it, run it,
144
+ and show the result. Good default picks, in order:
145
+
146
+ 1. A `PreToolUse` hook → **unit-test** that it blocks the thing it's meant to block (and allows a safe sibling).
147
+ 2. A `SessionStart` hook that injects context → **deterministic** test that the text actually reaches the model (`assertRequestContains`).
148
+ 3. A skill → **deterministic** test that it resolves via `pluginDir`, then offer the paid `measureTriggerRate` eval as a follow-up.
149
+
150
+ Then say which tier you used and why, and offer to climb a tier if the cheaper
151
+ test can't fully answer their question.
152
+
153
+ ## Reference
154
+
155
+ The full guide — every tier, testing skills for real, "fired ≠ landed", the
156
+ safe-by-default sandbox, the coverage matrix, and how it compares to promptfoo —
157
+ is in [`docs/harness-testing.md`](../../docs/harness-testing.md).
package/dist/action.d.ts DELETED
@@ -1,7 +0,0 @@
1
- /**
2
- * GitHub Action entry point for vigiles.
3
- *
4
- * Runs `compile` or `audit` depending on the action input.
5
- */
6
- export {};
7
- //# sourceMappingURL=action.d.ts.map
package/dist/action.js DELETED
@@ -1,180 +0,0 @@
1
- "use strict";
2
- /**
3
- * GitHub Action entry point for vigiles.
4
- *
5
- * Runs `compile` or `audit` depending on the action input.
6
- */
7
- Object.defineProperty(exports, "__esModule", { value: true });
8
- const node_fs_1 = require("node:fs");
9
- const node_path_1 = require("node:path");
10
- const node_child_process_1 = require("node:child_process");
11
- const glob_1 = require("glob");
12
- const compile_js_1 = require("./compile.js");
13
- // ---------------------------------------------------------------------------
14
- // Read action inputs
15
- // ---------------------------------------------------------------------------
16
- const command = process.env["INPUT_COMMAND"] ?? "audit";
17
- const pathsInput = process.env["INPUT_PATHS"];
18
- const maxRulesInput = process.env["INPUT_MAX-RULES"] ?? process.env["INPUT_MAX_RULES"];
19
- const catalogOnly = (process.env["INPUT_CATALOG-ONLY"] ??
20
- process.env["INPUT_CATALOG_ONLY"] ??
21
- "false") === "true";
22
- const maxRules = maxRulesInput ? Number(maxRulesInput) : undefined;
23
- // ---------------------------------------------------------------------------
24
- // Spec loading (from compiled dist/)
25
- // ---------------------------------------------------------------------------
26
- async function loadSpec(specPath) {
27
- // Try several resolution strategies:
28
- // 1. Replace /src/ with /dist/ (standard TS project layout)
29
- // 2. Look under dist/ directly (root-level specs like CLAUDE.md.spec.ts)
30
- // 3. Use the .js extension in-place (pre-compiled specs)
31
- const jsName = specPath.replace(/\.ts$/, ".js");
32
- const candidates = [
33
- (0, node_path_1.resolve)(process.cwd(), jsName.replace(/\/src\//, "/dist/")),
34
- (0, node_path_1.resolve)(process.cwd(), "dist", jsName),
35
- (0, node_path_1.resolve)(process.cwd(), jsName),
36
- ];
37
- // Deduplicate (candidate 1 and 3 may match when there's no /src/)
38
- const unique = [...new Set(candidates)];
39
- for (const distPath of unique) {
40
- if (!(0, node_fs_1.existsSync)(distPath))
41
- continue;
42
- try {
43
- const mod = (await import(distPath));
44
- // CJS modules (TypeScript with module: "Node16", no "type":
45
- // "module") produce `exports["default"] = spec`, so dynamic
46
- // import wraps it as `{ default: { default: spec } }`. Unwrap
47
- // the double-default when present.
48
- const raw = mod.default;
49
- if (raw && typeof raw === "object" && "default" in raw) {
50
- return raw.default;
51
- }
52
- return raw;
53
- }
54
- catch {
55
- continue;
56
- }
57
- }
58
- return null;
59
- }
60
- // ---------------------------------------------------------------------------
61
- // Commands
62
- // ---------------------------------------------------------------------------
63
- async function runCompile() {
64
- const specs = pathsInput
65
- ? pathsInput
66
- .split(",")
67
- .map((p) => p.trim())
68
- .filter(Boolean)
69
- : (0, glob_1.globSync)("**/*.spec.ts", { ignore: ["node_modules/**", "dist/**"] });
70
- if (specs.length === 0) {
71
- console.log("No .spec.ts files found.");
72
- return true;
73
- }
74
- let allValid = true;
75
- const basePath = process.cwd();
76
- for (const specPath of specs) {
77
- const spec = await loadSpec(specPath);
78
- if (!spec) {
79
- console.log(`::error file=${specPath}::Failed to load spec`);
80
- allValid = false;
81
- continue;
82
- }
83
- if (spec._specType === "claude") {
84
- const primaryOutput = specPath.replace(/\.spec\.ts$/, "");
85
- const { markdown, errors, targets } = (0, compile_js_1.compileClaude)(spec, {
86
- basePath,
87
- specFile: specPath,
88
- maxRules,
89
- catalogOnly,
90
- });
91
- if (errors.length > 0) {
92
- for (const err of errors) {
93
- console.log(`::error file=${specPath}::${err.message}`);
94
- }
95
- allValid = false;
96
- }
97
- // Write primary target
98
- (0, node_fs_1.writeFileSync)((0, node_path_1.resolve)(basePath, primaryOutput), markdown);
99
- const outputNames = [primaryOutput];
100
- // Write additional targets with swapped heading + recomputed hash
101
- for (const t of targets.slice(1)) {
102
- const body = markdown
103
- .replace(/^<!-- vigiles:[^\n]+\n\n?/, "")
104
- .replace(/^# [^\n]+/, `# ${t}`);
105
- const additional = (0, compile_js_1.addHash)(body, specPath);
106
- const dir = primaryOutput.substring(0, primaryOutput.lastIndexOf("/") + 1);
107
- const targetPath = dir + t;
108
- (0, node_fs_1.writeFileSync)((0, node_path_1.resolve)(basePath, targetPath), additional);
109
- outputNames.push(targetPath);
110
- }
111
- console.log(`Compiled: ${specPath} → ${outputNames.join(", ")}`);
112
- }
113
- else if (spec._specType === "skill") {
114
- const outputPath = specPath.replace(/\.spec\.ts$/, "");
115
- const { markdown, errors } = (0, compile_js_1.compileSkill)(spec, {
116
- basePath,
117
- specFile: specPath,
118
- });
119
- if (errors.length > 0) {
120
- for (const err of errors) {
121
- console.log(`::error file=${specPath}::${err.message}`);
122
- }
123
- allValid = false;
124
- }
125
- (0, node_fs_1.writeFileSync)((0, node_path_1.resolve)(basePath, outputPath), markdown);
126
- console.log(`Compiled: ${specPath} → ${outputPath}`);
127
- }
128
- }
129
- return allValid;
130
- }
131
- function runAudit() {
132
- // Delegate to the CLI so the action always runs the full audit flow
133
- // (hash verification, spec validation, duplicate detection, coverage
134
- // report, strengthen suggestions). Previously this only ran
135
- // checkFileHash, which silently skipped everything else.
136
- const files = pathsInput
137
- ? pathsInput
138
- .split(",")
139
- .map((p) => p.trim())
140
- .filter(Boolean)
141
- : [];
142
- const cliPath = (0, node_path_1.resolve)(__dirname, "cli.js");
143
- const args = ["audit", ...files];
144
- const result = (0, node_child_process_1.spawnSync)(process.execPath, [cliPath, ...args], {
145
- stdio: "inherit",
146
- env: {
147
- ...process.env,
148
- // Ensure the CLI emits GitHub annotations during the action run
149
- GITHUB_ACTIONS: process.env.GITHUB_ACTIONS ?? "true",
150
- },
151
- });
152
- if (result.error) {
153
- console.log(`::error::Failed to run vigiles audit: ${result.error.message}`);
154
- return false;
155
- }
156
- // Exit codes: 0 clean, 1 warnings, 2 hard errors.
157
- return result.status === 0;
158
- }
159
- // ---------------------------------------------------------------------------
160
- // Main
161
- // ---------------------------------------------------------------------------
162
- void (async () => {
163
- let valid;
164
- if (command === "compile") {
165
- valid = await runCompile();
166
- }
167
- else if (command === "audit") {
168
- valid = runAudit();
169
- }
170
- else {
171
- console.log(`::error::Unknown vigiles command "${command}". Valid commands: compile, audit.`);
172
- process.exit(1);
173
- }
174
- console.log(`::set-output name=valid::${String(valid)}`);
175
- if (!valid) {
176
- console.log("::error::vigiles failed — see errors above");
177
- process.exit(1);
178
- }
179
- })();
180
- //# sourceMappingURL=action.js.map
package/dist/refs.d.ts DELETED
@@ -1,44 +0,0 @@
1
- /** An inline code span with its 1-based source line. */
2
- export interface Span {
3
- readonly text: string;
4
- readonly line: number;
5
- }
6
- /**
7
- * Extract inline code spans, skipping fenced code blocks (R1). Returns each
8
- * span's trimmed text and 1-based line.
9
- */
10
- export declare function inlineSpans(markdown: string): Span[];
11
- /** A parsed file-qualified reference. */
12
- export interface SymbolRef {
13
- readonly file: string;
14
- readonly symbol: string;
15
- readonly line: number;
16
- }
17
- /** A reference that failed verification. */
18
- export interface SymbolRefError extends SymbolRef {
19
- readonly reason: string;
20
- }
21
- /** Extract the `vigiles:symbol` references from a markdown file. */
22
- export declare function symbolRefs(markdown: string): SymbolRef[];
23
- /**
24
- * Verify the file-qualified symbol references in a markdown file: the named
25
- * file must exist and define the named symbol. `basePath` is the directory the
26
- * paths resolve against (the instruction file's own directory).
27
- */
28
- export declare function verifySymbolRefs(markdown: string, basePath: string): SymbolRefError[];
29
- /**
30
- * Whether a span looks like a *code reference* that ought to carry a
31
- * file-qualified mark — a scoped name, or an identifier that isn't a bare
32
- * lowercase prose word. A function-call form `` `foo(args)` `` is treated as a
33
- * reference to its callee `foo`. Paths/filenames are excluded (they are `file`
34
- * refs).
35
- */
36
- export declare function isCodeShaped(text: string): boolean;
37
- /**
38
- * Code-shaped inline references that are NOT yet marked — the spans the
39
- * enforcement hook makes the agent mark as `` `vigiles:symbol path.ext#symbol` ``
40
- * or opt out of with `<!-- vigiles:ignore -->` (or `<!-- vigiles:ignore-file -->`
41
- * for the whole file).
42
- */
43
- export declare function unmarkedCodeRefs(markdown: string): Span[];
44
- //# sourceMappingURL=refs.d.ts.map
@@ -1,77 +0,0 @@
1
- /** A hook event payload (the JSON Claude Code writes to the hook's stdin). */
2
- export interface HookInput {
3
- /** e.g. "PreToolUse", "PostToolUse", "Stop", "SessionStart", "PreCompact". */
4
- readonly hook_event_name?: string;
5
- /** PreToolUse/PostToolUse. */
6
- readonly tool_name?: string;
7
- readonly tool_input?: unknown;
8
- readonly tool_response?: unknown;
9
- /** UserPromptSubmit. */
10
- readonly prompt?: string;
11
- /** SessionStart. */
12
- readonly source?: string;
13
- /** Stop / SubagentStop. */
14
- readonly stop_hook_active?: boolean;
15
- /** Any other event-specific fields. */
16
- readonly [k: string]: unknown;
17
- }
18
- /** The JSON a hook may print on stdout (all fields optional). */
19
- export interface HookOutput {
20
- readonly decision?: "approve" | "block";
21
- readonly reason?: string;
22
- readonly continue?: boolean;
23
- readonly stopReason?: string;
24
- readonly suppressOutput?: boolean;
25
- readonly systemMessage?: string;
26
- readonly hookSpecificOutput?: {
27
- readonly hookEventName?: string;
28
- readonly permissionDecision?: "allow" | "deny" | "ask";
29
- readonly permissionDecisionReason?: string;
30
- readonly additionalContext?: string;
31
- };
32
- readonly [k: string]: unknown;
33
- }
34
- export interface RunHookOptions {
35
- /** Working directory for the hook process. Default: a value won't be set. */
36
- readonly cwd?: string;
37
- /** Extra env vars (merged over process.env). `{cwd}` in values is left as-is. */
38
- readonly env?: Record<string, string>;
39
- /** Per-run timeout ms. Default 10000. */
40
- readonly timeoutMs?: number;
41
- }
42
- export interface HookRunResult {
43
- readonly exitCode: number;
44
- readonly stdout: string;
45
- readonly stderr: string;
46
- /** Parsed stdout JSON if the hook emitted a JSON decision, else null. */
47
- readonly json: HookOutput | null;
48
- /**
49
- * Normalized decision: a deny/block via exit 2, `decision:"block"`, or
50
- * `permissionDecision:"deny"` all set `blocked = true`.
51
- */
52
- readonly blocked: boolean;
53
- /**
54
- * The decision the hook expressed, preferring the structured
55
- * `permissionDecision` ("allow"|"deny"|"ask") then legacy `decision`
56
- * ("approve"|"block"), else undefined.
57
- */
58
- readonly decision: HookOutput["decision"] | "allow" | "deny" | "ask" | undefined;
59
- }
60
- /** Parse stdout as a hook JSON decision (pure, testable without a process). */
61
- export declare function parseHookOutput(stdout: string): HookOutput | null;
62
- /**
63
- * Decide whether a hook result blocked, and the normalized decision. Pure, so
64
- * the policy is unit-testable independent of spawning anything.
65
- */
66
- export declare function decideHook(exitCode: number, json: HookOutput | null): {
67
- blocked: boolean;
68
- decision: HookRunResult["decision"];
69
- };
70
- /**
71
- * Run a hook command, piping `input` as JSON to its stdin, and report the exit
72
- * code + parsed decision. Synchronous (so it can be used inside an eval's
73
- * `measure` too). `command` is run through a shell, so the same command string a
74
- * plugin ships (with args / env refs) works verbatim.
75
- */
76
- export declare function runHook(command: string, input: HookInput, opts?: RunHookOptions): HookRunResult;
77
- //# sourceMappingURL=run-hook.d.ts.map