vigiles 2.6.0 → 3.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude-plugin/plugin.json +9 -0
- package/README.md +74 -129
- package/action.yml +143 -7
- package/dist/action-gate.d.ts +1 -1
- package/dist/action-gate.js +1 -1
- package/dist/adapter-conformance.d.ts +30 -0
- package/dist/adapter-conformance.js +153 -0
- package/dist/adapter-registry.d.ts +42 -0
- package/dist/adapter-registry.js +55 -0
- package/dist/adapter.d.ts +26 -0
- package/dist/adapter.js +16 -0
- package/dist/adapters/claude-code/adapter.d.ts +3 -0
- package/dist/adapters/claude-code/adapter.js +46 -0
- package/dist/{agent-result.d.ts → adapters/claude-code/agent-result.d.ts} +1 -1
- package/dist/adapters/claude-code/dialect.d.ts +13 -0
- package/dist/adapters/claude-code/dialect.js +51 -0
- package/dist/adapters/claude-code/egress-entry.d.ts +2 -0
- package/dist/adapters/claude-code/egress-entry.js +115 -0
- package/dist/adapters/claude-code/egress.d.ts +114 -0
- package/dist/adapters/claude-code/egress.js +276 -0
- package/dist/{eval-cache.d.ts → adapters/claude-code/eval-cache.d.ts} +1 -1
- package/dist/{eval-cache.js → adapters/claude-code/eval-cache.js} +1 -1
- package/dist/{eval.d.ts → adapters/claude-code/eval.d.ts} +28 -4
- package/dist/{eval.js → adapters/claude-code/eval.js} +61 -25
- package/dist/{harness-test.d.ts → adapters/claude-code/harness-test.d.ts} +33 -37
- package/dist/{harness-test.js → adapters/claude-code/harness-test.js} +124 -43
- package/dist/adapters/claude-code/hook-protocol.d.ts +10 -0
- package/dist/adapters/claude-code/hook-protocol.js +10 -0
- package/dist/adapters/claude-code/layout.d.ts +8 -0
- package/dist/adapters/claude-code/layout.js +18 -0
- package/dist/{mock-model.d.ts → adapters/claude-code/mock-model.d.ts} +2 -24
- package/dist/adapters/claude-code/model-mock.d.ts +11 -0
- package/dist/adapters/claude-code/model-mock.js +10 -0
- package/dist/adapters/claude-code/plugin-loader.d.ts +25 -0
- package/dist/adapters/claude-code/plugin-loader.js +19 -0
- package/dist/{run-hook.d.ts → adapters/claude-code/run-hook.d.ts} +50 -3
- package/dist/{run-hook.js → adapters/claude-code/run-hook.js} +178 -18
- package/dist/adapters/claude-code/run-scripts.d.ts +52 -0
- package/dist/adapters/claude-code/run-scripts.js +150 -0
- package/dist/adapters/claude-code/runtime.d.ts +16 -0
- package/dist/adapters/claude-code/runtime.js +39 -0
- package/dist/{sandbox.d.ts → adapters/claude-code/sandbox.d.ts} +10 -0
- package/dist/{sandbox.js → adapters/claude-code/sandbox.js} +5 -3
- package/dist/{skill-driver.d.ts → adapters/claude-code/skill-driver.d.ts} +1 -1
- package/dist/adapters/codex/adapter.d.ts +3 -0
- package/dist/adapters/codex/adapter.js +49 -0
- package/dist/adapters/codex/dialect.d.ts +10 -0
- package/dist/adapters/codex/dialect.js +30 -0
- package/dist/adapters/codex/driver.d.ts +20 -0
- package/dist/adapters/codex/driver.js +89 -0
- package/dist/adapters/codex/hook-protocol.d.ts +10 -0
- package/dist/adapters/codex/hook-protocol.js +18 -0
- package/dist/adapters/codex/layout.d.ts +16 -0
- package/dist/adapters/codex/layout.js +18 -0
- package/dist/adapters/codex/mock-model.d.ts +52 -0
- package/dist/adapters/codex/mock-model.js +210 -0
- package/dist/adapters/codex/model-mock.d.ts +11 -0
- package/dist/adapters/codex/model-mock.js +10 -0
- package/dist/adapters/codex/runtime.d.ts +38 -0
- package/dist/adapters/codex/runtime.js +62 -0
- package/dist/adapters/opencode/adapter.d.ts +3 -0
- package/dist/adapters/opencode/adapter.js +48 -0
- package/dist/adapters/opencode/dialect.d.ts +10 -0
- package/dist/adapters/opencode/dialect.js +36 -0
- package/dist/adapters/opencode/layout.d.ts +9 -0
- package/dist/adapters/opencode/layout.js +25 -0
- package/dist/adapters/opencode/model-mock.d.ts +10 -0
- package/dist/adapters/opencode/model-mock.js +10 -0
- package/dist/adapters/opencode/runtime.d.ts +9 -0
- package/dist/adapters/opencode/runtime.js +21 -0
- package/dist/claude-code.d.ts +8 -2
- package/dist/claude-code.js +8 -2
- package/dist/cli-flags.d.ts +22 -0
- package/dist/cli-flags.js +38 -0
- package/dist/cli.js +406 -206
- package/dist/codex.d.ts +20 -0
- package/dist/codex.js +36 -0
- package/dist/community-skills.d.ts +1 -1
- package/dist/community-skills.js +2 -2
- package/dist/core/adapter.d.ts +88 -0
- package/dist/core/adapter.js +3 -0
- package/dist/{compile.d.ts → core/compile.d.ts} +11 -2
- package/dist/{compile.js → core/compile.js} +68 -53
- package/dist/core/compose.d.ts +79 -0
- package/dist/core/compose.js +145 -0
- package/dist/core/dialect.d.ts +51 -0
- package/dist/core/dialect.js +3 -0
- package/dist/core/harness-driver.d.ts +134 -0
- package/dist/core/harness-driver.js +3 -0
- package/dist/core/hook-protocol.d.ts +28 -0
- package/dist/core/hook-protocol.js +3 -0
- package/dist/core/layout.d.ts +43 -0
- package/dist/core/layout.js +3 -0
- package/dist/core/model-mock.d.ts +27 -0
- package/dist/core/model-mock.js +3 -0
- package/dist/core/refs.d.ts +62 -0
- package/dist/{refs.js → core/refs.js} +49 -23
- package/dist/core/runtime.d.ts +38 -0
- package/dist/core/runtime.js +3 -0
- package/dist/{types.d.ts → core/types.d.ts} +29 -0
- package/dist/{validate.d.ts → core/validate.d.ts} +1 -1
- package/dist/{validate.js → core/validate.js} +41 -4
- package/dist/e2e.d.ts +19 -0
- package/dist/e2e.js +39 -0
- package/dist/harness-assert.d.ts +29 -14
- package/dist/harness-assert.js +37 -10
- package/dist/integration.d.ts +16 -0
- package/dist/integration.js +32 -0
- package/dist/leaderboard.d.ts +33 -0
- package/dist/leaderboard.js +107 -0
- package/dist/linting.d.ts +2 -2
- package/dist/linting.js +2 -2
- package/dist/plugin-loader.d.ts +6 -3
- package/dist/plugin-loader.js +101 -49
- package/dist/scan.d.ts +49 -0
- package/dist/scan.js +177 -0
- package/dist/setup-plan.d.ts +51 -0
- package/dist/setup-plan.js +85 -0
- package/dist/skill-test.d.ts +1 -1
- package/dist/skill-test.js +1 -1
- package/dist/test-coverage.d.ts +71 -0
- package/dist/test-coverage.js +228 -0
- package/dist/testing.d.ts +3 -3
- package/dist/testing.js +3 -3
- package/dist/unit.d.ts +17 -0
- package/dist/unit.js +36 -0
- package/hooks/refs-nudge.sh +24 -0
- package/package.json +30 -15
- package/skills/audit-feedback-loop/SKILL.md +76 -0
- package/skills/edit-spec/SKILL.md +131 -0
- package/skills/enforce-rules-format/SKILL.md +71 -0
- package/skills/generate-logo/SKILL.md +103 -0
- package/skills/generate-rule/SKILL.md +64 -0
- package/skills/linter-docs/clippy.md +241 -0
- package/skills/linter-docs/eslint.md +384 -0
- package/skills/linter-docs/pylint.md +288 -0
- package/skills/linter-docs/rubocop.md +277 -0
- package/skills/linter-docs/ruff.md +187 -0
- package/skills/linter-docs/stylelint.md +247 -0
- package/skills/migrate-to-spec/SKILL.md +126 -0
- package/skills/pr-to-lint-rule/SKILL.md +97 -0
- package/skills/strengthen/SKILL.md +168 -0
- package/skills/test-harness/SKILL.md +157 -0
- package/dist/action.d.ts +0 -7
- package/dist/action.js +0 -180
- package/dist/refs.d.ts +0 -44
- package/dist/run-scripts.d.ts +0 -20
- package/dist/run-scripts.js +0 -70
- /package/dist/{agent-result.js → adapters/claude-code/agent-result.js} +0 -0
- /package/dist/{agent-runtime.d.ts → adapters/claude-code/agent-runtime.d.ts} +0 -0
- /package/dist/{agent-runtime.js → adapters/claude-code/agent-runtime.js} +0 -0
- /package/dist/{egress-proxy.d.ts → adapters/claude-code/egress-proxy.d.ts} +0 -0
- /package/dist/{egress-proxy.js → adapters/claude-code/egress-proxy.js} +0 -0
- /package/dist/{eval-baseline.d.ts → adapters/claude-code/eval-baseline.d.ts} +0 -0
- /package/dist/{eval-baseline.js → adapters/claude-code/eval-baseline.js} +0 -0
- /package/dist/{judge.d.ts → adapters/claude-code/judge.d.ts} +0 -0
- /package/dist/{judge.js → adapters/claude-code/judge.js} +0 -0
- /package/dist/{mock-entry.d.ts → adapters/claude-code/mock-entry.d.ts} +0 -0
- /package/dist/{mock-entry.js → adapters/claude-code/mock-entry.js} +0 -0
- /package/dist/{mock-model.js → adapters/claude-code/mock-model.js} +0 -0
- /package/dist/{skill-driver.js → adapters/claude-code/skill-driver.js} +0 -0
- /package/dist/{skill-runtime.d.ts → adapters/claude-code/skill-runtime.d.ts} +0 -0
- /package/dist/{skill-runtime.js → adapters/claude-code/skill-runtime.js} +0 -0
- /package/dist/{stats.d.ts → adapters/claude-code/stats.d.ts} +0 -0
- /package/dist/{stats.js → adapters/claude-code/stats.js} +0 -0
- /package/dist/{compile-generator.d.ts → core/compile-generator.d.ts} +0 -0
- /package/dist/{compile-generator.js → core/compile-generator.js} +0 -0
- /package/dist/{coverage.d.ts → core/coverage.d.ts} +0 -0
- /package/dist/{coverage.js → core/coverage.js} +0 -0
- /package/dist/{doc-refs.d.ts → core/doc-refs.d.ts} +0 -0
- /package/dist/{doc-refs.js → core/doc-refs.js} +0 -0
- /package/dist/{evolve.d.ts → core/evolve.d.ts} +0 -0
- /package/dist/{evolve.js → core/evolve.js} +0 -0
- /package/dist/{frontmatter.d.ts → core/frontmatter.d.ts} +0 -0
- /package/dist/{frontmatter.js → core/frontmatter.js} +0 -0
- /package/dist/{generate-schema.d.ts → core/generate-schema.d.ts} +0 -0
- /package/dist/{generate-schema.js → core/generate-schema.js} +0 -0
- /package/dist/{generate-types.d.ts → core/generate-types.d.ts} +0 -0
- /package/dist/{generate-types.js → core/generate-types.js} +0 -0
- /package/dist/{hash.d.ts → core/hash.d.ts} +0 -0
- /package/dist/{hash.js → core/hash.js} +0 -0
- /package/dist/{inline.d.ts → core/inline.d.ts} +0 -0
- /package/dist/{inline.js → core/inline.js} +0 -0
- /package/dist/{integrity.d.ts → core/integrity.d.ts} +0 -0
- /package/dist/{integrity.js → core/integrity.js} +0 -0
- /package/dist/{linters.d.ts → core/linters.d.ts} +0 -0
- /package/dist/{linters.js → core/linters.js} +0 -0
- /package/dist/{mcp.d.ts → core/mcp.d.ts} +0 -0
- /package/dist/{mcp.js → core/mcp.js} +0 -0
- /package/dist/{orphans.d.ts → core/orphans.d.ts} +0 -0
- /package/dist/{orphans.js → core/orphans.js} +0 -0
- /package/dist/{proofs.d.ts → core/proofs.d.ts} +0 -0
- /package/dist/{proofs.js → core/proofs.js} +0 -0
- /package/dist/{session.d.ts → core/session.d.ts} +0 -0
- /package/dist/{session.js → core/session.js} +0 -0
- /package/dist/{sidecar.d.ts → core/sidecar.d.ts} +0 -0
- /package/dist/{sidecar.js → core/sidecar.js} +0 -0
- /package/dist/{spec.d.ts → core/spec.d.ts} +0 -0
- /package/dist/{spec.js → core/spec.js} +0 -0
- /package/dist/{symbols.d.ts → core/symbols.d.ts} +0 -0
- /package/dist/{symbols.js → core/symbols.js} +0 -0
- /package/dist/{test-utils.d.ts → core/test-utils.d.ts} +0 -0
- /package/dist/{test-utils.js → core/test-utils.js} +0 -0
- /package/dist/{types.js → core/types.js} +0 -0
- /package/{.claude-plugin/hooks → hooks}/post-edit.sh +0 -0
- /package/{.claude-plugin/hooks → hooks}/pre-edit.sh +0 -0
- /package/{.claude-plugin/hooks → hooks}/session-start.sh +0 -0
|
@@ -30,6 +30,15 @@
|
|
|
30
30
|
"command": "bash ${CLAUDE_PLUGIN_ROOT}/hooks/post-edit.sh"
|
|
31
31
|
}
|
|
32
32
|
]
|
|
33
|
+
},
|
|
34
|
+
{
|
|
35
|
+
"matcher": "Edit|Write",
|
|
36
|
+
"hooks": [
|
|
37
|
+
{
|
|
38
|
+
"type": "command",
|
|
39
|
+
"command": "bash ${CLAUDE_PLUGIN_ROOT}/hooks/refs-nudge.sh"
|
|
40
|
+
}
|
|
41
|
+
]
|
|
33
42
|
}
|
|
34
43
|
],
|
|
35
44
|
"SessionStart": [
|
package/README.md
CHANGED
|
@@ -19,175 +19,120 @@
|
|
|
19
19
|
<a href="https://github.com/zernie/vigiles/blob/main/LICENSE"><img src="https://img.shields.io/github/license/zernie/vigiles" alt="License" /></a>
|
|
20
20
|
</p>
|
|
21
21
|
|
|
22
|
-
<p align="center">
|
|
23
|
-
<b>Two pillars →</b>
|
|
24
|
-
<a href="#verify-your-instruction-files">① Verify your instruction files</a>
|
|
25
|
-
·
|
|
26
|
-
<a href="#test-your-claude-code-harness">② Test your Claude Code harness</a>
|
|
27
|
-
</p>
|
|
28
|
-
|
|
29
22
|
---
|
|
30
23
|
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
- **Pillar 2** — [test your Claude Code harness](#test-your-claude-code-harness) · full guide: [docs/harness-testing.md](docs/harness-testing.md)
|
|
37
|
-
- [Quick start](#quick-start) · [CLI & CI](#cli--ci) · [Skills](#skills) · [Related tools](#related-tools)
|
|
38
|
-
|
|
39
|
-
</details>
|
|
40
|
-
|
|
41
|
-
## Two pillars — pick one or both
|
|
42
|
-
|
|
43
|
-
An agent runs real commands in your repo — it can delete the wrong files, leak a secret, or burn tokens looping on a stale instruction nobody checked. You'd never ship an app without a linter and a test suite. An AI agent steering your codebase is no different — so why is its harness trusted on vibes?
|
|
44
|
-
|
|
45
|
-
`Agent = Model + Harness`. Your harness is everything that steers a run — the **instructions** you write _and_ the **hooks, skills, and settings** that enforce them. vigiles is the **missing linting + testing layer for agentic coding**: it **lints** your instruction files and **tests** your harness. Two pillars of equal weight — adopt either on its own, or both:
|
|
46
|
-
|
|
47
|
-
| | Pillar | What it does |
|
|
48
|
-
| ----- | ------------------------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
|
|
49
|
-
| **①** | [**Verify your instruction files**](#verify-your-instruction-files) | Every linter rule, file path, script, and code symbol your CLAUDE.md cites is checked against reality, so stale references can't silently mislead the agent. |
|
|
50
|
-
| **②** | [**Test your harness**](#test-your-claude-code-harness) | Your hooks and skills are code — vigiles tests that they actually fire, **deterministically and for free** (no model, no API key) before you ever pay for a real-model eval. |
|
|
24
|
+
`Agent = Model + Harness`. You'd never ship an app without a linter and a test
|
|
25
|
+
suite — yet an AI agent steering your repo is trusted on vibes. vigiles is the
|
|
26
|
+
deterministic layer for the harness: it **lints** the references your instruction
|
|
27
|
+
files make and **tests** that your hooks and skills actually fire. Two independent
|
|
28
|
+
pillars — adopt either, or both:
|
|
51
29
|
|
|
52
|
-
|
|
30
|
+
| | Pillar | What it does |
|
|
31
|
+
| ----- | --------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
|
|
32
|
+
| **①** | **Verify your instruction files** | Every linter rule, file path, script, and code symbol your CLAUDE.md cites is checked against reality, so stale references can't silently mislead the agent. → [guide](docs/verifying-instruction-files.md) |
|
|
33
|
+
| **②** | **Test your harness** | Your hooks and skills are code — vigiles tests they actually fire, **deterministically and free** (no model, no API key) before you pay for an eval. → [guide](docs/harness-testing.md) |
|
|
53
34
|
|
|
54
|
-
|
|
35
|
+
Neither pillar depends on the other — pick the one that hurts today. **Works with
|
|
36
|
+
Claude Code and Codex** ([`vigiles/codex`](docs/harnesses.md)) behind a five-port
|
|
37
|
+
adapter; [custom adapters welcome](docs/authoring-an-adapter.md).
|
|
55
38
|
|
|
56
|
-
|
|
39
|
+
## ① Verify — your CLAUDE.md lies to your agent
|
|
57
40
|
|
|
58
|
-
|
|
41
|
+
Your CLAUDE.md says _"enforce `eslint/no-console`."_ But it was switched off
|
|
42
|
+
months ago — and the agent trusts the claim. (Same story for the file path it
|
|
43
|
+
cites that got renamed, and the script that was deleted.)
|
|
59
44
|
|
|
60
|
-
|
|
61
|
-
## Code Style
|
|
45
|
+
**Without vigiles:** nobody checks. The agent acts on fiction.
|
|
62
46
|
|
|
63
|
-
|
|
64
|
-
catches this. Always use `unknown` and narrow with type guards.
|
|
65
|
-
See `src/utils/type-helpers.ts` for project utilities.
|
|
47
|
+
**With vigiles:** `npx vigiles audit` resolves every reference against reality —
|
|
66
48
|
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
49
|
+
```text
|
|
50
|
+
CLAUDE.md (inline mode):
|
|
51
|
+
✗ line 1: Rule "eslint/no-console" exists but is disabled in eslint config
|
|
52
|
+
✓ line 2: eslint/eqeqeq
|
|
53
|
+
✗ line 3: Rule "no-consoel" not found in eslint. Did you mean: "eslint/no-console"?
|
|
71
54
|
```
|
|
72
55
|
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
2. `src/utils/type-helpers.ts` — renamed to `src/utils/narrowing.ts` last quarter
|
|
77
|
-
3. `npm run typecheck` — script removed from package.json
|
|
78
|
-
4. Service/test pairing — no automated check, just a hope
|
|
79
|
-
|
|
80
|
-
The agent reads this, trusts it, and writes code based on stale claims nobody verified. vigiles **verifies the references in your instruction files** — that each linter rule exists and is enabled, that every file path and script is real, and that referenced **code symbols** (functions, classes, constants) actually exist in the files that define them — and meets you at whatever commitment level you want.
|
|
81
|
-
|
|
82
|
-
> **Set it up — paste this into Claude Code:**
|
|
83
|
-
>
|
|
84
|
-
> > Install vigiles and verify my instruction files. Scan my CLAUDE.md / AGENTS.md, turn its real claims into checked references (linter rules, file paths, scripts, symbols), run `vigiles audit`, and show me what's already stale. Use sensible defaults, but **ask me first** whether to stay in markdown mode or generate a typed `.spec.ts`, and whether to wire the audit into CI and install the edit-blocking hooks.
|
|
85
|
-
|
|
86
|
-
Or do it by hand — add a marker to your existing CLAUDE.md and audit it, no install, no new files:
|
|
56
|
+
It resolves rule names across **7 linter catalogs** — the rule exists **and is
|
|
57
|
+
enabled** — and checks file paths, scripts, and code symbols the same way. Start
|
|
58
|
+
with one comment, no new files:
|
|
87
59
|
|
|
88
60
|
```md
|
|
89
61
|
<!-- vigiles:enforce eslint/no-console "Route output through logger.ts" -->
|
|
90
62
|
```
|
|
91
63
|
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
```
|
|
64
|
+
Step up to a typed `.spec.ts` (compiled to CLAUDE.md, compiler-grade) when you
|
|
65
|
+
want it. **[Full guide →](docs/verifying-instruction-files.md)**
|
|
95
66
|
|
|
96
|
-
|
|
67
|
+
## ② Test — does your harness actually fire?
|
|
97
68
|
|
|
98
|
-
|
|
69
|
+
A hook can be wired wrong, a skill's description can fail to trigger, injected
|
|
70
|
+
context can never reach the model — silently, all passing a naive "did it run?"
|
|
71
|
+
check.
|
|
99
72
|
|
|
100
|
-
|
|
73
|
+
**Without vigiles:** you assume your `--no-verify` guard blocks. You don't know.
|
|
101
74
|
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
```
|
|
105
|
-
|
|
106
|
-
The wizard auto-detects your project, creates a spec, scans your linters, compiles to markdown, adds a CI step, and installs Claude Code hooks. After install: the agent edits the spec (hooks block direct CLAUDE.md edits), the spec auto-compiles on save, and `vigiles audit` catches drift in CI. Prefer no new files? Stay in [markdown mode](docs/markdown-mode.md). Start with `guidance()` rules and `/strengthen` them to `enforce()` later; flags and agent usage are in the [CLI reference](docs/cli.md).
|
|
107
|
-
|
|
108
|
-
Companion repo for [Feedback Loop Is All You Need](https://zernie.com/blog/feedback-loop-is-all-you-need).
|
|
109
|
-
|
|
110
|
-
## Test your Claude Code harness
|
|
111
|
-
|
|
112
|
-
Verifying references proves your instructions are _true_ — but the hooks and
|
|
113
|
-
skills that enforce them still have to **actually fire**. A hook can be wired
|
|
114
|
-
wrong, a skill's description can fail to trigger, injected context can never reach
|
|
115
|
-
the model — all silently, all passing a naive "did it run?" check. So vigiles's
|
|
116
|
-
second pillar **tests the harness itself**, as the assembled machine it ships as.
|
|
117
|
-
|
|
118
|
-
> **Set it up — paste this into Claude Code:**
|
|
119
|
-
>
|
|
120
|
-
> > Install vigiles and use its `test-harness` skill to write and run a harness test for this project. If I didn't say what to test, pick something real from my hooks / skills / settings, choose the cheapest tier (unit / deterministic / eval), write the test, and run it. Use good defaults, but **ask me** whether to gate it in CI and whether to add a real-model eval.
|
|
121
|
-
|
|
122
|
-
It's a small library of plain async functions (drops into node:test / vitest /
|
|
123
|
-
jest, or a zero-setup `vigiles test`), with three tiers, cheapest first. The
|
|
124
|
-
design bet is **deterministic and cheap**: the first two tiers never call a model
|
|
125
|
-
or need an API key, so they run on every commit for free — the opposite of
|
|
126
|
-
eval-only frameworks like promptfoo, where every run hits a real model **by
|
|
127
|
-
design**. You only reach for the paid real-model tier when the question genuinely
|
|
128
|
-
needs it.
|
|
129
|
-
|
|
130
|
-
- **Unit-test a hook** — `runHook` hands a hook a fake event and checks block/allow. No `claude`, no model, milliseconds, reaches **every** event type.
|
|
131
|
-
- **Deterministic harness test** — `runHarnessTest` runs the **real** `claude` against a **scripted mock model**, so your hooks fire for real with no API key and the same result every time.
|
|
132
|
-
- **Eval** — `runEval` runs the real model A/B (change on vs off) and reports the gap as **mean ± se**, with a Welch-t-test [significance gate](docs/harness-testing.md#significance--is-the-gap-real), regression baselines, and cost/latency/token tracking.
|
|
75
|
+
**With vigiles:** a deterministic test proves it — no model, no API key,
|
|
76
|
+
milliseconds:
|
|
133
77
|
|
|
134
78
|
```typescript
|
|
135
79
|
import { runHook } from "vigiles/testing";
|
|
136
80
|
|
|
137
|
-
const r = runHook(
|
|
81
|
+
const r = runHook(guard, {
|
|
138
82
|
hook_event_name: "PreToolUse",
|
|
139
83
|
tool_name: "Bash",
|
|
140
84
|
tool_input: { command: "git commit --no-verify" },
|
|
141
85
|
});
|
|
142
|
-
assert(r.blocked); //
|
|
86
|
+
assert(r.blocked); // a red ✗ here means your hook silently lets it through
|
|
143
87
|
```
|
|
144
88
|
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
the safe-by-default sandbox for untrusted plugins, the surface × tier coverage
|
|
148
|
-
matrix, and how it compares to promptfoo. Also: [benchmarks](research/benchmarks-runtime-gates.md).
|
|
89
|
+
```text
|
|
90
|
+
✓ guard blocks --no-verify and allows a clean commit
|
|
149
91
|
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
```bash
|
|
153
|
-
npx vigiles init # Scaffold a spec (full setup wizard)
|
|
154
|
-
npx vigiles compile # Compile .spec.ts → .md
|
|
155
|
-
npx vigiles audit # Verify hashes + inline/frontmatter/spec rules + symbols + coverage
|
|
156
|
-
npx vigiles test # Run *.harness.mjs deterministic harness tests (no API key)
|
|
157
|
-
npx vigiles eval # Run *.eval.mjs real-model harness evals (--trials=N)
|
|
92
|
+
2 passed.
|
|
158
93
|
```
|
|
159
94
|
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
95
|
+
Three tiers, cheapest first: **`runHook`** (a hook's logic), **`runHarnessTest`**
|
|
96
|
+
(the real agent CLI against a scripted mock model), **`runEval`** (the real model
|
|
97
|
+
A/B with a significance gate). **Testing a skill?** `measureTriggerRate` checks
|
|
98
|
+
its description actually **fires** across varied prompts (recall) without
|
|
99
|
+
hijacking unrelated ones (precision). **[Full guide →](docs/harness-testing.md)**
|
|
165
100
|
|
|
166
|
-
|
|
167
|
-
<summary><b>The 8 skills</b></summary>
|
|
101
|
+
## Quick start
|
|
168
102
|
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
| `enforce-rules-format` | Validate all rules have enforcement classification |
|
|
177
|
-
| `audit-feedback-loop` | Score your repo's feedback loop maturity |
|
|
178
|
-
| `test-harness` | Test a Claude Code harness — pick the tier (unit / deterministic / eval) and write a passing test |
|
|
103
|
+
> **Paste into Claude Code or Codex:**
|
|
104
|
+
>
|
|
105
|
+
> > Install vigiles in this repo and run it. Verify my CLAUDE.md / AGENTS.md
|
|
106
|
+
> > references and show me what's stale, then write and run a harness test for one
|
|
107
|
+
> > of my hooks or skills. Use good defaults (both pillars, non-interactive), but
|
|
108
|
+
> > **ask me first** whether to gate it in CI, whether to add a real-model eval,
|
|
109
|
+
> > and whether to enforce strictly (`--strict`).
|
|
179
110
|
|
|
180
|
-
|
|
111
|
+
Or do it yourself:
|
|
181
112
|
|
|
182
|
-
|
|
113
|
+
```bash
|
|
114
|
+
npx vigiles init # sets up BOTH pillars: spec + harness test + CI + plugin
|
|
115
|
+
```
|
|
183
116
|
|
|
184
|
-
|
|
117
|
+
It's interactive in a terminal and non-interactive for agents/CI (or with
|
|
118
|
+
`--yes`), so "set up vigiles" from a Claude Code / Codex prompt Just Works — and
|
|
119
|
+
it installs a model-invocable **`test-harness` skill**, so afterward you can just
|
|
120
|
+
tell your agent _"test my skills"_ and it picks the tier and writes the test.
|
|
121
|
+
Scope `init` with `--pillars=verify|test|both`. Or write harness tests yourself in
|
|
122
|
+
JS **or** TS (`*.harness.{mjs,ts}`) and run `npx vigiles test`. `init` wires CI as
|
|
123
|
+
a `zernie/vigiles@v1` workflow — a composite over the same CLI:
|
|
124
|
+
|
|
125
|
+
```yaml
|
|
126
|
+
- uses: actions/checkout@v4
|
|
127
|
+
- uses: zernie/vigiles@v1 # audits by default; posts a sticky PR comment + a `valid` output
|
|
128
|
+
```
|
|
185
129
|
|
|
186
|
-
##
|
|
130
|
+
## More
|
|
187
131
|
|
|
188
|
-
- **
|
|
189
|
-
- **[
|
|
190
|
-
- **[
|
|
132
|
+
- **[CLI & GitHub Action →](docs/cli.md)** — every command, the Action (inputs / output / versioning), the Claude Code plugin, and the five `audit` rules.
|
|
133
|
+
- **[Skills →](docs/skills.md)** — 8 skills (`strengthen`, `migrate-to-spec`, `test-harness`, …) via `npx skills add zernie/vigiles`.
|
|
134
|
+
- **[Docs index →](docs/README.md)** · **[Research →](research/README.md)** · **[Related tools →](docs/related-tools.md)** (ast-grep, Dependency Cruiser, Ruler, rulesync).
|
|
135
|
+
- Companion to [Feedback Loop Is All You Need](https://zernie.com/blog/feedback-loop-is-all-you-need).
|
|
191
136
|
|
|
192
137
|
## License
|
|
193
138
|
|
package/action.yml
CHANGED
|
@@ -1,25 +1,161 @@
|
|
|
1
1
|
name: "vigiles"
|
|
2
|
-
description: "
|
|
2
|
+
description: "Verify the references in your agent instruction files (CLAUDE.md/AGENTS.md) and compile typed specs — in CI."
|
|
3
|
+
author: "zernie"
|
|
3
4
|
branding:
|
|
4
5
|
icon: "check-circle"
|
|
5
6
|
color: "orange"
|
|
6
7
|
|
|
7
8
|
inputs:
|
|
8
9
|
command:
|
|
9
|
-
description: "
|
|
10
|
+
description: "Which vigiles command to run: 'audit' (verify references + integrity + coverage) or 'compile' (specs → markdown)."
|
|
10
11
|
required: false
|
|
11
12
|
default: "audit"
|
|
12
13
|
paths:
|
|
13
|
-
description: "Comma-separated paths to .
|
|
14
|
+
description: "Comma- or space-separated paths to operate on (.md files for audit, .spec.ts files for compile). Auto-discovers when empty."
|
|
14
15
|
required: false
|
|
16
|
+
default: ""
|
|
17
|
+
version:
|
|
18
|
+
description: "Version of the vigiles npm package to run (e.g. '1', '1.2.3', 'latest'). Use 'local' to run the action's own checkout — for dogfooding via `uses: ./` in this repo's CI."
|
|
19
|
+
required: false
|
|
20
|
+
default: "latest"
|
|
15
21
|
max-rules:
|
|
16
|
-
description: "Maximum number of rules per spec file"
|
|
22
|
+
description: "Maximum number of rules per spec file (maps to the CLI's --max-rules)."
|
|
17
23
|
required: false
|
|
24
|
+
default: ""
|
|
18
25
|
catalog-only:
|
|
19
|
-
description: "Only check that linter rules exist
|
|
26
|
+
description: "Only check that linter rules exist; skip config-enabled checks (maps to the CLI's --catalog-only). 'true' or 'false'."
|
|
20
27
|
required: false
|
|
21
28
|
default: "false"
|
|
29
|
+
working-directory:
|
|
30
|
+
description: "Directory to run vigiles in."
|
|
31
|
+
required: false
|
|
32
|
+
default: "."
|
|
33
|
+
comment:
|
|
34
|
+
description: "On pull_request events, post/update a sticky PR comment with the result ('true' or 'false'). Requires `pull-requests: write`."
|
|
35
|
+
required: false
|
|
36
|
+
default: "true"
|
|
37
|
+
github-token:
|
|
38
|
+
description: "Token used to post the PR comment. Defaults to the workflow token."
|
|
39
|
+
required: false
|
|
40
|
+
default: ${{ github.token }}
|
|
41
|
+
|
|
42
|
+
outputs:
|
|
43
|
+
valid:
|
|
44
|
+
description: "'true' if vigiles passed (exit 0), 'false' otherwise."
|
|
45
|
+
value: ${{ steps.vigiles.outputs.valid }}
|
|
22
46
|
|
|
23
47
|
runs:
|
|
24
|
-
using: "
|
|
25
|
-
|
|
48
|
+
using: "composite"
|
|
49
|
+
steps:
|
|
50
|
+
- id: vigiles
|
|
51
|
+
shell: bash
|
|
52
|
+
working-directory: ${{ inputs.working-directory }}
|
|
53
|
+
env:
|
|
54
|
+
# Inputs are passed through the environment (never interpolated into
|
|
55
|
+
# the script body) so a value can't break out of the command line.
|
|
56
|
+
VIGILES_COMMAND: ${{ inputs.command }}
|
|
57
|
+
VIGILES_PATHS: ${{ inputs.paths }}
|
|
58
|
+
VIGILES_VERSION: ${{ inputs.version }}
|
|
59
|
+
VIGILES_MAX_RULES: ${{ inputs.max-rules }}
|
|
60
|
+
VIGILES_CATALOG_ONLY: ${{ inputs.catalog-only }}
|
|
61
|
+
VIGILES_ACTION_PATH: ${{ github.action_path }}
|
|
62
|
+
VIGILES_COMMENT: ${{ inputs.comment }}
|
|
63
|
+
GH_TOKEN: ${{ inputs.github-token }}
|
|
64
|
+
run: |
|
|
65
|
+
set -euo pipefail
|
|
66
|
+
|
|
67
|
+
cmd="${VIGILES_COMMAND:-audit}"
|
|
68
|
+
args=("$cmd")
|
|
69
|
+
|
|
70
|
+
# paths: split on commas and whitespace into positional args.
|
|
71
|
+
paths="${VIGILES_PATHS:-}"
|
|
72
|
+
if [[ -n "${paths// /}" ]]; then
|
|
73
|
+
IFS=', ' read -r -a _paths <<< "$paths"
|
|
74
|
+
args+=("${_paths[@]}")
|
|
75
|
+
fi
|
|
76
|
+
|
|
77
|
+
max_rules="${VIGILES_MAX_RULES:-}"
|
|
78
|
+
if [[ -n "${max_rules// /}" ]]; then
|
|
79
|
+
args+=("--max-rules=${max_rules}")
|
|
80
|
+
fi
|
|
81
|
+
|
|
82
|
+
if [[ "${VIGILES_CATALOG_ONLY:-false}" == "true" ]]; then
|
|
83
|
+
args+=("--catalog-only")
|
|
84
|
+
fi
|
|
85
|
+
|
|
86
|
+
# Resolve the runner. 'local' runs the action's own build (CI
|
|
87
|
+
# dogfood via `uses: ./`); any other value runs the published npm
|
|
88
|
+
# package at that version, so the Action reuses the same tested
|
|
89
|
+
# artifact users get from `npx vigiles`.
|
|
90
|
+
version="${VIGILES_VERSION:-latest}"
|
|
91
|
+
if [[ "$version" == "local" ]]; then
|
|
92
|
+
runner=(node "${VIGILES_ACTION_PATH}/dist/cli.js")
|
|
93
|
+
else
|
|
94
|
+
runner=(npx --yes "vigiles@${version}")
|
|
95
|
+
fi
|
|
96
|
+
|
|
97
|
+
echo "vigiles: ${runner[*]} ${args[*]}"
|
|
98
|
+
|
|
99
|
+
# Run, streaming to the log AND capturing for the summary/comment.
|
|
100
|
+
out="$(mktemp)"
|
|
101
|
+
set +e
|
|
102
|
+
GITHUB_ACTIONS=true "${runner[@]}" "${args[@]}" 2>&1 | tee "$out"
|
|
103
|
+
status="${PIPESTATUS[0]}"
|
|
104
|
+
set -e
|
|
105
|
+
|
|
106
|
+
if [[ "$status" -eq 0 ]]; then
|
|
107
|
+
echo "valid=true" >> "$GITHUB_OUTPUT"
|
|
108
|
+
headline="✅ \`vigiles ${cmd}\` passed"
|
|
109
|
+
else
|
|
110
|
+
echo "valid=false" >> "$GITHUB_OUTPUT"
|
|
111
|
+
headline="❌ \`vigiles ${cmd}\` failed (exit ${status})"
|
|
112
|
+
echo "::error::vigiles ${cmd} failed (exit ${status})"
|
|
113
|
+
fi
|
|
114
|
+
|
|
115
|
+
# Build a markdown report (cap the log tail so a huge run stays sane).
|
|
116
|
+
report="$(mktemp)"
|
|
117
|
+
{
|
|
118
|
+
echo "## vigiles"
|
|
119
|
+
echo
|
|
120
|
+
echo "$headline"
|
|
121
|
+
echo
|
|
122
|
+
echo '<details><summary>Output</summary>'
|
|
123
|
+
echo
|
|
124
|
+
echo '```'
|
|
125
|
+
tail -c 60000 "$out"
|
|
126
|
+
echo '```'
|
|
127
|
+
echo
|
|
128
|
+
echo '</details>'
|
|
129
|
+
} > "$report"
|
|
130
|
+
|
|
131
|
+
# Job summary (always, when available) — the modern run-page output.
|
|
132
|
+
if [[ -n "${GITHUB_STEP_SUMMARY:-}" ]]; then
|
|
133
|
+
cat "$report" >> "$GITHUB_STEP_SUMMARY"
|
|
134
|
+
fi
|
|
135
|
+
|
|
136
|
+
# Sticky PR comment (pull_request events only, best-effort): find the
|
|
137
|
+
# prior vigiles comment by marker and update it in place, else create.
|
|
138
|
+
marker="<!-- vigiles-action -->"
|
|
139
|
+
if [[ "${VIGILES_COMMENT:-true}" == "true" \
|
|
140
|
+
&& "${GITHUB_EVENT_NAME:-}" == "pull_request" \
|
|
141
|
+
&& -n "${GH_TOKEN:-}" ]] && command -v gh >/dev/null 2>&1; then
|
|
142
|
+
pr="$(jq -r '.pull_request.number // empty' "${GITHUB_EVENT_PATH:-/dev/null}")"
|
|
143
|
+
if [[ -n "$pr" ]]; then
|
|
144
|
+
body="$(printf '%s\n%s\n' "$marker" "$(cat "$report")")"
|
|
145
|
+
repo="${GITHUB_REPOSITORY}"
|
|
146
|
+
cid="$(gh api "repos/${repo}/issues/${pr}/comments" --paginate \
|
|
147
|
+
--jq "map(select(.body | startswith(\"${marker}\"))) | .[0].id // empty" \
|
|
148
|
+
2>/dev/null || true)"
|
|
149
|
+
if [[ -n "$cid" ]]; then
|
|
150
|
+
gh api -X PATCH "repos/${repo}/issues/comments/${cid}" -f body="$body" >/dev/null \
|
|
151
|
+
&& echo "Updated PR comment #${cid}" \
|
|
152
|
+
|| echo "::warning::could not update PR comment (need pull-requests: write?)"
|
|
153
|
+
else
|
|
154
|
+
gh api -X POST "repos/${repo}/issues/${pr}/comments" -f body="$body" >/dev/null \
|
|
155
|
+
&& echo "Posted PR comment" \
|
|
156
|
+
|| echo "::warning::could not post PR comment (need pull-requests: write?)"
|
|
157
|
+
fi
|
|
158
|
+
fi
|
|
159
|
+
fi
|
|
160
|
+
|
|
161
|
+
exit "$status"
|
package/dist/action-gate.d.ts
CHANGED
package/dist/action-gate.js
CHANGED
|
@@ -21,7 +21,7 @@ exports.loadActionGates = loadActionGates;
|
|
|
21
21
|
*/
|
|
22
22
|
const node_fs_1 = require("node:fs");
|
|
23
23
|
const node_path_1 = require("node:path");
|
|
24
|
-
const skill_runtime_js_1 = require("./skill-runtime.js");
|
|
24
|
+
const skill_runtime_js_1 = require("./adapters/claude-code/skill-runtime.js");
|
|
25
25
|
/** The file path an action touched, for `{file}` substitution. */
|
|
26
26
|
function fileOf(event) {
|
|
27
27
|
const i = event.input ?? {};
|
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
import type { HarnessAdapter } from "./core/adapter.js";
|
|
2
|
+
export interface ConformanceResult {
|
|
3
|
+
readonly ok: boolean;
|
|
4
|
+
readonly failures: readonly string[];
|
|
5
|
+
}
|
|
6
|
+
/** Check an adapter against the port contracts; returns the (possibly empty) failure list. */
|
|
7
|
+
export declare function checkAdapterConformance(adapter: HarnessAdapter): ConformanceResult;
|
|
8
|
+
/** Throw if the adapter fails conformance — drop this in an adapter's test suite. */
|
|
9
|
+
export declare function assertAdapterConformance(adapter: HarnessAdapter): void;
|
|
10
|
+
/**
|
|
11
|
+
* Guard for the pillar-2 entry points (runHarnessTest/runEval): a pillar-1-only
|
|
12
|
+
* adapter (Cursor, Devin, Amp, Amazon Q) has no mockable transport, so driving
|
|
13
|
+
* the deterministic/eval tiers against it would hang or spawn nothing. Calling
|
|
14
|
+
* this up front turns that into a clear, immediate error. Returns the narrowed
|
|
15
|
+
* runtime+modelMock so the caller can use them without re-checking for undefined.
|
|
16
|
+
*/
|
|
17
|
+
export declare function assertHarnessTestable(adapter: HarnessAdapter): {
|
|
18
|
+
runtime: NonNullable<HarnessAdapter["runtime"]>;
|
|
19
|
+
modelMock: NonNullable<HarnessAdapter["modelMock"]>;
|
|
20
|
+
};
|
|
21
|
+
/**
|
|
22
|
+
* Behavioural conformance the pure checks can't reach: write a minimal settings
|
|
23
|
+
* file in the adapter's declared `settingsFormat` (with a hook), load it through
|
|
24
|
+
* the adapter's `layout`, and assert the hooks actually came back. This is what
|
|
25
|
+
* catches a layout that points at the right file but in the wrong format (the
|
|
26
|
+
* JSON-vs-TOML trap) — the pure checker would pass it, the agent would silently
|
|
27
|
+
* run with zero hooks. Does filesystem IO, so it's a separate opt-in assert.
|
|
28
|
+
*/
|
|
29
|
+
export declare function assertAdapterLoadsHooks(adapter: HarnessAdapter): void;
|
|
30
|
+
//# sourceMappingURL=adapter-conformance.d.ts.map
|