vigiles 2.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (156) hide show
  1. package/.claude/settings.json +46 -0
  2. package/.claude/settings.local.json +8 -0
  3. package/.claude-plugin/hooks/post-edit.sh +34 -0
  4. package/.claude-plugin/hooks/pre-edit.sh +40 -0
  5. package/.claude-plugin/hooks/session-start.sh +38 -0
  6. package/.claude-plugin/marketplace.json +14 -0
  7. package/.claude-plugin/plugin.json +47 -0
  8. package/.github/workflows/ci.yml +81 -0
  9. package/.prettierignore +1 -0
  10. package/.vigiles/generated.d.ts +205 -0
  11. package/CLAUDE.md +95 -0
  12. package/CLAUDE.md.spec.ts +142 -0
  13. package/CONTRIBUTING.md +121 -0
  14. package/LICENSE +21 -0
  15. package/README.md +377 -0
  16. package/action.yml +25 -0
  17. package/dist/action.d.ts +7 -0
  18. package/dist/action.d.ts.map +1 -0
  19. package/dist/action.js +180 -0
  20. package/dist/action.js.map +1 -0
  21. package/dist/cli.d.ts +12 -0
  22. package/dist/cli.d.ts.map +1 -0
  23. package/dist/cli.js +1267 -0
  24. package/dist/cli.js.map +1 -0
  25. package/dist/cli.test.d.ts +2 -0
  26. package/dist/cli.test.d.ts.map +1 -0
  27. package/dist/cli.test.js +650 -0
  28. package/dist/cli.test.js.map +1 -0
  29. package/dist/compile.d.ts +101 -0
  30. package/dist/compile.d.ts.map +1 -0
  31. package/dist/compile.js +503 -0
  32. package/dist/compile.js.map +1 -0
  33. package/dist/evolve.d.ts +132 -0
  34. package/dist/evolve.d.ts.map +1 -0
  35. package/dist/evolve.js +599 -0
  36. package/dist/evolve.js.map +1 -0
  37. package/dist/freshness.d.ts +67 -0
  38. package/dist/freshness.d.ts.map +1 -0
  39. package/dist/freshness.js +244 -0
  40. package/dist/freshness.js.map +1 -0
  41. package/dist/freshness.test.d.ts +2 -0
  42. package/dist/freshness.test.d.ts.map +1 -0
  43. package/dist/freshness.test.js +356 -0
  44. package/dist/freshness.test.js.map +1 -0
  45. package/dist/generate-types.d.ts +34 -0
  46. package/dist/generate-types.d.ts.map +1 -0
  47. package/dist/generate-types.js +381 -0
  48. package/dist/generate-types.js.map +1 -0
  49. package/dist/inline.d.ts +58 -0
  50. package/dist/inline.d.ts.map +1 -0
  51. package/dist/inline.js +142 -0
  52. package/dist/inline.js.map +1 -0
  53. package/dist/inline.test.d.ts +5 -0
  54. package/dist/inline.test.d.ts.map +1 -0
  55. package/dist/inline.test.js +152 -0
  56. package/dist/inline.test.js.map +1 -0
  57. package/dist/linters.d.ts +38 -0
  58. package/dist/linters.d.ts.map +1 -0
  59. package/dist/linters.js +588 -0
  60. package/dist/linters.js.map +1 -0
  61. package/dist/proofs.d.ts +272 -0
  62. package/dist/proofs.d.ts.map +1 -0
  63. package/dist/proofs.js +622 -0
  64. package/dist/proofs.js.map +1 -0
  65. package/dist/proofs.test.d.ts +9 -0
  66. package/dist/proofs.test.d.ts.map +1 -0
  67. package/dist/proofs.test.js +952 -0
  68. package/dist/proofs.test.js.map +1 -0
  69. package/dist/spec.d.ts +258 -0
  70. package/dist/spec.d.ts.map +1 -0
  71. package/dist/spec.js +113 -0
  72. package/dist/spec.js.map +1 -0
  73. package/dist/spec.test.d.ts +2 -0
  74. package/dist/spec.test.d.ts.map +1 -0
  75. package/dist/spec.test.js +1222 -0
  76. package/dist/spec.test.js.map +1 -0
  77. package/dist/types.d.ts +101 -0
  78. package/dist/types.d.ts.map +1 -0
  79. package/dist/types.js +3 -0
  80. package/dist/types.js.map +1 -0
  81. package/dist/validate.d.ts +10 -0
  82. package/dist/validate.d.ts.map +1 -0
  83. package/dist/validate.js +286 -0
  84. package/dist/validate.js.map +1 -0
  85. package/dist/validate.test.d.ts +2 -0
  86. package/dist/validate.test.d.ts.map +1 -0
  87. package/dist/validate.test.js +531 -0
  88. package/dist/validate.test.js.map +1 -0
  89. package/docs/agent-setup.md +85 -0
  90. package/docs/agent-workflows.md +103 -0
  91. package/docs/comparison.md +71 -0
  92. package/docs/freshness.md +124 -0
  93. package/docs/inline-mode.md +119 -0
  94. package/docs/linter-support.md +166 -0
  95. package/docs/spec-format.md +194 -0
  96. package/eslint.config.mjs +79 -0
  97. package/examples/CLAUDE.md +54 -0
  98. package/examples/CLAUDE.md.spec.ts +65 -0
  99. package/examples/SKILL.md +50 -0
  100. package/examples/SKILL.md.spec.ts +57 -0
  101. package/fixtures/example-project/CLAUDE.md +11 -0
  102. package/fixtures/example-project/package.json +9 -0
  103. package/fixtures/example-project/src/index.ts +3 -0
  104. package/fixtures/example-project/src/utils.test.ts +2 -0
  105. package/fixtures/example-project/src/utils.ts +3 -0
  106. package/logo.png +0 -0
  107. package/package.json +42 -0
  108. package/research/adoption-strategy.md +111 -0
  109. package/research/agent-integration.md +145 -0
  110. package/research/ai-code-quality.md +197 -0
  111. package/research/code-search-for-agents.md +313 -0
  112. package/research/competitive-landscape.md +163 -0
  113. package/research/doc-freshness.md +516 -0
  114. package/research/executable-specs.md +368 -0
  115. package/research/feature-ideas.md +464 -0
  116. package/research/formal-proofs-for-agents.md +338 -0
  117. package/research/fp-for-agent-harness.md +150 -0
  118. package/research/fp-for-deterministic-ai.md +131 -0
  119. package/research/self-evolving-specs.md +298 -0
  120. package/schemas/claude-md-strict.yml +18 -0
  121. package/schemas/claude-md.yml +6 -0
  122. package/schemas/skill-strict.yml +12 -0
  123. package/schemas/skill.yml +5 -0
  124. package/skills/audit-feedback-loop/SKILL.md +76 -0
  125. package/skills/edit-spec/SKILL.md +131 -0
  126. package/skills/enforce-rules-format/SKILL.md +71 -0
  127. package/skills/generate-logo/SKILL.md +102 -0
  128. package/skills/generate-rule/SKILL.md +90 -0
  129. package/skills/linter-docs/clippy.md +241 -0
  130. package/skills/linter-docs/eslint.md +384 -0
  131. package/skills/linter-docs/pylint.md +288 -0
  132. package/skills/linter-docs/rubocop.md +277 -0
  133. package/skills/linter-docs/ruff.md +187 -0
  134. package/skills/linter-docs/stylelint.md +247 -0
  135. package/skills/migrate-to-spec/SKILL.md +124 -0
  136. package/skills/pr-to-lint-rule/SKILL.md +97 -0
  137. package/skills/strengthen/SKILL.md +168 -0
  138. package/src/action.ts +214 -0
  139. package/src/cli.test.ts +914 -0
  140. package/src/cli.ts +1631 -0
  141. package/src/compile.ts +691 -0
  142. package/src/evolve.ts +781 -0
  143. package/src/freshness.test.ts +449 -0
  144. package/src/freshness.ts +299 -0
  145. package/src/generate-types.ts +448 -0
  146. package/src/inline.test.ts +206 -0
  147. package/src/inline.ts +164 -0
  148. package/src/linters.ts +739 -0
  149. package/src/proofs.test.ts +1314 -0
  150. package/src/proofs.ts +849 -0
  151. package/src/spec.test.ts +1471 -0
  152. package/src/spec.ts +427 -0
  153. package/src/types.ts +117 -0
  154. package/src/validate.test.ts +701 -0
  155. package/src/validate.ts +381 -0
  156. package/tsconfig.json +23 -0
@@ -0,0 +1,111 @@
1
+ # vigiles Adoption Strategy
2
+
3
+ Goal: **`npx vigiles setup && npx skills add zernie/vigiles` works on first run with zero config. The agent starts editing specs automatically — no workflow change required. Start permissive, tighten over time.**
4
+
5
+ ### Adoption Principles
6
+
7
+ 1. **Works on first run.** Setup must succeed in any project without configuration. Auto-detect everything. Create reasonable defaults. Don't block on missing tools.
8
+ 2. **Zero workflow change.** After plugin install, the agent edits specs instead of markdown. The user doesn't need to learn a new workflow — they say "update CLAUDE.md" and the plugin handles the redirect.
9
+ 3. **Start permissive, tighten later.** First run creates `guidance()` rules (no enforcement). User upgrades to `enforce()` as they add linter rules. `require-spec: false` available for incremental migration.
10
+ 4. **Every surface tells you the next step.** `vigiles check` says "run setup." The hook says "edit the spec." The wizard says "install the plugin." No dead ends.
11
+
12
+ ---
13
+
14
+ ## Scope: What vigiles Does vs. Doesn't
15
+
16
+ vigiles compiles typed TypeScript specs to **markdown instruction files** (CLAUDE.md, AGENTS.md). It verifies linter rules, file paths, and commands at compile time. The compiled markdown is the artifact.
17
+
18
+ **vigiles handles:** CLAUDE.md (Claude Code), AGENTS.md (Codex, GitHub Copilot, any agent that reads AGENTS.md). These are both plain markdown — same compiler, same validation, different `target`.
19
+
20
+ **vigiles does NOT handle:** `.cursorrules`, `.copilot-instructions.md`, Windsurf format, or any non-markdown target. These have different structures. Use [rule-porter](https://github.com/nichochar/rule-porter) or [rulesync](https://github.com/dyoshikawa/rulesync) to convert compiled markdown to those formats. vigiles is the source; sync tools are the distribution layer.
21
+
22
+ **No symlinks needed.** AGENTS.md is a first-class target via `target: "AGENTS.md"` or `target: ["CLAUDE.md", "AGENTS.md"]`. The compiler outputs both from one spec.
23
+
24
+ ---
25
+
26
+ ## The Setup Wizard
27
+
28
+ `npx vigiles setup` is the single entry point. It does everything:
29
+
30
+ 1. **Creates spec** — scaffolds `CLAUDE.md.spec.ts` (or `--target=AGENTS.md` variant)
31
+ 2. **Generates types** — scans linters, package.json, project files → `.vigiles/generated.d.ts`
32
+ 3. **Compiles** — spec → markdown with SHA-256 hash
33
+ 4. **Adds CI step** — finds existing GHA workflow and appends `vigiles check` + `generate-types --check`
34
+ 5. **Prompts plugin install** — prints `npx skills add zernie/vigiles` with explanation
35
+
36
+ After setup, the user edits the spec, runs `vigiles compile`, and commits. The plugin handles everything else automatically.
37
+
38
+ ---
39
+
40
+ ## Adoption Levels
41
+
42
+ ### Level 0: Discovery
43
+
44
+ User runs `npx vigiles check` on an existing repo. `require-spec` fires: "No spec file found. Run `npx vigiles setup`." First nudge.
45
+
46
+ ### Level 1: Setup
47
+
48
+ ```bash
49
+ npx vigiles setup
50
+ ```
51
+
52
+ One command. Creates spec, types, compiled markdown, CI step. The user has a working pipeline in under a minute.
53
+
54
+ ### Level 2: Plugin
55
+
56
+ ```bash
57
+ npx skills add zernie/vigiles
58
+ ```
59
+
60
+ Two hooks activate:
61
+
62
+ - **PreToolUse**: Blocks direct edits to compiled `.md` files. Agent gets redirected to `.spec.ts`.
63
+ - **PostToolUse**: Auto-runs `generate-types` on config changes, `compile` on spec changes.
64
+
65
+ ### Level 3: Multi-Target
66
+
67
+ ```typescript
68
+ export default claude({
69
+ target: ["CLAUDE.md", "AGENTS.md"],
70
+ rules: { ... },
71
+ });
72
+ ```
73
+
74
+ One spec, multiple outputs. For non-markdown formats, pipe through rule-porter.
75
+
76
+ ### Level 4: Type Narrowing
77
+
78
+ Commit `.vigiles/generated.d.ts`. Now `enforce("eslint/no-consolee")` is a type error in the editor. Types narrow `enforce()`, `file()`, `cmd()` via declaration merging.
79
+
80
+ ---
81
+
82
+ ## Pain Points (Updated)
83
+
84
+ | Pain Point | Status | Resolution |
85
+ | --------------------------------- | --------- | ----------------------------------------------- |
86
+ | Multi-step installation | Fixed | `vigiles setup` does everything |
87
+ | No CI integration from wizard | Fixed | Wizard auto-adds GHA step |
88
+ | Plugin not mentioned as important | Fixed | README and wizard both prompt it |
89
+ | Agent edits compiled .md directly | Fixed | PreToolUse hook blocks with redirect |
90
+ | Cursor/Windsurf support | Won't fix | Out of scope — use sync tools |
91
+ | Codex / AGENTS.md | Fixed | First-class target |
92
+ | No interactive mode for agents | Open | `vigiles setup` works non-interactively already |
93
+
94
+ ## README Structure
95
+
96
+ The README should have:
97
+
98
+ 1. **Hook** — one compelling sentence
99
+ 2. **Problem** — realistic example of rot
100
+ 3. **Fix** — the spec that catches it
101
+ 4. **Quick Start** — `npx vigiles setup` (one command)
102
+ 5. **Three Rule Types** — enforce/check/guidance
103
+ 6. **Verified References** — file/cmd/ref
104
+ 7. **Type-Safe Rule References** — generate-types + narrowing
105
+ 8. **CLI** — reference for all commands
106
+ 9. **GitHub Action** — CI snippet
107
+ 10. **Plugin** — what it does, install command
108
+ 11. **Output Targets** — CLAUDE.md, AGENTS.md, multi-target
109
+ 12. **Related Tools** — sync tools for non-markdown formats
110
+
111
+ No separate installation steps. The wizard IS the installation.
@@ -0,0 +1,145 @@
1
+ # Agent Integration: Proofs, Hooks, and Static Analysis for AI Coding
2
+
3
+ ## Framing
4
+
5
+ vigiles today helps humans write instruction files that AI agents read. That is useful but thin. The larger opportunity is to make vigiles a **deterministic backstop** around coding agents: a layer that catches the specific failure modes AI models have, before they become committed code.
6
+
7
+ This doc collects research on where AI agents actually fail, what deterministic tooling can provably prevent, and proposes 10 concrete vigiles improvements. The through-line is: use the spec as the anchor for hooks, proofs, and static checks — so that unreliable probabilistic tools meet a reliable deterministic floor.
8
+
9
+ ## The problem: spiky intelligence
10
+
11
+ Andrej Karpathy's "jagged intelligence" framing describes it best: LLMs can solve olympiad-level problems in one prompt and fail at `9.11 > 9.9` in the next. Capability is not monotonic across tasks — it is spiky. For coding agents this means:
12
+
13
+ - The same model that writes a correct Raft implementation will happily `import leftpad` when the package does not exist
14
+ - An agent that refactors 500 lines cleanly will silently delete an unrelated test file one invocation later
15
+ - Context windows fix some of this, but hallucinated imports, stale file paths, and out-of-date API signatures survive even with full-file context
16
+
17
+ Deterministic tools do not have this problem. A TypeScript compiler does not "sometimes" miss a type error. A linter does not "occasionally" forget a rule. The thesis of this doc: **put deterministic tools on every edge where the agent touches reality**, and use vigiles specs as the schema that drives them.
18
+
19
+ ## What goes wrong in practice
20
+
21
+ ### 1. Documentation drift
22
+
23
+ ETH Zurich's AGENTS.md study (arxiv 2511.12884) found that well-intentioned AGENTS.md files produced a **3% reduction in task success** and a **20% increase in token cost** when the docs were stale or overlong. Specifically: files over ~300 lines hurt more than they helped. The model pays attention to prose that is wrong.
24
+
25
+ DAPLab's 9-failure-pattern study of coding agents called this out as pattern #1: "instructions the model treats as ground truth but which no longer reflect the code." A single file in a popular TS monorepo had **59 broken file-path references** in its CLAUDE.md — agents cheerfully followed them and wrote new code against paths that no longer existed.
26
+
27
+ ### 2. Package hallucination / slopsquatting
28
+
29
+ Lasso Security's 2024 study: **58% of hallucinated package names are reproducible** across runs of the same prompt. The same non-existent name appears over and over. Attackers have started registering those names on npm and PyPI — the so-called "slopsquatting" attack. If you ask an agent to add a CSV parser and it suggests `fast-csv-parser-lite`, that package might not exist today and might be malware tomorrow.
30
+
31
+ Neither TypeScript nor ESLint catches this. The import looks fine syntactically; it fails at install time or, worse, succeeds and runs attacker code.
32
+
33
+ ### 3. Missing linter backing
34
+
35
+ CodeRabbit's 2025 aggregate data: AI-generated code has **1.7× more reviewable issues** than human-written code. Of those issues, roughly 40% are things an appropriately configured linter would catch — the linter just wasn't configured. vigiles already addresses this for rules that appear in a spec (via `enforce()` cross-referencing), but the inverse problem is unsolved: rules enabled in the linter that the spec never mentions, and rules in the spec that are disabled in the linter but still appear in CLAUDE.md. Both cases are silent drift.
36
+
37
+ ### 4. Secrets and dangerous commands
38
+
39
+ GitGuardian's 2025 report: **3.2% of AI-assisted commits leak a secret**, up from 1.1% in pre-agent commits. Models are not malicious — they paste what they see. A `.env` file in the conversation becomes a `.env` file in the commit. Claude Code's PreToolUse hook is the only unbypassable enforcement point for this; vigiles currently does not ship one.
40
+
41
+ ### 5. Hook anti-patterns
42
+
43
+ Claude Code hook documentation explicitly warns about two anti-patterns people hit repeatedly:
44
+
45
+ - **Silent matcher ignore**: a PreToolUse matcher like `Edit|Write` written as `"Edit|Write"` with no regex semantics silently matches neither
46
+ - **Trailing-wildcard regex**: `Bash.*` without anchoring matches anything containing "Bash"
47
+
48
+ These failures do not error. They just quietly stop enforcing anything. Users believe they are protected when they are not.
49
+
50
+ ### 6. Missing product context
51
+
52
+ The DAPLab study's pattern #7: agents produce syntactically correct code that violates product conventions the codebase has never articulated anywhere. "Use our internal `http.Client` instead of `fetch`." "Log via `logger.ts`, never `console.log`." "All controllers live in `src/controllers/`, not `src/routes/`." These are real rules, but they live in the heads of senior engineers. The moment a rule is not written down, the agent cannot follow it.
53
+
54
+ vigiles's spec format is the right place for these — if they can be written quickly and enforced automatically.
55
+
56
+ ## Academic + industry landscape (2025–2026)
57
+
58
+ - **arxiv 2511.12884** (ETH Zurich) — Empirical analysis of AGENTS.md; shorter-and-enforced beats longer-and-prose-only
59
+ - **DAPLab 2025** — 9-failure-pattern taxonomy for agentic coding
60
+ - **Lasso Security 2024** — Package hallucination reproducibility (58%)
61
+ - **GitGuardian 2025** — Secret leak rates in AI-assisted commits (3.2%)
62
+ - **CodeRabbit 2025** — AI code has 1.7× issue density vs human code
63
+ - **Anthropic Claude Code docs** — PreToolUse / PostToolUse / SessionStart hook surface
64
+ - **Karpathy 2024** — "Jagged intelligence" framing for LLM capability distribution
65
+
66
+ Takeaway: the entire field has converged on the same conclusion. LLMs need deterministic rails. The question is where to put them and who writes the schema.
67
+
68
+ ## Ten vigiles improvements
69
+
70
+ Each of these maps to one or more failure modes above. All are scoped to vigiles's positioning (compilation + linter cross-reference + filesystem assertions). None of them require us to become a linter.
71
+
72
+ ### 1. Ship a hook pack — SHIPPED
73
+
74
+ A `vigiles init` flag that installs a curated `settings.json` block with:
75
+
76
+ - **PreToolUse (Edit|Write)** — block direct edits to any compiled markdown file (CLAUDE.md, AGENTS.md, the target of any discovered `.spec.ts`). Force edits through the spec.
77
+ - **PostToolUse (Edit|Write on \*.spec.ts)** — auto-run `vigiles compile` so the markdown is always in sync without the user thinking about it.
78
+ - **SessionStart (matcher: `compact`)** — run `vigiles audit --summary` and inject the result so the model sees stale-reference warnings every session for free.
79
+
80
+ Why this matters: PreToolUse is the only unbypassable enforcement point in the Claude Code harness. Without it, any rule is advisory. With it, vigiles becomes a real floor.
81
+
82
+ ### 2. Package-existence check inside `audit` — DROPPED
83
+
84
+ Reuse the linter-cross-reference machinery to scan imports in the repo against `package.json`/`requirements.txt`/`Cargo.toml`. Flag any import that (a) is not in the manifest and (b) is not a relative path or builtin. This catches slopsquatting at audit time before it reaches install. Cheap, fully local, zero false positives on builtins.
85
+
86
+ ### 3. Dead-enforcement detection — SHIPPED (already wired for all 6 linters)
87
+
88
+ Today `enforce("eslint/no-console")` verifies that `no-console` exists as a rule. It does not verify that it is **enabled** (severity > 0). Extend the linter engine to surface rules the spec claims are enforced but which the linter reports as `"off"`. This is a silent-drift bug that happens every time someone tweaks eslint config without touching the spec.
89
+
90
+ ### 4. Reverse coverage report — SHIPPED (built into `vigiles audit` coverage stage)
91
+
92
+ Right now `audit` tells you which files have no spec coverage. The inverse is just as useful: which **linter rules** are enabled in the project but not mentioned in any spec. Output as a sorted list so users can either add `enforce()` entries (promoting the rule to agent-visible prose) or mark it intentionally out-of-scope. Closes the gap that CodeRabbit's 1.7× issue-density number is mostly made of.
93
+
94
+ ### 5. SessionStart audit injection (compact matcher) — SHIPPED
95
+
96
+ Hook-pack component worth calling out separately: the `compact` SessionStart matcher fires every time context is compacted, which is effectively free in the session's token budget. Injecting a one-line "3 stale file refs, 2 disabled enforced rules, 1 duplicate rule" message at that moment means the model re-notices drift without anyone running a command. This is the single highest-leverage hook integration.
97
+
98
+ ### 6. Hook validation subcommand — NOT BUILT
99
+
100
+ `vigiles audit --hooks` parses the user's `settings.json`, runs the listed matchers against known-bad inputs, and reports silent failures (matchers that match nothing, trailing-wildcard regex, missing shell escapes). Directly addresses the silent-matcher-ignore anti-pattern documented in Claude Code's own hook docs. No one else ships this check.
101
+
102
+ ### 7. Token-budget compile-time enforcement — SHIPPED
103
+
104
+ ETH Zurich's finding was that CLAUDE.md files over ~300 lines actively hurt. Add a `maxTokens` / `maxLines` field to the spec (already partially present), but enforce it at **compile** time with a hard error rather than as advisory output. Force the user to choose what to cut. This is the simplest way to align vigiles with the empirical evidence that shorter + enforced beats longer + prose.
105
+
106
+ ### 8. Instruction diff subcommand — NOT BUILT
107
+
108
+ `vigiles audit --diff <base>` renders a semantic diff of the compiled markdown: rules added, rules **strengthened** (guidance → enforce), rules **weakened**, rules removed. Attach to every PR that touches a `.spec.ts`. Reviewers currently look at a raw markdown diff which hides the monotonicity property — a strengthen looks identical to a reword unless you read carefully. Semantic diff makes it obvious.
109
+
110
+ ### 9. Snapshot tests for structural changes — PARTIALLY SHIPPED (monotonicity lattice rejects weakening; full `--allow-weaken` flag not yet exposed as CLI)
111
+
112
+ Compile-time assertion: if a spec change would delete an `enforce()` rule, require an explicit `--allow-weaken` flag. This encodes the monotonicity lattice (`guidance < enforce`) from the proofs work as a hard rule rather than advisory output. Pairs with #8: the diff highlights it, the snapshot prevents accidental loss.
113
+
114
+ ### 10. Stale reference detection — DROPPED (compiler's job, not vigiles's)
115
+
116
+ Generalize the existing path-verification (`file("src/foo.ts")`) to a crawler that, during `audit`, walks every code-fenced path, every backticked identifier that looks like a file, every npm-script reference, every package name, and flags the ones that no longer resolve. The 59-broken-refs example was in a file no one had audited for 8 months. A cheap cron-friendly audit command closes that gap forever.
117
+
118
+ ### 11. Adversarial AI review loop — NOT BUILT
119
+
120
+ Ship a skill that takes a spec diff and runs a second LLM pass (same model, reviewer prompt) to critique it before the evolution engine accepts. The Codex-reviews-Claude cycle on PR #16 caught ~40 real bugs across 30+ review rounds — reference leaks, merge semantics, monotonicity bypasses, stale command references. That pattern works. Bake it in: `vigiles evolve --review` proposes a mutation, runs the proof suite, THEN asks a reviewer-prompt subagent "what did I miss?", feeds the response back as a second round of mutations, and only commits the final version. Two-LLM adversarial loop with deterministic gates between rounds.
121
+
122
+ ### 12. Inline-to-spec graduation detector — NOT BUILT
123
+
124
+ When a markdown file accumulates >N inline `<!-- vigiles:enforce ... -->` comments (default: 5), `audit --summary` includes a "graduate to spec mode" nudge. Bonus: print a ready-to-paste `.spec.ts` template pre-filled with all the inline rules already extracted. The adoption funnel becomes: add one comment → add five → see the nudge → paste the template → you're in spec mode. Zero cliff.
125
+
126
+ ### 13. Cross-file rule coherence — NOT BUILT
127
+
128
+ When a project has multiple specs (e.g., `CLAUDE.md.spec.ts` + `docs/AGENTS.md.spec.ts`), run NCD across specs (not just within) to detect cross-file near-duplicates. Also flag contradictions: spec A enforces `eslint/no-console` while spec B's prose says "use `console.log` for debugging." Today each spec is audited in isolation; coherence requires a cross-spec pass.
129
+
130
+ ## What we are NOT adding
131
+
132
+ - Not a linter. Architectural and per-file rules live in ESLint / Ruff / Steiger / ast-grep. vigiles references them.
133
+ - Not an agent sandbox. Claude Code and Codex already own the execution surface.
134
+ - Not a secret scanner. GitGuardian / trufflehog already do this well. We can recommend them via `enforce()` but we do not reimplement.
135
+ - Not an embedding / semantic-search layer. That belongs in the agent, not the spec compiler.
136
+
137
+ ## Next moves
138
+
139
+ 1. Hook pack (#1) + SessionStart audit injection (#5) — highest impact, lowest effort, unblocks everything else
140
+ 2. Dead-enforcement detection (#3) + reverse coverage report (#4) — extend existing linter engine, no new subsystem
141
+ 3. Package-existence check (#2) + stale reference detection (#10) — audit-time crawlers, reuse existing infrastructure
142
+ 4. Semantic diff (#8) + snapshot tests (#9) + token budget (#7) — compile-time guardrails
143
+ 5. Hook validation (#6) — stretch goal, depends on hook pack being stable first
144
+
145
+ The first two bullets could ship in one patch and would already move vigiles from "markdown compiler" to "deterministic backstop for coding agents." Everything after that is additive.
@@ -0,0 +1,197 @@
1
+ # AI Code Quality Research
2
+
3
+ Collected April 2026 during the vigiles v2 design session. This document captures the research, insights, and decisions that shaped vigiles v2's architecture.
4
+
5
+ ---
6
+
7
+ ## The Problem
8
+
9
+ Natural language instruction files (CLAUDE.md, AGENTS.md, .cursorrules) are the interface between humans and AI coding agents. We spent 50 years building compilers, type systems, and linters to escape ambiguity. Now the most powerful code generation tool communicates via markdown.
10
+
11
+ ### Empirical Data
12
+
13
+ | Finding | Source |
14
+ | -------------------------------------------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
15
+ | AI code produces 1.7x more issues than human code | [CodeRabbit report](https://www.coderabbit.ai/blog/state-of-ai-vs-human-code-generation-report) |
16
+ | 2.74x more security vulnerabilities in AI code | CodeRabbit report |
17
+ | 3x more readability problems, 2x naming inconsistencies | CodeRabbit report |
18
+ | 8x more excessive I/O operations | CodeRabbit report |
19
+ | LLM-generated instruction files reduce task success by ~3%, increase cost by 20% | [ETH Zurich SRI Lab](https://www.marktechpost.com/2026/02/25/new-eth-zurich-study-proves-your-ai-coding-agents-are-failing-because-your-agents-md-files-are-too-detailed/) |
20
+ | Over 50% of instruction file rules are noise | ETH Zurich study |
21
+ | Claude Code commits leak secrets at 3.2% (2x human baseline of ~1.5%) | [GitGuardian State of Secrets Sprawl 2026](https://blog.gitguardian.com/the-state-of-secrets-sprawl-2026/) |
22
+ | 60% of AI code faults are silent logic failures | [StackOverflow: Bugs with AI agents](https://stackoverflow.blog/2026/01/28/are-bugs-and-incidents-inevitable-with-ai-coding-agents/) |
23
+ | Instruction files over ~200-300 lines → compliance drops sharply | Multiple sources (lost-in-the-middle effect) |
24
+
25
+ ### Columbia DAPLab: 9 Critical Failure Patterns
26
+
27
+ From analysis of Cline, Claude, Cursor, Replit, and V0 across 15+ applications:
28
+
29
+ 1. Error handling suppression — agents suppress errors to make code run
30
+ 2. Business logic misunderstanding — pricing rules, constraints not tied in correctly
31
+ 3. Codebase awareness degradation — failure rates increase with file count
32
+ 4. Style drift — toward generic defaults for naming, architecture, formatting
33
+ 5. Deprecated/wrong pattern introduction — statistical inference, not semantic understanding
34
+ 6. Secret leakage — 2x higher rates with AI-assisted commits
35
+ 7. Invisible drift accumulation — builds up silently until production failure
36
+ 8. Review fatigue — 14x projected activity surge overwhelms human review
37
+ 9. Concurrency/dependency errors — 2x more likely in AI code
38
+
39
+ ---
40
+
41
+ ## The Core Tension
42
+
43
+ We can't lint natural language. But we CAN lint the claims natural language makes about the world.
44
+
45
+ - "Use the logger" → unlintable
46
+ - `enforce("eslint/no-console", "Use structured logger.")` → falsifiable claim, verified at compile time
47
+
48
+ ### Key Framings
49
+
50
+ - **Martin Fowler's "Harness Engineering"** (Feb 2026): Design deterministic constraints and feedback loops around AI agents. Not prompting. Not instructing. Constraining.
51
+ - **Factory.ai**: "Agents write the code; linters write the law."
52
+ - **The pattern**: "LLM proposes, deterministic tool disposes." The CI gate stays deterministic. AI helps write rules, but rules run deterministically.
53
+ - **Spotify's Honk system**: 650+ agent-generated PRs merged monthly with strong feedback loops.
54
+
55
+ Sources: [Martin Fowler](https://martinfowler.com/articles/exploring-gen-ai/context-engineering-coding-agents.html), [Factory.ai](https://factory.ai/news/using-linters-to-direct-agents), [Spotify](https://engineering.atspotify.com/2025/12/feedback-loops-background-coding-agents-part-3), [StackOverflow: Guidelines for AI](https://stackoverflow.blog/2026/03/26/coding-guidelines-for-ai-agents-and-people-too/)
56
+
57
+ ---
58
+
59
+ ## Ideas Explored
60
+
61
+ ### 10 Initial Ideas (from broad to medium ambition)
62
+
63
+ 1. **Convention Fingerprinting** — AST-analyze codebase, extract convention profiles, validate new code against them. Prior art: NATURALIZE (94% accuracy on naming), similarity-ts, IntelliCode.
64
+ 2. **Architectural Boundary Enforcement** — Define allowed dependency directions, validate import graph. Prior art: Nx, Sheriff, Dependency Cruiser.
65
+ 3. **"Guardrails as Tests"** — Lightweight DSL for structural code assertions via AST matching. Prior art: Semgrep, ast-grep.
66
+ 4. **PR Review Mining** — Cluster repeated review comments, auto-generate instruction entries. TF-IDF + cosine similarity (no LLM needed).
67
+ 5. **API Hallucination Detector** — Validate API calls against knowledge base via AST. Prior art: [arxiv 2601.19106](https://arxiv.org/html/2601.19106v1) — 100% precision, 87.6% recall.
68
+ 6. **Convention Compliance Scoring** — Per-PR audit of which instructions are followed. Cross-reference diff against rules.
69
+ 7. **Deprecation/Migration Fence** — Block deprecated patterns in new code, allow in existing. "allow-existing, block-new" semantic.
70
+ 8. **Context Sufficiency Validator** — Check instruction files reference real things, aren't vague.
71
+ 9. **Executable Examples** — Type-check code blocks in instruction files via tsc/py_compile/shellcheck.
72
+ 10. **Instruction Effectiveness Tracking** — Correlate rules with linter failures and review comments over time.
73
+
74
+ ### 4 Bold Bets
75
+
76
+ 1. **Executable Spec Files** → CHOSEN. TypeScript specs that compile to markdown. The spec IS the source of truth.
77
+ 2. **Convention Genome** → DEFERRED. Extract conventions from AST, encode as machine-readable genome. Promising but large research project.
78
+ 3. **Proof-Carrying Code** → REJECTED. Agents produce code + verification receipts. On reflection, this is just ESLint with extra steps.
79
+ 4. **Type System for Instructions** → ABSORBED into idea 1. Rich typed annotations become TypeScript types in the spec.
80
+
81
+ ---
82
+
83
+ ## Key Design Decisions
84
+
85
+ ### 1. Compile to markdown, don't lint markdown
86
+
87
+ The TypeScript type system catches errors at authoring time. `enforce()` must be called with a valid `LinterRule` template literal type. `file()` returns a `VerifiedPath` branded type. The compiler validates everything else. Nobody else does this.
88
+
89
+ ### 2. Three rule types: enforce, check, guidance
90
+
91
+ | Type | Builder | Responsibility |
92
+ | --------- | ------------ | ------------------------------------------------------------------------- |
93
+ | Delegated | `enforce()` | External tool (linter, ast-grep, etc.). vigiles verifies it's configured. |
94
+ | Checked | `check()` | Filesystem assertions only. vigiles owns what no other tool handles. |
95
+ | Guidance | `guidance()` | Prose. No enforcement pretended. |
96
+
97
+ **Originally had `prove()` with `no().matches()` and `layers()`.** Killed these — ast-grep, Dependency Cruiser, and Steiger already do architectural linting better. vigiles references their rules via `enforce()` instead of reimplementing them.
98
+
99
+ ### 3. Don't be an architectural linter
100
+
101
+ Existing tools that already handle this:
102
+
103
+ | Tool | Languages | What it does |
104
+ | -------------------------------------------------------------------- | ------------ | ------------------------------------------ |
105
+ | [ast-grep](https://ast-grep.github.io/) | 26 languages | Structural search/lint/rewrite, YAML rules |
106
+ | [Steiger](https://github.com/feature-sliced/steiger) | JS/TS | File structure + architecture linting |
107
+ | [Dependency Cruiser](https://github.com/sverweij/dependency-cruiser) | JS/TS | Import graph validation |
108
+ | [Sheriff](https://github.com/nicedoc/sheriff) | TS | Module boundary enforcement |
109
+ | Nx `enforce-module-boundaries` | JS/TS | Layer/tag-based import rules |
110
+ | eslint-plugin-boundaries | JS/TS | Import boundary rules in ESLint |
111
+
112
+ vigiles can reference all of these via `enforce("ast-grep/rule-name")` etc.
113
+
114
+ ### 4. generate-types as second moat
115
+
116
+ `vigiles generate-types` scans all 6 linter APIs + package.json + filesystem → emits `.d.ts`:
117
+
118
+ ```typescript
119
+ // .vigiles/generated.d.ts (auto-generated)
120
+ export type EslintRule = "no-console" | "no-unused-vars" | ...;
121
+ export type RuffRule = "E501" | "F401" | ...;
122
+ export type NpmScript = "build" | "test" | "fmt" | ...;
123
+ export type ProjectFile = "src/spec.ts" | "src/compile.ts" | ...;
124
+ ```
125
+
126
+ TS compiler proves references valid at authoring time. Like Prisma for databases, tRPC for APIs. Nobody else does this for linter rules.
127
+
128
+ ### 5. Branded types for references
129
+
130
+ `file()` returns `VerifiedPath`, not `string`. `cmd()` returns `VerifiedCmd`. Distinguishes "went through vigiles verification" from "random string" at the type level.
131
+
132
+ ### 6. Colocated spec files, not god-object config
133
+
134
+ - `CLAUDE.md.spec.ts` sits next to `CLAUDE.md`
135
+ - `SKILL.md.spec.ts` sits next to `SKILL.md`
136
+ - Hooks (.claude/settings.json) validated in place — no spec needed
137
+ - `vigiles.config.ts` only for project settings (maxRules, maxTokens)
138
+
139
+ ### 7. Token budget on compiled output
140
+
141
+ `maxTokens` constrains the compiled artifact. The spec can be any length. Estimated via ~4 chars/token heuristic (swappable for real tokenizer).
142
+
143
+ ### 8. SHA-256 hash on compiled output
144
+
145
+ Detects manual edits. `vigiles adopt` (future) to merge manual changes back into spec.
146
+
147
+ ---
148
+
149
+ ## Technology Choices
150
+
151
+ | Tool | Decision | Rationale |
152
+ | ------------------------------- | ----------------------------------------- | ----------------------------------------------------------- |
153
+ | **ast-grep** (`@ast-grep/napi`) | Future dependency for `check()` if needed | 26 languages, Rust-fast, Node.js API |
154
+ | **Semgrep** | REJECTED | Can't embed (OCaml binary), LGPL + restrictive rule license |
155
+ | **tree-sitter** | Too low-level | ast-grep wraps it with better API |
156
+ | **ESLint v10** | Potential output target | Inline plugins via flat config |
157
+ | **Pulumi model** | Architectural inspiration | TS code → declarative output |
158
+ | **Effect.ts Schema** | Pattern inspiration | Inspectable values = executable + documentable |
159
+
160
+ ---
161
+
162
+ ## Competitive Landscape
163
+
164
+ 15+ tools in the space across 4 categories. See `research/competitive-landscape.md` for full analysis.
165
+
166
+ **vigiles moat (v2):**
167
+
168
+ 1. Spec compilation (nobody else)
169
+ 2. Linter cross-referencing — 6 APIs (nobody else)
170
+ 3. Type generation from project state (nobody else)
171
+ 4. Branded reference types
172
+ 5. SHA-256 integrity hash
173
+
174
+ ---
175
+
176
+ ## What's NOT Worth Building
177
+
178
+ Based on research, these approaches don't work or are somebody else's job:
179
+
180
+ - **AI-in-CI for code review** — non-deterministic, can't be a gate. What works: AI writes the rules, deterministic tools run them.
181
+ - **Architectural linting** — ast-grep, Steiger, Dependency Cruiser do this better.
182
+ - **Markdown formatting** — markdownlint. We generate the markdown, structure is correct by construction.
183
+ - **Prose quality validation** — detecting "write clean code" as vague is cute but fragile. Better to just not generate it from specs.
184
+ - **Skill testing via LLM execution** — non-deterministic, expensive, side-effect-prone. Skill contracts are unreliable.
185
+ - **Per-file code linting** — that's what ESLint/Ruff/Clippy are for.
186
+
187
+ ---
188
+
189
+ ## Future Work
190
+
191
+ | Feature | Status | Notes |
192
+ | --------------------------- | ------------- | --------------------------------------------------------------------------- |
193
+ | `vigiles discover` | Not started | Scan linter configs, report coverage gaps, suggest rules to document |
194
+ | `vigiles adopt` | Not started | Detect manual edits via hash, merge changes back into spec |
195
+ | `vigiles.config.ts` loading | Not started | Needs jiti or tsx for TS config evaluation |
196
+ | Convention Genome | Research only | Extract conventions from AST, encode as types. High potential, large scope. |
197
+ | Real tokenizer | Not started | Replace ~4 chars/token heuristic with BPE (tiktoken/gpt-tokenizer) |