vigiles 8.0.0 → 9.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +63 -35
- package/dist/adoptability.d.ts +55 -0
- package/dist/adoptability.js +196 -0
- package/dist/audit-html.d.ts +20 -0
- package/dist/audit-html.js +61 -0
- package/dist/audit-prompts.d.ts +46 -0
- package/dist/audit-prompts.js +90 -0
- package/dist/audit-report.d.ts +70 -0
- package/dist/audit-report.js +51 -0
- package/dist/audit-report.template.html +110 -0
- package/dist/audit-score.d.ts +44 -0
- package/dist/audit-score.js +221 -0
- package/dist/cli-commands.d.ts +1 -1
- package/dist/cli-commands.js +1 -1
- package/dist/cli.js +340 -111
- package/dist/core/inline.js +11 -1
- package/dist/core/types.d.ts +12 -0
- package/dist/dialect-drift.js +1 -1
- package/dist/eval.d.ts +1 -1
- package/dist/eval.js +1 -1
- package/dist/optimize.d.ts +12 -5
- package/dist/optimize.js +27 -5
- package/dist/scan-behavioral.d.ts +8 -2
- package/dist/scan-behavioral.js +6 -4
- package/dist/scan-trigger-suggest.d.ts +68 -31
- package/dist/scan-trigger-suggest.js +66 -33
- package/dist/scan.d.ts +36 -11
- package/dist/scan.js +60 -14
- package/dist/score-explainer.d.ts +1 -1
- package/package.json +4 -2
- package/skills/test-harness/SKILL.md +1 -1
package/README.md
CHANGED
|
@@ -22,12 +22,14 @@
|
|
|
22
22
|
caveat, or a competitor, never the user's pain. SAME FOR THE SUBDOCS: open
|
|
23
23
|
every guide with the concrete pain, THEN the "what this doc is" line + the
|
|
24
24
|
README uplink (per docs-quality in CLAUDE.md).
|
|
25
|
-
2.
|
|
26
|
-
a
|
|
27
|
-
author + edit it,
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
25
|
+
2. SPEC-FIRST IS THE DEFAULT — but easy, never intimidating. `init` adopts your
|
|
26
|
+
existing CLAUDE.md INTO a spec, and model-invocable skills (edit-spec /
|
|
27
|
+
strengthen / test-harness) author + edit it, so you rarely hand-write a
|
|
28
|
+
.spec.ts and hooks auto-compile on save. Present the typed spec as the default
|
|
29
|
+
the agent manages for you, NOT an advanced opt-in or a "step up". Inline
|
|
30
|
+
markdown is the ZERO-TS FLOOR for anyone who skips `init` (progressive
|
|
31
|
+
adoption) — the on-ramp, not the default starting point. `eject` always
|
|
32
|
+
reverses. NEVER a wall.
|
|
31
33
|
3. THE INSTRUMENTS stay first-class — including Eval (measuring whether a skill
|
|
32
34
|
actually helps is core, not optional). NOTE: Guard / compiled hooks is PARKED
|
|
33
35
|
FOR LAUNCH (commented out below; see research/roadmap.md "Launch readiness") —
|
|
@@ -58,17 +60,41 @@
|
|
|
58
60
|
---
|
|
59
61
|
|
|
60
62
|
**You installed a bunch of plugins and wrote a few skills — but do they actually work?**
|
|
61
|
-
A skill that never fires, a
|
|
62
|
-
references — your harness fails **silently**, and you find out mid-task.
|
|
63
|
+
A skill that never fires, a subagent wired to a tool that doesn't exist, a CLAUDE.md
|
|
64
|
+
full of dead references — your harness fails **silently**, and you find out mid-task.
|
|
63
65
|
|
|
64
66
|
**It's a library with no tests.**
|
|
65
67
|
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
68
|
+
**One command shows you — like a Lighthouse report for your harness:**
|
|
69
|
+
|
|
70
|
+
```bash
|
|
71
|
+
npx vigiles audit # no key, no config, safe to run anywhere
|
|
72
|
+
```
|
|
73
|
+
|
|
74
|
+
It **reads** your harness deterministically — every reference checked, four categories
|
|
75
|
+
scored, each finding's fix inline — and writes a shareable HTML report:
|
|
70
76
|
|
|
71
|
-
|
|
77
|
+
<p align="center">
|
|
78
|
+
<img src="vigiles-audit.png" width="760" alt="vigiles audit report: an overall score with four category rings — Truthfulness, Triggering, Structure, Tested — and fix cards" />
|
|
79
|
+
</p>
|
|
80
|
+
|
|
81
|
+
| Ring | What it proves |
|
|
82
|
+
| ------------------- | --------------------------------------------------------------------- |
|
|
83
|
+
| **🔎 Truthfulness** | Every path / script / symbol / linter rule in your CLAUDE.md resolves |
|
|
84
|
+
| **🎯 Triggering** | Skills fire on the right prompts and don't collide |
|
|
85
|
+
| **🔧 Structure** | Tool contracts, MCP servers & frontmatter are sound |
|
|
86
|
+
| **🧪 Tested** | Every surface ships a test |
|
|
87
|
+
|
|
88
|
+
Like Lighthouse, `audit` is a **local report you run on your machine** — a
|
|
89
|
+
deterministic read, safe on any repo (even one wired to prod), identical on every
|
|
90
|
+
OS. **Not a CI step** (CI uses `lint`, the deterministic gate). The **executing
|
|
91
|
+
checks** (resolve MCP live · **do your skills fire?** · **what would vigiles
|
|
92
|
+
catch in your repo?**) run only when there's a human to consent: `audit` **asks
|
|
93
|
+
once** at a terminal and remembers. **[Audit a harness →](docs/for-plugin-authors.md)**
|
|
94
|
+
|
|
95
|
+
`Agent = Model + Harness` — the model gets the headlines, the harness is the half you
|
|
96
|
+
own. vigiles[^name] is how you make it prove itself: `audit` is the dashboard, and
|
|
97
|
+
**three instruments** fix and prove what it finds —
|
|
72
98
|
|
|
73
99
|
| | |
|
|
74
100
|
| ----------- | ----------------------------------------------------------------------------------------------------------------------------------------------- |
|
|
@@ -80,24 +106,17 @@ Three instruments, adopt any:
|
|
|
80
106
|
| **🛡 Guard** | A safety hook that **can't silently fail open** — write a typed function, get a guard that blocks. **[→](docs/compiled-hooks.md)** |
|
|
81
107
|
-->
|
|
82
108
|
|
|
83
|
-
<p align="center">
|
|
84
|
-
<img src="vigiles-demo.gif" width="720" alt="vigiles lint catching a file that moved and a script that was renamed" />
|
|
85
|
-
</p>
|
|
86
|
-
<!-- Regenerate the GIF: `python3 scripts/make-demo-gif.py` (output is verbatim CLI; see scripts/demo.sh for a live asciinema recording). -->
|
|
87
|
-
|
|
88
109
|
**Two ways in** — pick the pain that's yours:
|
|
89
110
|
|
|
90
|
-
- **Run agents on your own repo?**
|
|
91
|
-
- **Ship plugins to a marketplace?**
|
|
92
|
-
|
|
111
|
+
- **Run agents on your own repo?** `npx vigiles audit`, then `npx vigiles init`.
|
|
112
|
+
- **Ship plugins to a marketplace?** `npx vigiles audit ./plugins/*/` ranks a whole
|
|
113
|
+
marketplace (0–100, A–F) — see the **[plugin-author guide →](docs/for-plugin-authors.md)**.
|
|
93
114
|
|
|
94
|
-
**Your agent writes the spec — and you can always eject.**
|
|
95
|
-
`.spec.ts
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
harness](docs/authoring-an-adapter.md). _(New here? [the FAQ](docs/faq.md) covers the
|
|
100
|
-
"do I write TypeScript / will it touch my files" questions.)_
|
|
115
|
+
**Your agent writes the spec — and you can always eject.** Skills author the
|
|
116
|
+
`.spec.ts` for you, **`init` adopts an existing CLAUDE.md non-destructively**
|
|
117
|
+
(untouched until you compile), and plain markdown + inline `<!-- vigiles:enforce -->`
|
|
118
|
+
comments work with zero TypeScript. Works with **Claude Code and Codex**
|
|
119
|
+
([`vigiles/codex`](docs/harnesses.md)) or [your own harness](docs/authoring-an-adapter.md).
|
|
101
120
|
|
|
102
121
|
## Quick start
|
|
103
122
|
|
|
@@ -156,12 +175,18 @@ CLAUDE.md:
|
|
|
156
175
|
✓ @typescript-eslint/no-floating-promises — exists and enabled in eslint config
|
|
157
176
|
```
|
|
158
177
|
|
|
178
|
+
<p align="center">
|
|
179
|
+
<img src="vigiles-demo.gif" width="720" alt="vigiles lint catching a file that moved and a script that was renamed" />
|
|
180
|
+
</p>
|
|
181
|
+
<!-- Regenerate the GIF: `python3 scripts/make-demo-gif.py` (output is verbatim CLI; see scripts/demo.sh for a live asciinema recording). -->
|
|
182
|
+
|
|
159
183
|
File paths, scripts, code symbols — plus linter rules across **7 linters**
|
|
160
184
|
(ESLint, Ruff, Clippy + four more): each rule exists **and is enabled**.
|
|
161
185
|
|
|
162
|
-
**
|
|
163
|
-
|
|
164
|
-
|
|
186
|
+
**Your agent writes the spec — `init` adopts your existing CLAUDE.md into one,
|
|
187
|
+
faithfully and non-destructively.** Prefer zero new files? Plain markdown + one
|
|
188
|
+
inline `<!-- vigiles:enforce -->` comment lints too — no spec, no TypeScript. And
|
|
189
|
+
`vigiles eject` hands a spec back to markdown anytime.
|
|
165
190
|
**[Full guide →](docs/verifying-instruction-files.md)**
|
|
166
191
|
|
|
167
192
|
> **Want more? Bad states can stop compiling.** Opt in deeper and a broken
|
|
@@ -174,7 +199,9 @@ guarantees — and `vigiles eject` hands it back to plain markdown anytime.
|
|
|
174
199
|
**You wired the hook — but does it actually block?** A skill's description can fail
|
|
175
200
|
to trigger, or hijack unrelated prompts; injected context can silently never reach
|
|
176
201
|
the model. All of it passes a naive "did it run?" check. vigiles tests the
|
|
177
|
-
assembled harness for real
|
|
202
|
+
assembled harness for real.
|
|
203
|
+
|
|
204
|
+
Start with the cheapest tier — a hook, called directly. No model, no key:
|
|
178
205
|
|
|
179
206
|
```typescript
|
|
180
207
|
import { runHook } from "vigiles/testing";
|
|
@@ -191,18 +218,19 @@ It goes well past _"did it fire?"_:
|
|
|
191
218
|
|
|
192
219
|
- **Hooks block** what they must — `runHook`, or the real agent CLI via `runHarnessTest`.
|
|
193
220
|
- **Skills trigger** on the right prompts and stay quiet on the wrong ones — recall _and_ precision (`measureTriggerRate`).
|
|
221
|
+
- **Subagents finish right** — assert a subagent ended in the success (or error) outcome it promised, with a plain check, no LLM judge (`assertAgentOk` / `assertAgentErr`).
|
|
194
222
|
- **Behaviour is good** — score a skill's output, or A/B it on-vs-off for the real lift (`measure` / `runEval`, with significance testing).
|
|
195
223
|
- **Safety holds** — the agent _didn't_ push to the wrong branch or hit a paid API; `interceptTools` catches the attempt so the side effect never happens.
|
|
196
224
|
|
|
197
225
|
Almost every tier runs with **no model and no API key** — milliseconds, on every
|
|
198
|
-
commit; only the real-model
|
|
226
|
+
commit; only the real-model tier needs a model, on your own `claude` CLI.
|
|
199
227
|
**[How it works →](docs/harness-testing.md)**
|
|
200
228
|
|
|
201
229
|
## ③ Eval — does it actually help, or just cost more?
|
|
202
230
|
|
|
203
231
|
**"65% fewer tokens." "3× faster." Says who?** A skill claims it, a plugin promises
|
|
204
232
|
it — stars and vibes, **zero measurement**. vigiles A/Bs the claim on real coding
|
|
205
|
-
tasks, the harness loaded exactly as it ships, and reports
|
|
233
|
+
tasks, the harness loaded exactly as it ships, and reports **three numbers**:
|
|
206
234
|
|
|
207
235
|
```typescript
|
|
208
236
|
import { measureArms } from "vigiles/testing";
|
|
@@ -268,7 +296,7 @@ default, not an unbypassable wall._
|
|
|
268
296
|
## FAQ
|
|
269
297
|
|
|
270
298
|
- **Isn't this just a markdown linter?** No — it checks whether your instruction file is _true_ (every path/script/symbol/rule exists and is enabled), then tests and measures your harness. A style linter can't do any of that.
|
|
271
|
-
- **Do I have to write TypeScript?** No
|
|
299
|
+
- **Do I have to write TypeScript?** No — your agent writes the spec (`init` adopts your CLAUDE.md into one). Prefer zero new files? Plain markdown lints too. The deeper compiler-grade guarantees are the gradual, opt-in part — like TS's `strict`.
|
|
272
300
|
- **Does it overwrite my files?** No. `init` adopts an existing CLAUDE.md _non-destructively_ — untouched until you `compile`, and `eject` reverses it.
|
|
273
301
|
- **Need an API key?** No for almost everything (free, every commit). Real-model evals run on your Claude Pro/Max subscription — $0 metered tokens.
|
|
274
302
|
- **Non-JS repo?** `npx vigiles lint` verifies your CLAUDE.md with no install (Ruff/Clippy/Pylint/… too).
|
|
@@ -0,0 +1,55 @@
|
|
|
1
|
+
import { type AgentRunner, type ModelOutputParser } from "./eval.js";
|
|
2
|
+
/** A reference the model proposes as machine-verifiable. */
|
|
3
|
+
export interface DraftedRef {
|
|
4
|
+
readonly kind: "enforce" | "file" | "cmd" | "dir";
|
|
5
|
+
readonly ref: string;
|
|
6
|
+
}
|
|
7
|
+
/** A drafted ref that failed verification — the value proof. */
|
|
8
|
+
export interface BrokenRef {
|
|
9
|
+
readonly kind: DraftedRef["kind"];
|
|
10
|
+
readonly ref: string;
|
|
11
|
+
readonly issue: string;
|
|
12
|
+
}
|
|
13
|
+
export interface AdoptabilityResult {
|
|
14
|
+
/** Distinct verifiable references the model found (the surface a spec would protect). */
|
|
15
|
+
readonly total: number;
|
|
16
|
+
/** How many of those are broken in this repo right now. */
|
|
17
|
+
readonly broken: number;
|
|
18
|
+
readonly brokenRefs: readonly BrokenRef[];
|
|
19
|
+
}
|
|
20
|
+
/**
|
|
21
|
+
* Deterministic verdict over drafted refs — the "disposes" half. Dedupes
|
|
22
|
+
* (kind+ref), routes each to the real cross-ref/filesystem check, and counts the
|
|
23
|
+
* broken. Pure: a hallucinated rule resolves to broken, never trusted as a pass.
|
|
24
|
+
*/
|
|
25
|
+
export declare function verifyDraftedRefs(refs: readonly DraftedRef[], basePath: string): AdoptabilityResult;
|
|
26
|
+
/**
|
|
27
|
+
* Tolerant parse of the model's draft output into `DraftedRef[]`. The model is
|
|
28
|
+
* asked for a bare JSON array, but tolerate prose-wrapped / fenced output by
|
|
29
|
+
* extracting the outermost `[...]`. Drops any entry with an unknown kind or a
|
|
30
|
+
* non-string ref (the verifier is the guard, but a malformed shape is just noise).
|
|
31
|
+
*/
|
|
32
|
+
export declare function parseDraftJson(text: string): DraftedRef[];
|
|
33
|
+
/** Options for the real model draft (the one v8-ignored seam). */
|
|
34
|
+
export interface DraftOptions {
|
|
35
|
+
readonly model?: string;
|
|
36
|
+
readonly cwd?: string;
|
|
37
|
+
readonly runner?: AgentRunner;
|
|
38
|
+
readonly parse?: ModelOutputParser;
|
|
39
|
+
}
|
|
40
|
+
/** Injectable drafter — the real one calls a model; tests pass a fake. */
|
|
41
|
+
export type Drafter = (content: string) => Promise<DraftedRef[]>;
|
|
42
|
+
export interface AdoptabilityTierOptions {
|
|
43
|
+
readonly instructionContent: string;
|
|
44
|
+
readonly basePath: string;
|
|
45
|
+
/** Injectable for tests; defaults to the real one-shot model draft. */
|
|
46
|
+
readonly draft?: Drafter;
|
|
47
|
+
}
|
|
48
|
+
/**
|
|
49
|
+
* Run the preview: draft refs from the instruction file (model), then verify them
|
|
50
|
+
* (deterministic). The composition root of "LLM proposes, deterministic disposes".
|
|
51
|
+
*/
|
|
52
|
+
export declare function runAdoptabilityTier(opts: AdoptabilityTierOptions): Promise<AdoptabilityResult>;
|
|
53
|
+
/** Terminal section — the adoption invitation, not a graded ring. */
|
|
54
|
+
export declare function formatAdoptability(r: AdoptabilityResult, instructionFile: string): string;
|
|
55
|
+
//# sourceMappingURL=adoptability.d.ts.map
|
|
@@ -0,0 +1,196 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.verifyDraftedRefs = verifyDraftedRefs;
|
|
4
|
+
exports.parseDraftJson = parseDraftJson;
|
|
5
|
+
exports.runAdoptabilityTier = runAdoptabilityTier;
|
|
6
|
+
exports.formatAdoptability = formatAdoptability;
|
|
7
|
+
/**
|
|
8
|
+
* Adoptability preview — "what would vigiles catch in YOUR repo?"
|
|
9
|
+
*
|
|
10
|
+
* The audit's adoption front door for a NON-adopter: instead of grading the hygiene
|
|
11
|
+
* of an already-adopted spec, it shows the concrete bugs a spec WOULD catch today.
|
|
12
|
+
*
|
|
13
|
+
* Architecture (research/adoption-gateway-preview.md): **LLM proposes, deterministic
|
|
14
|
+
* disposes.** A model DRAFTS the verifiable references in an instruction file (high
|
|
15
|
+
* recall, incl. prose intent a regex can't see — `draftRefs`); the deterministic
|
|
16
|
+
* cross-reference engine VERIFIES each one (`verifyDraftedRefs`, reusing
|
|
17
|
+
* `checkLinterRule` + the compile validators). The model never gets to assert a
|
|
18
|
+
* pass — only the verifier does — so the "M broken right now" number is trustworthy
|
|
19
|
+
* even though the extraction was probabilistic.
|
|
20
|
+
*
|
|
21
|
+
* The verifier + parser + formatter are pure and model-free (fully unit-tested); the
|
|
22
|
+
* single real model call (`defaultDraft`) is the only v8-ignored seam, injected so
|
|
23
|
+
* the orchestration is testable without a model.
|
|
24
|
+
*/
|
|
25
|
+
const linters_js_1 = require("./core/linters.js");
|
|
26
|
+
const compile_js_1 = require("./core/compile.js");
|
|
27
|
+
const hash_js_1 = require("./core/hash.js");
|
|
28
|
+
const eval_js_1 = require("./eval.js");
|
|
29
|
+
/** Verify ONE drafted ref against the real repo; null = resolves, else the breakage. */
|
|
30
|
+
function verifyOne(r, basePath) {
|
|
31
|
+
switch (r.kind) {
|
|
32
|
+
case "enforce": {
|
|
33
|
+
const res = (0, linters_js_1.checkLinterRule)(r.ref, basePath);
|
|
34
|
+
if (!res.exists)
|
|
35
|
+
return {
|
|
36
|
+
...r,
|
|
37
|
+
issue: res.error ?? `linter rule "${r.ref}" does not exist`,
|
|
38
|
+
};
|
|
39
|
+
if (res.enabled === "disabled")
|
|
40
|
+
return { ...r, issue: `rule "${r.ref}" exists but is not enabled` };
|
|
41
|
+
return null;
|
|
42
|
+
}
|
|
43
|
+
case "file": {
|
|
44
|
+
const e = (0, compile_js_1.validateFileRef)(r.ref, basePath);
|
|
45
|
+
return e ? { kind: r.kind, ref: r.ref, issue: e.message } : null;
|
|
46
|
+
}
|
|
47
|
+
case "cmd": {
|
|
48
|
+
const e = (0, compile_js_1.validateCommandRef)(r.ref, basePath);
|
|
49
|
+
return e ? { kind: r.kind, ref: r.ref, issue: e.message } : null;
|
|
50
|
+
}
|
|
51
|
+
case "dir": {
|
|
52
|
+
const e = (0, compile_js_1.validateDirRef)(r.ref, basePath);
|
|
53
|
+
return e ? { kind: r.kind, ref: r.ref, issue: e.message } : null;
|
|
54
|
+
}
|
|
55
|
+
default:
|
|
56
|
+
return (0, hash_js_1.assertNever)(r.kind);
|
|
57
|
+
}
|
|
58
|
+
}
|
|
59
|
+
/**
|
|
60
|
+
* Deterministic verdict over drafted refs — the "disposes" half. Dedupes
|
|
61
|
+
* (kind+ref), routes each to the real cross-ref/filesystem check, and counts the
|
|
62
|
+
* broken. Pure: a hallucinated rule resolves to broken, never trusted as a pass.
|
|
63
|
+
*/
|
|
64
|
+
function verifyDraftedRefs(refs, basePath) {
|
|
65
|
+
const seen = new Set();
|
|
66
|
+
const unique = [];
|
|
67
|
+
for (const r of refs) {
|
|
68
|
+
const key = `${r.kind}:${r.ref}`;
|
|
69
|
+
if (seen.has(key))
|
|
70
|
+
continue;
|
|
71
|
+
seen.add(key);
|
|
72
|
+
unique.push(r);
|
|
73
|
+
}
|
|
74
|
+
const brokenRefs = unique
|
|
75
|
+
.map((r) => verifyOne(r, basePath))
|
|
76
|
+
.filter((b) => b !== null);
|
|
77
|
+
return { total: unique.length, broken: brokenRefs.length, brokenRefs };
|
|
78
|
+
}
|
|
79
|
+
const VALID_KINDS = new Set([
|
|
80
|
+
"enforce",
|
|
81
|
+
"file",
|
|
82
|
+
"cmd",
|
|
83
|
+
"dir",
|
|
84
|
+
]);
|
|
85
|
+
/**
|
|
86
|
+
* Tolerant parse of the model's draft output into `DraftedRef[]`. The model is
|
|
87
|
+
* asked for a bare JSON array, but tolerate prose-wrapped / fenced output by
|
|
88
|
+
* extracting the outermost `[...]`. Drops any entry with an unknown kind or a
|
|
89
|
+
* non-string ref (the verifier is the guard, but a malformed shape is just noise).
|
|
90
|
+
*/
|
|
91
|
+
function parseDraftJson(text) {
|
|
92
|
+
const raw = extractJsonArray(text);
|
|
93
|
+
if (raw === null)
|
|
94
|
+
return [];
|
|
95
|
+
let parsed;
|
|
96
|
+
try {
|
|
97
|
+
parsed = JSON.parse(raw);
|
|
98
|
+
}
|
|
99
|
+
catch {
|
|
100
|
+
return [];
|
|
101
|
+
}
|
|
102
|
+
if (!Array.isArray(parsed))
|
|
103
|
+
return [];
|
|
104
|
+
const out = [];
|
|
105
|
+
for (const item of parsed) {
|
|
106
|
+
if (typeof item !== "object" || item === null)
|
|
107
|
+
continue;
|
|
108
|
+
const rec = item;
|
|
109
|
+
const kind = rec.kind;
|
|
110
|
+
const ref = rec.ref;
|
|
111
|
+
if (typeof kind === "string" &&
|
|
112
|
+
VALID_KINDS.has(kind) &&
|
|
113
|
+
typeof ref === "string" &&
|
|
114
|
+
ref.trim()) {
|
|
115
|
+
out.push({ kind: kind, ref: ref.trim() });
|
|
116
|
+
}
|
|
117
|
+
}
|
|
118
|
+
return out;
|
|
119
|
+
}
|
|
120
|
+
/** Pull the outermost `[...]` from a possibly prose/fence-wrapped string. */
|
|
121
|
+
function extractJsonArray(text) {
|
|
122
|
+
const start = text.indexOf("[");
|
|
123
|
+
const end = text.lastIndexOf("]");
|
|
124
|
+
if (start === -1 || end === -1 || end < start)
|
|
125
|
+
return null;
|
|
126
|
+
return text.slice(start, end + 1);
|
|
127
|
+
}
|
|
128
|
+
/** The drafting prompt — reuse the strengthen/adopt-spec mapping intent. */
|
|
129
|
+
function draftPrompt(content) {
|
|
130
|
+
return [
|
|
131
|
+
"You are evaluating whether a coding-agent instruction file's references can be",
|
|
132
|
+
"machine-verified. Read the instruction file and identify every reference to a",
|
|
133
|
+
"CONCRETE, VERIFIABLE artifact:",
|
|
134
|
+
'- a linter rule (kind "enforce", ref like "eslint/no-console" or',
|
|
135
|
+
' "@typescript-eslint/no-floating-promises") — INCLUDING prose intent you can',
|
|
136
|
+
' confidently map to a real rule (e.g. "always await promises" ->',
|
|
137
|
+
' "@typescript-eslint/no-floating-promises", "no console.log" -> "eslint/no-console").',
|
|
138
|
+
'- a file path (kind "file", ref like "src/index.ts").',
|
|
139
|
+
'- an npm script (kind "cmd", ref like "npm run build" or "npm test").',
|
|
140
|
+
'- a directory (kind "dir", ref like "src/components").',
|
|
141
|
+
"",
|
|
142
|
+
'Output ONLY a JSON array of {"kind","ref"} objects — no markdown, no prose. If',
|
|
143
|
+
"none, output []. Do not invent references that aren't grounded in the text.",
|
|
144
|
+
"",
|
|
145
|
+
"Instruction file:",
|
|
146
|
+
"---",
|
|
147
|
+
content,
|
|
148
|
+
"---",
|
|
149
|
+
].join("\n");
|
|
150
|
+
}
|
|
151
|
+
/* v8 ignore start — the single real model call; the orchestration is tested with a fake draft. */
|
|
152
|
+
/** The "proposes" half: one model call drafting the verifiable refs from prose. */
|
|
153
|
+
async function defaultDraft(content, opts = {}) {
|
|
154
|
+
const runner = opts.runner ?? eval_js_1.spawnAgent;
|
|
155
|
+
const parse = opts.parse ?? eval_js_1.parseClaudeRun;
|
|
156
|
+
const out = await runner({
|
|
157
|
+
task: draftPrompt(content),
|
|
158
|
+
cwd: opts.cwd ?? process.cwd(),
|
|
159
|
+
model: opts.model ?? "sonnet",
|
|
160
|
+
tools: [], // the content is inline — no file tools needed (deterministic-ish)
|
|
161
|
+
hasSettings: false,
|
|
162
|
+
pluginDir: undefined,
|
|
163
|
+
timeoutMs: 120000,
|
|
164
|
+
env: process.env,
|
|
165
|
+
});
|
|
166
|
+
return parseDraftJson(parse(out).output);
|
|
167
|
+
}
|
|
168
|
+
/**
|
|
169
|
+
* Run the preview: draft refs from the instruction file (model), then verify them
|
|
170
|
+
* (deterministic). The composition root of "LLM proposes, deterministic disposes".
|
|
171
|
+
*/
|
|
172
|
+
async function runAdoptabilityTier(opts) {
|
|
173
|
+
const draft = opts.draft ?? ((c) => defaultDraft(c));
|
|
174
|
+
const refs = await draft(opts.instructionContent);
|
|
175
|
+
return verifyDraftedRefs(refs, opts.basePath);
|
|
176
|
+
}
|
|
177
|
+
/** Terminal section — the adoption invitation, not a graded ring. */
|
|
178
|
+
function formatAdoptability(r, instructionFile) {
|
|
179
|
+
const lines = ["Adoptability — what vigiles would lock in"];
|
|
180
|
+
if (r.total === 0) {
|
|
181
|
+
lines.push(` no machine-verifiable references found in ${instructionFile}.`);
|
|
182
|
+
return lines.join("\n");
|
|
183
|
+
}
|
|
184
|
+
lines.push(` vigiles drafted a spec from ${instructionFile}: ${String(r.total)} verifiable reference(s) found`);
|
|
185
|
+
if (r.broken === 0) {
|
|
186
|
+
lines.push(" ✓ all resolve right now — adopt a spec to keep it that way.");
|
|
187
|
+
return lines.join("\n");
|
|
188
|
+
}
|
|
189
|
+
lines.push(` ${String(r.broken)} broken right now:`);
|
|
190
|
+
for (const b of r.brokenRefs) {
|
|
191
|
+
lines.push(` ✗ ${b.issue}`);
|
|
192
|
+
}
|
|
193
|
+
lines.push(" → run `vigiles init` to adopt the spec and catch these at edit time.");
|
|
194
|
+
return lines.join("\n");
|
|
195
|
+
}
|
|
196
|
+
//# sourceMappingURL=adoptability.js.map
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
import type { AuditReport } from "./audit-report.js";
|
|
2
|
+
/**
|
|
3
|
+
* Candidate locations for the built template, relative to this module (`__dirname`
|
|
4
|
+
* is the compiled `dist/` at runtime, or `src/` under vitest). CommonJS output, so
|
|
5
|
+
* we use `__dirname`, not `import.meta`.
|
|
6
|
+
*/
|
|
7
|
+
export declare function templatePath(): string | null;
|
|
8
|
+
/**
|
|
9
|
+
* Inject the report JSON into a template by replacing the quoted placeholder
|
|
10
|
+
* string with the JSON object literal. Pure — the testable core. Throws if the
|
|
11
|
+
* template is missing the placeholder.
|
|
12
|
+
*/
|
|
13
|
+
export declare function injectReportData(template: string, report: AuditReport): string;
|
|
14
|
+
/**
|
|
15
|
+
* Render the self-contained HTML report (React template + injected data). Throws
|
|
16
|
+
* if the template hasn't been built — the caller (writeAuditHtml) catches that and
|
|
17
|
+
* skips the HTML, since the JSON + terminal report don't depend on it.
|
|
18
|
+
*/
|
|
19
|
+
export declare function renderAuditHtml(report: AuditReport): string;
|
|
20
|
+
//# sourceMappingURL=audit-html.d.ts.map
|
|
@@ -0,0 +1,61 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.templatePath = templatePath;
|
|
4
|
+
exports.injectReportData = injectReportData;
|
|
5
|
+
exports.renderAuditHtml = renderAuditHtml;
|
|
6
|
+
/**
|
|
7
|
+
* The shareable HTML audit report — the prebuilt **Vite + React + shadcn** template
|
|
8
|
+
* (`report/`, built to one self-contained file at `dist/audit-report.template.html`)
|
|
9
|
+
* with the {@link AuditReport} JSON injected. The React app runs in the reader's
|
|
10
|
+
* browser, so the CLI stays runtime-dependency-light and the output is still a
|
|
11
|
+
* single offline file. There is ONE renderer (pure shadcn/Tailwind) — no inline-CSS
|
|
12
|
+
* fallback; the build guarantees the template exists, and if it somehow doesn't the
|
|
13
|
+
* caller skips the HTML (the JSON + terminal report still work).
|
|
14
|
+
*
|
|
15
|
+
* `<`/`>`/`&` are escaped on injection so report text can never break out of the
|
|
16
|
+
* `<script>`. `injectReportData` is the pure, testable core.
|
|
17
|
+
*/
|
|
18
|
+
const node_fs_1 = require("node:fs");
|
|
19
|
+
const node_path_1 = require("node:path");
|
|
20
|
+
const PLACEHOLDER = "__VIGILES_DATA_PLACEHOLDER__";
|
|
21
|
+
/**
|
|
22
|
+
* Candidate locations for the built template, relative to this module (`__dirname`
|
|
23
|
+
* is the compiled `dist/` at runtime, or `src/` under vitest). CommonJS output, so
|
|
24
|
+
* we use `__dirname`, not `import.meta`.
|
|
25
|
+
*/
|
|
26
|
+
function templatePath() {
|
|
27
|
+
const candidates = [
|
|
28
|
+
(0, node_path_1.resolve)(__dirname, "audit-report.template.html"), // dist/ (shipped)
|
|
29
|
+
(0, node_path_1.resolve)(__dirname, "..", "dist", "audit-report.template.html"), // src/ under vitest
|
|
30
|
+
];
|
|
31
|
+
return candidates.find((p) => (0, node_fs_1.existsSync)(p)) ?? null;
|
|
32
|
+
}
|
|
33
|
+
/** Escape `<`, `>`, `&` so report text can never break out of the `<script>`. */
|
|
34
|
+
function escapeForScript(json) {
|
|
35
|
+
return json.replace(/[<>&]/g, (ch) => "\\u00" + ch.charCodeAt(0).toString(16).padStart(2, "0"));
|
|
36
|
+
}
|
|
37
|
+
/**
|
|
38
|
+
* Inject the report JSON into a template by replacing the quoted placeholder
|
|
39
|
+
* string with the JSON object literal. Pure — the testable core. Throws if the
|
|
40
|
+
* template is missing the placeholder.
|
|
41
|
+
*/
|
|
42
|
+
function injectReportData(template, report) {
|
|
43
|
+
const re = new RegExp(`(["'])${PLACEHOLDER}\\1`);
|
|
44
|
+
if (!re.test(template)) {
|
|
45
|
+
throw new Error("audit report template is missing the data placeholder");
|
|
46
|
+
}
|
|
47
|
+
return template.replace(re, escapeForScript(JSON.stringify(report)));
|
|
48
|
+
}
|
|
49
|
+
/**
|
|
50
|
+
* Render the self-contained HTML report (React template + injected data). Throws
|
|
51
|
+
* if the template hasn't been built — the caller (writeAuditHtml) catches that and
|
|
52
|
+
* skips the HTML, since the JSON + terminal report don't depend on it.
|
|
53
|
+
*/
|
|
54
|
+
function renderAuditHtml(report) {
|
|
55
|
+
const p = templatePath();
|
|
56
|
+
if (!p) {
|
|
57
|
+
throw new Error("audit report template not built — run `npm run build` (builds report/), or use --json / --no-html");
|
|
58
|
+
}
|
|
59
|
+
return injectReportData((0, node_fs_1.readFileSync)(p, "utf-8"), report);
|
|
60
|
+
}
|
|
61
|
+
//# sourceMappingURL=audit-html.js.map
|
|
@@ -0,0 +1,46 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Auto-generated trigger probes for `vigiles audit`'s model trigger tier.
|
|
3
|
+
*
|
|
4
|
+
* The trigger-rate eval needs a per-skill prompt set (does the description FIRE?
|
|
5
|
+
* — recall + precision). Authoring that set by hand was the friction that made
|
|
6
|
+
* the eval un-wowable. The trigger tier removes it: derive a small, DIVERSE probe
|
|
7
|
+
* set from each skill's own description — zero setup. `--prompts=<file>` still
|
|
8
|
+
* overrides for a rigorous, curated benchmark.
|
|
9
|
+
*
|
|
10
|
+
* Deterministic by design (no model needed to AUTHOR the probes — the model is
|
|
11
|
+
* spent RUNNING them). The trick that clears the diversity gate: extract a SHORT
|
|
12
|
+
* topic from the description (so the shared text stays small relative to the
|
|
13
|
+
* frame) and wrap it in lexically-distant frames. Measured min pairwise NCD
|
|
14
|
+
* ~0.27 across short/long descriptions — comfortably above {@link AUTO_MIN_DISTANCE}.
|
|
15
|
+
*/
|
|
16
|
+
import type { TriggerPromptSet } from "./scan-behavioral.js";
|
|
17
|
+
export interface PromptSkill {
|
|
18
|
+
readonly name: string;
|
|
19
|
+
readonly description: string;
|
|
20
|
+
}
|
|
21
|
+
/** How many recall probes we generate per skill (each a distinct frame). */
|
|
22
|
+
export declare const AUTO_RECALL_COUNT = 6;
|
|
23
|
+
/**
|
|
24
|
+
* The diversity floor for AUTO probes — relaxed from the default 0.3 because a
|
|
25
|
+
* templated-but-varied machine probe legitimately shares a topic phrase (the
|
|
26
|
+
* generator's measured min is ~0.27). Still well above 0 → genuine copy-paste
|
|
27
|
+
* is caught; the gate's "vary the phrasing" advice is for hand-authored sets.
|
|
28
|
+
*/
|
|
29
|
+
export declare const AUTO_MIN_DISTANCE = 0.2;
|
|
30
|
+
/**
|
|
31
|
+
* Extract a short, action-shaped topic from a description: drop boilerplate
|
|
32
|
+
* lead-ins ("A skill that…", "Use this skill to…"), take the first clause, cap
|
|
33
|
+
* at 8 words. Capping is load-bearing — a long verbatim topic makes the frames
|
|
34
|
+
* too similar (NCD collapses below the gate).
|
|
35
|
+
*/
|
|
36
|
+
export declare function topicOf(description: string): string;
|
|
37
|
+
/** Recall probes for one skill: distinct frames around its extracted topic. */
|
|
38
|
+
export declare function recallPrompts(description: string, count?: number): string[];
|
|
39
|
+
/**
|
|
40
|
+
* Build a {@link TriggerPromptSet} from skill descriptions — zero-setup trigger
|
|
41
|
+
* probes. Each skill gets `recallPrompts` derived from its description plus the
|
|
42
|
+
* shared irrelevant bank for precision. Skills with an empty description are
|
|
43
|
+
* skipped (nothing to derive a topic from).
|
|
44
|
+
*/
|
|
45
|
+
export declare function autoTriggerPrompts(skills: readonly PromptSkill[], count?: number): TriggerPromptSet;
|
|
46
|
+
//# sourceMappingURL=audit-prompts.d.ts.map
|
|
@@ -0,0 +1,90 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.AUTO_MIN_DISTANCE = exports.AUTO_RECALL_COUNT = void 0;
|
|
4
|
+
exports.topicOf = topicOf;
|
|
5
|
+
exports.recallPrompts = recallPrompts;
|
|
6
|
+
exports.autoTriggerPrompts = autoTriggerPrompts;
|
|
7
|
+
/** How many recall probes we generate per skill (each a distinct frame). */
|
|
8
|
+
exports.AUTO_RECALL_COUNT = 6;
|
|
9
|
+
/**
|
|
10
|
+
* The diversity floor for AUTO probes — relaxed from the default 0.3 because a
|
|
11
|
+
* templated-but-varied machine probe legitimately shares a topic phrase (the
|
|
12
|
+
* generator's measured min is ~0.27). Still well above 0 → genuine copy-paste
|
|
13
|
+
* is caught; the gate's "vary the phrasing" advice is for hand-authored sets.
|
|
14
|
+
*/
|
|
15
|
+
exports.AUTO_MIN_DISTANCE = 0.2;
|
|
16
|
+
// Lexically-distant frames around a short topic. Order matters: the first N are
|
|
17
|
+
// used, and they're arranged so any prefix stays diverse (verified in the test).
|
|
18
|
+
const RECALL_FRAMES = [
|
|
19
|
+
(t) => `I need help to ${t} in my project right now.`,
|
|
20
|
+
(t) => `How do I ${t}? Walk me through the steps.`,
|
|
21
|
+
(t) => `Please ${t} before I open this pull request.`,
|
|
22
|
+
(t) => `What's the recommended way to ${t} on a large team?`,
|
|
23
|
+
(t) => `Can you take a look and ${t} for me?`,
|
|
24
|
+
(t) => `My task today: ${t} across the whole repo.`,
|
|
25
|
+
(t) => `Is there a tool that will ${t} automatically?`,
|
|
26
|
+
(t) => `Give me a checklist to ${t} thoroughly.`,
|
|
27
|
+
];
|
|
28
|
+
// Unrelated requests for the precision arm — varied, clearly off-topic, so a
|
|
29
|
+
// well-scoped skill should NOT fire on them (a too-broad description that hijacks
|
|
30
|
+
// these fails precision). Generic on purpose, distant from any one skill's topic.
|
|
31
|
+
// Count is load-bearing: the auto trigger tier applies the diversity gate's
|
|
32
|
+
// `minPrompts` floor (= AUTO_RECALL_COUNT) to BOTH arms, so the bank must hold at
|
|
33
|
+
// least AUTO_RECALL_COUNT entries or the precision arm fails preflight and every
|
|
34
|
+
// skill reports "unmeasured" instead of running.
|
|
35
|
+
const IRRELEVANT_BANK = [
|
|
36
|
+
"What's the weather forecast for Tokyo this weekend?",
|
|
37
|
+
"Summarize the plot of Hamlet in two sentences.",
|
|
38
|
+
"Convert 100 US dollars to euros at today's rate.",
|
|
39
|
+
"Recommend a good pasta recipe for dinner tonight.",
|
|
40
|
+
"Who won the most Olympic gold medals in swimming?",
|
|
41
|
+
"Explain how photosynthesis works in plants.",
|
|
42
|
+
"Suggest a weekend hiking trail near Seattle.",
|
|
43
|
+
];
|
|
44
|
+
// Boilerplate lead-in words that carry no topical signal (skill descriptions
|
|
45
|
+
// open with a verb — "Reviews…", "Generate…" — so stripping these from the front
|
|
46
|
+
// never eats the real action). Applied iteratively until a content word remains.
|
|
47
|
+
const LEAD_WORD = /^(a|an|the|this|use|skill|agent|tool|command|helper|that|which|to|for|when|invoked?|invoke|used?|helps?|you|with)\b[\s,:-]*/i;
|
|
48
|
+
/**
|
|
49
|
+
* Extract a short, action-shaped topic from a description: drop boilerplate
|
|
50
|
+
* lead-ins ("A skill that…", "Use this skill to…"), take the first clause, cap
|
|
51
|
+
* at 8 words. Capping is load-bearing — a long verbatim topic makes the frames
|
|
52
|
+
* too similar (NCD collapses below the gate).
|
|
53
|
+
*/
|
|
54
|
+
function topicOf(description) {
|
|
55
|
+
let t = description.trim().toLowerCase();
|
|
56
|
+
let prev = "";
|
|
57
|
+
while (t !== prev) {
|
|
58
|
+
prev = t;
|
|
59
|
+
t = t.replace(LEAD_WORD, "");
|
|
60
|
+
}
|
|
61
|
+
const firstClause = t.split(/[.,;:!?]/)[0].trim();
|
|
62
|
+
const words = firstClause.split(/\s+/).filter(Boolean).slice(0, 8);
|
|
63
|
+
const topic = words.join(" ");
|
|
64
|
+
// Fall back to the raw (capped, lowercased) description if stripping left nothing.
|
|
65
|
+
return (topic || description.trim().toLowerCase().split(/\s+/).slice(0, 8).join(" "));
|
|
66
|
+
}
|
|
67
|
+
/** Recall probes for one skill: distinct frames around its extracted topic. */
|
|
68
|
+
function recallPrompts(description, count = exports.AUTO_RECALL_COUNT) {
|
|
69
|
+
const topic = topicOf(description);
|
|
70
|
+
return RECALL_FRAMES.slice(0, count).map((frame) => frame(topic));
|
|
71
|
+
}
|
|
72
|
+
/**
|
|
73
|
+
* Build a {@link TriggerPromptSet} from skill descriptions — zero-setup trigger
|
|
74
|
+
* probes. Each skill gets `recallPrompts` derived from its description plus the
|
|
75
|
+
* shared irrelevant bank for precision. Skills with an empty description are
|
|
76
|
+
* skipped (nothing to derive a topic from).
|
|
77
|
+
*/
|
|
78
|
+
function autoTriggerPrompts(skills, count = exports.AUTO_RECALL_COUNT) {
|
|
79
|
+
const set = {};
|
|
80
|
+
for (const s of skills) {
|
|
81
|
+
if (!s.description.trim())
|
|
82
|
+
continue;
|
|
83
|
+
set[s.name] = {
|
|
84
|
+
prompts: recallPrompts(s.description, count),
|
|
85
|
+
irrelevant: [...IRRELEVANT_BANK],
|
|
86
|
+
};
|
|
87
|
+
}
|
|
88
|
+
return set;
|
|
89
|
+
}
|
|
90
|
+
//# sourceMappingURL=audit-prompts.js.map
|