vigiles 11.0.0 → 12.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude-plugin/plugin.json +9 -0
- package/README.md +9 -4
- package/action.yml +13 -2
- package/dist/adapter-conformance.js +6 -0
- package/dist/adapter-registry.d.ts +20 -0
- package/dist/adapter-registry.js +27 -0
- package/dist/adapters/claude-code/hook-protocol.js +4 -0
- package/dist/adapters/claude-code/runtime.js +12 -0
- package/dist/adapters/codex/eval.js +3 -0
- package/dist/adapters/codex/hook-protocol.d.ts +9 -1
- package/dist/adapters/codex/hook-protocol.js +10 -0
- package/dist/adapters/codex/runtime.js +10 -0
- package/dist/adapters/opencode/runtime.js +4 -0
- package/dist/cli-commands.d.ts +1 -1
- package/dist/cli-commands.js +1 -0
- package/dist/cli.js +211 -29
- package/dist/core/hook-protocol.d.ts +15 -0
- package/dist/core/runtime.d.ts +20 -0
- package/dist/core/types.d.ts +12 -0
- package/dist/eval-cache.d.ts +6 -0
- package/dist/eval-cache.js +2 -0
- package/dist/eval-lock.d.ts +192 -0
- package/dist/eval-lock.js +286 -0
- package/dist/eval.d.ts +33 -20
- package/dist/eval.js +199 -51
- package/dist/setup-plan.d.ts +37 -0
- package/dist/setup-plan.js +66 -4
- package/hooks/eval-lock-nudge.sh +21 -0
- package/package.json +1 -1
- package/skills/test-harness/SKILL.md +27 -0
|
@@ -39,6 +39,15 @@
|
|
|
39
39
|
"command": "bash ${CLAUDE_PLUGIN_ROOT}/hooks/refs-nudge.sh"
|
|
40
40
|
}
|
|
41
41
|
]
|
|
42
|
+
},
|
|
43
|
+
{
|
|
44
|
+
"matcher": "Edit|Write",
|
|
45
|
+
"hooks": [
|
|
46
|
+
{
|
|
47
|
+
"type": "command",
|
|
48
|
+
"command": "bash ${CLAUDE_PLUGIN_ROOT}/hooks/eval-lock-nudge.sh"
|
|
49
|
+
}
|
|
50
|
+
]
|
|
42
51
|
}
|
|
43
52
|
],
|
|
44
53
|
"SessionStart": [
|
package/README.md
CHANGED
|
@@ -197,7 +197,8 @@ stray `git push` is caught before it happens. No model, no key, on every commit.
|
|
|
197
197
|
_"65% fewer tokens." Says who?_ vigiles[^name] A/Bs the claim on real coding tasks and reports
|
|
198
198
|
the token bill, whether it hit its target, and whether the code still works. promptfoo
|
|
199
199
|
and DeepEval bill **per token, every run**; vigiles runs on your own Claude Pro/Max
|
|
200
|
-
subscription.
|
|
200
|
+
subscription. Evals run locally — a committed lock then lets **CI catch stale results with no
|
|
201
|
+
model call**. **[Measure a skill →](docs/measuring-skills.md)**
|
|
201
202
|
|
|
202
203
|
## Quick start
|
|
203
204
|
|
|
@@ -225,13 +226,17 @@ npx vigiles init # adopts your files (non-destructive — eject reverses), add
|
|
|
225
226
|
|
|
226
227
|
Interactive in a terminal, non-interactive for agents/CI (or `--yes`).
|
|
227
228
|
|
|
228
|
-
**
|
|
229
|
-
|
|
229
|
+
**Adoption is smooth: one command, then your agent does the rest.** `init` installs
|
|
230
|
+
the **skills and hooks**, so a plain-English ask does the work — no specs to
|
|
231
|
+
hand-write, no hooks to wire:
|
|
230
232
|
|
|
231
|
-
- _"test my skills"_ → scaffolds **and runs** a trigger/behaviour test (`test-harness`)
|
|
233
|
+
- _"test my skills"_ → scaffolds **and runs** a trigger/behaviour test, then commits its result so CI can check it (`test-harness`)
|
|
232
234
|
- _"harden my rules"_ → upgrades prose guidance into enforced linter rules (`strengthen`)
|
|
233
235
|
- _"add a rule to my CLAUDE.md"_ → edits the source and recompiles (`edit-spec`)
|
|
234
236
|
|
|
237
|
+
The **hooks** keep it honest in-loop — nudging the agent to mark a reference or
|
|
238
|
+
refresh a stale eval — so there are no chores to remember.
|
|
239
|
+
|
|
235
240
|
<details>
|
|
236
241
|
<summary>What <code>init</code> sets up</summary>
|
|
237
242
|
|
package/action.yml
CHANGED
|
@@ -7,7 +7,11 @@ branding:
|
|
|
7
7
|
|
|
8
8
|
inputs:
|
|
9
9
|
command:
|
|
10
|
-
description:
|
|
10
|
+
description: >
|
|
11
|
+
Which vigiles command to run: 'lint' (verify references + integrity +
|
|
12
|
+
coverage), 'compile' (specs → markdown), or 'eval-check' (verify committed
|
|
13
|
+
eval locks against current inputs — the staleness gate; runs NO model, so
|
|
14
|
+
it is the CI-safe half of evals you produce locally with `vigiles eval --update`).
|
|
11
15
|
required: false
|
|
12
16
|
default: "lint"
|
|
13
17
|
paths:
|
|
@@ -65,7 +69,14 @@ runs:
|
|
|
65
69
|
set -euo pipefail
|
|
66
70
|
|
|
67
71
|
cmd="${VIGILES_COMMAND:-lint}"
|
|
68
|
-
|
|
72
|
+
# 'eval-check' is the CI staleness gate — it maps to the real verb
|
|
73
|
+
# `eval --check` (verify committed locks vs current inputs, NO model).
|
|
74
|
+
# Real evals run locally on a subscription (`eval --update`), never in CI.
|
|
75
|
+
if [[ "$cmd" == "eval-check" ]]; then
|
|
76
|
+
args=("eval" "--check")
|
|
77
|
+
else
|
|
78
|
+
args=("$cmd")
|
|
79
|
+
fi
|
|
69
80
|
|
|
70
81
|
# paths: split on commas and whitespace into positional args.
|
|
71
82
|
paths="${VIGILES_PATHS:-}"
|
|
@@ -67,6 +67,12 @@ function checkAdapterConformance(adapter) {
|
|
|
67
67
|
need(adapter.hookProtocol !== undefined, "capabilities.shellHooks is true but hookProtocol is missing");
|
|
68
68
|
if (adapter.hookProtocol) {
|
|
69
69
|
need(Number.isInteger(adapter.hookProtocol.blockExitCode), "hookProtocol.blockExitCode is not an integer");
|
|
70
|
+
// A shell-hook harness must declare WHICH events can inject developer
|
|
71
|
+
// context (`additionalContext`). Encoding it makes "can this harness
|
|
72
|
+
// deliver an inject hook?" a tested contract — the gap that let Codex's
|
|
73
|
+
// inject support sit unverified in prose. Empty would mean the harness
|
|
74
|
+
// can't inject context from a hook at all; every harness we support can.
|
|
75
|
+
need(adapter.hookProtocol.injectableEvents.length > 0, "hookProtocol.injectableEvents is empty — a shell-hook harness must declare the events that honor additionalContext injection (or it can't deliver an inject/nudge hook)");
|
|
70
76
|
portNames.push(["hookProtocol", adapter.hookProtocol.name]);
|
|
71
77
|
}
|
|
72
78
|
}
|
|
@@ -83,4 +83,24 @@ export declare function resolveHarnessSelection(opts: {
|
|
|
83
83
|
flag?: string;
|
|
84
84
|
configHarness?: string | readonly string[];
|
|
85
85
|
}): HarnessSelection;
|
|
86
|
+
/**
|
|
87
|
+
* The FULL adapter set a compile-time INSTALL should fan out to. Unlike
|
|
88
|
+
* `resolveHarnessSelection` (which picks ONE dialect for a single-output compile,
|
|
89
|
+
* since you emit a markdown file in one harness's format), an install writes the
|
|
90
|
+
* SAME artifact into EVERY enabled harness's native config — so a repo targeting
|
|
91
|
+
* both harnesses gets a compiled hook in `.claude/settings.json` AND
|
|
92
|
+
* `.codex/config.toml`, not just the first. Precedence mirrors the single picker:
|
|
93
|
+
*
|
|
94
|
+
* 1. `--harness=` flag → just that one (an explicit override is singular).
|
|
95
|
+
* 2. config `harness` list → ALL of them (the multi-harness fan-out).
|
|
96
|
+
* 3. no config → auto-detect → the one detected.
|
|
97
|
+
*
|
|
98
|
+
* Returns ≥1 adapter, de-duplicated by name (a config that lists a harness twice,
|
|
99
|
+
* or an alias + its canonical, collapses to one install).
|
|
100
|
+
*/
|
|
101
|
+
export declare function resolveHarnessAdapters(opts: {
|
|
102
|
+
root: string;
|
|
103
|
+
flag?: string;
|
|
104
|
+
configHarness?: string | readonly string[];
|
|
105
|
+
}): HarnessAdapter[];
|
|
86
106
|
//# sourceMappingURL=adapter-registry.d.ts.map
|
package/dist/adapter-registry.js
CHANGED
|
@@ -9,6 +9,7 @@ exports.adapterForInstructionFile = adapterForInstructionFile;
|
|
|
9
9
|
exports.resolveAdapter = resolveAdapter;
|
|
10
10
|
exports.normalizeHarnessList = normalizeHarnessList;
|
|
11
11
|
exports.resolveHarnessSelection = resolveHarnessSelection;
|
|
12
|
+
exports.resolveHarnessAdapters = resolveHarnessAdapters;
|
|
12
13
|
const adapter_js_1 = require("./adapters/claude-code/adapter.js");
|
|
13
14
|
const adapter_js_2 = require("./adapters/codex/adapter.js");
|
|
14
15
|
/** The default adapter when detection finds no harness markers. */
|
|
@@ -127,4 +128,30 @@ function resolveHarnessSelection(opts) {
|
|
|
127
128
|
}
|
|
128
129
|
return { kind: "ok", adapter: det.adapter };
|
|
129
130
|
}
|
|
131
|
+
/**
|
|
132
|
+
* The FULL adapter set a compile-time INSTALL should fan out to. Unlike
|
|
133
|
+
* `resolveHarnessSelection` (which picks ONE dialect for a single-output compile,
|
|
134
|
+
* since you emit a markdown file in one harness's format), an install writes the
|
|
135
|
+
* SAME artifact into EVERY enabled harness's native config — so a repo targeting
|
|
136
|
+
* both harnesses gets a compiled hook in `.claude/settings.json` AND
|
|
137
|
+
* `.codex/config.toml`, not just the first. Precedence mirrors the single picker:
|
|
138
|
+
*
|
|
139
|
+
* 1. `--harness=` flag → just that one (an explicit override is singular).
|
|
140
|
+
* 2. config `harness` list → ALL of them (the multi-harness fan-out).
|
|
141
|
+
* 3. no config → auto-detect → the one detected.
|
|
142
|
+
*
|
|
143
|
+
* Returns ≥1 adapter, de-duplicated by name (a config that lists a harness twice,
|
|
144
|
+
* or an alias + its canonical, collapses to one install).
|
|
145
|
+
*/
|
|
146
|
+
function resolveHarnessAdapters(opts) {
|
|
147
|
+
const { root, flag, configHarness } = opts;
|
|
148
|
+
if (flag !== undefined && flag !== "")
|
|
149
|
+
return [resolveAdapter(root, flag)];
|
|
150
|
+
const list = normalizeHarnessList(configHarness);
|
|
151
|
+
const adapters = list.length > 0
|
|
152
|
+
? list.map((h) => resolveAdapter(root, h))
|
|
153
|
+
: [detectAdapterResult(root).adapter];
|
|
154
|
+
const seen = new Set();
|
|
155
|
+
return adapters.filter((a) => !seen.has(a.name) && seen.add(a.name));
|
|
156
|
+
}
|
|
130
157
|
//# sourceMappingURL=adapter-registry.js.map
|
|
@@ -6,5 +6,9 @@ exports.claudeCodeHookProtocol = {
|
|
|
6
6
|
blockExitCode: 2,
|
|
7
7
|
denyDecisionValues: ["block", "deny"],
|
|
8
8
|
eventEnvVars: [],
|
|
9
|
+
// Events that honor `hookSpecificOutput.additionalContext` (developer-context
|
|
10
|
+
// injection). Covers vigiles's shipped inject hooks: the SessionStart lint
|
|
11
|
+
// summary and the PostToolUse refs / eval-lock nudges.
|
|
12
|
+
injectableEvents: ["SessionStart", "UserPromptSubmit", "PostToolUse"],
|
|
9
13
|
};
|
|
10
14
|
//# sourceMappingURL=hook-protocol.js.map
|
|
@@ -22,6 +22,18 @@ exports.claudeCodeRuntime = {
|
|
|
22
22
|
},
|
|
23
23
|
};
|
|
24
24
|
},
|
|
25
|
+
/**
|
|
26
|
+
* Claude Code keys on **major.minor**: a minor/major bump is where the system
|
|
27
|
+
* prompt + tool defs actually move (0.2 → 1.0 → 2.0 → 2.1, ~quarterly), while
|
|
28
|
+
* the daily patch stream rarely changes behavior — so keying patches would
|
|
29
|
+
* churn the cache for no signal. Falls back to the trimmed raw string when no
|
|
30
|
+
* semver is found. (If a specific patch is known to matter, clear the cache or
|
|
31
|
+
* bump `CACHE_FORMAT_VERSION`.)
|
|
32
|
+
*/
|
|
33
|
+
versionKey(raw) {
|
|
34
|
+
const m = /(\d+)\.(\d+)\.\d+/.exec(raw);
|
|
35
|
+
return m ? `${m[1]}.${m[2]}` : raw.trim();
|
|
36
|
+
},
|
|
25
37
|
};
|
|
26
38
|
/**
|
|
27
39
|
* Build the spawn env that points the agent CLI at the mock model: the caller's
|
|
@@ -195,6 +195,9 @@ exports.codexEvalDriver = {
|
|
|
195
195
|
runner: codexEvalAgentRunner,
|
|
196
196
|
parse: parseCodexEvalRun,
|
|
197
197
|
runError: codexRunError,
|
|
198
|
+
// The harness identity → folded into the trigger-rate lock hash, so a report
|
|
199
|
+
// recorded on Claude Code is STALE if the eval is switched to Codex (and v.v.).
|
|
200
|
+
harness: "codex",
|
|
198
201
|
};
|
|
199
202
|
/**
|
|
200
203
|
* Spawn real `codex exec --json` for the eval tier (real model, the user's codex
|
|
@@ -1,9 +1,17 @@
|
|
|
1
1
|
/**
|
|
2
|
-
* codexHookProtocol —
|
|
2
|
+
* codexHookProtocol — Codex's hook wire protocol.
|
|
3
3
|
* Finding: it is essentially IDENTICAL to Claude Code's (exit 2 / `decision:block`
|
|
4
4
|
* / `permissionDecision:deny`) — the thin `HookProtocol` port was the right call.
|
|
5
5
|
* The genuine deltas are the env vars a hook receives + the TOML config format
|
|
6
6
|
* (the latter lives in PluginLayout.settingsFormat, not here).
|
|
7
|
+
*
|
|
8
|
+
* Context injection (`hookSpecificOutput.additionalContext`) is ALSO shared — same
|
|
9
|
+
* shape, confirmed against the official Codex hooks docs
|
|
10
|
+
* (developers.openai.com/codex/hooks): supported on SessionStart, UserPromptSubmit,
|
|
11
|
+
* PreToolUse, PostToolUse, SubagentStart. (Earlier docs called this "deferred" —
|
|
12
|
+
* it is not.) So vigiles's PostToolUse nudges + SessionStart summary deliver on
|
|
13
|
+
* Codex unchanged. Caveats: Stop/SubagentStop/PreCompact carry no context, and
|
|
14
|
+
* Codex marks a hook run failed if it emits an unsupported field for the event.
|
|
7
15
|
*/
|
|
8
16
|
import type { HookProtocol } from "../../core/hook-protocol.js";
|
|
9
17
|
export declare const codexHookProtocol: HookProtocol;
|
|
@@ -8,6 +8,16 @@ exports.codexHookProtocol = {
|
|
|
8
8
|
// Codex matchers are anchored regexes (`matcher = "^Bash$"`), unlike Claude
|
|
9
9
|
// Code's exact tool name / `A|B` alternation.
|
|
10
10
|
matcherStyle: "regex",
|
|
11
|
+
// Events that honor `hookSpecificOutput.additionalContext` on Codex, per the
|
|
12
|
+
// official hooks docs. Includes the events vigiles's shipped hooks use
|
|
13
|
+
// (PostToolUse, SessionStart), so those nudges reach the Codex agent too.
|
|
14
|
+
injectableEvents: [
|
|
15
|
+
"SessionStart",
|
|
16
|
+
"UserPromptSubmit",
|
|
17
|
+
"PreToolUse",
|
|
18
|
+
"PostToolUse",
|
|
19
|
+
"SubagentStart",
|
|
20
|
+
],
|
|
11
21
|
eventEnvVars: [
|
|
12
22
|
"session_id",
|
|
13
23
|
"cwd",
|
|
@@ -22,6 +22,16 @@ exports.codexRuntime = {
|
|
|
22
22
|
wireMock(baseUrl) {
|
|
23
23
|
return { args: codexMockArgs(baseUrl), env: codexMockEnv() };
|
|
24
24
|
},
|
|
25
|
+
/**
|
|
26
|
+
* Codex opts OUT of version partitioning (`""`). It is perpetual `0.x` where
|
|
27
|
+
* the *minor* is the patch cadence (~2 bumps/week, 134 minors in 14 months), so
|
|
28
|
+
* keying `major.minor` like Claude Code would churn the cache/lock weekly. With
|
|
29
|
+
* no stable behavior boundary in the version string, Codex relies on the dated
|
|
30
|
+
* model id + `evalApiVersion` for staleness instead. See research/cache-invalidation.md.
|
|
31
|
+
*/
|
|
32
|
+
versionKey(_raw) {
|
|
33
|
+
return "";
|
|
34
|
+
},
|
|
25
35
|
};
|
|
26
36
|
/**
|
|
27
37
|
* The PROVEN `-c` flag array that points `codex exec` at a mock served at
|
package/dist/cli-commands.d.ts
CHANGED
|
@@ -13,7 +13,7 @@
|
|
|
13
13
|
/** Human-facing verbs (printed in help; typed by a human/agent/CI). */
|
|
14
14
|
export declare const VERBS: readonly ["init", "compile", "eject", "lint", "test", "eval", "audit", "scaffold-test", "generate", "hook-runtime"];
|
|
15
15
|
/** Runtime entrypoint kinds under `vigiles hook-runtime <kind>` (emitted, not typed). */
|
|
16
|
-
export declare const HOOK_RUNTIME_KINDS: readonly ["run-program", "agent", "agent-start", "agent-done", "skill", "skill-tool", "skill-start", "skill-done", "run-skill", "intercept-tool", "guard", "action", "refs", "effect-enter", "effect-exit"];
|
|
16
|
+
export declare const HOOK_RUNTIME_KINDS: readonly ["run-program", "agent", "agent-start", "agent-done", "skill", "skill-tool", "skill-start", "skill-done", "run-skill", "intercept-tool", "guard", "action", "refs", "eval-lock-nudge", "effect-enter", "effect-exit"];
|
|
17
17
|
export type Verb = (typeof VERBS)[number];
|
|
18
18
|
export type HookRuntimeKind = (typeof HOOK_RUNTIME_KINDS)[number];
|
|
19
19
|
//# sourceMappingURL=cli-commands.d.ts.map
|
package/dist/cli-commands.js
CHANGED
package/dist/cli.js
CHANGED
|
@@ -17,6 +17,7 @@ const generate_types_js_1 = require("./core/generate-types.js");
|
|
|
17
17
|
const generate_harness_js_1 = require("./core/generate-harness.js");
|
|
18
18
|
const capability_diff_js_1 = require("./core/capability-diff.js");
|
|
19
19
|
const validate_js_1 = require("./core/validate.js");
|
|
20
|
+
const eval_lock_js_1 = require("./eval-lock.js");
|
|
20
21
|
const cli_flags_js_1 = require("./cli-flags.js");
|
|
21
22
|
const setup_plan_js_1 = require("./setup-plan.js");
|
|
22
23
|
const types_js_1 = require("./core/types.js");
|
|
@@ -1601,7 +1602,19 @@ function specReferencedElsewhere(specFile, ejectedFile) {
|
|
|
1601
1602
|
/** Full GitHub Actions workflow that wires the production `zernie/vigiles@v1`
|
|
1602
1603
|
* Action (lint pillar) and, when the test pillar is set up, a deterministic
|
|
1603
1604
|
* harness job. */
|
|
1604
|
-
|
|
1605
|
+
/** The npm package(s) that provide each harness's CLI binary — the deterministic
|
|
1606
|
+
* harness tier spawns the real agent CLI against a mock model (no API key). A repo
|
|
1607
|
+
* targeting both harnesses installs both. */
|
|
1608
|
+
function harnessTestBinaries(harnesses) {
|
|
1609
|
+
const pkgs = [];
|
|
1610
|
+
if (harnesses.includes("claude"))
|
|
1611
|
+
pkgs.push("@anthropic-ai/claude-code");
|
|
1612
|
+
if (harnesses.includes("codex"))
|
|
1613
|
+
pkgs.push("@openai/codex");
|
|
1614
|
+
// Fall back to Claude Code if the set is somehow empty (back-compatible default).
|
|
1615
|
+
return (pkgs.length > 0 ? pkgs : ["@anthropic-ai/claude-code"]).join(" ");
|
|
1616
|
+
}
|
|
1617
|
+
function vigilesWorkflow(plan, harnesses) {
|
|
1605
1618
|
const harness = plan.test
|
|
1606
1619
|
? `
|
|
1607
1620
|
harness:
|
|
@@ -1615,8 +1628,23 @@ function vigilesWorkflow(plan) {
|
|
|
1615
1628
|
with:
|
|
1616
1629
|
node-version: "20"
|
|
1617
1630
|
- run: npm install
|
|
1618
|
-
- run: npm i -g
|
|
1631
|
+
- run: npm i -g ${harnessTestBinaries(harnesses)} # mock tier needs the binary, no API key
|
|
1619
1632
|
- run: npx vigiles test
|
|
1633
|
+
|
|
1634
|
+
eval-check:
|
|
1635
|
+
# Eval staleness gate — real-model evals run LOCALLY on your subscription
|
|
1636
|
+
# (\`npx vigiles eval --update\`, which commits a lock); this job VERIFIES those
|
|
1637
|
+
# committed results against the current inputs with NO model call. It stays a
|
|
1638
|
+
# green no-op until you commit your first lock. See docs/harness-testing.md.
|
|
1639
|
+
runs-on: ubuntu-latest
|
|
1640
|
+
steps:
|
|
1641
|
+
- uses: actions/checkout@v4
|
|
1642
|
+
- uses: actions/setup-node@v4
|
|
1643
|
+
with:
|
|
1644
|
+
node-version: "20"
|
|
1645
|
+
- uses: zernie/vigiles@v1
|
|
1646
|
+
with:
|
|
1647
|
+
command: eval-check
|
|
1620
1648
|
`
|
|
1621
1649
|
: "";
|
|
1622
1650
|
return `name: vigiles
|
|
@@ -1687,7 +1715,7 @@ function rewriteRemovedSubcommands(content) {
|
|
|
1687
1715
|
* commit hint). An existing workflow is never clobbered unless `--force`, but a
|
|
1688
1716
|
* STALE one (old bare-`npx vigiles` API, or a removed subcommand) is reported
|
|
1689
1717
|
* loudly instead of silently skipped — and rewritten in place with `--force`. */
|
|
1690
|
-
function wireGha(plan) {
|
|
1718
|
+
function wireGha(plan, harnesses) {
|
|
1691
1719
|
const dir = (0, node_path_1.resolve)(process.cwd(), ".github", "workflows");
|
|
1692
1720
|
const path = (0, node_path_1.resolve)(dir, "vigiles.yml");
|
|
1693
1721
|
const rel = ".github/workflows/vigiles.yml";
|
|
@@ -1708,7 +1736,7 @@ function wireGha(plan) {
|
|
|
1708
1736
|
}
|
|
1709
1737
|
else if (workflowUsesStaleApi(content)) {
|
|
1710
1738
|
if (plan.force) {
|
|
1711
|
-
(0, node_fs_1.writeFileSync)(path, vigilesWorkflow(plan));
|
|
1739
|
+
(0, node_fs_1.writeFileSync)(path, vigilesWorkflow(plan, harnesses));
|
|
1712
1740
|
console.log(`✓ Regenerated ${rel} (was a stale bare \`npx vigiles\`)`);
|
|
1713
1741
|
return [rel];
|
|
1714
1742
|
}
|
|
@@ -1724,7 +1752,7 @@ function wireGha(plan) {
|
|
|
1724
1752
|
}
|
|
1725
1753
|
if (!(0, node_fs_1.existsSync)(dir))
|
|
1726
1754
|
(0, node_fs_1.mkdirSync)(dir, { recursive: true });
|
|
1727
|
-
(0, node_fs_1.writeFileSync)(path, vigilesWorkflow(plan));
|
|
1755
|
+
(0, node_fs_1.writeFileSync)(path, vigilesWorkflow(plan, harnesses));
|
|
1728
1756
|
console.log("✓ Created .github/workflows/vigiles.yml (uses zernie/vigiles@v1)");
|
|
1729
1757
|
return [".github/workflows/vigiles.yml"];
|
|
1730
1758
|
}
|
|
@@ -2130,6 +2158,36 @@ function installPlugins(harnesses) {
|
|
|
2130
2158
|
console.log("");
|
|
2131
2159
|
reportInstall(plan, runInstall(plan, exec));
|
|
2132
2160
|
}
|
|
2161
|
+
// Claude Code gets its hooks from the global marketplace plugin; Codex has no
|
|
2162
|
+
// global store, so wire vigiles's proactive nudge hooks into the repo's
|
|
2163
|
+
// .codex/config.toml directly (the idiomatic, repo-committed place).
|
|
2164
|
+
if (harnesses.includes("codex"))
|
|
2165
|
+
wireCodexHooks();
|
|
2166
|
+
}
|
|
2167
|
+
/**
|
|
2168
|
+
* Wire vigiles's proactive nudge hooks into `.codex/config.toml` (idempotently).
|
|
2169
|
+
* Codex honors `additionalContext` on `PostToolUse`, and these run as direct
|
|
2170
|
+
* `npx vigiles hook-runtime …` commands (no plugin root / vendored script), so a
|
|
2171
|
+
* Codex user gets the same eval-lock + refs nudges a Claude Code user gets from
|
|
2172
|
+
* the marketplace plugin. The pure merge is `applyCodexPluginHooks` (unit-tested
|
|
2173
|
+
* in setup-plan.test.ts) — this only does the read/parse/write IO.
|
|
2174
|
+
*/
|
|
2175
|
+
function wireCodexHooks() {
|
|
2176
|
+
const path = (0, node_path_1.resolve)(process.cwd(), ".codex", "config.toml");
|
|
2177
|
+
let config = {};
|
|
2178
|
+
if ((0, node_fs_1.existsSync)(path)) {
|
|
2179
|
+
try {
|
|
2180
|
+
config = (0, toml_1.parse)((0, node_fs_1.readFileSync)(path, "utf-8"));
|
|
2181
|
+
}
|
|
2182
|
+
catch {
|
|
2183
|
+
console.log("⚠ .codex/config.toml is not valid TOML — skipping Codex hook wiring (fix it, then re-run `vigiles init`).");
|
|
2184
|
+
return;
|
|
2185
|
+
}
|
|
2186
|
+
}
|
|
2187
|
+
const merged = (0, setup_plan_js_1.applyCodexPluginHooks)(config);
|
|
2188
|
+
(0, node_fs_1.mkdirSync)((0, node_path_1.dirname)(path), { recursive: true });
|
|
2189
|
+
(0, node_fs_1.writeFileSync)(path, (0, hook_install_js_1.serializeConfig)(merged, "toml"));
|
|
2190
|
+
console.log("✓ Wired the eval-lock + refs nudge hooks into .codex/config.toml (commit it)");
|
|
2133
2191
|
}
|
|
2134
2192
|
/** Add/upgrade `vigiles` in the project's `devDependencies` (and move it out of
|
|
2135
2193
|
* `dependencies` if it's there). Returns the files it wrote (for the commit
|
|
@@ -2293,7 +2351,7 @@ async function setup(args) {
|
|
|
2293
2351
|
// CI — the production Action (+ a harness job when Pillar 2 is on).
|
|
2294
2352
|
if (plan.gha) {
|
|
2295
2353
|
console.log("");
|
|
2296
|
-
written.push(...wireGha(plan));
|
|
2354
|
+
written.push(...wireGha(plan, harnesses));
|
|
2297
2355
|
}
|
|
2298
2356
|
// Plugin/skill install — per-harness (Claude marketplace / Codex direct).
|
|
2299
2357
|
if (plan.plugin) {
|
|
@@ -3406,10 +3464,55 @@ async function handleGenerateHarness(args, restArgs) {
|
|
|
3406
3464
|
* tier needs it, just like the node:test suite). `--trials=N` is forwarded to
|
|
3407
3465
|
* eval scripts via the `VIGILES_TRIALS` env var.
|
|
3408
3466
|
*/
|
|
3467
|
+
/**
|
|
3468
|
+
* Resolve the eval LOCK env from the `eval` flags. `--update` records each named
|
|
3469
|
+
* eval's report to a committed `.vigiles/eval-locks/<name>.lock.json` (run locally
|
|
3470
|
+
* on your subscription); `--check` (CI) verifies the committed result against the
|
|
3471
|
+
* current inputs WITHOUT a model call. `--check` is a green NO-OP until the first
|
|
3472
|
+
* lock is committed (smooth adoption). Returns the env to thread, or `"skip"` to
|
|
3473
|
+
* exit green now. `--check`+`--update` together is a usage error (exit 2). The
|
|
3474
|
+
* behavior epoch comes from `.vigilesrc.json` `eval.apiVersion` (committed).
|
|
3475
|
+
*/
|
|
3476
|
+
function resolveEvalLockEnv(args) {
|
|
3477
|
+
const wantCheck = args.includes("--check");
|
|
3478
|
+
const wantUpdate = args.includes("--update");
|
|
3479
|
+
if (wantCheck && wantUpdate) {
|
|
3480
|
+
console.error("vigiles eval: --check and --update are mutually exclusive (one verifies, one records).");
|
|
3481
|
+
process.exit(2);
|
|
3482
|
+
}
|
|
3483
|
+
if (wantCheck &&
|
|
3484
|
+
!(0, eval_lock_js_1.anyLocksCommitted)((0, node_path_1.resolve)(process.cwd(), eval_lock_js_1.DEFAULT_LOCK_DIR))) {
|
|
3485
|
+
console.log("ℹ vigiles eval --check: no committed eval locks found — nothing to verify.\n" +
|
|
3486
|
+
" Run `vigiles eval --update` locally (on your subscription) and commit the\n" +
|
|
3487
|
+
" lock to enable the CI staleness gate.");
|
|
3488
|
+
return "skip";
|
|
3489
|
+
}
|
|
3490
|
+
const env = {};
|
|
3491
|
+
if (wantCheck)
|
|
3492
|
+
env.VIGILES_EVAL_LOCK = "check";
|
|
3493
|
+
if (wantUpdate)
|
|
3494
|
+
env.VIGILES_EVAL_LOCK = "update";
|
|
3495
|
+
if (wantCheck || wantUpdate) {
|
|
3496
|
+
const apiVersion = (0, validate_js_1.loadConfig)().eval?.apiVersion;
|
|
3497
|
+
if (apiVersion !== undefined)
|
|
3498
|
+
env.VIGILES_EVAL_API_VERSION = String(apiVersion);
|
|
3499
|
+
}
|
|
3500
|
+
return env;
|
|
3501
|
+
}
|
|
3409
3502
|
function handleRunScripts(kind, args, restArgs) {
|
|
3410
3503
|
const cwd = process.cwd();
|
|
3411
3504
|
// Harness/eval scripts may be authored in JS or TS (see run-scripts.ts).
|
|
3412
3505
|
const defaultGlob = (0, run_scripts_js_1.scriptGlob)(kind === "test" ? "harness" : "eval");
|
|
3506
|
+
// The eval LOCK flags (`--check`/`--update`) are resolved BEFORE file discovery
|
|
3507
|
+
// so mutual-exclusion + the cold-start no-op are honored regardless of file
|
|
3508
|
+
// count. Returns the env to thread to scripts, or `"skip"` to exit green now.
|
|
3509
|
+
let lockEnv = {};
|
|
3510
|
+
if (kind === "eval") {
|
|
3511
|
+
const r = resolveEvalLockEnv(args);
|
|
3512
|
+
if (r === "skip")
|
|
3513
|
+
return;
|
|
3514
|
+
lockEnv = r;
|
|
3515
|
+
}
|
|
3413
3516
|
const files = (0, run_scripts_js_1.discoverScripts)(restArgs, defaultGlob, cwd);
|
|
3414
3517
|
// `--min=N`: a CI gate asserts at least N scripts actually RAN — so a bad path,
|
|
3415
3518
|
// a renamed file, or a glob that matched nothing fails LOUD instead of passing
|
|
@@ -3438,7 +3541,7 @@ function handleRunScripts(kind, args, restArgs) {
|
|
|
3438
3541
|
// it's part of the measurement definition, so it belongs in the spec
|
|
3439
3542
|
// (`model` / `minModel`), version-controlled, not a hidden override.
|
|
3440
3543
|
const trialsFlag = args.find((a) => a.startsWith("--trials="));
|
|
3441
|
-
const env = {};
|
|
3544
|
+
const env = { ...lockEnv };
|
|
3442
3545
|
if (trialsFlag)
|
|
3443
3546
|
env.VIGILES_TRIALS = trialsFlag.split("=")[1];
|
|
3444
3547
|
console.log(`Running ${String(files.length)} ${kind} file(s):\n`);
|
|
@@ -3617,6 +3720,8 @@ function printUsage(command) {
|
|
|
3617
3720
|
console.log(" --serve opens a LIVE local report whose buttons create specs in one click (own repo only; loopback + token-guarded) · --no-serve to skip the prompt");
|
|
3618
3721
|
console.log(" vigiles test [files...] Run *.harness.mjs deterministic harness tests");
|
|
3619
3722
|
console.log(" vigiles eval [files...] Run *.eval.mjs real-model harness evals (--trials=N, --min=N, --no-skip)");
|
|
3723
|
+
console.log(" --update records each named eval's result to a committed lock (run locally on your subscription)");
|
|
3724
|
+
console.log(" --check verifies committed eval results against current inputs WITHOUT a model — the CI staleness gate");
|
|
3620
3725
|
console.log(" vigiles scaffold-test [dir] Generate a starter test for each untested skill/agent/hook (--write, --json)");
|
|
3621
3726
|
console.log("");
|
|
3622
3727
|
console.log("Examples:");
|
|
@@ -3932,6 +4037,9 @@ async function handleHookRuntime(kind, restArgs) {
|
|
|
3932
4037
|
case "refs":
|
|
3933
4038
|
refsHookCommand();
|
|
3934
4039
|
return;
|
|
4040
|
+
case "eval-lock-nudge":
|
|
4041
|
+
evalLockNudgeHookCommand();
|
|
4042
|
+
return;
|
|
3935
4043
|
case "effect-enter":
|
|
3936
4044
|
(0, effect_region_js_1.setEffectActive)(process.cwd());
|
|
3937
4045
|
console.log("Effect boundary entered.");
|
|
@@ -3978,6 +4086,45 @@ const INSTRUCTION_FILE = /^(SKILL|CLAUDE|AGENTS)\.md$/;
|
|
|
3978
4086
|
function isInstructionFile(file) {
|
|
3979
4087
|
return INSTRUCTION_FILE.test((0, node_path_1.basename)(file));
|
|
3980
4088
|
}
|
|
4089
|
+
/**
|
|
4090
|
+
* PostToolUse-hook entrypoint: when the agent edits an eval input (a `SKILL.md`
|
|
4091
|
+
* trigger surface or an `*.eval.*` script), and committed eval locks exist, inject
|
|
4092
|
+
* a NON-BLOCKING reminder to re-run `vigiles eval --update`. Self-gating (silent
|
|
4093
|
+
* until a lock is committed), never blocks, never runs an eval — a reminder, not a
|
|
4094
|
+
* gate (the gate is `eval --check` in CI). The harness-neutral nudge lives in
|
|
4095
|
+
* `evalLockNudge`; both CC and Codex deliver it as `additionalContext` on
|
|
4096
|
+
* `PostToolUse` (confirmed — see docs/harness-testing-codex.md).
|
|
4097
|
+
*/
|
|
4098
|
+
function evalLockNudgeHookCommand() {
|
|
4099
|
+
let raw = "";
|
|
4100
|
+
try {
|
|
4101
|
+
raw = (0, node_fs_1.readFileSync)(0, "utf-8");
|
|
4102
|
+
}
|
|
4103
|
+
catch {
|
|
4104
|
+
/* no stdin → nothing to do */
|
|
4105
|
+
}
|
|
4106
|
+
let file = "";
|
|
4107
|
+
try {
|
|
4108
|
+
const j = JSON.parse(raw);
|
|
4109
|
+
file = j.tool_input?.file_path ?? "";
|
|
4110
|
+
}
|
|
4111
|
+
catch {
|
|
4112
|
+
/* malformed → nothing to do */
|
|
4113
|
+
}
|
|
4114
|
+
if (!file)
|
|
4115
|
+
return;
|
|
4116
|
+
const cwd = process.cwd();
|
|
4117
|
+
const target = (0, node_path_1.relative)(cwd, (0, node_path_1.resolve)(cwd, file)) || file;
|
|
4118
|
+
const msg = (0, eval_lock_js_1.evalLockNudge)(target, (0, node_path_1.resolve)(cwd, eval_lock_js_1.DEFAULT_LOCK_DIR));
|
|
4119
|
+
if (!msg)
|
|
4120
|
+
return;
|
|
4121
|
+
process.stdout.write(JSON.stringify({
|
|
4122
|
+
hookSpecificOutput: {
|
|
4123
|
+
hookEventName: "PostToolUse",
|
|
4124
|
+
additionalContext: msg,
|
|
4125
|
+
},
|
|
4126
|
+
}) + "\n");
|
|
4127
|
+
}
|
|
3981
4128
|
/**
|
|
3982
4129
|
* PostToolUse-hook entrypoint: when the agent edits an instruction file, force
|
|
3983
4130
|
* every code reference to carry a file-qualified mark (`path.ext#symbol`) and
|
|
@@ -4153,31 +4300,63 @@ async function installHookFile(file, adapter, registeredProviders = []) {
|
|
|
4153
4300
|
: (0, hook_install_js_1.mergeHooksJson)(existing, compiled.hooks, file);
|
|
4154
4301
|
(0, node_fs_1.mkdirSync)((0, node_path_1.dirname)(settingsAbs), { recursive: true });
|
|
4155
4302
|
(0, node_fs_1.writeFileSync)(settingsAbs, (0, hook_install_js_1.serializeConfig)(merged, format));
|
|
4156
|
-
//
|
|
4157
|
-
//
|
|
4158
|
-
//
|
|
4159
|
-
//
|
|
4303
|
+
// No silent skips: warn loudly only where a hook's OUTPUT genuinely may not
|
|
4304
|
+
// apply on this harness. INJECT's `additionalContext` shape is now CONFIRMED
|
|
4305
|
+
// shared with Codex (per the official hooks docs), so an inject hook only
|
|
4306
|
+
// warns when its event isn't in the harness's `injectableEvents`. REACT's
|
|
4307
|
+
// output is still Claude-Code-confirmed only. The gate (deny→exit 2) path is
|
|
4308
|
+
// cross-harness and never warns.
|
|
4160
4309
|
const role = (0, hook_program_js_1.dispatchKind)(program);
|
|
4161
|
-
const
|
|
4162
|
-
|
|
4163
|
-
|
|
4164
|
-
|
|
4165
|
-
|
|
4166
|
-
|
|
4167
|
-
|
|
4310
|
+
const event = typeof program.on === "string" ? program.on : "";
|
|
4311
|
+
const injectable = adapter.hookProtocol?.injectableEvents ?? [];
|
|
4312
|
+
const matcher = (0, hook_program_js_1.hookRouting)(program).matcher;
|
|
4313
|
+
let warning;
|
|
4314
|
+
if (adapter.name !== "claude-code") {
|
|
4315
|
+
if (role === "inject" && !injectable.includes(event)) {
|
|
4316
|
+
warning =
|
|
4317
|
+
`this inject hook targets "${event}", which ${adapter.name} does not ` +
|
|
4318
|
+
`honor for additionalContext — the injected text won't reach the agent. ` +
|
|
4319
|
+
`Use an event ${adapter.name} supports: ${injectable.join(", ")}.`;
|
|
4320
|
+
}
|
|
4321
|
+
else if (role === "react") {
|
|
4322
|
+
warning =
|
|
4323
|
+
`react output is confirmed only for Claude Code; on ${adapter.name} this ` +
|
|
4324
|
+
`hook's react output is unverified (the gate deny→exit 2 path IS ` +
|
|
4325
|
+
`cross-harness). Confirm against the real binary first.`;
|
|
4326
|
+
}
|
|
4327
|
+
else if (matcher !== undefined) {
|
|
4328
|
+
// A tool-matched gate carries TOOL NAMES in its matcher. vigiles does not
|
|
4329
|
+
// yet translate tool vocabularies across dialects, so a matcher authored
|
|
4330
|
+
// with Claude Code names (`Edit`/`Write`/`Bash`) won't fire on a harness
|
|
4331
|
+
// that names the same tools differently (Codex: `apply_patch`/`shell`).
|
|
4332
|
+
// Warn LOUDLY rather than report a silently-non-firing success.
|
|
4333
|
+
warning =
|
|
4334
|
+
`this hook matches tool(s) "${matcher}" — if those are Claude Code tool ` +
|
|
4335
|
+
`names, they may not match ${adapter.name}'s vocabulary (e.g. ` +
|
|
4336
|
+
`apply_patch/shell), so the hook may not fire. Verify the matcher uses ` +
|
|
4337
|
+
`${adapter.name}'s tool names (cross-dialect matcher translation is not ` +
|
|
4338
|
+
`yet automatic).`;
|
|
4339
|
+
}
|
|
4340
|
+
}
|
|
4168
4341
|
return { role, settingsPath: adapter.layout.settingsPath, warning };
|
|
4169
4342
|
}
|
|
4170
4343
|
/**
|
|
4171
4344
|
* Compile + install every hook (explicit paths, else discovered under
|
|
4172
|
-
* `.vigiles/hooks/`) into
|
|
4173
|
-
* hook
|
|
4345
|
+
* `.vigiles/hooks/`) into EVERY enabled harness's config. A typed hook is
|
|
4346
|
+
* harness-neutral, so when a repo targets both harnesses the SAME hook is merged
|
|
4347
|
+
* into `.claude/settings.json` AND `.codex/config.toml` (each in its native
|
|
4348
|
+
* format, with per-harness warnings) — never just the first. The harness set is
|
|
4349
|
+
* resolved from the `--harness=` flag, else `config.harness`, else auto-detect.
|
|
4350
|
+
* Returns false if any hook failed to compile for any harness.
|
|
4174
4351
|
*/
|
|
4175
|
-
async function installHooks(hookFiles, harnessFlag) {
|
|
4352
|
+
async function installHooks(hookFiles, harnessFlag, configHarness) {
|
|
4176
4353
|
if (hookFiles.length === 0)
|
|
4177
4354
|
return true;
|
|
4178
|
-
const
|
|
4179
|
-
|
|
4180
|
-
:
|
|
4355
|
+
const adapters = (0, adapter_registry_js_1.resolveHarnessAdapters)({
|
|
4356
|
+
root: process.cwd(),
|
|
4357
|
+
flag: harnessFlag,
|
|
4358
|
+
configHarness,
|
|
4359
|
+
});
|
|
4181
4360
|
// Validate registered providers first → the names a hook's provider() ref may
|
|
4182
4361
|
// resolve to (an unsafe provider fails the whole compile, like a bad hook).
|
|
4183
4362
|
let registeredProviders;
|
|
@@ -4194,10 +4373,13 @@ async function installHooks(hookFiles, harnessFlag) {
|
|
|
4194
4373
|
let ok = true;
|
|
4195
4374
|
for (const file of hookFiles) {
|
|
4196
4375
|
try {
|
|
4197
|
-
|
|
4198
|
-
|
|
4199
|
-
|
|
4200
|
-
console.
|
|
4376
|
+
// Fan out: the same compiled hook lands in each enabled harness's config.
|
|
4377
|
+
for (const adapter of adapters) {
|
|
4378
|
+
const r = await installHookFile(file, adapter, registeredProviders);
|
|
4379
|
+
console.log(`✓ ${file} → ${r.settingsPath} (role: ${r.role}, harness: ${adapter.name})`);
|
|
4380
|
+
if (r.warning)
|
|
4381
|
+
console.warn(`⚠ ${r.warning}`);
|
|
4382
|
+
}
|
|
4201
4383
|
}
|
|
4202
4384
|
catch (e) {
|
|
4203
4385
|
if (e instanceof hook_program_js_1.HookCompileError) {
|
|
@@ -4781,7 +4963,7 @@ async function main() {
|
|
|
4781
4963
|
let valid = true;
|
|
4782
4964
|
if (specs.length > 0)
|
|
4783
4965
|
valid = (await compile(specs, config, { harnessFlag })) && valid;
|
|
4784
|
-
valid = (await installHooks(hooks, harnessFlag)) && valid;
|
|
4966
|
+
valid = (await installHooks(hooks, harnessFlag, config.harness)) && valid;
|
|
4785
4967
|
// Keep an existing whole-harness registry in sync (cheap, opt-in) so the
|
|
4786
4968
|
// user never hand-runs `generate-harness`. Skipped when no harness.gen.ts.
|
|
4787
4969
|
if (specs.length > 0)
|