vigiles 11.0.0 → 12.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude-plugin/plugin.json +9 -0
- package/README.md +9 -4
- package/action.yml +13 -2
- package/dist/adapter-conformance.js +6 -0
- package/dist/adapter-registry.d.ts +20 -0
- package/dist/adapter-registry.js +27 -0
- package/dist/adapters/claude-code/hook-protocol.js +4 -0
- package/dist/adapters/claude-code/runtime.js +12 -0
- package/dist/adapters/codex/eval.js +3 -0
- package/dist/adapters/codex/hook-protocol.d.ts +9 -1
- package/dist/adapters/codex/hook-protocol.js +10 -0
- package/dist/adapters/codex/runtime.js +10 -0
- package/dist/adapters/opencode/runtime.js +4 -0
- package/dist/cli-commands.d.ts +1 -1
- package/dist/cli-commands.js +1 -0
- package/dist/cli.js +211 -29
- package/dist/core/hook-protocol.d.ts +15 -0
- package/dist/core/runtime.d.ts +20 -0
- package/dist/core/types.d.ts +12 -0
- package/dist/eval-cache.d.ts +6 -0
- package/dist/eval-cache.js +2 -0
- package/dist/eval-lock.d.ts +192 -0
- package/dist/eval-lock.js +286 -0
- package/dist/eval.d.ts +33 -20
- package/dist/eval.js +199 -51
- package/dist/setup-plan.d.ts +37 -0
- package/dist/setup-plan.js +66 -4
- package/hooks/eval-lock-nudge.sh +21 -0
- package/package.json +1 -1
- package/skills/test-harness/SKILL.md +27 -0
|
@@ -31,5 +31,20 @@ export interface HookProtocol {
|
|
|
31
31
|
* Used by `compileHookProgram` when rendering the settings block.
|
|
32
32
|
*/
|
|
33
33
|
readonly matcherStyle?: "exact" | "regex";
|
|
34
|
+
/**
|
|
35
|
+
* The events whose hook can inject **developer context** into the agent by
|
|
36
|
+
* printing `{ hookSpecificOutput: { hookEventName, additionalContext } }` on
|
|
37
|
+
* stdout. The inject *shape* is shared across Claude Code and Codex (so the
|
|
38
|
+
* runtime emits it once, not per-harness); the genuinely per-harness fact is
|
|
39
|
+
* **which events honor it** — and encoding it here is what makes "this harness
|
|
40
|
+
* can deliver an inject hook" a TESTED contract instead of an assumption. Both
|
|
41
|
+
* Claude Code and Codex support the main lifecycle events (SessionStart,
|
|
42
|
+
* UserPromptSubmit, PostToolUse); a few (Stop, SubagentStop, PreCompact) carry
|
|
43
|
+
* no context on either. An empty list means the harness cannot inject context
|
|
44
|
+
* from a hook at all. Verified for Codex against the official hooks docs
|
|
45
|
+
* (developers.openai.com/codex/hooks). The conformance kit asserts a
|
|
46
|
+
* shell-hook harness declares a non-empty set.
|
|
47
|
+
*/
|
|
48
|
+
readonly injectableEvents: readonly string[];
|
|
34
49
|
}
|
|
35
50
|
//# sourceMappingURL=hook-protocol.d.ts.map
|
package/dist/core/runtime.d.ts
CHANGED
|
@@ -34,5 +34,25 @@ export interface HarnessRuntime {
|
|
|
34
34
|
readonly args: readonly string[];
|
|
35
35
|
readonly env: Record<string, string>;
|
|
36
36
|
};
|
|
37
|
+
/**
|
|
38
|
+
* Reduce a raw `--version` string to the **behaviorally-significant** token the
|
|
39
|
+
* cache + lock key on — so a harness upgrade that actually moves agent behavior
|
|
40
|
+
* (new system prompt / tool defs) invalidates a stale replay, while churn that
|
|
41
|
+
* doesn't shouldn't partition the key. **What counts as significant is
|
|
42
|
+
* per-harness**, which is exactly why this lives on the port rather than as a
|
|
43
|
+
* universal `major.minor` rule:
|
|
44
|
+
*
|
|
45
|
+
* - **Claude Code** is semver-ish — `major.minor` bumps roughly quarterly
|
|
46
|
+
* (0.2 → 1.0 → 2.0 → 2.1 over 16 months) while patches ship ~daily — so it
|
|
47
|
+
* returns `major.minor`: a real behavior boundary, rare enough not to churn.
|
|
48
|
+
* - **Codex** is perpetual `0.x` where the *minor* IS the patch cadence (~2
|
|
49
|
+
* bumps/week), so `major.minor` would churn weekly — it returns `""`, opting
|
|
50
|
+
* out of version partitioning and relying on the dated model id +
|
|
51
|
+
* `evalApiVersion` instead.
|
|
52
|
+
*
|
|
53
|
+
* `""` means "don't partition on the harness version." Pure (no spawn — the
|
|
54
|
+
* caller resolves the raw string via `agentBinary --version`); unit-testable.
|
|
55
|
+
*/
|
|
56
|
+
versionKey(raw: string): string;
|
|
37
57
|
}
|
|
38
58
|
//# sourceMappingURL=runtime.d.ts.map
|
package/dist/core/types.d.ts
CHANGED
|
@@ -346,6 +346,18 @@ export interface VigilesConfig {
|
|
|
346
346
|
audit?: {
|
|
347
347
|
measure?: boolean;
|
|
348
348
|
};
|
|
349
|
+
/**
|
|
350
|
+
* `vigiles eval` preferences. `apiVersion` is the hand-bumped **behavior epoch**
|
|
351
|
+
* folded into the eval LOCK's input hash (`src/eval-lock.ts`): bump it when a
|
|
352
|
+
* harness-side change YOU made (a CLAUDE.md edit, a global hook) would shift
|
|
353
|
+
* eval outputs but isn't otherwise visible to the lock — so `vigiles eval
|
|
354
|
+
* --check` reports the committed eval results STALE and forces a local re-run.
|
|
355
|
+
* Default 1. Distinct from the (auto-resolved) `claude` CLI version, which is
|
|
356
|
+
* recorded as provenance but deliberately NOT hashed.
|
|
357
|
+
*/
|
|
358
|
+
eval?: {
|
|
359
|
+
apiVersion?: number;
|
|
360
|
+
};
|
|
349
361
|
}
|
|
350
362
|
/** Valid marker types for rule detection. */
|
|
351
363
|
export type MarkerType = "headings" | "checkboxes";
|
package/dist/eval-cache.d.ts
CHANGED
|
@@ -43,6 +43,12 @@ export interface CacheRecord {
|
|
|
43
43
|
/** Text files present in the cwd after the run (relative path → contents). */
|
|
44
44
|
readonly files: Record<string, string>;
|
|
45
45
|
}
|
|
46
|
+
/**
|
|
47
|
+
* Canonicalize a value so the key is stable regardless of object key order —
|
|
48
|
+
* recursively sorts object keys. Arrays keep order (it's significant for tools).
|
|
49
|
+
* Exported so the eval LOCK ({@link ./eval-lock}) hashes its inputs the same way.
|
|
50
|
+
*/
|
|
51
|
+
export declare function canonical(value: unknown): unknown;
|
|
46
52
|
/**
|
|
47
53
|
* Cache record-format version, SALTED into every key (Jest `CACHE_VERSION` /
|
|
48
54
|
* webpack `cache.version` pattern). Bump when the `CacheRecord` shape — or how a
|
package/dist/eval-cache.js
CHANGED
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
3
|
exports.CACHE_FORMAT_VERSION = void 0;
|
|
4
|
+
exports.canonical = canonical;
|
|
4
5
|
exports.cacheKey = cacheKey;
|
|
5
6
|
exports.readCache = readCache;
|
|
6
7
|
exports.writeCache = writeCache;
|
|
@@ -32,6 +33,7 @@ const SKIP_DIRS = new Set(["node_modules", ".git"]);
|
|
|
32
33
|
/**
|
|
33
34
|
* Canonicalize a value so the key is stable regardless of object key order —
|
|
34
35
|
* recursively sorts object keys. Arrays keep order (it's significant for tools).
|
|
36
|
+
* Exported so the eval LOCK ({@link ./eval-lock}) hashes its inputs the same way.
|
|
35
37
|
*/
|
|
36
38
|
function canonical(value) {
|
|
37
39
|
if (Array.isArray(value))
|
|
@@ -0,0 +1,192 @@
|
|
|
1
|
+
import { type SHA256Hash } from "./core/hash.js";
|
|
2
|
+
/** Lock mode: never touch the lock / verify-only (CI) / record-and-write (local). */
|
|
3
|
+
export type LockMode = "off" | "check" | "update";
|
|
4
|
+
/**
|
|
5
|
+
* Per-spec lock overrides (additive on `EvalSpec`/`TriggerRateSpec`). Normally the
|
|
6
|
+
* mode comes from the CLI (`eval --check`/`--update` → `VIGILES_EVAL_LOCK`) and
|
|
7
|
+
* the dir/epoch from defaults/config; set these to drive the lock programmatically
|
|
8
|
+
* (or to point a test at a throwaway dir). Each field falls back to its env/default.
|
|
9
|
+
*/
|
|
10
|
+
export interface EvalLockOptions {
|
|
11
|
+
/** Override the lock mode (else `VIGILES_EVAL_LOCK`, else `off`). */
|
|
12
|
+
readonly mode?: LockMode;
|
|
13
|
+
/** Override the lock directory (else `<cwd>/.vigiles/eval-locks`). */
|
|
14
|
+
readonly dir?: string;
|
|
15
|
+
/** Override the behavior epoch (else `VIGILES_EVAL_API_VERSION`, else 1). */
|
|
16
|
+
readonly evalApiVersion?: number;
|
|
17
|
+
}
|
|
18
|
+
/**
|
|
19
|
+
* On-disk lock-format version, salted into nothing (the lock is keyed by name,
|
|
20
|
+
* not by hash) but VALIDATED on read so an incompatible shape fails loud rather
|
|
21
|
+
* than deserializing into a stale structure. Bump on a breaking shape change.
|
|
22
|
+
*/
|
|
23
|
+
export declare const LOCK_VERSION = 1;
|
|
24
|
+
/** Default directory for committed eval locks (tracked, NOT gitignored). */
|
|
25
|
+
export declare const DEFAULT_LOCK_DIR = ".vigiles/eval-locks";
|
|
26
|
+
/**
|
|
27
|
+
* The model-affecting inputs hashed into a lock's `inputsHash`. Everything here
|
|
28
|
+
* is something that, if it changes, means the recorded model behavior is stale
|
|
29
|
+
* and you MUST re-drive the model (→ subscription → local). Deliberately EXCLUDED:
|
|
30
|
+
* the scoring `measure`/assertions (re-run live against the replayed report), the
|
|
31
|
+
* trial count (a sample-size knob, not a behavior input), and per-run env noise.
|
|
32
|
+
*/
|
|
33
|
+
export interface EvalLockInputs {
|
|
34
|
+
/** Model id used (folded in; a floating alias can't detect weight drift — warned). */
|
|
35
|
+
readonly model: string;
|
|
36
|
+
/**
|
|
37
|
+
* A hand-bumped behavior epoch the project owns (`.vigilesrc.json`
|
|
38
|
+
* `eval.apiVersion`), bumped when a harness-side change YOU made (a CLAUDE.md
|
|
39
|
+
* edit, a global hook) would shift eval outputs but isn't otherwise in the
|
|
40
|
+
* inputs. The escape hatch for "force a re-eval."
|
|
41
|
+
*/
|
|
42
|
+
readonly evalApiVersion: number;
|
|
43
|
+
/**
|
|
44
|
+
* The seam-specific canonical input object — the tasks/prompts/files/settings/
|
|
45
|
+
* sorted-tools/pluginDirHash/serialized-checks that steer the model. Assembled
|
|
46
|
+
* by each entry point (it knows its own shape) and hashed opaquely here.
|
|
47
|
+
*/
|
|
48
|
+
readonly inputs: unknown;
|
|
49
|
+
}
|
|
50
|
+
/**
|
|
51
|
+
* Why the harness binary version is **NOT** hashed (only recorded as provenance):
|
|
52
|
+
* `--check` runs in CI where `claude` is PINNED to a fixed version, while a dev's
|
|
53
|
+
* local `claude` is whatever they have — folding the version into the hash would
|
|
54
|
+
* false-trip `--check` on every PR where those differ. It is also the lock's
|
|
55
|
+
* honest scope: the gate verifies your committed results match your current
|
|
56
|
+
* *author-controlled inputs*, not current model/harness behavior (there is no
|
|
57
|
+
* automated live run). Harness/model drift is caught when YOU re-run `--update`
|
|
58
|
+
* locally and review the moved numbers in the git diff. Keeping the version out
|
|
59
|
+
* of the hash is what lets `--check` stay binary-free + deterministic in CI.
|
|
60
|
+
* (The eval CACHE still keys on it — that's local replay soundness, a different
|
|
61
|
+
* axis.) See research/cache-invalidation.md.
|
|
62
|
+
*/
|
|
63
|
+
/** Deterministic content hash of a lock's model-affecting inputs. */
|
|
64
|
+
export declare function evalInputsHash(input: EvalLockInputs): SHA256Hash;
|
|
65
|
+
/** A committed eval lock: the integrity stamp + the replayable recorded report. */
|
|
66
|
+
export interface EvalLock {
|
|
67
|
+
readonly version: number;
|
|
68
|
+
/** The eval's report name (human-facing; also the lock filename slug source). */
|
|
69
|
+
readonly name: string;
|
|
70
|
+
/** Hash of the model-affecting inputs ({@link evalInputsHash}). */
|
|
71
|
+
readonly inputsHash: string;
|
|
72
|
+
/** The model id the report was produced against (for the drift warning). */
|
|
73
|
+
readonly model: string;
|
|
74
|
+
/** The harness version token at record time (provenance; already in the hash). */
|
|
75
|
+
readonly harnessVersionKey: string;
|
|
76
|
+
/** The behavior epoch at record time (provenance; already in the hash). */
|
|
77
|
+
readonly evalApiVersion: number;
|
|
78
|
+
/** ISO-8601 timestamp the lock was recorded (provenance; NOT in the hash). */
|
|
79
|
+
readonly builtAt: string;
|
|
80
|
+
/**
|
|
81
|
+
* The entry point's recorded report — the model's observed behavior, REPLAYED
|
|
82
|
+
* verbatim on `--check` so the script's own assertions judge it. Stored as the
|
|
83
|
+
* exact return type of the entry point (`EvalReport` / `TriggerRateReport` /
|
|
84
|
+
* `CheckReport`) so replay is transparent to the caller.
|
|
85
|
+
*/
|
|
86
|
+
readonly report: unknown;
|
|
87
|
+
}
|
|
88
|
+
/** Filesystem-safe slug for a report name (the lock filename). */
|
|
89
|
+
export declare function lockSlug(name: string): string;
|
|
90
|
+
/** Path to a named eval's lock file under `dir`. */
|
|
91
|
+
export declare function lockPath(dir: string, name: string): string;
|
|
92
|
+
/**
|
|
93
|
+
* Read a named eval's lock. A MISS (no file) returns `null`. A CORRUPT or
|
|
94
|
+
* wrong-version file **throws** — a broken lock is a real failure the CI gate
|
|
95
|
+
* must surface, not silently treat as "no lock" (which would let a stale eval
|
|
96
|
+
* pass). The message says how to recover.
|
|
97
|
+
*/
|
|
98
|
+
export declare function readLock(dir: string, name: string): EvalLock | null;
|
|
99
|
+
/**
|
|
100
|
+
* Whether ANY lock has been committed under `dir`. The CI staleness gate
|
|
101
|
+
* (`eval --check`) uses this to stay a NO-OP until the feature is in use: a repo
|
|
102
|
+
* that has never run `eval --update` has no locks, so there is nothing to verify
|
|
103
|
+
* and CI passes green. Once the first lock is committed, every named eval is held
|
|
104
|
+
* to having a fresh one (a new unlocked eval then reads as stale). The graduated,
|
|
105
|
+
* opt-in-by-committing behavior that keeps a fresh `init` from going red.
|
|
106
|
+
*/
|
|
107
|
+
export declare function anyLocksCommitted(dir: string): boolean;
|
|
108
|
+
/**
|
|
109
|
+
* Does an edited path plausibly change an eval's INPUTS — so a committed lock may
|
|
110
|
+
* now be stale? Two surfaces feed the hash: a skill's trigger surface (`SKILL.md`)
|
|
111
|
+
* and the eval script that holds the prompts/spec (`*.eval.{mjs,cjs,js,mts,cts,ts}`).
|
|
112
|
+
* Pure (string-only) so the nudge hook stays cheap and never runs an eval script.
|
|
113
|
+
*/
|
|
114
|
+
export declare function isEvalInputFile(path: string): boolean;
|
|
115
|
+
/**
|
|
116
|
+
* The NON-BLOCKING nudge to emit after an eval-input edit when committed locks
|
|
117
|
+
* exist, or `null` for no nudge. Self-gating: it stays silent until you've opted
|
|
118
|
+
* into the lock (committed one), so it can't annoy a repo that doesn't use evals.
|
|
119
|
+
* It deliberately does NOT recompute staleness (that needs the eval script + is
|
|
120
|
+
* the job of `eval --check`) — a reminder, not a gate. The honest harness-neutral
|
|
121
|
+
* reminder; how it reaches the agent (both CC and Codex inject `additionalContext`
|
|
122
|
+
* on `PostToolUse`) is the caller's concern. See docs/harness-testing-*.md.
|
|
123
|
+
*/
|
|
124
|
+
export declare function evalLockNudge(filePath: string, lockDir: string): string | null;
|
|
125
|
+
/** Write a named eval's lock (pretty JSON for a reviewable git diff). */
|
|
126
|
+
export declare function writeLock(dir: string, lock: EvalLock): void;
|
|
127
|
+
/**
|
|
128
|
+
* Build a fresh lock envelope from a just-recorded report (the `--update` write).
|
|
129
|
+
* `builtAt` is passed in (never read from the clock here) so the module stays
|
|
130
|
+
* pure + deterministically testable; the CLI stamps the real timestamp.
|
|
131
|
+
*/
|
|
132
|
+
export declare function buildLock(args: {
|
|
133
|
+
readonly name: string;
|
|
134
|
+
readonly inputsHash: string;
|
|
135
|
+
readonly model: string;
|
|
136
|
+
readonly harnessVersionKey: string;
|
|
137
|
+
readonly evalApiVersion: number;
|
|
138
|
+
readonly builtAt: string;
|
|
139
|
+
readonly report: unknown;
|
|
140
|
+
}): EvalLock;
|
|
141
|
+
/** What the lock layer decides an entry point should do for this run. */
|
|
142
|
+
export type LockDecision =
|
|
143
|
+
/** Drive the model normally (mode `off`, or `update`, or `check` with no lock-skip). */
|
|
144
|
+
{
|
|
145
|
+
readonly kind: "run";
|
|
146
|
+
}
|
|
147
|
+
/** `check` + a matching fresh lock → return the recorded report, NO model call. */
|
|
148
|
+
| {
|
|
149
|
+
readonly kind: "replay";
|
|
150
|
+
readonly report: unknown;
|
|
151
|
+
}
|
|
152
|
+
/** `check` + a missing/stale lock → fail; the caller throws `reason`. */
|
|
153
|
+
| {
|
|
154
|
+
readonly kind: "stale";
|
|
155
|
+
readonly reason: string;
|
|
156
|
+
};
|
|
157
|
+
/**
|
|
158
|
+
* Decide what `check` mode should do given the current input hash and the
|
|
159
|
+
* committed lock. `off`/`update` always `run` (update records afterwards). `check`
|
|
160
|
+
* replays a matching lock (no model) and is `stale` on a missing lock or a hash
|
|
161
|
+
* mismatch — the deterministic CI gate.
|
|
162
|
+
*/
|
|
163
|
+
export declare function decideLock(mode: LockMode, name: string, currentHash: string, existing: EvalLock | null): LockDecision;
|
|
164
|
+
/** A single numeric leaf that moved between the prior lock and a fresh `--update`. */
|
|
165
|
+
export interface NumberDelta {
|
|
166
|
+
readonly path: string;
|
|
167
|
+
readonly before: number;
|
|
168
|
+
readonly after: number;
|
|
169
|
+
}
|
|
170
|
+
/**
|
|
171
|
+
* Collect the numeric leaves that changed between two recorded reports — the
|
|
172
|
+
* human-facing delta printed at `--update` time (e.g. `rate: 0.900 → 0.650`).
|
|
173
|
+
* Generic over any report shape (walks numbers by dotted path), so it works for
|
|
174
|
+
* `EvalReport`, `TriggerRateReport`, and `CheckReport` without per-type code. The
|
|
175
|
+
* committed git diff is the primary review surface; this is the at-a-glance echo.
|
|
176
|
+
*/
|
|
177
|
+
export declare function diffReportNumbers(before: unknown, after: unknown): NumberDelta[];
|
|
178
|
+
/** Render the `--update` result for a human: NEW lock, or the per-number deltas. */
|
|
179
|
+
export declare function formatLockUpdate(name: string, deltas: readonly NumberDelta[], isNew: boolean): string;
|
|
180
|
+
/**
|
|
181
|
+
* Read the lock mode from the environment (`VIGILES_EVAL_LOCK`), set by the CLI's
|
|
182
|
+
* `eval --check` / `--update` flags. A run knob (like `VIGILES_TRIALS`): the CLI
|
|
183
|
+
* is the only place that should set it. Anything unrecognized → `off`.
|
|
184
|
+
*/
|
|
185
|
+
export declare function lockModeFromEnv(env?: NodeJS.ProcessEnv): LockMode;
|
|
186
|
+
/**
|
|
187
|
+
* The behavior epoch (`evalApiVersion`) for this run, read from the env the CLI
|
|
188
|
+
* populates from `.vigilesrc.json` `eval.apiVersion`. Default 1. A malformed
|
|
189
|
+
* value falls back to 1 (never throws) — the lock stays usable.
|
|
190
|
+
*/
|
|
191
|
+
export declare function evalApiVersionFromEnv(env?: NodeJS.ProcessEnv): number;
|
|
192
|
+
//# sourceMappingURL=eval-lock.d.ts.map
|
|
@@ -0,0 +1,286 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.DEFAULT_LOCK_DIR = exports.LOCK_VERSION = void 0;
|
|
4
|
+
exports.evalInputsHash = evalInputsHash;
|
|
5
|
+
exports.lockSlug = lockSlug;
|
|
6
|
+
exports.lockPath = lockPath;
|
|
7
|
+
exports.readLock = readLock;
|
|
8
|
+
exports.anyLocksCommitted = anyLocksCommitted;
|
|
9
|
+
exports.isEvalInputFile = isEvalInputFile;
|
|
10
|
+
exports.evalLockNudge = evalLockNudge;
|
|
11
|
+
exports.writeLock = writeLock;
|
|
12
|
+
exports.buildLock = buildLock;
|
|
13
|
+
exports.decideLock = decideLock;
|
|
14
|
+
exports.diffReportNumbers = diffReportNumbers;
|
|
15
|
+
exports.formatLockUpdate = formatLockUpdate;
|
|
16
|
+
exports.lockModeFromEnv = lockModeFromEnv;
|
|
17
|
+
exports.evalApiVersionFromEnv = evalApiVersionFromEnv;
|
|
18
|
+
/**
|
|
19
|
+
* vigiles — the eval LOCK (the CI staleness gate for evals run on a subscription).
|
|
20
|
+
*
|
|
21
|
+
* Real-model evals authenticate as your own `claude` CLI on your Claude
|
|
22
|
+
* subscription, so they only run **locally** — never in CI (no metered key, and
|
|
23
|
+
* the subscription can't be driven from a headless runner). That leaves a hole:
|
|
24
|
+
* how does CI know the committed eval numbers still match the current inputs?
|
|
25
|
+
* Someone edits a skill, forgets to re-eval, and ships stale results.
|
|
26
|
+
*
|
|
27
|
+
* The lock closes it with an **integrity hash**, NOT a cache. The two are
|
|
28
|
+
* different mechanisms and must not be confused:
|
|
29
|
+
*
|
|
30
|
+
* - the eval CACHE ({@link ./eval-cache}) is a LOCAL speed optimization
|
|
31
|
+
* (gitignored, keyed store, skips the model call when re-scoring `measure`);
|
|
32
|
+
* - the eval LOCK is a COMMITTED staleness stamp (`.vigiles/eval-locks/<slug>.lock.json`),
|
|
33
|
+
* reviewed in the git diff, checked in CI without ever touching the model.
|
|
34
|
+
*
|
|
35
|
+
* It is the snapshot/lockfile pattern (`Cargo.lock` + `npm ci`; `jest --ci` /
|
|
36
|
+
* `cargo-insta`): you produce numbers locally with `--update`, commit the lock,
|
|
37
|
+
* and CI runs `--check` — recompute the input hash, compare, and fail "stale,
|
|
38
|
+
* re-run `--update`" on a mismatch. The committed diff of `recall: 0.90 → 0.65`
|
|
39
|
+
* IS the quality gate a human reviews. The same integrity-hash-of-inputs idea
|
|
40
|
+
* vigiles already ships in `core/integrity.ts` (compiled markdown) and
|
|
41
|
+
* `core/sidecar.ts` (spec inputs), applied a third time to eval results.
|
|
42
|
+
*
|
|
43
|
+
* Honest scope (no fiction): the lock promises "your committed results match your
|
|
44
|
+
* current inputs," NOT "your results reflect current model behavior." Model /
|
|
45
|
+
* harness drift is only caught when YOU re-run `--update` locally — there is no
|
|
46
|
+
* automated live run, by design. The clean split that makes replay sound: the
|
|
47
|
+
* lock stores only the model's OBSERVED BEHAVIOR (the report); the script's own
|
|
48
|
+
* assertions (`assertTriggerRate` / `assertSignificant`) re-run live against the
|
|
49
|
+
* replayed report, so a threshold-only edit is a valid replay (no model call)
|
|
50
|
+
* while an input change is stale. See `research/cache-invalidation.md`.
|
|
51
|
+
*
|
|
52
|
+
* Pure + model-free (the only side effects are the two small fs helpers); the
|
|
53
|
+
* inputs hash reuses `canonical` from `eval-cache.ts` so the lock and the cache
|
|
54
|
+
* canonicalize identically.
|
|
55
|
+
*/
|
|
56
|
+
const node_fs_1 = require("node:fs");
|
|
57
|
+
const node_path_1 = require("node:path");
|
|
58
|
+
const hash_js_1 = require("./core/hash.js");
|
|
59
|
+
const eval_cache_js_1 = require("./eval-cache.js");
|
|
60
|
+
/**
|
|
61
|
+
* On-disk lock-format version, salted into nothing (the lock is keyed by name,
|
|
62
|
+
* not by hash) but VALIDATED on read so an incompatible shape fails loud rather
|
|
63
|
+
* than deserializing into a stale structure. Bump on a breaking shape change.
|
|
64
|
+
*/
|
|
65
|
+
exports.LOCK_VERSION = 1;
|
|
66
|
+
/** Default directory for committed eval locks (tracked, NOT gitignored). */
|
|
67
|
+
exports.DEFAULT_LOCK_DIR = ".vigiles/eval-locks";
|
|
68
|
+
/**
|
|
69
|
+
* Why the harness binary version is **NOT** hashed (only recorded as provenance):
|
|
70
|
+
* `--check` runs in CI where `claude` is PINNED to a fixed version, while a dev's
|
|
71
|
+
* local `claude` is whatever they have — folding the version into the hash would
|
|
72
|
+
* false-trip `--check` on every PR where those differ. It is also the lock's
|
|
73
|
+
* honest scope: the gate verifies your committed results match your current
|
|
74
|
+
* *author-controlled inputs*, not current model/harness behavior (there is no
|
|
75
|
+
* automated live run). Harness/model drift is caught when YOU re-run `--update`
|
|
76
|
+
* locally and review the moved numbers in the git diff. Keeping the version out
|
|
77
|
+
* of the hash is what lets `--check` stay binary-free + deterministic in CI.
|
|
78
|
+
* (The eval CACHE still keys on it — that's local replay soundness, a different
|
|
79
|
+
* axis.) See research/cache-invalidation.md.
|
|
80
|
+
*/
|
|
81
|
+
/** Deterministic content hash of a lock's model-affecting inputs. */
|
|
82
|
+
function evalInputsHash(input) {
|
|
83
|
+
return (0, hash_js_1.sha256short)(JSON.stringify((0, eval_cache_js_1.canonical)(input)));
|
|
84
|
+
}
|
|
85
|
+
/** Filesystem-safe slug for a report name (the lock filename). */
|
|
86
|
+
function lockSlug(name) {
|
|
87
|
+
const slug = name
|
|
88
|
+
.toLowerCase()
|
|
89
|
+
.replace(/[^a-z0-9]+/g, "-")
|
|
90
|
+
.replace(/^-+|-+$/g, "");
|
|
91
|
+
return slug || "eval";
|
|
92
|
+
}
|
|
93
|
+
/** Path to a named eval's lock file under `dir`. */
|
|
94
|
+
function lockPath(dir, name) {
|
|
95
|
+
return (0, node_path_1.join)(dir, `${lockSlug(name)}.lock.json`);
|
|
96
|
+
}
|
|
97
|
+
/**
|
|
98
|
+
* Read a named eval's lock. A MISS (no file) returns `null`. A CORRUPT or
|
|
99
|
+
* wrong-version file **throws** — a broken lock is a real failure the CI gate
|
|
100
|
+
* must surface, not silently treat as "no lock" (which would let a stale eval
|
|
101
|
+
* pass). The message says how to recover.
|
|
102
|
+
*/
|
|
103
|
+
function readLock(dir, name) {
|
|
104
|
+
const path = lockPath(dir, name);
|
|
105
|
+
if (!(0, node_fs_1.existsSync)(path))
|
|
106
|
+
return null;
|
|
107
|
+
const raw = (0, node_fs_1.readFileSync)(path, "utf-8");
|
|
108
|
+
let data;
|
|
109
|
+
try {
|
|
110
|
+
data = JSON.parse(raw);
|
|
111
|
+
}
|
|
112
|
+
catch {
|
|
113
|
+
throw new Error(`eval lock: corrupt lock ${path} (invalid JSON) — delete it and re-run \`vigiles eval --update\``);
|
|
114
|
+
}
|
|
115
|
+
if (typeof data !== "object" || data === null)
|
|
116
|
+
throw new Error(`eval lock: ${path} is not a JSON object`);
|
|
117
|
+
const obj = data;
|
|
118
|
+
if (obj.version !== exports.LOCK_VERSION)
|
|
119
|
+
throw new Error(`eval lock: ${path} has unsupported version ${String(obj.version)} ` +
|
|
120
|
+
`(expected ${String(exports.LOCK_VERSION)}) — re-run \`vigiles eval --update\``);
|
|
121
|
+
// Slug collision guard: two distinct names can normalize to the same file
|
|
122
|
+
// (`foo/bar` and `foo bar` → `foo-bar.lock.json`). A lock whose stored `name`
|
|
123
|
+
// differs from the one requested belongs to the OTHER eval — treat it as a
|
|
124
|
+
// MISS (not this eval's lock) so `--check` degrades to "stale → re-run" and
|
|
125
|
+
// NEVER replays the wrong eval's report. The stored name is the source of truth.
|
|
126
|
+
if (obj.name !== name)
|
|
127
|
+
return null;
|
|
128
|
+
return obj;
|
|
129
|
+
}
|
|
130
|
+
/**
|
|
131
|
+
* Whether ANY lock has been committed under `dir`. The CI staleness gate
|
|
132
|
+
* (`eval --check`) uses this to stay a NO-OP until the feature is in use: a repo
|
|
133
|
+
* that has never run `eval --update` has no locks, so there is nothing to verify
|
|
134
|
+
* and CI passes green. Once the first lock is committed, every named eval is held
|
|
135
|
+
* to having a fresh one (a new unlocked eval then reads as stale). The graduated,
|
|
136
|
+
* opt-in-by-committing behavior that keeps a fresh `init` from going red.
|
|
137
|
+
*/
|
|
138
|
+
function anyLocksCommitted(dir) {
|
|
139
|
+
if (!(0, node_fs_1.existsSync)(dir))
|
|
140
|
+
return false;
|
|
141
|
+
return (0, node_fs_1.readdirSync)(dir).some((f) => f.endsWith(".lock.json"));
|
|
142
|
+
}
|
|
143
|
+
/**
|
|
144
|
+
* Does an edited path plausibly change an eval's INPUTS — so a committed lock may
|
|
145
|
+
* now be stale? Two surfaces feed the hash: a skill's trigger surface (`SKILL.md`)
|
|
146
|
+
* and the eval script that holds the prompts/spec (`*.eval.{mjs,cjs,js,mts,cts,ts}`).
|
|
147
|
+
* Pure (string-only) so the nudge hook stays cheap and never runs an eval script.
|
|
148
|
+
*/
|
|
149
|
+
function isEvalInputFile(path) {
|
|
150
|
+
const p = path.replace(/\\/g, "/");
|
|
151
|
+
if (/(^|\/)SKILL\.md$/.test(p))
|
|
152
|
+
return true;
|
|
153
|
+
return /\.eval\.(mjs|cjs|js|mts|cts|ts)$/.test(p);
|
|
154
|
+
}
|
|
155
|
+
/**
|
|
156
|
+
* The NON-BLOCKING nudge to emit after an eval-input edit when committed locks
|
|
157
|
+
* exist, or `null` for no nudge. Self-gating: it stays silent until you've opted
|
|
158
|
+
* into the lock (committed one), so it can't annoy a repo that doesn't use evals.
|
|
159
|
+
* It deliberately does NOT recompute staleness (that needs the eval script + is
|
|
160
|
+
* the job of `eval --check`) — a reminder, not a gate. The honest harness-neutral
|
|
161
|
+
* reminder; how it reaches the agent (both CC and Codex inject `additionalContext`
|
|
162
|
+
* on `PostToolUse`) is the caller's concern. See docs/harness-testing-*.md.
|
|
163
|
+
*/
|
|
164
|
+
function evalLockNudge(filePath, lockDir) {
|
|
165
|
+
if (!isEvalInputFile(filePath))
|
|
166
|
+
return null;
|
|
167
|
+
if (!anyLocksCommitted(lockDir))
|
|
168
|
+
return null;
|
|
169
|
+
return (`vigiles: you edited ${filePath}, which can change an eval's inputs — a ` +
|
|
170
|
+
`committed eval lock may now be stale. When you're done, run ` +
|
|
171
|
+
`\`vigiles eval --update\` (local, on your subscription) and commit the ` +
|
|
172
|
+
`updated lock; CI's \`vigiles eval --check\` will otherwise flag it stale. ` +
|
|
173
|
+
`This is a reminder, not a block.`);
|
|
174
|
+
}
|
|
175
|
+
/** Write a named eval's lock (pretty JSON for a reviewable git diff). */
|
|
176
|
+
function writeLock(dir, lock) {
|
|
177
|
+
(0, node_fs_1.mkdirSync)(dir, { recursive: true });
|
|
178
|
+
(0, node_fs_1.writeFileSync)(lockPath(dir, lock.name), JSON.stringify(lock, null, 2) + "\n");
|
|
179
|
+
}
|
|
180
|
+
/**
|
|
181
|
+
* Build a fresh lock envelope from a just-recorded report (the `--update` write).
|
|
182
|
+
* `builtAt` is passed in (never read from the clock here) so the module stays
|
|
183
|
+
* pure + deterministically testable; the CLI stamps the real timestamp.
|
|
184
|
+
*/
|
|
185
|
+
function buildLock(args) {
|
|
186
|
+
return { version: exports.LOCK_VERSION, ...args };
|
|
187
|
+
}
|
|
188
|
+
/**
|
|
189
|
+
* Decide what `check` mode should do given the current input hash and the
|
|
190
|
+
* committed lock. `off`/`update` always `run` (update records afterwards). `check`
|
|
191
|
+
* replays a matching lock (no model) and is `stale` on a missing lock or a hash
|
|
192
|
+
* mismatch — the deterministic CI gate.
|
|
193
|
+
*/
|
|
194
|
+
function decideLock(mode, name, currentHash, existing) {
|
|
195
|
+
if (mode !== "check")
|
|
196
|
+
return { kind: "run" };
|
|
197
|
+
if (!existing)
|
|
198
|
+
return {
|
|
199
|
+
kind: "stale",
|
|
200
|
+
reason: `eval lock missing for "${name}" — no committed results to verify against. ` +
|
|
201
|
+
`Run \`vigiles eval --update\` locally (on your subscription) and commit the lock.`,
|
|
202
|
+
};
|
|
203
|
+
if (existing.inputsHash !== currentHash)
|
|
204
|
+
return {
|
|
205
|
+
kind: "stale",
|
|
206
|
+
reason: `eval lock STALE for "${name}" — the inputs changed since the committed results ` +
|
|
207
|
+
`were recorded (skill/prompts/model/harness/apiVersion). Re-run ` +
|
|
208
|
+
`\`vigiles eval --update\` locally and commit the updated lock.`,
|
|
209
|
+
};
|
|
210
|
+
return { kind: "replay", report: existing.report };
|
|
211
|
+
}
|
|
212
|
+
/**
|
|
213
|
+
* Collect the numeric leaves that changed between two recorded reports — the
|
|
214
|
+
* human-facing delta printed at `--update` time (e.g. `rate: 0.900 → 0.650`).
|
|
215
|
+
* Generic over any report shape (walks numbers by dotted path), so it works for
|
|
216
|
+
* `EvalReport`, `TriggerRateReport`, and `CheckReport` without per-type code. The
|
|
217
|
+
* committed git diff is the primary review surface; this is the at-a-glance echo.
|
|
218
|
+
*/
|
|
219
|
+
function diffReportNumbers(before, after) {
|
|
220
|
+
const out = [];
|
|
221
|
+
walkNumberLeaves(before, after, "", out);
|
|
222
|
+
return out;
|
|
223
|
+
}
|
|
224
|
+
function walkNumberLeaves(a, b, path, out) {
|
|
225
|
+
if (typeof a === "number" && typeof b === "number") {
|
|
226
|
+
if (a !== b)
|
|
227
|
+
out.push({ path, before: a, after: b });
|
|
228
|
+
}
|
|
229
|
+
else if (Array.isArray(a) && Array.isArray(b)) {
|
|
230
|
+
walkArrayLeaves(a, b, path, out);
|
|
231
|
+
}
|
|
232
|
+
else if (isRecord(a) && isRecord(b)) {
|
|
233
|
+
walkRecordLeaves(a, b, path, out);
|
|
234
|
+
}
|
|
235
|
+
}
|
|
236
|
+
function walkArrayLeaves(a, b, path, out) {
|
|
237
|
+
const len = Math.min(a.length, b.length);
|
|
238
|
+
for (let i = 0; i < len; i++)
|
|
239
|
+
walkNumberLeaves(a[i], b[i], `${path}[${String(i)}]`, out);
|
|
240
|
+
}
|
|
241
|
+
function walkRecordLeaves(a, b, path, out) {
|
|
242
|
+
for (const k of Object.keys(a))
|
|
243
|
+
if (k in b)
|
|
244
|
+
walkNumberLeaves(a[k], b[k], path ? `${path}.${k}` : k, out);
|
|
245
|
+
}
|
|
246
|
+
function isRecord(v) {
|
|
247
|
+
return v !== null && typeof v === "object";
|
|
248
|
+
}
|
|
249
|
+
/** Render the `--update` result for a human: NEW lock, or the per-number deltas. */
|
|
250
|
+
function formatLockUpdate(name, deltas, isNew) {
|
|
251
|
+
if (isNew)
|
|
252
|
+
return `eval lock: recorded NEW lock for "${name}"`;
|
|
253
|
+
if (deltas.length === 0)
|
|
254
|
+
return `eval lock: "${name}" updated — no numeric change vs the prior lock`;
|
|
255
|
+
const lines = [
|
|
256
|
+
`eval lock: "${name}" updated — ${String(deltas.length)} value(s) moved:`,
|
|
257
|
+
];
|
|
258
|
+
for (const d of deltas) {
|
|
259
|
+
const dir = d.after > d.before ? "▲" : "▼";
|
|
260
|
+
lines.push(` ${dir} ${d.path}: ${d.before.toFixed(3)} → ${d.after.toFixed(3)}`);
|
|
261
|
+
}
|
|
262
|
+
lines.push(" review the committed lock diff — this is the eval quality gate.");
|
|
263
|
+
return lines.join("\n");
|
|
264
|
+
}
|
|
265
|
+
/**
|
|
266
|
+
* Read the lock mode from the environment (`VIGILES_EVAL_LOCK`), set by the CLI's
|
|
267
|
+
* `eval --check` / `--update` flags. A run knob (like `VIGILES_TRIALS`): the CLI
|
|
268
|
+
* is the only place that should set it. Anything unrecognized → `off`.
|
|
269
|
+
*/
|
|
270
|
+
function lockModeFromEnv(env = process.env) {
|
|
271
|
+
const v = env.VIGILES_EVAL_LOCK;
|
|
272
|
+
return v === "check" || v === "update" ? v : "off";
|
|
273
|
+
}
|
|
274
|
+
/**
|
|
275
|
+
* The behavior epoch (`evalApiVersion`) for this run, read from the env the CLI
|
|
276
|
+
* populates from `.vigilesrc.json` `eval.apiVersion`. Default 1. A malformed
|
|
277
|
+
* value falls back to 1 (never throws) — the lock stays usable.
|
|
278
|
+
*/
|
|
279
|
+
function evalApiVersionFromEnv(env = process.env) {
|
|
280
|
+
const raw = env.VIGILES_EVAL_API_VERSION;
|
|
281
|
+
if (raw === undefined)
|
|
282
|
+
return 1;
|
|
283
|
+
const n = Number.parseInt(raw, 10);
|
|
284
|
+
return Number.isFinite(n) && n >= 0 ? n : 1;
|
|
285
|
+
}
|
|
286
|
+
//# sourceMappingURL=eval-lock.js.map
|