vigiles 16.1.3 → 17.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/adapters/claude-code/run-scripts.d.ts +19 -3
- package/dist/adapters/claude-code/run-scripts.js +17 -7
- package/dist/adapters/codex/eval.js +2 -0
- package/dist/audit-score.js +1 -1
- package/dist/cli.js +7 -2
- package/dist/core/eval-load-phase.d.ts +78 -0
- package/dist/core/eval-load-phase.js +104 -0
- package/dist/eval-define.d.ts +166 -0
- package/dist/eval-define.js +182 -0
- package/dist/eval-entry.d.ts +41 -0
- package/dist/eval-entry.js +203 -0
- package/dist/eval.js +2 -0
- package/dist/judge.js +2 -0
- package/dist/scan-behavioral.js +2 -0
- package/dist/scan.d.ts +21 -0
- package/dist/scan.js +5 -0
- package/dist/test-coverage.d.ts +45 -0
- package/dist/test-coverage.js +91 -3
- package/dist/test.d.ts +2 -0
- package/dist/test.js +8 -1
- package/package.json +1 -1
- package/skills/test-harness/SKILL.md +31 -22
|
@@ -74,10 +74,16 @@ import type { NodeCaps } from "../../ts-runner-caps.js";
|
|
|
74
74
|
* type stripping. Throws a clear, actionable error when neither is available.
|
|
75
75
|
* Pure — exported for testing.
|
|
76
76
|
*
|
|
77
|
+
* `entry` interposes a program that takes the script as its ARGUMENT instead of
|
|
78
|
+
* running the script as the program. `vigiles eval` passes one: an eval file
|
|
79
|
+
* describes its eval rather than running it (see `src/eval-define.ts`), so
|
|
80
|
+
* something has to import the description and execute what it declares. Harness
|
|
81
|
+
* scripts pass nothing and are launched exactly as before.
|
|
82
|
+
*
|
|
77
83
|
* 🔴 The disjunction below is `canRunTypeScript` — keep them together. When they
|
|
78
84
|
* drifted, the tool recommended a `.ts` file and then refused to run it.
|
|
79
85
|
*/
|
|
80
|
-
export declare function interpreterArgs(file: string, caps: NodeCaps): string[];
|
|
86
|
+
export declare function interpreterArgs(file: string, caps: NodeCaps, entry?: string): string[];
|
|
81
87
|
/**
|
|
82
88
|
* Expand the given path/glob patterns into concrete script files. A pattern
|
|
83
89
|
* that is an existing file passes through unchanged; anything else is treated
|
|
@@ -85,8 +91,18 @@ export declare function interpreterArgs(file: string, caps: NodeCaps): string[];
|
|
|
85
91
|
* are deduped and sorted; `node_modules` and `dist` are always ignored.
|
|
86
92
|
*/
|
|
87
93
|
export declare function discoverScripts(patterns: readonly string[], defaultGlob: string, cwd: string): string[];
|
|
94
|
+
/** Extra wiring for {@link runScripts}. */
|
|
95
|
+
export interface RunScriptsOptions {
|
|
96
|
+
/**
|
|
97
|
+
* A program to run INSTEAD of each script, with the script's path as its one
|
|
98
|
+
* argument. `vigiles eval` passes `dist/eval-entry.js`; `vigiles test` passes
|
|
99
|
+
* nothing. See {@link interpreterArgs}.
|
|
100
|
+
*/
|
|
101
|
+
readonly entry?: string;
|
|
102
|
+
}
|
|
88
103
|
/**
|
|
89
|
-
* Run each script as `node <file
|
|
104
|
+
* Run each script as `node <file>` (or `node <entry> <file>`, see
|
|
105
|
+
* {@link RunScriptsOptions}), inheriting stdio so the script's own report
|
|
90
106
|
* streams to the console. `env` is merged over `process.env` for every child
|
|
91
107
|
* (e.g. `VIGILES_TRIALS`). Returns the per-file exit codes + check counts.
|
|
92
108
|
*
|
|
@@ -100,7 +116,7 @@ export declare function discoverScripts(patterns: readonly string[], defaultGlob
|
|
|
100
116
|
* record them (`.vigiles/coverage.json`) and coverage can answer "tested?" from
|
|
101
117
|
* execution rather than from a matching file name.
|
|
102
118
|
*/
|
|
103
|
-
export declare function runScripts(files: readonly string[], cwd: string, env?: NodeJS.ProcessEnv): ScriptRunResult[];
|
|
119
|
+
export declare function runScripts(files: readonly string[], cwd: string, env?: NodeJS.ProcessEnv, opts?: RunScriptsOptions): ScriptRunResult[];
|
|
104
120
|
/**
|
|
105
121
|
* Whether any script FAILED. Neither a skip nor a vacuous run counts: the first
|
|
106
122
|
* declined to run, the second ran and verified nothing, and neither is evidence
|
|
@@ -88,16 +88,25 @@ const ts_runner_caps_js_2 = require("../../ts-runner-caps.js");
|
|
|
88
88
|
* type stripping. Throws a clear, actionable error when neither is available.
|
|
89
89
|
* Pure — exported for testing.
|
|
90
90
|
*
|
|
91
|
+
* `entry` interposes a program that takes the script as its ARGUMENT instead of
|
|
92
|
+
* running the script as the program. `vigiles eval` passes one: an eval file
|
|
93
|
+
* describes its eval rather than running it (see `src/eval-define.ts`), so
|
|
94
|
+
* something has to import the description and execute what it declares. Harness
|
|
95
|
+
* scripts pass nothing and are launched exactly as before.
|
|
96
|
+
*
|
|
91
97
|
* 🔴 The disjunction below is `canRunTypeScript` — keep them together. When they
|
|
92
98
|
* drifted, the tool recommended a `.ts` file and then refused to run it.
|
|
93
99
|
*/
|
|
94
|
-
function interpreterArgs(file, caps) {
|
|
100
|
+
function interpreterArgs(file, caps, entry) {
|
|
101
|
+
// The TS flags are chosen from FILE's extension even when `entry` runs — a
|
|
102
|
+
// JavaScript entry importing a `.ts` eval still needs the loader installed.
|
|
103
|
+
const tail = entry === undefined ? [file] : [entry, file];
|
|
95
104
|
if (!TS_EXT.test(file))
|
|
96
|
-
return
|
|
105
|
+
return tail;
|
|
97
106
|
if (caps.tsx)
|
|
98
|
-
return ["--import", "tsx",
|
|
107
|
+
return ["--import", "tsx", ...tail];
|
|
99
108
|
if (caps.stripTypes)
|
|
100
|
-
return ["--experimental-strip-types",
|
|
109
|
+
return ["--experimental-strip-types", ...tail];
|
|
101
110
|
throw new Error(`Cannot run TypeScript test script "${file}": install tsx ` +
|
|
102
111
|
`(npm i -D tsx) or use Node >= 22.6, or author it as a .mjs file.`);
|
|
103
112
|
}
|
|
@@ -154,7 +163,8 @@ function readCheckReport(path) {
|
|
|
154
163
|
}
|
|
155
164
|
}
|
|
156
165
|
/**
|
|
157
|
-
* Run each script as `node <file
|
|
166
|
+
* Run each script as `node <file>` (or `node <entry> <file>`, see
|
|
167
|
+
* {@link RunScriptsOptions}), inheriting stdio so the script's own report
|
|
158
168
|
* streams to the console. `env` is merged over `process.env` for every child
|
|
159
169
|
* (e.g. `VIGILES_TRIALS`). Returns the per-file exit codes + check counts.
|
|
160
170
|
*
|
|
@@ -168,7 +178,7 @@ function readCheckReport(path) {
|
|
|
168
178
|
* record them (`.vigiles/coverage.json`) and coverage can answer "tested?" from
|
|
169
179
|
* execution rather than from a matching file name.
|
|
170
180
|
*/
|
|
171
|
-
function runScripts(files, cwd, env = {}) {
|
|
181
|
+
function runScripts(files, cwd, env = {}, opts = {}) {
|
|
172
182
|
const caps = (0, ts_runner_caps_js_2.detectNodeCaps)(cwd);
|
|
173
183
|
const results = [];
|
|
174
184
|
const countDir = (0, node_fs_1.mkdtempSync)((0, node_path_1.join)((0, node_os_1.tmpdir)(), "vigiles-checks-"));
|
|
@@ -176,7 +186,7 @@ function runScripts(files, cwd, env = {}) {
|
|
|
176
186
|
files.forEach((file, i) => {
|
|
177
187
|
let argv;
|
|
178
188
|
try {
|
|
179
|
-
argv = interpreterArgs(file, caps);
|
|
189
|
+
argv = interpreterArgs(file, caps, opts.entry);
|
|
180
190
|
}
|
|
181
191
|
catch (e) {
|
|
182
192
|
console.error(`✗ ${file}: ${e.message}`);
|
|
@@ -36,6 +36,7 @@ exports.codexEvalAgentRunner = codexEvalAgentRunner;
|
|
|
36
36
|
exports.codexEvalRunner = codexEvalRunner;
|
|
37
37
|
const node_child_process_1 = require("node:child_process");
|
|
38
38
|
const foreign_runner_js_1 = require("../../core/foreign-runner.js");
|
|
39
|
+
const eval_load_phase_js_1 = require("../../core/eval-load-phase.js");
|
|
39
40
|
const node_fs_1 = require("node:fs");
|
|
40
41
|
const node_path_1 = require("node:path");
|
|
41
42
|
const str = (v) => (typeof v === "string" ? v : "");
|
|
@@ -229,6 +230,7 @@ function codexEvalRunner(args) {
|
|
|
229
230
|
// The DOCUMENTED alternative to the default Claude runner, and the one the
|
|
230
231
|
// untested-skill nudge now recommends — so it is exactly as able to be
|
|
231
232
|
// collected by a stray `npx vitest run`, and exactly as expensive.
|
|
233
|
+
(0, eval_load_phase_js_1.refuseDuringEvalLoad)("driving `codex exec`");
|
|
232
234
|
(0, foreign_runner_js_1.refuseUnderForeignRunner)("driving `codex exec`");
|
|
233
235
|
const r = (0, node_child_process_1.spawnSync)("codex", [
|
|
234
236
|
"exec",
|
package/dist/audit-score.js
CHANGED
|
@@ -49,7 +49,7 @@ const score_core_js_1 = require("./score-core.js");
|
|
|
49
49
|
const W_UNTESTED = 3;
|
|
50
50
|
/** The command that answers the firing question — named, not alluded to. */
|
|
51
51
|
const MEASURE_FIRING_COMMAND = "run `npx vigiles audit` interactively to measure, or add a `*.eval.mjs` " +
|
|
52
|
-
"(`
|
|
52
|
+
"(`export default defineEval({ measureTriggerRate: … })`, vigiles)";
|
|
53
53
|
/** Resolve the terse "thing(s)" plural placeholder against a count:
|
|
54
54
|
* n===1 drops the "(s)" ("1 tool"); otherwise it becomes "s" ("3 tools"). */
|
|
55
55
|
function pluralizeLabel(n, label) {
|
package/dist/cli.js
CHANGED
|
@@ -1706,7 +1706,7 @@ function formatTriggerNudge(triggerableSkills) {
|
|
|
1706
1706
|
return "";
|
|
1707
1707
|
const n = triggerableSkills;
|
|
1708
1708
|
return (`ℹ Do your ${String(n)} skill${n === 1 ? "" : "s"} actually fire? The deterministic read can't tell — ` +
|
|
1709
|
-
`run \`audit\` interactively to measure, or
|
|
1709
|
+
`run \`audit\` interactively to measure, or add a \`*.eval.mjs\` declaring \`measureTriggerRate\` (vigiles).`);
|
|
1710
1710
|
}
|
|
1711
1711
|
/** A terminal summary of the rule map: the CONFIDENT lane counts + the POSSIBLE
|
|
1712
1712
|
* (review) and SKIPPED tiers, with the honest caveat that detection is a
|
|
@@ -4336,7 +4336,12 @@ async function handleRunScripts(kind, args, restArgs) {
|
|
|
4336
4336
|
if (trialsFlag)
|
|
4337
4337
|
env.VIGILES_TRIALS = trialsFlag.split("=")[1];
|
|
4338
4338
|
console.log(`Running ${String(files.length)} ${kind} file(s):\n`);
|
|
4339
|
-
|
|
4339
|
+
// An eval file DESCRIBES its eval; `dist/eval-entry.js` is what imports the
|
|
4340
|
+
// description and runs what it declares. A harness script is still its own
|
|
4341
|
+
// program (it is free, so "import spends money" never applied to it).
|
|
4342
|
+
const results = (0, run_scripts_js_1.runScripts)(files, cwd, env, {
|
|
4343
|
+
...(kind === "eval" ? { entry: (0, node_path_1.resolve)(__dirname, "eval-entry.js") } : {}),
|
|
4344
|
+
});
|
|
4340
4345
|
// Write down WHAT the run exercised, so `lint`/`audit` can answer "tested?"
|
|
4341
4346
|
// from execution instead of from a matching file name. Not a new verb and not
|
|
4342
4347
|
// a flag: the run already happened, and this is the runner recording what it
|
|
@@ -0,0 +1,78 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The paid tier is closed while an eval file is being LOADED.
|
|
3
|
+
*
|
|
4
|
+
* ## The failure this removes
|
|
5
|
+
*
|
|
6
|
+
* An eval file used to do its work at the top level:
|
|
7
|
+
*
|
|
8
|
+
* const report = await measureTriggerRate({ … }); // ← module body
|
|
9
|
+
*
|
|
10
|
+
* In ESM, `import` IS execution, so *reading* such a file spends real money.
|
|
11
|
+
* Measured 2026-08-12 (and reproduced 2026-08-18 with an inert stand-in that
|
|
12
|
+
* writes a marker file instead of spawning): `node -e 'import("./x.eval.mjs")'`
|
|
13
|
+
* ran the whole body and the one guard that exists — `refuseUnderForeignRunner`
|
|
14
|
+
* — stayed SILENT, because under `node -e` there is no `process.argv[1]` at all:
|
|
15
|
+
*
|
|
16
|
+
* node -e 'import(x)' argv[1] = undefined → foreignRunner(…) = null
|
|
17
|
+
*
|
|
18
|
+
* That guard is not broken; it answers a different question ("does somebody
|
|
19
|
+
* else's test runner own this process?"). No process fact distinguishes
|
|
20
|
+
* `node -e 'import(x)'` from a legitimate runner doing `import(x)`, so no
|
|
21
|
+
* argv-shaped guard can close this door without also refusing the correct
|
|
22
|
+
* invocation — the exact failure mode that got `process.env.VITEST` rejected in
|
|
23
|
+
* `foreign-runner.ts`.
|
|
24
|
+
*
|
|
25
|
+
* ## What closes it instead
|
|
26
|
+
*
|
|
27
|
+
* The SHAPE changed: an eval file now DESCRIBES its eval (`defineEval`) and
|
|
28
|
+
* `vigiles eval` runs it. A description cannot spend, so for a conforming file
|
|
29
|
+
* the door is shut by construction and this module is not needed.
|
|
30
|
+
*
|
|
31
|
+
* This module covers the file that is only HALF migrated — a `defineEval` export
|
|
32
|
+
* plus a leftover top-level `measure(…)`. During the import that the eval runner
|
|
33
|
+
* performs, the paid tier refuses. So the usual way anyone runs an eval turns a
|
|
34
|
+
* silent bill into a loud error that names the fix, instead of paying it.
|
|
35
|
+
*
|
|
36
|
+
* ⚠️ HONEST BOUNDARY, stated rather than implied. This is scoped to the runner's
|
|
37
|
+
* own import. `node -e 'import("./half-migrated.eval.mjs")'` still spends,
|
|
38
|
+
* because that file still contains the original defect — the redesign made the
|
|
39
|
+
* defect *avoidable and detectable*, it cannot retroactively fix a file that did
|
|
40
|
+
* not adopt it. The free syntax check is `node --check <file>`, which never
|
|
41
|
+
* executes anything.
|
|
42
|
+
*
|
|
43
|
+
* ## Why the default is OPEN
|
|
44
|
+
*
|
|
45
|
+
* `paid_runEval` and friends are public API; somebody's own script calling them
|
|
46
|
+
* at top level is their business. A gate that defaulted to closed would refuse
|
|
47
|
+
* every correct direct call — again, "a guard that fires on correct input is a
|
|
48
|
+
* guard people delete". So the window is opened by nobody and closed by exactly
|
|
49
|
+
* one caller: `eval-entry.ts`, around its `import()`.
|
|
50
|
+
*
|
|
51
|
+
* Imports NOTHING, so every spawn door (including the adapters) can call it
|
|
52
|
+
* without a cycle — the same reason `foreign-runner.ts` is a leaf.
|
|
53
|
+
*/
|
|
54
|
+
/**
|
|
55
|
+
* Open the no-spend window. Called by `eval-entry.ts` immediately before it
|
|
56
|
+
* imports an eval file, and paired with {@link endEvalLoad} in a `finally`.
|
|
57
|
+
*/
|
|
58
|
+
export declare function beginEvalLoad(): void;
|
|
59
|
+
/** Close the no-spend window: the description is loaded, the runner may spend. */
|
|
60
|
+
export declare function endEvalLoad(): void;
|
|
61
|
+
/** Whether the no-spend window is open (exported for tests + the entry). */
|
|
62
|
+
export declare function inEvalLoad(): boolean;
|
|
63
|
+
/**
|
|
64
|
+
* The refusal message. Separate from the check so a test can assert the WORDS:
|
|
65
|
+
* this fires on a file whose author has not seen the new shape yet, so it has to
|
|
66
|
+
* teach it, not just stop.
|
|
67
|
+
*/
|
|
68
|
+
export declare function evalLoadRefusal(what: string): string;
|
|
69
|
+
/**
|
|
70
|
+
* Refuse to spend model budget while an eval file is being imported.
|
|
71
|
+
*
|
|
72
|
+
* 🔴 CALL THIS AT EVERY REAL-MODEL SPAWN, beside `refuseUnderForeignRunner`, and
|
|
73
|
+
* OUTSIDE any `try` that swallows — `judge` and `deriveAttackReal` both wrap
|
|
74
|
+
* their spawn in `try { … } catch { return fallback }`, so a refusal thrown
|
|
75
|
+
* inside would be downgraded to a score of 0 instead of stopping the run.
|
|
76
|
+
*/
|
|
77
|
+
export declare function refuseDuringEvalLoad(what: string): void;
|
|
78
|
+
//# sourceMappingURL=eval-load-phase.d.ts.map
|
|
@@ -0,0 +1,104 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
/**
|
|
3
|
+
* The paid tier is closed while an eval file is being LOADED.
|
|
4
|
+
*
|
|
5
|
+
* ## The failure this removes
|
|
6
|
+
*
|
|
7
|
+
* An eval file used to do its work at the top level:
|
|
8
|
+
*
|
|
9
|
+
* const report = await measureTriggerRate({ … }); // ← module body
|
|
10
|
+
*
|
|
11
|
+
* In ESM, `import` IS execution, so *reading* such a file spends real money.
|
|
12
|
+
* Measured 2026-08-12 (and reproduced 2026-08-18 with an inert stand-in that
|
|
13
|
+
* writes a marker file instead of spawning): `node -e 'import("./x.eval.mjs")'`
|
|
14
|
+
* ran the whole body and the one guard that exists — `refuseUnderForeignRunner`
|
|
15
|
+
* — stayed SILENT, because under `node -e` there is no `process.argv[1]` at all:
|
|
16
|
+
*
|
|
17
|
+
* node -e 'import(x)' argv[1] = undefined → foreignRunner(…) = null
|
|
18
|
+
*
|
|
19
|
+
* That guard is not broken; it answers a different question ("does somebody
|
|
20
|
+
* else's test runner own this process?"). No process fact distinguishes
|
|
21
|
+
* `node -e 'import(x)'` from a legitimate runner doing `import(x)`, so no
|
|
22
|
+
* argv-shaped guard can close this door without also refusing the correct
|
|
23
|
+
* invocation — the exact failure mode that got `process.env.VITEST` rejected in
|
|
24
|
+
* `foreign-runner.ts`.
|
|
25
|
+
*
|
|
26
|
+
* ## What closes it instead
|
|
27
|
+
*
|
|
28
|
+
* The SHAPE changed: an eval file now DESCRIBES its eval (`defineEval`) and
|
|
29
|
+
* `vigiles eval` runs it. A description cannot spend, so for a conforming file
|
|
30
|
+
* the door is shut by construction and this module is not needed.
|
|
31
|
+
*
|
|
32
|
+
* This module covers the file that is only HALF migrated — a `defineEval` export
|
|
33
|
+
* plus a leftover top-level `measure(…)`. During the import that the eval runner
|
|
34
|
+
* performs, the paid tier refuses. So the usual way anyone runs an eval turns a
|
|
35
|
+
* silent bill into a loud error that names the fix, instead of paying it.
|
|
36
|
+
*
|
|
37
|
+
* ⚠️ HONEST BOUNDARY, stated rather than implied. This is scoped to the runner's
|
|
38
|
+
* own import. `node -e 'import("./half-migrated.eval.mjs")'` still spends,
|
|
39
|
+
* because that file still contains the original defect — the redesign made the
|
|
40
|
+
* defect *avoidable and detectable*, it cannot retroactively fix a file that did
|
|
41
|
+
* not adopt it. The free syntax check is `node --check <file>`, which never
|
|
42
|
+
* executes anything.
|
|
43
|
+
*
|
|
44
|
+
* ## Why the default is OPEN
|
|
45
|
+
*
|
|
46
|
+
* `paid_runEval` and friends are public API; somebody's own script calling them
|
|
47
|
+
* at top level is their business. A gate that defaulted to closed would refuse
|
|
48
|
+
* every correct direct call — again, "a guard that fires on correct input is a
|
|
49
|
+
* guard people delete". So the window is opened by nobody and closed by exactly
|
|
50
|
+
* one caller: `eval-entry.ts`, around its `import()`.
|
|
51
|
+
*
|
|
52
|
+
* Imports NOTHING, so every spawn door (including the adapters) can call it
|
|
53
|
+
* without a cycle — the same reason `foreign-runner.ts` is a leaf.
|
|
54
|
+
*/
|
|
55
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
56
|
+
exports.beginEvalLoad = beginEvalLoad;
|
|
57
|
+
exports.endEvalLoad = endEvalLoad;
|
|
58
|
+
exports.inEvalLoad = inEvalLoad;
|
|
59
|
+
exports.evalLoadRefusal = evalLoadRefusal;
|
|
60
|
+
exports.refuseDuringEvalLoad = refuseDuringEvalLoad;
|
|
61
|
+
/** Whether an eval module is being imported by the eval runner right now. */
|
|
62
|
+
let loading = false;
|
|
63
|
+
/**
|
|
64
|
+
* Open the no-spend window. Called by `eval-entry.ts` immediately before it
|
|
65
|
+
* imports an eval file, and paired with {@link endEvalLoad} in a `finally`.
|
|
66
|
+
*/
|
|
67
|
+
function beginEvalLoad() {
|
|
68
|
+
loading = true;
|
|
69
|
+
}
|
|
70
|
+
/** Close the no-spend window: the description is loaded, the runner may spend. */
|
|
71
|
+
function endEvalLoad() {
|
|
72
|
+
loading = false;
|
|
73
|
+
}
|
|
74
|
+
/** Whether the no-spend window is open (exported for tests + the entry). */
|
|
75
|
+
function inEvalLoad() {
|
|
76
|
+
return loading;
|
|
77
|
+
}
|
|
78
|
+
/**
|
|
79
|
+
* The refusal message. Separate from the check so a test can assert the WORDS:
|
|
80
|
+
* this fires on a file whose author has not seen the new shape yet, so it has to
|
|
81
|
+
* teach it, not just stop.
|
|
82
|
+
*/
|
|
83
|
+
function evalLoadRefusal(what) {
|
|
84
|
+
return (`vigiles refused to spawn a model: ${what} while an eval file was being IMPORTED.\n` +
|
|
85
|
+
` An eval file DESCRIBES its eval; it must not run one at the top level — importing such a\n` +
|
|
86
|
+
` file spends real money (that is the defect this shape removes).\n` +
|
|
87
|
+
` Move the runner call into the description, keyed by the runner's own name:\n` +
|
|
88
|
+
` import { defineEval } from "vigiles";\n` +
|
|
89
|
+
` export default defineEval({ measureTriggerRate: { …the spec you passed… } });\n` +
|
|
90
|
+
` and read the report in \`assert(report)\`. See docs/harness-testing.md § Eval files.`);
|
|
91
|
+
}
|
|
92
|
+
/**
|
|
93
|
+
* Refuse to spend model budget while an eval file is being imported.
|
|
94
|
+
*
|
|
95
|
+
* 🔴 CALL THIS AT EVERY REAL-MODEL SPAWN, beside `refuseUnderForeignRunner`, and
|
|
96
|
+
* OUTSIDE any `try` that swallows — `judge` and `deriveAttackReal` both wrap
|
|
97
|
+
* their spawn in `try { … } catch { return fallback }`, so a refusal thrown
|
|
98
|
+
* inside would be downgraded to a score of 0 instead of stopping the run.
|
|
99
|
+
*/
|
|
100
|
+
function refuseDuringEvalLoad(what) {
|
|
101
|
+
if (loading)
|
|
102
|
+
throw new Error(evalLoadRefusal(what));
|
|
103
|
+
}
|
|
104
|
+
//# sourceMappingURL=eval-load-phase.js.map
|
|
@@ -0,0 +1,166 @@
|
|
|
1
|
+
import type { ArmsCheckReport, EvalDriver, ArmsMeasureSpec, CheckReport, EvalReport, EvalSpec, MeasureSpec, Metrics, TriggerRateReport, TriggerRateSpec } from "./eval.js";
|
|
2
|
+
import type { SelectionMatrixOptions, SelectionReport } from "./scan-behavioral.js";
|
|
3
|
+
/**
|
|
4
|
+
* Brand marking a value as built by {@link defineEval}. A registered symbol, so
|
|
5
|
+
* a descriptor still reads as one across two copies of the package on disk —
|
|
6
|
+
* the shape a monorepo produces routinely.
|
|
7
|
+
*/
|
|
8
|
+
export declare const EVAL_DEFINITION: unique symbol;
|
|
9
|
+
/** `measureSelectionMatrix`'s two arguments as one declarable object. */
|
|
10
|
+
export interface SelectionMatrixSpec extends SelectionMatrixOptions {
|
|
11
|
+
/** The plugin root whose skills are measured against each other. */
|
|
12
|
+
readonly pluginDir: string;
|
|
13
|
+
}
|
|
14
|
+
/**
|
|
15
|
+
* The five measurements an eval file may declare — the runners that already
|
|
16
|
+
* exist, keyed by their own names, so migrating is renaming a call to a key.
|
|
17
|
+
* EXACTLY ONE must be present: zero is a file that declares nothing, and two is
|
|
18
|
+
* a file whose author expected both to run.
|
|
19
|
+
*/
|
|
20
|
+
export interface EvalMeasurements {
|
|
21
|
+
/** A/B across arms with derived metrics — `paid_runEval`. */
|
|
22
|
+
readonly runEval?: EvalSpec<Metrics>;
|
|
23
|
+
/** Checks scored over N trials of one task — `paid_measure`. */
|
|
24
|
+
readonly measure?: MeasureSpec;
|
|
25
|
+
/** The same checks per arm — `paid_measureArms`. */
|
|
26
|
+
readonly measureArms?: ArmsMeasureSpec;
|
|
27
|
+
/** Does a skill's description actually fire — `paid_measureTriggerRate`. */
|
|
28
|
+
readonly measureTriggerRate?: TriggerRateSpec;
|
|
29
|
+
/**
|
|
30
|
+
* The N×N skill-selection collision matrix — `measureSelectionMatrix` from
|
|
31
|
+
* `vigiles/claude-code`. Declared here because a description is a description
|
|
32
|
+
* whatever harness answers it; only `eval-entry.ts` resolves it to a runner.
|
|
33
|
+
*/
|
|
34
|
+
readonly measureSelectionMatrix?: SelectionMatrixSpec;
|
|
35
|
+
}
|
|
36
|
+
/** The report type produced by each declared measurement. */
|
|
37
|
+
export interface EvalReports {
|
|
38
|
+
readonly runEval: EvalReport;
|
|
39
|
+
readonly measure: CheckReport;
|
|
40
|
+
readonly measureArms: ArmsCheckReport;
|
|
41
|
+
readonly measureTriggerRate: TriggerRateReport;
|
|
42
|
+
readonly measureSelectionMatrix: SelectionReport;
|
|
43
|
+
}
|
|
44
|
+
/** The measurement names, as a type. */
|
|
45
|
+
export type EvalKind = keyof EvalMeasurements;
|
|
46
|
+
/** Everything an eval file declares beyond the measurement itself. */
|
|
47
|
+
export interface EvalHooks<K extends EvalKind> {
|
|
48
|
+
/**
|
|
49
|
+
* A lazy precondition. Return a reason to SKIP (exit 77, loud — never a silent
|
|
50
|
+
* green); return nothing to run. Lazy is the point: a probe at the top of the
|
|
51
|
+
* file would be work at import time, which is the shape being removed.
|
|
52
|
+
*/
|
|
53
|
+
readonly skipIf?: () => string | false | undefined | null;
|
|
54
|
+
/**
|
|
55
|
+
* What the report has to show. Throw to fail. The runner has already printed
|
|
56
|
+
* the standard report by the time this is called, so this is for the derived
|
|
57
|
+
* reads — gates, per-arm significance, a verdict line.
|
|
58
|
+
*/
|
|
59
|
+
readonly assert?: (report: EvalReports[K]) => void | Promise<void>;
|
|
60
|
+
/**
|
|
61
|
+
* Drive the measurement with this harness instead of the real `claude` CLI —
|
|
62
|
+
* the declarative home for the second argument of
|
|
63
|
+
* `measureTriggerRate(spec, { evalDriver })`, e.g. `codexEvalDriver` from
|
|
64
|
+
* `vigiles/codex`, or a fake runner in a test.
|
|
65
|
+
*
|
|
66
|
+
* ⚠️ `measureTriggerRate` ONLY. That is not a limitation of this shape but of
|
|
67
|
+
* the product as it stands: `measureTriggerRate` is the one measurement with a
|
|
68
|
+
* public driver seam, and `runEval(spec)` always drives Claude Code (see
|
|
69
|
+
* docs/harnesses.md, footnote 2). Declaring it with any other measurement is a
|
|
70
|
+
* loud error rather than a field that silently does nothing.
|
|
71
|
+
*/
|
|
72
|
+
readonly evalDriver?: EvalDriver;
|
|
73
|
+
}
|
|
74
|
+
/** One measurement + its hooks: what an eval file default-exports. */
|
|
75
|
+
export type EvalDefinitionInput<K extends EvalKind> = Pick<EvalMeasurements, K> & EvalHooks<K>;
|
|
76
|
+
/** The branded value {@link defineEval} returns. */
|
|
77
|
+
export type EvalDefinition<K extends EvalKind = EvalKind> = EvalDefinitionInput<K> & {
|
|
78
|
+
readonly [EVAL_DEFINITION]: true;
|
|
79
|
+
};
|
|
80
|
+
/**
|
|
81
|
+
* What a module's default export declares. Total: every answer the runner has to
|
|
82
|
+
* distinguish is a case here, so no caller can forget one.
|
|
83
|
+
*/
|
|
84
|
+
export type DeclaredEval = {
|
|
85
|
+
readonly ok: true;
|
|
86
|
+
readonly kind: EvalKind;
|
|
87
|
+
readonly spec: unknown;
|
|
88
|
+
} | {
|
|
89
|
+
readonly ok: false;
|
|
90
|
+
readonly why: "not-a-definition";
|
|
91
|
+
} | {
|
|
92
|
+
readonly ok: false;
|
|
93
|
+
readonly why: "declares-nothing";
|
|
94
|
+
} | {
|
|
95
|
+
readonly ok: false;
|
|
96
|
+
readonly why: "declares-several";
|
|
97
|
+
readonly kinds: readonly EvalKind[];
|
|
98
|
+
};
|
|
99
|
+
/** The measurement keys, in a fixed order — the one list, read by everything. */
|
|
100
|
+
export declare const EVAL_KINDS: readonly EvalKind[];
|
|
101
|
+
/** Whether a value came from {@link defineEval}. */
|
|
102
|
+
export declare function isEvalDefinition(v: unknown): v is EvalDefinition;
|
|
103
|
+
/**
|
|
104
|
+
* Read a module's default export as a declaration. Pure — this is the whole
|
|
105
|
+
* reason the descriptor is data: the runner answers "what does this file
|
|
106
|
+
* declare?" without executing anything and without spending anything.
|
|
107
|
+
*/
|
|
108
|
+
export declare function declaredEval(def: unknown): DeclaredEval;
|
|
109
|
+
/**
|
|
110
|
+
* The definition a module namespace carries, through the CJS/ESM interop layer.
|
|
111
|
+
*
|
|
112
|
+
* 🔴 THE SECOND `.default` IS NOT DEFENSIVE — it is the only way a TypeScript
|
|
113
|
+
* eval file works, and it was found by a test, not by reading. Three shapes
|
|
114
|
+
* reach this function and they are genuinely different objects:
|
|
115
|
+
*
|
|
116
|
+
* x.eval.mjs real ESM → `mod.default` IS the definition
|
|
117
|
+
* x.eval.cjs `module.exports = defineEval(…)`
|
|
118
|
+
* → `mod.default` is `module.exports`, the definition
|
|
119
|
+
* x.eval.ts `export default …`, transpiled to CJS by tsx
|
|
120
|
+
* → `mod.default` is `module.exports`, and the
|
|
121
|
+
* definition sits at `mod.default.default`
|
|
122
|
+
*
|
|
123
|
+
* Measured 2026-08-18: without the unwrap a `.eval.ts` file reported
|
|
124
|
+
* "no `export default defineEval({…})` found" — a correct file, refused, with a
|
|
125
|
+
* message that sent its author looking in the wrong place.
|
|
126
|
+
*
|
|
127
|
+
* Brand-directed rather than shape-directed: it reaches deeper ONLY when the
|
|
128
|
+
* outer value is not a definition, so a definition that happens to carry a
|
|
129
|
+
* `default` field of its own is never skipped over.
|
|
130
|
+
*/
|
|
131
|
+
export declare function moduleDefault(mod: unknown): unknown;
|
|
132
|
+
/**
|
|
133
|
+
* Was node pointed STRAIGHT at an eval file? `argv1` is `process.argv[1]`: the
|
|
134
|
+
* path node was started with, which no stray configuration can forge.
|
|
135
|
+
*
|
|
136
|
+
* node x.eval.mjs → the file → true
|
|
137
|
+
* vigiles eval x.eval.mjs → dist/eval-entry.js → false
|
|
138
|
+
* node -e 'import("x.eval.mjs")' → undefined → false
|
|
139
|
+
* npx vitest run → …/vitest/…/forks.js → false
|
|
140
|
+
*
|
|
141
|
+
* Pure: a fact in, a boolean out.
|
|
142
|
+
*/
|
|
143
|
+
export declare function ranAsEntry(argv1: string | undefined): boolean;
|
|
144
|
+
/** The words shown when someone runs an eval file directly. Asserted by a test:
|
|
145
|
+
* a refusal that stops a run without saying what to do instead is a ticket. */
|
|
146
|
+
export declare function ranAsEntryRefusal(argv1: string): string;
|
|
147
|
+
/**
|
|
148
|
+
* Declare the eval a file describes. Returns a plain, branded value; it starts
|
|
149
|
+
* nothing, spends nothing, and touches no filesystem.
|
|
150
|
+
*
|
|
151
|
+
* ```js
|
|
152
|
+
* import { defineEval, assertRates } from "vigiles";
|
|
153
|
+
* import { skill } from "vigiles";
|
|
154
|
+
*
|
|
155
|
+
* export default defineEval({
|
|
156
|
+
* measure: { pluginDir, task: "…", checks: [skill("my:skill")], trials: 3 },
|
|
157
|
+
* assert: (report) => assertRates(report, { min: 0.6 }),
|
|
158
|
+
* });
|
|
159
|
+
* ```
|
|
160
|
+
*
|
|
161
|
+
* @throws if node was pointed straight at the eval file — see the module doc.
|
|
162
|
+
* That is the ONE thing this function does besides build a value, and it is
|
|
163
|
+
* here rather than in each file precisely so that no author can forget it.
|
|
164
|
+
*/
|
|
165
|
+
export declare function defineEval<K extends EvalKind>(def: EvalDefinitionInput<K>): EvalDefinition<K>;
|
|
166
|
+
//# sourceMappingURL=eval-define.d.ts.map
|