vigiles 24.0.0 → 25.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -61,6 +61,24 @@ export declare function codexSkillFired(run: {
61
61
  * turn. Pure fs — unit-testable without a binary.
62
62
  */
63
63
  export declare function installCodexSkills(pluginDir: string, cwd: string): number;
64
+ /**
65
+ * Refuse an `effort` this adapter cannot honour, LOUDLY.
66
+ *
67
+ * The Claude runner pins the reasoning budget through `--effort` plus the env var
68
+ * it sits under. Codex exposes no mapping we have measured — its config carries a
69
+ * `model_reasoning_effort` key, but nothing here has driven the real binary
70
+ * through it, so claiming support would assert something unverified.
71
+ *
72
+ * The alternative — forwarding `task`/`cwd`/`timeoutMs` and dropping `effort` on
73
+ * the floor, as this runner does today — is the silent CC-only path the
74
+ * harness-parity rule forbids: a spec would declare `effort: "low"`, the lock
75
+ * would RECORD `low`, and the run would happen at whatever Codex defaults to.
76
+ * A stale number is recoverable; a confidently mislabelled one is not.
77
+ *
78
+ * Pure so the deferral itself is tested rather than living inside the ignored
79
+ * subprocess region.
80
+ */
81
+ export declare function refuseCodexEffort(effort: string | number | undefined): void;
64
82
  /**
65
83
  * The Codex eval-tier `AgentRunner`: install the run's skills into `.codex/skills`
66
84
  * (Codex's discovery path, vs Claude's `--plugin-dir`), then drive a real
@@ -32,6 +32,7 @@ exports.parseCodexEvalRun = parseCodexEvalRun;
32
32
  exports.codexRunError = codexRunError;
33
33
  exports.codexSkillFired = codexSkillFired;
34
34
  exports.installCodexSkills = installCodexSkills;
35
+ exports.refuseCodexEffort = refuseCodexEffort;
35
36
  exports.codexEvalAgentRunner = codexEvalAgentRunner;
36
37
  exports.codexEvalRunner = codexEvalRunner;
37
38
  const node_child_process_1 = require("node:child_process");
@@ -172,6 +173,31 @@ function installCodexSkills(pluginDir, cwd) {
172
173
  }
173
174
  return n;
174
175
  }
176
+ /**
177
+ * Refuse an `effort` this adapter cannot honour, LOUDLY.
178
+ *
179
+ * The Claude runner pins the reasoning budget through `--effort` plus the env var
180
+ * it sits under. Codex exposes no mapping we have measured — its config carries a
181
+ * `model_reasoning_effort` key, but nothing here has driven the real binary
182
+ * through it, so claiming support would assert something unverified.
183
+ *
184
+ * The alternative — forwarding `task`/`cwd`/`timeoutMs` and dropping `effort` on
185
+ * the floor, as this runner does today — is the silent CC-only path the
186
+ * harness-parity rule forbids: a spec would declare `effort: "low"`, the lock
187
+ * would RECORD `low`, and the run would happen at whatever Codex defaults to.
188
+ * A stale number is recoverable; a confidently mislabelled one is not.
189
+ *
190
+ * Pure so the deferral itself is tested rather than living inside the ignored
191
+ * subprocess region.
192
+ */
193
+ function refuseCodexEffort(effort) {
194
+ if (effort === undefined)
195
+ return;
196
+ throw new Error(`effort (${JSON.stringify(effort)}) is not supported on the Codex adapter: ` +
197
+ `vigiles has not measured a mapping for it, so honouring the spec here ` +
198
+ `would record an effort the run did not use. Drop \`effort\` for this ` +
199
+ `harness, or run the eval on Claude Code.`);
200
+ }
175
201
  /* v8 ignore start -- real codex subprocess; validated against the binary, not the unit gate */
176
202
  /**
177
203
  * The Codex eval-tier `AgentRunner`: install the run's skills into `.codex/skills`
@@ -180,6 +206,7 @@ function installCodexSkills(pluginDir, cwd) {
180
206
  * codexEvalDriver })` dispatches through.
181
207
  */
182
208
  function codexEvalAgentRunner(args) {
209
+ refuseCodexEffort(args.effort);
183
210
  if (args.pluginDir)
184
211
  installCodexSkills(args.pluginDir, args.cwd);
185
212
  return Promise.resolve(codexEvalRunner({
package/dist/cli.js CHANGED
@@ -17,6 +17,7 @@ exports.discoverNestedBundles = discoverNestedBundles;
17
17
  const node_fs_1 = require("node:fs");
18
18
  const node_path_1 = require("node:path");
19
19
  const minimatch_1 = require("minimatch");
20
+ const repo_path_js_1 = require("./core/repo-path.js");
20
21
  const node_child_process_1 = require("node:child_process");
21
22
  const glob_1 = require("glob");
22
23
  const generate_types_js_1 = require("./core/generate-types.js");
@@ -288,11 +289,14 @@ function printWarnings(specFile, warnings) {
288
289
  /** Compile a generator-skill spec from source → SKILL.md. Returns validity. */
289
290
  function compileGeneratorSkillToFile(specPath, source) {
290
291
  const outputPath = specPath.replace(/\.spec\.ts$/, "");
291
- const { markdown, errors } = (0, compile_generator_js_1.compileGeneratorSkill)(source, {
292
+ const { artifact, errors } = (0, compile_generator_js_1.compileGeneratorSkill)(source, {
292
293
  basePath: process.cwd(),
293
294
  specFile: specPath,
294
295
  });
295
- (0, node_fs_1.writeFileSync)((0, node_path_1.resolve)(process.cwd(), outputPath), markdown);
296
+ // Written only when the compile is clean: `artifact` is null otherwise, and
297
+ // `writeArtifact` takes nothing else.
298
+ if (artifact)
299
+ writeArtifact(outputPath, artifact);
296
300
  if (errors.length === 0) {
297
301
  console.log(`\n✓ ${specPath} → ${outputPath} (generator skill)`);
298
302
  return true;
@@ -393,14 +397,17 @@ function writeInstructionMirrors(primaryOutput, harnesses) {
393
397
  /** Compile a declarative SkillSpec → SKILL.md. */
394
398
  function compileSkillToFile(spec, specPath, dialect) {
395
399
  const outputPath = specPath.replace(/\.spec\.ts$/, "");
396
- const { markdown, errors, warnings } = (0, compile_js_1.compileSkill)(spec, {
400
+ const { artifact, errors, warnings } = (0, compile_js_1.compileSkill)(spec, {
397
401
  basePath: process.cwd(),
398
402
  specFile: specPath,
399
403
  // The SKILL.md frontmatter profile comes from the resolved harness — a Codex
400
404
  // repo gets a minimal (name + description) SKILL.md; CC gets the full set.
401
405
  dialect,
402
406
  });
403
- (0, node_fs_1.writeFileSync)((0, node_path_1.resolve)(process.cwd(), outputPath), markdown);
407
+ // Written only when the compile is clean: `artifact` is null otherwise, and
408
+ // `writeArtifact` takes nothing else.
409
+ if (artifact)
410
+ writeArtifact(outputPath, artifact);
404
411
  if (errors.length === 0) {
405
412
  console.log(`\n✓ ${specPath} → ${outputPath}`);
406
413
  printWarnings(specPath, warnings);
@@ -414,12 +421,15 @@ function compileSkillToFile(spec, specPath, dialect) {
414
421
  /** Compile a subagent spec → agents/<name>.md (with its result-contract section). */
415
422
  function compileAgentToFile(spec, specPath, dialect) {
416
423
  const outputPath = specPath.replace(/\.spec\.ts$/, "");
417
- const { markdown, errors, warnings } = (0, compile_js_1.compileAgent)(spec, {
424
+ const { artifact, errors, warnings } = (0, compile_js_1.compileAgent)(spec, {
418
425
  basePath: process.cwd(),
419
426
  specFile: specPath,
420
427
  dialect,
421
428
  });
422
- (0, node_fs_1.writeFileSync)((0, node_path_1.resolve)(process.cwd(), outputPath), markdown);
429
+ // Written only when the compile is clean: `artifact` is null otherwise, and
430
+ // `writeArtifact` takes nothing else.
431
+ if (artifact)
432
+ writeArtifact(outputPath, artifact);
423
433
  if (errors.length === 0) {
424
434
  console.log(`\n✓ ${specPath} → ${outputPath}`);
425
435
  printWarnings(specPath, warnings);
@@ -437,11 +447,14 @@ function compileAgentToFile(spec, specPath, dialect) {
437
447
  */
438
448
  function compileRailwayToFile(spec, specPath, knownAgents) {
439
449
  const outputPath = specPath.replace(/\.spec\.ts$/, "");
440
- const { markdown, errors } = (0, compile_js_1.compileRailway)(spec, {
450
+ const { artifact, errors } = (0, compile_js_1.compileRailway)(spec, {
441
451
  specFile: specPath,
442
452
  knownAgents,
443
453
  });
444
- (0, node_fs_1.writeFileSync)((0, node_path_1.resolve)(process.cwd(), outputPath), markdown);
454
+ // Written only when the compile is clean: `artifact` is null otherwise, and
455
+ // `writeArtifact` takes nothing else.
456
+ if (artifact)
457
+ writeArtifact(outputPath, artifact);
445
458
  if (errors.length === 0) {
446
459
  console.log(`\n✓ ${specPath} → ${outputPath}`);
447
460
  return true;
@@ -1294,6 +1307,18 @@ async function checkSpecRefs(config, silent, dialect) {
1294
1307
  }
1295
1308
  return { issues: found.length, errors: sev === "error" ? found.length : 0 };
1296
1309
  }
1310
+ /**
1311
+ * Write a compiled artifact. Accepts ONLY a {@link StampedMarkdown}, so a body
1312
+ * that failed to compile cannot reach the disk — there is no stamp to pass.
1313
+ *
1314
+ * This replaces four copies of `writeFileSync(path, markdown)` that ran BEFORE
1315
+ * their error check (#173, reproduced in the skill/subagent/railway/generator
1316
+ * compilers after the CLAUDE.md one was fixed). Guarding four call sites would
1317
+ * have left the fifth writable; the type leaves nothing to remember.
1318
+ */
1319
+ function writeArtifact(outputPath, artifact) {
1320
+ (0, node_fs_1.writeFileSync)((0, node_path_1.resolve)(process.cwd(), outputPath), artifact);
1321
+ }
1297
1322
  async function runLint(restArgs, flags, config) {
1298
1323
  const summary = flags.includes("--summary");
1299
1324
  const json = flags.includes("--json");
@@ -6222,14 +6247,12 @@ async function runHookProgramCommand(file) {
6222
6247
  function ensureReportGitignored(cwd, entries) {
6223
6248
  if (entries.length === 0)
6224
6249
  return;
6225
- // An `--out` outside the repo produced entries like
6226
- // `../../../../private/tmp/x/vigiles-report.json`, which ignore NOTHING
6227
- // .gitignore does not reach outside its own tree and accumulate one dead
6228
- // block per output path. Worse in principle than in practice: `audit` is
6229
- // documented as a read-only report, and this made it edit a tracked file for
6230
- // no benefit at all. Inside the repo the write is expected and documented.
6231
- if (entries.some((e) => e.startsWith("..") || (0, node_path_1.isAbsolute)(e)))
6232
- return;
6250
+ // 🔴 THE GUARD THAT USED TO BE HERE IS GONE, and its absence is the point.
6251
+ // It checked at the write site that no entry escaped the repo (#176.8) which
6252
+ // worked, and left the bug writable: the next caller to build an entry list
6253
+ // still got a bare `string[]`. `RepoRelativePath` moves the check into the
6254
+ // TYPE, so an escaping path cannot be handed to this function at all. One
6255
+ // place mints them (`repoRelative`), and it returns null instead.
6233
6256
  const gi = (0, node_path_1.resolve)(cwd, ".gitignore");
6234
6257
  try {
6235
6258
  if (!(0, node_fs_1.existsSync)(gi)) {
@@ -7135,10 +7158,17 @@ async function main() {
7135
7158
  // backslashes (`relative()` yields `reports\x` on Windows, which would
7136
7159
  // never match `reports/x`).
7137
7160
  if (wroteReports.length > 0) {
7138
- const rel = wroteReports.map((f) => {
7139
- const r = (0, node_path_1.relative)(process.cwd(), (0, node_path_1.resolve)(outDir, f)) || f;
7140
- return node_path_1.sep === "/" ? r : r.split(node_path_1.sep).join("/");
7141
- });
7161
+ // Only paths that PROVABLY sit inside the repo can be minted, so an
7162
+ // `--out` pointing elsewhere yields nothing to write rather than a
7163
+ // dead `../../..` entry.
7164
+ const rel = wroteReports
7165
+ .map((f) => (0, repo_path_js_1.repoRelative)(process.cwd(), (0, node_path_1.resolve)(outDir, f), {
7166
+ relative: node_path_1.relative,
7167
+ resolve: node_path_1.resolve,
7168
+ isAbsolute: node_path_1.isAbsolute,
7169
+ sep: node_path_1.sep,
7170
+ }))
7171
+ .filter((p) => p !== null);
7142
7172
  ensureReportGitignored(process.cwd(), rel);
7143
7173
  }
7144
7174
  // A shareable deep-link for a public GitHub repo: the in-browser demo
@@ -25,6 +25,7 @@
25
25
  * Gate references (`cmd`/`file`/`project`) are collected and verified, so the
26
26
  * cross-referencing moat works on generators too (literal args only).
27
27
  */
28
+ import { type StampedMarkdown } from "./compile.js";
28
29
  export interface GeneratorError {
29
30
  type: "stale-file" | "stale-command";
30
31
  message: string;
@@ -34,6 +35,11 @@ export interface CompileGeneratorResult {
34
35
  errors: GeneratorError[];
35
36
  }
36
37
  export interface CompileGeneratorSkillResult {
38
+ /**
39
+ * The stamped artifact — present ONLY when `errors` is empty. `null` is what
40
+ * makes a failed compile unwritable: `writeArtifact` accepts nothing else.
41
+ */
42
+ artifact: StampedMarkdown | null;
37
43
  markdown: string;
38
44
  errors: GeneratorError[];
39
45
  }
@@ -278,6 +278,9 @@ function compileGeneratorSkill(source, options = {}) {
278
278
  !genArg.body) {
279
279
  return {
280
280
  markdown: "",
281
+ // No stamp for a spec that did not compile — the error branch has nothing
282
+ // to write, which is the whole point of the field.
283
+ artifact: null,
281
284
  errors: [
282
285
  {
283
286
  type: "stale-command",
@@ -300,7 +303,7 @@ function compileGeneratorSkill(source, options = {}) {
300
303
  }
301
304
  fm.push("", "---");
302
305
  const content = `${fm.join("\n")}\n\n${body.trim()}\n`;
303
- return { markdown: (0, compile_js_1.addHash)(content, specFile), errors };
306
+ return { ...(0, compile_js_1.seal)(content, errors, specFile), errors };
304
307
  }
305
308
  /**
306
309
  * Compile a generator's SOURCE text to SKILL.md markdown + verified-ref
@@ -10,7 +10,44 @@ import type { HarnessDialect } from "./dialect.js";
10
10
  /** @internal Compute SHA-256 hash of content (excluding any existing hash line). */
11
11
  export declare function computeHash(content: string): string;
12
12
  /** @internal Prepend a hash comment to compiled content. */
13
- export declare function addHash(content: string, specFile: string): string;
13
+ /**
14
+ * A compiled body that carries a valid integrity stamp — mintable ONLY by
15
+ * {@link addHash}, and by construction only handed out for a CLEAN compile.
16
+ *
17
+ * 🔴 WHY A BRAND AND NOT A CHECK AT THE WRITE SITE. #173 was a `CLAUDE.md`
18
+ * written while its refs were known-dead: `compile` printed the errors, exited
19
+ * 1, and wrote the file anyway, stamped. `lint` then verified the stamp and
20
+ * exited 0 over an artifact that names files which do not exist. The fix that
21
+ * shipped moved the write behind an error check in ONE compiler — and the same
22
+ * three lines sat unchanged in four siblings (skill, subagent, railway,
23
+ * generator), where the lint-side backstop does not even reach because
24
+ * `spec-refs` only inspects `claude` specs. Reproduced end to end after that
25
+ * fix: a skill spec with a stale ref still produced a stamped `SKILL.md` and a
26
+ * green `lint`.
27
+ *
28
+ * Fixing five write sites leaves the class writable — the sixth compiler would
29
+ * be written the same way. Branding the STAMP moves the guarantee to where it is
30
+ * produced: `writeArtifact` accepts nothing else, and an erroring compile has no
31
+ * stamp to give it.
32
+ */
33
+ export type StampedMarkdown = string & {
34
+ readonly __stamped: unique symbol;
35
+ };
36
+ export declare function addHash(content: string, specFile: string): StampedMarkdown;
37
+ /**
38
+ * How every compiler finishes: the rendered body, plus a STAMPED artifact that
39
+ * exists only when the compile is clean.
40
+ *
41
+ * `markdown` stays a plain string because callers legitimately want the draft
42
+ * even when it is wrong — `adoptDiff` diffs "what the spec WOULD produce"
43
+ * against the file on disk, and a stale ref must not stop that. `artifact` is
44
+ * what a writer needs, and it is `null` the moment there is an error, so the
45
+ * write cannot happen without narrowing.
46
+ */
47
+ export declare function seal(body: string, errors: readonly CompileError[], specFile: string): {
48
+ markdown: string;
49
+ artifact: StampedMarkdown | null;
50
+ };
14
51
  /** @internal Check if a file's hash matches its content. Returns null if no hash found. */
15
52
  export declare function verifyHash(content: string): {
16
53
  valid: boolean;
@@ -74,6 +111,11 @@ export interface CompileClaudeOptions {
74
111
  */
75
112
  export declare function compileClaude(spec: ClaudeSpec, options?: CompileClaudeOptions): CompileClaudeResult;
76
113
  export interface CompileSkillResult {
114
+ /**
115
+ * The stamped artifact — present ONLY when `errors` is empty. `null` is what
116
+ * makes a failed compile unwritable: `writeArtifact` accepts nothing else.
117
+ */
118
+ artifact: StampedMarkdown | null;
77
119
  markdown: string;
78
120
  errors: CompileError[];
79
121
  /** Non-blocking advisories (e.g. an over-long inline code block). */
@@ -90,6 +132,11 @@ export declare function compileSkill(spec: SkillSpec, options?: {
90
132
  dialect?: HarnessDialect;
91
133
  }): CompileSkillResult;
92
134
  export interface CompileAgentResult {
135
+ /**
136
+ * The stamped artifact — present ONLY when `errors` is empty. `null` is what
137
+ * makes a failed compile unwritable: `writeArtifact` accepts nothing else.
138
+ */
139
+ artifact: StampedMarkdown | null;
93
140
  markdown: string;
94
141
  errors: CompileError[];
95
142
  /** Non-blocking advisories (e.g. an over-long inline code block). */
@@ -113,6 +160,11 @@ export interface CompileRailwayOptions {
113
160
  specFile?: string;
114
161
  }
115
162
  export interface CompileRailwayResult {
163
+ /**
164
+ * The stamped artifact — present ONLY when `errors` is empty. `null` is what
165
+ * makes a failed compile unwritable: `writeArtifact` accepts nothing else.
166
+ */
167
+ artifact: StampedMarkdown | null;
116
168
  markdown: string;
117
169
  errors: CompileError[];
118
170
  }
@@ -11,6 +11,7 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
11
11
  Object.defineProperty(exports, "__esModule", { value: true });
12
12
  exports.computeHash = computeHash;
13
13
  exports.addHash = addHash;
14
+ exports.seal = seal;
14
15
  exports.verifyHash = verifyHash;
15
16
  exports.estimateTokens = estimateTokens;
16
17
  exports.validateFileRef = validateFileRef;
@@ -53,7 +54,6 @@ const DEFAULT_TARGET = "CLAUDE.md";
53
54
  function computeHash(content) {
54
55
  return (0, hash_js_1.sha256short)((0, integrity_js_1.findIntegrityHeader)(content)?.withoutHeader ?? content);
55
56
  }
56
- /** @internal Prepend a hash comment to compiled content. */
57
57
  function addHash(content, specFile) {
58
58
  // 🔴 THE LAST GATE BEFORE A COMPILED FILE IS WRITTEN. Every compile path returns through here
59
59
  // (four call sites), which makes it the one place a whole class of defect can be stopped.
@@ -72,8 +72,24 @@ function addHash(content, specFile) {
72
72
  `where a string was expected, and JavaScript stringified it. Check the arguments to ` +
73
73
  `input()/file()/cmd() and friends: they take strings, not option objects.`);
74
74
  }
75
+ // The ONE mint. Every stamped artifact in the codebase originates here, which
76
+ // is what makes the brand meaningful rather than decorative.
75
77
  return (0, integrity_js_1.placeIntegrityHeader)(content, computeHash(content), specFile);
76
78
  }
79
+ /**
80
+ * How every compiler finishes: the rendered body, plus a STAMPED artifact that
81
+ * exists only when the compile is clean.
82
+ *
83
+ * `markdown` stays a plain string because callers legitimately want the draft
84
+ * even when it is wrong — `adoptDiff` diffs "what the spec WOULD produce"
85
+ * against the file on disk, and a stale ref must not stop that. `artifact` is
86
+ * what a writer needs, and it is `null` the moment there is an error, so the
87
+ * write cannot happen without narrowing.
88
+ */
89
+ function seal(body, errors, specFile) {
90
+ const stamped = addHash(body, specFile);
91
+ return { markdown: stamped, artifact: errors.length === 0 ? stamped : null };
92
+ }
77
93
  /** @internal Check if a file's hash matches its content. Returns null if no hash found. */
78
94
  function verifyHash(content) {
79
95
  const found = (0, integrity_js_1.findIntegrityHeader)(content);
@@ -859,7 +875,7 @@ function compileSkill(spec, options = {}) {
859
875
  (marker ? marker + "\n\n" : "") +
860
876
  sections.trim() +
861
877
  "\n";
862
- return { markdown: addHash(content, specFile), errors, warnings };
878
+ return { ...seal(content, errors, specFile), errors, warnings };
863
879
  }
864
880
  // ---------------------------------------------------------------------------
865
881
  // Compile a subagent spec → agents/<name>.md
@@ -1058,7 +1074,7 @@ function compileAgent(spec, options) {
1058
1074
  (marker ? marker + "\n\n" : "") +
1059
1075
  body.trim() +
1060
1076
  "\n";
1061
- return { markdown: addHash(content, specFile), errors, warnings };
1077
+ return { ...seal(content, errors, specFile), errors, warnings };
1062
1078
  }
1063
1079
  /** Verify a railway: non-empty, bounded recovery, every delegate target real. */
1064
1080
  function validateRailway(rw, knownAgents) {
@@ -1126,7 +1142,7 @@ function compileRailway(rw, options = {}) {
1126
1142
  const errors = validateRailway(rw, options.knownAgents);
1127
1143
  const specFile = options.specFile ?? `${rw.name}.railway.spec.ts`;
1128
1144
  const content = renderRailwayMarkdown(rw) + "\n";
1129
- return { markdown: addHash(content, specFile), errors };
1145
+ return { ...seal(content, errors, specFile), errors };
1130
1146
  }
1131
1147
  /** Check if a generated file's hash is intact. */
1132
1148
  function checkFileHash(filePath) {
@@ -0,0 +1,41 @@
1
+ /**
2
+ * `RepoRelativePath` — a path that is provably INSIDE the repository.
3
+ *
4
+ * 🔴 WHY A TYPE AND NOT A CHECK. `audit --out=/tmp/x` appended entries like
5
+ * `../../../../private/tmp/x/vigiles-report.json` to the user's `.gitignore`
6
+ * (#176.8). Those ignore nothing — `.gitignore` does not reach outside its own
7
+ * tree — and accumulate one dead block per output path, in a file the tool is
8
+ * documented as never writing. The fix that shipped was a guard at the write
9
+ * site, which works and leaves the bug WRITABLE: the next caller to build an
10
+ * entry list gets a `string[]` and no reason to think twice.
11
+ *
12
+ * This makes it unwritable instead. `ensureReportGitignored` accepts only
13
+ * `RepoRelativePath[]`, and the only way to obtain one is `repoRelative()`,
14
+ * which returns `null` for anything that escapes the root. There is no cast at
15
+ * the call site and no second guard to keep in sync — a path that leaves the
16
+ * repo cannot reach the writer, because it cannot be given the type.
17
+ *
18
+ * The brand is the pattern this codebase already uses for `VerifiedPath` and
19
+ * friends: a nominal marker on `string` that only a smart constructor mints.
20
+ */
21
+ declare const REPO_RELATIVE: unique symbol;
22
+ /** A POSIX-separated path known to resolve inside the repository root. */
23
+ export type RepoRelativePath = string & {
24
+ readonly [REPO_RELATIVE]: true;
25
+ };
26
+ /**
27
+ * Mint a {@link RepoRelativePath}, or `null` when the target escapes `root`.
28
+ *
29
+ * Rejects an absolute path and anything whose relative form starts with `..`.
30
+ * Normalizes to POSIX separators, because `.gitignore` patterns are POSIX and a
31
+ * Windows `reports\x` would never match `reports/x` — a second silent-miss that
32
+ * lived next to the first.
33
+ */
34
+ export declare function repoRelative(root: string, target: string, io: {
35
+ relative: (from: string, to: string) => string;
36
+ resolve: (...parts: string[]) => string;
37
+ isAbsolute: (p: string) => boolean;
38
+ sep: string;
39
+ }): RepoRelativePath | null;
40
+ export {};
41
+ //# sourceMappingURL=repo-path.d.ts.map
@@ -0,0 +1,19 @@
1
+ "use strict";
2
+ Object.defineProperty(exports, "__esModule", { value: true });
3
+ exports.repoRelative = repoRelative;
4
+ /**
5
+ * Mint a {@link RepoRelativePath}, or `null` when the target escapes `root`.
6
+ *
7
+ * Rejects an absolute path and anything whose relative form starts with `..`.
8
+ * Normalizes to POSIX separators, because `.gitignore` patterns are POSIX and a
9
+ * Windows `reports\x` would never match `reports/x` — a second silent-miss that
10
+ * lived next to the first.
11
+ */
12
+ function repoRelative(root, target, io) {
13
+ const rel = io.relative(root, io.resolve(root, target));
14
+ if (rel === "" || rel.startsWith("..") || io.isAbsolute(rel))
15
+ return null;
16
+ const posix = io.sep === "/" ? rel : rel.split(io.sep).join("/");
17
+ return posix;
18
+ }
19
+ //# sourceMappingURL=repo-path.js.map
@@ -6,6 +6,13 @@ export type CacheMode = "off" | "read" | "readwrite";
6
6
  export interface CacheKeyInput {
7
7
  readonly task: string;
8
8
  readonly model: string;
9
+ /**
10
+ * Reasoning budget (`--effort`). Keyed for the same reason `model` is: it moves
11
+ * the output distribution, so a replay across effort levels would serve a result
12
+ * the caller did not ask for. `undefined` (the harness default) drops out of the
13
+ * hash via JSON, so entries recorded before effort existed stay valid.
14
+ */
15
+ readonly effort?: string | number;
9
16
  readonly tools: readonly string[];
10
17
  /** The resolved fixture + arm + plugin files written before the run. */
11
18
  readonly files: Record<string, string>;
@@ -33,6 +33,18 @@ export declare const DEFAULT_LOCK_DIR = ".vigiles/eval-locks";
33
33
  export interface EvalLockInputs {
34
34
  /** Model id used (folded in; a floating alias can't detect weight drift — warned). */
35
35
  readonly model: string;
36
+ /**
37
+ * Reasoning budget (`--effort`) the run was pinned to, or undefined for the
38
+ * harness default. Hashed because it steers the model — the criterion this
39
+ * interface already states — so a committed report recorded at one effort is
40
+ * STALE for a run at another. `undefined` is dropped by `JSON.stringify`, so
41
+ * locks committed before effort existed keep their hash and still replay.
42
+ *
43
+ * Caveat kept honest: "omitted" means the harness's own default, which is
44
+ * per-model and can move between builds — reproducible only modulo that, the
45
+ * same class of provenance caveat as `harnessVersion` below.
46
+ */
47
+ readonly effort?: string | number;
36
48
  /**
37
49
  * A hand-bumped behavior epoch the project owns (`.vigilesrc.json`
38
50
  * `eval.apiVersion`), bumped when a harness-side change YOU made (a CLAUDE.md
@@ -71,6 +83,13 @@ export interface EvalLock {
71
83
  readonly inputsHash: string;
72
84
  /** The model id the report was produced against (for the drift warning). */
73
85
  readonly model: string;
86
+ /**
87
+ * The effort the report was produced at, or undefined for the harness default
88
+ * (provenance; already in the hash). Recorded because the complaint that
89
+ * motivated effort support was not only that it could not be SET — it was that
90
+ * nothing in the run record said which effort produced the numbers.
91
+ */
92
+ readonly effort?: string | number;
74
93
  /** The harness version token at record time (provenance; already in the hash). */
75
94
  readonly harnessVersionKey: string;
76
95
  /** The behavior epoch at record time (provenance; already in the hash). */
@@ -142,6 +161,7 @@ export declare function buildLock(args: {
142
161
  readonly name: string;
143
162
  readonly inputsHash: string;
144
163
  readonly model: string;
164
+ readonly effort?: string | number;
145
165
  readonly harnessVersionKey: string;
146
166
  readonly evalApiVersion: number;
147
167
  readonly builtAt: string;
package/dist/eval.d.ts CHANGED
@@ -46,6 +46,13 @@ export interface EvalArm {
46
46
  * use the eval-level model. See `research/eval-architecture.md` (model strategy).
47
47
  */
48
48
  readonly model?: string;
49
+ /**
50
+ * Reasoning budget for the run (`claude --effort`, e.g. `"low"` or an integer).
51
+ * Lives here beside `model` because it is part of the MEASUREMENT — it moves the
52
+ * output distribution, not the sample size — so it is hashed into the lock and
53
+ * the cache, and never read from an env var. Omit for the harness default.
54
+ */
55
+ readonly effort?: string | number;
49
56
  }
50
57
  /** Per-run resource use, parsed from the terminal `result` event (0 when absent). */
51
58
  export interface EvalUsage {
@@ -95,6 +102,13 @@ export interface EvalSpec<M extends Metrics> {
95
102
  readonly trials?: number;
96
103
  /** Model alias. Default "haiku". */
97
104
  readonly model?: string;
105
+ /**
106
+ * Reasoning budget for the run (`claude --effort`, e.g. `"low"` or an integer).
107
+ * Lives here beside `model` because it is part of the MEASUREMENT — it moves the
108
+ * output distribution, not the sample size — so it is hashed into the lock and
109
+ * the cache, and never read from an env var. Omit for the harness default.
110
+ */
111
+ readonly effort?: string | number;
98
112
  /** Tools the agent may use. Default: Read Edit Write Bash. */
99
113
  readonly allowedTools?: readonly string[];
100
114
  /** Per-run timeout ms. Default 240000. */
@@ -229,6 +243,19 @@ export interface AgentRunArgs {
229
243
  readonly task: string;
230
244
  readonly cwd: string;
231
245
  readonly model: string;
246
+ /**
247
+ * Reasoning-budget level for the run (`claude --effort`). Part of the
248
+ * MEASUREMENT, not a run knob: it changes the model's output distribution, not
249
+ * the sample size — so it lives on the spec next to `model` (never an env),
250
+ * and it is hashed into both the cache key and the eval lock. Deliberately
251
+ * `string | number` rather than a literal union: the binary accepts an alias
252
+ * map, is case-insensitive, and takes an integer budget, and its own valid set
253
+ * MOVED between builds (2.1.42 had no `xhigh`, 2.1.257 does) — a hard-coded
254
+ * union would reject a valid level after any upstream addition. A wrong value
255
+ * is caught at RUNTIME instead, by {@link effortRejection}, which is what the
256
+ * binary actually tells us. Omit for the harness default.
257
+ */
258
+ readonly effort?: string | number;
232
259
  readonly tools: readonly string[];
233
260
  readonly hasSettings: boolean;
234
261
  readonly pluginDir: string | undefined;
@@ -260,10 +287,74 @@ export type AgentRunner = (args: AgentRunArgs) => Promise<RunOut>;
260
287
  * regression to an always-merge would otherwise silently defeat ephemerality and
261
288
  * leak the host environment into an untrusted, model-driven run.
262
289
  */
263
- export declare function resolveSpawnEnv(a: Pick<AgentRunArgs, "env" | "replaceEnv">, base?: NodeJS.ProcessEnv): NodeJS.ProcessEnv;
264
- /** The real `claude`-spawning runner (composition root). Exported so other
265
- * real-model entries (e.g. the `audit` trigger tier) bind the same runner. */
266
- export declare function spawnAgent(a: AgentRunArgs): Promise<RunOut>;
290
+ export declare function resolveSpawnEnv(a: Pick<AgentRunArgs, "env" | "replaceEnv" | "effort">, base?: NodeJS.ProcessEnv): NodeJS.ProcessEnv;
291
+ /**
292
+ * The env var name the harness reads for the reasoning budget. It sits ABOVE the
293
+ * `--effort` flag in the CLI's own precedence chain, so passing the flag alone
294
+ * does NOT pin the level.
295
+ */
296
+ export declare const EFFORT_ENV_VAR = "CLAUDE_CODE_EFFORT_LEVEL";
297
+ /**
298
+ * Pin the effort the run actually gets, so the recorded effort is the effort
299
+ * that ran.
300
+ *
301
+ * WHY THIS EXISTS AND WHY IT IS NOT OPTIONAL. Effort has THREE inputs — the
302
+ * `--effort` flag, the `effortLevel` settings key, and `CLAUDE_CODE_EFFORT_LEVEL`
303
+ * — and the env var wins over the flag. `EPHEMERAL_ALLOW_PREFIXES` passes
304
+ * `CLAUDE_*` through by design (the CLI reads several such knobs and dropping one
305
+ * is the failure mode), so an ambient `CLAUDE_CODE_EFFORT_LEVEL=max` in the
306
+ * author's shell survives even the SCRUBBED ephemeral env. Without this pin,
307
+ * hashing effort into the lock would make the lock CONFIDENTLY WRONG: it would
308
+ * record `low` over a run that executed at `max` — the exact defect the feature
309
+ * exists to prevent, reintroduced by the fix for it.
310
+ *
311
+ * Both directions matter, so both are handled:
312
+ * - effort DECLARED → set the var, overriding whatever the shell had.
313
+ * - effort OMITTED → DELETE an inherited var, so "omit" means the harness
314
+ * default rather than "whatever this machine happened to
315
+ * export". An omitted effort must not be a hidden input.
316
+ */
317
+ export declare function pinEffortEnv(env: NodeJS.ProcessEnv, effort: string | number | undefined): NodeJS.ProcessEnv;
318
+ /**
319
+ * The harness's own rejection of an `--effort` value, or null. Pure.
320
+ *
321
+ * The CLI does NOT fail on a bad level — it prints this to stderr and silently
322
+ * runs at its default. That silent substitution is precisely the bug class this
323
+ * feature addresses (a number produced by a configuration nobody asked for), so
324
+ * a rejected value must never become a sample. Matched on the binary's own
325
+ * wording, the same shape as {@link isRateLimited}.
326
+ */
327
+ export declare function effortRejection(out: RunOut): string | null;
328
+ /**
329
+ * Wrap a runner so a run the harness rejected on `--effort` FAILS LOUDLY.
330
+ *
331
+ * Applied ONCE, around the real runner, rather than as a guard repeated at each
332
+ * of the five `runner(...)` call sites — a guard per call site is the shape that
333
+ * left four of five compilers unprotected in #173.
334
+ *
335
+ * It THROWS rather than counting the trial as `runError`. A `runError` trial is
336
+ * dropped from the denominator, which is right for a transient (a rate limit) and
337
+ * wrong here: an unusable effort value is deterministic and repeatable, so every
338
+ * trial fails it and the run would report a rate computed over ZERO samples. A
339
+ * configuration mistake should stop the run and name itself.
340
+ */
341
+ export declare function withEffortGuard(runner: AgentRunner): AgentRunner;
342
+ /**
343
+ * Build the real runner's argv. Pure and exported so the FLAGS are provable —
344
+ * `spawnAgentRaw` is `v8 ignore`d (it spawns a subprocess), so an argv assembled
345
+ * inline there could not be asserted at all. Mirrors `buildCodexArgs`.
346
+ */
347
+ export declare function buildAgentArgs(a: AgentRunArgs): string[];
348
+ /**
349
+ * The real `claude`-spawning runner (composition root). Exported so other
350
+ * real-model entries (e.g. the `audit` trigger tier) bind the same runner.
351
+ *
352
+ * The effort guard is composed in HERE, at the single definition, rather than at
353
+ * each of the places that bind this runner — so every consumer, including ones
354
+ * not yet written, is covered by construction. Guarding each call site instead is
355
+ * the shape that left four of five compilers unprotected in #173.
356
+ */
357
+ export declare const spawnAgent: AgentRunner;
267
358
  /**
268
359
  * Run the eval: every arm × every trial against the real `claude` CLI, with the
269
360
  * metric computed per run and aggregated per arm. Requires `claude` on PATH and
@@ -320,6 +411,13 @@ export interface MeasureSpec {
320
411
  readonly trials?: number;
321
412
  /** Model alias. Default "sonnet" — measure on the model your users run. */
322
413
  readonly model?: string;
414
+ /**
415
+ * Reasoning budget for the run (`claude --effort`, e.g. `"low"` or an integer).
416
+ * Lives here beside `model` because it is part of the MEASUREMENT — it moves the
417
+ * output distribution, not the sample size — so it is hashed into the lock and
418
+ * the cache, and never read from an env var. Omit for the harness default.
419
+ */
420
+ readonly effort?: string | number;
323
421
  /** Tools the agent may use. */
324
422
  readonly allowedTools?: readonly string[];
325
423
  /** Per-run timeout ms. */
@@ -375,6 +473,13 @@ export interface ArmsMeasureSpec {
375
473
  readonly model?: string;
376
474
  readonly allowedTools?: readonly string[];
377
475
  readonly timeoutMs?: number;
476
+ /**
477
+ * Reasoning budget for the run (`claude --effort`, e.g. `"low"` or an integer).
478
+ * Lives here beside `model` because it is part of the MEASUREMENT — it moves the
479
+ * output distribution, not the sample size — so it is hashed into the lock and
480
+ * the cache, and never read from an env var. Omit for the harness default.
481
+ */
482
+ readonly effort?: string | number;
378
483
  readonly spacingSec?: number;
379
484
  }
380
485
  /** Per-arm {@link CheckReport}s — `arms[name].perCheck[i]` aligns across arms. */
@@ -471,6 +576,7 @@ export declare function runSkillSelectionTrial(args: {
471
576
  readonly runner: AgentRunner;
472
577
  readonly parse?: ModelOutputParser;
473
578
  readonly model: string;
579
+ readonly effort?: string | number;
474
580
  readonly tools?: readonly string[];
475
581
  readonly timeoutMs?: number;
476
582
  readonly fixture?: Record<string, string>;
@@ -658,6 +764,13 @@ export interface TriggerRateSpec {
658
764
  * 0.50 on haiku vs 0.90 on Sonnet). Override for a cheaper-but-pessimistic run.
659
765
  */
660
766
  readonly model?: string;
767
+ /**
768
+ * Reasoning budget for the run (`claude --effort`, e.g. `"low"` or an integer).
769
+ * Lives here beside `model` because it is part of the MEASUREMENT — it moves the
770
+ * output distribution, not the sample size — so it is hashed into the lock and
771
+ * the cache, and never read from an env var. Omit for the harness default.
772
+ */
773
+ readonly effort?: string | number;
661
774
  /**
662
775
  * Minimum model tier this eval may run on (haiku<sonnet<opus by family). The
663
776
  * run **fails** if the resolved `model` is weaker — trigger-rate under-measures
package/dist/eval.js CHANGED
@@ -1,8 +1,11 @@
1
1
  "use strict";
2
2
  Object.defineProperty(exports, "__esModule", { value: true });
3
- exports.claudeEvalDriver = exports.EPHEMERAL_HOME_KEEP = void 0;
3
+ exports.claudeEvalDriver = exports.EPHEMERAL_HOME_KEEP = exports.spawnAgent = exports.EFFORT_ENV_VAR = void 0;
4
4
  exports.resolveSpawnEnv = resolveSpawnEnv;
5
- exports.spawnAgent = spawnAgent;
5
+ exports.pinEffortEnv = pinEffortEnv;
6
+ exports.effortRejection = effortRejection;
7
+ exports.withEffortGuard = withEffortGuard;
8
+ exports.buildAgentArgs = buildAgentArgs;
6
9
  exports.unregisteredSkillFiles = unregisteredSkillFiles;
7
10
  exports.runEval = runEval;
8
11
  exports.measureWith = measureWith;
@@ -99,35 +102,136 @@ function writeFiles(cwd, files) {
99
102
  * leak the host environment into an untrusted, model-driven run.
100
103
  */
101
104
  function resolveSpawnEnv(a, base = process.env) {
102
- return a.replaceEnv ? (a.env ?? {}) : { ...base, ...a.env };
105
+ const resolved = a.replaceEnv ? (a.env ?? {}) : { ...base, ...a.env };
106
+ return pinEffortEnv(resolved, a.effort);
103
107
  }
108
+ /**
109
+ * The env var name the harness reads for the reasoning budget. It sits ABOVE the
110
+ * `--effort` flag in the CLI's own precedence chain, so passing the flag alone
111
+ * does NOT pin the level.
112
+ */
113
+ exports.EFFORT_ENV_VAR = "CLAUDE_CODE_EFFORT_LEVEL";
114
+ /**
115
+ * Pin the effort the run actually gets, so the recorded effort is the effort
116
+ * that ran.
117
+ *
118
+ * WHY THIS EXISTS AND WHY IT IS NOT OPTIONAL. Effort has THREE inputs — the
119
+ * `--effort` flag, the `effortLevel` settings key, and `CLAUDE_CODE_EFFORT_LEVEL`
120
+ * — and the env var wins over the flag. `EPHEMERAL_ALLOW_PREFIXES` passes
121
+ * `CLAUDE_*` through by design (the CLI reads several such knobs and dropping one
122
+ * is the failure mode), so an ambient `CLAUDE_CODE_EFFORT_LEVEL=max` in the
123
+ * author's shell survives even the SCRUBBED ephemeral env. Without this pin,
124
+ * hashing effort into the lock would make the lock CONFIDENTLY WRONG: it would
125
+ * record `low` over a run that executed at `max` — the exact defect the feature
126
+ * exists to prevent, reintroduced by the fix for it.
127
+ *
128
+ * Both directions matter, so both are handled:
129
+ * - effort DECLARED → set the var, overriding whatever the shell had.
130
+ * - effort OMITTED → DELETE an inherited var, so "omit" means the harness
131
+ * default rather than "whatever this machine happened to
132
+ * export". An omitted effort must not be a hidden input.
133
+ */
134
+ function pinEffortEnv(env, effort) {
135
+ // Rebuilt WITHOUT the key rather than deleting or assigning `undefined`:
136
+ // omission has to be provable here, and whether a spawn drops an
137
+ // `undefined`-valued env entry is a Node-version detail we should not lean on.
138
+ const { [exports.EFFORT_ENV_VAR]: _inherited, ...rest } = env;
139
+ return effort === undefined
140
+ ? rest
141
+ : { ...rest, [exports.EFFORT_ENV_VAR]: String(effort) };
142
+ }
143
+ /**
144
+ * The harness's own rejection of an `--effort` value, or null. Pure.
145
+ *
146
+ * The CLI does NOT fail on a bad level — it prints this to stderr and silently
147
+ * runs at its default. That silent substitution is precisely the bug class this
148
+ * feature addresses (a number produced by a configuration nobody asked for), so
149
+ * a rejected value must never become a sample. Matched on the binary's own
150
+ * wording, the same shape as {@link isRateLimited}.
151
+ */
152
+ function effortRejection(out) {
153
+ const text = `${out.stderr ?? ""}\n${out.stdout}`;
154
+ const m = /Unknown --effort value[^\n]*/.exec(text);
155
+ return m ? m[0].trim() : null;
156
+ }
157
+ /**
158
+ * Wrap a runner so a run the harness rejected on `--effort` FAILS LOUDLY.
159
+ *
160
+ * Applied ONCE, around the real runner, rather than as a guard repeated at each
161
+ * of the five `runner(...)` call sites — a guard per call site is the shape that
162
+ * left four of five compilers unprotected in #173.
163
+ *
164
+ * It THROWS rather than counting the trial as `runError`. A `runError` trial is
165
+ * dropped from the denominator, which is right for a transient (a rate limit) and
166
+ * wrong here: an unusable effort value is deterministic and repeatable, so every
167
+ * trial fails it and the run would report a rate computed over ZERO samples. A
168
+ * configuration mistake should stop the run and name itself.
169
+ */
170
+ function withEffortGuard(runner) {
171
+ // 🔴 DELIBERATELY NOT `async`. An `async` wrapper turns the wrapped runner's
172
+ // SYNCHRONOUS throws into rejected promises, and the real runner refuses
173
+ // synchronously on purpose — `refuseDuringEvalLoad` / `refuseUnderForeignRunner`
174
+ // stop a paid eval from billing when a foreign test runner collects it. Making
175
+ // this `async` silently downgraded those refusals from "throws at the call" to
176
+ // "returns a promise that rejects", which `assert.throws` cannot see and an
177
+ // un-awaited caller would not notice. Caught by the full suite, not by the
178
+ // targeted one; pinned below by `withEffortGuard preserves a SYNCHRONOUS throw`.
179
+ return (a) => {
180
+ const pending = runner(a);
181
+ return pending.then((out) => {
182
+ const rejection = effortRejection(out);
183
+ if (rejection !== null) {
184
+ throw new Error(`the harness rejected effort ${JSON.stringify(a.effort)}: ${rejection}`);
185
+ }
186
+ return out;
187
+ });
188
+ };
189
+ }
190
+ /**
191
+ * Build the real runner's argv. Pure and exported so the FLAGS are provable —
192
+ * `spawnAgentRaw` is `v8 ignore`d (it spawns a subprocess), so an argv assembled
193
+ * inline there could not be asserted at all. Mirrors `buildCodexArgs`.
194
+ */
195
+ function buildAgentArgs(a) {
196
+ return [
197
+ "-p",
198
+ a.task,
199
+ // stream-json (+ --verbose, required with -p) so the per-turn tool_use
200
+ // events survive into `ctx.toolCalls` — the unified Trace, same as the
201
+ // harness tier. The terminal `result` event still carries num_turns/output.
202
+ "--output-format",
203
+ "stream-json",
204
+ "--verbose",
205
+ "--model",
206
+ a.model,
207
+ ...(a.effort !== undefined ? ["--effort", String(a.effort)] : []),
208
+ "--permission-mode",
209
+ "acceptEdits",
210
+ ...(a.pluginDir !== undefined
211
+ ? ["--plugin-dir", (0, node_path_1.resolve)(a.pluginDir)]
212
+ : []),
213
+ ...(a.hasSettings ? ["--settings", "settings.json"] : []),
214
+ "--allowedTools",
215
+ ...a.tools,
216
+ ];
217
+ }
218
+ /**
219
+ * The real `claude`-spawning runner (composition root). Exported so other
220
+ * real-model entries (e.g. the `audit` trigger tier) bind the same runner.
221
+ *
222
+ * The effort guard is composed in HERE, at the single definition, rather than at
223
+ * each of the places that bind this runner — so every consumer, including ones
224
+ * not yet written, is covered by construction. Guarding each call site instead is
225
+ * the shape that left four of five compilers unprotected in #173.
226
+ */
227
+ exports.spawnAgent = withEffortGuard(spawnAgentRaw);
104
228
  /* v8 ignore start -- real claude subprocess; exercised by bench/, not the unit gate */
105
- /** The real `claude`-spawning runner (composition root). Exported so other
106
- * real-model entries (e.g. the `audit` trigger tier) bind the same runner. */
107
- function spawnAgent(a) {
229
+ /** The unguarded spawn itself; wrapped by {@link spawnAgent}, never bound raw. */
230
+ function spawnAgentRaw(a) {
108
231
  (0, eval_load_phase_js_1.refuseDuringEvalLoad)("spawning `claude`");
109
232
  (0, foreign_runner_js_1.refuseUnderForeignRunner)("spawning `claude`");
110
233
  return new Promise((resolvePromise) => {
111
- const args = [
112
- "-p",
113
- a.task,
114
- // stream-json (+ --verbose, required with -p) so the per-turn tool_use
115
- // events survive into `ctx.toolCalls` — the unified Trace, same as the
116
- // harness tier. The terminal `result` event still carries num_turns/output.
117
- "--output-format",
118
- "stream-json",
119
- "--verbose",
120
- "--model",
121
- a.model,
122
- "--permission-mode",
123
- "acceptEdits",
124
- ...(a.pluginDir !== undefined
125
- ? ["--plugin-dir", (0, node_path_1.resolve)(a.pluginDir)]
126
- : []),
127
- ...(a.hasSettings ? ["--settings", "settings.json"] : []),
128
- "--allowedTools",
129
- ...a.tools,
130
- ];
234
+ const args = buildAgentArgs(a);
131
235
  const child = (0, node_child_process_1.spawn)(runtime_js_1.claudeCodeRuntime.agentBinary, args, {
132
236
  cwd: a.cwd,
133
237
  // The security-critical env resolution (overlay vs. scrubbed replacement)
@@ -194,7 +298,7 @@ function warnUnregisteredSkillArms(arms) {
194
298
  }
195
299
  async function runEval(spec) {
196
300
  warnUnregisteredSkillArms(spec.arms);
197
- const report = await runEvalWith(spec, spawnAgent);
301
+ const report = await runEvalWith(spec, exports.spawnAgent);
198
302
  // Surface what the run spent — tokens + API-equivalent $, and a LOUD warning if
199
303
  // it was billed to a metered API key instead of the subscription. See eval-cost.ts.
200
304
  (0, eval_cost_js_1.emitCostSummary)((0, eval_cost_js_1.costFromEvalReport)(report));
@@ -237,6 +341,7 @@ async function measureWith(spec, runner) {
237
341
  task: spec.task,
238
342
  trials: spec.trials ?? 5,
239
343
  model: spec.model ?? "sonnet",
344
+ effort: spec.effort,
240
345
  allowedTools: spec.allowedTools,
241
346
  timeoutMs: spec.timeoutMs,
242
347
  spacingSec: spec.spacingSec,
@@ -266,7 +371,7 @@ async function measureWith(spec, runner) {
266
371
  /* v8 ignore start -- real claude subprocess; thin wrapper over measureWith */
267
372
  /** Score a check vocabulary across trials against the real `claude` CLI. */
268
373
  async function measure(spec) {
269
- const report = await measureWith(spec, spawnAgent);
374
+ const report = await measureWith(spec, exports.spawnAgent);
270
375
  (0, eval_cost_js_1.emitCostSummary)((0, eval_cost_js_1.costFromArm)(report.usage));
271
376
  return report;
272
377
  }
@@ -283,6 +388,7 @@ async function measureArmsWith(spec, runner) {
283
388
  task: spec.task,
284
389
  trials: spec.trials ?? 5,
285
390
  model: spec.model ?? "sonnet",
391
+ effort: spec.effort,
286
392
  allowedTools: spec.allowedTools,
287
393
  timeoutMs: spec.timeoutMs,
288
394
  spacingSec: spec.spacingSec,
@@ -337,7 +443,7 @@ function stubArmPluginDirs(arms) {
337
443
  /** Score checks across arms against the real `claude` CLI. */
338
444
  async function measureArms(spec) {
339
445
  warnUnregisteredSkillArms(spec.arms);
340
- const report = await measureArmsWith(spec, spawnAgent);
446
+ const report = await measureArmsWith(spec, exports.spawnAgent);
341
447
  // Sum every arm's spend — an A/B run pays for both arms.
342
448
  (0, eval_cost_js_1.emitCostSummary)((0, eval_cost_js_1.sumCosts)(Object.values(report.arms).map((a) => (0, eval_cost_js_1.costFromArm)(a.usage))));
343
449
  return report;
@@ -552,6 +658,7 @@ async function runSkillSelectionTrial(args) {
552
658
  task: args.prompt,
553
659
  cwd,
554
660
  model: args.model,
661
+ effort: args.effort,
555
662
  tools: args.tools ?? ["Read", "Edit", "Write", "Bash", "Skill"],
556
663
  hasSettings: false,
557
664
  pluginDir: args.pluginDir,
@@ -637,6 +744,7 @@ async function runWithCache(runArgs, keyParts, runner, cfg) {
637
744
  const key = (0, eval_cache_js_1.cacheKey)({
638
745
  task: runArgs.task,
639
746
  model: runArgs.model,
747
+ effort: runArgs.effort,
640
748
  tools: runArgs.tools,
641
749
  files: keyParts.files,
642
750
  settings: keyParts.settings,
@@ -967,6 +1075,7 @@ async function executeTrial(spec, arm, trialIndex, runner, cfg) {
967
1075
  cwd,
968
1076
  // A model comparison is a harness A/B: an arm may override the model.
969
1077
  model: arm.model ?? cfg.model,
1078
+ effort: arm.effort ?? cfg.effort,
970
1079
  tools: cfg.tools,
971
1080
  hasSettings,
972
1081
  pluginDir: arm.pluginDir,
@@ -1100,8 +1209,16 @@ async function withEvalLock(args, produce) {
1100
1209
  }
1101
1210
  if (!isDatedModel(args.model))
1102
1211
  warnFloatingModel(args.model);
1212
+ // OVERLAP, DELIBERATE — do not delete this as dead. Effort reaches the hash
1213
+ // twice: here (the CHOKEPOINT every seam passes through, so a future seam that
1214
+ // forgets to fold effort into its own `inputs` is still covered) and inside
1215
+ // each seam's `inputs` (which alone can see a PER-ARM override this line
1216
+ // cannot). Measured 2026-09-01: removing either one alone leaves the suite
1217
+ // green; removing BOTH fails `changing effort makes a committed lock STALE`.
1218
+ // That is two populations covered, not one line duplicated.
1103
1219
  const inputsHash = (0, eval_lock_js_1.evalInputsHash)({
1104
1220
  model: args.model,
1221
+ effort: args.effort,
1105
1222
  evalApiVersion: lock.evalApiVersion,
1106
1223
  inputs: args.inputs,
1107
1224
  });
@@ -1116,6 +1233,7 @@ async function withEvalLock(args, produce) {
1116
1233
  name: args.name,
1117
1234
  inputsHash,
1118
1235
  model: args.model,
1236
+ effort: args.effort,
1119
1237
  harnessVersionKey: harnessVersion(),
1120
1238
  evalApiVersion: lock.evalApiVersion,
1121
1239
  builtAt: new Date().toISOString(),
@@ -1166,6 +1284,10 @@ function evalArmsInputs(spec, cfg) {
1166
1284
  const absRoot = arm.plugin ? (0, node_path_1.resolve)(process.cwd(), arm.plugin) : "";
1167
1285
  arms[name] = {
1168
1286
  model: arm.model ?? cfg.model,
1287
+ // The per-ARM half of the overlap documented at `inputsHash` — the
1288
+ // chokepoint sees only the eval-level effort, so an arm that overrides it
1289
+ // would otherwise hash identically to its sibling.
1290
+ effort: arm.effort ?? cfg.effort,
1169
1291
  tools: [...cfg.tools].sort(),
1170
1292
  files: stripPluginRoot(resolved.files, absRoot),
1171
1293
  settings: stripPluginRoot(resolved.settings, absRoot),
@@ -1203,6 +1325,7 @@ async function runEvalWith(spec, runner) {
1203
1325
  const backoffMs = spec.retryBackoffMs ?? 1000;
1204
1326
  const cfg = {
1205
1327
  model: spec.model ?? "haiku",
1328
+ effort: spec.effort,
1206
1329
  tools: spec.allowedTools ?? ["Read", "Edit", "Write", "Bash"],
1207
1330
  timeoutMs: spec.timeoutMs ?? 240000,
1208
1331
  cache: spec.cache ?? "off",
@@ -1232,7 +1355,7 @@ async function runEvalWith(spec, runner) {
1232
1355
  // resolveHarness/hashDir work). `check` replays the committed report below
1233
1356
  // without ever entering the run pool — so no model is driven in CI.
1234
1357
  const inputs = lock.mode === "off" ? undefined : evalArmsInputs(spec, cfg);
1235
- return withEvalLock({ name: spec.name, inputs, model: cfg.model, lock }, async () => {
1358
+ return withEvalLock({ name: spec.name, inputs, model: cfg.model, effort: cfg.effort, lock }, async () => {
1236
1359
  const results = await runPool(units, concurrency, worker);
1237
1360
  const { arms, totalCostUsd } = aggregateArms(Object.keys(spec.arms), results);
1238
1361
  return { name: spec.name ?? "eval", trials, arms, totalCostUsd, aborted };
@@ -1285,7 +1408,7 @@ function formatEvalReport(report) {
1285
1408
  * The asymmetry reflects default-vs-injected, not a hexagonal violation.
1286
1409
  */
1287
1410
  exports.claudeEvalDriver = {
1288
- runner: spawnAgent,
1411
+ runner: exports.spawnAgent,
1289
1412
  parse: parseClaudeRun,
1290
1413
  harness: "claude-code",
1291
1414
  };
@@ -1591,6 +1714,7 @@ async function runTriggerTrial(prompt, cfg, runner) {
1591
1714
  task: prompt,
1592
1715
  cwd,
1593
1716
  model: cfg.model,
1717
+ effort: cfg.effort,
1594
1718
  tools: cfg.tools,
1595
1719
  hasSettings: false,
1596
1720
  pluginDir: cfg.pluginDir,
@@ -1690,6 +1814,7 @@ async function measureTriggerRateWith(spec, runner, parse = parseClaudeRun, runE
1690
1814
  // Sonnet, not haiku: trigger-rate is a selection measurement and haiku
1691
1815
  // under-selects, producing false-negative recall (see TriggerRateSpec.model).
1692
1816
  model,
1817
+ effort: spec.effort,
1693
1818
  tools: spec.allowedTools ?? ["Read", "Edit", "Write", "Bash", "Skill"],
1694
1819
  timeoutMs: spec.timeoutMs ?? 240000,
1695
1820
  spacing: (spec.spacingSec ?? 4) * 1000,
@@ -1715,6 +1840,7 @@ async function measureTriggerRateWith(spec, runner, parse = parseClaudeRun, runE
1715
1840
  ? [...spec.irrelevantPrompts]
1716
1841
  : undefined,
1717
1842
  model: cfg.model,
1843
+ effort: cfg.effort,
1718
1844
  tools: [...cfg.tools].sort(),
1719
1845
  fixture: spec.fixture,
1720
1846
  competitors,
@@ -1723,7 +1849,13 @@ async function measureTriggerRateWith(spec, runner, parse = parseClaudeRun, runE
1723
1849
  // STALE if the eval is switched to another harness.
1724
1850
  harness,
1725
1851
  };
1726
- return await withEvalLock({ name: spec.name, inputs: triggerInputs, model: cfg.model, lock }, async () => {
1852
+ return await withEvalLock({
1853
+ name: spec.name,
1854
+ inputs: triggerInputs,
1855
+ model: cfg.model,
1856
+ effort: cfg.effort,
1857
+ lock,
1858
+ }, async () => {
1727
1859
  const relevant = await runTriggerSet(spec.prompts, cfg, runner);
1728
1860
  const base = {
1729
1861
  rate: relevant.n > 0 ? relevant.fired / relevant.n : 0,
@@ -187,6 +187,13 @@ export interface HookRunResult extends ScriptRunResult {
187
187
  * a halt sets both.
188
188
  */
189
189
  readonly haltsTurn: boolean;
190
+ /**
191
+ * Every block mechanism that fired, from the closed {@link BlockMechanism}
192
+ * set. Reported so the next mechanism needs no new boolean on this interface —
193
+ * `haltsTurn` exists because the halt case was added as a one-off, and this is
194
+ * the shape that stops the pattern repeating.
195
+ */
196
+ readonly blockedBy: readonly BlockMechanism[];
190
197
  /**
191
198
  * The decision the hook expressed, preferring the structured
192
199
  * `permissionDecision` ("allow"|"deny"|"ask") then legacy `decision`
@@ -196,14 +203,34 @@ export interface HookRunResult extends ScriptRunResult {
196
203
  }
197
204
  /** Parse stdout as a hook JSON decision (pure, testable without a process). */
198
205
  export declare function parseHookOutput(stdout: string): HookOutput | null;
206
+ /**
207
+ * The ways a hook can stop an action — a CLOSED set, listed once.
208
+ *
209
+ * 🔴 WHY A TABLE AND NOT THREE `||` TERMS. `decideHook` used to be a boolean
210
+ * expression over three mechanisms while `HookOutput` declared a fourth,
211
+ * `continue`, that nothing read (#174). The consequence was not a missed
212
+ * detection but an INVERTED verdict in the flagship feature: a real guard that
213
+ * stopped every command in the disaster battery was reported by
214
+ * `assertBlocksDisasters` as blocking none of them.
215
+ *
216
+ * A `||` chain has no shape that can be incomplete — every term is optional by
217
+ * construction, so nothing can say "you declared a mechanism and did not handle
218
+ * it". `Record<BlockMechanism, …>` can: adding a member to the union without
219
+ * adding its row is a tsc error, the same device that keeps `RULE_META` honest.
220
+ */
221
+ export type BlockMechanism = "exit-code" | "deny-decision" | "halt-field";
199
222
  /**
200
223
  * Decide whether a hook result blocked, and the normalized decision. Pure, so
201
224
  * the policy is unit-testable independent of spawning anything.
225
+ *
226
+ * Folds the closed {@link BLOCK_MECHANISMS} table rather than testing three
227
+ * conditions inline, so a mechanism cannot be declared and left unhandled.
202
228
  */
203
229
  export declare function decideHook(exitCode: number, json: HookOutput | null, protocol?: HookProtocol): {
204
230
  blocked: boolean;
205
231
  decision: HookRunResult["decision"];
206
232
  haltsTurn: boolean;
233
+ blockedBy: readonly BlockMechanism[];
207
234
  };
208
235
  /**
209
236
  * The hook layer over {@link runScriptWith}: serialize the event to stdin, run
package/dist/run-hook.js CHANGED
@@ -123,23 +123,42 @@ function parseHookOutput(stdout) {
123
123
  return null;
124
124
  }
125
125
  }
126
+ const BLOCK_MECHANISMS = {
127
+ "exit-code": ({ exitCode, protocol }) => exitCode === protocol.blockExitCode,
128
+ "deny-decision": ({ json, protocol }) => {
129
+ const decision = json?.hookSpecificOutput?.permissionDecision ?? json?.decision;
130
+ return (decision !== undefined && protocol.denyDecisionValues.includes(decision));
131
+ },
132
+ // Read from the PORT, never hard-coded: `"continue"` is a documented Claude
133
+ // Code fact and unverified for Codex, so the harness that has it declares it
134
+ // (core ⊄ adapter). `=== false` and not falsy — an absent field is not a halt.
135
+ "halt-field": ({ json, protocol }) => {
136
+ const field = protocol.haltsTurnField;
137
+ return field !== undefined && json?.[field] === false;
138
+ },
139
+ };
126
140
  /**
127
141
  * Decide whether a hook result blocked, and the normalized decision. Pure, so
128
142
  * the policy is unit-testable independent of spawning anything.
143
+ *
144
+ * Folds the closed {@link BLOCK_MECHANISMS} table rather than testing three
145
+ * conditions inline, so a mechanism cannot be declared and left unhandled.
129
146
  */
130
147
  function decideHook(exitCode, json, protocol = hook_protocol_js_1.claudeCodeHookProtocol) {
131
148
  const permission = json?.hookSpecificOutput?.permissionDecision;
132
149
  const decision = permission ?? json?.decision;
133
- // The halt field is read from the PORT, never hard-coded: `"continue"` is a
134
- // documented Claude Code fact and an unverified one for Codex, so the harness
135
- // that has it declares it (core adapter). `=== false` and not falsy —
136
- // an absent field must not read as a halt.
137
- const haltField = protocol.haltsTurnField;
138
- const haltsTurn = haltField !== undefined && json?.[haltField] === false;
139
- const blocked = exitCode === protocol.blockExitCode ||
140
- haltsTurn ||
141
- (decision !== undefined && protocol.denyDecisionValues.includes(decision));
142
- return { blocked, decision, haltsTurn };
150
+ const ctx = { exitCode, json, protocol };
151
+ // EVERY mechanism is evaluated, not the first match: a hook may both exit 2
152
+ // and halt the turn, and reporting only the first would make `haltsTurn`
153
+ // depend on the table's order.
154
+ const blockedBy = Object.keys(BLOCK_MECHANISMS).filter((kind) => BLOCK_MECHANISMS[kind](ctx));
155
+ return {
156
+ blocked: blockedBy.length > 0,
157
+ decision,
158
+ // Derived, not computed twice — the next mechanism needs no new boolean.
159
+ haltsTurn: blockedBy.includes("halt-field"),
160
+ blockedBy,
161
+ };
143
162
  }
144
163
  /**
145
164
  * The hook layer over {@link runScriptWith}: serialize the event to stdin, run
@@ -150,8 +169,8 @@ function decideHook(exitCode, json, protocol = hook_protocol_js_1.claudeCodeHook
150
169
  function runHookWith(command, input, opts, deps) {
151
170
  const res = (0, run_script_js_1.runScriptWith)(command, JSON.stringify(input), opts, deps);
152
171
  const json = parseHookOutput(res.stdout);
153
- const { blocked, decision, haltsTurn } = decideHook(res.exitCode, json);
154
- return { ...res, json, blocked, decision, haltsTurn };
172
+ const { blocked, decision, haltsTurn, blockedBy } = decideHook(res.exitCode, json);
173
+ return { ...res, json, blocked, decision, haltsTurn, blockedBy };
155
174
  }
156
175
  /**
157
176
  * Run a hook command, piping `input` as JSON to its stdin, and report the exit
@@ -89,6 +89,16 @@ export interface SelectionOptions {
89
89
  readonly trials?: number;
90
90
  /** Selector model — defaults to Sonnet (a weaker model under-selects). */
91
91
  readonly model?: string;
92
+ /**
93
+ * Reasoning budget (`claude --effort`) for the selector, or undefined for the
94
+ * harness default. Present for {@link measureSelectionMatrix}, the ASSERTABLE
95
+ * test primitive, where the configuration a number came from has to be pinnable.
96
+ *
97
+ * Deliberately NOT exposed as an `audit` CLI flag: `audit` is a local report,
98
+ * not a reproducibility surface, and a flag nobody can act on is surface without
99
+ * a use. The audit probe therefore leaves this unset and runs at the default.
100
+ */
101
+ readonly effort?: string | number;
92
102
  /** Parallel runs across the prompts × trials grid (default 1). */
93
103
  readonly concurrency?: number;
94
104
  /** Which harness drives it (default `"claude-code"`; others report n/a). */
@@ -338,6 +338,7 @@ async function measurePluginSelectionWith(dir, promptSet, probe, opts = {}) {
338
338
  parse: d.parse,
339
339
  runError: d.runError,
340
340
  model: opts.model ?? "sonnet",
341
+ effort: opts.effort,
341
342
  }));
342
343
  const runs = [];
343
344
  jobs.forEach((job, k) => {
package/dist/test.d.ts CHANGED
@@ -54,7 +54,7 @@ export { recordCheck } from "./check-count.js";
54
54
  export { runScript } from "./run-script.js";
55
55
  export type { RunScriptOptions, ScriptRunResult } from "./run-script.js";
56
56
  export { runHook, parseHookOutput, decideHook, propertyHook, fileToolEvents, egressRoutes, } from "./run-hook.js";
57
- export type { HookRunResult, RunHookOptions, HookInput, HookOutput, HookPropertyResult, FileToolEventOptions, } from "./run-hook.js";
57
+ export type { HookRunResult, BlockMechanism, RunHookOptions, HookInput, HookOutput, HookPropertyResult, FileToolEventOptions, } from "./run-hook.js";
58
58
  export * from "./harness-assert.js";
59
59
  export { experimental_emitTool, experimental_parseEmitted, experimental_assertEmittedOk, type EmitFieldSchema, type EmitObjectSchema, type EmitPropertySchema, type EmitTrackSchema, type EmitToolDefinition, type ExperimentalEmitTool, } from "./experimental-emit.js";
60
60
  export { loadHook } from "./load-hook.js";
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "vigiles",
3
- "version": "24.0.0",
3
+ "version": "25.1.0",
4
4
  "description": "Audit, test and measure the harness your AI agent runs on — grade your CLAUDE.md / AGENTS.md, skills, subagents and hooks, run them against a scripted model, and measure whether they actually fire.",
5
5
  "keywords": [
6
6
  "claude-code",