vigiles 34.0.0 → 36.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -7,6 +7,9 @@ import type { HarnessTestDriver, HarnessDriverContext, ParsedRun } from "../../c
7
7
  * `--dangerously-bypass-approvals-and-sandbox` make it run unattended;
8
8
  * `--skip-git-repo-check` lets it run in a bare temp dir; `--ignore-user-config`
9
9
  * keeps the host's `~/.codex` out. The prompt is the trailing positional.
10
+ * `ctx.allowedTools` is ignored: `codex exec` runs with approvals bypassed, so
11
+ * everything is already approved. `ctx.tools` is refused (throws): Codex has no
12
+ * per-run tool availability.
10
13
  * Pure, so the arg shape is unit-tested.
11
14
  */
12
15
  export declare function buildCodexArgs(ctx: HarnessDriverContext): string[];
@@ -26,9 +26,15 @@ const mock_model_js_1 = require("./mock-model.js");
26
26
  * `--dangerously-bypass-approvals-and-sandbox` make it run unattended;
27
27
  * `--skip-git-repo-check` lets it run in a bare temp dir; `--ignore-user-config`
28
28
  * keeps the host's `~/.codex` out. The prompt is the trailing positional.
29
+ * `ctx.allowedTools` is ignored: `codex exec` runs with approvals bypassed, so
30
+ * everything is already approved. `ctx.tools` is refused (throws): Codex has no
31
+ * per-run tool availability.
29
32
  * Pure, so the arg shape is unit-tested.
30
33
  */
31
34
  function buildCodexArgs(ctx) {
35
+ if (ctx.tools !== undefined) {
36
+ throw new Error("tools is not supported for codex: it has no per-run tool availability, and a list that withheld nothing would let a test pass over a run where nothing was withheld. Drop tools, or assert on what the agent did.");
37
+ }
32
38
  return [
33
39
  "exec",
34
40
  "--ignore-user-config",
@@ -75,7 +81,8 @@ async function startCodexMockHandle(script) {
75
81
  }));
76
82
  },
77
83
  get count() {
78
- return mock.requests.length;
84
+ // Turns SERVED: a request past the end of the script gets a 400.
85
+ return Math.min(mock.requests.length, script.length);
79
86
  },
80
87
  close: () => mock.close(),
81
88
  };
@@ -42,8 +42,9 @@ export declare function parseResponsesRequest(body: string): {
42
42
  };
43
43
  /**
44
44
  * Start the scripted Codex Responses mock on a free port. Each
45
- * `POST /v1/responses` consumes the next scripted turn (the last turn repeats if
46
- * codex asks for more); HEAD and any other path get `{}`. Resolves to a handle
45
+ * `POST /v1/responses` consumes the next scripted turn; one past the end of the
46
+ * script is a 400, never a copy of the last turn (#340); HEAD and any other path
47
+ * get `{}`. Resolves to a handle
47
48
  * with the base `url`, the `port`, the recorded `requests`, and `close()`.
48
49
  */
49
50
  export declare function startCodexMock(script: readonly CodexTurn[], opts?: {
@@ -22,6 +22,7 @@ exports.startCodexMock = startCodexMock;
22
22
  * `src/adapters/codex/runtime.ts` (`codexMockArgs`/`codexMockEnv`).
23
23
  */
24
24
  const node_http_1 = __importDefault(require("node:http"));
25
+ const script_overrun_js_1 = require("../../core/script-overrun.js");
25
26
  const MSG_ID = "msg_mock_1";
26
27
  const RESP_ID = "resp_mock_1";
27
28
  /**
@@ -183,8 +184,9 @@ function parseResponsesRequest(body) {
183
184
  }
184
185
  /**
185
186
  * Start the scripted Codex Responses mock on a free port. Each
186
- * `POST /v1/responses` consumes the next scripted turn (the last turn repeats if
187
- * codex asks for more); HEAD and any other path get `{}`. Resolves to a handle
187
+ * `POST /v1/responses` consumes the next scripted turn; one past the end of the
188
+ * script is a 400, never a copy of the last turn (#340); HEAD and any other path
189
+ * get `{}`. Resolves to a handle
188
190
  * with the base `url`, the `port`, the recorded `requests`, and `close()`.
189
191
  */
190
192
  function startCodexMock(script, opts = {}) {
@@ -204,7 +206,17 @@ function startCodexMock(script, opts = {}) {
204
206
  const parsed = parseResponsesRequest(body);
205
207
  requests.push(parsed);
206
208
  opts.onRequest?.(parsed);
207
- const turn = script[Math.min(i, script.length - 1)] ?? { text: "" };
209
+ const turn = script[i];
210
+ if (turn === undefined) {
211
+ const message = (0, script_overrun_js_1.scriptOverrunMessage)({
212
+ scripted: script.length,
213
+ request: script.length + 1,
214
+ lastMessage: parsed.prompt,
215
+ });
216
+ res.writeHead(400, { "content-type": "application/json" });
217
+ res.end(JSON.stringify({ error: { type: "invalid_request_error", message } }));
218
+ return;
219
+ }
208
220
  i++;
209
221
  const model = parsed.model || "gpt-5-codex";
210
222
  res.writeHead(200, {
package/dist/cli-main.js CHANGED
@@ -2671,17 +2671,21 @@ function vigilesWorkflow(plan, harnesses, hasPackageJson) {
2671
2671
  eval-check:
2672
2672
  # Eval staleness gate — real-model evals run LOCALLY on your subscription
2673
2673
  # (\`npx vigiles eval --update\`, which commits a lock); this job VERIFIES those
2674
- # committed results against the current inputs with NO model call. It stays a
2675
- # green no-op until you commit your first lock. See docs/harness-testing.md.
2674
+ # committed results against the current inputs with NO model call.
2675
+ # \`--min=0\` is deliberate: a repo with no eval file yet matches nothing, and
2676
+ # \`vigiles eval --check\` fails on an empty match by default (a gate that
2677
+ # verified nothing must not read as a pass). With \`--min=0\` this job is green
2678
+ # until the first \`*.eval.*\` file exists; from then on it is red until that
2679
+ # eval's lock is committed. Raise the floor (\`--min=N\`) once you want
2680
+ # a vanished eval file to fail it. See docs/harness-testing.md.
2676
2681
  runs-on: ubuntu-latest
2677
2682
  steps:
2678
2683
  - uses: actions/checkout@v4
2679
2684
  - uses: actions/setup-node@v4
2680
2685
  with:
2681
2686
  node-version: "20"
2682
- - uses: zernie/vigiles@v1
2683
- with:
2684
- command: eval-check
2687
+ - run: npm install
2688
+ - run: npx vigiles eval --check --min=0
2685
2689
  `
2686
2690
  : "";
2687
2691
  // No selectable jobs (e.g. `--no-lint --no-test`, or `--test` on a repo with no
@@ -5195,10 +5199,11 @@ async function promptYesNo(question) {
5195
5199
  * Resolve the eval LOCK env from the `eval` flags. `--update` records each named
5196
5200
  * eval's report to a committed `.vigiles/eval-locks/<name>.lock.json` (run locally
5197
5201
  * on your subscription); `--check` (CI) verifies the committed result against the
5198
- * current inputs WITHOUT a model call. `--check` is a green NO-OP until the first
5199
- * lock is committed (smooth adoption). Returns the env to thread, or `"skip"` to
5200
- * exit green now. `--check`+`--update` together is a usage error (exit 2). The
5201
- * behavior epoch comes from `.vigilesrc.json` `eval.apiVersion` (committed).
5202
+ * current inputs WITHOUT a model call. `--check` with no lock committed at all is
5203
+ * a FAILURE, not a green no-op ({@link refuseEvalCheckWithoutLocks}, #197): a gate
5204
+ * that has verified nothing must not read as one that passed.
5205
+ * `--check`+`--update` together is a usage error (exit 2). The behavior epoch
5206
+ * comes from `.vigilesrc.json` `eval.apiVersion` (committed).
5202
5207
  */
5203
5208
  function resolveEvalLockEnv(args) {
5204
5209
  const wantCheck = args.includes("--check");
@@ -5207,13 +5212,6 @@ function resolveEvalLockEnv(args) {
5207
5212
  console.error("vigiles eval: --check and --update are mutually exclusive (one verifies, one records).");
5208
5213
  process.exit(2);
5209
5214
  }
5210
- if (wantCheck &&
5211
- !(0, eval_lock_js_1.anyLocksCommitted)((0, node_path_1.resolve)(process.cwd(), eval_lock_js_1.DEFAULT_LOCK_DIR))) {
5212
- console.log("ℹ vigiles eval --check: no committed eval locks found — nothing to verify.\n" +
5213
- " Run `vigiles eval --update` locally (on your subscription) and commit the\n" +
5214
- " lock to enable the CI staleness gate.");
5215
- return "skip";
5216
- }
5217
5215
  const env = {};
5218
5216
  if (wantCheck)
5219
5217
  env.VIGILES_EVAL_LOCK = "check";
@@ -5226,6 +5224,22 @@ function resolveEvalLockEnv(args) {
5226
5224
  }
5227
5225
  return env;
5228
5226
  }
5227
+ /**
5228
+ * `eval --check` over eval files when no lock is committed anywhere: every one of
5229
+ * them has no recorded result to compare the current inputs against, so the gate
5230
+ * would verify nothing and pass. It used to return green BEFORE discovery, which
5231
+ * also bypassed the empty-match failure. Once any lock exists, a file without one
5232
+ * already fails as "lock missing" in the eval itself. Exits 1; returns otherwise.
5233
+ */
5234
+ function refuseEvalCheckWithoutLocks(args, evalCount) {
5235
+ if (!args.includes("--check"))
5236
+ return;
5237
+ if ((0, eval_lock_js_1.anyLocksCommitted)((0, node_path_1.resolve)(process.cwd(), eval_lock_js_1.DEFAULT_LOCK_DIR)))
5238
+ return;
5239
+ console.error(`✗ vigiles eval --check: ${String(evalCount)} eval file(s) have no recorded result — no eval lock is committed, so there is nothing to verify the current inputs against.\n` +
5240
+ " Run `vigiles eval --update` locally (on your subscription) and commit the lock(s).");
5241
+ process.exit(1);
5242
+ }
5229
5243
  /**
5230
5244
  * Turn a completed run into `.vigiles/coverage.json` — the composition root for
5231
5245
  * the execution tier of coverage (`coverage-artifact.ts`).
@@ -5400,6 +5414,31 @@ function gitHead(cwd) {
5400
5414
  return "";
5401
5415
  }
5402
5416
  }
5417
+ /**
5418
+ * Why a `vigiles test` / `vigiles eval` run matched no file — the message names
5419
+ * what was looked for, so a stale path, a wrong glob and an empty default
5420
+ * discovery each say so. A DIRECTORY is the one stale-looking target whose cause
5421
+ * we actually know, so it is said instead of the three guesses: `vigiles test .`
5422
+ * used to reach `spawn("node", ["."])` and surface Node's module-resolution stack.
5423
+ */
5424
+ function emptyMatchMessage(kind, targets, defaultGlob) {
5425
+ const dirs = targets.filter((a) => (0, node_fs_1.lstatSync)(a, { throwIfNoEntry: false })?.isDirectory() === true);
5426
+ if (dirs.length > 0) {
5427
+ return (`✗ vigiles ${kind}: ${dirs.join(", ")} ${dirs.length === 1 ? "is a directory" : "are directories"} — ` +
5428
+ `pass a file, or a glob like "${defaultGlob}".\n` +
5429
+ ` A directory is not a script; nothing ran.`);
5430
+ }
5431
+ const expected = ` If an empty match is expected here, say so with --min=0.`;
5432
+ if (targets.length === 0) {
5433
+ return (`✗ vigiles ${kind}: NOTHING matched the default glob ${defaultGlob}\n` +
5434
+ ` Nothing ran. An empty run is not a pass: a renamed or moved file looks like this.\n` +
5435
+ expected);
5436
+ }
5437
+ return (`✗ vigiles ${kind}: ${String(targets.length)} target(s) given and NOTHING matched — ` +
5438
+ `${targets.join(", ")}\n` +
5439
+ ` Nothing ran. A stale path, a wrong glob, or a moved file all look like this.\n` +
5440
+ expected);
5441
+ }
5403
5442
  async function handleRunScripts(kind, args, restArgs, excludes) {
5404
5443
  const cwd = process.cwd();
5405
5444
  // The ignore file keeps NEW copies of `.vigiles/` local files out of git, but
@@ -5420,14 +5459,10 @@ async function handleRunScripts(kind, args, restArgs, excludes) {
5420
5459
  // Harness/eval scripts may be authored in JS or TS (see run-scripts.ts).
5421
5460
  const defaultGlob = (0, run_scripts_js_1.scriptGlob)(kind === "test" ? "harness" : "eval");
5422
5461
  // The eval LOCK flags (`--check`/`--update`) are resolved BEFORE file discovery
5423
- // so mutual-exclusion + the cold-start no-op are honored regardless of file
5424
- // count. Returns the env to thread to scripts, or `"skip"` to exit green now.
5462
+ // so mutual-exclusion (exit 2) is honored regardless of file count.
5425
5463
  let lockEnv = {};
5426
5464
  if (kind === "eval") {
5427
- const r = resolveEvalLockEnv(args);
5428
- if (r === "skip")
5429
- return;
5430
- lockEnv = r;
5465
+ lockEnv = resolveEvalLockEnv(args);
5431
5466
  }
5432
5467
  // A script under an excluded path is not DISCOVERED (a vendored corpus's own
5433
5468
  // harness must not run as ours), but a script you NAME still runs (#192).
@@ -5449,44 +5484,33 @@ async function handleRunScripts(kind, args, restArgs, excludes) {
5449
5484
  process.exit(1);
5450
5485
  }
5451
5486
  if (files.length === 0) {
5452
- // 🔴 ASKING FOR SOMETHING AND GETTING NOTHING IS A FAILURE; FINDING NOTHING IS NOT.
5453
- // The two cases were collapsed into one silent exit 0, and the collapse cost a real
5454
- // repository three days of green CI verifying zero files: a named step ran
5487
+ // 🔴 A RUN THAT MATCHES NO FILE IS A FAILURE, NAMED OR NOT.
5488
+ // An empty match used to be one silent exit 0, and it cost a real repository three
5489
+ // days of green CI verifying zero files: a named step ran
5455
5490
  // `vigiles test .claude/pipeline/skills.harness.mjs` after that file had been split
5456
5491
  // into one-per-skill, printed "No **/*.harness.* files found" and passed, right next
5457
- // to a step that was red for the same root cause.
5458
- //
5459
- // They are different states. A POSITIONAL argument is a claim that something is there —
5460
- // when nothing matches it, the path is stale, the glob is wrong, or the run never
5461
- // reached its target, and every one of those is a defect. Bare discovery finding
5462
- // nothing is just an empty repository, which is a legitimate place to stand and must
5463
- // stay quiet.
5492
+ // to a step that was red for the same root cause. The first repair failed only the
5493
+ // NAMED case and kept bare discovery quiet ("an empty repository is a legitimate
5494
+ // place to stand") — and bare `vigiles test` is the line `init` writes into CI, so
5495
+ // the same green no-op was one rename away from the default setup (#197).
5464
5496
  //
5465
- // This is the default the field settled on: Jest and Vitest FAIL on no tests found and
5466
- // make you opt in with `--passWithNoTests`; pytest exits 5. `--min=0` remains the
5467
- // explicit opt-out here, so no new flag is introduced by this change.
5468
- // `minFlag`, not `minRequired`: 0 is both the DEFAULT and the explicit opt-out, so the
5469
- // VALUE cannot tell them apart — only the flag's presence can. (Caught by a control:
5470
- // the first version read `minRequired === 0` and made `--min=0` do nothing.)
5471
- if (restArgs.length > 0 && minFlag === undefined) {
5472
- // A DIRECTORY is the one stale-looking target whose cause we actually know,
5473
- // so say it instead of listing the three guesses. `vigiles test .` used to
5474
- // reach `spawn("node", ["."])` and surface Node's module-resolution stack;
5475
- // the generic message above would now be true but unhelpful.
5476
- const dirs = restArgs.filter((a) => (0, node_fs_1.lstatSync)(a, { throwIfNoEntry: false })?.isDirectory() === true);
5477
- console.error(dirs.length > 0
5478
- ? `✗ vigiles ${kind}: ${dirs.join(", ")} ${dirs.length === 1 ? "is a directory" : "are directories"} — ` +
5479
- `pass a file, or a glob like "${defaultGlob}".\n` +
5480
- ` A directory is not a script; nothing ran.`
5481
- : `✗ vigiles ${kind}: ${String(restArgs.length)} target(s) given and NOTHING matched — ` +
5482
- `${restArgs.join(", ")}\n` +
5483
- ` Nothing ran. A stale path, a wrong glob, or a moved file all look like this.\n` +
5484
- ` If an empty match is expected here, say so with --min=0.`);
5497
+ // This is the default the field settled on: Jest and Vitest FAIL on no tests found
5498
+ // and make you opt in with `--passWithNoTests`; pytest exits 5. Here there is no
5499
+ // such flag (a person who wants an empty run does not call the command), and
5500
+ // `--min=0` — the existing floor, spelled as zero — is the only way to say an empty
5501
+ // match is expected. `minFlag`, not `minRequired`: 0 is both the DEFAULT and that
5502
+ // explicit opt-out, so the VALUE cannot tell them apart — only the flag's presence
5503
+ // can. (Caught by a control: the first version read `minRequired === 0` and made
5504
+ // `--min=0` do nothing.)
5505
+ if (minFlag === undefined) {
5506
+ console.error(emptyMatchMessage(kind, restArgs, defaultGlob));
5485
5507
  process.exit(1);
5486
5508
  }
5487
5509
  console.log(`No ${defaultGlob} files found.`);
5488
5510
  return;
5489
5511
  }
5512
+ if (kind === "eval")
5513
+ refuseEvalCheckWithoutLocks(args, files.length);
5490
5514
  // Consent gate for a bare `vigiles eval`: it runs the REAL model on your
5491
5515
  // subscription, and a no-target run discovered the whole tree — so never fan out
5492
5516
  // over an unbounded glob without explicit intent. Mirrors audit's read-vs-run
@@ -107,8 +107,18 @@ export interface HarnessDriverContext {
107
107
  readonly cwd: string;
108
108
  /** Whether a settings file was written (CC: pass `--settings`). */
109
109
  readonly hasSettings: boolean;
110
- /** Tools the agent may use. */
111
- readonly tools: readonly string[];
110
+ /**
111
+ * Which tools EXIST in the session, when the spec lists them (CC: `--tools`);
112
+ * undefined = every tool exists. A driver whose harness has no such concept
113
+ * (Codex) throws on one rather than ignore it.
114
+ */
115
+ readonly tools?: readonly string[];
116
+ /**
117
+ * Which tools are PRE-APPROVED (CC: `--allowedTools`); undefined = the
118
+ * default four. It never removes a tool. A driver with no such concept
119
+ * ignores it.
120
+ */
121
+ readonly allowedTools?: readonly string[];
112
122
  /** Capture the full event transcript instead of just the final result. */
113
123
  readonly transcript: boolean;
114
124
  /** Path to a plugin dir to install natively, if any. */
@@ -0,0 +1,32 @@
1
+ /**
2
+ * Running past the end of a scripted model is an error.
3
+ *
4
+ * A harness test scripts the turns the agent should take, then asserts on what
5
+ * happened. When the agent asks for more turns than the script has, the mock
6
+ * used to answer with a copy of the LAST turn: a looping agent (a Stop hook
7
+ * that keeps blocking, a retry, a reaction to a tool result) then ran on turns
8
+ * nobody scripted, a final `{ tool }` re-ran its side effect, and the test went
9
+ * green over a run it never described (#340). Pure and shared by both mocks
10
+ * (Anthropic Messages, OpenAI Responses) and the runner that fails the test.
11
+ */
12
+ import type { ModelRequest } from "./harness-driver.js";
13
+ /** The first request the script could not answer. */
14
+ export interface ScriptOverrun {
15
+ /** How many turns the script had. */
16
+ readonly scripted: number;
17
+ /** 1-based number of the agent request that had no turn left. */
18
+ readonly request: number;
19
+ /** The last message that request carried (what the agent was reacting to). */
20
+ readonly lastMessage: string;
21
+ }
22
+ /** Why a test failed because the agent outran its script. */
23
+ export declare function scriptOverrunMessage(o: ScriptOverrun): string;
24
+ /**
25
+ * The first agent request past the end of a script of `scripted` turns, read
26
+ * off the recorded requests (side-channel calls never consume a turn, so they
27
+ * do not count). Undefined when the agent stayed inside the script.
28
+ */
29
+ export declare function findScriptOverrun(scripted: number, requests: readonly ModelRequest[]): ScriptOverrun | undefined;
30
+ /** {@link findScriptOverrun} and {@link scriptOverrunMessage} in one: the message, or undefined. */
31
+ export declare function overrunMessageFor(scripted: number, requests: readonly ModelRequest[]): string | undefined;
32
+ //# sourceMappingURL=script-overrun.d.ts.map
@@ -0,0 +1,42 @@
1
+ "use strict";
2
+ Object.defineProperty(exports, "__esModule", { value: true });
3
+ exports.scriptOverrunMessage = scriptOverrunMessage;
4
+ exports.findScriptOverrun = findScriptOverrun;
5
+ exports.overrunMessageFor = overrunMessageFor;
6
+ const MAX_QUOTED = 160;
7
+ const quote = (text) => {
8
+ const flat = text.replace(/\s+/g, " ").trim();
9
+ return flat.length > MAX_QUOTED ? `${flat.slice(0, MAX_QUOTED)}...` : flat;
10
+ };
11
+ /** Why a test failed because the agent outran its script. */
12
+ function scriptOverrunMessage(o) {
13
+ const last = o.lastMessage.trim() === ""
14
+ ? ""
15
+ : ` The request carried this last message: "${quote(o.lastMessage)}".`;
16
+ return (`the agent asked for more model turns than the script has: ${String(o.scripted)} turn(s) scripted, ` +
17
+ `and agent request #${String(o.request)} had none left.${last} ` +
18
+ "The mock does not repeat the last turn, because a looping agent would then pass on turns nobody scripted. " +
19
+ "Script every turn the agent should take.");
20
+ }
21
+ /**
22
+ * The first agent request past the end of a script of `scripted` turns, read
23
+ * off the recorded requests (side-channel calls never consume a turn, so they
24
+ * do not count). Undefined when the agent stayed inside the script.
25
+ */
26
+ function findScriptOverrun(scripted, requests) {
27
+ const agent = requests.filter((r) => r.sideChannel !== true);
28
+ const first = agent[scripted];
29
+ if (first === undefined)
30
+ return undefined;
31
+ return {
32
+ scripted,
33
+ request: scripted + 1,
34
+ lastMessage: first.messages.at(-1)?.text ?? "",
35
+ };
36
+ }
37
+ /** {@link findScriptOverrun} and {@link scriptOverrunMessage} in one: the message, or undefined. */
38
+ function overrunMessageFor(scripted, requests) {
39
+ const overrun = findScriptOverrun(scripted, requests);
40
+ return overrun === undefined ? undefined : scriptOverrunMessage(overrun);
41
+ }
42
+ //# sourceMappingURL=script-overrun.js.map
@@ -20,7 +20,7 @@
20
20
  * anywhere in the prose counts, because a name like `test:harness` cannot
21
21
  * collide with an English word the way `test` and `audit` do. The only thing the
22
22
  * lookarounds buy is that a LONGER script name is never read as a shorter one
23
- * (`test:cli-e2e` does not document `test:e2e`).
23
+ * (`test:unit-extra` does not document `test:unit`).
24
24
  *
25
25
  * Scope is the `test:` PREFIX, deliberately. Bare `npm test` is the vitest
26
26
  * suite, named in the map as prose; every other script (`build`, `lint`,
@@ -21,7 +21,7 @@
21
21
  * anywhere in the prose counts, because a name like `test:harness` cannot
22
22
  * collide with an English word the way `test` and `audit` do. The only thing the
23
23
  * lookarounds buy is that a LONGER script name is never read as a shorter one
24
- * (`test:cli-e2e` does not document `test:e2e`).
24
+ * (`test:unit-extra` does not document `test:unit`).
25
25
  *
26
26
  * Scope is the `test:` PREFIX, deliberately. Bare `npm test` is the vitest
27
27
  * suite, named in the map as prose; every other script (`build`, `lint`,
@@ -116,12 +116,11 @@ export declare function lockPath(dir: string, name: string): string;
116
116
  */
117
117
  export declare function readLock(dir: string, name: string): EvalLock | null;
118
118
  /**
119
- * Whether ANY lock has been committed under `dir`. The CI staleness gate
120
- * (`eval --check`) uses this to stay a NO-OP until the feature is in use: a repo
121
- * that has never run `eval --update` has no locks, so there is nothing to verify
122
- * and CI passes green. Once the first lock is committed, every named eval is held
123
- * to having a fresh one (a new unlocked eval then reads as stale). The graduated,
124
- * opt-in-by-committing behavior that keeps a fresh `init` from going red.
119
+ * Whether ANY lock has been committed under `dir`. `eval --check` fails when none
120
+ * is (#197): with no lock there is no recorded result to verify the current inputs
121
+ * against, so a pass would mean nothing was checked. Once the first lock is
122
+ * committed, every named eval is held to having a fresh one (a new unlocked eval
123
+ * then reads as stale). The edit-time nudge also stays silent until then.
125
124
  */
126
125
  export declare function anyLocksCommitted(dir: string): boolean;
127
126
  /**
package/dist/eval-lock.js CHANGED
@@ -130,12 +130,11 @@ function readLock(dir, name) {
130
130
  return obj;
131
131
  }
132
132
  /**
133
- * Whether ANY lock has been committed under `dir`. The CI staleness gate
134
- * (`eval --check`) uses this to stay a NO-OP until the feature is in use: a repo
135
- * that has never run `eval --update` has no locks, so there is nothing to verify
136
- * and CI passes green. Once the first lock is committed, every named eval is held
137
- * to having a fresh one (a new unlocked eval then reads as stale). The graduated,
138
- * opt-in-by-committing behavior that keeps a fresh `init` from going red.
133
+ * Whether ANY lock has been committed under `dir`. `eval --check` fails when none
134
+ * is (#197): with no lock there is no recorded result to verify the current inputs
135
+ * against, so a pass would mean nothing was checked. Once the first lock is
136
+ * committed, every named eval is held to having a fresh one (a new unlocked eval
137
+ * then reads as stale). The edit-time nudge also stays silent until then.
139
138
  */
140
139
  function anyLocksCommitted(dir) {
141
140
  return countLocks(dir) > 0;
@@ -1,5 +1,5 @@
1
1
  import type { HarnessAdapter } from "./core/adapter.js";
2
- import type { HarnessTestDriver, ToolCall, HookFire, ParsedRun, ModelTurn } from "./core/harness-driver.js";
2
+ import type { HarnessTestDriver, ToolCall, HookFire, ParsedRun, ModelTurn, ModelRequest } from "./core/harness-driver.js";
3
3
  import type { Trace, SubagentTrace } from "./core/eval-driver.js";
4
4
  export type { Trace, SubagentTrace } from "./core/eval-driver.js";
5
5
  import { parseReplies, reachedModel } from "./adapters/claude-code/replies.js";
@@ -52,7 +52,38 @@ export interface HarnessTestSpec {
52
52
  readonly model: readonly ModelTurn[];
53
53
  /** The user prompt. Default: "go". */
54
54
  readonly prompt?: string;
55
- /** Tools the agent may use. Default: Read Edit Write Bash. */
55
+ /**
56
+ * Which tools EXIST in the session (`claude --tools`, Claude Code 2.0.31+). A
57
+ * tool left out is not offered at all, so a call to it is "No such tool
58
+ * available". Omitted, every tool exists. This is the field that withholds a
59
+ * tool. Names only: a permission rule such as `Bash(git *)` is refused (put the
60
+ * rule in `allowedTools`). `[]` is refused (an agent with no tools is never
61
+ * served a scripted turn), and a scripted call to a tool the list leaves out
62
+ * throws before the run starts. An MCP tool is not a built-in, so `--tools`
63
+ * does not withhold it. Codex has no equivalent, so a run there refuses it.
64
+ *
65
+ * | field | Claude Code flag | meaning |
66
+ * | -------------- | ----------------- | ---------------------------- |
67
+ * | `tools` | `--tools` | which tools exist |
68
+ * | `allowedTools` | `--allowedTools` | which are pre-approved |
69
+ */
70
+ readonly tools?: readonly string[];
71
+ /**
72
+ * Which tools are PRE-APPROVED (`claude --allowedTools`), so they do not stop
73
+ * on a permission prompt. Takes permission rules (`Bash(git *)`). Omitted:
74
+ * Read, Edit, Write and Bash are approved. An explicit list REPLACES that
75
+ * default, so `tools: ["Read", "Write"]` with `allowedTools: ["Skill"]` leaves
76
+ * Write unapproved. `[]`: nothing is approved.
77
+ *
78
+ * It never removes a tool, and leaving a tool out is not a fence: in headless
79
+ * mode Claude Code refuses only the calls it would have asked about. For Bash
80
+ * that is a command that creates, changes or removes a file (`touch`, `>`),
81
+ * inside or outside the working directory; a command with no file effect
82
+ * (`ls`, `printf`, `id`) runs unapproved. So a refused `touch` proves the
83
+ * file-change guard is live, not that Bash is withheld. To withhold a tool use
84
+ * `tools`; to forbid its use put a deny rule in the fixture's
85
+ * `settings.permissions.deny`.
86
+ */
56
87
  readonly allowedTools?: readonly string[];
57
88
  /**
58
89
  * Capture the full event transcript (`--output-format stream-json`) into
@@ -119,6 +150,41 @@ export declare function parseResultEvent(stdout: string): Record<string, unknown
119
150
  /** The agent's final answer text from a transcript / result object, or "". */
120
151
  export declare function parseOutput(stdout: string): string;
121
152
  export declare function parseHooks(stdout: string): HookFire[];
153
+ /**
154
+ * The names `--tools` takes for a `tools` list: trimmed and deduplicated.
155
+ * `--allowedTools` takes permission rules (`Bash(git *)`); `--tools` takes tool
156
+ * NAMES, and a rule there makes the CLI offer no tools at all. Reducing it to
157
+ * `Bash` would offer more than the author wrote, so a rule is refused: it
158
+ * belongs in `allowedTools`. Pure.
159
+ */
160
+ export declare function toolAvailabilityList(tools: readonly string[]): readonly string[];
161
+ /**
162
+ * A scripted call to a tool an EXPLICIT `tools` list leaves out. The CLI answers
163
+ * it with "No such tool available" and the run goes on, so a test that scripts
164
+ * the call and then asserts on something else passes over a step that never
165
+ * happened. A tool that exists but is not in `allowedTools` is NOT this: the CLI
166
+ * refuses it for permission, which is a result a test can assert on. Returns the
167
+ * message to throw, or undefined. MCP tools are not built-ins, so `--tools` does
168
+ * not withhold them and they are skipped. Pure.
169
+ */
170
+ export declare function unofferedScriptedTool(model: readonly ModelTurn[], tools: readonly string[] | undefined): string | undefined;
171
+ /**
172
+ * The agent's own tool results, read off the requests it sent back to the model,
173
+ * that say a scripted tool does not exist. The CLI answers an unknown name
174
+ * (`Bsh` for `Bash`, an `mcp__` server that is not configured) with an error
175
+ * RESULT and carries on, exit 0, so a test that asserts "the call errored and
176
+ * left no side effect" passes on a typo. `tools` cannot catch it: the pre-flight
177
+ * skips `mcp__` names and only runs when `tools` is given. Side-channel requests
178
+ * are the CLI's own bookkeeping and are not read. Returns the message to throw,
179
+ * or undefined. Pure.
180
+ */
181
+ export declare function noSuchToolMessage(model: readonly ModelTurn[], requests: readonly ModelRequest[]): string | undefined;
182
+ /**
183
+ * A run on a `claude` that predates `--tools` exits at argument parsing, and
184
+ * commander's "unknown option" says nothing about WHY vigiles passed it. Returns
185
+ * the message to throw, or undefined when the run did not die that way. Pure.
186
+ */
187
+ export declare function unsupportedToolsFlag(stderr: string): string | undefined;
122
188
  /**
123
189
  * The `claude` CLI argv for a harness run (shared by the direct and sandboxed
124
190
  * paths). `ANTHROPIC_BASE_URL` is set by the caller's environment / wrapper, not
@@ -170,6 +236,12 @@ export interface RunHarnessTestOptions {
170
236
  */
171
237
  readonly adapter?: HarnessAdapter;
172
238
  }
239
+ /** Say so on stderr when side-channel calls arrived and no script turn was served. */
240
+ export declare function warnUnconsumed(count: number, sideChannelCount: number): void;
241
+ /**
242
+ * What makes a finished run unsound, most specific first, or undefined. Pure.
243
+ */
244
+ export declare function runProblem(stderr: string, model: readonly ModelTurn[], modelRequests: readonly ModelRequest[], tools?: readonly string[]): string | undefined;
173
245
  /**
174
246
  * Run the real agent CLI against a scripted mock model, with the given fixture
175
247
  * and settings (hooks). Deterministic — same script, same result. Adapter-driven
@@ -6,10 +6,16 @@ exports.parseSubagents = parseSubagents;
6
6
  exports.parseResultEvent = parseResultEvent;
7
7
  exports.parseOutput = parseOutput;
8
8
  exports.parseHooks = parseHooks;
9
+ exports.toolAvailabilityList = toolAvailabilityList;
10
+ exports.unofferedScriptedTool = unofferedScriptedTool;
11
+ exports.noSuchToolMessage = noSuchToolMessage;
12
+ exports.unsupportedToolsFlag = unsupportedToolsFlag;
9
13
  exports.buildClaudeArgs = buildClaudeArgs;
10
14
  exports.parseClaudeRun = parseClaudeRun;
11
15
  exports.harnessSpawnEnv = harnessSpawnEnv;
12
16
  exports.claudeAvailable = claudeAvailable;
17
+ exports.warnUnconsumed = warnUnconsumed;
18
+ exports.runProblem = runProblem;
13
19
  exports.runHarnessTest = runHarnessTest;
14
20
  exports.runHarnessTestIn = runHarnessTestIn;
15
21
  exports.runHarness = runHarness;
@@ -49,6 +55,7 @@ const node_fs_1 = require("node:fs");
49
55
  const node_path_1 = require("node:path");
50
56
  const adapter_conformance_js_1 = require("./adapter-conformance.js");
51
57
  const check_count_js_1 = require("./check-count.js");
58
+ const script_overrun_js_1 = require("./core/script-overrun.js");
52
59
  const coverage_probe_js_1 = require("./coverage-probe.js");
53
60
  const runtime_js_1 = require("./adapters/claude-code/runtime.js");
54
61
  const mock_model_js_1 = require("./mock-model.js");
@@ -274,13 +281,92 @@ function parseHooks(stdout) {
274
281
  }
275
282
  return hooks;
276
283
  }
284
+ /**
285
+ * The first `claude` that has `--tools` (print mode). Found by diffing the
286
+ * published `@anthropic-ai/claude-code` tarballs: 2.0.30 has no such option,
287
+ * 2.0.31 does (the changelog does not list it; its 2.1.0 entry only extends the
288
+ * flag to interactive mode).
289
+ */
290
+ const TOOLS_FLAG_MIN_CLAUDE = "2.0.31";
291
+ /**
292
+ * The names `--tools` takes for a `tools` list: trimmed and deduplicated.
293
+ * `--allowedTools` takes permission rules (`Bash(git *)`); `--tools` takes tool
294
+ * NAMES, and a rule there makes the CLI offer no tools at all. Reducing it to
295
+ * `Bash` would offer more than the author wrote, so a rule is refused: it
296
+ * belongs in `allowedTools`. Pure.
297
+ */
298
+ function toolAvailabilityList(tools) {
299
+ const names = tools.map((entry) => entry.trim()).filter((n) => n !== "");
300
+ const rule = names.find((name) => name.includes("("));
301
+ if (rule !== undefined) {
302
+ throw new Error(`tools takes bare tool names, and "${rule}" is a permission rule — put the rule in allowedTools and the name ("${rule.replace(/\(.*$/, "")}") in tools`);
303
+ }
304
+ return [...new Set(names)];
305
+ }
306
+ /**
307
+ * A scripted call to a tool an EXPLICIT `tools` list leaves out. The CLI answers
308
+ * it with "No such tool available" and the run goes on, so a test that scripts
309
+ * the call and then asserts on something else passes over a step that never
310
+ * happened. A tool that exists but is not in `allowedTools` is NOT this: the CLI
311
+ * refuses it for permission, which is a result a test can assert on. Returns the
312
+ * message to throw, or undefined. MCP tools are not built-ins, so `--tools` does
313
+ * not withhold them and they are skipped. Pure.
314
+ */
315
+ function unofferedScriptedTool(model, tools) {
316
+ if (tools === undefined)
317
+ return undefined;
318
+ const exist = toolAvailabilityList(tools);
319
+ const at = model.findIndex((t) => t.tool !== undefined &&
320
+ !t.tool.startsWith("mcp__") &&
321
+ !exist.includes(t.tool));
322
+ return at < 0
323
+ ? undefined
324
+ : `scripted call to "${model[at]?.tool ?? ""}" on turn ${String(at + 1)}, but this run's tools are only: ${exist.join(", ")} — add it to tools`;
325
+ }
326
+ /**
327
+ * The agent's own tool results, read off the requests it sent back to the model,
328
+ * that say a scripted tool does not exist. The CLI answers an unknown name
329
+ * (`Bsh` for `Bash`, an `mcp__` server that is not configured) with an error
330
+ * RESULT and carries on, exit 0, so a test that asserts "the call errored and
331
+ * left no side effect" passes on a typo. `tools` cannot catch it: the pre-flight
332
+ * skips `mcp__` names and only runs when `tools` is given. Side-channel requests
333
+ * are the CLI's own bookkeeping and are not read. Returns the message to throw,
334
+ * or undefined. Pure.
335
+ */
336
+ function noSuchToolMessage(model, requests) {
337
+ // The CLI's own wording, inside its error wrapper, for a name the script
338
+ // calls: the agent calls tools only from the script, so the same words in the
339
+ // prompt or in a hook's injected text are not a result of this run.
340
+ const pattern = /<tool_use_error>Error: No such tool available: ([^\s<]+)<\/tool_use_error>/g;
341
+ const scripted = new Set(model.flatMap((t) => (t.tool === undefined ? [] : [t.tool])));
342
+ const hit = requests
343
+ .filter((r) => r.sideChannel !== true)
344
+ .flatMap((r) => r.messages)
345
+ .flatMap((m) => [...m.text.matchAll(pattern)].map((x) => x[1]))
346
+ .find((name) => name !== undefined && scripted.has(name));
347
+ if (hit === undefined)
348
+ return undefined;
349
+ const at = model.findIndex((t) => t.tool === hit);
350
+ const turn = at < 0 ? "" : ` on turn ${String(at + 1)}`;
351
+ return `scripted call to "${hit}"${turn}: the CLI answered "No such tool available: ${hit}", so the call never ran and the test would carry on over a step that did not happen. Check the name for a typo; an mcp__ tool also needs its server configured in the fixture.`;
352
+ }
353
+ /**
354
+ * A run on a `claude` that predates `--tools` exits at argument parsing, and
355
+ * commander's "unknown option" says nothing about WHY vigiles passed it. Returns
356
+ * the message to throw, or undefined when the run did not die that way. Pure.
357
+ */
358
+ function unsupportedToolsFlag(stderr) {
359
+ return /unknown option '--tools'/.test(stderr)
360
+ ? `this \`claude\` does not know \`--tools\`, which runHarnessTest passes for the \`tools\` field (which tools exist). Update Claude Code to ${TOOLS_FLAG_MIN_CLAUDE} or newer. Running without it would hand the agent every tool while the test reads as if it were restricted.`
361
+ : undefined;
362
+ }
277
363
  /**
278
364
  * The `claude` CLI argv for a harness run (shared by the direct and sandboxed
279
365
  * paths). `ANTHROPIC_BASE_URL` is set by the caller's environment / wrapper, not
280
366
  * here. Pure, so the arg shape is unit-tested.
281
367
  */
282
368
  function buildClaudeArgs(spec, hasSettings) {
283
- const tools = spec.allowedTools ?? ["Read", "Edit", "Write", "Bash"];
369
+ const approved = spec.allowedTools ?? ["Read", "Edit", "Write", "Bash"];
284
370
  return [
285
371
  "-p",
286
372
  spec.prompt ?? "go",
@@ -293,9 +379,15 @@ function buildClaudeArgs(spec, hasSettings) {
293
379
  ? ["--plugin-dir", (0, node_path_1.resolve)(spec.pluginDir)]
294
380
  : []),
295
381
  ...(hasSettings ? ["--settings", "settings.json"] : []),
296
- // A permission allowlist: an empty one approves nothing, so the flag is
297
- // left out (a bare `--allowedTools` exits before any model turn).
298
- ...(tools.length === 0 ? [] : ["--allowedTools", ...tools]),
382
+ // `tools` = which tools EXIST (`--tools`, the availability list). Omitted:
383
+ // nothing is withheld.
384
+ ...(spec.tools === undefined
385
+ ? []
386
+ : ["--tools", toolAvailabilityList(spec.tools).join(",")]),
387
+ // `allowedTools` = which are PRE-APPROVED (`--allowedTools`); it never
388
+ // restricts. An empty list approves nothing, so the flag is left out (a bare
389
+ // `--allowedTools` exits before any model turn).
390
+ ...(approved.length === 0 ? [] : ["--allowedTools", ...approved]),
299
391
  ];
300
392
  }
301
393
  /** Build the `claude` argv from the driver context — wraps `buildClaudeArgs`. */
@@ -305,7 +397,8 @@ function buildClaudeArgsFromCtx(ctx) {
305
397
  prompt: ctx.prompt,
306
398
  transcript: ctx.transcript,
307
399
  pluginDir: ctx.pluginDir,
308
- allowedTools: ctx.tools,
400
+ tools: ctx.tools,
401
+ allowedTools: ctx.allowedTools,
309
402
  }, ctx.hasSettings);
310
403
  }
311
404
  /** Parse the `claude` stdout/stream into the unified trace fields. */
@@ -415,6 +508,52 @@ exports.claudeCodeDriver = {
415
508
  parseRun: parseClaudeRun,
416
509
  available: claudeAvailable,
417
510
  };
511
+ /** Say so on stderr when side-channel calls arrived and no script turn was served. */
512
+ function warnUnconsumed(count, sideChannelCount) {
513
+ const unconsumed = (0, mock_model_js_1.scriptUnconsumedWarning)(count, sideChannelCount);
514
+ if (unconsumed !== undefined)
515
+ console.error(unconsumed);
516
+ }
517
+ /**
518
+ * `tools` made only of names the CLI does not know leaves the agent with none:
519
+ * every request then declares no tools, the mock files it as side-channel
520
+ * traffic, and the script is never served, so the run decides nothing. Requests
521
+ * that all came in that way, under an explicit `tools`, mean exactly that. No
522
+ * request at all (a hook stopped the prompt) is a different run. Pure.
523
+ */
524
+ function toolsResolvedToNothing(tools, requests) {
525
+ if (tools === undefined || requests.length === 0)
526
+ return undefined;
527
+ if (!requests.every((r) => r.sideChannel === true))
528
+ return undefined;
529
+ return `tools ${JSON.stringify(tools)} gave the agent no tools: no name in it is a tool this CLI knows, so no request offered a tool and the script was never served. Check the names for typos.`;
530
+ }
531
+ /**
532
+ * What makes a finished run unsound, most specific first, or undefined. Pure.
533
+ */
534
+ function runProblem(stderr, model, modelRequests, tools) {
535
+ return (unsupportedToolsFlag(stderr) ??
536
+ toolsResolvedToNothing(tools, modelRequests) ??
537
+ // A typo'd last turn makes the CLI ask once more, so the overrun follows
538
+ // from it: report the cause, not its effect.
539
+ noSuchToolMessage(model, modelRequests) ??
540
+ (0, script_overrun_js_1.overrunMessageFor)(model.length, modelRequests));
541
+ }
542
+ /**
543
+ * A run that proves nothing about the script fails the test instead of coming
544
+ * back as a result: the `claude` on PATH predates `--tools` (needed for `tools`), or the agent
545
+ * asked for a model turn the script did not have and the mock answered with an
546
+ * error rather than an invented turn (#340), or a scripted call named a tool the
547
+ * CLI does not have. Removes the run's directory, since
548
+ * no result is handed back to clean it up.
549
+ */
550
+ function assertRunSound(cwd, stderr, model, modelRequests, tools) {
551
+ const problem = runProblem(stderr, model, modelRequests, tools);
552
+ if (problem === undefined)
553
+ return;
554
+ (0, node_fs_1.rmSync)(cwd, { recursive: true, force: true });
555
+ throw new Error(problem);
556
+ }
418
557
  /* v8 ignore start -- spawns the real agent CLI + filesystem; exercised by the
419
558
  claude-backed + gated codex suites, excluded from the deterministic coverage
420
559
  gate (the parse helpers above carry the testable logic). */
@@ -457,8 +596,36 @@ function makeResult(cwd, out, parsed, turns, modelRequests) {
457
596
  * only — requesting confinement for another harness throws.
458
597
  */
459
598
  async function runHarnessTest(spec, opts = {}) {
599
+ refuseUnrunnableSpec(spec);
460
600
  return runHarnessTestIn(spec, opts, "inherit");
461
601
  }
602
+ /** The driver's view of one run of `spec`. Pure. */
603
+ function driverContext(spec, cwd, hasSettings, mockArgs) {
604
+ return {
605
+ prompt: spec.prompt ?? "go",
606
+ cwd,
607
+ hasSettings,
608
+ tools: spec.tools,
609
+ allowedTools: spec.allowedTools,
610
+ transcript: spec.transcript ?? false,
611
+ pluginDir: spec.pluginDir,
612
+ mockArgs,
613
+ };
614
+ }
615
+ /**
616
+ * The scripted model tells an agent turn from the CLI's own bookkeeping calls by
617
+ * the tools the request declares (`isMainLoopRequest`). An agent with none is
618
+ * never served a script turn, and the run would decide on nothing.
619
+ */
620
+ function refuseUnrunnableSpec(spec) {
621
+ if (spec.tools !== undefined &&
622
+ toolAvailabilityList(spec.tools).length === 0) {
623
+ throw new Error("tools: [] leaves the agent with no tools (a list of blank names counts as empty), and the scripted model only serves a turn to a request that declares tools — no script turn would be consumed. Name the tools the agent has.");
624
+ }
625
+ const unoffered = unofferedScriptedTool(spec.model, spec.tools);
626
+ if (unoffered !== undefined)
627
+ throw new Error(unoffered);
628
+ }
462
629
  /**
463
630
  * {@link runHarnessTest} with the run's HOME chosen by the caller — see
464
631
  * {@link HarnessRunHome}. Not on a public entry point: the one caller that needs
@@ -491,17 +658,19 @@ async function runHarnessTestIn(spec, opts, home) {
491
658
  const { files, settings, check } = fixtureFor(spec, opts.adapter);
492
659
  const cwd = (0, tmp_root_js_1.makeTmpDir)("harness");
493
660
  writeFixture(cwd, files, settings);
494
- const tools = spec.allowedTools ?? ["Read", "Edit", "Write", "Bash"];
495
661
  const timeoutMs = spec.timeoutMs ?? 60000;
496
- const buildArgs = (mockArgs) => driver.buildArgs({
497
- prompt: spec.prompt ?? "go",
498
- cwd,
499
- hasSettings: settings !== undefined,
500
- tools,
501
- transcript: spec.transcript ?? false,
502
- pluginDir: spec.pluginDir,
503
- mockArgs,
504
- });
662
+ // A driver refuses a spec it cannot run (Codex and `tools`) inside buildArgs.
663
+ // That refusal must not leave the run's temp dir behind; the mock, if started,
664
+ // is closed by the caller's finally.
665
+ const buildArgs = (mockArgs) => {
666
+ try {
667
+ return driver.buildArgs(driverContext(spec, cwd, settings !== undefined, mockArgs));
668
+ }
669
+ catch (error) {
670
+ (0, node_fs_1.rmSync)(cwd, { recursive: true, force: true });
671
+ throw error;
672
+ }
673
+ };
505
674
  // Confined path (Claude Code only): the mock is co-launched in the netns, so
506
675
  // the agent reaches it over the loopback URL the sandbox sets — Claude Code is
507
676
  // env-only (empty mockArgs), so the argv needs no mock-wiring flags.
@@ -519,9 +688,8 @@ async function runHarnessTestIn(spec, opts, home) {
519
688
  // the direct one stays right. Same recovery feeds the unconsumed-script
520
689
  // warning, which the sandbox path could not emit at all before.
521
690
  const { count, sideChannelCount } = (0, mock_model_js_1.splitRequestCounts)(out.requests);
522
- const unconsumed = (0, mock_model_js_1.scriptUnconsumedWarning)(count, sideChannelCount);
523
- if (unconsumed !== undefined)
524
- console.error(unconsumed);
691
+ warnUnconsumed(count, sideChannelCount);
692
+ assertRunSound(cwd, "", spec.model, out.requests, spec.tools);
525
693
  return check(makeResult(cwd, out, parseClaudeRun(out.stdout), count, out.requests));
526
694
  }
527
695
  // Direct path: mock runs in this process; the agent reaches it over localhost.
@@ -539,9 +707,8 @@ async function runHarnessTestIn(spec, opts, home) {
539
707
  // looks empty. `scriptUnconsumedWarning` names the one shape that produces
540
708
  // it (every request arriving without tool declarations, so nothing looked
541
709
  // like an agent turn) instead of leaving it to be rediscovered.
542
- const unconsumed = (0, mock_model_js_1.scriptUnconsumedWarning)(mock.count, mock.sideChannelCount ?? 0);
543
- if (unconsumed !== undefined)
544
- console.error(unconsumed);
710
+ warnUnconsumed(mock.count, mock.sideChannelCount ?? 0);
711
+ assertRunSound(cwd, out.stderr, spec.model, mock.requests, spec.tools);
545
712
  return check(makeResult(cwd, out, driver.parseRun(out.stdout), mock.count, [
546
713
  ...mock.requests,
547
714
  ]));
@@ -78,7 +78,10 @@ export declare function extractRequest(body: {
78
78
  }): ModelRequest;
79
79
  /**
80
80
  * Start the scripted mock on a free port. Each MAIN-LOOP `/v1/messages` POST
81
- * consumes the next turn (the last turn repeats if the client asks for more);
81
+ * consumes the next turn; one past the end of the script is answered with a
82
+ * non-retryable API error, never a copy of the last turn (#340 — a looping
83
+ * agent would pass on turns nobody scripted), and `runHarnessTest` then fails
84
+ * the test from the recorded requests ({@link findScriptOverrun});
82
85
  * a SIDE-CHANNEL POST — the CLI's own bookkeeping calls, see
83
86
  * {@link isMainLoopRequest} — is answered from outside the script and consumes
84
87
  * nothing. Resolves to a handle with the base `url` and a `close()`.
@@ -28,6 +28,7 @@ exports.startMock = startMock;
28
28
  * and echoing the requested model.
29
29
  */
30
30
  const node_http_1 = __importDefault(require("node:http"));
31
+ const script_overrun_js_1 = require("./core/script-overrun.js");
31
32
  /** Build a scripted model from an ordered list of turns. */
32
33
  function scriptModel(turns) {
33
34
  return [...turns];
@@ -388,9 +389,41 @@ function extractRequest(body) {
388
389
  : [];
389
390
  return { system: flattenContent(body.system), messages };
390
391
  }
392
+ /**
393
+ * The 400 body for an agent request past the end of the script: an API error,
394
+ * because the CLI retries 429/5xx and an invented turn would let the agent act on
395
+ * a step nobody scripted (#340). Pure.
396
+ */
397
+ function pastScriptBody(scripted, request) {
398
+ const message = (0, script_overrun_js_1.scriptOverrunMessage)({
399
+ scripted,
400
+ request: scripted + 1,
401
+ lastMessage: request.messages.at(-1)?.text ?? "",
402
+ });
403
+ return JSON.stringify({
404
+ type: "error",
405
+ error: { type: "invalid_request_error", message },
406
+ });
407
+ }
408
+ /** Does a message's content carry a tool result (the agent reacting to a tool)? */
409
+ function endsWithToolResult(content) {
410
+ return JSON.stringify(content ?? "").includes('"tool_result"');
411
+ }
412
+ /** The JSON body of a request, `{}` for the HEAD / health checks that have none. */
413
+ function parseReqBody(body) {
414
+ try {
415
+ return JSON.parse(body);
416
+ }
417
+ catch {
418
+ return {};
419
+ }
420
+ }
391
421
  /**
392
422
  * Start the scripted mock on a free port. Each MAIN-LOOP `/v1/messages` POST
393
- * consumes the next turn (the last turn repeats if the client asks for more);
423
+ * consumes the next turn; one past the end of the script is answered with a
424
+ * non-retryable API error, never a copy of the last turn (#340 — a looping
425
+ * agent would pass on turns nobody scripted), and `runHarnessTest` then fails
426
+ * the test from the recorded requests ({@link findScriptOverrun});
394
427
  * a SIDE-CHANNEL POST — the CLI's own bookkeeping calls, see
395
428
  * {@link isMainLoopRequest} — is answered from outside the script and consumes
396
429
  * nothing. Resolves to a handle with the base `url` and a `close()`.
@@ -406,13 +439,7 @@ function startMock(script, opts = {}) {
406
439
  const url = req.url ?? "";
407
440
  const isCount = url.includes("count_tokens");
408
441
  const isMessages = url.includes("/v1/messages") && !isCount;
409
- let reqBody = {};
410
- try {
411
- reqBody = JSON.parse(body);
412
- }
413
- catch {
414
- /* HEAD / health checks have no JSON body */
415
- }
442
+ const reqBody = parseReqBody(body);
416
443
  if (req.method === "HEAD" || (!isMessages && !isCount)) {
417
444
  res.writeHead(200, { "content-type": "application/json" });
418
445
  res.end("{}");
@@ -445,13 +472,17 @@ function startMock(script, opts = {}) {
445
472
  jsonTurn(res, reply, model);
446
473
  return;
447
474
  }
448
- const last = JSON.stringify(reqBody.messages?.at(-1)?.content ?? "");
475
+ const turn = script[i];
476
+ if (turn === undefined) {
477
+ res.writeHead(400, { "content-type": "application/json" });
478
+ res.end(pastScriptBody(script.length, request));
479
+ return;
480
+ }
449
481
  opts.onTurn?.({
450
482
  n: i,
451
483
  stream: reqBody.stream === true,
452
- hasToolResult: last.includes('"tool_result"'),
484
+ hasToolResult: endsWithToolResult(reqBody.messages?.at(-1)?.content),
453
485
  });
454
- const turn = script[Math.min(i, script.length - 1)] ?? { text: "" };
455
486
  i++;
456
487
  if (reqBody.stream === true)
457
488
  streamTurn(res, turn, model);
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "vigiles",
3
- "version": "34.0.0",
3
+ "version": "36.0.0",
4
4
  "description": "Audit, test and measure the harness your AI agent runs on — grade your CLAUDE.md / AGENTS.md, skills, subagents and hooks, run them against a scripted model, and measure whether they actually fire.",
5
5
  "keywords": [
6
6
  "claude-code",
@@ -83,7 +83,6 @@
83
83
  "test:unit": "npm run build && vitest run --project unit",
84
84
  "test:integration": "npm run build && vitest run --project integration",
85
85
  "test:e2e": "npm run build && vitest run --project e2e",
86
- "test:cli-e2e": "bash test/e2e/run.sh",
87
86
  "test:harness": "npm run build && node dist/cli.js test",
88
87
  "test:eval": "npm run build && node dist/cli.js eval --all",
89
88
  "test:vitest": "npm run build && vitest run --project runners",