vigiles 34.0.0 → 36.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/adapters/codex/driver.d.ts +3 -0
- package/dist/adapters/codex/driver.js +8 -1
- package/dist/adapters/codex/mock-model.d.ts +3 -2
- package/dist/adapters/codex/mock-model.js +15 -3
- package/dist/cli-main.js +76 -52
- package/dist/core/harness-driver.d.ts +12 -2
- package/dist/core/script-overrun.d.ts +32 -0
- package/dist/core/script-overrun.js +42 -0
- package/dist/doc-test-script-coverage.d.ts +1 -1
- package/dist/doc-test-script-coverage.js +1 -1
- package/dist/eval-lock.d.ts +5 -6
- package/dist/eval-lock.js +5 -6
- package/dist/harness-test.d.ts +74 -2
- package/dist/harness-test.js +188 -21
- package/dist/mock-model.d.ts +4 -1
- package/dist/mock-model.js +42 -11
- package/package.json +1 -2
|
@@ -7,6 +7,9 @@ import type { HarnessTestDriver, HarnessDriverContext, ParsedRun } from "../../c
|
|
|
7
7
|
* `--dangerously-bypass-approvals-and-sandbox` make it run unattended;
|
|
8
8
|
* `--skip-git-repo-check` lets it run in a bare temp dir; `--ignore-user-config`
|
|
9
9
|
* keeps the host's `~/.codex` out. The prompt is the trailing positional.
|
|
10
|
+
* `ctx.allowedTools` is ignored: `codex exec` runs with approvals bypassed, so
|
|
11
|
+
* everything is already approved. `ctx.tools` is refused (throws): Codex has no
|
|
12
|
+
* per-run tool availability.
|
|
10
13
|
* Pure, so the arg shape is unit-tested.
|
|
11
14
|
*/
|
|
12
15
|
export declare function buildCodexArgs(ctx: HarnessDriverContext): string[];
|
|
@@ -26,9 +26,15 @@ const mock_model_js_1 = require("./mock-model.js");
|
|
|
26
26
|
* `--dangerously-bypass-approvals-and-sandbox` make it run unattended;
|
|
27
27
|
* `--skip-git-repo-check` lets it run in a bare temp dir; `--ignore-user-config`
|
|
28
28
|
* keeps the host's `~/.codex` out. The prompt is the trailing positional.
|
|
29
|
+
* `ctx.allowedTools` is ignored: `codex exec` runs with approvals bypassed, so
|
|
30
|
+
* everything is already approved. `ctx.tools` is refused (throws): Codex has no
|
|
31
|
+
* per-run tool availability.
|
|
29
32
|
* Pure, so the arg shape is unit-tested.
|
|
30
33
|
*/
|
|
31
34
|
function buildCodexArgs(ctx) {
|
|
35
|
+
if (ctx.tools !== undefined) {
|
|
36
|
+
throw new Error("tools is not supported for codex: it has no per-run tool availability, and a list that withheld nothing would let a test pass over a run where nothing was withheld. Drop tools, or assert on what the agent did.");
|
|
37
|
+
}
|
|
32
38
|
return [
|
|
33
39
|
"exec",
|
|
34
40
|
"--ignore-user-config",
|
|
@@ -75,7 +81,8 @@ async function startCodexMockHandle(script) {
|
|
|
75
81
|
}));
|
|
76
82
|
},
|
|
77
83
|
get count() {
|
|
78
|
-
|
|
84
|
+
// Turns SERVED: a request past the end of the script gets a 400.
|
|
85
|
+
return Math.min(mock.requests.length, script.length);
|
|
79
86
|
},
|
|
80
87
|
close: () => mock.close(),
|
|
81
88
|
};
|
|
@@ -42,8 +42,9 @@ export declare function parseResponsesRequest(body: string): {
|
|
|
42
42
|
};
|
|
43
43
|
/**
|
|
44
44
|
* Start the scripted Codex Responses mock on a free port. Each
|
|
45
|
-
* `POST /v1/responses` consumes the next scripted turn
|
|
46
|
-
*
|
|
45
|
+
* `POST /v1/responses` consumes the next scripted turn; one past the end of the
|
|
46
|
+
* script is a 400, never a copy of the last turn (#340); HEAD and any other path
|
|
47
|
+
* get `{}`. Resolves to a handle
|
|
47
48
|
* with the base `url`, the `port`, the recorded `requests`, and `close()`.
|
|
48
49
|
*/
|
|
49
50
|
export declare function startCodexMock(script: readonly CodexTurn[], opts?: {
|
|
@@ -22,6 +22,7 @@ exports.startCodexMock = startCodexMock;
|
|
|
22
22
|
* `src/adapters/codex/runtime.ts` (`codexMockArgs`/`codexMockEnv`).
|
|
23
23
|
*/
|
|
24
24
|
const node_http_1 = __importDefault(require("node:http"));
|
|
25
|
+
const script_overrun_js_1 = require("../../core/script-overrun.js");
|
|
25
26
|
const MSG_ID = "msg_mock_1";
|
|
26
27
|
const RESP_ID = "resp_mock_1";
|
|
27
28
|
/**
|
|
@@ -183,8 +184,9 @@ function parseResponsesRequest(body) {
|
|
|
183
184
|
}
|
|
184
185
|
/**
|
|
185
186
|
* Start the scripted Codex Responses mock on a free port. Each
|
|
186
|
-
* `POST /v1/responses` consumes the next scripted turn
|
|
187
|
-
*
|
|
187
|
+
* `POST /v1/responses` consumes the next scripted turn; one past the end of the
|
|
188
|
+
* script is a 400, never a copy of the last turn (#340); HEAD and any other path
|
|
189
|
+
* get `{}`. Resolves to a handle
|
|
188
190
|
* with the base `url`, the `port`, the recorded `requests`, and `close()`.
|
|
189
191
|
*/
|
|
190
192
|
function startCodexMock(script, opts = {}) {
|
|
@@ -204,7 +206,17 @@ function startCodexMock(script, opts = {}) {
|
|
|
204
206
|
const parsed = parseResponsesRequest(body);
|
|
205
207
|
requests.push(parsed);
|
|
206
208
|
opts.onRequest?.(parsed);
|
|
207
|
-
const turn = script[
|
|
209
|
+
const turn = script[i];
|
|
210
|
+
if (turn === undefined) {
|
|
211
|
+
const message = (0, script_overrun_js_1.scriptOverrunMessage)({
|
|
212
|
+
scripted: script.length,
|
|
213
|
+
request: script.length + 1,
|
|
214
|
+
lastMessage: parsed.prompt,
|
|
215
|
+
});
|
|
216
|
+
res.writeHead(400, { "content-type": "application/json" });
|
|
217
|
+
res.end(JSON.stringify({ error: { type: "invalid_request_error", message } }));
|
|
218
|
+
return;
|
|
219
|
+
}
|
|
208
220
|
i++;
|
|
209
221
|
const model = parsed.model || "gpt-5-codex";
|
|
210
222
|
res.writeHead(200, {
|
package/dist/cli-main.js
CHANGED
|
@@ -2671,17 +2671,21 @@ function vigilesWorkflow(plan, harnesses, hasPackageJson) {
|
|
|
2671
2671
|
eval-check:
|
|
2672
2672
|
# Eval staleness gate — real-model evals run LOCALLY on your subscription
|
|
2673
2673
|
# (\`npx vigiles eval --update\`, which commits a lock); this job VERIFIES those
|
|
2674
|
-
# committed results against the current inputs with NO model call.
|
|
2675
|
-
#
|
|
2674
|
+
# committed results against the current inputs with NO model call.
|
|
2675
|
+
# \`--min=0\` is deliberate: a repo with no eval file yet matches nothing, and
|
|
2676
|
+
# \`vigiles eval --check\` fails on an empty match by default (a gate that
|
|
2677
|
+
# verified nothing must not read as a pass). With \`--min=0\` this job is green
|
|
2678
|
+
# until the first \`*.eval.*\` file exists; from then on it is red until that
|
|
2679
|
+
# eval's lock is committed. Raise the floor (\`--min=N\`) once you want
|
|
2680
|
+
# a vanished eval file to fail it. See docs/harness-testing.md.
|
|
2676
2681
|
runs-on: ubuntu-latest
|
|
2677
2682
|
steps:
|
|
2678
2683
|
- uses: actions/checkout@v4
|
|
2679
2684
|
- uses: actions/setup-node@v4
|
|
2680
2685
|
with:
|
|
2681
2686
|
node-version: "20"
|
|
2682
|
-
-
|
|
2683
|
-
|
|
2684
|
-
command: eval-check
|
|
2687
|
+
- run: npm install
|
|
2688
|
+
- run: npx vigiles eval --check --min=0
|
|
2685
2689
|
`
|
|
2686
2690
|
: "";
|
|
2687
2691
|
// No selectable jobs (e.g. `--no-lint --no-test`, or `--test` on a repo with no
|
|
@@ -5195,10 +5199,11 @@ async function promptYesNo(question) {
|
|
|
5195
5199
|
* Resolve the eval LOCK env from the `eval` flags. `--update` records each named
|
|
5196
5200
|
* eval's report to a committed `.vigiles/eval-locks/<name>.lock.json` (run locally
|
|
5197
5201
|
* on your subscription); `--check` (CI) verifies the committed result against the
|
|
5198
|
-
* current inputs WITHOUT a model call. `--check`
|
|
5199
|
-
*
|
|
5200
|
-
*
|
|
5201
|
-
*
|
|
5202
|
+
* current inputs WITHOUT a model call. `--check` with no lock committed at all is
|
|
5203
|
+
* a FAILURE, not a green no-op ({@link refuseEvalCheckWithoutLocks}, #197): a gate
|
|
5204
|
+
* that has verified nothing must not read as one that passed.
|
|
5205
|
+
* `--check`+`--update` together is a usage error (exit 2). The behavior epoch
|
|
5206
|
+
* comes from `.vigilesrc.json` `eval.apiVersion` (committed).
|
|
5202
5207
|
*/
|
|
5203
5208
|
function resolveEvalLockEnv(args) {
|
|
5204
5209
|
const wantCheck = args.includes("--check");
|
|
@@ -5207,13 +5212,6 @@ function resolveEvalLockEnv(args) {
|
|
|
5207
5212
|
console.error("vigiles eval: --check and --update are mutually exclusive (one verifies, one records).");
|
|
5208
5213
|
process.exit(2);
|
|
5209
5214
|
}
|
|
5210
|
-
if (wantCheck &&
|
|
5211
|
-
!(0, eval_lock_js_1.anyLocksCommitted)((0, node_path_1.resolve)(process.cwd(), eval_lock_js_1.DEFAULT_LOCK_DIR))) {
|
|
5212
|
-
console.log("ℹ vigiles eval --check: no committed eval locks found — nothing to verify.\n" +
|
|
5213
|
-
" Run `vigiles eval --update` locally (on your subscription) and commit the\n" +
|
|
5214
|
-
" lock to enable the CI staleness gate.");
|
|
5215
|
-
return "skip";
|
|
5216
|
-
}
|
|
5217
5215
|
const env = {};
|
|
5218
5216
|
if (wantCheck)
|
|
5219
5217
|
env.VIGILES_EVAL_LOCK = "check";
|
|
@@ -5226,6 +5224,22 @@ function resolveEvalLockEnv(args) {
|
|
|
5226
5224
|
}
|
|
5227
5225
|
return env;
|
|
5228
5226
|
}
|
|
5227
|
+
/**
|
|
5228
|
+
* `eval --check` over eval files when no lock is committed anywhere: every one of
|
|
5229
|
+
* them has no recorded result to compare the current inputs against, so the gate
|
|
5230
|
+
* would verify nothing and pass. It used to return green BEFORE discovery, which
|
|
5231
|
+
* also bypassed the empty-match failure. Once any lock exists, a file without one
|
|
5232
|
+
* already fails as "lock missing" in the eval itself. Exits 1; returns otherwise.
|
|
5233
|
+
*/
|
|
5234
|
+
function refuseEvalCheckWithoutLocks(args, evalCount) {
|
|
5235
|
+
if (!args.includes("--check"))
|
|
5236
|
+
return;
|
|
5237
|
+
if ((0, eval_lock_js_1.anyLocksCommitted)((0, node_path_1.resolve)(process.cwd(), eval_lock_js_1.DEFAULT_LOCK_DIR)))
|
|
5238
|
+
return;
|
|
5239
|
+
console.error(`✗ vigiles eval --check: ${String(evalCount)} eval file(s) have no recorded result — no eval lock is committed, so there is nothing to verify the current inputs against.\n` +
|
|
5240
|
+
" Run `vigiles eval --update` locally (on your subscription) and commit the lock(s).");
|
|
5241
|
+
process.exit(1);
|
|
5242
|
+
}
|
|
5229
5243
|
/**
|
|
5230
5244
|
* Turn a completed run into `.vigiles/coverage.json` — the composition root for
|
|
5231
5245
|
* the execution tier of coverage (`coverage-artifact.ts`).
|
|
@@ -5400,6 +5414,31 @@ function gitHead(cwd) {
|
|
|
5400
5414
|
return "";
|
|
5401
5415
|
}
|
|
5402
5416
|
}
|
|
5417
|
+
/**
|
|
5418
|
+
* Why a `vigiles test` / `vigiles eval` run matched no file — the message names
|
|
5419
|
+
* what was looked for, so a stale path, a wrong glob and an empty default
|
|
5420
|
+
* discovery each say so. A DIRECTORY is the one stale-looking target whose cause
|
|
5421
|
+
* we actually know, so it is said instead of the three guesses: `vigiles test .`
|
|
5422
|
+
* used to reach `spawn("node", ["."])` and surface Node's module-resolution stack.
|
|
5423
|
+
*/
|
|
5424
|
+
function emptyMatchMessage(kind, targets, defaultGlob) {
|
|
5425
|
+
const dirs = targets.filter((a) => (0, node_fs_1.lstatSync)(a, { throwIfNoEntry: false })?.isDirectory() === true);
|
|
5426
|
+
if (dirs.length > 0) {
|
|
5427
|
+
return (`✗ vigiles ${kind}: ${dirs.join(", ")} ${dirs.length === 1 ? "is a directory" : "are directories"} — ` +
|
|
5428
|
+
`pass a file, or a glob like "${defaultGlob}".\n` +
|
|
5429
|
+
` A directory is not a script; nothing ran.`);
|
|
5430
|
+
}
|
|
5431
|
+
const expected = ` If an empty match is expected here, say so with --min=0.`;
|
|
5432
|
+
if (targets.length === 0) {
|
|
5433
|
+
return (`✗ vigiles ${kind}: NOTHING matched the default glob ${defaultGlob}\n` +
|
|
5434
|
+
` Nothing ran. An empty run is not a pass: a renamed or moved file looks like this.\n` +
|
|
5435
|
+
expected);
|
|
5436
|
+
}
|
|
5437
|
+
return (`✗ vigiles ${kind}: ${String(targets.length)} target(s) given and NOTHING matched — ` +
|
|
5438
|
+
`${targets.join(", ")}\n` +
|
|
5439
|
+
` Nothing ran. A stale path, a wrong glob, or a moved file all look like this.\n` +
|
|
5440
|
+
expected);
|
|
5441
|
+
}
|
|
5403
5442
|
async function handleRunScripts(kind, args, restArgs, excludes) {
|
|
5404
5443
|
const cwd = process.cwd();
|
|
5405
5444
|
// The ignore file keeps NEW copies of `.vigiles/` local files out of git, but
|
|
@@ -5420,14 +5459,10 @@ async function handleRunScripts(kind, args, restArgs, excludes) {
|
|
|
5420
5459
|
// Harness/eval scripts may be authored in JS or TS (see run-scripts.ts).
|
|
5421
5460
|
const defaultGlob = (0, run_scripts_js_1.scriptGlob)(kind === "test" ? "harness" : "eval");
|
|
5422
5461
|
// The eval LOCK flags (`--check`/`--update`) are resolved BEFORE file discovery
|
|
5423
|
-
// so mutual-exclusion
|
|
5424
|
-
// count. Returns the env to thread to scripts, or `"skip"` to exit green now.
|
|
5462
|
+
// so mutual-exclusion (exit 2) is honored regardless of file count.
|
|
5425
5463
|
let lockEnv = {};
|
|
5426
5464
|
if (kind === "eval") {
|
|
5427
|
-
|
|
5428
|
-
if (r === "skip")
|
|
5429
|
-
return;
|
|
5430
|
-
lockEnv = r;
|
|
5465
|
+
lockEnv = resolveEvalLockEnv(args);
|
|
5431
5466
|
}
|
|
5432
5467
|
// A script under an excluded path is not DISCOVERED (a vendored corpus's own
|
|
5433
5468
|
// harness must not run as ours), but a script you NAME still runs (#192).
|
|
@@ -5449,44 +5484,33 @@ async function handleRunScripts(kind, args, restArgs, excludes) {
|
|
|
5449
5484
|
process.exit(1);
|
|
5450
5485
|
}
|
|
5451
5486
|
if (files.length === 0) {
|
|
5452
|
-
// 🔴
|
|
5453
|
-
//
|
|
5454
|
-
//
|
|
5487
|
+
// 🔴 A RUN THAT MATCHES NO FILE IS A FAILURE, NAMED OR NOT.
|
|
5488
|
+
// An empty match used to be one silent exit 0, and it cost a real repository three
|
|
5489
|
+
// days of green CI verifying zero files: a named step ran
|
|
5455
5490
|
// `vigiles test .claude/pipeline/skills.harness.mjs` after that file had been split
|
|
5456
5491
|
// into one-per-skill, printed "No **/*.harness.* files found" and passed, right next
|
|
5457
|
-
// to a step that was red for the same root cause.
|
|
5458
|
-
//
|
|
5459
|
-
//
|
|
5460
|
-
//
|
|
5461
|
-
// reached its target, and every one of those is a defect. Bare discovery finding
|
|
5462
|
-
// nothing is just an empty repository, which is a legitimate place to stand and must
|
|
5463
|
-
// stay quiet.
|
|
5492
|
+
// to a step that was red for the same root cause. The first repair failed only the
|
|
5493
|
+
// NAMED case and kept bare discovery quiet ("an empty repository is a legitimate
|
|
5494
|
+
// place to stand") — and bare `vigiles test` is the line `init` writes into CI, so
|
|
5495
|
+
// the same green no-op was one rename away from the default setup (#197).
|
|
5464
5496
|
//
|
|
5465
|
-
// This is the default the field settled on: Jest and Vitest FAIL on no tests found
|
|
5466
|
-
// make you opt in with `--passWithNoTests`; pytest exits 5.
|
|
5467
|
-
//
|
|
5468
|
-
// `
|
|
5469
|
-
//
|
|
5470
|
-
// the
|
|
5471
|
-
|
|
5472
|
-
|
|
5473
|
-
|
|
5474
|
-
|
|
5475
|
-
// the generic message above would now be true but unhelpful.
|
|
5476
|
-
const dirs = restArgs.filter((a) => (0, node_fs_1.lstatSync)(a, { throwIfNoEntry: false })?.isDirectory() === true);
|
|
5477
|
-
console.error(dirs.length > 0
|
|
5478
|
-
? `✗ vigiles ${kind}: ${dirs.join(", ")} ${dirs.length === 1 ? "is a directory" : "are directories"} — ` +
|
|
5479
|
-
`pass a file, or a glob like "${defaultGlob}".\n` +
|
|
5480
|
-
` A directory is not a script; nothing ran.`
|
|
5481
|
-
: `✗ vigiles ${kind}: ${String(restArgs.length)} target(s) given and NOTHING matched — ` +
|
|
5482
|
-
`${restArgs.join(", ")}\n` +
|
|
5483
|
-
` Nothing ran. A stale path, a wrong glob, or a moved file all look like this.\n` +
|
|
5484
|
-
` If an empty match is expected here, say so with --min=0.`);
|
|
5497
|
+
// This is the default the field settled on: Jest and Vitest FAIL on no tests found
|
|
5498
|
+
// and make you opt in with `--passWithNoTests`; pytest exits 5. Here there is no
|
|
5499
|
+
// such flag (a person who wants an empty run does not call the command), and
|
|
5500
|
+
// `--min=0` — the existing floor, spelled as zero — is the only way to say an empty
|
|
5501
|
+
// match is expected. `minFlag`, not `minRequired`: 0 is both the DEFAULT and that
|
|
5502
|
+
// explicit opt-out, so the VALUE cannot tell them apart — only the flag's presence
|
|
5503
|
+
// can. (Caught by a control: the first version read `minRequired === 0` and made
|
|
5504
|
+
// `--min=0` do nothing.)
|
|
5505
|
+
if (minFlag === undefined) {
|
|
5506
|
+
console.error(emptyMatchMessage(kind, restArgs, defaultGlob));
|
|
5485
5507
|
process.exit(1);
|
|
5486
5508
|
}
|
|
5487
5509
|
console.log(`No ${defaultGlob} files found.`);
|
|
5488
5510
|
return;
|
|
5489
5511
|
}
|
|
5512
|
+
if (kind === "eval")
|
|
5513
|
+
refuseEvalCheckWithoutLocks(args, files.length);
|
|
5490
5514
|
// Consent gate for a bare `vigiles eval`: it runs the REAL model on your
|
|
5491
5515
|
// subscription, and a no-target run discovered the whole tree — so never fan out
|
|
5492
5516
|
// over an unbounded glob without explicit intent. Mirrors audit's read-vs-run
|
|
@@ -107,8 +107,18 @@ export interface HarnessDriverContext {
|
|
|
107
107
|
readonly cwd: string;
|
|
108
108
|
/** Whether a settings file was written (CC: pass `--settings`). */
|
|
109
109
|
readonly hasSettings: boolean;
|
|
110
|
-
/**
|
|
111
|
-
|
|
110
|
+
/**
|
|
111
|
+
* Which tools EXIST in the session, when the spec lists them (CC: `--tools`);
|
|
112
|
+
* undefined = every tool exists. A driver whose harness has no such concept
|
|
113
|
+
* (Codex) throws on one rather than ignore it.
|
|
114
|
+
*/
|
|
115
|
+
readonly tools?: readonly string[];
|
|
116
|
+
/**
|
|
117
|
+
* Which tools are PRE-APPROVED (CC: `--allowedTools`); undefined = the
|
|
118
|
+
* default four. It never removes a tool. A driver with no such concept
|
|
119
|
+
* ignores it.
|
|
120
|
+
*/
|
|
121
|
+
readonly allowedTools?: readonly string[];
|
|
112
122
|
/** Capture the full event transcript instead of just the final result. */
|
|
113
123
|
readonly transcript: boolean;
|
|
114
124
|
/** Path to a plugin dir to install natively, if any. */
|
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Running past the end of a scripted model is an error.
|
|
3
|
+
*
|
|
4
|
+
* A harness test scripts the turns the agent should take, then asserts on what
|
|
5
|
+
* happened. When the agent asks for more turns than the script has, the mock
|
|
6
|
+
* used to answer with a copy of the LAST turn: a looping agent (a Stop hook
|
|
7
|
+
* that keeps blocking, a retry, a reaction to a tool result) then ran on turns
|
|
8
|
+
* nobody scripted, a final `{ tool }` re-ran its side effect, and the test went
|
|
9
|
+
* green over a run it never described (#340). Pure and shared by both mocks
|
|
10
|
+
* (Anthropic Messages, OpenAI Responses) and the runner that fails the test.
|
|
11
|
+
*/
|
|
12
|
+
import type { ModelRequest } from "./harness-driver.js";
|
|
13
|
+
/** The first request the script could not answer. */
|
|
14
|
+
export interface ScriptOverrun {
|
|
15
|
+
/** How many turns the script had. */
|
|
16
|
+
readonly scripted: number;
|
|
17
|
+
/** 1-based number of the agent request that had no turn left. */
|
|
18
|
+
readonly request: number;
|
|
19
|
+
/** The last message that request carried (what the agent was reacting to). */
|
|
20
|
+
readonly lastMessage: string;
|
|
21
|
+
}
|
|
22
|
+
/** Why a test failed because the agent outran its script. */
|
|
23
|
+
export declare function scriptOverrunMessage(o: ScriptOverrun): string;
|
|
24
|
+
/**
|
|
25
|
+
* The first agent request past the end of a script of `scripted` turns, read
|
|
26
|
+
* off the recorded requests (side-channel calls never consume a turn, so they
|
|
27
|
+
* do not count). Undefined when the agent stayed inside the script.
|
|
28
|
+
*/
|
|
29
|
+
export declare function findScriptOverrun(scripted: number, requests: readonly ModelRequest[]): ScriptOverrun | undefined;
|
|
30
|
+
/** {@link findScriptOverrun} and {@link scriptOverrunMessage} in one: the message, or undefined. */
|
|
31
|
+
export declare function overrunMessageFor(scripted: number, requests: readonly ModelRequest[]): string | undefined;
|
|
32
|
+
//# sourceMappingURL=script-overrun.d.ts.map
|
|
@@ -0,0 +1,42 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.scriptOverrunMessage = scriptOverrunMessage;
|
|
4
|
+
exports.findScriptOverrun = findScriptOverrun;
|
|
5
|
+
exports.overrunMessageFor = overrunMessageFor;
|
|
6
|
+
const MAX_QUOTED = 160;
|
|
7
|
+
const quote = (text) => {
|
|
8
|
+
const flat = text.replace(/\s+/g, " ").trim();
|
|
9
|
+
return flat.length > MAX_QUOTED ? `${flat.slice(0, MAX_QUOTED)}...` : flat;
|
|
10
|
+
};
|
|
11
|
+
/** Why a test failed because the agent outran its script. */
|
|
12
|
+
function scriptOverrunMessage(o) {
|
|
13
|
+
const last = o.lastMessage.trim() === ""
|
|
14
|
+
? ""
|
|
15
|
+
: ` The request carried this last message: "${quote(o.lastMessage)}".`;
|
|
16
|
+
return (`the agent asked for more model turns than the script has: ${String(o.scripted)} turn(s) scripted, ` +
|
|
17
|
+
`and agent request #${String(o.request)} had none left.${last} ` +
|
|
18
|
+
"The mock does not repeat the last turn, because a looping agent would then pass on turns nobody scripted. " +
|
|
19
|
+
"Script every turn the agent should take.");
|
|
20
|
+
}
|
|
21
|
+
/**
|
|
22
|
+
* The first agent request past the end of a script of `scripted` turns, read
|
|
23
|
+
* off the recorded requests (side-channel calls never consume a turn, so they
|
|
24
|
+
* do not count). Undefined when the agent stayed inside the script.
|
|
25
|
+
*/
|
|
26
|
+
function findScriptOverrun(scripted, requests) {
|
|
27
|
+
const agent = requests.filter((r) => r.sideChannel !== true);
|
|
28
|
+
const first = agent[scripted];
|
|
29
|
+
if (first === undefined)
|
|
30
|
+
return undefined;
|
|
31
|
+
return {
|
|
32
|
+
scripted,
|
|
33
|
+
request: scripted + 1,
|
|
34
|
+
lastMessage: first.messages.at(-1)?.text ?? "",
|
|
35
|
+
};
|
|
36
|
+
}
|
|
37
|
+
/** {@link findScriptOverrun} and {@link scriptOverrunMessage} in one: the message, or undefined. */
|
|
38
|
+
function overrunMessageFor(scripted, requests) {
|
|
39
|
+
const overrun = findScriptOverrun(scripted, requests);
|
|
40
|
+
return overrun === undefined ? undefined : scriptOverrunMessage(overrun);
|
|
41
|
+
}
|
|
42
|
+
//# sourceMappingURL=script-overrun.js.map
|
|
@@ -20,7 +20,7 @@
|
|
|
20
20
|
* anywhere in the prose counts, because a name like `test:harness` cannot
|
|
21
21
|
* collide with an English word the way `test` and `audit` do. The only thing the
|
|
22
22
|
* lookarounds buy is that a LONGER script name is never read as a shorter one
|
|
23
|
-
* (`test:
|
|
23
|
+
* (`test:unit-extra` does not document `test:unit`).
|
|
24
24
|
*
|
|
25
25
|
* Scope is the `test:` PREFIX, deliberately. Bare `npm test` is the vitest
|
|
26
26
|
* suite, named in the map as prose; every other script (`build`, `lint`,
|
|
@@ -21,7 +21,7 @@
|
|
|
21
21
|
* anywhere in the prose counts, because a name like `test:harness` cannot
|
|
22
22
|
* collide with an English word the way `test` and `audit` do. The only thing the
|
|
23
23
|
* lookarounds buy is that a LONGER script name is never read as a shorter one
|
|
24
|
-
* (`test:
|
|
24
|
+
* (`test:unit-extra` does not document `test:unit`).
|
|
25
25
|
*
|
|
26
26
|
* Scope is the `test:` PREFIX, deliberately. Bare `npm test` is the vitest
|
|
27
27
|
* suite, named in the map as prose; every other script (`build`, `lint`,
|
package/dist/eval-lock.d.ts
CHANGED
|
@@ -116,12 +116,11 @@ export declare function lockPath(dir: string, name: string): string;
|
|
|
116
116
|
*/
|
|
117
117
|
export declare function readLock(dir: string, name: string): EvalLock | null;
|
|
118
118
|
/**
|
|
119
|
-
* Whether ANY lock has been committed under `dir`.
|
|
120
|
-
* (
|
|
121
|
-
*
|
|
122
|
-
*
|
|
123
|
-
*
|
|
124
|
-
* opt-in-by-committing behavior that keeps a fresh `init` from going red.
|
|
119
|
+
* Whether ANY lock has been committed under `dir`. `eval --check` fails when none
|
|
120
|
+
* is (#197): with no lock there is no recorded result to verify the current inputs
|
|
121
|
+
* against, so a pass would mean nothing was checked. Once the first lock is
|
|
122
|
+
* committed, every named eval is held to having a fresh one (a new unlocked eval
|
|
123
|
+
* then reads as stale). The edit-time nudge also stays silent until then.
|
|
125
124
|
*/
|
|
126
125
|
export declare function anyLocksCommitted(dir: string): boolean;
|
|
127
126
|
/**
|
package/dist/eval-lock.js
CHANGED
|
@@ -130,12 +130,11 @@ function readLock(dir, name) {
|
|
|
130
130
|
return obj;
|
|
131
131
|
}
|
|
132
132
|
/**
|
|
133
|
-
* Whether ANY lock has been committed under `dir`.
|
|
134
|
-
* (
|
|
135
|
-
*
|
|
136
|
-
*
|
|
137
|
-
*
|
|
138
|
-
* opt-in-by-committing behavior that keeps a fresh `init` from going red.
|
|
133
|
+
* Whether ANY lock has been committed under `dir`. `eval --check` fails when none
|
|
134
|
+
* is (#197): with no lock there is no recorded result to verify the current inputs
|
|
135
|
+
* against, so a pass would mean nothing was checked. Once the first lock is
|
|
136
|
+
* committed, every named eval is held to having a fresh one (a new unlocked eval
|
|
137
|
+
* then reads as stale). The edit-time nudge also stays silent until then.
|
|
139
138
|
*/
|
|
140
139
|
function anyLocksCommitted(dir) {
|
|
141
140
|
return countLocks(dir) > 0;
|
package/dist/harness-test.d.ts
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import type { HarnessAdapter } from "./core/adapter.js";
|
|
2
|
-
import type { HarnessTestDriver, ToolCall, HookFire, ParsedRun, ModelTurn } from "./core/harness-driver.js";
|
|
2
|
+
import type { HarnessTestDriver, ToolCall, HookFire, ParsedRun, ModelTurn, ModelRequest } from "./core/harness-driver.js";
|
|
3
3
|
import type { Trace, SubagentTrace } from "./core/eval-driver.js";
|
|
4
4
|
export type { Trace, SubagentTrace } from "./core/eval-driver.js";
|
|
5
5
|
import { parseReplies, reachedModel } from "./adapters/claude-code/replies.js";
|
|
@@ -52,7 +52,38 @@ export interface HarnessTestSpec {
|
|
|
52
52
|
readonly model: readonly ModelTurn[];
|
|
53
53
|
/** The user prompt. Default: "go". */
|
|
54
54
|
readonly prompt?: string;
|
|
55
|
-
/**
|
|
55
|
+
/**
|
|
56
|
+
* Which tools EXIST in the session (`claude --tools`, Claude Code 2.0.31+). A
|
|
57
|
+
* tool left out is not offered at all, so a call to it is "No such tool
|
|
58
|
+
* available". Omitted, every tool exists. This is the field that withholds a
|
|
59
|
+
* tool. Names only: a permission rule such as `Bash(git *)` is refused (put the
|
|
60
|
+
* rule in `allowedTools`). `[]` is refused (an agent with no tools is never
|
|
61
|
+
* served a scripted turn), and a scripted call to a tool the list leaves out
|
|
62
|
+
* throws before the run starts. An MCP tool is not a built-in, so `--tools`
|
|
63
|
+
* does not withhold it. Codex has no equivalent, so a run there refuses it.
|
|
64
|
+
*
|
|
65
|
+
* | field | Claude Code flag | meaning |
|
|
66
|
+
* | -------------- | ----------------- | ---------------------------- |
|
|
67
|
+
* | `tools` | `--tools` | which tools exist |
|
|
68
|
+
* | `allowedTools` | `--allowedTools` | which are pre-approved |
|
|
69
|
+
*/
|
|
70
|
+
readonly tools?: readonly string[];
|
|
71
|
+
/**
|
|
72
|
+
* Which tools are PRE-APPROVED (`claude --allowedTools`), so they do not stop
|
|
73
|
+
* on a permission prompt. Takes permission rules (`Bash(git *)`). Omitted:
|
|
74
|
+
* Read, Edit, Write and Bash are approved. An explicit list REPLACES that
|
|
75
|
+
* default, so `tools: ["Read", "Write"]` with `allowedTools: ["Skill"]` leaves
|
|
76
|
+
* Write unapproved. `[]`: nothing is approved.
|
|
77
|
+
*
|
|
78
|
+
* It never removes a tool, and leaving a tool out is not a fence: in headless
|
|
79
|
+
* mode Claude Code refuses only the calls it would have asked about. For Bash
|
|
80
|
+
* that is a command that creates, changes or removes a file (`touch`, `>`),
|
|
81
|
+
* inside or outside the working directory; a command with no file effect
|
|
82
|
+
* (`ls`, `printf`, `id`) runs unapproved. So a refused `touch` proves the
|
|
83
|
+
* file-change guard is live, not that Bash is withheld. To withhold a tool use
|
|
84
|
+
* `tools`; to forbid its use put a deny rule in the fixture's
|
|
85
|
+
* `settings.permissions.deny`.
|
|
86
|
+
*/
|
|
56
87
|
readonly allowedTools?: readonly string[];
|
|
57
88
|
/**
|
|
58
89
|
* Capture the full event transcript (`--output-format stream-json`) into
|
|
@@ -119,6 +150,41 @@ export declare function parseResultEvent(stdout: string): Record<string, unknown
|
|
|
119
150
|
/** The agent's final answer text from a transcript / result object, or "". */
|
|
120
151
|
export declare function parseOutput(stdout: string): string;
|
|
121
152
|
export declare function parseHooks(stdout: string): HookFire[];
|
|
153
|
+
/**
|
|
154
|
+
* The names `--tools` takes for a `tools` list: trimmed and deduplicated.
|
|
155
|
+
* `--allowedTools` takes permission rules (`Bash(git *)`); `--tools` takes tool
|
|
156
|
+
* NAMES, and a rule there makes the CLI offer no tools at all. Reducing it to
|
|
157
|
+
* `Bash` would offer more than the author wrote, so a rule is refused: it
|
|
158
|
+
* belongs in `allowedTools`. Pure.
|
|
159
|
+
*/
|
|
160
|
+
export declare function toolAvailabilityList(tools: readonly string[]): readonly string[];
|
|
161
|
+
/**
|
|
162
|
+
* A scripted call to a tool an EXPLICIT `tools` list leaves out. The CLI answers
|
|
163
|
+
* it with "No such tool available" and the run goes on, so a test that scripts
|
|
164
|
+
* the call and then asserts on something else passes over a step that never
|
|
165
|
+
* happened. A tool that exists but is not in `allowedTools` is NOT this: the CLI
|
|
166
|
+
* refuses it for permission, which is a result a test can assert on. Returns the
|
|
167
|
+
* message to throw, or undefined. MCP tools are not built-ins, so `--tools` does
|
|
168
|
+
* not withhold them and they are skipped. Pure.
|
|
169
|
+
*/
|
|
170
|
+
export declare function unofferedScriptedTool(model: readonly ModelTurn[], tools: readonly string[] | undefined): string | undefined;
|
|
171
|
+
/**
|
|
172
|
+
* The agent's own tool results, read off the requests it sent back to the model,
|
|
173
|
+
* that say a scripted tool does not exist. The CLI answers an unknown name
|
|
174
|
+
* (`Bsh` for `Bash`, an `mcp__` server that is not configured) with an error
|
|
175
|
+
* RESULT and carries on, exit 0, so a test that asserts "the call errored and
|
|
176
|
+
* left no side effect" passes on a typo. `tools` cannot catch it: the pre-flight
|
|
177
|
+
* skips `mcp__` names and only runs when `tools` is given. Side-channel requests
|
|
178
|
+
* are the CLI's own bookkeeping and are not read. Returns the message to throw,
|
|
179
|
+
* or undefined. Pure.
|
|
180
|
+
*/
|
|
181
|
+
export declare function noSuchToolMessage(model: readonly ModelTurn[], requests: readonly ModelRequest[]): string | undefined;
|
|
182
|
+
/**
|
|
183
|
+
* A run on a `claude` that predates `--tools` exits at argument parsing, and
|
|
184
|
+
* commander's "unknown option" says nothing about WHY vigiles passed it. Returns
|
|
185
|
+
* the message to throw, or undefined when the run did not die that way. Pure.
|
|
186
|
+
*/
|
|
187
|
+
export declare function unsupportedToolsFlag(stderr: string): string | undefined;
|
|
122
188
|
/**
|
|
123
189
|
* The `claude` CLI argv for a harness run (shared by the direct and sandboxed
|
|
124
190
|
* paths). `ANTHROPIC_BASE_URL` is set by the caller's environment / wrapper, not
|
|
@@ -170,6 +236,12 @@ export interface RunHarnessTestOptions {
|
|
|
170
236
|
*/
|
|
171
237
|
readonly adapter?: HarnessAdapter;
|
|
172
238
|
}
|
|
239
|
+
/** Say so on stderr when side-channel calls arrived and no script turn was served. */
|
|
240
|
+
export declare function warnUnconsumed(count: number, sideChannelCount: number): void;
|
|
241
|
+
/**
|
|
242
|
+
* What makes a finished run unsound, most specific first, or undefined. Pure.
|
|
243
|
+
*/
|
|
244
|
+
export declare function runProblem(stderr: string, model: readonly ModelTurn[], modelRequests: readonly ModelRequest[], tools?: readonly string[]): string | undefined;
|
|
173
245
|
/**
|
|
174
246
|
* Run the real agent CLI against a scripted mock model, with the given fixture
|
|
175
247
|
* and settings (hooks). Deterministic — same script, same result. Adapter-driven
|
package/dist/harness-test.js
CHANGED
|
@@ -6,10 +6,16 @@ exports.parseSubagents = parseSubagents;
|
|
|
6
6
|
exports.parseResultEvent = parseResultEvent;
|
|
7
7
|
exports.parseOutput = parseOutput;
|
|
8
8
|
exports.parseHooks = parseHooks;
|
|
9
|
+
exports.toolAvailabilityList = toolAvailabilityList;
|
|
10
|
+
exports.unofferedScriptedTool = unofferedScriptedTool;
|
|
11
|
+
exports.noSuchToolMessage = noSuchToolMessage;
|
|
12
|
+
exports.unsupportedToolsFlag = unsupportedToolsFlag;
|
|
9
13
|
exports.buildClaudeArgs = buildClaudeArgs;
|
|
10
14
|
exports.parseClaudeRun = parseClaudeRun;
|
|
11
15
|
exports.harnessSpawnEnv = harnessSpawnEnv;
|
|
12
16
|
exports.claudeAvailable = claudeAvailable;
|
|
17
|
+
exports.warnUnconsumed = warnUnconsumed;
|
|
18
|
+
exports.runProblem = runProblem;
|
|
13
19
|
exports.runHarnessTest = runHarnessTest;
|
|
14
20
|
exports.runHarnessTestIn = runHarnessTestIn;
|
|
15
21
|
exports.runHarness = runHarness;
|
|
@@ -49,6 +55,7 @@ const node_fs_1 = require("node:fs");
|
|
|
49
55
|
const node_path_1 = require("node:path");
|
|
50
56
|
const adapter_conformance_js_1 = require("./adapter-conformance.js");
|
|
51
57
|
const check_count_js_1 = require("./check-count.js");
|
|
58
|
+
const script_overrun_js_1 = require("./core/script-overrun.js");
|
|
52
59
|
const coverage_probe_js_1 = require("./coverage-probe.js");
|
|
53
60
|
const runtime_js_1 = require("./adapters/claude-code/runtime.js");
|
|
54
61
|
const mock_model_js_1 = require("./mock-model.js");
|
|
@@ -274,13 +281,92 @@ function parseHooks(stdout) {
|
|
|
274
281
|
}
|
|
275
282
|
return hooks;
|
|
276
283
|
}
|
|
284
|
+
/**
|
|
285
|
+
* The first `claude` that has `--tools` (print mode). Found by diffing the
|
|
286
|
+
* published `@anthropic-ai/claude-code` tarballs: 2.0.30 has no such option,
|
|
287
|
+
* 2.0.31 does (the changelog does not list it; its 2.1.0 entry only extends the
|
|
288
|
+
* flag to interactive mode).
|
|
289
|
+
*/
|
|
290
|
+
const TOOLS_FLAG_MIN_CLAUDE = "2.0.31";
|
|
291
|
+
/**
|
|
292
|
+
* The names `--tools` takes for a `tools` list: trimmed and deduplicated.
|
|
293
|
+
* `--allowedTools` takes permission rules (`Bash(git *)`); `--tools` takes tool
|
|
294
|
+
* NAMES, and a rule there makes the CLI offer no tools at all. Reducing it to
|
|
295
|
+
* `Bash` would offer more than the author wrote, so a rule is refused: it
|
|
296
|
+
* belongs in `allowedTools`. Pure.
|
|
297
|
+
*/
|
|
298
|
+
function toolAvailabilityList(tools) {
|
|
299
|
+
const names = tools.map((entry) => entry.trim()).filter((n) => n !== "");
|
|
300
|
+
const rule = names.find((name) => name.includes("("));
|
|
301
|
+
if (rule !== undefined) {
|
|
302
|
+
throw new Error(`tools takes bare tool names, and "${rule}" is a permission rule — put the rule in allowedTools and the name ("${rule.replace(/\(.*$/, "")}") in tools`);
|
|
303
|
+
}
|
|
304
|
+
return [...new Set(names)];
|
|
305
|
+
}
|
|
306
|
+
/**
|
|
307
|
+
* A scripted call to a tool an EXPLICIT `tools` list leaves out. The CLI answers
|
|
308
|
+
* it with "No such tool available" and the run goes on, so a test that scripts
|
|
309
|
+
* the call and then asserts on something else passes over a step that never
|
|
310
|
+
* happened. A tool that exists but is not in `allowedTools` is NOT this: the CLI
|
|
311
|
+
* refuses it for permission, which is a result a test can assert on. Returns the
|
|
312
|
+
* message to throw, or undefined. MCP tools are not built-ins, so `--tools` does
|
|
313
|
+
* not withhold them and they are skipped. Pure.
|
|
314
|
+
*/
|
|
315
|
+
function unofferedScriptedTool(model, tools) {
|
|
316
|
+
if (tools === undefined)
|
|
317
|
+
return undefined;
|
|
318
|
+
const exist = toolAvailabilityList(tools);
|
|
319
|
+
const at = model.findIndex((t) => t.tool !== undefined &&
|
|
320
|
+
!t.tool.startsWith("mcp__") &&
|
|
321
|
+
!exist.includes(t.tool));
|
|
322
|
+
return at < 0
|
|
323
|
+
? undefined
|
|
324
|
+
: `scripted call to "${model[at]?.tool ?? ""}" on turn ${String(at + 1)}, but this run's tools are only: ${exist.join(", ")} — add it to tools`;
|
|
325
|
+
}
|
|
326
|
+
/**
|
|
327
|
+
* The agent's own tool results, read off the requests it sent back to the model,
|
|
328
|
+
* that say a scripted tool does not exist. The CLI answers an unknown name
|
|
329
|
+
* (`Bsh` for `Bash`, an `mcp__` server that is not configured) with an error
|
|
330
|
+
* RESULT and carries on, exit 0, so a test that asserts "the call errored and
|
|
331
|
+
* left no side effect" passes on a typo. `tools` cannot catch it: the pre-flight
|
|
332
|
+
* skips `mcp__` names and only runs when `tools` is given. Side-channel requests
|
|
333
|
+
* are the CLI's own bookkeeping and are not read. Returns the message to throw,
|
|
334
|
+
* or undefined. Pure.
|
|
335
|
+
*/
|
|
336
|
+
function noSuchToolMessage(model, requests) {
|
|
337
|
+
// The CLI's own wording, inside its error wrapper, for a name the script
|
|
338
|
+
// calls: the agent calls tools only from the script, so the same words in the
|
|
339
|
+
// prompt or in a hook's injected text are not a result of this run.
|
|
340
|
+
const pattern = /<tool_use_error>Error: No such tool available: ([^\s<]+)<\/tool_use_error>/g;
|
|
341
|
+
const scripted = new Set(model.flatMap((t) => (t.tool === undefined ? [] : [t.tool])));
|
|
342
|
+
const hit = requests
|
|
343
|
+
.filter((r) => r.sideChannel !== true)
|
|
344
|
+
.flatMap((r) => r.messages)
|
|
345
|
+
.flatMap((m) => [...m.text.matchAll(pattern)].map((x) => x[1]))
|
|
346
|
+
.find((name) => name !== undefined && scripted.has(name));
|
|
347
|
+
if (hit === undefined)
|
|
348
|
+
return undefined;
|
|
349
|
+
const at = model.findIndex((t) => t.tool === hit);
|
|
350
|
+
const turn = at < 0 ? "" : ` on turn ${String(at + 1)}`;
|
|
351
|
+
return `scripted call to "${hit}"${turn}: the CLI answered "No such tool available: ${hit}", so the call never ran and the test would carry on over a step that did not happen. Check the name for a typo; an mcp__ tool also needs its server configured in the fixture.`;
|
|
352
|
+
}
|
|
353
|
+
/**
|
|
354
|
+
* A run on a `claude` that predates `--tools` exits at argument parsing, and
|
|
355
|
+
* commander's "unknown option" says nothing about WHY vigiles passed it. Returns
|
|
356
|
+
* the message to throw, or undefined when the run did not die that way. Pure.
|
|
357
|
+
*/
|
|
358
|
+
function unsupportedToolsFlag(stderr) {
|
|
359
|
+
return /unknown option '--tools'/.test(stderr)
|
|
360
|
+
? `this \`claude\` does not know \`--tools\`, which runHarnessTest passes for the \`tools\` field (which tools exist). Update Claude Code to ${TOOLS_FLAG_MIN_CLAUDE} or newer. Running without it would hand the agent every tool while the test reads as if it were restricted.`
|
|
361
|
+
: undefined;
|
|
362
|
+
}
|
|
277
363
|
/**
|
|
278
364
|
* The `claude` CLI argv for a harness run (shared by the direct and sandboxed
|
|
279
365
|
* paths). `ANTHROPIC_BASE_URL` is set by the caller's environment / wrapper, not
|
|
280
366
|
* here. Pure, so the arg shape is unit-tested.
|
|
281
367
|
*/
|
|
282
368
|
function buildClaudeArgs(spec, hasSettings) {
|
|
283
|
-
const
|
|
369
|
+
const approved = spec.allowedTools ?? ["Read", "Edit", "Write", "Bash"];
|
|
284
370
|
return [
|
|
285
371
|
"-p",
|
|
286
372
|
spec.prompt ?? "go",
|
|
@@ -293,9 +379,15 @@ function buildClaudeArgs(spec, hasSettings) {
|
|
|
293
379
|
? ["--plugin-dir", (0, node_path_1.resolve)(spec.pluginDir)]
|
|
294
380
|
: []),
|
|
295
381
|
...(hasSettings ? ["--settings", "settings.json"] : []),
|
|
296
|
-
//
|
|
297
|
-
//
|
|
298
|
-
...(tools
|
|
382
|
+
// `tools` = which tools EXIST (`--tools`, the availability list). Omitted:
|
|
383
|
+
// nothing is withheld.
|
|
384
|
+
...(spec.tools === undefined
|
|
385
|
+
? []
|
|
386
|
+
: ["--tools", toolAvailabilityList(spec.tools).join(",")]),
|
|
387
|
+
// `allowedTools` = which are PRE-APPROVED (`--allowedTools`); it never
|
|
388
|
+
// restricts. An empty list approves nothing, so the flag is left out (a bare
|
|
389
|
+
// `--allowedTools` exits before any model turn).
|
|
390
|
+
...(approved.length === 0 ? [] : ["--allowedTools", ...approved]),
|
|
299
391
|
];
|
|
300
392
|
}
|
|
301
393
|
/** Build the `claude` argv from the driver context — wraps `buildClaudeArgs`. */
|
|
@@ -305,7 +397,8 @@ function buildClaudeArgsFromCtx(ctx) {
|
|
|
305
397
|
prompt: ctx.prompt,
|
|
306
398
|
transcript: ctx.transcript,
|
|
307
399
|
pluginDir: ctx.pluginDir,
|
|
308
|
-
|
|
400
|
+
tools: ctx.tools,
|
|
401
|
+
allowedTools: ctx.allowedTools,
|
|
309
402
|
}, ctx.hasSettings);
|
|
310
403
|
}
|
|
311
404
|
/** Parse the `claude` stdout/stream into the unified trace fields. */
|
|
@@ -415,6 +508,52 @@ exports.claudeCodeDriver = {
|
|
|
415
508
|
parseRun: parseClaudeRun,
|
|
416
509
|
available: claudeAvailable,
|
|
417
510
|
};
|
|
511
|
+
/** Say so on stderr when side-channel calls arrived and no script turn was served. */
|
|
512
|
+
function warnUnconsumed(count, sideChannelCount) {
|
|
513
|
+
const unconsumed = (0, mock_model_js_1.scriptUnconsumedWarning)(count, sideChannelCount);
|
|
514
|
+
if (unconsumed !== undefined)
|
|
515
|
+
console.error(unconsumed);
|
|
516
|
+
}
|
|
517
|
+
/**
|
|
518
|
+
* `tools` made only of names the CLI does not know leaves the agent with none:
|
|
519
|
+
* every request then declares no tools, the mock files it as side-channel
|
|
520
|
+
* traffic, and the script is never served, so the run decides nothing. Requests
|
|
521
|
+
* that all came in that way, under an explicit `tools`, mean exactly that. No
|
|
522
|
+
* request at all (a hook stopped the prompt) is a different run. Pure.
|
|
523
|
+
*/
|
|
524
|
+
function toolsResolvedToNothing(tools, requests) {
|
|
525
|
+
if (tools === undefined || requests.length === 0)
|
|
526
|
+
return undefined;
|
|
527
|
+
if (!requests.every((r) => r.sideChannel === true))
|
|
528
|
+
return undefined;
|
|
529
|
+
return `tools ${JSON.stringify(tools)} gave the agent no tools: no name in it is a tool this CLI knows, so no request offered a tool and the script was never served. Check the names for typos.`;
|
|
530
|
+
}
|
|
531
|
+
/**
|
|
532
|
+
* What makes a finished run unsound, most specific first, or undefined. Pure.
|
|
533
|
+
*/
|
|
534
|
+
function runProblem(stderr, model, modelRequests, tools) {
|
|
535
|
+
return (unsupportedToolsFlag(stderr) ??
|
|
536
|
+
toolsResolvedToNothing(tools, modelRequests) ??
|
|
537
|
+
// A typo'd last turn makes the CLI ask once more, so the overrun follows
|
|
538
|
+
// from it: report the cause, not its effect.
|
|
539
|
+
noSuchToolMessage(model, modelRequests) ??
|
|
540
|
+
(0, script_overrun_js_1.overrunMessageFor)(model.length, modelRequests));
|
|
541
|
+
}
|
|
542
|
+
/**
|
|
543
|
+
* A run that proves nothing about the script fails the test instead of coming
|
|
544
|
+
* back as a result: the `claude` on PATH predates `--tools` (needed for `tools`), or the agent
|
|
545
|
+
* asked for a model turn the script did not have and the mock answered with an
|
|
546
|
+
* error rather than an invented turn (#340), or a scripted call named a tool the
|
|
547
|
+
* CLI does not have. Removes the run's directory, since
|
|
548
|
+
* no result is handed back to clean it up.
|
|
549
|
+
*/
|
|
550
|
+
function assertRunSound(cwd, stderr, model, modelRequests, tools) {
|
|
551
|
+
const problem = runProblem(stderr, model, modelRequests, tools);
|
|
552
|
+
if (problem === undefined)
|
|
553
|
+
return;
|
|
554
|
+
(0, node_fs_1.rmSync)(cwd, { recursive: true, force: true });
|
|
555
|
+
throw new Error(problem);
|
|
556
|
+
}
|
|
418
557
|
/* v8 ignore start -- spawns the real agent CLI + filesystem; exercised by the
|
|
419
558
|
claude-backed + gated codex suites, excluded from the deterministic coverage
|
|
420
559
|
gate (the parse helpers above carry the testable logic). */
|
|
@@ -457,8 +596,36 @@ function makeResult(cwd, out, parsed, turns, modelRequests) {
|
|
|
457
596
|
* only — requesting confinement for another harness throws.
|
|
458
597
|
*/
|
|
459
598
|
async function runHarnessTest(spec, opts = {}) {
|
|
599
|
+
refuseUnrunnableSpec(spec);
|
|
460
600
|
return runHarnessTestIn(spec, opts, "inherit");
|
|
461
601
|
}
|
|
602
|
+
/** The driver's view of one run of `spec`. Pure. */
|
|
603
|
+
function driverContext(spec, cwd, hasSettings, mockArgs) {
|
|
604
|
+
return {
|
|
605
|
+
prompt: spec.prompt ?? "go",
|
|
606
|
+
cwd,
|
|
607
|
+
hasSettings,
|
|
608
|
+
tools: spec.tools,
|
|
609
|
+
allowedTools: spec.allowedTools,
|
|
610
|
+
transcript: spec.transcript ?? false,
|
|
611
|
+
pluginDir: spec.pluginDir,
|
|
612
|
+
mockArgs,
|
|
613
|
+
};
|
|
614
|
+
}
|
|
615
|
+
/**
|
|
616
|
+
* The scripted model tells an agent turn from the CLI's own bookkeeping calls by
|
|
617
|
+
* the tools the request declares (`isMainLoopRequest`). An agent with none is
|
|
618
|
+
* never served a script turn, and the run would decide on nothing.
|
|
619
|
+
*/
|
|
620
|
+
function refuseUnrunnableSpec(spec) {
|
|
621
|
+
if (spec.tools !== undefined &&
|
|
622
|
+
toolAvailabilityList(spec.tools).length === 0) {
|
|
623
|
+
throw new Error("tools: [] leaves the agent with no tools (a list of blank names counts as empty), and the scripted model only serves a turn to a request that declares tools — no script turn would be consumed. Name the tools the agent has.");
|
|
624
|
+
}
|
|
625
|
+
const unoffered = unofferedScriptedTool(spec.model, spec.tools);
|
|
626
|
+
if (unoffered !== undefined)
|
|
627
|
+
throw new Error(unoffered);
|
|
628
|
+
}
|
|
462
629
|
/**
|
|
463
630
|
* {@link runHarnessTest} with the run's HOME chosen by the caller — see
|
|
464
631
|
* {@link HarnessRunHome}. Not on a public entry point: the one caller that needs
|
|
@@ -491,17 +658,19 @@ async function runHarnessTestIn(spec, opts, home) {
|
|
|
491
658
|
const { files, settings, check } = fixtureFor(spec, opts.adapter);
|
|
492
659
|
const cwd = (0, tmp_root_js_1.makeTmpDir)("harness");
|
|
493
660
|
writeFixture(cwd, files, settings);
|
|
494
|
-
const tools = spec.allowedTools ?? ["Read", "Edit", "Write", "Bash"];
|
|
495
661
|
const timeoutMs = spec.timeoutMs ?? 60000;
|
|
496
|
-
|
|
497
|
-
|
|
498
|
-
|
|
499
|
-
|
|
500
|
-
|
|
501
|
-
|
|
502
|
-
|
|
503
|
-
|
|
504
|
-
|
|
662
|
+
// A driver refuses a spec it cannot run (Codex and `tools`) inside buildArgs.
|
|
663
|
+
// That refusal must not leave the run's temp dir behind; the mock, if started,
|
|
664
|
+
// is closed by the caller's finally.
|
|
665
|
+
const buildArgs = (mockArgs) => {
|
|
666
|
+
try {
|
|
667
|
+
return driver.buildArgs(driverContext(spec, cwd, settings !== undefined, mockArgs));
|
|
668
|
+
}
|
|
669
|
+
catch (error) {
|
|
670
|
+
(0, node_fs_1.rmSync)(cwd, { recursive: true, force: true });
|
|
671
|
+
throw error;
|
|
672
|
+
}
|
|
673
|
+
};
|
|
505
674
|
// Confined path (Claude Code only): the mock is co-launched in the netns, so
|
|
506
675
|
// the agent reaches it over the loopback URL the sandbox sets — Claude Code is
|
|
507
676
|
// env-only (empty mockArgs), so the argv needs no mock-wiring flags.
|
|
@@ -519,9 +688,8 @@ async function runHarnessTestIn(spec, opts, home) {
|
|
|
519
688
|
// the direct one stays right. Same recovery feeds the unconsumed-script
|
|
520
689
|
// warning, which the sandbox path could not emit at all before.
|
|
521
690
|
const { count, sideChannelCount } = (0, mock_model_js_1.splitRequestCounts)(out.requests);
|
|
522
|
-
|
|
523
|
-
|
|
524
|
-
console.error(unconsumed);
|
|
691
|
+
warnUnconsumed(count, sideChannelCount);
|
|
692
|
+
assertRunSound(cwd, "", spec.model, out.requests, spec.tools);
|
|
525
693
|
return check(makeResult(cwd, out, parseClaudeRun(out.stdout), count, out.requests));
|
|
526
694
|
}
|
|
527
695
|
// Direct path: mock runs in this process; the agent reaches it over localhost.
|
|
@@ -539,9 +707,8 @@ async function runHarnessTestIn(spec, opts, home) {
|
|
|
539
707
|
// looks empty. `scriptUnconsumedWarning` names the one shape that produces
|
|
540
708
|
// it (every request arriving without tool declarations, so nothing looked
|
|
541
709
|
// like an agent turn) instead of leaving it to be rediscovered.
|
|
542
|
-
|
|
543
|
-
|
|
544
|
-
console.error(unconsumed);
|
|
710
|
+
warnUnconsumed(mock.count, mock.sideChannelCount ?? 0);
|
|
711
|
+
assertRunSound(cwd, out.stderr, spec.model, mock.requests, spec.tools);
|
|
545
712
|
return check(makeResult(cwd, out, driver.parseRun(out.stdout), mock.count, [
|
|
546
713
|
...mock.requests,
|
|
547
714
|
]));
|
package/dist/mock-model.d.ts
CHANGED
|
@@ -78,7 +78,10 @@ export declare function extractRequest(body: {
|
|
|
78
78
|
}): ModelRequest;
|
|
79
79
|
/**
|
|
80
80
|
* Start the scripted mock on a free port. Each MAIN-LOOP `/v1/messages` POST
|
|
81
|
-
* consumes the next turn
|
|
81
|
+
* consumes the next turn; one past the end of the script is answered with a
|
|
82
|
+
* non-retryable API error, never a copy of the last turn (#340 — a looping
|
|
83
|
+
* agent would pass on turns nobody scripted), and `runHarnessTest` then fails
|
|
84
|
+
* the test from the recorded requests ({@link findScriptOverrun});
|
|
82
85
|
* a SIDE-CHANNEL POST — the CLI's own bookkeeping calls, see
|
|
83
86
|
* {@link isMainLoopRequest} — is answered from outside the script and consumes
|
|
84
87
|
* nothing. Resolves to a handle with the base `url` and a `close()`.
|
package/dist/mock-model.js
CHANGED
|
@@ -28,6 +28,7 @@ exports.startMock = startMock;
|
|
|
28
28
|
* and echoing the requested model.
|
|
29
29
|
*/
|
|
30
30
|
const node_http_1 = __importDefault(require("node:http"));
|
|
31
|
+
const script_overrun_js_1 = require("./core/script-overrun.js");
|
|
31
32
|
/** Build a scripted model from an ordered list of turns. */
|
|
32
33
|
function scriptModel(turns) {
|
|
33
34
|
return [...turns];
|
|
@@ -388,9 +389,41 @@ function extractRequest(body) {
|
|
|
388
389
|
: [];
|
|
389
390
|
return { system: flattenContent(body.system), messages };
|
|
390
391
|
}
|
|
392
|
+
/**
|
|
393
|
+
* The 400 body for an agent request past the end of the script: an API error,
|
|
394
|
+
* because the CLI retries 429/5xx and an invented turn would let the agent act on
|
|
395
|
+
* a step nobody scripted (#340). Pure.
|
|
396
|
+
*/
|
|
397
|
+
function pastScriptBody(scripted, request) {
|
|
398
|
+
const message = (0, script_overrun_js_1.scriptOverrunMessage)({
|
|
399
|
+
scripted,
|
|
400
|
+
request: scripted + 1,
|
|
401
|
+
lastMessage: request.messages.at(-1)?.text ?? "",
|
|
402
|
+
});
|
|
403
|
+
return JSON.stringify({
|
|
404
|
+
type: "error",
|
|
405
|
+
error: { type: "invalid_request_error", message },
|
|
406
|
+
});
|
|
407
|
+
}
|
|
408
|
+
/** Does a message's content carry a tool result (the agent reacting to a tool)? */
|
|
409
|
+
function endsWithToolResult(content) {
|
|
410
|
+
return JSON.stringify(content ?? "").includes('"tool_result"');
|
|
411
|
+
}
|
|
412
|
+
/** The JSON body of a request, `{}` for the HEAD / health checks that have none. */
|
|
413
|
+
function parseReqBody(body) {
|
|
414
|
+
try {
|
|
415
|
+
return JSON.parse(body);
|
|
416
|
+
}
|
|
417
|
+
catch {
|
|
418
|
+
return {};
|
|
419
|
+
}
|
|
420
|
+
}
|
|
391
421
|
/**
|
|
392
422
|
* Start the scripted mock on a free port. Each MAIN-LOOP `/v1/messages` POST
|
|
393
|
-
* consumes the next turn
|
|
423
|
+
* consumes the next turn; one past the end of the script is answered with a
|
|
424
|
+
* non-retryable API error, never a copy of the last turn (#340 — a looping
|
|
425
|
+
* agent would pass on turns nobody scripted), and `runHarnessTest` then fails
|
|
426
|
+
* the test from the recorded requests ({@link findScriptOverrun});
|
|
394
427
|
* a SIDE-CHANNEL POST — the CLI's own bookkeeping calls, see
|
|
395
428
|
* {@link isMainLoopRequest} — is answered from outside the script and consumes
|
|
396
429
|
* nothing. Resolves to a handle with the base `url` and a `close()`.
|
|
@@ -406,13 +439,7 @@ function startMock(script, opts = {}) {
|
|
|
406
439
|
const url = req.url ?? "";
|
|
407
440
|
const isCount = url.includes("count_tokens");
|
|
408
441
|
const isMessages = url.includes("/v1/messages") && !isCount;
|
|
409
|
-
|
|
410
|
-
try {
|
|
411
|
-
reqBody = JSON.parse(body);
|
|
412
|
-
}
|
|
413
|
-
catch {
|
|
414
|
-
/* HEAD / health checks have no JSON body */
|
|
415
|
-
}
|
|
442
|
+
const reqBody = parseReqBody(body);
|
|
416
443
|
if (req.method === "HEAD" || (!isMessages && !isCount)) {
|
|
417
444
|
res.writeHead(200, { "content-type": "application/json" });
|
|
418
445
|
res.end("{}");
|
|
@@ -445,13 +472,17 @@ function startMock(script, opts = {}) {
|
|
|
445
472
|
jsonTurn(res, reply, model);
|
|
446
473
|
return;
|
|
447
474
|
}
|
|
448
|
-
const
|
|
475
|
+
const turn = script[i];
|
|
476
|
+
if (turn === undefined) {
|
|
477
|
+
res.writeHead(400, { "content-type": "application/json" });
|
|
478
|
+
res.end(pastScriptBody(script.length, request));
|
|
479
|
+
return;
|
|
480
|
+
}
|
|
449
481
|
opts.onTurn?.({
|
|
450
482
|
n: i,
|
|
451
483
|
stream: reqBody.stream === true,
|
|
452
|
-
hasToolResult:
|
|
484
|
+
hasToolResult: endsWithToolResult(reqBody.messages?.at(-1)?.content),
|
|
453
485
|
});
|
|
454
|
-
const turn = script[Math.min(i, script.length - 1)] ?? { text: "" };
|
|
455
486
|
i++;
|
|
456
487
|
if (reqBody.stream === true)
|
|
457
488
|
streamTurn(res, turn, model);
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "vigiles",
|
|
3
|
-
"version": "
|
|
3
|
+
"version": "36.0.0",
|
|
4
4
|
"description": "Audit, test and measure the harness your AI agent runs on — grade your CLAUDE.md / AGENTS.md, skills, subagents and hooks, run them against a scripted model, and measure whether they actually fire.",
|
|
5
5
|
"keywords": [
|
|
6
6
|
"claude-code",
|
|
@@ -83,7 +83,6 @@
|
|
|
83
83
|
"test:unit": "npm run build && vitest run --project unit",
|
|
84
84
|
"test:integration": "npm run build && vitest run --project integration",
|
|
85
85
|
"test:e2e": "npm run build && vitest run --project e2e",
|
|
86
|
-
"test:cli-e2e": "bash test/e2e/run.sh",
|
|
87
86
|
"test:harness": "npm run build && node dist/cli.js test",
|
|
88
87
|
"test:eval": "npm run build && node dist/cli.js eval --all",
|
|
89
88
|
"test:vitest": "npm run build && vitest run --project runners",
|