scenescout 3.14.1 → 3.16.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +94 -0
- package/README.md +90 -18
- package/dist/browsers.js +28 -0
- package/dist/check-run.js +211 -24
- package/dist/ci-run.js +374 -62
- package/dist/cli.js +221 -47
- package/dist/commands.js +42 -4
- package/dist/engine/baseline.js +377 -0
- package/dist/engine/brief.js +16 -7
- package/dist/engine/browser.js +1660 -386
- package/dist/engine/calibration.js +61 -30
- package/dist/engine/capture.js +164 -0
- package/dist/engine/check.js +258 -68
- package/dist/engine/ci-lanes.js +215 -0
- package/dist/engine/ci.js +233 -18
- package/dist/engine/claims.js +234 -12
- package/dist/engine/collector.js +851 -78
- package/dist/engine/crawl.js +49 -0
- package/dist/engine/dedup.js +329 -9
- package/dist/engine/design.js +355 -43
- package/dist/engine/export.js +877 -0
- package/dist/engine/fingerprint.js +92 -4
- package/dist/engine/flow.js +18 -6
- package/dist/engine/forms.js +198 -18
- package/dist/engine/journey.js +29 -1
- package/dist/engine/lane.js +13 -3
- package/dist/engine/launch.js +45 -6
- package/dist/engine/limits.js +7 -0
- package/dist/engine/live-page.js +49 -2
- package/dist/engine/live.js +4 -1
- package/dist/engine/memory.js +668 -70
- package/dist/engine/open.js +118 -0
- package/dist/engine/oracles.js +167 -2
- package/dist/engine/plain.js +268 -0
- package/dist/engine/png.js +127 -0
- package/dist/engine/policy.js +379 -9
- package/dist/engine/probes.js +3 -2
- package/dist/engine/profiles.js +45 -9
- package/dist/engine/project-folder.js +191 -0
- package/dist/engine/provider.js +4 -3
- package/dist/engine/refresh.js +338 -25
- package/dist/engine/replay.js +63 -10
- package/dist/engine/report.js +248 -40
- package/dist/engine/request.js +317 -23
- package/dist/engine/sarif.js +120 -0
- package/dist/engine/scripted-login.js +342 -62
- package/dist/engine/settle.js +67 -0
- package/dist/engine/signed-in.js +256 -0
- package/dist/engine/tickets.js +671 -0
- package/dist/engine/unload.js +3 -2
- package/dist/export-run.js +633 -0
- package/dist/first-run.js +581 -0
- package/dist/installer.js +267 -3
- package/dist/intake.js +104 -0
- package/dist/login-run.js +386 -71
- package/dist/mcp-server.js +637 -68
- package/dist/playbook.js +5 -0
- package/package.json +6 -3
- package/skills/scenescout/SKILL.md +51 -18
package/dist/ci-run.js
CHANGED
|
@@ -1,8 +1,9 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* Runs `scenescout ci`: starts the SceneScout MCP server as a child process,
|
|
3
3
|
* attaches it to the app, lets a model drive the scout_* tools until it is
|
|
4
|
-
* done or a cap ends the run
|
|
5
|
-
*
|
|
4
|
+
* done or a cap ends the run (with --lanes, several conversations with the
|
|
5
|
+
* model at once, one per part of the app, each in its own session), then has
|
|
6
|
+
* the report written and writes the CI files. The rules (options, caps, redaction, which tools, the files) are in
|
|
6
7
|
* engine/ci.ts and the message shapes in engine/provider.ts; this file only
|
|
7
8
|
* moves bytes between the model, the server and the disk.
|
|
8
9
|
*
|
|
@@ -17,10 +18,14 @@ import path from "node:path";
|
|
|
17
18
|
import { fileURLToPath } from "node:url";
|
|
18
19
|
import { Client } from "@modelcontextprotocol/sdk/client/index.js";
|
|
19
20
|
import { StdioClientTransport } from "@modelcontextprotocol/sdk/client/stdio.js";
|
|
21
|
+
import { CreateMessageRequestSchema, ErrorCode, McpError } from "@modelcontextprotocol/sdk/types.js";
|
|
22
|
+
import { DEDUP_JUDGE_CAPABILITY, durationText, JUDGE_CALL_MS, JUDGE_MAX_OUTPUT_TOKENS, JUDGE_SYSTEM, JUDGE_TOOL, judgeKickoffOf, samplingResultOf, } from "./engine/dedup.js";
|
|
20
23
|
import { CAPTURE_MARGIN, parseCaptureResult, rebaseUrl, SHOT_FILES, SHOTS_DIRNAME } from "./engine/capture.js";
|
|
21
|
-
import { addUsage, wallLeftMs, guardToolArgs,
|
|
24
|
+
import { addUsage, attachFailure, budgetSpend, wallLeftMs, guardToolArgs, CAPTURE_TOOLS, childEnv, ciCaptureKickoff, ciCaptureSystemPrompt, CI_DIRNAME, ciExitCode, ciKickoff, ciSarif, ciSummaryJson, ciSummaryMarkdown, ciSystemPrompt, ciToolArgs, ciTools, describeStop, findingsThisRun, newBudget, NO_USAGE, readFindings, redactKeys, settleTurn, takeTurn, toolResultText, usageLine, } from "./engine/ci.js";
|
|
25
|
+
import { ciLaneKickoff, ciLaneSystemPrompt, crawlFoundNothing, crawlNotes, LANE_TOOLS, mergeLaneStops, PLAN_CRAWL_ROUNDS, planCiLanes, PLANNER_SESSION, } from "./engine/ci-lanes.js";
|
|
22
26
|
import { resolveTimeLimits } from "./engine/limits.js";
|
|
23
27
|
import { MEMORY_DIRNAME, writeSelfIgnore } from "./engine/memory.js";
|
|
28
|
+
import { sarifFilesFor } from "./engine/sarif.js";
|
|
24
29
|
import { decodePng, diffImages, encodePng } from "./engine/png.js";
|
|
25
30
|
import { AnthropicConversation, backoffMs, errorMessage, MalformedReply, OpenAIConversation, retryable, retryAfterMs, } from "./engine/provider.js";
|
|
26
31
|
import { loadPlaybook } from "./playbook.js";
|
|
@@ -135,22 +140,88 @@ export class HttpModelClient {
|
|
|
135
140
|
this.conversation.addResults(results);
|
|
136
141
|
}
|
|
137
142
|
}
|
|
143
|
+
/** One conversation with the provider `resolved` names. */
|
|
144
|
+
function conversationFor(resolved, o, kickoff) {
|
|
145
|
+
return resolved.provider === "anthropic" ? new AnthropicConversation(o, kickoff) : new OpenAIConversation(o, kickoff);
|
|
146
|
+
}
|
|
138
147
|
export function httpClient(resolved, key, system, tools, kickoff) {
|
|
139
148
|
const o = { baseUrl: resolved.baseUrl, model: resolved.model, effort: resolved.effort, system, tools };
|
|
140
|
-
|
|
141
|
-
return new HttpModelClient(conversation, key);
|
|
149
|
+
return new HttpModelClient(conversationFor(resolved, o, kickoff), key);
|
|
142
150
|
}
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
151
|
+
/**
|
|
152
|
+
* The dedup judge's model call over HTTP: one short conversation per pair, at
|
|
153
|
+
* `resolved`'s effort, with its output capped and every attempt and retry
|
|
154
|
+
* inside `callMs`. Used by `scenescout ci` (the run's provider and model at
|
|
155
|
+
* the judge's effort) and by the MCP server when a key is in its environment.
|
|
156
|
+
*/
|
|
157
|
+
export function httpJudgeAsk(resolved, key, deps = {}) {
|
|
158
|
+
return async (system, tools, kickoff, limitMs) => {
|
|
159
|
+
const ms = Math.min(deps.callMs ?? JUDGE_CALL_MS, limitMs ?? Infinity);
|
|
160
|
+
const o = { baseUrl: resolved.baseUrl, model: resolved.model, effort: resolved.effort, system, tools, maxOutputTokens: JUDGE_MAX_OUTPUT_TOKENS };
|
|
161
|
+
try {
|
|
162
|
+
return await new HttpModelClient(conversationFor(resolved, o, kickoff), key, { fetch: deps.fetch, sleep: deps.sleep, attemptMs: ms }).next(ms);
|
|
163
|
+
}
|
|
164
|
+
catch (err) {
|
|
165
|
+
// The "time cap" here is the judge call's own limit, not the run's: say which.
|
|
166
|
+
if (err instanceof OutOfTime)
|
|
167
|
+
throw new Error(err.message.replace(/^the time cap was reached/, `no answer within ${durationText(ms)}`));
|
|
168
|
+
throw err;
|
|
169
|
+
}
|
|
170
|
+
};
|
|
171
|
+
}
|
|
172
|
+
/**
|
|
173
|
+
* The client's answer to the server's dedup judge (a sampling request): the
|
|
174
|
+
* question goes to the run's model through `ask`, under this side's own
|
|
175
|
+
* JUDGE_SYSTEM and JUDGE_TOOL, and the answer goes back as the tool call. Its
|
|
176
|
+
* tokens are added to the run's usage, so the caps count them, and no call
|
|
177
|
+
* runs past the time cap: one asked after it is refused, and one asked before
|
|
178
|
+
* it gets only the time left. A request that is not the judge's question is
|
|
179
|
+
* refused; a failed call is reported to the server, whose judge then leaves
|
|
180
|
+
* the rule's decision and logs it.
|
|
181
|
+
*/
|
|
182
|
+
export function judgeHandler(o) {
|
|
183
|
+
const now = o.now ?? Date.now;
|
|
184
|
+
return async (request) => {
|
|
185
|
+
o.calls.calls += 1;
|
|
186
|
+
const question = judgeKickoffOf(request.params);
|
|
187
|
+
if (!question.ok) {
|
|
188
|
+
o.calls.failed += 1;
|
|
189
|
+
throw new McpError(ErrorCode.InvalidRequest, `this client answers only the dedup judge's question: ${question.error}`);
|
|
190
|
+
}
|
|
191
|
+
const left = wallLeftMs(o.spend, o.caps, now());
|
|
192
|
+
if (left <= 0) {
|
|
193
|
+
o.calls.failed += 1;
|
|
194
|
+
throw new McpError(ErrorCode.InternalError, "the run's time cap was reached, so the judge was not asked");
|
|
195
|
+
}
|
|
196
|
+
const started = now();
|
|
197
|
+
try {
|
|
198
|
+
const turn = await o.ask(JUDGE_SYSTEM, [JUDGE_TOOL], question.kickoff, Math.min(JUDGE_CALL_MS, left));
|
|
199
|
+
o.spend.usage = addUsage(o.spend.usage, turn.usage);
|
|
200
|
+
o.calls.usage = addUsage(o.calls.usage, turn.usage);
|
|
201
|
+
return samplingResultOf(turn, o.model);
|
|
202
|
+
}
|
|
203
|
+
catch (err) {
|
|
204
|
+
o.calls.failed += 1;
|
|
205
|
+
throw new McpError(ErrorCode.InternalError, redactKeys(err instanceof Error ? err.message : String(err), o.secrets ?? []));
|
|
206
|
+
}
|
|
207
|
+
finally {
|
|
208
|
+
o.calls.ms += now() - started;
|
|
209
|
+
}
|
|
210
|
+
};
|
|
211
|
+
}
|
|
212
|
+
/**
|
|
213
|
+
* The run's MCP client. With a judge handler it declares sampling with tools
|
|
214
|
+
* and DEDUP_JUDGE_CAPABILITY, and answers the server's dedup judge with it;
|
|
215
|
+
* without one it declares nothing, and the server dedups by the rule.
|
|
216
|
+
*/
|
|
217
|
+
export function ciClient(judge) {
|
|
218
|
+
const client = new Client({ name: "scenescout-ci", version: "1" }, judge ? { capabilities: { sampling: { tools: {} }, experimental: { [DEDUP_JUDGE_CAPABILITY]: {} } } } : undefined);
|
|
219
|
+
if (judge)
|
|
220
|
+
client.setRequestHandler(CreateMessageRequestSchema, judge);
|
|
221
|
+
return client;
|
|
222
|
+
}
|
|
223
|
+
/** A connected client as the loop's tool host. */
|
|
224
|
+
export function toolHost(client) {
|
|
154
225
|
return {
|
|
155
226
|
tools: async () => (await client.listTools()).tools,
|
|
156
227
|
call: async (name, args, timeoutMs) => {
|
|
@@ -162,6 +233,19 @@ async function startServer(log) {
|
|
|
162
233
|
},
|
|
163
234
|
};
|
|
164
235
|
}
|
|
236
|
+
async function startServer(log, judge) {
|
|
237
|
+
if (!fs.existsSync(serverPath))
|
|
238
|
+
throw new Error(`${serverPath} is missing: run \`npm run build\` first`);
|
|
239
|
+
const transport = new StdioClientTransport({ command: process.execPath, args: [serverPath], env: childEnv(process.env), stderr: "pipe" });
|
|
240
|
+
transport.stderr?.on("data", (chunk) => {
|
|
241
|
+
for (const line of chunk.toString("utf8").split(/\r?\n/))
|
|
242
|
+
if (line.trim())
|
|
243
|
+
log(` [server] ${line}`);
|
|
244
|
+
});
|
|
245
|
+
const client = ciClient(judge);
|
|
246
|
+
await client.connect(transport);
|
|
247
|
+
return toolHost(client);
|
|
248
|
+
}
|
|
165
249
|
function readMemoryFindings(projectDir) {
|
|
166
250
|
try {
|
|
167
251
|
return readFindings(JSON.parse(fs.readFileSync(path.join(projectDir, MEMORY_DIRNAME, "memory.json"), "utf8")));
|
|
@@ -172,31 +256,37 @@ function readMemoryFindings(projectDir) {
|
|
|
172
256
|
}
|
|
173
257
|
}
|
|
174
258
|
/**
|
|
175
|
-
* The agent loop. Before each model call
|
|
176
|
-
*
|
|
177
|
-
*
|
|
178
|
-
*
|
|
179
|
-
*
|
|
180
|
-
*
|
|
181
|
-
*
|
|
259
|
+
* The agent loop. Before each model call a turn is taken from the budget, or
|
|
260
|
+
* the cap that refuses it ends the loop; the call runs with at most the time
|
|
261
|
+
* that is left; each tool call it asks for is run in order, on the loop's one
|
|
262
|
+
* session, and none runs past the time cap: a call reached after it is
|
|
263
|
+
* answered as not run, as is any beyond MAX_TOOL_CALLS_PER_TURN. A call the
|
|
264
|
+
* loop cannot run (a tool it was not given, arguments that are not JSON, a
|
|
265
|
+
* scan of another directory) goes back to the model as an error result, so a
|
|
266
|
+
* malformed reply costs a turn, never the run.
|
|
182
267
|
*/
|
|
183
268
|
export async function agentLoop(o) {
|
|
184
269
|
const now = o.now ?? Date.now;
|
|
185
|
-
const
|
|
270
|
+
const budget = o.budget ?? newBudget(o.caps, o.startedAt ?? now());
|
|
271
|
+
// This loop's own turns and tokens; the budget holds every loop's.
|
|
272
|
+
const spend = { turns: 0, usage: { ...NO_USAGE }, startedAt: budget.startedAt };
|
|
273
|
+
const timeLeft = () => wallLeftMs(spend, budget.caps, now());
|
|
186
274
|
const allowed = new Set(o.tools.map((t) => t.name));
|
|
187
275
|
for (;;) {
|
|
188
|
-
const cap =
|
|
276
|
+
const cap = takeTurn(budget, now());
|
|
189
277
|
if (cap)
|
|
190
278
|
return { stop: cap, spend };
|
|
191
279
|
let turn;
|
|
192
280
|
try {
|
|
193
|
-
turn = await o.client.next(
|
|
281
|
+
turn = await o.client.next(timeLeft());
|
|
194
282
|
}
|
|
195
283
|
catch (err) {
|
|
284
|
+
settleTurn(budget);
|
|
196
285
|
if (err instanceof OutOfTime)
|
|
197
286
|
return { stop: "time", spend };
|
|
198
287
|
return { stop: "provider-error", stopDetail: err instanceof Error ? err.message : String(err), spend };
|
|
199
288
|
}
|
|
289
|
+
settleTurn(budget, turn.usage);
|
|
200
290
|
spend.turns += 1;
|
|
201
291
|
spend.usage = addUsage(spend.usage, turn.usage);
|
|
202
292
|
if (turn.resume)
|
|
@@ -228,15 +318,16 @@ export async function agentLoop(o) {
|
|
|
228
318
|
results.push({ id: call.id, isError: true, text: `${call.name} was not run: ${guarded.error}.` });
|
|
229
319
|
continue;
|
|
230
320
|
}
|
|
231
|
-
const left =
|
|
321
|
+
const left = timeLeft();
|
|
232
322
|
if (left <= 0) {
|
|
233
323
|
results.push({ id: call.id, isError: true, text: `${call.name} was not run: the time cap was reached.` });
|
|
234
324
|
continue;
|
|
235
325
|
}
|
|
326
|
+
const sent = o.session && o.session.tools.has(call.name) ? { ...guarded.args, session: o.session.name } : guarded.args;
|
|
236
327
|
try {
|
|
237
|
-
const r = await o.host.call(call.name,
|
|
328
|
+
const r = await o.host.call(call.name, sent, Math.min(600_000, left));
|
|
238
329
|
results.push({ id: call.id, isError: r.isError, text: r.text });
|
|
239
|
-
o.onResult?.(call.name,
|
|
330
|
+
o.onResult?.(call.name, sent, r);
|
|
240
331
|
}
|
|
241
332
|
catch (err) {
|
|
242
333
|
results.push({ id: call.id, isError: true, text: `${call.name} failed: ${err instanceof Error ? err.message : String(err)}` });
|
|
@@ -336,10 +427,181 @@ export async function captureShots(o) {
|
|
|
336
427
|
return { ...outcome, detail: `the base URL could not be captured: ${err instanceof Error ? err.message : String(err)}` };
|
|
337
428
|
}
|
|
338
429
|
}
|
|
430
|
+
/** Whether a listed tool takes `session`: a lane's calls to it are sent to the lane's own. */
|
|
431
|
+
function takesSession(t) {
|
|
432
|
+
const props = t.inputSchema?.properties;
|
|
433
|
+
return !!props && "session" in props;
|
|
434
|
+
}
|
|
435
|
+
const messageOf = (err) => (err instanceof Error ? err.message : String(err));
|
|
436
|
+
/**
|
|
437
|
+
* A run split into lanes (--lanes): plan, then run every lane at once, then
|
|
438
|
+
* fold what they did. The plan is a snapshot and a crawl from the planner's
|
|
439
|
+
* session (no model call: only their time counts), the crawl repeated while it
|
|
440
|
+
* finds routes, and the split brief.ts makes of them. Each lane attaches its
|
|
441
|
+
* own session on the run's target URL, as the planner did (the engine resolves
|
|
442
|
+
* every path against the URL a session attached with), opens its first route,
|
|
443
|
+
* runs the agent loop there with its own conversation, draws its turns from
|
|
444
|
+
* the run's one budget, and is closed when it ends. Their findings are already
|
|
445
|
+
* one: every session files into the project's one memory, whose dedup folds a
|
|
446
|
+
* defect two lanes filed. `outcome` is absent when there was nothing to split,
|
|
447
|
+
* and the run explores in one loop instead.
|
|
448
|
+
*/
|
|
449
|
+
async function exploreInLanes(o) {
|
|
450
|
+
const { host, options, log, now, budget } = o;
|
|
451
|
+
const timeLeft = () => wallLeftMs(budgetSpend(budget), options.caps, now());
|
|
452
|
+
// ── plan ──
|
|
453
|
+
// What went wrong while planning, so a plan left with nothing to split says why rather than blaming the app.
|
|
454
|
+
let planningFailed;
|
|
455
|
+
/** One planner call: its text, or undefined after saying why it failed. */
|
|
456
|
+
const plannerCall = async (tool, maxMs, what) => {
|
|
457
|
+
try {
|
|
458
|
+
const r = await host.call(tool, { session: PLANNER_SESSION }, Math.min(maxMs, timeLeft()));
|
|
459
|
+
if (r.isError)
|
|
460
|
+
throw new Error(r.text.replace(/^ERROR:\s*/, ""));
|
|
461
|
+
return r.text;
|
|
462
|
+
}
|
|
463
|
+
catch (err) {
|
|
464
|
+
planningFailed = `the planning ${what} failed: ${messageOf(err).slice(0, 300)}`;
|
|
465
|
+
log(`Lanes: ${planningFailed}.`);
|
|
466
|
+
return undefined;
|
|
467
|
+
}
|
|
468
|
+
};
|
|
469
|
+
// Attaching harvests no links; a snapshot of the page it landed on does, so the first crawl has routes to visit.
|
|
470
|
+
if (timeLeft() > 0)
|
|
471
|
+
await plannerCall("scout_snapshot", 120_000, "snapshot");
|
|
472
|
+
const notes = new Map();
|
|
473
|
+
for (let round = 0; round < PLAN_CRAWL_ROUNDS && timeLeft() > 0; round += 1) {
|
|
474
|
+
const text = await plannerCall("scout_crawl", 600_000, "crawl");
|
|
475
|
+
if (text === undefined)
|
|
476
|
+
break;
|
|
477
|
+
const known = notes.size;
|
|
478
|
+
for (const [route, lines] of crawlNotes(text))
|
|
479
|
+
if (!notes.has(route))
|
|
480
|
+
notes.set(route, lines);
|
|
481
|
+
if (crawlFoundNothing(text) || notes.size === known)
|
|
482
|
+
break;
|
|
483
|
+
}
|
|
484
|
+
const plan = planCiLanes({
|
|
485
|
+
target: options.url,
|
|
486
|
+
notes,
|
|
487
|
+
count: options.lanes,
|
|
488
|
+
focus: options.focus,
|
|
489
|
+
mode: options.mode,
|
|
490
|
+
...(planningFailed ? { planningFailed } : {}),
|
|
491
|
+
});
|
|
492
|
+
if (plan.oneLoop) {
|
|
493
|
+
log(`Lanes: ${plan.oneLoop}. Exploring in one loop.`);
|
|
494
|
+
return { lanes: { asked: options.lanes, sessions: [], oneLoop: plan.oneLoop } };
|
|
495
|
+
}
|
|
496
|
+
log(`Lanes: ${plan.lanes.length} of ${options.lanes} asked, sharing the caps: ` +
|
|
497
|
+
plan.lanes.map((l) => `${l.session} (${l.modules.join(", ")}; ${l.routes.length} route(s))`).join("; "));
|
|
498
|
+
// ── run ──
|
|
499
|
+
const tools = ciTools(o.listed, LANE_TOOLS);
|
|
500
|
+
const sessionTools = new Set(o.listed.filter(takesSession).map((t) => t.name));
|
|
501
|
+
const system = ciLaneSystemPrompt(loadPlaybook(packageRoot), options);
|
|
502
|
+
const runLane = async (lane) => {
|
|
503
|
+
const say = (line) => log(line.replace(/^(\s*)/, `$1[${lane.session}] `));
|
|
504
|
+
const result = {
|
|
505
|
+
session: lane.session,
|
|
506
|
+
modules: lane.modules,
|
|
507
|
+
routes: lane.routes.length,
|
|
508
|
+
attached: false,
|
|
509
|
+
stop: "could-not-start",
|
|
510
|
+
turns: 0,
|
|
511
|
+
usage: { ...NO_USAGE },
|
|
512
|
+
};
|
|
513
|
+
let attached = false;
|
|
514
|
+
try {
|
|
515
|
+
if (timeLeft() <= 0) {
|
|
516
|
+
say("the time cap was reached before it attached.");
|
|
517
|
+
return { ...result, stop: "time", stopDetail: "the time cap was reached before it attached" };
|
|
518
|
+
}
|
|
519
|
+
const r = await host
|
|
520
|
+
.call("scout_attach", o.attachArgs({ session: lane.session, url: options.url, objective: lane.objective, task: `Starting lane ${lane.session}` }), Math.min(o.attachMs, timeLeft()))
|
|
521
|
+
.catch((err) => ({ text: `ERROR: ${messageOf(err)}`, isError: true }));
|
|
522
|
+
const failed = attachFailure(r);
|
|
523
|
+
if (failed !== null) {
|
|
524
|
+
say(`could not attach: ${failed.slice(0, 300)}`);
|
|
525
|
+
return { ...result, stopDetail: failed.slice(0, 300) };
|
|
526
|
+
}
|
|
527
|
+
attached = true;
|
|
528
|
+
// Its own first route, by its full URL. A landing that does not open costs the lane nothing but the detour: it starts from the target.
|
|
529
|
+
let on = options.url;
|
|
530
|
+
if (lane.url !== options.url && timeLeft() > 0) {
|
|
531
|
+
const opened = await host
|
|
532
|
+
.call("scout_navigate", { session: lane.session, target: lane.url, task: `Opening lane ${lane.session}'s first route` }, Math.min(o.attachMs, timeLeft()))
|
|
533
|
+
.catch((err) => ({ text: `ERROR: ${messageOf(err)}`, isError: true }));
|
|
534
|
+
// With several sessions live the server puts a "[session …]" line first; the verdict is on the line after it.
|
|
535
|
+
const said = opened.text.replace(/^\[session [^\]\n]*\]\n/, "");
|
|
536
|
+
if (opened.isError || /^(ERROR|REFUSED):/.test(said))
|
|
537
|
+
say(`could not open ${lane.landing} (${said.replace(/^ERROR:\s*/, "").slice(0, 200)}); starting from the target instead.`);
|
|
538
|
+
else
|
|
539
|
+
on = lane.url;
|
|
540
|
+
}
|
|
541
|
+
say(`attached on ${new URL(on).pathname}, owning ${lane.modules.join(", ")}.`);
|
|
542
|
+
return await exploreLane(lane, on, say, { ...result, attached: true });
|
|
543
|
+
}
|
|
544
|
+
catch (err) {
|
|
545
|
+
// Not a cap and not the model's API: the lane itself broke. Reported as the lane's, and the run's (mergeLaneStops), never dropped.
|
|
546
|
+
const why = `the lane failed: ${messageOf(err).slice(0, 300)}`;
|
|
547
|
+
say(why);
|
|
548
|
+
return { ...result, attached, stop: "could-not-start", stopDetail: why };
|
|
549
|
+
}
|
|
550
|
+
};
|
|
551
|
+
const exploreLane = async (lane, on, say, result) => {
|
|
552
|
+
const outcome = await agentLoop({
|
|
553
|
+
client: o.makeClient(system, tools, ciLaneKickoff({
|
|
554
|
+
lane: { ...lane, url: on },
|
|
555
|
+
laneCount: plan.lanes.length,
|
|
556
|
+
url: options.url,
|
|
557
|
+
projectDir: options.projectDir,
|
|
558
|
+
mode: options.mode,
|
|
559
|
+
level: options.level,
|
|
560
|
+
focus: options.focus,
|
|
561
|
+
caps: options.caps,
|
|
562
|
+
})),
|
|
563
|
+
host,
|
|
564
|
+
tools,
|
|
565
|
+
caps: options.caps,
|
|
566
|
+
budget,
|
|
567
|
+
log: say,
|
|
568
|
+
now,
|
|
569
|
+
projectDir: options.projectDir,
|
|
570
|
+
session: { name: lane.session, tools: sessionTools },
|
|
571
|
+
});
|
|
572
|
+
say(`ended: ${outcome.stop === "done" ? "the model finished the lane" : describeStop(outcome.stop, options.caps, outcome.stopDetail)}.`);
|
|
573
|
+
// Closed as soon as it is done, so a lane that finished early holds no browser while the others work.
|
|
574
|
+
// Past the time cap nothing more runs here: the run's own close, within FINISH_MS, collects it.
|
|
575
|
+
if (timeLeft() > 0)
|
|
576
|
+
await host
|
|
577
|
+
.call("scout_close", { session: lane.session }, Math.min(30_000, timeLeft()))
|
|
578
|
+
.catch((err) => say(`closing its browser failed: ${messageOf(err)}`));
|
|
579
|
+
return {
|
|
580
|
+
...result,
|
|
581
|
+
stop: outcome.stop,
|
|
582
|
+
...(outcome.stopDetail ? { stopDetail: outcome.stopDetail } : {}),
|
|
583
|
+
turns: outcome.spend.turns,
|
|
584
|
+
usage: outcome.spend.usage,
|
|
585
|
+
};
|
|
586
|
+
};
|
|
587
|
+
// ── merge ──
|
|
588
|
+
// Settled, not raced: every lane is waited out before the run moves on. runLane reports its own failures, so none rejects.
|
|
589
|
+
const settled = await Promise.allSettled(plan.lanes.map(runLane));
|
|
590
|
+
const broken = settled.find((s) => s.status === "rejected");
|
|
591
|
+
if (broken)
|
|
592
|
+
throw new Error(`a lane failed outside its own handling: ${messageOf(broken.reason)}`);
|
|
593
|
+
const sessions = settled.map((s) => s.value);
|
|
594
|
+
const merged = mergeLaneStops(sessions);
|
|
595
|
+
return {
|
|
596
|
+
lanes: { asked: options.lanes, sessions },
|
|
597
|
+
outcome: { stop: merged.stop, ...(merged.stopDetail ? { stopDetail: merged.stopDetail } : {}), spend: budgetSpend(budget) },
|
|
598
|
+
};
|
|
599
|
+
}
|
|
339
600
|
/**
|
|
340
601
|
* The whole run. `makeClient` is how the model is reached: the HTTP client in
|
|
341
|
-
* the CLI, a scripted one in the tests.
|
|
342
|
-
* through the key
|
|
602
|
+
* the CLI, a scripted one in the tests. It is called once per conversation:
|
|
603
|
+
* once, or once per lane. Everything logged or written passes through the key
|
|
604
|
+
* redaction first.
|
|
343
605
|
*/
|
|
344
606
|
export async function runCi(options, resolved, deps) {
|
|
345
607
|
const secrets = deps.secrets ?? [];
|
|
@@ -350,10 +612,19 @@ export async function runCi(options, resolved, deps) {
|
|
|
350
612
|
const before = readMemoryFindings(options.projectDir);
|
|
351
613
|
// Pictures an earlier run left in the same output are not this run's: they must never be uploaded as its.
|
|
352
614
|
fs.rmSync(path.join(outDir, SHOTS_DIRNAME), { recursive: true, force: true });
|
|
353
|
-
|
|
615
|
+
// One budget for the run: every loop (one, or each lane) takes its turns from it, and the dedup judge's calls add their tokens, which the caps count.
|
|
616
|
+
const budget = newBudget(options.caps, startedAt);
|
|
617
|
+
// A run asked to show an element files no findings, so it has nothing to deduplicate.
|
|
618
|
+
const wantsJudge = options.dedup === "judge" && !options.show;
|
|
619
|
+
if (wantsJudge && !deps.judge)
|
|
620
|
+
log("No model was given for the dedup judge; the rule decides duplicates.");
|
|
621
|
+
const judgeAsk = wantsJudge ? deps.judge : undefined;
|
|
622
|
+
const judgeCalls = { calls: 0, failed: 0, usage: { ...NO_USAGE }, ms: 0 };
|
|
623
|
+
let outcome = { stop: "could-not-start", spend: budgetSpend(budget) };
|
|
354
624
|
let contractMet = false;
|
|
355
625
|
let reportWritten = false;
|
|
356
626
|
let capture;
|
|
627
|
+
let lanes;
|
|
357
628
|
let host = null;
|
|
358
629
|
// Set when the exploration ends (or never starts): the report and the close share FINISH_MS from then.
|
|
359
630
|
let finishBy = 0;
|
|
@@ -361,42 +632,67 @@ export async function runCi(options, resolved, deps) {
|
|
|
361
632
|
try {
|
|
362
633
|
// The page-load limit may be longer than the usual attach budget; the attach gets that limit and a minute to launch.
|
|
363
634
|
const attachMs = Math.max(ATTACH_MS, resolveTimeLimits(options, process.env).navMs + 60_000);
|
|
364
|
-
|
|
365
|
-
const
|
|
366
|
-
|
|
635
|
+
// One judge handler for the run's client: it answers the server's dedup questions and adds their tokens to the run's budget.
|
|
636
|
+
const judge = judgeAsk
|
|
637
|
+
? judgeHandler({ ask: judgeAsk, model: resolved.model, spend: budget, caps: options.caps, calls: judgeCalls, secrets, now })
|
|
638
|
+
: undefined;
|
|
639
|
+
host = await (deps.startHost ?? startServer)(log, judge);
|
|
640
|
+
// What every session of the run attaches with: the planner's here, and each lane's when the run is split.
|
|
641
|
+
const attachArgs = (a) => ({
|
|
642
|
+
url: a.url,
|
|
367
643
|
projectPath: options.projectDir,
|
|
368
644
|
mode: options.mode,
|
|
369
|
-
|
|
370
|
-
|
|
645
|
+
// Named either way, so a SCENESCOUT_DEDUP in the job's environment never decides for the option. Every session alike: they share one memory.
|
|
646
|
+
dedup: judgeAsk ? "judge" : "rule",
|
|
647
|
+
objective: a.objective.slice(0, 300),
|
|
648
|
+
task: a.task,
|
|
649
|
+
...(a.session ? { session: a.session } : {}),
|
|
371
650
|
...(options.storageStatePath ? { storageStatePath: options.storageStatePath } : {}),
|
|
372
651
|
...(options.browser ? { browser: options.browser } : {}),
|
|
373
652
|
...(options.actionTimeoutMs !== undefined ? { actionTimeoutMs: options.actionTimeoutMs } : {}),
|
|
374
653
|
...(options.navTimeoutMs !== undefined ? { navTimeoutMs: options.navTimeoutMs } : {}),
|
|
375
|
-
}
|
|
376
|
-
const
|
|
377
|
-
|
|
378
|
-
|
|
654
|
+
});
|
|
655
|
+
const attached = await host.call("scout_attach", attachArgs({
|
|
656
|
+
url: options.url,
|
|
657
|
+
objective: `CI run: explore at level ${options.level}${options.focus ? `, focusing on ${options.focus}` : ""}`,
|
|
658
|
+
task: "Starting the CI run",
|
|
659
|
+
}), Math.min(attachMs, options.caps.wallMs));
|
|
660
|
+
const failed = attachFailure(attached);
|
|
661
|
+
if (failed !== null) {
|
|
662
|
+
outcome = { ...outcome, stopDetail: failed.slice(0, 400) };
|
|
379
663
|
}
|
|
380
664
|
else {
|
|
381
665
|
log(`Attached to ${options.url} in ${options.mode} mode.`);
|
|
382
|
-
const
|
|
383
|
-
const
|
|
384
|
-
const kickoff = options.show ? ciCaptureKickoff({ url: options.url, show: options.show }) : ciKickoff(options);
|
|
666
|
+
const listed = await host.tools();
|
|
667
|
+
const toolHost = host;
|
|
385
668
|
let captured = null;
|
|
386
|
-
|
|
387
|
-
|
|
388
|
-
|
|
389
|
-
|
|
390
|
-
|
|
391
|
-
|
|
392
|
-
|
|
393
|
-
|
|
394
|
-
|
|
395
|
-
|
|
396
|
-
|
|
397
|
-
|
|
398
|
-
|
|
399
|
-
|
|
669
|
+
const oneLoop = () => {
|
|
670
|
+
const tools = ciTools(listed, options.show ? CAPTURE_TOOLS : undefined);
|
|
671
|
+
const system = options.show ? ciCaptureSystemPrompt() : ciSystemPrompt(loadPlaybook(packageRoot), options);
|
|
672
|
+
const kickoff = options.show ? ciCaptureKickoff({ url: options.url, show: options.show }) : ciKickoff(options);
|
|
673
|
+
return agentLoop({
|
|
674
|
+
client: deps.makeClient(system, tools, kickoff),
|
|
675
|
+
host: toolHost,
|
|
676
|
+
tools,
|
|
677
|
+
caps: options.caps,
|
|
678
|
+
log,
|
|
679
|
+
now,
|
|
680
|
+
budget,
|
|
681
|
+
projectDir: options.projectDir,
|
|
682
|
+
onResult: (name, _args, r) => {
|
|
683
|
+
if (name === "scout_capture" && !r.isError)
|
|
684
|
+
captured = parseCaptureResult(r.text) ?? captured;
|
|
685
|
+
},
|
|
686
|
+
});
|
|
687
|
+
};
|
|
688
|
+
if (options.lanes > 1 && !options.show) {
|
|
689
|
+
const split = await exploreInLanes({ host, listed, options, makeClient: deps.makeClient, log, now, budget, attachArgs, attachMs });
|
|
690
|
+
lanes = split.lanes;
|
|
691
|
+
outcome = split.outcome ?? (await oneLoop());
|
|
692
|
+
}
|
|
693
|
+
else {
|
|
694
|
+
outcome = await oneLoop();
|
|
695
|
+
}
|
|
400
696
|
log(`Run ended: ${describeStop(outcome.stop, options.caps, outcome.stopDetail)}.`);
|
|
401
697
|
finishBy = now() + FINISH_MS;
|
|
402
698
|
if (options.show) {
|
|
@@ -405,10 +701,11 @@ export async function runCi(options, resolved, deps) {
|
|
|
405
701
|
}
|
|
406
702
|
else {
|
|
407
703
|
// The report is written whatever ended the run. A forced report still prints its gaps.
|
|
408
|
-
|
|
704
|
+
// From the planner's session by name: a lane attaching made itself the server's default.
|
|
705
|
+
let report = await host.call("scout_report", { level: options.level, session: PLANNER_SESSION }, finishLeft());
|
|
409
706
|
contractMet = !report.isError && !/NOT GENERATED/.test(report.text);
|
|
410
707
|
if (!contractMet && !report.isError)
|
|
411
|
-
report = await host.call("scout_report", { level: options.level, force: true }, finishLeft());
|
|
708
|
+
report = await host.call("scout_report", { level: options.level, force: true, session: PLANNER_SESSION }, finishLeft());
|
|
412
709
|
reportWritten = !report.isError && !/^ERROR:/.test(report.text) && fs.existsSync(path.join(options.projectDir, MEMORY_DIRNAME, "report.md"));
|
|
413
710
|
if (!reportWritten)
|
|
414
711
|
log(`The report could not be generated: ${report.text.slice(0, 400)}`);
|
|
@@ -445,10 +742,17 @@ export async function runCi(options, resolved, deps) {
|
|
|
445
742
|
stop: outcome.stop,
|
|
446
743
|
...(outcome.stopDetail ? { stopDetail: outcome.stopDetail } : {}),
|
|
447
744
|
contractMet,
|
|
448
|
-
|
|
745
|
+
// The run's, not the last loop's: every lane's turns, and the judge's tokens.
|
|
746
|
+
spend: budgetSpend(budget),
|
|
449
747
|
endedAt,
|
|
450
748
|
findings: findingsThisRun(before, readMemoryFindings(options.projectDir)),
|
|
451
749
|
...(capture ? { capture } : {}),
|
|
750
|
+
...(lanes ? { lanes } : {}),
|
|
751
|
+
...(options.show
|
|
752
|
+
? {}
|
|
753
|
+
: {
|
|
754
|
+
dedup: judgeAsk ? { by: "judge", ...(deps.judgeEffort ? { effort: deps.judgeEffort } : {}), ...judgeCalls } : { by: "rule" },
|
|
755
|
+
}),
|
|
452
756
|
};
|
|
453
757
|
const written = [];
|
|
454
758
|
try {
|
|
@@ -468,7 +772,15 @@ export async function runCi(options, resolved, deps) {
|
|
|
468
772
|
const summary = ciSummaryMarkdown(result, secrets);
|
|
469
773
|
write("summary.md", summary);
|
|
470
774
|
write("ci.json", JSON.stringify(ciSummaryJson(result, deps.version, secrets), null, 2) + "\n");
|
|
471
|
-
|
|
775
|
+
const { anchor, warning } = sarifFilesFor({
|
|
776
|
+
option: options.sarifFileAnchor,
|
|
777
|
+
env: process.env,
|
|
778
|
+
projectDir: options.projectDir,
|
|
779
|
+
exists: (p) => fs.existsSync(p),
|
|
780
|
+
});
|
|
781
|
+
if (warning)
|
|
782
|
+
log(warning);
|
|
783
|
+
write("ci.sarif", JSON.stringify(ciSarif(result, deps.version, secrets, anchor), null, 2) + "\n");
|
|
472
784
|
// A capture run's outcome is its pictures and ci.json: it wrote no report by design.
|
|
473
785
|
if (options.show)
|
|
474
786
|
reportWritten = true;
|