scenescout 3.8.0 → 3.10.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +23 -0
- package/README.md +52 -3
- package/dist/browsers.js +12 -0
- package/dist/check-run.js +76 -0
- package/dist/cli.js +71 -6
- package/dist/engine/browser.js +514 -45
- package/dist/engine/check.js +487 -0
- package/dist/engine/collector.js +194 -25
- package/dist/engine/design.js +24 -8
- package/dist/engine/forms.js +290 -0
- package/dist/engine/lane.js +179 -2
- package/dist/engine/memory.js +70 -3
- package/dist/engine/oracles.js +101 -4
- package/dist/engine/pace.js +14 -1
- package/dist/engine/policy.js +118 -0
- package/dist/engine/report.js +54 -5
- package/dist/mcp-server.js +49 -9
- package/package.json +6 -4
- package/skills/scenescout/SKILL.md +3 -3
package/dist/engine/report.js
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import fs from "node:fs";
|
|
2
2
|
import path from "node:path";
|
|
3
|
-
import { SHARED_CHROME_ROUTE } from "./memory.js";
|
|
3
|
+
import { SHARED_CHROME_ROUTE, isEmbedKey } from "./memory.js";
|
|
4
4
|
import { sayVerification } from "./verify.js";
|
|
5
5
|
import { feedForSession } from "./live.js";
|
|
6
6
|
import { buildReplayHtml, evidenceFor } from "./replay.js";
|
|
@@ -59,6 +59,38 @@ function violationRollup(oracleLog) {
|
|
|
59
59
|
``,
|
|
60
60
|
];
|
|
61
61
|
}
|
|
62
|
+
/**
|
|
63
|
+
* What came from other sites' frames: their controls, and the violations
|
|
64
|
+
* they caused, by origin. Kept apart from the app's coverage and rollup —
|
|
65
|
+
* an embed's failing request is the embed's behaviour, and its controls are
|
|
66
|
+
* not the app's to cover — but listed, since the app chose to embed them.
|
|
67
|
+
*/
|
|
68
|
+
export function embedSection(violations, coverage) {
|
|
69
|
+
if (violations.length === 0 && coverage.total === 0)
|
|
70
|
+
return [];
|
|
71
|
+
const lines = [`## Embeds (other sites' frames)`, ``];
|
|
72
|
+
if (coverage.total > 0) {
|
|
73
|
+
lines.push(`${coverage.exercised}/${coverage.total} of their controls exercised; not counted in the app's coverage or its gap ledger.`, ``);
|
|
74
|
+
}
|
|
75
|
+
if (violations.length > 0) {
|
|
76
|
+
const byOrigin = new Map();
|
|
77
|
+
for (const v of violations) {
|
|
78
|
+
const origin = v.embed ?? "";
|
|
79
|
+
const sig = `${v.kind}: ${v.detail.replace(/\b\d+\b/g, ":n").slice(0, 120)}`;
|
|
80
|
+
const sigs = byOrigin.get(origin) ?? new Map();
|
|
81
|
+
sigs.set(sig, (sigs.get(sig) ?? 0) + 1);
|
|
82
|
+
byOrigin.set(origin, sigs);
|
|
83
|
+
}
|
|
84
|
+
lines.push(`Violations inside them (${violations.length}), reported as the embed's behaviour, not the app's:`, ``, `| Embed | Count | Signature |`, `|---|---|---|`);
|
|
85
|
+
for (const [origin, sigs] of byOrigin) {
|
|
86
|
+
for (const [sig, count] of [...sigs.entries()].sort((a, b) => b[1] - a[1]).slice(0, 8)) {
|
|
87
|
+
lines.push(`| \`${escapeTableCell(origin)}\` | ${count} | \`${escapeTableCell(sig)}\` |`);
|
|
88
|
+
}
|
|
89
|
+
}
|
|
90
|
+
lines.push(``);
|
|
91
|
+
}
|
|
92
|
+
return lines;
|
|
93
|
+
}
|
|
62
94
|
const SEVERITY_ORDER = { high: 0, medium: 1, low: 2 };
|
|
63
95
|
const SEVERITY_ICON = { high: "🔴", medium: "🟠", low: "🟡" };
|
|
64
96
|
/**
|
|
@@ -253,13 +285,15 @@ export function classifyFilledStates(memory, facts) {
|
|
|
253
285
|
const unsubmitted = new Set();
|
|
254
286
|
const noSubmitControl = new Set();
|
|
255
287
|
for (const st of Object.values(memory.states)) {
|
|
256
|
-
|
|
288
|
+
// Another site's frame is not the app's form: typing into an embed's chat leaves no app form unsubmitted.
|
|
289
|
+
const own = Object.fromEntries(Object.entries(st.elements).filter(([key]) => !isEmbedKey(key)));
|
|
290
|
+
const keys = Object.keys(own);
|
|
257
291
|
const filledForReal = keys.some((key) => st.elements[key].exercised && /^(type|select|upload|plan:(type|select|upload))/.test(st.elements[key].lastAction ?? "") && !isFilterKey(key));
|
|
258
292
|
if (!filledForReal)
|
|
259
293
|
continue;
|
|
260
294
|
if (facts[st.route]?.mutated || mutatedSiblingStep(st.route, facts))
|
|
261
295
|
continue;
|
|
262
|
-
if (offersSubmit(
|
|
296
|
+
if (offersSubmit(own) || keys.length >= COLLECTOR_CAP)
|
|
263
297
|
unsubmitted.add(st.route);
|
|
264
298
|
else
|
|
265
299
|
noSubmitControl.add(st.route);
|
|
@@ -431,7 +465,8 @@ export function generateReport(memory, oracleLog, extras, opts = {}) {
|
|
|
431
465
|
lines.push(`| States explored | ${cov.states} |`);
|
|
432
466
|
if (extras)
|
|
433
467
|
lines.push(`| Design audits this run (all sessions) | ${extras.designAudits} |`);
|
|
434
|
-
|
|
468
|
+
const fromEmbeds = oracleLog.filter((v) => v.embed).length;
|
|
469
|
+
lines.push(`| Oracle violations this session | ${oracleLog.length - fromEmbeds}${fromEmbeds > 0 ? ` (plus ${fromEmbeds} inside other sites' frames)` : ""} |`);
|
|
435
470
|
if (extras?.policyAttributed) {
|
|
436
471
|
lines.push(`| Errors caused by the tester's own write-policy blocks (not counted above) | ${extras.policyAttributed} |`);
|
|
437
472
|
}
|
|
@@ -614,7 +649,21 @@ export function generateReport(memory, oracleLog, extras, opts = {}) {
|
|
|
614
649
|
lines.push(`- \`${r}\``);
|
|
615
650
|
lines.push(``);
|
|
616
651
|
}
|
|
617
|
-
|
|
652
|
+
if (extras?.trustedEmbeds && extras.trustedEmbeds.length > 0) {
|
|
653
|
+
lines.push(`## Trusted embeds`);
|
|
654
|
+
lines.push(``);
|
|
655
|
+
lines.push(extras.mode === "safe-write"
|
|
656
|
+
? `Writes that frames of these origins sent outside the app were allowed, as the user asked; anything they created lives with that provider, and SceneScout cannot list it:`
|
|
657
|
+
: extras.mode === "destructive"
|
|
658
|
+
? `Named as trusted, and not needed: destructive mode let every embed's writes out:`
|
|
659
|
+
: `Named as trusted, but not applied: trust only counts in safe-write mode, and this run was ${extras.mode ?? "read-only"}:`);
|
|
660
|
+
lines.push(``);
|
|
661
|
+
for (const o of extras.trustedEmbeds)
|
|
662
|
+
lines.push(`- \`${o}\``);
|
|
663
|
+
lines.push(``);
|
|
664
|
+
}
|
|
665
|
+
lines.push(...violationRollup(oracleLog.filter((v) => !v.embed)));
|
|
666
|
+
lines.push(...embedSection(oracleLog.filter((v) => v.embed), cov.embeds));
|
|
618
667
|
if (cov.unexercised.length > 0) {
|
|
619
668
|
lines.push(`## Unexplored surface (for the next run)`);
|
|
620
669
|
lines.push(``);
|
package/dist/mcp-server.js
CHANGED
|
@@ -35,12 +35,13 @@ import { z } from "zod";
|
|
|
35
35
|
import { BrowserEngine } from "./engine/browser.js";
|
|
36
36
|
import { reapOrphanBrowsers } from "./engine/reaper.js";
|
|
37
37
|
import { FINDING_CATEGORIES, MemoryStore, redactSecrets } from "./engine/memory.js";
|
|
38
|
-
import { LANE_NAME_MAX, laneReportInstruction, parseLaneReport, summarizeLaneReport } from "./engine/lane.js";
|
|
38
|
+
import { decodedEntitiesNote, LANE_NAME_MAX, LaneLedger, laneCloseGuard, laneReportInstruction, parseLaneReport, summarizeLaneReport } from "./engine/lane.js";
|
|
39
39
|
import { MAX_UNFILED_NAMED, unfiledDefects } from "./engine/calibration.js";
|
|
40
40
|
import { SessionQueue, withWatchdog } from "./engine/dispatch.js";
|
|
41
41
|
import { FIXTURE_KINDS } from "./engine/fixtures.js";
|
|
42
42
|
import { feedForSession, LIVE_ENV, writeStatusFile, LIVE_TOKEN_FILE, liveEngines, liveTokenFileName, pidAlive, statusFileName, LiveServer, StatusBoard, } from "./engine/live.js";
|
|
43
43
|
import { formatBriefs, MAX_LANES, planLanes } from "./engine/brief.js";
|
|
44
|
+
import { formatNeverSubmittedEmpty } from "./engine/forms.js";
|
|
44
45
|
import { computeGaps, formatRouteCoverage, formatUnchosenOptions, generateReport, replayDocument, reportEvidence } from "./engine/report.js";
|
|
45
46
|
import { describeVerdict, formatWorklist, unknownIds, VERDICTS, verifyWorklist } from "./engine/verify.js";
|
|
46
47
|
import { RECORD_MAX_FRAMES, resolveFrame } from "./engine/replay.js";
|
|
@@ -243,6 +244,7 @@ function reportExtras(eng) {
|
|
|
243
244
|
createdResources: eng.createdResources,
|
|
244
245
|
unvisitedRoutes: unvisited,
|
|
245
246
|
mode: eng.mode,
|
|
247
|
+
trustedEmbeds: [...eng.trustedEmbeds],
|
|
246
248
|
policyAttributed: eng.oracleLog.policyAttributed,
|
|
247
249
|
version: PKG_VERSION,
|
|
248
250
|
attachedSessions: [...engines.keys()],
|
|
@@ -499,6 +501,9 @@ server.server.setRequestHandler(GetPromptRequestSchema, (request) => {
|
|
|
499
501
|
// Splitting the app between lanes: the other half of the parallel protocol.
|
|
500
502
|
// scout_lane_report is how a lane hands its answers back; this is what the
|
|
501
503
|
// planner hands it in the first place.
|
|
504
|
+
// Which sessions those two tools have named as lanes, and whose report was
|
|
505
|
+
// folded, so scout_close can refuse to drop a lane's decisions.
|
|
506
|
+
const laneLedger = new LaneLedger();
|
|
502
507
|
server.registerTool("scout_lane_brief", {
|
|
503
508
|
description: "For a run split across parallel agents (lanes). Divides the app's known routes between N lanes and returns each lane's session name, the `objective` to attach it with, and the routes it owns — whole modules per lane, balanced by route count, so no two lanes audit the same area and none is left unopened. Call it after the first crawl, when route knowledge is complete. Touches no browser; pass the briefs to your lane agents, then use scout_lane_report for what they hand back.",
|
|
504
509
|
inputSchema: {
|
|
@@ -511,7 +516,9 @@ server.registerTool("scout_lane_brief", {
|
|
|
511
516
|
try {
|
|
512
517
|
const eng = engineFor(session);
|
|
513
518
|
const all = routes && routes.length > 0 ? routes : eng.allKnownRoutes();
|
|
514
|
-
|
|
519
|
+
const briefs = planLanes(all, lanes, { goal, mode: eng.mode, role: eng.role });
|
|
520
|
+
laneLedger.nameBriefed(briefs.map((b) => b.lane), (s) => engines.has(s));
|
|
521
|
+
return text(formatBriefs(briefs, { goal, mode: eng.mode, role: eng.role }), session);
|
|
515
522
|
}
|
|
516
523
|
catch (err) {
|
|
517
524
|
return errorText(err);
|
|
@@ -526,10 +533,13 @@ server.registerTool("scout_lane_report", {
|
|
|
526
533
|
},
|
|
527
534
|
}, async ({ lane, reply }) => {
|
|
528
535
|
try {
|
|
529
|
-
if (reply === undefined)
|
|
536
|
+
if (reply === undefined) {
|
|
537
|
+
laneLedger.name(lane);
|
|
530
538
|
return { content: [{ type: "text", text: laneReportInstruction(lane) }] };
|
|
539
|
+
}
|
|
531
540
|
const parsed = parseLaneReport(reply, lane);
|
|
532
541
|
if (!parsed.ok) {
|
|
542
|
+
laneLedger.refuse(lane, engines.get(lane)?.attached === true);
|
|
533
543
|
return {
|
|
534
544
|
content: [
|
|
535
545
|
{ type: "text", text: `Lane report REFUSED: ${parsed.reason}. Ask the lane once for the corrected object; do not re-judge its prose.` },
|
|
@@ -574,8 +584,12 @@ server.registerTool("scout_lane_report", {
|
|
|
574
584
|
(unfiled.length > MAX_UNFILED_NAMED ? `\n … +${unfiled.length - MAX_UNFILED_NAMED} more` : "") +
|
|
575
585
|
`\nFile each with scout_finding (the same evidence), or confirm which finding already covers it, before closing the lane's session. A judged defect that is never filed is not in the report.`
|
|
576
586
|
: "";
|
|
587
|
+
laneLedger.fold(lane, engines.get(lane)?.attached === true);
|
|
577
588
|
const around = parsed.aroundIgnored ? `\n(The text around the report's JSON block was discarded unread.)` : "";
|
|
578
|
-
|
|
589
|
+
const decoded = decodedEntitiesNote(parsed.entitiesDecoded);
|
|
590
|
+
return {
|
|
591
|
+
content: [{ type: "text", text: `Lane report accepted — ${summarizeLaneReport(parsed.report)}${note}${around}${decoded}${followUp}` }],
|
|
592
|
+
};
|
|
579
593
|
}
|
|
580
594
|
catch (err) {
|
|
581
595
|
return errorText(err);
|
|
@@ -628,6 +642,12 @@ server.registerTool("scout_attach", {
|
|
|
628
642
|
.max(60000)
|
|
629
643
|
.optional()
|
|
630
644
|
.describe("A floor between actions, in milliseconds, for when a person is watching and needs to keep up — following a flow, taking notes, demonstrating. Default 0: as fast as the page allows, which is what a run wants otherwise. Changeable mid-run with scout_session {paceMs}."),
|
|
645
|
+
trustedEmbeds: z
|
|
646
|
+
.array(z.string().max(200))
|
|
647
|
+
.max(10)
|
|
648
|
+
.optional()
|
|
649
|
+
.describe('Origins of embedded frames (e.g. "https://pay.example.com") whose writes out of the app may go out — ONLY when the user named them, typically a provider in test mode, and only in safe-write mode. ' +
|
|
650
|
+
"Never add one yourself. Hostile input, repeated-click probes and uploads stay refused in them."),
|
|
631
651
|
record: z
|
|
632
652
|
.boolean()
|
|
633
653
|
.default(false)
|
|
@@ -639,7 +659,7 @@ server.registerTool("scout_attach", {
|
|
|
639
659
|
.optional()
|
|
640
660
|
.describe("Session name for multi-role runs (e.g. 'admin', 'qa'). Creates/replaces that session's browser and makes it the default. Default: 'default'."),
|
|
641
661
|
},
|
|
642
|
-
}, serializedControl(async ({ url, projectPath, storageStatePath, mode, headed, browser, viewportWidth, viewportHeight, objective, task, record, paceMs, session, }) => {
|
|
662
|
+
}, serializedControl(async ({ url, projectPath, storageStatePath, mode, headed, browser, viewportWidth, viewportHeight, objective, task, record, trustedEmbeds, paceMs, session, }) => {
|
|
643
663
|
try {
|
|
644
664
|
const target = session ?? activeName;
|
|
645
665
|
if (session) {
|
|
@@ -722,6 +742,7 @@ server.registerTool("scout_attach", {
|
|
|
722
742
|
paceMs,
|
|
723
743
|
task: objective ? task : undefined,
|
|
724
744
|
record,
|
|
745
|
+
trustedEmbeds,
|
|
725
746
|
memoryStore: store,
|
|
726
747
|
});
|
|
727
748
|
eng.role = storageStatePath ? path.basename(storageStatePath).replace(/\.json$/i, "") : "anonymous";
|
|
@@ -1154,7 +1175,7 @@ server.registerTool("scout_finding", {
|
|
|
1154
1175
|
}
|
|
1155
1176
|
}));
|
|
1156
1177
|
server.registerTool("scout_coverage", {
|
|
1157
|
-
description: "Show exploration coverage: states visited across all runs, which elements remain unexercised,
|
|
1178
|
+
description: "Show exploration coverage: states visited across all runs, which elements remain unexercised, which options of a dropdown used this run no session has chosen yet, and which forms seen this run no session has submitted with every text field blank. Use to decide where to explore next and when the level's budget is satisfied.",
|
|
1158
1179
|
inputSchema: { session: sessionParam },
|
|
1159
1180
|
}, serializedPerSession("scout_coverage", async (_args, session) => {
|
|
1160
1181
|
try {
|
|
@@ -1169,11 +1190,12 @@ server.registerTool("scout_coverage", {
|
|
|
1169
1190
|
`⚠ MEMORY WRITE FAILING: ${eng.memory.lastSaveError} — coverage/findings since the last successful write are NOT persisted to disk. If this doesn't clear on its own, check the project directory still exists and is writable.`,
|
|
1170
1191
|
]
|
|
1171
1192
|
: []),
|
|
1172
|
-
`States known: ${cov.states} · Elements exercised: ${cov.elementsExercised}/${cov.elementsTotal}`,
|
|
1193
|
+
`States known: ${cov.states} · Elements exercised: ${cov.elementsExercised}/${cov.elementsTotal}${cov.embeds.total > 0 ? ` (plus ${cov.embeds.exercised}/${cov.embeds.total} inside other sites' frames, not counted)` : ""}`,
|
|
1173
1194
|
formatRouteCoverage(eng.allKnownRoutes(), unvisited),
|
|
1174
1195
|
`Unexercised elements by route:`,
|
|
1175
1196
|
...cov.unexercised.slice(0, 25).map((u) => ` ${u.state}: ${u.keys.slice(0, 6).join(", ")}${u.keys.length > 6 ? ` … +${u.keys.length - 6}` : ""}`),
|
|
1176
1197
|
...formatUnchosenOptions(eng.memory.unchosenOptions()),
|
|
1198
|
+
...formatNeverSubmittedEmpty(eng.memory.formsNeverSubmittedEmpty()),
|
|
1177
1199
|
];
|
|
1178
1200
|
return text(lines.join("\n"), session);
|
|
1179
1201
|
}
|
|
@@ -1254,6 +1276,7 @@ server.registerTool("scout_report", {
|
|
|
1254
1276
|
createdResources: eng.createdResources,
|
|
1255
1277
|
unvisitedRoutes: unvisited,
|
|
1256
1278
|
mode: eng.mode,
|
|
1279
|
+
trustedEmbeds: [...eng.trustedEmbeds],
|
|
1257
1280
|
policyAttributed: eng.oracleLog.policyAttributed,
|
|
1258
1281
|
// Which sessions are still open decides whether a quiet one is holding a browser, and how long its trailing idle runs.
|
|
1259
1282
|
attachedSessions: [...engines.keys()],
|
|
@@ -1324,15 +1347,23 @@ server.registerTool("scout_verify", {
|
|
|
1324
1347
|
}
|
|
1325
1348
|
}));
|
|
1326
1349
|
server.registerTool("scout_close", {
|
|
1327
|
-
description: "Close a session's browser (memory persists on disk). Default: the DEFAULT session. Pass session to close a specific one, or all=true to close every live session at the end of a multi-role run."
|
|
1350
|
+
description: "Close a session's browser (memory persists on disk). Default: the DEFAULT session. Pass session to close a specific one, or all=true to close every live session at the end of a multi-role run. " +
|
|
1351
|
+
"A session that is a lane of a parallel run (named by scout_lane_brief or scout_lane_report) is not closed until its lane report has been accepted by scout_lane_report, since folding needs the session attached; the refusal names every such lane.",
|
|
1328
1352
|
inputSchema: {
|
|
1329
1353
|
session: z.string().max(40).optional().describe("Session to close (default: the default session)"),
|
|
1330
1354
|
all: z.boolean().default(false).describe("Close every live session"),
|
|
1355
|
+
force: z
|
|
1356
|
+
.boolean()
|
|
1357
|
+
.default(false)
|
|
1358
|
+
.describe("Close even a lane whose report has not been folded yet. Its decisions are then lost: nothing is kept for calibration and nothing checks its defects were filed. Prefer folding it with scout_lane_report first."),
|
|
1331
1359
|
},
|
|
1332
|
-
}, serializedControl(async ({ session, all }) => {
|
|
1360
|
+
}, serializedControl(async ({ session, all, force }) => {
|
|
1333
1361
|
try {
|
|
1334
1362
|
if (all) {
|
|
1335
1363
|
const names = [...engines.keys()];
|
|
1364
|
+
const guard = laneCloseGuard(names, laneLedger, force);
|
|
1365
|
+
if (!guard.ok)
|
|
1366
|
+
return text(`${guard.message}\nNo session was closed.`, activeName);
|
|
1336
1367
|
// Closes are independent per-browser — run them in parallel so N wedged
|
|
1337
1368
|
// sessions cost one 8s teardown cap total, not N of them.
|
|
1338
1369
|
const dirs = new Set();
|
|
@@ -1350,6 +1381,7 @@ server.registerTool("scout_close", {
|
|
|
1350
1381
|
store.endRun();
|
|
1351
1382
|
sessionQueue.clear();
|
|
1352
1383
|
board.clear();
|
|
1384
|
+
laneLedger.clear();
|
|
1353
1385
|
lastWriter = null;
|
|
1354
1386
|
for (const dir of dirs)
|
|
1355
1387
|
flushStatus(dir);
|
|
@@ -1359,6 +1391,9 @@ server.registerTool("scout_close", {
|
|
|
1359
1391
|
const eng = engines.get(name);
|
|
1360
1392
|
if (!eng)
|
|
1361
1393
|
return text(`No live session "${name}".`, name);
|
|
1394
|
+
const guard = laneCloseGuard([name], laneLedger, force);
|
|
1395
|
+
if (!guard.ok)
|
|
1396
|
+
return text(guard.message, name);
|
|
1362
1397
|
keepReport(eng);
|
|
1363
1398
|
live?.dropSession(name);
|
|
1364
1399
|
await eng.close();
|
|
@@ -1369,6 +1404,11 @@ server.registerTool("scout_close", {
|
|
|
1369
1404
|
eng.memory.endRun();
|
|
1370
1405
|
sessionQueue.forget(name);
|
|
1371
1406
|
board.remove(name);
|
|
1407
|
+
laneLedger.forget(name);
|
|
1408
|
+
// The last session closing ends the run, and its lanes with it: a brief's
|
|
1409
|
+
// names that never attached must not make a later run's session a lane.
|
|
1410
|
+
if (engines.size === 0)
|
|
1411
|
+
laneLedger.clear();
|
|
1372
1412
|
if (lastWriter?.session === name)
|
|
1373
1413
|
lastWriter = null;
|
|
1374
1414
|
if (eng.memory?.dir)
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "scenescout",
|
|
3
|
-
"version": "3.
|
|
3
|
+
"version": "3.10.0",
|
|
4
4
|
"description": "SceneScout — exploratory UI testing for AI coding agents. An MCP server that gives any agent (Claude Code, Cursor, VS Code Copilot, Codex, Gemini CLI and others) a structured view of a running web app, always-on oracles, a network-level write policy, memory across runs and a gap-checked report.",
|
|
5
5
|
"license": "MIT",
|
|
6
6
|
"author": "brunoboto96",
|
|
@@ -71,13 +71,14 @@
|
|
|
71
71
|
"mcp-check": "npm run build && npm run mcp-check:run",
|
|
72
72
|
"mcp-check:run": "tsx scripts/mcp-check.ts",
|
|
73
73
|
"test": "npm run build && npm run test:unit && npm run smoke:run && npm run mcp-check:run",
|
|
74
|
-
"test:unit": "npm run scan-test && npm run oracle-test && npm run policy-test && npm run fixture-test && npm run dispatch-test && npm run design-test && npm run contract-test && npm run pace-test && npm run request-test && npm run settle-test && npm run claims-test && npm run verify-test && npm run brief-test && npm run lane-test && npm run calibration-test && npm run bench-test && npm run memory-test && npm run install-test && npm run live-test && npm run demo-test && npm run hygiene-test",
|
|
74
|
+
"test:unit": "npm run scan-test && npm run oracle-test && npm run policy-test && npm run fixture-test && npm run dispatch-test && npm run design-test && npm run check-test && npm run contract-test && npm run pace-test && npm run request-test && npm run settle-test && npm run claims-test && npm run verify-test && npm run brief-test && npm run lane-test && npm run calibration-test && npm run bench-test && npm run memory-test && npm run install-test && npm run live-test && npm run demo-test && npm run hygiene-test",
|
|
75
75
|
"scan-test": "tsx scripts/scan-test.ts",
|
|
76
76
|
"oracle-test": "tsx --test scripts/oracle-test.ts",
|
|
77
77
|
"policy-test": "tsx --test scripts/policy-test.ts",
|
|
78
78
|
"fixture-test": "tsx --test scripts/fixture-test.ts",
|
|
79
79
|
"dispatch-test": "tsx --test scripts/dispatch-test.ts",
|
|
80
80
|
"design-test": "tsx --test scripts/design-test.ts",
|
|
81
|
+
"check-test": "tsx --test scripts/check-test.ts",
|
|
81
82
|
"contract-test": "tsx --test scripts/contract-test.ts",
|
|
82
83
|
"lane-test": "tsx --test scripts/lane-test.ts",
|
|
83
84
|
"memory-test": "tsx --test scripts/memory-test.ts",
|
|
@@ -104,7 +105,8 @@
|
|
|
104
105
|
"@changesets/cli": "3.0.3",
|
|
105
106
|
"@types/node": "^20.19.0",
|
|
106
107
|
"prettier": "3.9.8",
|
|
107
|
-
"tsx": "^4.23.
|
|
108
|
-
"typescript": "^5.9.3"
|
|
108
|
+
"tsx": "^4.23.15",
|
|
109
|
+
"typescript": "^5.9.3",
|
|
110
|
+
"yaml": "^2.9.1"
|
|
109
111
|
}
|
|
110
112
|
}
|
|
@@ -34,7 +34,7 @@ You are the brain of an exploratory UI tester. The SceneScout MCP server gives y
|
|
|
34
34
|
1. **`scout_crawl` first, always.** One call visits every known route (pass `paths` to sweep a specific subset instead), records coverage, and returns per-route health. This is the whole breadth pass — do not visit routes one-by-one with navigate+snapshot.
|
|
35
35
|
2. **Investigate what the crawl flagged.** For each problem route (violations, dead-ends, auth-redirects): navigate there, `scout_snapshot`, reproduce, then `scout_finding`.
|
|
36
36
|
3. **Run journeys with `scout_run_plan {steps}`.** Mechanical sequences (fill form → submit → check) go in ONE plan call — `steps` is an ordered list of `{action, target, value}` — with `testid=`/`text=`/`label=` targets — not one LLM turn per click. The plan aborts at the first violation and tells you where; that's your cue to investigate interactively.
|
|
37
|
-
4. **Snapshot economics:** `scout_snapshot` after landing somewhere new; re-snapshots of the same route return *diffs* with stable refs — "No element changes" costs you almost nothing. `scout_screenshot` ONLY for suspected pixel-native issues (a canvas, a rendering glitch); geometry problems (overlap, off-screen, a covered control) are already in the snapshot as GEOMETRY issues, and images that failed to load are listed under BROKEN IMAGES — file those, quoting the line.
|
|
37
|
+
4. **Snapshot economics:** `scout_snapshot` after landing somewhere new; re-snapshots of the same route return *diffs* with stable refs — "No element changes" costs you almost nothing. `scout_screenshot` ONLY for suspected pixel-native issues (a canvas, a rendering glitch); geometry problems (overlap, off-screen, a covered control) are already in the snapshot as GEOMETRY issues, and images that failed to load are listed under BROKEN IMAGES — file those, quoting the line. Controls inside embeds (`<iframe>`) are listed with the page's, each marked `⟨in … frame⟩`, and act by ref like any other. A frame of the app's own origin is the app: test it fully. A frame of another site (`cross-origin`) is someone else's system that the user has not authorised you to test: click and type there as a user would, to see the embed render and respond, but never send it hostile input — the engine refuses markup, values over 200 characters, control characters, repeated-click probes and uploads there, and you must not try SQL, template or other injection shapes either — and file what you find there as the embed's behaviour, naming its origin, not as the app's bug. A failing request such a frame sent outside the app says so (`in an embed of …`) and is kept at medium; console errors cannot be told apart by frame and stay the app's; its controls are counted apart and never enter the app's coverage or gap ledger. Its container text is masked, and the writes it sends outside the app are refused in every mode but destructive — unless the user names that origin as a trusted embed (a provider in test mode, say): pass it in `scout_attach {trustedEmbeds: ["https://…"]}` and, in safe-write only, its writes go out. Never add an origin the user did not name. The FRAMES line names frames that were not read; say in your summary that their contents were not explored.
|
|
38
38
|
5. **Native-user behaviours.** `scout_type {ref, textValue}` (or its alias `value`, matching `scout_select` and a plan step) APPENDS when a field already has content (menu clicks often insert @-mention chips or commands into composers — appending preserves them; the result reports what was already there); pass `replace=true` only to deliberately clear, and `pressEnter=true` to submit from the field the way a user would. Before concluding a badge, icon, or "N errors" indicator *does nothing*, `scout_hover` it — tooltips and hover cards are invisible to snapshots and clicks, and hover output includes what appeared. In HEADED mode (`scout_attach {headed:true}`, which the user asks for when they want to watch) the user's physical mouse competes with the synthetic pointer: if a hover reveals nothing and the finding matters, ask the user to move their mouse off the browser window and retry before filing. **Scroll long pages with `scout_scroll`** — the design audit and snapshot measure at the current scroll position, so judge deep sections by scrolling then re-auditing; it refuses to scroll where a real user couldn't and reports SCROLL LOCKED (the leaked modal scroll-lock that silently amputates everything below the fold — snapshots also flag it passively as an OVERLAY line), and scrolling triggers lazy-loaded content whose failures surface as fresh oracle violations. Elements fully clipped inside an overflow-hidden container are flagged UNREACHABLE in GEOMETRY issues — no amount of scrolling reveals them; that's a high-value layout bug, distinct from merely below-the-fold content. **A page can hold SEVERAL independent scroll regions** and plain `scout_scroll` moves the largest one, so a sidebar nav beside a taller main pane never budges: pass `scout_scroll {target:"testid=…"}` to scroll one region. Never report a nav item, tab or list row as missing/truncated until you have scrolled ITS container — content scrolled out of a secondary pane looks exactly like content that was cut off.
|
|
39
39
|
6. **The rest of the input vocabulary.** `scout_select` sets a `<select>` option by value or visible label — use it rather than clicking a native dropdown open, which does not render as page DOM. `scout_press` sends a real key to the focused element (`Escape` to dismiss a modal, `Tab` to walk focus order, `Enter` to submit from a field); it is also how the keyboard-only pass at `extensive` is performed, and it vets the focused control first so a destructive action cannot be triggered blind in read-only mode. **`scout_upload {ref}` attaches a file the way a user does** — `ref` is a visible `<input type=file>` (snapshots list these with role `file`; `scout_type` on one redirects here) OR the button/label/dropzone that opens the file chooser (the chooser is intercepted and answered — that is how the hidden input behind a styled "Choose file" control is reached); omit `ref` when the page has exactly one file input, hidden or not (snapshots disclose hidden ones on a FILE INPUTS line). Nothing needs to exist on disk: a small VALID fixture is generated in memory, its kind inferred from the input's `accept` attribute or chosen with `fixture` (`pdf`, `png`, `txt`, `csv`, `json`); `filePath` uploads a real file but must live inside the attached project (fenced, like navigation is fenced to the origin); `name` overrides the filename. The result flags a file that violates `accept` (a mismatch the app then ACCEPTS is a validation finding), warns when the app cleared the input after selection, and says whether a state-changing request fired on selection — if none did, either click the form's submit or read the next snapshot for a client-side rejection. Plans take `{action:"upload", target, value:"pdf"}` steps (`target` required). When the input or its trigger was addressed by `ref`, the gap ledger counts an attached-but-unsent file as filled-never-submitted; the ref-less path has no listed element to mark.
|
|
40
40
|
7. **Say what you are doing: `task` is required before a tool acts.** A session shows two lines to whoever is watching. Its **objective** is the whole remit you were given, set once at `scout_attach {objective}` ("Admin lane: §2 registers, §7 plan gating", "Approve and reject orders as a manager"). Its **task** is what you are doing *right now*, and every tool that changes the app or the page — `scout_navigate`, `scout_back`, `scout_click`, `scout_type`, `scout_select`, `scout_press`, `scout_upload`, `scout_run_plan` — takes it: a few words for the batch in front of you ("Filtering the documents register by status", "Filling the deviation form with invalid dates", "Signing in as QA_Team"). Say what you are DOING, not what you are checking — "§2.4 filtering narrows the set and is reflected in the URL" is the acceptance criteria, which is the result you will judge, not the batch you are running; naming the item is fine ("§2.4: filtering the documents register"). The task STAYS SET until you pass a different one, so a batch costs a few words, not one per call — pass a fresh one whenever you move on. Acting with none standing is refused: the person watching would otherwise see a session clicking through their app with nothing to say why. `scout_journey {action:"start", goal:…}` sets the task too while it runs — use a journey when you are MEASURING a whole user task, the parameter for everything else.
|
|
@@ -42,7 +42,7 @@ You are the brain of an exploratory UI tester. The SceneScout MCP server gives y
|
|
|
42
42
|
8. **Design-connoisseur pass without pixels: `scout_design_audit`.** Run it once per representative page (dashboard, a form, a detail view, a data table). Its output has two tiers: **⚠ measurable defects** (WCAG contrast, tiny targets, clipped text, aspect-distorted images, horizontal overflow, keyboard tab stops with no visible focus indicator — sampled with real Tab presses) and **→ craft suggestions** (line measure and line-height rhythm, spacing-grid adherence, typography entropy, gray census and accent-hue count, pure-#000 body text, elevation/control consistency, heading structure, indistinguishable links, AI-slop tells like gradient text/glassmorphism/side-stripe borders/identical card grids), closing with a SYSTEM SUMMARY of design-system coherence. Judge every line with product context (dense tables legitimately have small targets; a chart page legitimately uses many hues). File ⚠ defects as `visual`/`a11y`, and genuine → opportunities as `ux-polish` findings **quoting the concrete numbers** — "~142 characters per line (65–75 ideal)" beats "text feels wide". Every audit ends with a **PAGE SCORE** (0–100 overall + a11y/craft/consistency/task-clarity subscores) persisted per route — the report ranks pages worst-first, so re-runs show whether pages got better or worse. Separately, every `scout_snapshot` runs an **overlay/modal probe** automatically: an empty dialog over a grayed page, a backdrop with no dialog, a far-off-centre dialog leaving a blank band, or a dialog extending unreachably below the viewport appear as OVERLAY lines in GEOMETRY issues — treat these as high-value findings (the user is visually stuck). This is where "how could this page be better" gets answered, not just "is it broken".
|
|
43
43
|
9. **Measure task EASE with `scout_journey`, not just correctness.** Wrap each module's primary task (`{action:"start", goal:"Create an order"}` → do it → `{action:"end", completed:…}`). Navigate by CLICKING like a first-time user — typing a known deep URL shortcuts the very thing being measured (a route you can only reach by editing the address bar is itself a finding). The result gives interaction count, distinct screens, the path taken, and BACKTRACKS — returning to a screen already left is the clearest evidence the next step wasn't discoverable. An abandoned journey (`completed:false`) is a high-severity finding: the task is blocked or undiscoverable, which no passing e2e suite would ever reveal.
|
|
44
44
|
10. **Walk the auth surface too — anonymously.** Attach a second session WITHOUT a storage-state file (a fresh logged-out profile) and exercise signup, login failure states, and forgot/reset-password **as far as they physically go**. The mailbox wall is expected — reaching "check your email" IS the success condition; everything before it is what you're testing: does submit actually fire (a dead signup button is a high finding), are errors specific and actionable, can the user resend or recover from a typo, does the flow dead-end. Use plausible synthetic identities only (invent `qa-<runid>@example.com`-style addresses, never a real person's), submit each form valid AND invalid, and judge the feedback. Two classic findings live here: a forgot-password that answers "no account with that email" is an **account-enumeration leak** (file as security; "if an account exists, we sent a link" is the correct shape), and a signup that accepts the form then lands on a blank or logged-out page with no guidance is a **journey dead-end**. Signup creates a record, so what this pass may do depends on the mode. In `observe`, fill and submit the auth forms for their CLIENT-SIDE behaviour only: the engine blocks signup, password change and reset, and lets only a login itself go out. Disclose the server-side half as a gap. Actually creating an account needs the user's explicit okay and safe-write mode; the engine tracks the created account like any other creation.
|
|
45
|
-
11. **`scout_coverage` decides what's next** — it lists unvisited routes, unexercised elements, and the options of each dropdown you used that no session has chosen this run (a filter counts as exercised after one choice, and the option you skipped can be the one whose request fails). Trust it over your memory. Prefer reaching routes by clicking real navigation; fall back to direct URLs for coverage completeness and re-verification, and say which you used when it affects the finding (see the provenance rule below).
|
|
45
|
+
11. **`scout_coverage` decides what's next** — it lists unvisited routes, unexercised elements, and the options of each dropdown you used that no session has chosen this run (a filter counts as exercised after one choice, and the option you skipped can be the one whose request fails), and each `<form>` seen this run that no session has submitted with every text field blank (submit it once empty: a submit that silently does nothing — no request, no message — is the commonest defect there, and filling a form in first never finds it). Trust it over your memory. Prefer reaching routes by clicking real navigation; fall back to direct URLs for coverage completeness and re-verification, and say which you used when it affects the finding (see the provenance rule below).
|
|
46
46
|
|
|
47
47
|
## Levels (completion contracts — the engine ENFORCES them via `scout_report {level}`)
|
|
48
48
|
|
|
@@ -72,7 +72,7 @@ Some flows need a TEAM — a document one role submits and another approves, a r
|
|
|
72
72
|
- While roles are NOT collaborating, use each one productively where its permissions matter (admin in /admin surfaces, low-privilege probing for permission leaks) — same coverage contract, different vantage points.
|
|
73
73
|
- **Infer the PERSONA behind each role, and write it down.** From what a role can see and do (its nav, its dashboard, the capability matrix in the report), state what this person is FOR: "qa = reviewer — approves orders, assigns reviewers, no admin" / "user = front-line user — reads documents, completes reviews, raises orders". Record it with `scout_note {section:'roles'}`. Then test the persona's WORLD, not just the permissions: does the operator's landing page serve an operator? Is anything they need N clicks deep? The capability matrix's divergent rows are questions, not verdicts — each is either a correct boundary or a gap ("should this role be able to do this?"); say which you believe it is and why.
|
|
74
74
|
- `scout_close {all: true}` at the end of a multi-role run; `scout_close {session}` to drop one role early.
|
|
75
|
-
- **Several agents in parallel** (subagents or a workflow, each driving its own session): each agent attaches its session when it STARTS, and the PLANNER closes it by name once it has folded that lane's report. A lane must NOT close its own session before reporting: `scout_lane_report` keeps its decisions against that session's project memory, so a lane that closes first hands back decisions with nowhere to write — the tool says so rather than accepting silently, and the run's calibration section ends up with nothing to measure. Fold, then close. Never open sessions ahead for agents that have not started, and never hand an open session from one agent to the next: an agent waiting for its turn should hold no browser. Give each session an `objective` when you attach it (`scout_attach {session, objective:"Approve and reject orders as a manager"}`) and wrap each goal in `scout_journey`: the live view shows that objective and the task it is on beside the session's feed, which is how the person watching knows what every agent is for. Keep that goal TRUE: one journey per goal, one goal per thing you are checking ("Save a settings change as the auditor", not "Check every page"), ended the moment it is decided and the next one started before you move on. A journey that outlives its goal shows the viewer an objective the session left behind minutes ago. Run roughly as many agents at once as the machine has cores, less two, since each drives a real browser; beyond that they only queue. Exploring one area is well within a mid-tier model, so run these agents on one (Sonnet or its equivalent in your client) unless the user names a model; keep the larger model for the agent that plans the split and writes the report. No agent may call `scout_close {all: true}` while others run — only the last step, once every agent has finished.
|
|
75
|
+
- **Several agents in parallel** (subagents or a workflow, each driving its own session): each agent attaches its session when it STARTS, and the PLANNER closes it by name once it has folded that lane's report. A lane must NOT close its own session before reporting: `scout_lane_report` keeps its decisions against that session's project memory, so a lane that closes first hands back decisions with nowhere to write — the tool says so rather than accepting silently, and the run's calibration section ends up with nothing to measure. Fold, then close. `scout_close` enforces it: a session that `scout_lane_brief` or `scout_lane_report` named as a lane is not closed until a report from it has been ACCEPTED (a refused one does not count, so re-send the corrected object before closing), and `scout_close {all: true}` names every such lane and closes nothing. Pass `force: true` only when a lane will never report, knowing its decisions are lost. Keep the planner's own session open until the last lane is folded: closing every session ends the run, and with it the record of which sessions are lanes. Never open sessions ahead for agents that have not started, and never hand an open session from one agent to the next: an agent waiting for its turn should hold no browser. Give each session an `objective` when you attach it (`scout_attach {session, objective:"Approve and reject orders as a manager"}`) and wrap each goal in `scout_journey`: the live view shows that objective and the task it is on beside the session's feed, which is how the person watching knows what every agent is for. Keep that goal TRUE: one journey per goal, one goal per thing you are checking ("Save a settings change as the auditor", not "Check every page"), ended the moment it is decided and the next one started before you move on. A journey that outlives its goal shows the viewer an objective the session left behind minutes ago. Run roughly as many agents at once as the machine has cores, less two, since each drives a real browser; beyond that they only queue. Exploring one area is well within a mid-tier model, so run these agents on one (Sonnet or its equivalent in your client) unless the user names a model; keep the larger model for the agent that plans the split and writes the report. No agent may call `scout_close {all: true}` while others run — only the last step, once every agent has finished.
|
|
76
76
|
- **Let the engine divide the app: `scout_lane_brief {lanes, goal}`.** Call it after the first crawl, when route knowledge is complete. It splits the known routes into whole modules — everything under `/orders` goes to one lane, so that lane carries state between its own steps instead of re-learning the app on every route — deals the modules out so the lanes come out within a route or two of each other, and returns each lane's session name, the `objective` to attach it with, the routes it owns, the route it lands on (its own first route: lanes that all attached on the home page all met its defects first, and several filed the same one), and the rules each lane must follow — file each defect as it is judged, check a list's request status before calling it empty or stuck, submit one markup value on a create form and open where it is listed, probe a withheld control's endpoint, check coverage, and leave the session open for you to fold. Pass `routes` to split a subset instead. Hand each brief to its agent verbatim. Dividing by hand fails in two ways that a finished run cannot tell apart from success: two lanes audit the same register while a third module is never opened, and route coverage reads complete either way; and lanes launch without an objective, so the person watching sees browsers clicking through their app with nothing to say why.
|
|
77
77
|
- **Lanes report in typed decisions, not prose.** Each lane files its findings with `scout_finding` as it goes, so the report already has them; what the lane hands BACK to the planner is one JSON object, the lane report, and nothing else. Get the paragraph to put in a lane's prompt from `scout_lane_report {lane}`: it names the shape (`status`, one decision per observation with `verdict`, `severity`, `category`, a calibrated `confidence` and a machine-signature `evidence`, the `routes` covered, `blocked_by`), every value's closed set (the categories are `scout_finding`'s) and every length limit, because a limit a lane is not told refuses good replies. When the lane hands back, pass its text to `scout_lane_report {lane, reply}`: it returns the one-line fold (defects, highs, unsure, mean confidence, routes, what blocked it) or the reason the reply was refused, and keeps each decision so the confidence the lane stated can be checked against what the run went on to file — the report's calibration section is built from that, and it only appears once enough decisions exist to mean something. **So the confidence is worth stating honestly**: a lane that writes 0.95 on everything makes the section say so. The fold also lists every defect the lane judged that no finding on the project matches yet: have the lane file each with the same `evidence` (a finding filed without evidence cannot be matched), or name the finding that covers it, before you close its session, because a defect that is only in a lane report never reaches the run's report. Prose around ONE fenced JSON block is discarded unread and the report accepted; anything less clear-cut is refused. Relay a refusal to the lane once and ask for the corrected object; never re-judge its prose yourself, and tell every lane that the object IS its final report, since a lane that writes the object and then hands back a summary of it has failed. The planner then folds lanes by counting, not by reading: a lane's reply is a few hundred tokens whatever it found, and a value the schema refuses is caught at the boundary instead of becoming a severity like "Low-Medium" in the report. Where the client lets you set a lane's reasoning effort, use **medium**: in this project's benchmark it judged as well as high in three-quarters of the time and under half of xhigh, low inflated severities, and max took three minutes per lane for no better agreement.
|
|
78
78
|
|