scenescout 3.4.0 → 3.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +15 -0
- package/README.md +38 -12
- package/dist/engine/bench.js +485 -0
- package/dist/engine/brief.js +20 -5
- package/dist/engine/browser.js +84 -27
- package/dist/engine/calibration.js +57 -0
- package/dist/engine/claims.js +10 -3
- package/dist/engine/collector.js +17 -0
- package/dist/engine/lane.js +26 -11
- package/dist/engine/memory.js +29 -0
- package/dist/engine/oracles.js +15 -8
- package/dist/engine/pace.js +57 -11
- package/dist/engine/policy.js +40 -0
- package/dist/engine/report.js +1 -1
- package/dist/engine/request.js +14 -1
- package/dist/mcp-server.js +40 -7
- package/package.json +5 -3
- package/skills/scenescout/SKILL.md +6 -6
package/dist/engine/pace.js
CHANGED
|
@@ -4,7 +4,7 @@
|
|
|
4
4
|
* counting them inflates the action count and drags the median gap toward
|
|
5
5
|
* zero. One real run logged 325 entries of which 107 were stated tasks.
|
|
6
6
|
*/
|
|
7
|
-
const MARKER_ACTIONS = new Set(["task", "attach", "created-resource", "journey:start", "journey:end", "record:full", "record:failed"]);
|
|
7
|
+
const MARKER_ACTIONS = new Set(["task", "attach", "close", "created-resource", "journey:start", "journey:end", "record:full", "record:failed"]);
|
|
8
8
|
/** Whether a log entry represents work the browser actually did. */
|
|
9
9
|
export function isActing(action) {
|
|
10
10
|
return !MARKER_ACTIONS.has(action);
|
|
@@ -27,23 +27,37 @@ function median(sorted) {
|
|
|
27
27
|
export function measurePace(log, nowMs, attached = []) {
|
|
28
28
|
const stillOpen = new Set(attached);
|
|
29
29
|
const bySession = new Map();
|
|
30
|
+
const lifecycle = new Map();
|
|
30
31
|
for (const entry of log) {
|
|
31
|
-
if (!isActing(entry.action))
|
|
32
|
-
continue;
|
|
33
32
|
const name = entry.session ?? "default";
|
|
34
|
-
const
|
|
33
|
+
const into = isActing(entry.action) ? bySession : entry.action === "attach" || entry.action === "close" ? lifecycle : null;
|
|
34
|
+
if (!into)
|
|
35
|
+
continue;
|
|
36
|
+
const list = into.get(name);
|
|
35
37
|
if (list)
|
|
36
38
|
list.push(entry);
|
|
37
39
|
else
|
|
38
|
-
|
|
40
|
+
into.set(name, [entry]);
|
|
39
41
|
}
|
|
42
|
+
// A session that attached and never acted still held a browser.
|
|
43
|
+
for (const name of lifecycle.keys())
|
|
44
|
+
if (!bySession.has(name))
|
|
45
|
+
bySession.set(name, []);
|
|
40
46
|
const sessions = [];
|
|
41
47
|
let first = Number.POSITIVE_INFINITY;
|
|
42
48
|
let last = 0;
|
|
43
49
|
for (const [session, entries] of bySession) {
|
|
44
|
-
const times = entries
|
|
45
|
-
|
|
50
|
+
const times = entries
|
|
51
|
+
.map((e) => Date.parse(e.at))
|
|
52
|
+
.filter((t) => Number.isFinite(t))
|
|
53
|
+
.sort((a, b) => a - b);
|
|
54
|
+
const heldIdleMs = heldIdle(lifecycle.get(session) ?? [], times, nowMs, stillOpen.has(session));
|
|
55
|
+
if (times.length === 0) {
|
|
56
|
+
if (heldIdleMs > 0) {
|
|
57
|
+
sessions.push({ session, actions: 0, spanMs: 0, medianGapMs: 0, maxGapMs: 0, idleShare: 0, quietMs: 0, framed: 0, heldIdleMs });
|
|
58
|
+
}
|
|
46
59
|
continue;
|
|
60
|
+
}
|
|
47
61
|
times.sort((a, b) => a - b);
|
|
48
62
|
first = Math.min(first, times[0]);
|
|
49
63
|
last = Math.max(last, times[times.length - 1]);
|
|
@@ -61,6 +75,7 @@ export function measurePace(log, nowMs, attached = []) {
|
|
|
61
75
|
idleShare: spanMs > 0 ? idleMs / spanMs : 0,
|
|
62
76
|
quietMs: Math.max(0, nowMs - times[times.length - 1]),
|
|
63
77
|
framed: entries.filter((e) => e.frame).length,
|
|
78
|
+
heldIdleMs,
|
|
64
79
|
});
|
|
65
80
|
}
|
|
66
81
|
sessions.sort((a, b) => b.actions - a.actions);
|
|
@@ -75,6 +90,37 @@ export function measurePace(log, nowMs, attached = []) {
|
|
|
75
90
|
quiet: sessions.filter((s) => s.quietMs > STALE_SESSION_MS && stillOpen.has(s.session)).map((s) => s.session),
|
|
76
91
|
};
|
|
77
92
|
}
|
|
93
|
+
/**
|
|
94
|
+
* Idle time at the edges of each attach: attach to first action, last action
|
|
95
|
+
* to close. An attach with no close after it ends at the next attach, or at
|
|
96
|
+
* now while the session is still open. A log written before close was
|
|
97
|
+
* recorded has no close marker; its closed sessions count no trailing idle,
|
|
98
|
+
* because when they closed is unknown.
|
|
99
|
+
*/
|
|
100
|
+
function heldIdle(markers, actionTimes, nowMs, stillOpen) {
|
|
101
|
+
const events = markers
|
|
102
|
+
.map((m) => ({ kind: m.action, at: Date.parse(m.at) }))
|
|
103
|
+
.filter((e) => Number.isFinite(e.at))
|
|
104
|
+
.sort((a, b) => a.at - b.at);
|
|
105
|
+
let idle = 0;
|
|
106
|
+
for (let i = 0; i < events.length; i += 1) {
|
|
107
|
+
if (events[i].kind !== "attach")
|
|
108
|
+
continue;
|
|
109
|
+
const start = events[i].at;
|
|
110
|
+
const next = events.slice(i + 1).find((e) => e.kind === "attach" || e.kind === "close");
|
|
111
|
+
const end = next ? next.at : stillOpen ? nowMs : null;
|
|
112
|
+
const inside = actionTimes.filter((t) => t >= start && (end === null || t <= end));
|
|
113
|
+
if (inside.length === 0) {
|
|
114
|
+
if (end !== null)
|
|
115
|
+
idle += end - start;
|
|
116
|
+
continue;
|
|
117
|
+
}
|
|
118
|
+
idle += inside[0] - start;
|
|
119
|
+
if (end !== null)
|
|
120
|
+
idle += end - inside[inside.length - 1];
|
|
121
|
+
}
|
|
122
|
+
return Math.max(0, idle);
|
|
123
|
+
}
|
|
78
124
|
/** Milliseconds as a person says them: 45s, 4m12s, 1h03m. */
|
|
79
125
|
export function sayDuration(ms) {
|
|
80
126
|
const total = Math.max(0, Math.round(ms / 1000));
|
|
@@ -96,13 +142,13 @@ export function formatPace(pace) {
|
|
|
96
142
|
const lines = [
|
|
97
143
|
`## How the run was paced`,
|
|
98
144
|
``,
|
|
99
|
-
`${pace.actions} action(s) over ${sayDuration(pace.spanMs)}. Stated tasks and attaches are not counted: they take no time. Idle share is time the browser stood still waiting for the agent, not time the engine spent working.`,
|
|
145
|
+
`${pace.actions} action(s) over ${sayDuration(pace.spanMs)}. Stated tasks and attaches are not counted: they take no time. Idle share is time the browser stood still waiting for the agent, not time the engine spent working. Held idle is time a browser was open before a session's first action or after its last one — a lane waiting to be folded, or attached and never used.`,
|
|
100
146
|
``,
|
|
101
|
-
`| Session | Actions | Span | Median gap | Longest gap | Idle | Frames |`,
|
|
102
|
-
|
|
147
|
+
`| Session | Actions | Span | Median gap | Longest gap | Idle | Held idle | Frames |`,
|
|
148
|
+
`|---|---:|---:|---:|---:|---:|---:|---:|`,
|
|
103
149
|
];
|
|
104
150
|
for (const s of pace.sessions) {
|
|
105
|
-
lines.push(`| ${s.session} | ${s.actions} | ${sayDuration(s.spanMs)} | ${sayDuration(s.medianGapMs)} | ${sayDuration(s.maxGapMs)} | ${Math.round(s.idleShare * 100)}% | ${s.framed} |`);
|
|
151
|
+
lines.push(`| ${s.session} | ${s.actions} | ${sayDuration(s.spanMs)} | ${sayDuration(s.medianGapMs)} | ${sayDuration(s.maxGapMs)} | ${Math.round(s.idleShare * 100)}% | ${sayDuration(s.heldIdleMs)} | ${s.framed} |`);
|
|
106
152
|
}
|
|
107
153
|
lines.push(``);
|
|
108
154
|
if (pace.quiet.length > 0) {
|
package/dist/engine/policy.js
CHANGED
|
@@ -185,3 +185,43 @@ export function allowsWrite(mode, method, destructiveWire, owned) {
|
|
|
185
185
|
// PUT/PATCH/DELETE: only in safe-write, only on this run's own records.
|
|
186
186
|
return mode === "safe-write" && owned;
|
|
187
187
|
}
|
|
188
|
+
/**
|
|
189
|
+
* Which blocked requests the write policy answers rather than drops.
|
|
190
|
+
*
|
|
191
|
+
* A dropped request tells the page nothing it would ever meet in production:
|
|
192
|
+
* `fetch` rejects with a network error, and the code that handles a refusal —
|
|
193
|
+
* the branch that should say "couldn't save" — never runs. Answering a script's
|
|
194
|
+
* request with a refusal keeps the server untouched and exercises that branch,
|
|
195
|
+
* so a page that reports a refused save as saved is caught. A navigation (a
|
|
196
|
+
* native form post) is still dropped: answering it would replace the page the
|
|
197
|
+
* user was on with the stand-in body.
|
|
198
|
+
*/
|
|
199
|
+
export function answersWithRefusal(resourceType) {
|
|
200
|
+
return resourceType === "fetch" || resourceType === "xhr";
|
|
201
|
+
}
|
|
202
|
+
/** Header on every refusal the policy writes, so the stand-in can be told from the server's own answer. */
|
|
203
|
+
export const POLICY_REFUSAL_HEADER = "x-scenescout-policy";
|
|
204
|
+
/**
|
|
205
|
+
* The response the write policy sends in the server's place. 403, because the
|
|
206
|
+
* request is forbidden, not failed: a 5xx invites retries, and a 401 is what
|
|
207
|
+
* many apps read as "signed out". The CORS headers let a cross-origin API
|
|
208
|
+
* call read the refusal instead of failing as a network error, which would
|
|
209
|
+
* drop it all over again. `origin` is the request's own Origin header, echoed
|
|
210
|
+
* only when there is one.
|
|
211
|
+
*/
|
|
212
|
+
export function policyRefusal(mode, method, pathname, origin) {
|
|
213
|
+
const headers = { "content-type": "application/json", [POLICY_REFUSAL_HEADER]: `refused; mode=${mode}` };
|
|
214
|
+
if (origin) {
|
|
215
|
+
headers["access-control-allow-origin"] = origin;
|
|
216
|
+
headers["access-control-allow-credentials"] = "true";
|
|
217
|
+
headers["access-control-expose-headers"] = POLICY_REFUSAL_HEADER;
|
|
218
|
+
}
|
|
219
|
+
return {
|
|
220
|
+
status: 403,
|
|
221
|
+
headers,
|
|
222
|
+
body: JSON.stringify({
|
|
223
|
+
error: "Forbidden",
|
|
224
|
+
message: `${method} ${pathname} was refused by the tester's ${mode} write policy. The server never received it.`,
|
|
225
|
+
}),
|
|
226
|
+
};
|
|
227
|
+
}
|
package/dist/engine/report.js
CHANGED
|
@@ -415,7 +415,7 @@ export function generateReport(memory, oracleLog, extras, opts = {}) {
|
|
|
415
415
|
lines.push(`| Route coverage | ${extras.routesVisited}/${extras.routesTotal} |`);
|
|
416
416
|
lines.push(`| States explored | ${cov.states} |`);
|
|
417
417
|
if (extras)
|
|
418
|
-
lines.push(`| Design audits this
|
|
418
|
+
lines.push(`| Design audits this run (all sessions) | ${extras.designAudits} |`);
|
|
419
419
|
lines.push(`| Oracle violations this session | ${oracleLog.length} |`);
|
|
420
420
|
if (extras?.policyAttributed) {
|
|
421
421
|
lines.push(`| Errors caused by the tester's own write-policy blocks (not counted above) | ${extras.policyAttributed} |`);
|
package/dist/engine/request.js
CHANGED
|
@@ -22,6 +22,7 @@
|
|
|
22
22
|
* Everything in this file is pure so it can be table-tested; the one
|
|
23
23
|
* `page.evaluate` lives in browser.ts.
|
|
24
24
|
*/
|
|
25
|
+
import { POLICY_REFUSAL_HEADER } from "./policy.js";
|
|
25
26
|
/** Methods a session may replay. Anything else is refused before it reaches the page. */
|
|
26
27
|
export const REPLAYABLE_METHODS = ["GET", "HEAD", "POST", "PUT", "PATCH", "DELETE", "OPTIONS"];
|
|
27
28
|
/** Most of a response body that reaches the agent. A JSON list can be megabytes; the signature is in the first lines. */
|
|
@@ -114,6 +115,7 @@ export function toReplayResult(raw) {
|
|
|
114
115
|
truncated: raw.full > BODY_MAX,
|
|
115
116
|
ms: raw.ms,
|
|
116
117
|
url: raw.url,
|
|
118
|
+
refusedByPolicy: raw.headers[POLICY_REFUSAL_HEADER] ?? null,
|
|
117
119
|
};
|
|
118
120
|
}
|
|
119
121
|
/** The signature a finding carries for this call: the line that dedups the same refusal across runs. */
|
|
@@ -128,8 +130,19 @@ export function replaySignature(method, url, status) {
|
|
|
128
130
|
}
|
|
129
131
|
return `${method} ${path} ${status}`;
|
|
130
132
|
}
|
|
131
|
-
/**
|
|
133
|
+
/**
|
|
134
|
+
* What the agent reads back. Leads with the signature, because that is what a
|
|
135
|
+
* finding quotes — except when the write policy answered, where there is no
|
|
136
|
+
* server signature to quote. Printing the stand-in's 403 as one would read as
|
|
137
|
+
* the server enforcing a permission it was never asked about.
|
|
138
|
+
*/
|
|
132
139
|
export function formatReplay(method, result) {
|
|
140
|
+
if (result.refusedByPolicy) {
|
|
141
|
+
const path = replaySignature(method, result.url, result.status).replace(/ \d+$/, "");
|
|
142
|
+
return (`REFUSED by the write policy (${result.refusedByPolicy}): ${path} never reached the server.\n` +
|
|
143
|
+
`This is the engine's safety net, not the server's answer, so it says nothing about whether the server enforces this rule. ` +
|
|
144
|
+
`To test that, re-attach in a mode that allows the request on a record this session owns, or leave it out of the finding.`);
|
|
145
|
+
}
|
|
133
146
|
const lines = [replaySignature(method, result.url, result.status) + (result.statusText ? ` ${result.statusText}` : ""), `took ${result.ms} ms`];
|
|
134
147
|
const headers = Object.entries(result.headers);
|
|
135
148
|
if (headers.length > 0)
|
package/dist/mcp-server.js
CHANGED
|
@@ -36,6 +36,7 @@ import { BrowserEngine } from "./engine/browser.js";
|
|
|
36
36
|
import { reapOrphanBrowsers } from "./engine/reaper.js";
|
|
37
37
|
import { FINDING_CATEGORIES, MemoryStore, redactSecrets } from "./engine/memory.js";
|
|
38
38
|
import { LANE_NAME_MAX, laneReportInstruction, parseLaneReport, summarizeLaneReport } from "./engine/lane.js";
|
|
39
|
+
import { MAX_UNFILED_NAMED, unfiledDefects } from "./engine/calibration.js";
|
|
39
40
|
import { SessionQueue, withWatchdog } from "./engine/dispatch.js";
|
|
40
41
|
import { FIXTURE_KINDS } from "./engine/fixtures.js";
|
|
41
42
|
import { feedForSession, LIVE_ENV, writeStatusFile, LIVE_TOKEN_FILE, liveEngines, liveTokenFileName, pidAlive, statusFileName, LiveServer, StatusBoard, } from "./engine/live.js";
|
|
@@ -238,7 +239,7 @@ function reportExtras(eng) {
|
|
|
238
239
|
return {
|
|
239
240
|
routesVisited: all.length - unvisited.length,
|
|
240
241
|
routesTotal: all.length,
|
|
241
|
-
designAudits: eng.designAuditCount,
|
|
242
|
+
designAudits: eng.memory?.auditsThisRun ?? eng.designAuditCount,
|
|
242
243
|
createdResources: eng.createdResources,
|
|
243
244
|
unvisitedRoutes: unvisited,
|
|
244
245
|
mode: eng.mode,
|
|
@@ -555,8 +556,23 @@ server.registerTool("scout_lane_report", {
|
|
|
555
556
|
// makes a later calibration section look wrong rather than absent.
|
|
556
557
|
const note = kept > 0
|
|
557
558
|
? ` (${kept} decision(s) kept for calibration)`
|
|
558
|
-
: ` (nothing kept for calibration — session ${JSON.stringify(lane)} is not attached here, so there is no project memory to write to. Fold a lane report before closing that lane's session.)`;
|
|
559
|
-
|
|
559
|
+
: ` (nothing kept for calibration, and no check that its defects were filed — session ${JSON.stringify(lane)} is not attached here, so there is no project memory to write to or read. Fold a lane report before closing that lane's session.)`;
|
|
560
|
+
// Follow-through: a defect judged and never filed never reaches the
|
|
561
|
+
// report. Checked against every finding on the lane's project, so one
|
|
562
|
+
// the planner or another lane filed counts. Only that project's: the
|
|
563
|
+
// same reason decisions are kept only there.
|
|
564
|
+
const unfiled = owner ? unfiledDefects(parsed.report.decisions, owner.findings) : [];
|
|
565
|
+
const followUp = unfiled.length > 0
|
|
566
|
+
? `\n⚠ ${unfiled.length} judged defect(s) have no finding with matching evidence yet:\n` +
|
|
567
|
+
unfiled
|
|
568
|
+
.slice(0, MAX_UNFILED_NAMED)
|
|
569
|
+
.map((u) => ` · ${u}`)
|
|
570
|
+
.join("\n") +
|
|
571
|
+
(unfiled.length > MAX_UNFILED_NAMED ? `\n … +${unfiled.length - MAX_UNFILED_NAMED} more` : "") +
|
|
572
|
+
`\nFile each with scout_finding (the same evidence), or confirm which finding already covers it, before closing the lane's session. A judged defect that is never filed is not in the report.`
|
|
573
|
+
: "";
|
|
574
|
+
const around = parsed.aroundIgnored ? `\n(The text around the report's JSON block was discarded unread.)` : "";
|
|
575
|
+
return { content: [{ type: "text", text: `Lane report accepted — ${summarizeLaneReport(parsed.report)}${note}${around}${followUp}` }] };
|
|
560
576
|
}
|
|
561
577
|
catch (err) {
|
|
562
578
|
return errorText(err);
|
|
@@ -680,6 +696,12 @@ server.registerTool("scout_attach", {
|
|
|
680
696
|
catch {
|
|
681
697
|
/* conflict detection is best-effort */
|
|
682
698
|
}
|
|
699
|
+
// Moving this session to another project leaves its old one; if it was
|
|
700
|
+
// the last session there, that run is over. Re-attaching to the SAME
|
|
701
|
+
// project is the same run, and keeps what the run has learned.
|
|
702
|
+
const previous = eng.memory;
|
|
703
|
+
if (previous && previous !== store && ![...engines.values()].some((e) => e !== eng && e.memory === previous))
|
|
704
|
+
previous.endRun();
|
|
683
705
|
const viewport = viewportWidth && viewportHeight ? { width: viewportWidth, height: viewportHeight } : undefined;
|
|
684
706
|
const out = await eng.attach({
|
|
685
707
|
url,
|
|
@@ -1185,8 +1207,11 @@ server.registerTool("scout_report", {
|
|
|
1185
1207
|
(unvisited.length > 30 ? `\n … +${unvisited.length - 30} more` : "") +
|
|
1186
1208
|
`\n→ Run scout_crawl (no args) to cover them in one call.`);
|
|
1187
1209
|
}
|
|
1188
|
-
|
|
1189
|
-
|
|
1210
|
+
// Counted across every session on this project, not just the one
|
|
1211
|
+
// asking: in a parallel run the lanes audit and the planner reports.
|
|
1212
|
+
const auditsThisRun = eng.memory.auditsThisRun;
|
|
1213
|
+
if (auditsThisRun === 0) {
|
|
1214
|
+
gates.push(`No scout_design_audit was run in this run, by any session — run it on at least one representative page (visual/a11y coverage is part of every level).`);
|
|
1190
1215
|
}
|
|
1191
1216
|
const lvl = level ?? "medium";
|
|
1192
1217
|
const auditedRoutes = Object.values(eng.memory.routeFacts).filter((f) => f.audited).length;
|
|
@@ -1201,7 +1226,7 @@ server.registerTool("scout_report", {
|
|
|
1201
1226
|
const gapList = computeGaps(eng.memory, {
|
|
1202
1227
|
routesVisited: all.length - unvisited.length,
|
|
1203
1228
|
routesTotal: all.length,
|
|
1204
|
-
designAudits:
|
|
1229
|
+
designAudits: auditsThisRun,
|
|
1205
1230
|
unvisitedRoutes: unvisited,
|
|
1206
1231
|
mode: eng.mode,
|
|
1207
1232
|
});
|
|
@@ -1221,11 +1246,13 @@ server.registerTool("scout_report", {
|
|
|
1221
1246
|
history,
|
|
1222
1247
|
routesVisited: all.length - unvisited.length,
|
|
1223
1248
|
routesTotal: all.length,
|
|
1224
|
-
designAudits:
|
|
1249
|
+
designAudits: auditsThisRun,
|
|
1225
1250
|
createdResources: eng.createdResources,
|
|
1226
1251
|
unvisitedRoutes: unvisited,
|
|
1227
1252
|
mode: eng.mode,
|
|
1228
1253
|
policyAttributed: eng.oracleLog.policyAttributed,
|
|
1254
|
+
// Which sessions are still open decides whether a quiet one is holding a browser, and how long its trailing idle runs.
|
|
1255
|
+
attachedSessions: [...engines.keys()],
|
|
1229
1256
|
});
|
|
1230
1257
|
void p;
|
|
1231
1258
|
return text(summary, session);
|
|
@@ -1312,8 +1339,11 @@ server.registerTool("scout_close", {
|
|
|
1312
1339
|
keepReport(e);
|
|
1313
1340
|
for (const name of engines.keys())
|
|
1314
1341
|
live?.dropSession(name);
|
|
1342
|
+
const stores = new Set([...engines.values()].map((e) => e.memory).filter((m) => m !== null && m !== undefined));
|
|
1315
1343
|
await Promise.allSettled([...engines.values()].map((e) => e.close()));
|
|
1316
1344
|
engines.clear();
|
|
1345
|
+
for (const store of stores)
|
|
1346
|
+
store.endRun();
|
|
1317
1347
|
sessionQueue.clear();
|
|
1318
1348
|
board.clear();
|
|
1319
1349
|
lastWriter = null;
|
|
@@ -1330,6 +1360,9 @@ server.registerTool("scout_close", {
|
|
|
1330
1360
|
await eng.close();
|
|
1331
1361
|
const saveError = eng.memory?.lastSaveError;
|
|
1332
1362
|
engines.delete(name);
|
|
1363
|
+
// The last session on this project ends its run.
|
|
1364
|
+
if (eng.memory && ![...engines.values()].some((e) => e.memory === eng.memory))
|
|
1365
|
+
eng.memory.endRun();
|
|
1333
1366
|
sessionQueue.forget(name);
|
|
1334
1367
|
board.remove(name);
|
|
1335
1368
|
if (lastWriter?.session === name)
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "scenescout",
|
|
3
|
-
"version": "3.
|
|
3
|
+
"version": "3.5.0",
|
|
4
4
|
"description": "SceneScout — exploratory UI testing for AI coding agents. An MCP server that gives any agent (Claude Code, Cursor, VS Code Copilot, Codex, Gemini CLI and others) a structured view of a running web app, always-on oracles, a network-level write policy, memory across runs and a gap-checked report.",
|
|
5
5
|
"license": "MIT",
|
|
6
6
|
"author": "brunoboto96",
|
|
@@ -71,7 +71,7 @@
|
|
|
71
71
|
"mcp-check": "npm run build && npm run mcp-check:run",
|
|
72
72
|
"mcp-check:run": "tsx scripts/mcp-check.ts",
|
|
73
73
|
"test": "npm run build && npm run test:unit && npm run smoke:run && npm run mcp-check:run",
|
|
74
|
-
"test:unit": "npm run scan-test && npm run oracle-test && npm run policy-test && npm run fixture-test && npm run dispatch-test && npm run design-test && npm run contract-test && npm run pace-test && npm run request-test && npm run settle-test && npm run claims-test && npm run verify-test && npm run brief-test && npm run lane-test && npm run calibration-test && npm run memory-test && npm run install-test && npm run live-test && npm run demo-test && npm run hygiene-test",
|
|
74
|
+
"test:unit": "npm run scan-test && npm run oracle-test && npm run policy-test && npm run fixture-test && npm run dispatch-test && npm run design-test && npm run contract-test && npm run pace-test && npm run request-test && npm run settle-test && npm run claims-test && npm run verify-test && npm run brief-test && npm run lane-test && npm run calibration-test && npm run bench-test && npm run memory-test && npm run install-test && npm run live-test && npm run demo-test && npm run hygiene-test",
|
|
75
75
|
"scan-test": "tsx scripts/scan-test.ts",
|
|
76
76
|
"oracle-test": "tsx --test scripts/oracle-test.ts",
|
|
77
77
|
"policy-test": "tsx --test scripts/policy-test.ts",
|
|
@@ -91,7 +91,9 @@
|
|
|
91
91
|
"claims-test": "tsx --test scripts/claims-test.ts",
|
|
92
92
|
"verify-test": "tsx --test scripts/verify-test.ts",
|
|
93
93
|
"brief-test": "tsx --test scripts/brief-test.ts",
|
|
94
|
-
"calibration-test": "tsx --test scripts/calibration-test.ts"
|
|
94
|
+
"calibration-test": "tsx --test scripts/calibration-test.ts",
|
|
95
|
+
"bench": "tsx scripts/bench.ts",
|
|
96
|
+
"bench-test": "tsx --test scripts/bench-test.ts"
|
|
95
97
|
},
|
|
96
98
|
"dependencies": {
|
|
97
99
|
"@modelcontextprotocol/sdk": "^1.12.0",
|
|
@@ -25,7 +25,7 @@ You are the brain of an exploratory UI tester. The SceneScout MCP server gives y
|
|
|
25
25
|
- `read-only` (default when the source is here, i.e. a local or dev app): destructive labels blocked in the UI AND all PUT/PATCH/DELETE + destructive POSTs blocked on the wire. Ordinary form POSTs still go through. Use unless told otherwise.
|
|
26
26
|
- `safe-write` (`--safe-write`, or the user asks to test creating/editing things): create freely — **prioritize testing CREATE flows** — then edit/delete ONLY the records you created (the engine tracks your creations and blocks mutations on anything else). Never attempt to clean up or modify pre-existing data.
|
|
27
27
|
- `destructive` (`--allow-destructive` only): everything allowed. Requires the user to explicitly confirm the environment is disposable/seeded. Never decide this yourself.
|
|
28
|
-
A `🛡 WRITE-POLICY blocked` notice in a tool result is the engine's safety net, NOT an app bug — never file a finding for the error UI it causes; note it and move on (or suggest the user re-run with a laxer mode if that flow matters).
|
|
28
|
+
A `🛡 WRITE-POLICY blocked` notice in a tool result is the engine's safety net, NOT an app bug — never file a finding for the error UI it causes; note it and move on (or suggest the user re-run with a laxer mode if that flow matters). The page's own requests are answered with a 403 in the server's place, so the page's handling of a refusal is real: an error message after a block is correct behaviour, but a page that claims the change was saved or deleted is lying to its user — the engine reports that as `false_success`, and it IS a finding.
|
|
29
29
|
**`⚠ AUTH FAILED` on attach means the storage state is stale** — its token has expired and the browser is sitting on a login page. Stop and ask the user to regenerate it (usually their Playwright auth-setup project); do not explore, and do not file findings from a logged-out session. The message quotes whatever an earlier run wrote down about regenerating it, so relay that first rather than asking from scratch — and the moment the user tells you what actually worked, write it down with `scout_note {section:'setup', note}` (the exact command, what has to be on PATH, how long a token lasts). A login state expires on a timer nobody remembers, and this is the single learning a run most reliably pays for and then forgets. Mid-run, `⚠ SESSION AUTH LOST` means the same thing happened after N navigations: everything since is meaningless, so re-attach rather than pressing on. Routes bounced this way are recorded as NOT covered, so the gap ledger will still show them.
|
|
30
30
|
6. **`scout_note {action:'read'}` immediately after attaching.** ASSUMPTIONS.md is the run-over-run written memory: what the app is, who each role is FOR, constraints discovered the hard way ("an order can only ship once approved"). Starting without reading it means re-learning what a previous run already paid to find out. Throughout the run, `scout_note {action:'add'}` every DURABLE learning the moment you confirm it — app model, role personas, conventions, constraints, risks, glossary. Not session facts (ids, counts); knowledge a future run should start with.
|
|
31
31
|
|
|
@@ -72,9 +72,9 @@ Some flows need a TEAM — a document one role submits and another approves, a r
|
|
|
72
72
|
- While roles are NOT collaborating, use each one productively where its permissions matter (admin in /admin surfaces, low-privilege probing for permission leaks) — same coverage contract, different vantage points.
|
|
73
73
|
- **Infer the PERSONA behind each role, and write it down.** From what a role can see and do (its nav, its dashboard, the capability matrix in the report), state what this person is FOR: "qa = reviewer — approves orders, assigns reviewers, no admin" / "user = front-line user — reads documents, completes reviews, raises orders". Record it with `scout_note {section:'roles'}`. Then test the persona's WORLD, not just the permissions: does the operator's landing page serve an operator? Is anything they need N clicks deep? The capability matrix's divergent rows are questions, not verdicts — each is either a correct boundary or a gap ("should this role be able to do this?"); say which you believe it is and why.
|
|
74
74
|
- `scout_close {all: true}` at the end of a multi-role run; `scout_close {session}` to drop one role early.
|
|
75
|
-
- **Several agents in parallel** (subagents or a workflow, each driving its own session): each agent attaches its session when it STARTS and closes it by name
|
|
76
|
-
- **Let the engine divide the app: `scout_lane_brief {lanes, goal}`.** Call it after the first crawl, when route knowledge is complete. It splits the known routes into whole modules — everything under `/orders` goes to one lane, so that lane carries state between its own steps instead of re-learning the app on every route — deals the modules out so the lanes come out within a route or two of each other, and returns each lane's session name, the `objective` to attach it with, and the
|
|
77
|
-
- **Lanes report in typed decisions, not prose.** Each lane files its findings with `scout_finding` as it goes, so the report already has them; what the lane hands BACK to the planner is one JSON object, the lane report, and nothing else. Get the paragraph to put in a lane's prompt from `scout_lane_report {lane}`: it names the shape (`status`, one decision per observation with `verdict`, `severity`, `category`, a calibrated `confidence` and a machine-signature `evidence`, the `routes` covered, `blocked_by`), every value's closed set (the categories are `scout_finding`'s) and every length limit, because a limit a lane is not told refuses good replies. When the lane hands back, pass its text to `scout_lane_report {lane, reply}`: it returns the one-line fold (defects, highs, unsure, mean confidence, routes, what blocked it) or the reason the reply was refused, and keeps each decision so the confidence the lane stated can be checked against what the run went on to file — the report's calibration section is built from that, and it only appears once enough decisions exist to mean something. **So the confidence is worth stating honestly**: a lane that writes 0.95 on everything makes the section say so. Relay a refusal to the lane once and ask for the corrected object; never re-judge its prose yourself, and tell every lane that the object IS its final report, since a lane that writes the object and then hands back a summary of it has failed. The planner then folds lanes by counting, not by reading: a lane's reply is a few hundred tokens whatever it found, and a value the schema refuses is caught at the boundary instead of becoming a severity like "Low-Medium" in the report. Where the client lets you set a lane's reasoning effort, use **medium**: in this project's benchmark it judged as well as high in three-quarters of the time and under half of xhigh, low inflated severities, and max took three minutes per lane for no better agreement.
|
|
75
|
+
- **Several agents in parallel** (subagents or a workflow, each driving its own session): each agent attaches its session when it STARTS, and the PLANNER closes it by name once it has folded that lane's report. A lane must NOT close its own session before reporting: `scout_lane_report` keeps its decisions against that session's project memory, so a lane that closes first hands back decisions with nowhere to write — the tool says so rather than accepting silently, and the run's calibration section ends up with nothing to measure. Fold, then close. Never open sessions ahead for agents that have not started, and never hand an open session from one agent to the next: an agent waiting for its turn should hold no browser. Give each session an `objective` when you attach it (`scout_attach {session, objective:"Approve and reject orders as a manager"}`) and wrap each goal in `scout_journey`: the live view shows that objective and the task it is on beside the session's feed, which is how the person watching knows what every agent is for. Keep that goal TRUE: one journey per goal, one goal per thing you are checking ("Save a settings change as the auditor", not "Check every page"), ended the moment it is decided and the next one started before you move on. A journey that outlives its goal shows the viewer an objective the session left behind minutes ago. Run roughly as many agents at once as the machine has cores, less two, since each drives a real browser; beyond that they only queue. Exploring one area is well within a mid-tier model, so run these agents on one (Sonnet or its equivalent in your client) unless the user names a model; keep the larger model for the agent that plans the split and writes the report. No agent may call `scout_close {all: true}` while others run — only the last step, once every agent has finished.
|
|
76
|
+
- **Let the engine divide the app: `scout_lane_brief {lanes, goal}`.** Call it after the first crawl, when route knowledge is complete. It splits the known routes into whole modules — everything under `/orders` goes to one lane, so that lane carries state between its own steps instead of re-learning the app on every route — deals the modules out so the lanes come out within a route or two of each other, and returns each lane's session name, the `objective` to attach it with, the routes it owns, the route it lands on (its own first route: lanes that all attached on the home page all met its defects first, and several filed the same one), and the rules each lane must follow — file each defect as it is judged, check a list's request status before calling it empty or stuck, submit one markup value on a create form and open where it is listed, probe a withheld control's endpoint, check coverage, and leave the session open for you to fold. Pass `routes` to split a subset instead. Hand each brief to its agent verbatim. Dividing by hand fails in two ways that a finished run cannot tell apart from success: two lanes audit the same register while a third module is never opened, and route coverage reads complete either way; and lanes launch without an objective, so the person watching sees browsers clicking through their app with nothing to say why.
|
|
77
|
+
- **Lanes report in typed decisions, not prose.** Each lane files its findings with `scout_finding` as it goes, so the report already has them; what the lane hands BACK to the planner is one JSON object, the lane report, and nothing else. Get the paragraph to put in a lane's prompt from `scout_lane_report {lane}`: it names the shape (`status`, one decision per observation with `verdict`, `severity`, `category`, a calibrated `confidence` and a machine-signature `evidence`, the `routes` covered, `blocked_by`), every value's closed set (the categories are `scout_finding`'s) and every length limit, because a limit a lane is not told refuses good replies. When the lane hands back, pass its text to `scout_lane_report {lane, reply}`: it returns the one-line fold (defects, highs, unsure, mean confidence, routes, what blocked it) or the reason the reply was refused, and keeps each decision so the confidence the lane stated can be checked against what the run went on to file — the report's calibration section is built from that, and it only appears once enough decisions exist to mean something. **So the confidence is worth stating honestly**: a lane that writes 0.95 on everything makes the section say so. The fold also lists every defect the lane judged that no finding on the project matches yet: have the lane file each with the same `evidence` (a finding filed without evidence cannot be matched), or name the finding that covers it, before you close its session, because a defect that is only in a lane report never reaches the run's report. Prose around ONE fenced JSON block is discarded unread and the report accepted; anything less clear-cut is refused. Relay a refusal to the lane once and ask for the corrected object; never re-judge its prose yourself, and tell every lane that the object IS its final report, since a lane that writes the object and then hands back a summary of it has failed. The planner then folds lanes by counting, not by reading: a lane's reply is a few hundred tokens whatever it found, and a value the schema refuses is caught at the boundary instead of becoming a severity like "Low-Medium" in the report. Where the client lets you set a lane's reasoning effort, use **medium**: in this project's benchmark it judged as well as high in three-quarters of the time and under half of xhigh, low inflated severities, and max took three minutes per lane for no better agreement.
|
|
78
78
|
|
|
79
79
|
## The impatient-user pass (extensive)
|
|
80
80
|
|
|
@@ -89,10 +89,10 @@ The engine is self-healing (orphaned browsers reaped, wedged calls time out with
|
|
|
89
89
|
|
|
90
90
|
## Judgment (what the engine can't do)
|
|
91
91
|
|
|
92
|
-
- **A hidden or disabled control proves nothing: `scout_request` is how a refusal becomes evidence.** Any finding that a role CANNOT do something must show the server refusing it, not just the UI withholding the button — a control hidden client-side is a decision the browser made, and the endpoint behind it may answer 200 to anyone who asks. `scout_request {session, method, path}` calls the app's own API as that session, with the UI bypassed: it runs inside the page, so it carries the session's cookies, replays the Authorization header the app itself last sent, and meets the same write policy a click does (in safe-write a mutation on a record the session did not create is refused here too,
|
|
92
|
+
- **A hidden or disabled control proves nothing: `scout_request` is how a refusal becomes evidence.** Any finding that a role CANNOT do something must show the server refusing it, not just the UI withholding the button — a control hidden client-side is a decision the browser made, and the endpoint behind it may answer 200 to anyone who asks. `scout_request {session, method, path}` calls the app's own API as that session, with the UI bypassed: it runs inside the page, so it carries the session's cookies, replays the Authorization header the app itself last sent, and meets the same write policy a click does (in safe-write a mutation on a record the session did not create is refused here too, and the result says `REFUSED by the write policy` instead of a status: the server was never asked, so it is not evidence either way and never a finding). It returns the signature to quote (`GET /api/admin/users 403`), the timing, the headers that decide whether two responses are genuinely identical, and the body. Use it for every permission boundary, for comparing one route across roles, and for confirming a 402/403 the UI turned into a friendly empty state. Paths are fenced to the attached origin. Unlike a shell call, every request is recorded in the run's trail and shows up in the report.
|
|
93
93
|
- **Ground absence-claims in the source before filing.** Black-box behaviour tells you what happened, never *why*. Any finding asserting a control is MISSING ("no permission check", "no validation", "no segregation of duties", "not implemented") must be checked against the codebase first — you have Read/Grep/Bash, use them. Three outcomes, all valuable: (1) genuinely absent → file it with the file:line you checked, which makes it actionable instead of speculative; (2) implemented but not reached on your path → you found a *routing/config* bug, a different and often better finding; (3) implemented elsewhere in the same codebase but not here → **the strongest kind of finding**, an inconsistency against the team's own established pattern, with the reference implementation attached as the fix. A grep costs seconds and is the difference between "I think this is wrong" and "here is the line". Behaviour-only findings (a crash, a 500, a dead end) need no such check — the evidence is self-contained.
|
|
94
94
|
- Oracle violations are *evidence* — reproduce/contextualize, then `scout_finding`. **Severity anchors:** crash, data loss, security/permission breach, or a flow the user cannot complete or escape (infinite spinner, no-way-out page, broken primary journey) = **high** — a dead end is high even without a crash. Broken but recoverable = medium. Polish/friction = low.
|
|
95
|
-
- **A `refused_empty` or `false_success` violation is the highest-value thing the engine hands you — file it, always.** Both mean the page CONTRADICTED the server. `refused_empty`: a list request was refused (401/403/5xx) and the page showed its empty state with no error, so the user is told they have nothing when the truth is that nothing could be loaded — this is how a permission regression ships without anyone noticing, and it is a `data-inconsistency` or `security` finding at **high**, not a cosmetic one. `false_success`: a save was refused and the page said it worked, so the user walks away believing their change was kept — **high**, `data-inconsistency`. The engine only raises either when a request genuinely failed AND the page admitted nothing, so do not re-litigate whether the refusal happened; reproduce it, name the endpoint and the message the user saw, and read the handler in the source to say which branch swallowed the response. A page that shows an error is correct behaviour and never raises these — quote that contrast in the finding, since it is usually the fix.
|
|
95
|
+
- **A `refused_empty` or `false_success` violation is the highest-value thing the engine hands you — file it, always.** Both mean the page CONTRADICTED the server. `refused_empty`: a list request was refused (401/403/5xx) and the page showed its empty state with no error, so the user is told they have nothing when the truth is that nothing could be loaded — this is how a permission regression ships without anyone noticing, and it is a `data-inconsistency` or `security` finding at **high**, not a cosmetic one. `false_success`: a save was refused and the page said it worked, so the user walks away believing their change was kept — **high**, `data-inconsistency`. The engine only raises either when a request genuinely failed AND the page admitted nothing, so do not re-litigate whether the refusal happened; reproduce it, name the endpoint and the message the user saw, and read the handler in the source to say which branch swallowed the response. A page that shows an error is correct behaviour and never raises these — quote that contrast in the finding, since it is usually the fix. When the violation says the refusal was the write policy's **stand-in**, the server was never asked: the finding is about the page's handling of a refusal (`data-inconsistency`), not about the server's permissions, so never file it as a permission regression or `security`.
|
|
96
96
|
- **Provenance for deep-link findings:** if you reached a state by direct URL (crawl or scout_navigate) and it looks broken (no app shell, dead end), check whether real users can reach it through the UI before filing — many apps render tab content at internal URLs that users only ever reach via `?tab=` links. Always state in the finding how the state was reached.
|
|
97
97
|
- Tabs and views (`?tab=…`) are distinct states in coverage — prefer clicking the actual tab controls when testing a flow; deep-link only to re-verify.
|
|
98
98
|
- **Lifecycle:** if the user says something is fixed, or the evidence no longer reproduces on a route you re-tested, mark it with `scout_resolve {id}` (spelled `findingId` if you prefer the long form). Historical findings in the report are unverified — don't re-report them as new.
|