scenescout 3.4.0 → 3.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +21 -0
- package/README.md +38 -12
- package/dist/engine/bench.js +485 -0
- package/dist/engine/brief.js +20 -5
- package/dist/engine/browser.js +84 -27
- package/dist/engine/calibration.js +57 -0
- package/dist/engine/claims.js +10 -3
- package/dist/engine/collector.js +17 -0
- package/dist/engine/lane.js +26 -11
- package/dist/engine/memory.js +33 -1
- package/dist/engine/oracles.js +15 -8
- package/dist/engine/pace.js +91 -11
- package/dist/engine/policy.js +40 -0
- package/dist/engine/report.js +1 -1
- package/dist/engine/request.js +14 -1
- package/dist/mcp-server.js +43 -7
- package/package.json +5 -3
- package/skills/scenescout/SKILL.md +6 -6
package/dist/engine/pace.js
CHANGED
|
@@ -4,7 +4,7 @@
|
|
|
4
4
|
* counting them inflates the action count and drags the median gap toward
|
|
5
5
|
* zero. One real run logged 325 entries of which 107 were stated tasks.
|
|
6
6
|
*/
|
|
7
|
-
const MARKER_ACTIONS = new Set(["task", "attach", "created-resource", "journey:start", "journey:end", "record:full", "record:failed"]);
|
|
7
|
+
const MARKER_ACTIONS = new Set(["task", "attach", "close", "lane-report", "created-resource", "journey:start", "journey:end", "record:full", "record:failed"]);
|
|
8
8
|
/** Whether a log entry represents work the browser actually did. */
|
|
9
9
|
export function isActing(action) {
|
|
10
10
|
return !MARKER_ACTIONS.has(action);
|
|
@@ -27,23 +27,41 @@ function median(sorted) {
|
|
|
27
27
|
export function measurePace(log, nowMs, attached = []) {
|
|
28
28
|
const stillOpen = new Set(attached);
|
|
29
29
|
const bySession = new Map();
|
|
30
|
+
const lifecycle = new Map();
|
|
30
31
|
for (const entry of log) {
|
|
31
|
-
if (!isActing(entry.action))
|
|
32
|
-
continue;
|
|
33
32
|
const name = entry.session ?? "default";
|
|
34
|
-
const
|
|
33
|
+
const into = isActing(entry.action)
|
|
34
|
+
? bySession
|
|
35
|
+
: entry.action === "attach" || entry.action === "close" || entry.action === "lane-report"
|
|
36
|
+
? lifecycle
|
|
37
|
+
: null;
|
|
38
|
+
if (!into)
|
|
39
|
+
continue;
|
|
40
|
+
const list = into.get(name);
|
|
35
41
|
if (list)
|
|
36
42
|
list.push(entry);
|
|
37
43
|
else
|
|
38
|
-
|
|
44
|
+
into.set(name, [entry]);
|
|
39
45
|
}
|
|
46
|
+
// A session that attached and never acted still held a browser.
|
|
47
|
+
for (const name of lifecycle.keys())
|
|
48
|
+
if (!bySession.has(name))
|
|
49
|
+
bySession.set(name, []);
|
|
40
50
|
const sessions = [];
|
|
41
51
|
let first = Number.POSITIVE_INFINITY;
|
|
42
52
|
let last = 0;
|
|
43
53
|
for (const [session, entries] of bySession) {
|
|
44
|
-
const times = entries
|
|
45
|
-
|
|
54
|
+
const times = entries
|
|
55
|
+
.map((e) => Date.parse(e.at))
|
|
56
|
+
.filter((t) => Number.isFinite(t))
|
|
57
|
+
.sort((a, b) => a - b);
|
|
58
|
+
const edge = edges(lifecycle.get(session) ?? [], times, nowMs, stillOpen.has(session));
|
|
59
|
+
if (times.length === 0) {
|
|
60
|
+
if (edge.leadInMs > 0) {
|
|
61
|
+
sessions.push({ session, actions: 0, spanMs: 0, medianGapMs: 0, maxGapMs: 0, idleShare: 0, quietMs: 0, framed: 0, ...edge });
|
|
62
|
+
}
|
|
46
63
|
continue;
|
|
64
|
+
}
|
|
47
65
|
times.sort((a, b) => a - b);
|
|
48
66
|
first = Math.min(first, times[0]);
|
|
49
67
|
last = Math.max(last, times[times.length - 1]);
|
|
@@ -61,10 +79,20 @@ export function measurePace(log, nowMs, attached = []) {
|
|
|
61
79
|
idleShare: spanMs > 0 ? idleMs / spanMs : 0,
|
|
62
80
|
quietMs: Math.max(0, nowMs - times[times.length - 1]),
|
|
63
81
|
framed: entries.filter((e) => e.frame).length,
|
|
82
|
+
...edge,
|
|
64
83
|
});
|
|
65
84
|
}
|
|
66
85
|
sessions.sort((a, b) => b.actions - a.actions);
|
|
86
|
+
const folds = sessions.filter((s) => s.afterFoldMs !== null);
|
|
87
|
+
const waiting = {
|
|
88
|
+
workingMs: sessions.reduce((sum, s) => sum + s.spanMs, 0),
|
|
89
|
+
workingIdleMs: sessions.reduce((sum, s) => sum + s.idleShare * s.spanMs, 0),
|
|
90
|
+
leadInMs: sessions.reduce((sum, s) => sum + s.leadInMs, 0),
|
|
91
|
+
afterFinishMs: sessions.reduce((sum, s) => sum + s.afterFinishMs, 0),
|
|
92
|
+
afterFoldMs: folds.length > 0 ? folds.reduce((sum, s) => sum + (s.afterFoldMs ?? 0), 0) : null,
|
|
93
|
+
};
|
|
67
94
|
return {
|
|
95
|
+
waiting,
|
|
68
96
|
sessions,
|
|
69
97
|
spanMs: sessions.length > 0 ? last - first : 0,
|
|
70
98
|
actions: sessions.reduce((sum, x) => sum + x.actions, 0),
|
|
@@ -75,6 +103,46 @@ export function measurePace(log, nowMs, attached = []) {
|
|
|
75
103
|
quiet: sessions.filter((s) => s.quietMs > STALE_SESSION_MS && stillOpen.has(s.session)).map((s) => s.session),
|
|
76
104
|
};
|
|
77
105
|
}
|
|
106
|
+
/**
|
|
107
|
+
* Time at the edges of each attach: attach to first action (lead-in), and last
|
|
108
|
+
* action to close (after finishing). An attach with no close after it ends at
|
|
109
|
+
* the next attach, or at now while the session is still open. A log written
|
|
110
|
+
* before close was recorded has no close marker; its closed sessions count no
|
|
111
|
+
* trailing time, because when they closed is unknown. A fold marker after the
|
|
112
|
+
* last action splits the trailing time at the moment the browser stopped being
|
|
113
|
+
* needed.
|
|
114
|
+
*/
|
|
115
|
+
function edges(markers, actionTimes, nowMs, stillOpen) {
|
|
116
|
+
const events = markers
|
|
117
|
+
.map((m) => ({ kind: m.action, at: Date.parse(m.at) }))
|
|
118
|
+
.filter((e) => Number.isFinite(e.at))
|
|
119
|
+
.sort((a, b) => a.at - b.at);
|
|
120
|
+
let leadInMs = 0;
|
|
121
|
+
let afterFinishMs = 0;
|
|
122
|
+
let afterFoldMs = null;
|
|
123
|
+
for (let i = 0; i < events.length; i += 1) {
|
|
124
|
+
if (events[i].kind !== "attach")
|
|
125
|
+
continue;
|
|
126
|
+
const start = events[i].at;
|
|
127
|
+
const next = events.slice(i + 1).find((e) => e.kind === "attach" || e.kind === "close");
|
|
128
|
+
const end = next ? next.at : stillOpen ? nowMs : null;
|
|
129
|
+
const inside = actionTimes.filter((t) => t >= start && (end === null || t <= end));
|
|
130
|
+
if (inside.length === 0) {
|
|
131
|
+
if (end !== null)
|
|
132
|
+
leadInMs += end - start;
|
|
133
|
+
continue;
|
|
134
|
+
}
|
|
135
|
+
leadInMs += inside[0] - start;
|
|
136
|
+
if (end === null)
|
|
137
|
+
continue;
|
|
138
|
+
const lastAction = inside[inside.length - 1];
|
|
139
|
+
afterFinishMs += end - lastAction;
|
|
140
|
+
const fold = events.filter((e) => e.kind === "lane-report" && e.at >= lastAction && e.at <= end).at(-1);
|
|
141
|
+
if (fold)
|
|
142
|
+
afterFoldMs = (afterFoldMs ?? 0) + (end - fold.at);
|
|
143
|
+
}
|
|
144
|
+
return { leadInMs: Math.max(0, leadInMs), afterFinishMs: Math.max(0, afterFinishMs), afterFoldMs };
|
|
145
|
+
}
|
|
78
146
|
/** Milliseconds as a person says them: 45s, 4m12s, 1h03m. */
|
|
79
147
|
export function sayDuration(ms) {
|
|
80
148
|
const total = Math.max(0, Math.round(ms / 1000));
|
|
@@ -93,16 +161,28 @@ export function sayDuration(ms) {
|
|
|
93
161
|
export function formatPace(pace) {
|
|
94
162
|
if (pace.sessions.length === 0)
|
|
95
163
|
return [];
|
|
164
|
+
const w = pace.waiting;
|
|
165
|
+
const pct = (part, whole) => (whole > 0 ? `${Math.round((part / whole) * 100)}%` : "0%");
|
|
96
166
|
const lines = [
|
|
97
167
|
`## How the run was paced`,
|
|
98
168
|
``,
|
|
99
|
-
`${pace.actions} action(s) over ${sayDuration(pace.spanMs)}. Stated tasks and attaches are not counted: they take no time
|
|
169
|
+
`${pace.actions} action(s) over ${sayDuration(pace.spanMs)}. Stated tasks and attaches are not counted: they take no time.`,
|
|
170
|
+
``,
|
|
171
|
+
`**While working** — first action to last, ${sayDuration(w.workingMs)} across sessions — ${pct(w.workingIdleMs, w.workingMs)} was spent in gaps over ${sayDuration(IDLE_GAP_MS)}, the agent thinking at length. That is how closely the sessions kept working.`,
|
|
172
|
+
``,
|
|
173
|
+
`**After finishing**, sessions held their browsers for a further ${sayDuration(w.afterFinishMs)}` +
|
|
174
|
+
(w.afterFoldMs !== null ? `, ${sayDuration(w.afterFoldMs)} of it after their report was already folded` : ``) +
|
|
175
|
+
(w.afterFoldMs !== null
|
|
176
|
+
? `. That is time waiting to be collected and closed: it grows with the number of lanes and with the slowest one, not with how well any lane worked. Closing each lane as soon as its report is folded removes it.`
|
|
177
|
+
: `: time from the last action to close.`) +
|
|
178
|
+
` (Before the first action: ${sayDuration(w.leadInMs)}.)`,
|
|
100
179
|
``,
|
|
101
|
-
`| Session | Actions |
|
|
102
|
-
|
|
180
|
+
`| Session | Actions | Working | Median gap | Longest gap | Idle while working | After finishing | Frames |`,
|
|
181
|
+
`|---|---:|---:|---:|---:|---:|---:|---:|`,
|
|
103
182
|
];
|
|
104
183
|
for (const s of pace.sessions) {
|
|
105
|
-
|
|
184
|
+
const after = sayDuration(s.afterFinishMs) + (s.afterFoldMs !== null ? ` (${sayDuration(s.afterFoldMs)} after fold)` : ``);
|
|
185
|
+
lines.push(`| ${s.session} | ${s.actions} | ${sayDuration(s.spanMs)} | ${sayDuration(s.medianGapMs)} | ${sayDuration(s.maxGapMs)} | ${Math.round(s.idleShare * 100)}% | ${after} | ${s.framed} |`);
|
|
106
186
|
}
|
|
107
187
|
lines.push(``);
|
|
108
188
|
if (pace.quiet.length > 0) {
|
package/dist/engine/policy.js
CHANGED
|
@@ -185,3 +185,43 @@ export function allowsWrite(mode, method, destructiveWire, owned) {
|
|
|
185
185
|
// PUT/PATCH/DELETE: only in safe-write, only on this run's own records.
|
|
186
186
|
return mode === "safe-write" && owned;
|
|
187
187
|
}
|
|
188
|
+
/**
|
|
189
|
+
* Which blocked requests the write policy answers rather than drops.
|
|
190
|
+
*
|
|
191
|
+
* A dropped request tells the page nothing it would ever meet in production:
|
|
192
|
+
* `fetch` rejects with a network error, and the code that handles a refusal —
|
|
193
|
+
* the branch that should say "couldn't save" — never runs. Answering a script's
|
|
194
|
+
* request with a refusal keeps the server untouched and exercises that branch,
|
|
195
|
+
* so a page that reports a refused save as saved is caught. A navigation (a
|
|
196
|
+
* native form post) is still dropped: answering it would replace the page the
|
|
197
|
+
* user was on with the stand-in body.
|
|
198
|
+
*/
|
|
199
|
+
export function answersWithRefusal(resourceType) {
|
|
200
|
+
return resourceType === "fetch" || resourceType === "xhr";
|
|
201
|
+
}
|
|
202
|
+
/** Header on every refusal the policy writes, so the stand-in can be told from the server's own answer. */
|
|
203
|
+
export const POLICY_REFUSAL_HEADER = "x-scenescout-policy";
|
|
204
|
+
/**
|
|
205
|
+
* The response the write policy sends in the server's place. 403, because the
|
|
206
|
+
* request is forbidden, not failed: a 5xx invites retries, and a 401 is what
|
|
207
|
+
* many apps read as "signed out". The CORS headers let a cross-origin API
|
|
208
|
+
* call read the refusal instead of failing as a network error, which would
|
|
209
|
+
* drop it all over again. `origin` is the request's own Origin header, echoed
|
|
210
|
+
* only when there is one.
|
|
211
|
+
*/
|
|
212
|
+
export function policyRefusal(mode, method, pathname, origin) {
|
|
213
|
+
const headers = { "content-type": "application/json", [POLICY_REFUSAL_HEADER]: `refused; mode=${mode}` };
|
|
214
|
+
if (origin) {
|
|
215
|
+
headers["access-control-allow-origin"] = origin;
|
|
216
|
+
headers["access-control-allow-credentials"] = "true";
|
|
217
|
+
headers["access-control-expose-headers"] = POLICY_REFUSAL_HEADER;
|
|
218
|
+
}
|
|
219
|
+
return {
|
|
220
|
+
status: 403,
|
|
221
|
+
headers,
|
|
222
|
+
body: JSON.stringify({
|
|
223
|
+
error: "Forbidden",
|
|
224
|
+
message: `${method} ${pathname} was refused by the tester's ${mode} write policy. The server never received it.`,
|
|
225
|
+
}),
|
|
226
|
+
};
|
|
227
|
+
}
|
package/dist/engine/report.js
CHANGED
|
@@ -415,7 +415,7 @@ export function generateReport(memory, oracleLog, extras, opts = {}) {
|
|
|
415
415
|
lines.push(`| Route coverage | ${extras.routesVisited}/${extras.routesTotal} |`);
|
|
416
416
|
lines.push(`| States explored | ${cov.states} |`);
|
|
417
417
|
if (extras)
|
|
418
|
-
lines.push(`| Design audits this
|
|
418
|
+
lines.push(`| Design audits this run (all sessions) | ${extras.designAudits} |`);
|
|
419
419
|
lines.push(`| Oracle violations this session | ${oracleLog.length} |`);
|
|
420
420
|
if (extras?.policyAttributed) {
|
|
421
421
|
lines.push(`| Errors caused by the tester's own write-policy blocks (not counted above) | ${extras.policyAttributed} |`);
|
package/dist/engine/request.js
CHANGED
|
@@ -22,6 +22,7 @@
|
|
|
22
22
|
* Everything in this file is pure so it can be table-tested; the one
|
|
23
23
|
* `page.evaluate` lives in browser.ts.
|
|
24
24
|
*/
|
|
25
|
+
import { POLICY_REFUSAL_HEADER } from "./policy.js";
|
|
25
26
|
/** Methods a session may replay. Anything else is refused before it reaches the page. */
|
|
26
27
|
export const REPLAYABLE_METHODS = ["GET", "HEAD", "POST", "PUT", "PATCH", "DELETE", "OPTIONS"];
|
|
27
28
|
/** Most of a response body that reaches the agent. A JSON list can be megabytes; the signature is in the first lines. */
|
|
@@ -114,6 +115,7 @@ export function toReplayResult(raw) {
|
|
|
114
115
|
truncated: raw.full > BODY_MAX,
|
|
115
116
|
ms: raw.ms,
|
|
116
117
|
url: raw.url,
|
|
118
|
+
refusedByPolicy: raw.headers[POLICY_REFUSAL_HEADER] ?? null,
|
|
117
119
|
};
|
|
118
120
|
}
|
|
119
121
|
/** The signature a finding carries for this call: the line that dedups the same refusal across runs. */
|
|
@@ -128,8 +130,19 @@ export function replaySignature(method, url, status) {
|
|
|
128
130
|
}
|
|
129
131
|
return `${method} ${path} ${status}`;
|
|
130
132
|
}
|
|
131
|
-
/**
|
|
133
|
+
/**
|
|
134
|
+
* What the agent reads back. Leads with the signature, because that is what a
|
|
135
|
+
* finding quotes — except when the write policy answered, where there is no
|
|
136
|
+
* server signature to quote. Printing the stand-in's 403 as one would read as
|
|
137
|
+
* the server enforcing a permission it was never asked about.
|
|
138
|
+
*/
|
|
132
139
|
export function formatReplay(method, result) {
|
|
140
|
+
if (result.refusedByPolicy) {
|
|
141
|
+
const path = replaySignature(method, result.url, result.status).replace(/ \d+$/, "");
|
|
142
|
+
return (`REFUSED by the write policy (${result.refusedByPolicy}): ${path} never reached the server.\n` +
|
|
143
|
+
`This is the engine's safety net, not the server's answer, so it says nothing about whether the server enforces this rule. ` +
|
|
144
|
+
`To test that, re-attach in a mode that allows the request on a record this session owns, or leave it out of the finding.`);
|
|
145
|
+
}
|
|
133
146
|
const lines = [replaySignature(method, result.url, result.status) + (result.statusText ? ` ${result.statusText}` : ""), `took ${result.ms} ms`];
|
|
134
147
|
const headers = Object.entries(result.headers);
|
|
135
148
|
if (headers.length > 0)
|
package/dist/mcp-server.js
CHANGED
|
@@ -36,6 +36,7 @@ import { BrowserEngine } from "./engine/browser.js";
|
|
|
36
36
|
import { reapOrphanBrowsers } from "./engine/reaper.js";
|
|
37
37
|
import { FINDING_CATEGORIES, MemoryStore, redactSecrets } from "./engine/memory.js";
|
|
38
38
|
import { LANE_NAME_MAX, laneReportInstruction, parseLaneReport, summarizeLaneReport } from "./engine/lane.js";
|
|
39
|
+
import { MAX_UNFILED_NAMED, unfiledDefects } from "./engine/calibration.js";
|
|
39
40
|
import { SessionQueue, withWatchdog } from "./engine/dispatch.js";
|
|
40
41
|
import { FIXTURE_KINDS } from "./engine/fixtures.js";
|
|
41
42
|
import { feedForSession, LIVE_ENV, writeStatusFile, LIVE_TOKEN_FILE, liveEngines, liveTokenFileName, pidAlive, statusFileName, LiveServer, StatusBoard, } from "./engine/live.js";
|
|
@@ -238,7 +239,7 @@ function reportExtras(eng) {
|
|
|
238
239
|
return {
|
|
239
240
|
routesVisited: all.length - unvisited.length,
|
|
240
241
|
routesTotal: all.length,
|
|
241
|
-
designAudits: eng.designAuditCount,
|
|
242
|
+
designAudits: eng.memory?.auditsThisRun ?? eng.designAuditCount,
|
|
242
243
|
createdResources: eng.createdResources,
|
|
243
244
|
unvisitedRoutes: unvisited,
|
|
244
245
|
mode: eng.mode,
|
|
@@ -545,6 +546,9 @@ server.registerTool("scout_lane_report", {
|
|
|
545
546
|
// already closed makes that the ordinary case, not an edge one.
|
|
546
547
|
const owner = engines.get(lane)?.memory;
|
|
547
548
|
const at = new Date().toISOString();
|
|
549
|
+
// Marks the moment the lane's browser stopped being needed, so the pace
|
|
550
|
+
// section can tell a lane still reporting from one waiting to be closed.
|
|
551
|
+
owner?.logAction({ action: "lane-report", url: engines.get(lane)?.currentUrl ?? "", session: lane });
|
|
548
552
|
const kept = owner
|
|
549
553
|
? owner.addLaneDecisions(lane, parsed.report.decisions.map((d) => ({ ...d, lane, at })))
|
|
550
554
|
: 0;
|
|
@@ -555,8 +559,23 @@ server.registerTool("scout_lane_report", {
|
|
|
555
559
|
// makes a later calibration section look wrong rather than absent.
|
|
556
560
|
const note = kept > 0
|
|
557
561
|
? ` (${kept} decision(s) kept for calibration)`
|
|
558
|
-
: ` (nothing kept for calibration — session ${JSON.stringify(lane)} is not attached here, so there is no project memory to write to. Fold a lane report before closing that lane's session.)`;
|
|
559
|
-
|
|
562
|
+
: ` (nothing kept for calibration, and no check that its defects were filed — session ${JSON.stringify(lane)} is not attached here, so there is no project memory to write to or read. Fold a lane report before closing that lane's session.)`;
|
|
563
|
+
// Follow-through: a defect judged and never filed never reaches the
|
|
564
|
+
// report. Checked against every finding on the lane's project, so one
|
|
565
|
+
// the planner or another lane filed counts. Only that project's: the
|
|
566
|
+
// same reason decisions are kept only there.
|
|
567
|
+
const unfiled = owner ? unfiledDefects(parsed.report.decisions, owner.findings) : [];
|
|
568
|
+
const followUp = unfiled.length > 0
|
|
569
|
+
? `\n⚠ ${unfiled.length} judged defect(s) have no finding with matching evidence yet:\n` +
|
|
570
|
+
unfiled
|
|
571
|
+
.slice(0, MAX_UNFILED_NAMED)
|
|
572
|
+
.map((u) => ` · ${u}`)
|
|
573
|
+
.join("\n") +
|
|
574
|
+
(unfiled.length > MAX_UNFILED_NAMED ? `\n … +${unfiled.length - MAX_UNFILED_NAMED} more` : "") +
|
|
575
|
+
`\nFile each with scout_finding (the same evidence), or confirm which finding already covers it, before closing the lane's session. A judged defect that is never filed is not in the report.`
|
|
576
|
+
: "";
|
|
577
|
+
const around = parsed.aroundIgnored ? `\n(The text around the report's JSON block was discarded unread.)` : "";
|
|
578
|
+
return { content: [{ type: "text", text: `Lane report accepted — ${summarizeLaneReport(parsed.report)}${note}${around}${followUp}` }] };
|
|
560
579
|
}
|
|
561
580
|
catch (err) {
|
|
562
581
|
return errorText(err);
|
|
@@ -680,6 +699,12 @@ server.registerTool("scout_attach", {
|
|
|
680
699
|
catch {
|
|
681
700
|
/* conflict detection is best-effort */
|
|
682
701
|
}
|
|
702
|
+
// Moving this session to another project leaves its old one; if it was
|
|
703
|
+
// the last session there, that run is over. Re-attaching to the SAME
|
|
704
|
+
// project is the same run, and keeps what the run has learned.
|
|
705
|
+
const previous = eng.memory;
|
|
706
|
+
if (previous && previous !== store && ![...engines.values()].some((e) => e !== eng && e.memory === previous))
|
|
707
|
+
previous.endRun();
|
|
683
708
|
const viewport = viewportWidth && viewportHeight ? { width: viewportWidth, height: viewportHeight } : undefined;
|
|
684
709
|
const out = await eng.attach({
|
|
685
710
|
url,
|
|
@@ -1185,8 +1210,11 @@ server.registerTool("scout_report", {
|
|
|
1185
1210
|
(unvisited.length > 30 ? `\n … +${unvisited.length - 30} more` : "") +
|
|
1186
1211
|
`\n→ Run scout_crawl (no args) to cover them in one call.`);
|
|
1187
1212
|
}
|
|
1188
|
-
|
|
1189
|
-
|
|
1213
|
+
// Counted across every session on this project, not just the one
|
|
1214
|
+
// asking: in a parallel run the lanes audit and the planner reports.
|
|
1215
|
+
const auditsThisRun = eng.memory.auditsThisRun;
|
|
1216
|
+
if (auditsThisRun === 0) {
|
|
1217
|
+
gates.push(`No scout_design_audit was run in this run, by any session — run it on at least one representative page (visual/a11y coverage is part of every level).`);
|
|
1190
1218
|
}
|
|
1191
1219
|
const lvl = level ?? "medium";
|
|
1192
1220
|
const auditedRoutes = Object.values(eng.memory.routeFacts).filter((f) => f.audited).length;
|
|
@@ -1201,7 +1229,7 @@ server.registerTool("scout_report", {
|
|
|
1201
1229
|
const gapList = computeGaps(eng.memory, {
|
|
1202
1230
|
routesVisited: all.length - unvisited.length,
|
|
1203
1231
|
routesTotal: all.length,
|
|
1204
|
-
designAudits:
|
|
1232
|
+
designAudits: auditsThisRun,
|
|
1205
1233
|
unvisitedRoutes: unvisited,
|
|
1206
1234
|
mode: eng.mode,
|
|
1207
1235
|
});
|
|
@@ -1221,11 +1249,13 @@ server.registerTool("scout_report", {
|
|
|
1221
1249
|
history,
|
|
1222
1250
|
routesVisited: all.length - unvisited.length,
|
|
1223
1251
|
routesTotal: all.length,
|
|
1224
|
-
designAudits:
|
|
1252
|
+
designAudits: auditsThisRun,
|
|
1225
1253
|
createdResources: eng.createdResources,
|
|
1226
1254
|
unvisitedRoutes: unvisited,
|
|
1227
1255
|
mode: eng.mode,
|
|
1228
1256
|
policyAttributed: eng.oracleLog.policyAttributed,
|
|
1257
|
+
// Which sessions are still open decides whether a quiet one is holding a browser, and how long its trailing idle runs.
|
|
1258
|
+
attachedSessions: [...engines.keys()],
|
|
1229
1259
|
});
|
|
1230
1260
|
void p;
|
|
1231
1261
|
return text(summary, session);
|
|
@@ -1312,8 +1342,11 @@ server.registerTool("scout_close", {
|
|
|
1312
1342
|
keepReport(e);
|
|
1313
1343
|
for (const name of engines.keys())
|
|
1314
1344
|
live?.dropSession(name);
|
|
1345
|
+
const stores = new Set([...engines.values()].map((e) => e.memory).filter((m) => m !== null && m !== undefined));
|
|
1315
1346
|
await Promise.allSettled([...engines.values()].map((e) => e.close()));
|
|
1316
1347
|
engines.clear();
|
|
1348
|
+
for (const store of stores)
|
|
1349
|
+
store.endRun();
|
|
1317
1350
|
sessionQueue.clear();
|
|
1318
1351
|
board.clear();
|
|
1319
1352
|
lastWriter = null;
|
|
@@ -1330,6 +1363,9 @@ server.registerTool("scout_close", {
|
|
|
1330
1363
|
await eng.close();
|
|
1331
1364
|
const saveError = eng.memory?.lastSaveError;
|
|
1332
1365
|
engines.delete(name);
|
|
1366
|
+
// The last session on this project ends its run.
|
|
1367
|
+
if (eng.memory && ![...engines.values()].some((e) => e.memory === eng.memory))
|
|
1368
|
+
eng.memory.endRun();
|
|
1333
1369
|
sessionQueue.forget(name);
|
|
1334
1370
|
board.remove(name);
|
|
1335
1371
|
if (lastWriter?.session === name)
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "scenescout",
|
|
3
|
-
"version": "3.
|
|
3
|
+
"version": "3.6.0",
|
|
4
4
|
"description": "SceneScout — exploratory UI testing for AI coding agents. An MCP server that gives any agent (Claude Code, Cursor, VS Code Copilot, Codex, Gemini CLI and others) a structured view of a running web app, always-on oracles, a network-level write policy, memory across runs and a gap-checked report.",
|
|
5
5
|
"license": "MIT",
|
|
6
6
|
"author": "brunoboto96",
|
|
@@ -71,7 +71,7 @@
|
|
|
71
71
|
"mcp-check": "npm run build && npm run mcp-check:run",
|
|
72
72
|
"mcp-check:run": "tsx scripts/mcp-check.ts",
|
|
73
73
|
"test": "npm run build && npm run test:unit && npm run smoke:run && npm run mcp-check:run",
|
|
74
|
-
"test:unit": "npm run scan-test && npm run oracle-test && npm run policy-test && npm run fixture-test && npm run dispatch-test && npm run design-test && npm run contract-test && npm run pace-test && npm run request-test && npm run settle-test && npm run claims-test && npm run verify-test && npm run brief-test && npm run lane-test && npm run calibration-test && npm run memory-test && npm run install-test && npm run live-test && npm run demo-test && npm run hygiene-test",
|
|
74
|
+
"test:unit": "npm run scan-test && npm run oracle-test && npm run policy-test && npm run fixture-test && npm run dispatch-test && npm run design-test && npm run contract-test && npm run pace-test && npm run request-test && npm run settle-test && npm run claims-test && npm run verify-test && npm run brief-test && npm run lane-test && npm run calibration-test && npm run bench-test && npm run memory-test && npm run install-test && npm run live-test && npm run demo-test && npm run hygiene-test",
|
|
75
75
|
"scan-test": "tsx scripts/scan-test.ts",
|
|
76
76
|
"oracle-test": "tsx --test scripts/oracle-test.ts",
|
|
77
77
|
"policy-test": "tsx --test scripts/policy-test.ts",
|
|
@@ -91,7 +91,9 @@
|
|
|
91
91
|
"claims-test": "tsx --test scripts/claims-test.ts",
|
|
92
92
|
"verify-test": "tsx --test scripts/verify-test.ts",
|
|
93
93
|
"brief-test": "tsx --test scripts/brief-test.ts",
|
|
94
|
-
"calibration-test": "tsx --test scripts/calibration-test.ts"
|
|
94
|
+
"calibration-test": "tsx --test scripts/calibration-test.ts",
|
|
95
|
+
"bench": "tsx scripts/bench.ts",
|
|
96
|
+
"bench-test": "tsx --test scripts/bench-test.ts"
|
|
95
97
|
},
|
|
96
98
|
"dependencies": {
|
|
97
99
|
"@modelcontextprotocol/sdk": "^1.12.0",
|
|
@@ -25,7 +25,7 @@ You are the brain of an exploratory UI tester. The SceneScout MCP server gives y
|
|
|
25
25
|
- `read-only` (default when the source is here, i.e. a local or dev app): destructive labels blocked in the UI AND all PUT/PATCH/DELETE + destructive POSTs blocked on the wire. Ordinary form POSTs still go through. Use unless told otherwise.
|
|
26
26
|
- `safe-write` (`--safe-write`, or the user asks to test creating/editing things): create freely — **prioritize testing CREATE flows** — then edit/delete ONLY the records you created (the engine tracks your creations and blocks mutations on anything else). Never attempt to clean up or modify pre-existing data.
|
|
27
27
|
- `destructive` (`--allow-destructive` only): everything allowed. Requires the user to explicitly confirm the environment is disposable/seeded. Never decide this yourself.
|
|
28
|
-
A `🛡 WRITE-POLICY blocked` notice in a tool result is the engine's safety net, NOT an app bug — never file a finding for the error UI it causes; note it and move on (or suggest the user re-run with a laxer mode if that flow matters).
|
|
28
|
+
A `🛡 WRITE-POLICY blocked` notice in a tool result is the engine's safety net, NOT an app bug — never file a finding for the error UI it causes; note it and move on (or suggest the user re-run with a laxer mode if that flow matters). The page's own requests are answered with a 403 in the server's place, so the page's handling of a refusal is real: an error message after a block is correct behaviour, but a page that claims the change was saved or deleted is lying to its user — the engine reports that as `false_success`, and it IS a finding.
|
|
29
29
|
**`⚠ AUTH FAILED` on attach means the storage state is stale** — its token has expired and the browser is sitting on a login page. Stop and ask the user to regenerate it (usually their Playwright auth-setup project); do not explore, and do not file findings from a logged-out session. The message quotes whatever an earlier run wrote down about regenerating it, so relay that first rather than asking from scratch — and the moment the user tells you what actually worked, write it down with `scout_note {section:'setup', note}` (the exact command, what has to be on PATH, how long a token lasts). A login state expires on a timer nobody remembers, and this is the single learning a run most reliably pays for and then forgets. Mid-run, `⚠ SESSION AUTH LOST` means the same thing happened after N navigations: everything since is meaningless, so re-attach rather than pressing on. Routes bounced this way are recorded as NOT covered, so the gap ledger will still show them.
|
|
30
30
|
6. **`scout_note {action:'read'}` immediately after attaching.** ASSUMPTIONS.md is the run-over-run written memory: what the app is, who each role is FOR, constraints discovered the hard way ("an order can only ship once approved"). Starting without reading it means re-learning what a previous run already paid to find out. Throughout the run, `scout_note {action:'add'}` every DURABLE learning the moment you confirm it — app model, role personas, conventions, constraints, risks, glossary. Not session facts (ids, counts); knowledge a future run should start with.
|
|
31
31
|
|
|
@@ -72,9 +72,9 @@ Some flows need a TEAM — a document one role submits and another approves, a r
|
|
|
72
72
|
- While roles are NOT collaborating, use each one productively where its permissions matter (admin in /admin surfaces, low-privilege probing for permission leaks) — same coverage contract, different vantage points.
|
|
73
73
|
- **Infer the PERSONA behind each role, and write it down.** From what a role can see and do (its nav, its dashboard, the capability matrix in the report), state what this person is FOR: "qa = reviewer — approves orders, assigns reviewers, no admin" / "user = front-line user — reads documents, completes reviews, raises orders". Record it with `scout_note {section:'roles'}`. Then test the persona's WORLD, not just the permissions: does the operator's landing page serve an operator? Is anything they need N clicks deep? The capability matrix's divergent rows are questions, not verdicts — each is either a correct boundary or a gap ("should this role be able to do this?"); say which you believe it is and why.
|
|
74
74
|
- `scout_close {all: true}` at the end of a multi-role run; `scout_close {session}` to drop one role early.
|
|
75
|
-
- **Several agents in parallel** (subagents or a workflow, each driving its own session): each agent attaches its session when it STARTS and closes it by name
|
|
76
|
-
- **Let the engine divide the app: `scout_lane_brief {lanes, goal}`.** Call it after the first crawl, when route knowledge is complete. It splits the known routes into whole modules — everything under `/orders` goes to one lane, so that lane carries state between its own steps instead of re-learning the app on every route — deals the modules out so the lanes come out within a route or two of each other, and returns each lane's session name, the `objective` to attach it with, and the
|
|
77
|
-
- **Lanes report in typed decisions, not prose.** Each lane files its findings with `scout_finding` as it goes, so the report already has them; what the lane hands BACK to the planner is one JSON object, the lane report, and nothing else. Get the paragraph to put in a lane's prompt from `scout_lane_report {lane}`: it names the shape (`status`, one decision per observation with `verdict`, `severity`, `category`, a calibrated `confidence` and a machine-signature `evidence`, the `routes` covered, `blocked_by`), every value's closed set (the categories are `scout_finding`'s) and every length limit, because a limit a lane is not told refuses good replies. When the lane hands back, pass its text to `scout_lane_report {lane, reply}`: it returns the one-line fold (defects, highs, unsure, mean confidence, routes, what blocked it) or the reason the reply was refused, and keeps each decision so the confidence the lane stated can be checked against what the run went on to file — the report's calibration section is built from that, and it only appears once enough decisions exist to mean something. **So the confidence is worth stating honestly**: a lane that writes 0.95 on everything makes the section say so. Relay a refusal to the lane once and ask for the corrected object; never re-judge its prose yourself, and tell every lane that the object IS its final report, since a lane that writes the object and then hands back a summary of it has failed. The planner then folds lanes by counting, not by reading: a lane's reply is a few hundred tokens whatever it found, and a value the schema refuses is caught at the boundary instead of becoming a severity like "Low-Medium" in the report. Where the client lets you set a lane's reasoning effort, use **medium**: in this project's benchmark it judged as well as high in three-quarters of the time and under half of xhigh, low inflated severities, and max took three minutes per lane for no better agreement.
|
|
75
|
+
- **Several agents in parallel** (subagents or a workflow, each driving its own session): each agent attaches its session when it STARTS, and the PLANNER closes it by name once it has folded that lane's report. A lane must NOT close its own session before reporting: `scout_lane_report` keeps its decisions against that session's project memory, so a lane that closes first hands back decisions with nowhere to write — the tool says so rather than accepting silently, and the run's calibration section ends up with nothing to measure. Fold, then close. Never open sessions ahead for agents that have not started, and never hand an open session from one agent to the next: an agent waiting for its turn should hold no browser. Give each session an `objective` when you attach it (`scout_attach {session, objective:"Approve and reject orders as a manager"}`) and wrap each goal in `scout_journey`: the live view shows that objective and the task it is on beside the session's feed, which is how the person watching knows what every agent is for. Keep that goal TRUE: one journey per goal, one goal per thing you are checking ("Save a settings change as the auditor", not "Check every page"), ended the moment it is decided and the next one started before you move on. A journey that outlives its goal shows the viewer an objective the session left behind minutes ago. Run roughly as many agents at once as the machine has cores, less two, since each drives a real browser; beyond that they only queue. Exploring one area is well within a mid-tier model, so run these agents on one (Sonnet or its equivalent in your client) unless the user names a model; keep the larger model for the agent that plans the split and writes the report. No agent may call `scout_close {all: true}` while others run — only the last step, once every agent has finished.
|
|
76
|
+
- **Let the engine divide the app: `scout_lane_brief {lanes, goal}`.** Call it after the first crawl, when route knowledge is complete. It splits the known routes into whole modules — everything under `/orders` goes to one lane, so that lane carries state between its own steps instead of re-learning the app on every route — deals the modules out so the lanes come out within a route or two of each other, and returns each lane's session name, the `objective` to attach it with, the routes it owns, the route it lands on (its own first route: lanes that all attached on the home page all met its defects first, and several filed the same one), and the rules each lane must follow — file each defect as it is judged, check a list's request status before calling it empty or stuck, submit one markup value on a create form and open where it is listed, probe a withheld control's endpoint, check coverage, and leave the session open for you to fold. Pass `routes` to split a subset instead. Hand each brief to its agent verbatim. Dividing by hand fails in two ways that a finished run cannot tell apart from success: two lanes audit the same register while a third module is never opened, and route coverage reads complete either way; and lanes launch without an objective, so the person watching sees browsers clicking through their app with nothing to say why.
|
|
77
|
+
- **Lanes report in typed decisions, not prose.** Each lane files its findings with `scout_finding` as it goes, so the report already has them; what the lane hands BACK to the planner is one JSON object, the lane report, and nothing else. Get the paragraph to put in a lane's prompt from `scout_lane_report {lane}`: it names the shape (`status`, one decision per observation with `verdict`, `severity`, `category`, a calibrated `confidence` and a machine-signature `evidence`, the `routes` covered, `blocked_by`), every value's closed set (the categories are `scout_finding`'s) and every length limit, because a limit a lane is not told refuses good replies. When the lane hands back, pass its text to `scout_lane_report {lane, reply}`: it returns the one-line fold (defects, highs, unsure, mean confidence, routes, what blocked it) or the reason the reply was refused, and keeps each decision so the confidence the lane stated can be checked against what the run went on to file — the report's calibration section is built from that, and it only appears once enough decisions exist to mean something. **So the confidence is worth stating honestly**: a lane that writes 0.95 on everything makes the section say so. The fold also lists every defect the lane judged that no finding on the project matches yet: have the lane file each with the same `evidence` (a finding filed without evidence cannot be matched), or name the finding that covers it, before you close its session, because a defect that is only in a lane report never reaches the run's report. Prose around ONE fenced JSON block is discarded unread and the report accepted; anything less clear-cut is refused. Relay a refusal to the lane once and ask for the corrected object; never re-judge its prose yourself, and tell every lane that the object IS its final report, since a lane that writes the object and then hands back a summary of it has failed. The planner then folds lanes by counting, not by reading: a lane's reply is a few hundred tokens whatever it found, and a value the schema refuses is caught at the boundary instead of becoming a severity like "Low-Medium" in the report. Where the client lets you set a lane's reasoning effort, use **medium**: in this project's benchmark it judged as well as high in three-quarters of the time and under half of xhigh, low inflated severities, and max took three minutes per lane for no better agreement.
|
|
78
78
|
|
|
79
79
|
## The impatient-user pass (extensive)
|
|
80
80
|
|
|
@@ -89,10 +89,10 @@ The engine is self-healing (orphaned browsers reaped, wedged calls time out with
|
|
|
89
89
|
|
|
90
90
|
## Judgment (what the engine can't do)
|
|
91
91
|
|
|
92
|
-
- **A hidden or disabled control proves nothing: `scout_request` is how a refusal becomes evidence.** Any finding that a role CANNOT do something must show the server refusing it, not just the UI withholding the button — a control hidden client-side is a decision the browser made, and the endpoint behind it may answer 200 to anyone who asks. `scout_request {session, method, path}` calls the app's own API as that session, with the UI bypassed: it runs inside the page, so it carries the session's cookies, replays the Authorization header the app itself last sent, and meets the same write policy a click does (in safe-write a mutation on a record the session did not create is refused here too,
|
|
92
|
+
- **A hidden or disabled control proves nothing: `scout_request` is how a refusal becomes evidence.** Any finding that a role CANNOT do something must show the server refusing it, not just the UI withholding the button — a control hidden client-side is a decision the browser made, and the endpoint behind it may answer 200 to anyone who asks. `scout_request {session, method, path}` calls the app's own API as that session, with the UI bypassed: it runs inside the page, so it carries the session's cookies, replays the Authorization header the app itself last sent, and meets the same write policy a click does (in safe-write a mutation on a record the session did not create is refused here too, and the result says `REFUSED by the write policy` instead of a status: the server was never asked, so it is not evidence either way and never a finding). It returns the signature to quote (`GET /api/admin/users 403`), the timing, the headers that decide whether two responses are genuinely identical, and the body. Use it for every permission boundary, for comparing one route across roles, and for confirming a 402/403 the UI turned into a friendly empty state. Paths are fenced to the attached origin. Unlike a shell call, every request is recorded in the run's trail and shows up in the report.
|
|
93
93
|
- **Ground absence-claims in the source before filing.** Black-box behaviour tells you what happened, never *why*. Any finding asserting a control is MISSING ("no permission check", "no validation", "no segregation of duties", "not implemented") must be checked against the codebase first — you have Read/Grep/Bash, use them. Three outcomes, all valuable: (1) genuinely absent → file it with the file:line you checked, which makes it actionable instead of speculative; (2) implemented but not reached on your path → you found a *routing/config* bug, a different and often better finding; (3) implemented elsewhere in the same codebase but not here → **the strongest kind of finding**, an inconsistency against the team's own established pattern, with the reference implementation attached as the fix. A grep costs seconds and is the difference between "I think this is wrong" and "here is the line". Behaviour-only findings (a crash, a 500, a dead end) need no such check — the evidence is self-contained.
|
|
94
94
|
- Oracle violations are *evidence* — reproduce/contextualize, then `scout_finding`. **Severity anchors:** crash, data loss, security/permission breach, or a flow the user cannot complete or escape (infinite spinner, no-way-out page, broken primary journey) = **high** — a dead end is high even without a crash. Broken but recoverable = medium. Polish/friction = low.
|
|
95
|
-
- **A `refused_empty` or `false_success` violation is the highest-value thing the engine hands you — file it, always.** Both mean the page CONTRADICTED the server. `refused_empty`: a list request was refused (401/403/5xx) and the page showed its empty state with no error, so the user is told they have nothing when the truth is that nothing could be loaded — this is how a permission regression ships without anyone noticing, and it is a `data-inconsistency` or `security` finding at **high**, not a cosmetic one. `false_success`: a save was refused and the page said it worked, so the user walks away believing their change was kept — **high**, `data-inconsistency`. The engine only raises either when a request genuinely failed AND the page admitted nothing, so do not re-litigate whether the refusal happened; reproduce it, name the endpoint and the message the user saw, and read the handler in the source to say which branch swallowed the response. A page that shows an error is correct behaviour and never raises these — quote that contrast in the finding, since it is usually the fix.
|
|
95
|
+
- **A `refused_empty` or `false_success` violation is the highest-value thing the engine hands you — file it, always.** Both mean the page CONTRADICTED the server. `refused_empty`: a list request was refused (401/403/5xx) and the page showed its empty state with no error, so the user is told they have nothing when the truth is that nothing could be loaded — this is how a permission regression ships without anyone noticing, and it is a `data-inconsistency` or `security` finding at **high**, not a cosmetic one. `false_success`: a save was refused and the page said it worked, so the user walks away believing their change was kept — **high**, `data-inconsistency`. The engine only raises either when a request genuinely failed AND the page admitted nothing, so do not re-litigate whether the refusal happened; reproduce it, name the endpoint and the message the user saw, and read the handler in the source to say which branch swallowed the response. A page that shows an error is correct behaviour and never raises these — quote that contrast in the finding, since it is usually the fix. When the violation says the refusal was the write policy's **stand-in**, the server was never asked: the finding is about the page's handling of a refusal (`data-inconsistency`), not about the server's permissions, so never file it as a permission regression or `security`.
|
|
96
96
|
- **Provenance for deep-link findings:** if you reached a state by direct URL (crawl or scout_navigate) and it looks broken (no app shell, dead end), check whether real users can reach it through the UI before filing — many apps render tab content at internal URLs that users only ever reach via `?tab=` links. Always state in the finding how the state was reached.
|
|
97
97
|
- Tabs and views (`?tab=…`) are distinct states in coverage — prefer clicking the actual tab controls when testing a flow; deep-link only to re-verify.
|
|
98
98
|
- **Lifecycle:** if the user says something is fixed, or the evidence no longer reproduces on a route you re-tested, mark it with `scout_resolve {id}` (spelled `findingId` if you prefer the long form). Historical findings in the report are unverified — don't re-report them as new.
|