scenescout 3.8.0 → 3.10.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +23 -0
- package/README.md +52 -3
- package/dist/browsers.js +12 -0
- package/dist/check-run.js +76 -0
- package/dist/cli.js +71 -6
- package/dist/engine/browser.js +514 -45
- package/dist/engine/check.js +487 -0
- package/dist/engine/collector.js +194 -25
- package/dist/engine/design.js +24 -8
- package/dist/engine/forms.js +290 -0
- package/dist/engine/lane.js +179 -2
- package/dist/engine/memory.js +70 -3
- package/dist/engine/oracles.js +101 -4
- package/dist/engine/pace.js +14 -1
- package/dist/engine/policy.js +118 -0
- package/dist/engine/report.js +54 -5
- package/dist/mcp-server.js +49 -9
- package/package.json +6 -4
- package/skills/scenescout/SKILL.md +3 -3
package/dist/engine/browser.js
CHANGED
|
@@ -7,7 +7,7 @@ import { normalizeTask } from "./task.js";
|
|
|
7
7
|
import { CLAIM_SCAN_SCRIPT, findContradictions } from "./claims.js";
|
|
8
8
|
import { describeInjection, newInjections, probeQueries, probeScript, probeShape, rememberProbe } from "./injection.js";
|
|
9
9
|
import { AuthLossTracker } from "./authloss.js";
|
|
10
|
-
import { COLLECT_INTERACTABLES_SCRIPT, VISIBLE_SRC, geometryIssues, BROKEN_IMAGES_SCRIPT, brokenImageIssues, frameLines, hasVisibleFrame, displayName, missingName, } from "./collector.js";
|
|
10
|
+
import { COLLECT_INTERACTABLES_SCRIPT, VISIBLE_SRC, geometryIssues, BROKEN_IMAGES_SCRIPT, brokenImageIssues, frameLines, hasVisibleFrame, frameElementKey, frameLabel, masksForeignName, MASKED_NAME, capForeignName, stripForeignHref, frameToPageRect, displayName, missingName, placeholderOnly, placeholderEvidence, describeControl, labelFlag, } from "./collector.js";
|
|
11
11
|
import { OracleMonitor, formatViolations } from "./oracles.js";
|
|
12
12
|
import { extractCreatedIds, isOwnedResource, normalizeId } from "./ownership.js";
|
|
13
13
|
import { formatJourney, measureJourney } from "./journey.js";
|
|
@@ -16,11 +16,12 @@ import { keepWatchingUrl, normalizePace, SETTLE_TICK_MS, shouldKeepWaiting } fro
|
|
|
16
16
|
import { buildRequestScript, formatReplay, replaySignature, requestHeaders, resolveMethod, resolveRequestUrl, toReplayResult } from "./request.js";
|
|
17
17
|
import { defaultEngine, focusAdvanceKey, REMOVE_SHARED_WORKER_SCRIPT, screencastSupport, serviceWorkerPolicy, sharedWorkersAllowed, } from "../browsers.js";
|
|
18
18
|
import { revealedLines } from "./hover.js";
|
|
19
|
+
import { FORMS_INVENTORY_SCRIPT, FORM_PROBE_BODY, formProbeExpression, formIdentity, formStatus, FORMS_READ_FAILED, FORMS_SUBMIT_UNMATCHED, isEmptySubmit, sameControl, isNavigationTeardown, submits, tracksForm, } from "./forms.js";
|
|
19
20
|
import { explainLaunchFailure, isMissingBrowser } from "./launch.js";
|
|
20
21
|
import { ACTION_TIMEOUT_MS, performScroll, probeFocusIndicators, probeOverlays, scrollContainer } from "./probes.js";
|
|
21
22
|
import { BROWSER_MARKER, reapOrphanBrowsers } from "./reaper.js";
|
|
22
23
|
import { planUploadOptions, resolveDiskUpload } from "./uploads.js";
|
|
23
|
-
import { answersWithRefusal, destructiveRefusal, isDestructive, isDestructiveWire, allowsWrite, policyRefusal, foreignFrameOrigin, foreignWrite, withForeignFrameSandbox, offAppPageWrite, EmbedMoveTracker, sandboxedRedirectPage, allowsForeignWriteOnSignIn, isAuthExempt, } from "./policy.js";
|
|
24
|
+
import { answersWithRefusal, destructiveRefusal, isDestructive, isDestructiveWire, allowsWrite, policyRefusal, foreignFrameOrigin, foreignWrite, withForeignFrameSandbox, offAppPageWrite, embedOfRequest, hostileForEmbed, trustedEmbedOrigins, MAX_TRUSTED_EMBEDS, trustsForeignWrite, embedProbeRefusal, EmbedMoveTracker, sandboxedRedirectPage, allowsForeignWriteOnSignIn, isAuthExempt, } from "./policy.js";
|
|
24
25
|
import { scanProject } from "../scan.js";
|
|
25
26
|
import { analyzeDesign, DESIGN_COLLECT_SCRIPT } from "./design.js";
|
|
26
27
|
import { acceptMatches, generatedUpload } from "./fixtures.js";
|
|
@@ -77,8 +78,27 @@ function requestSource(req) {
|
|
|
77
78
|
}
|
|
78
79
|
/** The `why` of a top-window navigation refused as a possible frame escape; the notice words it on its own. */
|
|
79
80
|
const ESCAPE_REFUSAL = "a possible frame escape";
|
|
81
|
+
/** The most frames a snapshot reads controls from. */
|
|
82
|
+
const MAX_READ_FRAMES = 10;
|
|
80
83
|
/** How long a snapshot waits for its frames' elements to answer. */
|
|
81
84
|
const FRAME_READ_MS = 1500;
|
|
85
|
+
/** The coverage key of the page's own element at this xpath (never one in a frame: its xpath names a node in another document). */
|
|
86
|
+
function keyAtXpath(elements, xpath) {
|
|
87
|
+
return elements.find((el) => !el.frame && el.xpath === xpath)?.key ?? null;
|
|
88
|
+
}
|
|
89
|
+
/** The page's own element at the probe's submit xpath, when it really is that control (forms.ts sameControl). */
|
|
90
|
+
function submitControlIn(elements, probe) {
|
|
91
|
+
const el = elements.find((e) => !e.frame && e.xpath === probe.submit);
|
|
92
|
+
return el && sameControl(el, probe) ? el : null;
|
|
93
|
+
}
|
|
94
|
+
/** Page-side: the form an element belongs to, and what its fields hold now (forms.ts). */
|
|
95
|
+
const probeFormOf = new Function("node", FORM_PROBE_BODY);
|
|
96
|
+
/**
|
|
97
|
+
* How long a form read may take: the inventory on every collect, the probe
|
|
98
|
+
* before a submit-capable action. One pass over document.forms takes a few
|
|
99
|
+
* milliseconds; this only stops a wedged page stalling the action.
|
|
100
|
+
*/
|
|
101
|
+
const FORM_READ_MS = 1000;
|
|
82
102
|
/** In-page XPath lookup fragment for string-expression evaluates. */
|
|
83
103
|
function xpathLookup(xpath) {
|
|
84
104
|
return `document.evaluate(${JSON.stringify(xpath)}, document, null, XPathResult.FIRST_ORDERED_NODE_TYPE, null).singleNodeValue`;
|
|
@@ -180,6 +200,9 @@ export class BrowserEngine {
|
|
|
180
200
|
projectDirNote = "";
|
|
181
201
|
memory = null;
|
|
182
202
|
mode = "read-only";
|
|
203
|
+
/** Origins named as trusted embeds (policy.ts trustsEmbedWrite decides when that counts). */
|
|
204
|
+
trustedEmbeds = new Set();
|
|
205
|
+
trustNotice = "";
|
|
183
206
|
/** Human label for the auth identity driving this session (the server sets it from the storage-state filename). */
|
|
184
207
|
role = "anonymous";
|
|
185
208
|
/** Named-session id (the server sets it; every engine shares one MemoryStore, so
|
|
@@ -628,6 +651,18 @@ export class BrowserEngine {
|
|
|
628
651
|
throw new Error(`storageStatePath does not exist: ${opts.storageStatePath}`);
|
|
629
652
|
}
|
|
630
653
|
this.mode = opts.mode ?? "read-only";
|
|
654
|
+
const trust = trustedEmbedOrigins(opts.trustedEmbeds);
|
|
655
|
+
this.trustedEmbeds = new Set(trust.origins);
|
|
656
|
+
this.trustNotice =
|
|
657
|
+
(trust.origins.length > 0
|
|
658
|
+
? this.mode === "safe-write"
|
|
659
|
+
? ` Trusted embeds: ${trust.origins.join(", ")} — writes their frames send outside the app go out in this mode; hostile input, repeated-click probes and uploads stay refused.`
|
|
660
|
+
: this.mode === "destructive"
|
|
661
|
+
? ` Trusted embeds (${trust.origins.join(", ")}) are not needed in destructive mode, which lets every embed's writes out.`
|
|
662
|
+
: ` Trusted embeds (${trust.origins.join(", ")}) are ignored in ${this.mode} mode: they apply in safe-write only.`
|
|
663
|
+
: "") +
|
|
664
|
+
(trust.rejected.length > 0 ? ` Not a plain http(s) origin, so not trusted: ${trust.rejected.join(", ")}.` : "") +
|
|
665
|
+
(trust.overflow.length > 0 ? ` More than ${MAX_TRUSTED_EMBEDS} trusted embeds; not trusted: ${trust.overflow.join(", ")}.` : "");
|
|
631
666
|
this.sessionObjective = (opts.objective ?? "").trim().replace(/\s+/g, " ").slice(0, 300);
|
|
632
667
|
// An agent-supplied task counts as stated; the placeholder does not.
|
|
633
668
|
this.setTask(opts.task ?? "Attaching and taking stock", opts.task !== undefined);
|
|
@@ -665,8 +700,22 @@ export class BrowserEngine {
|
|
|
665
700
|
}
|
|
666
701
|
this.oracles = new OracleMonitor();
|
|
667
702
|
this.oracles.setPolicyRefusalCheck((req) => this.refusedByPolicy.has(req));
|
|
703
|
+
// A request an embed sends to the app is the app's to answer: only one headed outside the app is the embed's.
|
|
704
|
+
this.oracles.setEmbedAttribution((req) => {
|
|
705
|
+
let site = null;
|
|
706
|
+
try {
|
|
707
|
+
const frame = req.frame();
|
|
708
|
+
site = frame === frame.page().mainFrame() ? null : this.foreignOriginOf(frame);
|
|
709
|
+
}
|
|
710
|
+
catch {
|
|
711
|
+
/* no frame: a service worker's or a new window's first request */
|
|
712
|
+
}
|
|
713
|
+
return embedOfRequest(this.baseUrl, req.url(), site);
|
|
714
|
+
});
|
|
668
715
|
this.lastSnap = null;
|
|
669
716
|
this.designAuditCount = 0;
|
|
717
|
+
// A new attach may be a restarted app: what failed to load before gets another try.
|
|
718
|
+
this.loadFailedRoutes = new Set();
|
|
670
719
|
// The engine learns the route list itself so completion is an objective,
|
|
671
720
|
// enforceable contract (scout_report refuses while known routes are unvisited)
|
|
672
721
|
// rather than a prompt suggestion the driver may ignore.
|
|
@@ -996,7 +1045,7 @@ export class BrowserEngine {
|
|
|
996
1045
|
`Memory: ${this.memory.dir}.${this.memory.loadWarning ? ` WARNING: ${this.memory.loadWarning}` : ""}` +
|
|
997
1046
|
(this.memory.prunedStates > 0 ? ` Trimmed ${this.memory.prunedStates} old page state(s) from the history; coverage is unchanged.` : "") +
|
|
998
1047
|
`${this.memory.legacyDirNote ? ` ${this.memory.legacyDirNote}` : ""}` +
|
|
999
|
-
`${this.memory.gitIgnoreNote ? ` ${this.memory.gitIgnoreNote}` : ""} Call scout_snapshot to see the current state.` +
|
|
1048
|
+
`${this.memory.gitIgnoreNote ? ` ${this.memory.gitIgnoreNote}` : ""}${this.trustNotice} Call scout_snapshot to see the current state.` +
|
|
1000
1049
|
authWarning);
|
|
1001
1050
|
}
|
|
1002
1051
|
/** Whether the page was moved off the app by one of its embeds (policy.ts EmbedMoveTracker). */
|
|
@@ -1154,31 +1203,169 @@ export class BrowserEngine {
|
|
|
1154
1203
|
if (stable)
|
|
1155
1204
|
break;
|
|
1156
1205
|
}
|
|
1206
|
+
const mainCount = rawElements.length;
|
|
1207
|
+
// The page's forms, read in the same settled page as its elements. The
|
|
1208
|
+
// page's own document only: an xpath names a node in one document.
|
|
1209
|
+
const rawForms = ((await this.formRead("form inventory", page.evaluate(FORMS_INVENTORY_SCRIPT))) ?? []);
|
|
1210
|
+
// Then each frame's own controls, read once, each within its own limit.
|
|
1211
|
+
const framed = await this.collectFrames(page);
|
|
1157
1212
|
// Element identity is the coverage key (testid or role+name, ordinal-
|
|
1158
1213
|
// disambiguated). When a key persists across snapshots of the same route,
|
|
1159
1214
|
// its ref is REUSED — diffs stay meaningful and the agent's mental model
|
|
1160
|
-
// (and previously issued refs) survive re-snapshots.
|
|
1215
|
+
// (and previously issued refs) survive re-snapshots. An element inside a
|
|
1216
|
+
// frame carries the frame in its key (collector.ts frameElementKey).
|
|
1161
1217
|
const route = normalizePath(page.url());
|
|
1162
1218
|
const prevByKey = this.lastSnap?.route === route ? this.lastSnap.byKey : null;
|
|
1163
1219
|
this.refs.clear();
|
|
1220
|
+
this.refFrames.clear();
|
|
1164
1221
|
const keyCounts = new Map();
|
|
1165
|
-
const
|
|
1166
|
-
|
|
1222
|
+
const all = [
|
|
1223
|
+
...rawElements.map((raw) => ({ raw })),
|
|
1224
|
+
...framed.flatMap((g) => g.raws.map((raw) => ({ raw: raw, frame: g.frame, tag: g.tag }))),
|
|
1225
|
+
];
|
|
1226
|
+
const elements = all.map(({ raw: el, frame, tag }) => {
|
|
1227
|
+
const baseKey = frameElementKey(elementKey(el), tag);
|
|
1167
1228
|
const count = keyCounts.get(baseKey) ?? 0;
|
|
1168
1229
|
keyCounts.set(baseKey, count + 1);
|
|
1169
1230
|
const key = count === 0 ? baseKey : `${baseKey}~${count}`;
|
|
1170
1231
|
const ref = prevByKey?.get(key)?.ref ?? `e${++this.refCounter}`;
|
|
1171
1232
|
const full = {
|
|
1172
1233
|
...el,
|
|
1234
|
+
...(tag ? { frame: tag } : {}),
|
|
1235
|
+
// Judged on the real label, then masked: the policy must see what a click would press.
|
|
1236
|
+
destructive: isDestructive(el.name, el.testid),
|
|
1237
|
+
...(tag?.foreign ? { name: masksForeignName(el.tag, el.role) ? MASKED_NAME : capForeignName(el.name) } : {}),
|
|
1238
|
+
...(tag?.foreign && el.href ? { href: stripForeignHref(el.href) } : {}),
|
|
1173
1239
|
ref,
|
|
1174
1240
|
key,
|
|
1175
|
-
destructive: isDestructive(el.name, el.testid),
|
|
1176
1241
|
};
|
|
1177
1242
|
this.refs.set(ref, full);
|
|
1243
|
+
if (frame)
|
|
1244
|
+
this.refFrames.set(ref, frame);
|
|
1178
1245
|
return full;
|
|
1179
1246
|
});
|
|
1180
|
-
this.harvestRoutes(elements);
|
|
1181
|
-
|
|
1247
|
+
this.harvestRoutes(elements.filter((el) => !el.frame?.foreign));
|
|
1248
|
+
this.framesRead = new Set(framed.map((g) => g.frame));
|
|
1249
|
+
// A form is known by its own attributes, else by its submit control's coverage
|
|
1250
|
+
// key (forms.ts formIdentity); a form with neither is not tracked.
|
|
1251
|
+
const forms = rawForms.flatMap((f) => {
|
|
1252
|
+
const status = formStatus(f);
|
|
1253
|
+
const key = status === "untracked" ? null : (formIdentity(f.attrs) ?? keyAtXpath(elements, f.submit));
|
|
1254
|
+
return key ? [{ key, guarded: status === "guarded" }] : [];
|
|
1255
|
+
});
|
|
1256
|
+
this.lastCollectTruncated = mainCount >= 150;
|
|
1257
|
+
return { elements, truncated: mainCount >= 150, forms };
|
|
1258
|
+
}
|
|
1259
|
+
/** Whether the latest collect stopped at the element cap, so some controls have no key at all. */
|
|
1260
|
+
lastCollectTruncated = false;
|
|
1261
|
+
/**
|
|
1262
|
+
* The controls inside the page's frames, up to MAX_READ_FRAMES visible ones,
|
|
1263
|
+
* each read within FRAME_READ_MS so one busy frame cannot stall a snapshot.
|
|
1264
|
+
* Positions are moved into the page's coordinates (the frame element's box
|
|
1265
|
+
* on screen, the frame's own scroll, the page's scroll), so geometry and
|
|
1266
|
+
* the design audit can place them.
|
|
1267
|
+
*/
|
|
1268
|
+
async collectFrames(page) {
|
|
1269
|
+
const top = page.mainFrame();
|
|
1270
|
+
const frames = page.frames().filter((f) => f !== top && !f.isDetached());
|
|
1271
|
+
if (frames.length === 0)
|
|
1272
|
+
return [];
|
|
1273
|
+
const pageScroll = (await page.evaluate("({ x: window.scrollX, y: window.scrollY })").catch(() => null)) ?? {
|
|
1274
|
+
x: 0,
|
|
1275
|
+
y: 0,
|
|
1276
|
+
};
|
|
1277
|
+
const read = async (frame) => {
|
|
1278
|
+
let timer;
|
|
1279
|
+
const work = (async () => {
|
|
1280
|
+
const el = await frame.frameElement();
|
|
1281
|
+
const box = await el.boundingBox();
|
|
1282
|
+
if (!box || box.width < 2 || box.height < 2)
|
|
1283
|
+
return null;
|
|
1284
|
+
const title = ((await el.getAttribute("title")) || (await el.getAttribute("name")) || "").trim();
|
|
1285
|
+
const got = (await frame.evaluate(`(() => ({ els: ${COLLECT_INTERACTABLES_SCRIPT}, sx: window.scrollX, sy: window.scrollY }))()`));
|
|
1286
|
+
const raws = got.els.map((r) => ({ ...r, rect: frameToPageRect(r.rect, box, { x: got.sx, y: got.sy }, pageScroll) }));
|
|
1287
|
+
let origin = "";
|
|
1288
|
+
try {
|
|
1289
|
+
const u = new URL(frame.url());
|
|
1290
|
+
if (u.protocol === "http:" || u.protocol === "https:")
|
|
1291
|
+
origin = u.origin;
|
|
1292
|
+
}
|
|
1293
|
+
catch {
|
|
1294
|
+
/* no web address */
|
|
1295
|
+
}
|
|
1296
|
+
const foreignOrigin = this.foreignOriginOf(frame);
|
|
1297
|
+
return { frame, tag: { url: frame.url(), origin: foreignOrigin ?? origin, title, foreign: foreignOrigin !== null }, raws };
|
|
1298
|
+
})().catch(() => null);
|
|
1299
|
+
const limit = new Promise((resolve) => {
|
|
1300
|
+
timer = setTimeout(() => resolve(null), FRAME_READ_MS);
|
|
1301
|
+
});
|
|
1302
|
+
return Promise.race([work, limit]).finally(() => clearTimeout(timer));
|
|
1303
|
+
};
|
|
1304
|
+
// Visible frames first: a page's hidden plumbing frames must not use up the budget.
|
|
1305
|
+
const boxes = await Promise.all(frames.map((f) => this.withinFrameLimit(f.frameElement().then((el) => el.boundingBox()))));
|
|
1306
|
+
const visible = frames.filter((_, i) => {
|
|
1307
|
+
const b = boxes[i];
|
|
1308
|
+
return !!b && b.width >= 2 && b.height >= 2;
|
|
1309
|
+
});
|
|
1310
|
+
const got = await Promise.all(visible.slice(0, MAX_READ_FRAMES).map(read));
|
|
1311
|
+
return got.filter((g) => g !== null);
|
|
1312
|
+
}
|
|
1313
|
+
/** The origin of another site's frame an element is in, or null for the page and the app's own frames. */
|
|
1314
|
+
foreignEmbedOf(el) {
|
|
1315
|
+
const frame = this.refFrames.get(el.ref);
|
|
1316
|
+
const now = frame ? this.foreignOriginOf(frame) : null;
|
|
1317
|
+
if (now)
|
|
1318
|
+
return now;
|
|
1319
|
+
return el.frame?.foreign ? el.frame.origin || el.frame.url : null;
|
|
1320
|
+
}
|
|
1321
|
+
/**
|
|
1322
|
+
* The other site a frame belongs to, judged on the whole chain up to the
|
|
1323
|
+
* page: a blank or srcdoc frame inside another site's frame is that site's,
|
|
1324
|
+
* and so is a frame that once held its document (the foreign-frame record).
|
|
1325
|
+
*/
|
|
1326
|
+
foreignOriginOf(frame) {
|
|
1327
|
+
const top = frame.page().mainFrame();
|
|
1328
|
+
const chain = [];
|
|
1329
|
+
for (let f = frame; f && f !== top; f = f.parentFrame()) {
|
|
1330
|
+
const recorded = this.foreignFrames.get(f);
|
|
1331
|
+
if (recorded)
|
|
1332
|
+
return recorded;
|
|
1333
|
+
chain.push(f.url());
|
|
1334
|
+
}
|
|
1335
|
+
return foreignFrameOrigin(this.baseUrl, chain);
|
|
1336
|
+
}
|
|
1337
|
+
/** A frame read that gives up after FRAME_READ_MS, answering null. */
|
|
1338
|
+
async withinFrameLimit(work) {
|
|
1339
|
+
let timer;
|
|
1340
|
+
const limit = new Promise((resolve) => {
|
|
1341
|
+
timer = setTimeout(() => resolve(null), FRAME_READ_MS);
|
|
1342
|
+
});
|
|
1343
|
+
return Promise.race([work.catch(() => null), limit]).finally(() => clearTimeout(timer));
|
|
1344
|
+
}
|
|
1345
|
+
/** Frames not directly under the page, less those whose controls were read anyway. */
|
|
1346
|
+
unreadNestedFrames(page, nested) {
|
|
1347
|
+
const top = page.mainFrame();
|
|
1348
|
+
const readNested = [...this.framesRead].filter((f) => !f.isDetached() && f.parentFrame() !== top).length;
|
|
1349
|
+
return Math.max(0, nested - readNested);
|
|
1350
|
+
}
|
|
1351
|
+
/** The frames whose controls the last snapshot listed. */
|
|
1352
|
+
framesRead = new Set();
|
|
1353
|
+
/** The frame each ref of the last snapshot lives in; refs of the page itself are absent. */
|
|
1354
|
+
refFrames = new Map();
|
|
1355
|
+
/**
|
|
1356
|
+
* Where to look for a ref's element: its frame, or the page. A frame that has
|
|
1357
|
+
* gone makes the ref stale, like a page that has navigated.
|
|
1358
|
+
*/
|
|
1359
|
+
scopeOf(el) {
|
|
1360
|
+
const frame = this.refFrames.get(el.ref);
|
|
1361
|
+
if (!frame)
|
|
1362
|
+
return this.requirePage();
|
|
1363
|
+
if (frame.isDetached())
|
|
1364
|
+
throw new Error(`The frame holding ${el.ref} has gone — take a new scout_snapshot.`);
|
|
1365
|
+
// A frame that navigated holds a new document: the ref, and what it knew of the frame's site, are stale.
|
|
1366
|
+
if (el.frame && frame.url() !== el.frame.url)
|
|
1367
|
+
throw new Error(`The frame holding ${el.ref} has navigated — take a new scout_snapshot.`);
|
|
1368
|
+
return frame;
|
|
1182
1369
|
}
|
|
1183
1370
|
/**
|
|
1184
1371
|
* Collect the current page and REGISTER it as a visited state, returning the
|
|
@@ -1189,10 +1376,12 @@ export class BrowserEngine {
|
|
|
1189
1376
|
*/
|
|
1190
1377
|
async captureCoverageState() {
|
|
1191
1378
|
const page = this.requirePage();
|
|
1192
|
-
const { elements } = await this.collect();
|
|
1379
|
+
const { elements, forms } = await this.collect();
|
|
1193
1380
|
const url = page.url();
|
|
1194
1381
|
const fp = fingerprintState(url, elements);
|
|
1195
1382
|
this.memory?.visitState(fp, url, normalizePath(url), elements.map((el) => el.key));
|
|
1383
|
+
for (const f of forms)
|
|
1384
|
+
this.memory?.recordForm(fp, f.key, f.guarded);
|
|
1196
1385
|
return { fp, elements, url };
|
|
1197
1386
|
}
|
|
1198
1387
|
/**
|
|
@@ -1260,13 +1449,15 @@ export class BrowserEngine {
|
|
|
1260
1449
|
const page = this.requirePage();
|
|
1261
1450
|
const memory = this.memory;
|
|
1262
1451
|
await this.settle();
|
|
1263
|
-
const { elements, truncated } = await this.collect();
|
|
1452
|
+
const { elements, truncated, forms } = await this.collect();
|
|
1264
1453
|
const url = page.url();
|
|
1265
1454
|
this.snapshotUrl = url;
|
|
1266
1455
|
const route = normalizePath(url);
|
|
1267
1456
|
const fp = fingerprintState(url, elements);
|
|
1268
1457
|
this.currentFingerprint = fp;
|
|
1269
1458
|
const isNew = memory.visitState(fp, url, route, elements.map((el) => el.key));
|
|
1459
|
+
for (const f of forms)
|
|
1460
|
+
memory.recordForm(fp, f.key, f.guarded);
|
|
1270
1461
|
memory.recordRoleAccess(this.role, route, "reached");
|
|
1271
1462
|
// A snapshot is what an agent takes when it wants to LOOK at something, so
|
|
1272
1463
|
// it is the frame a reader most wants beside the step. Recording only the
|
|
@@ -1282,10 +1473,11 @@ export class BrowserEngine {
|
|
|
1282
1473
|
dup ? `copy#${Number(dup[1]) + 1}` : null,
|
|
1283
1474
|
el.disabled ? "disabled" : null,
|
|
1284
1475
|
el.destructive ? "DESTRUCTIVE" : null,
|
|
1476
|
+
labelFlag(el),
|
|
1285
1477
|
memory.wasExercised(fp, el.key) ? "done" : null,
|
|
1286
1478
|
el.href ? `href=${el.href.slice(0, 60)}` : null,
|
|
1287
1479
|
].filter(Boolean);
|
|
1288
|
-
return `${el.ref} ${el.role} "${displayName(el)}"${flags.length ? ` [${flags.join(", ")}]` : ""}`;
|
|
1480
|
+
return `${el.ref} ${el.role} "${displayName(el)}"${flags.length ? ` [${flags.join(", ")}]` : ""}${el.frame ? ` ⟨in ${frameLabel(el.frame)}⟩` : ""}`;
|
|
1289
1481
|
};
|
|
1290
1482
|
// Diff mode: when re-snapshotting the same route, report only what
|
|
1291
1483
|
// changed — same idea as UI reconciliation, applied to agent context.
|
|
@@ -1334,10 +1526,10 @@ export class BrowserEngine {
|
|
|
1334
1526
|
`Interactables (${elements.length}${truncated ? "+ — TRUNCATED at 150, dense page" : ""}${missingTestids ? `, ${missingTestids} missing data-testid` : ""}):\n` +
|
|
1335
1527
|
elements.map(line).join("\n");
|
|
1336
1528
|
}
|
|
1337
|
-
|
|
1338
|
-
|
|
1529
|
+
// Per document: the page's own controls together, and each frame's apart —
|
|
1530
|
+
// an embed's controls are not laid out against the page's.
|
|
1531
|
+
const { geometry, brokenImages } = await this.measureLayout(page, elements, url);
|
|
1339
1532
|
const hiddenFileInputs = await this.hiddenFileInputs(page);
|
|
1340
|
-
const brokenImages = brokenImageIssues((await page.evaluate(BROKEN_IMAGES_SCRIPT).catch(() => null)) ?? { images: [], total: 0 }, url);
|
|
1341
1533
|
const { frames, nested: nestedFrames } = await this.frameInventory(page);
|
|
1342
1534
|
const cov = memory.coverage();
|
|
1343
1535
|
const unvisited = this.unvisitedKnownRoutes();
|
|
@@ -1350,7 +1542,13 @@ export class BrowserEngine {
|
|
|
1350
1542
|
(geometry.length > 0 ? `\nGEOMETRY issues:\n` + geometry.map((g) => ` ⚠ ${g}`).join("\n") : "") +
|
|
1351
1543
|
(brokenImages.length > 0 ? `\nBROKEN IMAGES:\n` + brokenImages.map((b) => ` ⚠ ${b}`).join("\n") : "") +
|
|
1352
1544
|
(frames.length > 0 || nestedFrames > 0
|
|
1353
|
-
? `\n` +
|
|
1545
|
+
? `\n` +
|
|
1546
|
+
frameLines(this.baseUrl, frames, {
|
|
1547
|
+
nested: this.unreadNestedFrames(page, nestedFrames),
|
|
1548
|
+
writesRefused: this.mode !== "destructive",
|
|
1549
|
+
trustedWrites: this.mode === "safe-write" ? this.trustedEmbeds : undefined,
|
|
1550
|
+
read: new Set([...this.framesRead].map((f) => f.url())),
|
|
1551
|
+
}).join("\n")
|
|
1354
1552
|
: "") +
|
|
1355
1553
|
(hiddenFileInputs.length > 0
|
|
1356
1554
|
? `\nFILE INPUTS not listed above (hidden behind a styled control — a user never sees the input itself): ${hiddenFileInputs.join("; ")}. ` +
|
|
@@ -1384,7 +1582,7 @@ export class BrowserEngine {
|
|
|
1384
1582
|
}
|
|
1385
1583
|
// String EXPRESSION via page.evaluate (locator.evaluate treats a string as
|
|
1386
1584
|
// an expression, not a function — the element arg never binds).
|
|
1387
|
-
const live = (await
|
|
1585
|
+
const live = (await this.scopeOf(el)
|
|
1388
1586
|
.evaluate(`(() => { const node = ${xpathLookup(el.xpath)}; if (!node) return null; ` +
|
|
1389
1587
|
`return { testid: node.getAttribute('data-testid'), label: (node.getAttribute('aria-label') || node.innerText || node.textContent || node.getAttribute('placeholder') || '').trim().slice(0, 120) }; })()`)
|
|
1390
1588
|
.catch(() => null));
|
|
@@ -1605,15 +1803,14 @@ export class BrowserEngine {
|
|
|
1605
1803
|
}
|
|
1606
1804
|
// Frames still attached that hold, or held, another site: one that moved itself to data: is no longer foreign by its URL.
|
|
1607
1805
|
const pageHasForeignFrame = this.embeddedSites().size > 0;
|
|
1608
|
-
const
|
|
1609
|
-
|
|
1610
|
-
|
|
1611
|
-
unadoptedPageUrl,
|
|
1612
|
-
pageHasForeignFrame,
|
|
1613
|
-
...requestSource(req),
|
|
1614
|
-
});
|
|
1806
|
+
const source = requestSource(req);
|
|
1807
|
+
const originHeader = req.headers()["origin"];
|
|
1808
|
+
const foreign = foreignWrite(this.baseUrl, { url: req.url(), originHeader, unadoptedPageUrl, pageHasForeignFrame, ...source });
|
|
1615
1809
|
if (foreign && allowsForeignWriteOnSignIn(this.mode, this.page?.url() ?? "", this.baseUrl))
|
|
1616
1810
|
return null;
|
|
1811
|
+
// Frames of origins the user named as trusted, in safe-write, every one involved: the ordinary rules.
|
|
1812
|
+
if (foreign && trustsForeignWrite(this.mode, this.trustedEmbeds, this.baseUrl, { frameChain: source.frameChain, originHeader, unadoptedPageUrl }))
|
|
1813
|
+
return null;
|
|
1617
1814
|
return foreign;
|
|
1618
1815
|
}
|
|
1619
1816
|
/** Report (and clear) write-policy blocks since the last action. */
|
|
@@ -1732,6 +1929,88 @@ export class BrowserEngine {
|
|
|
1732
1929
|
return { forced: true };
|
|
1733
1930
|
}
|
|
1734
1931
|
}
|
|
1932
|
+
/**
|
|
1933
|
+
* A form read (the inventory, or the probe before a submit), bounded by
|
|
1934
|
+
* FORM_READ_MS. This bookkeeping must never fail or stall the action, so a
|
|
1935
|
+
* failure yields null — but only a page navigating away under the read is
|
|
1936
|
+
* expected; a timeout or any other error goes in the action log.
|
|
1937
|
+
*/
|
|
1938
|
+
async formRead(what, work) {
|
|
1939
|
+
let timer;
|
|
1940
|
+
const limit = new Promise((resolve) => {
|
|
1941
|
+
timer = setTimeout(() => resolve("timeout"), FORM_READ_MS);
|
|
1942
|
+
});
|
|
1943
|
+
const outcome = await Promise.race([
|
|
1944
|
+
work.then((value) => ({ value }), (err) => ({ err })),
|
|
1945
|
+
limit,
|
|
1946
|
+
]).finally(() => clearTimeout(timer));
|
|
1947
|
+
const url = this.page?.url() ?? "";
|
|
1948
|
+
if (outcome === "timeout") {
|
|
1949
|
+
this.logAction({ action: FORMS_READ_FAILED, target: `${what} took over ${FORM_READ_MS} ms`, url });
|
|
1950
|
+
return null;
|
|
1951
|
+
}
|
|
1952
|
+
if ("err" in outcome) {
|
|
1953
|
+
const message = outcome.err instanceof Error ? outcome.err.message.split("\n")[0] : String(outcome.err);
|
|
1954
|
+
if (!isNavigationTeardown(message))
|
|
1955
|
+
this.logAction({ action: FORMS_READ_FAILED, target: `${what}: ${message}`.slice(0, 300), url });
|
|
1956
|
+
return null;
|
|
1957
|
+
}
|
|
1958
|
+
return outcome.value;
|
|
1959
|
+
}
|
|
1960
|
+
/** The form a located element belongs to, read just before an action that may submit it (forms.ts). */
|
|
1961
|
+
probeForm(target) {
|
|
1962
|
+
return this.formRead("form probe", target.evaluate(probeFormOf, undefined, { timeout: FORM_READ_MS }));
|
|
1963
|
+
}
|
|
1964
|
+
/** The form the focused element belongs to, for a key press that may submit through it. */
|
|
1965
|
+
probeFocusedForm() {
|
|
1966
|
+
return this.formRead("form probe", this.requirePage().evaluate(formProbeExpression("document.activeElement")));
|
|
1967
|
+
}
|
|
1968
|
+
/**
|
|
1969
|
+
* A plan's pre-action state, fit to record a submit of this form against.
|
|
1970
|
+
* A plan reuses its previous capture while the URL has not changed, but the
|
|
1971
|
+
* page can shift under the same URL (a control inserted above the form
|
|
1972
|
+
* moves every xpath after it). Only a REUSED capture is suspect: when the
|
|
1973
|
+
* form has no identity of its own and the reused state does not hold its
|
|
1974
|
+
* submit control, the state is captured afresh. A capture this step just
|
|
1975
|
+
* took is never taken twice, and a step with no state gets one.
|
|
1976
|
+
*/
|
|
1977
|
+
async stateHolding(probe, state, reused) {
|
|
1978
|
+
if (state && (!reused || formIdentity(probe.attrs) || submitControlIn(state.elements, probe)))
|
|
1979
|
+
return state;
|
|
1980
|
+
return (await this.captureCoverageState().catch(() => null)) ?? state;
|
|
1981
|
+
}
|
|
1982
|
+
/** Whether this probe describes a submit of a tracked form, so the submit is worth recording. */
|
|
1983
|
+
static isTrackedSubmit(kind, probe) {
|
|
1984
|
+
return probe !== null && tracksForm(probe.fields) && submits(kind, probe);
|
|
1985
|
+
}
|
|
1986
|
+
/**
|
|
1987
|
+
* Mark a submit of a listed form against the state it was made in. The
|
|
1988
|
+
* form is known by its own attributes when it has any; otherwise by the
|
|
1989
|
+
* snapshot element at its submit control's xpath, accepted only when that
|
|
1990
|
+
* element's testid and name agree with the control the probe found (for a
|
|
1991
|
+
* click on that control, the element clicked). A submit that matches no
|
|
1992
|
+
* listed form is logged, never dropped silently and never added to the
|
|
1993
|
+
* list. Whether it was a submit, and an empty one, is forms.ts's rule.
|
|
1994
|
+
*/
|
|
1995
|
+
noteFormSubmit(state, kind, probe) {
|
|
1996
|
+
if (!this.memory || !BrowserEngine.isTrackedSubmit(kind, probe))
|
|
1997
|
+
return;
|
|
1998
|
+
const empty = isEmptySubmit(kind, probe);
|
|
1999
|
+
const key = formIdentity(probe.attrs) ?? (state ? (submitControlIn(state.elements, probe)?.key ?? null) : null);
|
|
2000
|
+
if (state?.fp && key && this.memory.recordFormSubmit(state.fp, key, empty))
|
|
2001
|
+
return;
|
|
2002
|
+
// A page past the snapshot's element cap may never list this control, so
|
|
2003
|
+
// a fresh snapshot would not help: say it is untracked, not what to retry.
|
|
2004
|
+
const beyondCap = !key && this.lastCollectTruncated && !state?.elements.some((e) => !e.frame && e.xpath === probe.submit);
|
|
2005
|
+
const what = `${empty ? "empty " : ""}${kind} submit of a form (submit control ${JSON.stringify(probe.submitName)} at ${probe.submit})`;
|
|
2006
|
+
this.logAction({
|
|
2007
|
+
action: FORMS_SUBMIT_UNMATCHED,
|
|
2008
|
+
target: beyondCap
|
|
2009
|
+
? `${what} is untracked: the page has more controls than a snapshot lists, so this form, which has no id, name or action, cannot be keyed`
|
|
2010
|
+
: `${what} matches no form listed for this page, so it is not counted — take a scout_snapshot and submit again`,
|
|
2011
|
+
url: this.page?.url() ?? "",
|
|
2012
|
+
});
|
|
2013
|
+
}
|
|
1735
2014
|
async click(ref, clicks = 1) {
|
|
1736
2015
|
const page = this.requirePage();
|
|
1737
2016
|
const { el, liveLabel } = await this.resolveForAction(ref);
|
|
@@ -1740,12 +2019,23 @@ export class BrowserEngine {
|
|
|
1740
2019
|
this.logAction({ action: "click:refused", target: liveLabel || el.name, url: page.url() });
|
|
1741
2020
|
return refusal;
|
|
1742
2021
|
}
|
|
2022
|
+
const embed = this.foreignEmbedOf(el);
|
|
2023
|
+
if (embed && clicks > 1) {
|
|
2024
|
+
this.logAction({ action: "click:refused", target: `${clicks} clicks in a frame of ${embed}`, url: page.url() });
|
|
2025
|
+
return embedProbeRefusal(`a ${clicks}-click probe`, embed);
|
|
2026
|
+
}
|
|
1743
2027
|
// Submit-shaped clicks that fire zero network requests are a smell
|
|
1744
2028
|
// (silent no-op forms): capture the count before to compare after.
|
|
1745
2029
|
const xhrBefore = this.xhrCount;
|
|
1746
2030
|
const submitLike = el.role === "button" && /submit|send|save|create|apply|subscribe|register|sign|post|add\b/i.test(el.name + " " + (el.testid ?? ""));
|
|
1747
|
-
const
|
|
2031
|
+
const clickTarget = this.scopeOf(el).locator(`xpath=${el.xpath}`);
|
|
2032
|
+
// Read before the click: what the form's fields hold when it goes. Only a
|
|
2033
|
+
// button or an input can submit a form; nothing else is asked.
|
|
2034
|
+
const form = !el.frame && (el.tag === "button" || el.tag === "input") ? await this.probeForm(clickTarget) : null;
|
|
2035
|
+
const formState = { fp: this.currentFingerprint, elements: [...this.refs.values()] };
|
|
2036
|
+
const { forced } = await this.resilientClick(clickTarget, ACTION_TIMEOUT_MS, clicks);
|
|
1748
2037
|
this.memory.markExercised(this.currentFingerprint, el.key, clicks > 1 ? `click×${clicks}` : "click");
|
|
2038
|
+
this.noteFormSubmit(formState, "click", form);
|
|
1749
2039
|
const result = await this.afterAction(clicks > 1 ? `click×${clicks}` : "click", `${el.role} "${el.name}"`);
|
|
1750
2040
|
// Impatient-user probe: a rapid multi-click that fires the SAME
|
|
1751
2041
|
// state-changing request more than once means the action is not guarded
|
|
@@ -1854,15 +2144,28 @@ export class BrowserEngine {
|
|
|
1854
2144
|
const refusal = this.actionPolicyCheck(el, liveLabel);
|
|
1855
2145
|
if (refusal)
|
|
1856
2146
|
return refusal;
|
|
1857
|
-
const
|
|
1858
|
-
|
|
1859
|
-
|
|
1860
|
-
|
|
2147
|
+
const embed = this.foreignEmbedOf(el);
|
|
2148
|
+
if (embed) {
|
|
2149
|
+
const hostile = hostileForEmbed(text);
|
|
2150
|
+
if (hostile) {
|
|
2151
|
+
this.logAction({ action: "type:refused", target: `${el.role} in a frame of ${embed} (${hostile})`, url: page.url() });
|
|
2152
|
+
return embedProbeRefusal(`typing this value (${hostile})`, embed);
|
|
2153
|
+
}
|
|
2154
|
+
}
|
|
2155
|
+
const locator = this.scopeOf(el).locator(`xpath=${el.xpath}`);
|
|
2156
|
+
// Before the fill: a page that reflects input as it is typed already holds
|
|
2157
|
+
// the element afterwards. Never for another site's frame: no probe is placed there.
|
|
2158
|
+
if (!embed)
|
|
2159
|
+
await this.noteProbe(text, `${el.role} "${el.name}"`);
|
|
2160
|
+
let fillNote = await this.fillOrAppend(locator, text, replace);
|
|
2161
|
+
// What another site's field already held is its business, not the report's.
|
|
2162
|
+
if (embed)
|
|
2163
|
+
fillNote = fillNote.replace(/existing content "(?:[^"\\]|\\.)*"/g, "existing content (masked: another site's frame)");
|
|
1861
2164
|
if (pressEnter) {
|
|
1862
2165
|
// Enter inside a form submits it — check the form's submit target, or
|
|
1863
2166
|
// pressEnter becomes a read-only bypass for destructive submits.
|
|
1864
2167
|
if (this.readOnly) {
|
|
1865
|
-
const submitLabel = (await
|
|
2168
|
+
const submitLabel = (await this.scopeOf(el)
|
|
1866
2169
|
.evaluate(`(() => { const node = ${xpathLookup(el.xpath)}; if (!node) return ''; ` +
|
|
1867
2170
|
`const f = node.form || node.closest('form'); if (!f) return ''; ` +
|
|
1868
2171
|
`const s = f.querySelector('[type="submit"], button:not([type="button"]):not([type="reset"])'); ` +
|
|
@@ -1873,7 +2176,11 @@ export class BrowserEngine {
|
|
|
1873
2176
|
return `Filled ${el.role} "${el.name}" but did NOT press Enter. ` + destructiveRefusal(submitLabel, this.mode);
|
|
1874
2177
|
}
|
|
1875
2178
|
}
|
|
2179
|
+
// After the fill, before the key: the fields as they are when it submits.
|
|
2180
|
+
const form = !el.frame && el.tag === "input" ? await this.probeForm(locator) : null;
|
|
2181
|
+
const formState = { fp: this.currentFingerprint, elements: [...this.refs.values()] };
|
|
1876
2182
|
await locator.press("Enter", { timeout: ACTION_TIMEOUT_MS });
|
|
2183
|
+
this.noteFormSubmit(formState, "enter", form);
|
|
1877
2184
|
}
|
|
1878
2185
|
this.memory.markExercised(this.currentFingerprint, el.key, "type");
|
|
1879
2186
|
return this.afterAction("type", `${el.role} "${el.name}" ← ${JSON.stringify(text.slice(0, 60))}${pressEnter ? " + Enter" : ""}${fillNote}`);
|
|
@@ -1897,7 +2204,10 @@ export class BrowserEngine {
|
|
|
1897
2204
|
const refusal = this.actionPolicyCheck(el, resolved.liveLabel);
|
|
1898
2205
|
if (refusal)
|
|
1899
2206
|
return refusal;
|
|
1900
|
-
|
|
2207
|
+
const embed = this.foreignEmbedOf(el);
|
|
2208
|
+
if (embed)
|
|
2209
|
+
return embedProbeRefusal("a file upload", embed);
|
|
2210
|
+
locator = this.scopeOf(el).locator(`xpath=${el.xpath}`);
|
|
1901
2211
|
}
|
|
1902
2212
|
const outcome = await this.performUpload(locator, opts);
|
|
1903
2213
|
if (outcome.refused)
|
|
@@ -2023,6 +2333,23 @@ export class BrowserEngine {
|
|
|
2023
2333
|
static OVERLAY_SELECTOR = '[role="tooltip"], [data-tippy-root], [data-radix-popper-content-wrapper], [data-floating-ui-portal], ' +
|
|
2024
2334
|
'[data-slot*="tooltip" i], [data-slot*="popover" i], ' +
|
|
2025
2335
|
'[class*="tooltip" i], [class*="hovercard" i], [class*="popover" i]';
|
|
2336
|
+
/**
|
|
2337
|
+
* The layout checks a snapshot prints: geometry per document (the page's own
|
|
2338
|
+
* controls together, each frame's apart, since an embed's controls are not
|
|
2339
|
+
* laid out against the page's), the overlay probe, and images that failed.
|
|
2340
|
+
*/
|
|
2341
|
+
async measureLayout(page, elements, url) {
|
|
2342
|
+
const viewport = page.viewportSize() ?? { width: 1280, height: 900 };
|
|
2343
|
+
const byDocument = new Map();
|
|
2344
|
+
for (const el of elements) {
|
|
2345
|
+
const doc = el.frame ? el.key.slice(0, el.key.indexOf("|") + 1) : "";
|
|
2346
|
+
byDocument.set(doc, [...(byDocument.get(doc) ?? []), el]);
|
|
2347
|
+
}
|
|
2348
|
+
const geometry = [...byDocument.values()].flatMap((group) => geometryIssues(group, viewport));
|
|
2349
|
+
geometry.push(...(await probeOverlays(page)));
|
|
2350
|
+
const brokenImages = brokenImageIssues((await page.evaluate(BROKEN_IMAGES_SCRIPT).catch(() => null)) ?? { images: [], total: 0 }, url);
|
|
2351
|
+
return { geometry, brokenImages };
|
|
2352
|
+
}
|
|
2026
2353
|
/**
|
|
2027
2354
|
* Visible overlay texts right now — diffed before/after a hover to isolate
|
|
2028
2355
|
* what the hover revealed. Returns null when the read itself failed, so a
|
|
@@ -2094,7 +2421,7 @@ export class BrowserEngine {
|
|
|
2094
2421
|
const page = this.requirePage();
|
|
2095
2422
|
const { el } = await this.resolveForAction(ref);
|
|
2096
2423
|
const { before, bodyBefore, churning } = await this.hoverBaselines();
|
|
2097
|
-
const locator =
|
|
2424
|
+
const locator = this.scopeOf(el).locator(`xpath=${el.xpath}`);
|
|
2098
2425
|
await locator.hover({ timeout: ACTION_TIMEOUT_MS });
|
|
2099
2426
|
// Wiggle inside the element: pointer-tracking libraries distinguish real
|
|
2100
2427
|
// movement from a single synthetic hover event.
|
|
@@ -2104,7 +2431,7 @@ export class BrowserEngine {
|
|
|
2104
2431
|
await page.mouse.move(box.x + box.width / 2 - 2, box.y + box.height / 2 - 1);
|
|
2105
2432
|
}
|
|
2106
2433
|
const { revealed, fallbackUsed } = await this.detectHoverReveal(before, bodyBefore, churning);
|
|
2107
|
-
const attrTexts = (await
|
|
2434
|
+
const attrTexts = (await this.scopeOf(el)
|
|
2108
2435
|
.evaluate(`(() => { const node = ${xpathLookup(el.xpath)}; if (!node) return []; const out = []; ` +
|
|
2109
2436
|
`const t = node.getAttribute('title'); if (t) out.push('title: ' + t.slice(0, 300)); ` +
|
|
2110
2437
|
`const d = node.getAttribute('aria-describedby'); if (d) { for (const id of d.split(/\\s+/)) { ` +
|
|
@@ -2136,7 +2463,7 @@ export class BrowserEngine {
|
|
|
2136
2463
|
return refusal;
|
|
2137
2464
|
if (this.readOnly) {
|
|
2138
2465
|
// Bulk-action dropdowns fire on change — vet the chosen option itself.
|
|
2139
|
-
const optionLabel = (await
|
|
2466
|
+
const optionLabel = (await this.scopeOf(el)
|
|
2140
2467
|
.evaluate(`(() => { const node = ${xpathLookup(el.xpath)}; if (!node) return ''; const v = ${JSON.stringify(value)}; ` +
|
|
2141
2468
|
`const opts = Array.from(node.options || []); ` +
|
|
2142
2469
|
`const o = opts.find(o => o.value === v || o.label === v || (o.textContent || '').trim() === v); ` +
|
|
@@ -2147,7 +2474,7 @@ export class BrowserEngine {
|
|
|
2147
2474
|
return destructiveRefusal(optionLabel || value, this.mode);
|
|
2148
2475
|
}
|
|
2149
2476
|
}
|
|
2150
|
-
const loc =
|
|
2477
|
+
const loc = this.scopeOf(el).locator(`xpath=${el.xpath}`);
|
|
2151
2478
|
const options = el.tag === "select" ? await readSelectOptions(loc) : null;
|
|
2152
2479
|
const picked = await loc.selectOption(value, { timeout: ACTION_TIMEOUT_MS });
|
|
2153
2480
|
this.memory.markExercised(this.currentFingerprint, el.key, "select");
|
|
@@ -2160,11 +2487,37 @@ export class BrowserEngine {
|
|
|
2160
2487
|
* policy as click, or the keyboard becomes a read-only bypass. Shared by
|
|
2161
2488
|
* scout_press and plan press steps; returns a refusal message or null.
|
|
2162
2489
|
*/
|
|
2490
|
+
/**
|
|
2491
|
+
* The frame that holds keyboard focus: the page, or — when the page's
|
|
2492
|
+
* focused element is a frame — that frame, followed down as far as focus goes.
|
|
2493
|
+
*/
|
|
2494
|
+
async focusedFrame(page) {
|
|
2495
|
+
let scope = page;
|
|
2496
|
+
let current = page.mainFrame();
|
|
2497
|
+
for (let depth = 0; depth < 5; depth++) {
|
|
2498
|
+
let next = null;
|
|
2499
|
+
for (const child of current.childFrames()) {
|
|
2500
|
+
// Within the frame limit: a frame that never answers must not hold up a key press.
|
|
2501
|
+
const holds = await this.withinFrameLimit(child.frameElement().then((el) => el.evaluate((node) => node === node.ownerDocument?.activeElement)));
|
|
2502
|
+
if (holds) {
|
|
2503
|
+
next = child;
|
|
2504
|
+
break;
|
|
2505
|
+
}
|
|
2506
|
+
}
|
|
2507
|
+
if (!next)
|
|
2508
|
+
break;
|
|
2509
|
+
scope = next;
|
|
2510
|
+
current = next;
|
|
2511
|
+
}
|
|
2512
|
+
return scope;
|
|
2513
|
+
}
|
|
2163
2514
|
async vetFocusedActivation(key) {
|
|
2164
2515
|
if (!(this.readOnly && /^(Enter|NumpadEnter|Space| )$/i.test(key)))
|
|
2165
2516
|
return null;
|
|
2166
2517
|
const page = this.requirePage();
|
|
2167
|
-
|
|
2518
|
+
// Focus inside a frame reads, from the page, as the <iframe> itself: follow it down.
|
|
2519
|
+
const focused = await this.focusedFrame(page);
|
|
2520
|
+
const focusedLabel = await focused
|
|
2168
2521
|
.evaluate(`(() => { const el = document.activeElement; if (!el) return ""; ` +
|
|
2169
2522
|
`return (el.getAttribute("aria-label") || el.getAttribute("data-testid") || el.innerText || el.textContent || "").trim().slice(0, 120); })()`)
|
|
2170
2523
|
.catch(() => "");
|
|
@@ -2199,7 +2552,13 @@ export class BrowserEngine {
|
|
|
2199
2552
|
// Identify the focused control BEFORE the key lands — activating it may
|
|
2200
2553
|
// navigate, close a dialog, or otherwise destroy the element.
|
|
2201
2554
|
const focused = BrowserEngine.ACTIVATION_KEY_RE.test(key) ? await this.focusedInteractable() : null;
|
|
2555
|
+
// Enter submits from a field or a submit control; Space only activates a control.
|
|
2556
|
+
const formKind = BrowserEngine.ACTIVATION_KEY_RE.test(key) ? (/enter/i.test(key) ? "enter" : "click") : null;
|
|
2557
|
+
const form = formKind ? await this.probeFocusedForm() : null;
|
|
2558
|
+
const formState = { fp: this.currentFingerprint, elements: [...this.refs.values()] };
|
|
2202
2559
|
await page.keyboard.press(key);
|
|
2560
|
+
if (formKind)
|
|
2561
|
+
this.noteFormSubmit(formState, formKind, form);
|
|
2203
2562
|
if (focused && this.memory && this.currentFingerprint) {
|
|
2204
2563
|
this.memory.markExercised(this.currentFingerprint, focused, `press:${key}`);
|
|
2205
2564
|
}
|
|
@@ -2340,17 +2699,51 @@ export class BrowserEngine {
|
|
|
2340
2699
|
navigablePath(routeClass) {
|
|
2341
2700
|
return this.memory?.discoveredRoutes[routeClass] ?? routeClass;
|
|
2342
2701
|
}
|
|
2702
|
+
/** Routes whose page failed to load in this process: still unvisited, but not tried again by a crawl that picks its own targets. */
|
|
2703
|
+
loadFailedRoutes = new Set();
|
|
2704
|
+
/** Unvisited known routes a crawl would still try: those that already failed to load in this process are left out. */
|
|
2705
|
+
crawlableRoutes() {
|
|
2706
|
+
return this.unvisitedKnownRoutes().filter((r) => !this.loadFailedRoutes.has(normalizePath(r)));
|
|
2707
|
+
}
|
|
2708
|
+
/** What the last crawl measured on each route, as data: `scenescout check` builds its verdict from this. */
|
|
2709
|
+
crawlHealth = [];
|
|
2710
|
+
/** The per-route results of the most recent crawl. */
|
|
2711
|
+
get lastCrawlHealth() {
|
|
2712
|
+
return this.crawlHealth;
|
|
2713
|
+
}
|
|
2714
|
+
/** A check's extra measurements on a crawled page: the snapshot's layout checks and the design audit's defects. */
|
|
2715
|
+
async inspectRoute(page, elements, url) {
|
|
2716
|
+
const { geometry, brokenImages } = await this.measureLayout(page, elements, url);
|
|
2717
|
+
try {
|
|
2718
|
+
const { defects, sampled } = await this.auditPage();
|
|
2719
|
+
// Nothing styled to read is not a clean page: contrast, focus and target size went unmeasured.
|
|
2720
|
+
return sampled > 0
|
|
2721
|
+
? { geometry, brokenImages, design: defects }
|
|
2722
|
+
: { geometry, brokenImages, design: [], auditError: "no visible styled elements to measure" };
|
|
2723
|
+
}
|
|
2724
|
+
catch (err) {
|
|
2725
|
+
// A page that navigated away mid-audit has nothing to measure. The
|
|
2726
|
+
// route's other results still stand, and the report says the audit is missing.
|
|
2727
|
+
return { geometry, brokenImages, design: [], auditError: err instanceof Error ? err.message.split("\n")[0] : String(err) };
|
|
2728
|
+
}
|
|
2729
|
+
}
|
|
2343
2730
|
/**
|
|
2344
2731
|
* Engine-side route sweep: visit each path, record the state in memory, and
|
|
2345
2732
|
* collect per-route health — one tool call instead of one LLM turn per route.
|
|
2346
2733
|
* Output is anomaly-oriented: a summary line per route, details only where
|
|
2347
2734
|
* something is wrong. Navigation-only, so it is safe in read-only mode.
|
|
2348
2735
|
*/
|
|
2349
|
-
async crawl(paths) {
|
|
2736
|
+
async crawl(paths, opts = {}) {
|
|
2350
2737
|
const page = this.requirePage();
|
|
2351
2738
|
const memory = this.memory;
|
|
2352
|
-
|
|
2739
|
+
this.crawlHealth = [];
|
|
2740
|
+
const targets = (paths && paths.length > 0 ? paths : this.crawlableRoutes().map((r) => this.navigablePath(r))).slice(0, Math.min(150, opts.limit ?? 150));
|
|
2353
2741
|
if (targets.length === 0) {
|
|
2742
|
+
const failed = this.unvisitedKnownRoutes().filter((r) => this.loadFailedRoutes.has(normalizePath(r)));
|
|
2743
|
+
if (failed.length > 0) {
|
|
2744
|
+
return (`Nothing new to crawl. ${failed.length} route(s) failed to load earlier in this session and are still unvisited: ${failed.slice(0, 20).join(", ")}${failed.length > 20 ? " …" : ""}. ` +
|
|
2745
|
+
`A crawl without paths does not retry them; pass them as paths once the app is reachable, e.g. scout_crawl {paths:${JSON.stringify(failed.slice(0, 3).map((r) => this.navigablePath(r)))}}.`);
|
|
2746
|
+
}
|
|
2354
2747
|
return this.allKnownRoutes().length > 0
|
|
2355
2748
|
? "Nothing to crawl: every known route has been visited. Use scout_coverage for remaining unexercised elements."
|
|
2356
2749
|
: "No routes to crawl yet: no scanned or link-discovered routes. Take a snapshot first (links harvest routes) or pass explicit paths.";
|
|
@@ -2371,16 +2764,42 @@ export class BrowserEngine {
|
|
|
2371
2764
|
status = resp?.status() ?? "no-response";
|
|
2372
2765
|
}
|
|
2373
2766
|
catch (err) {
|
|
2767
|
+
const reason = err instanceof Error ? err.message.split("\n")[0] : String(err);
|
|
2768
|
+
// Not retried by later crawls in this process (a check would otherwise try it
|
|
2769
|
+
// on every discovery round). Deliberately not written to memory: one outage
|
|
2770
|
+
// must not count a route as covered in every later run's gap ledger.
|
|
2771
|
+
this.loadFailedRoutes.add(normalizePath(url));
|
|
2772
|
+
// The browser commits its own error page for this failure tens of
|
|
2773
|
+
// milliseconds after goto has thrown, and that commit interrupts the next
|
|
2774
|
+
// navigation, charging this route's failure to the next one. Wait for it;
|
|
2775
|
+
// a browser that shows no error page simply lets the wait time out.
|
|
2776
|
+
await page.waitForEvent("framenavigated", { predicate: (f) => f === page.mainFrame(), timeout: 1500 }).catch(() => undefined);
|
|
2374
2777
|
summary.push(`${path} — LOAD FAILED`);
|
|
2375
|
-
problems.push(`${path}: ${
|
|
2778
|
+
problems.push(`${path}: ${reason}`);
|
|
2779
|
+
this.crawlHealth.push({
|
|
2780
|
+
path,
|
|
2781
|
+
url,
|
|
2782
|
+
status: null,
|
|
2783
|
+
loadError: reason,
|
|
2784
|
+
loginRedirect: false,
|
|
2785
|
+
elements: 0,
|
|
2786
|
+
unnamed: [],
|
|
2787
|
+
placeholderOnly: [],
|
|
2788
|
+
violations: [],
|
|
2789
|
+
geometry: [],
|
|
2790
|
+
brokenImages: [],
|
|
2791
|
+
design: [],
|
|
2792
|
+
});
|
|
2376
2793
|
continue;
|
|
2377
2794
|
}
|
|
2378
2795
|
await this.settle();
|
|
2379
|
-
const { elements } = await this.collect();
|
|
2796
|
+
const { elements, forms } = await this.collect();
|
|
2380
2797
|
const finalUrl = page.url();
|
|
2381
2798
|
const route = normalizePath(finalUrl);
|
|
2382
2799
|
const fp = fingerprintState(finalUrl, elements);
|
|
2383
2800
|
memory.visitState(fp, finalUrl, route, elements.map((el) => el.key));
|
|
2801
|
+
for (const f of forms)
|
|
2802
|
+
memory.recordForm(fp, f.key, f.guarded);
|
|
2384
2803
|
memory.recordRoleAccess(this.role, route, "reached");
|
|
2385
2804
|
// If we landed somewhere else (auth wall, canonical redirect), the
|
|
2386
2805
|
// REQUESTED route still counts as covered for THIS role — a role that
|
|
@@ -2405,9 +2824,28 @@ export class BrowserEngine {
|
|
|
2405
2824
|
this.logAction({ action: "crawl", target: path, url: finalUrl, ...(await this.frameFor("crawl")) });
|
|
2406
2825
|
await this.scanForInjections();
|
|
2407
2826
|
await this.scanForContradictions();
|
|
2827
|
+
// Measured before the drain: the audit's Tab presses can load things too.
|
|
2828
|
+
const inspected = opts.inspect ? await this.inspectRoute(page, elements, finalUrl) : null;
|
|
2408
2829
|
const violations = this.oracles.drain();
|
|
2830
|
+
const own = elements.filter((el) => !el.frame?.foreign);
|
|
2831
|
+
this.crawlHealth.push({
|
|
2832
|
+
path,
|
|
2833
|
+
url: finalUrl,
|
|
2834
|
+
status: typeof status === "number" ? status : null,
|
|
2835
|
+
loginRedirect,
|
|
2836
|
+
elements: elements.length,
|
|
2837
|
+
unnamed: own.filter(missingName).map(describeControl),
|
|
2838
|
+
placeholderOnly: own.filter(placeholderOnly).map(placeholderEvidence),
|
|
2839
|
+
violations: violations.map(({ kind, severity, detail, url, embed }) => ({ kind, severity, detail, url, ...(embed ? { embed } : {}) })),
|
|
2840
|
+
geometry: inspected?.geometry ?? [],
|
|
2841
|
+
brokenImages: inspected?.brokenImages ?? [],
|
|
2842
|
+
design: inspected?.design ?? [],
|
|
2843
|
+
...(inspected?.auditError ? { auditError: inspected.auditError } : {}),
|
|
2844
|
+
});
|
|
2409
2845
|
const deadEnd = elements.length === 0;
|
|
2410
|
-
|
|
2846
|
+
// A field labelled only by its placeholder counts here too: to someone reading the crawl, it has no label.
|
|
2847
|
+
// Counted over the page's own controls, as the check's lists are: another site's frame is not this app's to fix.
|
|
2848
|
+
const unnamed = own.filter((el) => missingName(el) || placeholderOnly(el)).length;
|
|
2411
2849
|
const missingTestid = elements.filter((el) => !el.testid && !el.disabled).length;
|
|
2412
2850
|
const flags = [loginRedirect ? "AUTH-REDIRECT" : null, deadEnd ? "DEAD-END" : null, violations.length > 0 ? `${violations.length}⚠` : null].filter(Boolean);
|
|
2413
2851
|
summary.push(`${path} — ${status} · ${elements.length} el` +
|
|
@@ -2483,6 +2921,8 @@ export class BrowserEngine {
|
|
|
2483
2921
|
// A select step's options and choice, recorded against the dropdown the bookkeeping below finds.
|
|
2484
2922
|
let chose = null;
|
|
2485
2923
|
let preState = null;
|
|
2924
|
+
// The form this step may submit, read before it went (forms.ts).
|
|
2925
|
+
let form = null;
|
|
2486
2926
|
try {
|
|
2487
2927
|
if (step.action === "navigate") {
|
|
2488
2928
|
const result = await this.navigate(step.target ?? step.value ?? "/");
|
|
@@ -2512,6 +2952,15 @@ export class BrowserEngine {
|
|
|
2512
2952
|
transcript.push(`${desc} → ${refusal}`);
|
|
2513
2953
|
break;
|
|
2514
2954
|
}
|
|
2955
|
+
if (BrowserEngine.ACTIVATION_KEY_RE.test(key)) {
|
|
2956
|
+
const kind = /enter/i.test(key) ? "enter" : "click";
|
|
2957
|
+
const probe = await this.probeFocusedForm();
|
|
2958
|
+
if (BrowserEngine.isTrackedSubmit(kind, probe)) {
|
|
2959
|
+
const reusable = lastCapture?.url === page.url() ? lastCapture : null;
|
|
2960
|
+
preState = await this.stateHolding(probe, reusable, reusable !== null);
|
|
2961
|
+
form = { kind, probe };
|
|
2962
|
+
}
|
|
2963
|
+
}
|
|
2515
2964
|
await page.keyboard.press(key);
|
|
2516
2965
|
}
|
|
2517
2966
|
else {
|
|
@@ -2540,8 +2989,14 @@ export class BrowserEngine {
|
|
|
2540
2989
|
transcript.push(`${desc} → ${destructiveRefusal(label || step.target, this.mode)}`);
|
|
2541
2990
|
break;
|
|
2542
2991
|
}
|
|
2543
|
-
if (step.action === "click")
|
|
2992
|
+
if (step.action === "click") {
|
|
2993
|
+
const probe = await this.probeForm(loc);
|
|
2994
|
+
if (BrowserEngine.isTrackedSubmit("click", probe)) {
|
|
2995
|
+
preState = await this.stateHolding(probe, preState, preState !== null && preState === lastCapture);
|
|
2996
|
+
form = { kind: "click", probe };
|
|
2997
|
+
}
|
|
2544
2998
|
forcedClick = (await this.resilientClick(loc, ACTION_TIMEOUT_MS)).forced;
|
|
2999
|
+
}
|
|
2545
3000
|
else if (step.action === "hover") {
|
|
2546
3001
|
const { before, bodyBefore, churning } = await this.hoverBaselines();
|
|
2547
3002
|
await loc.hover({ timeout: ACTION_TIMEOUT_MS });
|
|
@@ -2563,6 +3018,11 @@ export class BrowserEngine {
|
|
|
2563
3018
|
break;
|
|
2564
3019
|
}
|
|
2565
3020
|
}
|
|
3021
|
+
const probe = await this.probeForm(loc);
|
|
3022
|
+
if (BrowserEngine.isTrackedSubmit("enter", probe)) {
|
|
3023
|
+
preState = await this.stateHolding(probe, preState, preState !== null && preState === lastCapture);
|
|
3024
|
+
form = { kind: "enter", probe };
|
|
3025
|
+
}
|
|
2566
3026
|
await loc.press("Enter", { timeout: ACTION_TIMEOUT_MS });
|
|
2567
3027
|
}
|
|
2568
3028
|
}
|
|
@@ -2582,6 +3042,8 @@ export class BrowserEngine {
|
|
|
2582
3042
|
}
|
|
2583
3043
|
}
|
|
2584
3044
|
await this.settle();
|
|
3045
|
+
if (form)
|
|
3046
|
+
this.noteFormSubmit(preState, form.kind, form.probe);
|
|
2585
3047
|
// A plan is the RECOMMENDED way to run a mechanical sequence, so its
|
|
2586
3048
|
// steps are where most of a recorded run actually happens. Leaving
|
|
2587
3049
|
// them unframed reproduced, inside run_plan, the same hole that crawl
|
|
@@ -2596,10 +3058,12 @@ export class BrowserEngine {
|
|
|
2596
3058
|
try {
|
|
2597
3059
|
// Record the state the action LANDED on (it may be a new screen
|
|
2598
3060
|
// this plan just reached, and it deserves coverage of its own)…
|
|
2599
|
-
const { elements } = await this.collect();
|
|
3061
|
+
const { elements, forms } = await this.collect();
|
|
2600
3062
|
const url = page.url();
|
|
2601
3063
|
const fp = fingerprintState(url, elements);
|
|
2602
3064
|
this.memory.visitState(fp, url, normalizePath(url), elements.map((el) => el.key));
|
|
3065
|
+
for (const f of forms)
|
|
3066
|
+
this.memory.recordForm(fp, f.key, f.guarded);
|
|
2603
3067
|
this.memory.recordRoleAccess(this.role, normalizePath(url), "reached");
|
|
2604
3068
|
lastCapture = { fp, elements, url };
|
|
2605
3069
|
// …but mark the acted-on element in the state it came FROM, using
|
|
@@ -2676,6 +3140,11 @@ export class BrowserEngine {
|
|
|
2676
3140
|
}
|
|
2677
3141
|
/** Computed-style design audit of the current page — visual judgment material without pixels. */
|
|
2678
3142
|
async designAudit() {
|
|
3143
|
+
const { url, report } = await this.auditPage();
|
|
3144
|
+
return `URL: ${url}\n` + report;
|
|
3145
|
+
}
|
|
3146
|
+
/** The design audit's measurements, as data as well as prose; scout_design_audit and a check's crawl share it. */
|
|
3147
|
+
async auditPage() {
|
|
2679
3148
|
const page = this.requirePage();
|
|
2680
3149
|
await this.settle();
|
|
2681
3150
|
const payload = (await page.evaluate(DESIGN_COLLECT_SCRIPT));
|
|
@@ -2687,7 +3156,7 @@ export class BrowserEngine {
|
|
|
2687
3156
|
// score with chrome included and later ones don't. That is the same warm-up
|
|
2688
3157
|
// the coverage census has: nothing is knowable as "shared" until it has been
|
|
2689
3158
|
// seen on several routes.
|
|
2690
|
-
const { report, score, signatures } = analyzeDesign(payload, page.viewportSize() ?? { width: 1280, height: 900 }, this.memory?.designChromeKeys() ?? new Set());
|
|
3159
|
+
const { report, score, signatures, defects } = analyzeDesign(payload, page.viewportSize() ?? { width: 1280, height: 900 }, this.memory?.designChromeKeys() ?? new Set());
|
|
2691
3160
|
const route = normalizePath(page.url());
|
|
2692
3161
|
this.memory?.recordDesignElements(route, signatures);
|
|
2693
3162
|
// Only a page that actually produced a score counts as audited. An empty or
|
|
@@ -2699,7 +3168,7 @@ export class BrowserEngine {
|
|
|
2699
3168
|
this.memory?.markRouteFact(route, { audited: true });
|
|
2700
3169
|
}
|
|
2701
3170
|
this.logAction({ action: "design-audit", url: page.url(), result: score ? `score:${score.overall}` : undefined, ...(await this.frameFor("design-audit")) });
|
|
2702
|
-
return
|
|
3171
|
+
return { url: page.url(), report, defects, sampled: payload.records.length };
|
|
2703
3172
|
}
|
|
2704
3173
|
/**
|
|
2705
3174
|
* Scroll like a user — and notice when the page won't let you. Reports the
|