scenescout 3.14.0 → 3.15.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +25 -0
- package/README.md +27 -4
- package/dist/check-run.js +22 -11
- package/dist/ci-run.js +113 -17
- package/dist/cli.js +150 -75
- package/dist/commands.js +141 -0
- package/dist/engine/browser.js +546 -133
- package/dist/engine/check.js +77 -30
- package/dist/engine/ci.js +97 -0
- package/dist/engine/claims.js +81 -15
- package/dist/engine/collector.js +307 -65
- package/dist/engine/dedup.js +329 -9
- package/dist/engine/design.js +81 -12
- package/dist/engine/forms.js +17 -0
- package/dist/engine/memory.js +180 -36
- package/dist/engine/oracles.js +126 -1
- package/dist/engine/provider.js +4 -3
- package/dist/engine/refresh.js +270 -22
- package/dist/engine/report.js +8 -1
- package/dist/engine/scripted-login.js +342 -62
- package/dist/first-run.js +577 -0
- package/dist/login-run.js +136 -35
- package/dist/mcp-server.js +75 -16
- package/package.json +3 -2
- package/skills/scenescout/SKILL.md +3 -3
package/dist/engine/browser.js
CHANGED
|
@@ -4,13 +4,13 @@ import path from "node:path";
|
|
|
4
4
|
import { elementKey, fingerprintState, isNonPageRoute, normalizePath } from "./fingerprint.js";
|
|
5
5
|
import { AUTH_LOSS_PREFIX, JOURNEY_END, JOURNEY_START, MemoryStore, redactSecrets, TASK_SET } from "./memory.js";
|
|
6
6
|
import { normalizeTask } from "./task.js";
|
|
7
|
-
import { CLAIM_SCAN_SCRIPT, findContradictions } from "./claims.js";
|
|
7
|
+
import { CLAIM_SCAN_SCRIPT, findContradictions, INFRASTRUCTURE_WRITE_RE, OPEN_DIALOGS_SCRIPT } from "./claims.js";
|
|
8
8
|
import { POSTMESSAGE_BINDING, describeTokenPost, postMessageCaptureScript, tokenHits, tokenPostKey } from "./postmessage.js";
|
|
9
9
|
import { describeInjection, newInjections, probeQueries, probeScript, probeShape, rememberProbe } from "./injection.js";
|
|
10
10
|
import { AuthLossTracker } from "./authloss.js";
|
|
11
11
|
import { captureClip } from "./capture.js";
|
|
12
|
-
import { COLLECT_INTERACTABLES_SCRIPT, VISIBLE_SRC, geometryIssues, BROKEN_IMAGES_SCRIPT, brokenImageIssues, frameLines, hasVisibleFrame, frameElementKey, frameLabel, masksForeignName, MASKED_NAME, capForeignName, stripForeignHref, frameToPageRect, displayName, missingName, placeholderOnly, placeholderEvidence, describeControl, labelFlag, } from "./collector.js";
|
|
13
|
-
import { OracleMonitor, formatViolations, httpErrorDetail } from "./oracles.js";
|
|
12
|
+
import { COLLECT_INTERACTABLES_SCRIPT, VISIBLE_SRC, geometryIssues, BROKEN_IMAGES_SCRIPT, brokenImageIssues, frameLines, hasVisibleFrame, frameElementKey, frameLabel, masksForeignName, MASKED_NAME, capForeignName, stripForeignHref, frameToPageRect, displayName, missingName, placeholderOnly, placeholderEvidence, describeControl, labelFlag, stateFlags, stateChange, trackedElements, inertKeys, MAIN_REGION_SCRIPT, mainRegionLine, mainRegionTag, } from "./collector.js";
|
|
13
|
+
import { OracleMonitor, formatViolations, httpErrorDetail, requestKey } from "./oracles.js";
|
|
14
14
|
import { extractCreatedIds, isOwnedResource, normalizeId } from "./ownership.js";
|
|
15
15
|
import { formatJourney, measureJourney } from "./journey.js";
|
|
16
16
|
import { describeStep, FLOW_AFTER_LAST_STEP_MS, isAction, matchRequest, parseTarget, splitRefusals, TARGET_HELP, urlMatches, } from "./flow.js";
|
|
@@ -19,7 +19,7 @@ import { keepWatchingUrl, normalizePace, SETTLE_TICK_MS, shouldKeepWaiting } fro
|
|
|
19
19
|
import { buildRequestScript, formatReplay, replaySignature, requestHeaders, resolveMethod, resolveRequestUrl, toReplayResult } from "./request.js";
|
|
20
20
|
import { defaultEngine, focusAdvanceKey, REMOVE_SHARED_WORKER_SCRIPT, screencastSupport, serviceWorkerPolicy, sharedWorkersAllowed, unloadWriteInterception, closeWaitsForLeavingWrites, } from "../browsers.js";
|
|
21
21
|
import { revealedLines } from "./hover.js";
|
|
22
|
-
import { FORMS_INVENTORY_SCRIPT, FORM_PROBE_BODY, formProbeExpression, formIdentity, formStatus, FORMS_READ_FAILED, FORMS_SUBMIT_UNMATCHED, isEmptySubmit, sameControl, isNavigationTeardown, submits, tracksForm, } from "./forms.js";
|
|
22
|
+
import { FORMS_INVENTORY_SCRIPT, FORM_PROBE_BODY, formProbeExpression, formIdentity, formStatus, FORMS_READ_FAILED, FORMS_SUBMIT_UNMATCHED, isEmptySubmit, isSubmitLike, sameControl, isNavigationTeardown, submits, tracksForm, } from "./forms.js";
|
|
23
23
|
import { explainLaunchFailure, isMissingBrowser } from "./launch.js";
|
|
24
24
|
import { DEFAULT_TIME_LIMITS, explainTimeout, limitHint, resolveTimeLimits } from "./limits.js";
|
|
25
25
|
import { performScroll, probeFocusIndicators, probeOverlays, scrollContainer } from "./probes.js";
|
|
@@ -32,7 +32,7 @@ import { analyzeDesign, DESIGN_COLLECT_SCRIPT } from "./design.js";
|
|
|
32
32
|
import { acceptMatches, generatedUpload } from "./fixtures.js";
|
|
33
33
|
import { bodyDigest, headerOf, isReadMethod, judgeUnseenWrite, pausedRequestBytes, RoutedWrites, UnseenRefusals, unseenWriteSource, } from "./unload.js";
|
|
34
34
|
import { loginCommand, permissionNote, resolveAttachAuth, roleLabel, sessionStorageInitScript, splitProfile, summarizeState, writeProfile, } from "./profiles.js";
|
|
35
|
-
import { acquireLock, brokerEnabled, lockPathFor, planRefresh,
|
|
35
|
+
import { acquireLock, brokerDecision, brokerEnabled, cookieMayCount, endpointKey, endpointsPathFor, headersForResend, isStaticAsset, learnableEndpoint, lockPathFor, planRefresh, profileAfterRotation, readLearnedEndpoints, REFRESH_BROKER_ENV, refreshTokenSlots, rotatedCookies, rotatedFromResponse, rotationStored, swapProfileToken, swapRequest, swapToken, withLearnedEndpoint, writeLearnedEndpoints, } from "./refresh.js";
|
|
36
36
|
const SETTLE_MS = 400;
|
|
37
37
|
/**
|
|
38
38
|
* Request URL → pathname, falling back to the raw string for anything
|
|
@@ -59,8 +59,8 @@ const FORCED_CLICK_TIMEOUT_MS = 1500;
|
|
|
59
59
|
const CHOOSER_GRACE_MS = 2000;
|
|
60
60
|
/** How long a hover waits for delay-gated tooltips (component libraries warm up for as long as ~1500ms). */
|
|
61
61
|
const HOVER_REVEAL_WINDOW_MS = 2500;
|
|
62
|
-
/** Non-GET traffic that is auth/telemetry plumbing, not tester-caused state mutation. */
|
|
63
|
-
const BENIGN_MUTATION_RE =
|
|
62
|
+
/** Non-GET traffic that is auth/telemetry plumbing, not tester-caused state mutation. One list, shared with the false-success rule. */
|
|
63
|
+
const BENIGN_MUTATION_RE = INFRASTRUCTURE_WRITE_RE;
|
|
64
64
|
/**
|
|
65
65
|
* What the policy needs to know about where a request came from: the URLs of
|
|
66
66
|
* the frame that sent it and of its parents, up to but not including the top
|
|
@@ -239,10 +239,12 @@ export class BrowserEngine {
|
|
|
239
239
|
* profile holds no refresh token.
|
|
240
240
|
*/
|
|
241
241
|
refresh = null;
|
|
242
|
+
/** Requests the broker took the lock for: their write-back is its own, so the rotation watcher leaves them alone. */
|
|
243
|
+
brokeredRequests = new WeakSet();
|
|
242
244
|
/** Write-backs still waiting on a response, so close() lets them release their lock. */
|
|
243
245
|
refreshTasks = new Set();
|
|
244
246
|
/** What the broker did this session, for the lane report: counts only. */
|
|
245
|
-
refreshCounts = { refreshed: 0, swapped: 0, failed: 0 };
|
|
247
|
+
refreshCounts = { refreshed: 0, swapped: 0, failed: 0, learned: 0 };
|
|
246
248
|
/** UI-label blocking applies only in read-only mode (safe-write enforces at the network layer instead). */
|
|
247
249
|
get readOnly() {
|
|
248
250
|
// observe is read-only and then some: every UI-level refusal applies to it too.
|
|
@@ -367,9 +369,12 @@ export class BrowserEngine {
|
|
|
367
369
|
headers: requestHeaders({ given: input.headers, auth: this.lastAuthHeader, body: input.body }),
|
|
368
370
|
});
|
|
369
371
|
let raw;
|
|
372
|
+
// Claimed by the first matching request the page sends (the request
|
|
373
|
+
// listener), so the oracles and the contradiction ledger know it is ours.
|
|
374
|
+
this.pendingReplay = { method: method.method, key: requestKey(target.url) };
|
|
375
|
+
this.oracles.replayStarted(target.url);
|
|
370
376
|
try {
|
|
371
377
|
raw = (await page.evaluate(script));
|
|
372
|
-
this.forgetReplay(method.method, target.url);
|
|
373
378
|
}
|
|
374
379
|
catch (err) {
|
|
375
380
|
// The page could not run the fetch at all (a navigation mid-call, a
|
|
@@ -378,6 +383,12 @@ export class BrowserEngine {
|
|
|
378
383
|
this.logAction({ action: "request", target: `${method.method} ${input.path}`, url: page.url(), result: `blocked: ${message.split("\n")[0]}` });
|
|
379
384
|
return `${method.method} ${input.path} — the request did not complete: ${message.split("\n")[0]}`;
|
|
380
385
|
}
|
|
386
|
+
finally {
|
|
387
|
+
this.pendingReplay = null;
|
|
388
|
+
this.oracles.replayEnded(target.url);
|
|
389
|
+
for (const hop of this.replayHops.splice(0))
|
|
390
|
+
this.oracles.replayEnded(hop);
|
|
391
|
+
}
|
|
381
392
|
const result = toReplayResult(raw);
|
|
382
393
|
this.logAction({
|
|
383
394
|
action: "request",
|
|
@@ -388,21 +399,34 @@ export class BrowserEngine {
|
|
|
388
399
|
});
|
|
389
400
|
return formatReplay(method.method, result);
|
|
390
401
|
}
|
|
402
|
+
/** The scout_request call in flight, until the page's request for it is seen. */
|
|
403
|
+
pendingReplay = null;
|
|
404
|
+
/** Addresses the call in flight was redirected to, ended with it. */
|
|
405
|
+
replayHops = [];
|
|
391
406
|
/**
|
|
392
|
-
*
|
|
393
|
-
* agent's
|
|
394
|
-
*
|
|
395
|
-
*
|
|
396
|
-
*
|
|
407
|
+
* The requests scout_request sent, by identity (redirect hops included). They
|
|
408
|
+
* are the agent's calls, not the page's: their failures are not the page's
|
|
409
|
+
* violations, and whatever the page says next is not their answer. Left in
|
|
410
|
+
* the contradiction ledger, a replay the policy refused was blamed on the
|
|
411
|
+
* next click as that click's false success.
|
|
397
412
|
*/
|
|
398
|
-
|
|
399
|
-
|
|
400
|
-
|
|
401
|
-
|
|
402
|
-
|
|
403
|
-
|
|
404
|
-
|
|
413
|
+
replayRequests = new WeakSet();
|
|
414
|
+
/** When each request started, so the contradiction rules can tell the action's own writes from ones already in flight. */
|
|
415
|
+
requestStartedAt = new WeakMap();
|
|
416
|
+
/** Note whether a request the page is sending is this session's scout_request call. */
|
|
417
|
+
claimReplay(req) {
|
|
418
|
+
const from = req.redirectedFrom();
|
|
419
|
+
if (from && this.replayRequests.has(from)) {
|
|
420
|
+
this.replayRequests.add(req);
|
|
421
|
+
this.oracles.replayStarted(req.url());
|
|
422
|
+
this.replayHops.push(req.url());
|
|
423
|
+
return;
|
|
405
424
|
}
|
|
425
|
+
const pending = this.pendingReplay;
|
|
426
|
+
if (!pending || req.method() !== pending.method || requestKey(req.url()) !== pending.key)
|
|
427
|
+
return;
|
|
428
|
+
this.replayRequests.add(req);
|
|
429
|
+
this.pendingReplay = null;
|
|
406
430
|
}
|
|
407
431
|
/**
|
|
408
432
|
* The frame field for a log entry, or nothing. Spread into logAction so a
|
|
@@ -618,18 +642,55 @@ export class BrowserEngine {
|
|
|
618
642
|
* element on the current page? Runs wherever violations are drained, so the
|
|
619
643
|
* finding reaches the agent in the result of the action that revealed it.
|
|
620
644
|
*/
|
|
621
|
-
/**
|
|
645
|
+
/**
|
|
646
|
+
* Record one answered request for the contradiction rules. Ones the policy
|
|
647
|
+
* stopped are marked, never dropped: the rules need to know they were ours.
|
|
648
|
+
* The agent's own scout_request calls are left out: the page did not send them.
|
|
649
|
+
*/
|
|
622
650
|
watchResponse(req, status) {
|
|
651
|
+
if (this.replayRequests.has(req))
|
|
652
|
+
return;
|
|
623
653
|
if (this.watchedResponses.length >= BrowserEngine.MAX_WATCHED_RESPONSES)
|
|
624
654
|
return;
|
|
655
|
+
const started = this.requestStartedAt.get(req) ?? 0;
|
|
625
656
|
this.watchedResponses.push({
|
|
626
657
|
method: req.method(),
|
|
627
658
|
url: req.url(),
|
|
628
659
|
status,
|
|
629
660
|
resourceType: req.resourceType(),
|
|
630
661
|
blockedByPolicy: this.refusedByAnyPolicy(req),
|
|
662
|
+
// The current action is a user input only when it is the one that set the input mark.
|
|
663
|
+
background: this.inputSince === null || this.inputSince < this.actionStartedAt || started < this.inputSince,
|
|
631
664
|
});
|
|
632
665
|
}
|
|
666
|
+
/**
|
|
667
|
+
* When the current user input (a click, a keypress, typing, a choice) began,
|
|
668
|
+
* or null before the first. An action that is not input (a navigation, a
|
|
669
|
+
* scroll, a crawl) moves actionStartedAt past it, which is how a request is
|
|
670
|
+
* known to have no input behind it.
|
|
671
|
+
*/
|
|
672
|
+
inputSince = null;
|
|
673
|
+
/** What the page said as the current input began, for the contradiction rules; null when it could not be read. */
|
|
674
|
+
claimBaseline = null;
|
|
675
|
+
/**
|
|
676
|
+
* Mark the start of a user input: read what the page says now, so a claim
|
|
677
|
+
* already on screen (a status badge, a heading) is not taken for the
|
|
678
|
+
* page's answer to this input's write. Never fails the action: with no
|
|
679
|
+
* baseline every text counts, as before.
|
|
680
|
+
*/
|
|
681
|
+
async beginInput() {
|
|
682
|
+
this.inputSince = this.actionStartedAt;
|
|
683
|
+
this.claimBaseline = null;
|
|
684
|
+
const page = this.page;
|
|
685
|
+
if (!page || page.isClosed())
|
|
686
|
+
return;
|
|
687
|
+
try {
|
|
688
|
+
this.claimBaseline = (await page.evaluate(CLAIM_SCAN_SCRIPT));
|
|
689
|
+
}
|
|
690
|
+
catch {
|
|
691
|
+
// A page mid-navigation has nothing on screen to excuse.
|
|
692
|
+
}
|
|
693
|
+
}
|
|
633
694
|
/**
|
|
634
695
|
* Did the page agree with what the network just did? Runs in the same slot
|
|
635
696
|
* as the injection scan, so it sees the DOM the action settled on.
|
|
@@ -655,7 +716,9 @@ export class BrowserEngine {
|
|
|
655
716
|
// A page mid-navigation has no DOM to ask.
|
|
656
717
|
return;
|
|
657
718
|
}
|
|
658
|
-
|
|
719
|
+
// A baseline belongs to the input that took it, and to its page.
|
|
720
|
+
const before = this.inputSince !== null && this.inputSince >= this.actionStartedAt ? this.claimBaseline : null;
|
|
721
|
+
for (const found of findContradictions(requests, state, before)) {
|
|
659
722
|
if (this.contradictionsReported.has(found.evidence))
|
|
660
723
|
continue;
|
|
661
724
|
this.contradictionsReported.add(found.evidence);
|
|
@@ -754,7 +817,8 @@ export class BrowserEngine {
|
|
|
754
817
|
this.authLoss.beginSession(auth.kind, this.role);
|
|
755
818
|
let profileNote = "";
|
|
756
819
|
this.refresh = null;
|
|
757
|
-
this.refreshCounts = { refreshed: 0, swapped: 0, failed: 0 };
|
|
820
|
+
this.refreshCounts = { refreshed: 0, swapped: 0, failed: 0, learned: 0 };
|
|
821
|
+
this.brokeredRequests = new WeakSet();
|
|
758
822
|
if (auth.kind === "role") {
|
|
759
823
|
const note = permissionNote(auth.storageStatePath, fs.statSync(auth.storageStatePath).mode);
|
|
760
824
|
if (note)
|
|
@@ -762,8 +826,19 @@ export class BrowserEngine {
|
|
|
762
826
|
if (brokered) {
|
|
763
827
|
const read = this.readRoleProfile();
|
|
764
828
|
const known = read.ok ? refreshTokenSlots(read.state) : [];
|
|
765
|
-
if (known.length > 0)
|
|
766
|
-
this.refresh = {
|
|
829
|
+
if (known.length > 0) {
|
|
830
|
+
this.refresh = {
|
|
831
|
+
projectDir: opts.projectDir,
|
|
832
|
+
role: auth.role,
|
|
833
|
+
file: auth.storageStatePath,
|
|
834
|
+
known,
|
|
835
|
+
spent: [],
|
|
836
|
+
learned: new Set(),
|
|
837
|
+
learnedCheckedAt: 0,
|
|
838
|
+
learnedMtimeMs: 0,
|
|
839
|
+
};
|
|
840
|
+
this.learnedEndpoints();
|
|
841
|
+
}
|
|
767
842
|
}
|
|
768
843
|
}
|
|
769
844
|
this.currentMode = opts.mode ?? "read-only";
|
|
@@ -818,6 +893,7 @@ export class BrowserEngine {
|
|
|
818
893
|
}
|
|
819
894
|
this.oracles = new OracleMonitor();
|
|
820
895
|
this.oracles.setPolicyRefusalCheck((req) => this.refusedByAnyPolicy(req));
|
|
896
|
+
this.oracles.setReplayCheck((req) => this.replayRequests.has(req));
|
|
821
897
|
// A request an embed sends to the app is the app's to answer: only one headed outside the app is the embed's.
|
|
822
898
|
this.oracles.setEmbedAttribution((req) => {
|
|
823
899
|
let site = null;
|
|
@@ -902,6 +978,8 @@ export class BrowserEngine {
|
|
|
902
978
|
}
|
|
903
979
|
this.inFlight += 1;
|
|
904
980
|
this.lastRequestStart = Date.now();
|
|
981
|
+
this.requestStartedAt.set(req, this.lastRequestStart);
|
|
982
|
+
this.claimReplay(req);
|
|
905
983
|
const type = req.resourceType();
|
|
906
984
|
if (type === "xhr" || type === "fetch")
|
|
907
985
|
this.xhrCount += 1;
|
|
@@ -946,6 +1024,14 @@ export class BrowserEngine {
|
|
|
946
1024
|
}
|
|
947
1025
|
}
|
|
948
1026
|
});
|
|
1027
|
+
// The refresh broker. Registered before the write policy, so it runs after
|
|
1028
|
+
// it: the policy judges each request as the page sent it and hands on
|
|
1029
|
+
// (route.fallback) only what it lets out, and the broker sees only those.
|
|
1030
|
+
// A refresh the policy refuses is never sent, so it needs no lock.
|
|
1031
|
+
if (this.refresh) {
|
|
1032
|
+
await this.context.route("**/*", (route) => this.brokerRefresh(route));
|
|
1033
|
+
this.context.on("response", (res) => this.watchRotation(res));
|
|
1034
|
+
}
|
|
949
1035
|
// Write policy — enforced on the wire, where the truth lives.
|
|
950
1036
|
if (this.mode !== "destructive") {
|
|
951
1037
|
const judge = async (route) => {
|
|
@@ -1026,11 +1112,13 @@ export class BrowserEngine {
|
|
|
1026
1112
|
catch {
|
|
1027
1113
|
/* could not ask: load it as it would have loaded */
|
|
1028
1114
|
}
|
|
1029
|
-
await route.
|
|
1115
|
+
await route.fallback().catch(() => { });
|
|
1030
1116
|
return;
|
|
1031
1117
|
}
|
|
1118
|
+
// What the policy lets out it hands on with route.fallback, never route.continue: the
|
|
1119
|
+
// refresh broker, registered before this handler, runs next. With no broker it goes out.
|
|
1032
1120
|
if (method === "GET" || method === "HEAD" || method === "OPTIONS")
|
|
1033
|
-
return route.
|
|
1121
|
+
return route.fallback();
|
|
1034
1122
|
const url = req.url();
|
|
1035
1123
|
const pathname = pathnameOf(url);
|
|
1036
1124
|
const refuse = (why) => {
|
|
@@ -1062,7 +1150,7 @@ export class BrowserEngine {
|
|
|
1062
1150
|
// one, and in observe only the requests a login itself needs.
|
|
1063
1151
|
if (isAuthExempt(rule, method, pathname, destructiveWire)) {
|
|
1064
1152
|
this.routedWrites.note(rule, method, url, bodyDigest(req.postDataBuffer()));
|
|
1065
|
-
return route.
|
|
1153
|
+
return route.fallback();
|
|
1066
1154
|
}
|
|
1067
1155
|
let owned = this.isOwnedResource(pathname);
|
|
1068
1156
|
// A single UI action commonly fires create-then-immediately-save
|
|
@@ -1098,7 +1186,7 @@ export class BrowserEngine {
|
|
|
1098
1186
|
this.pendingCreations.add(task);
|
|
1099
1187
|
}
|
|
1100
1188
|
this.routedWrites.note(rule, method, url, bodyDigest(req.postDataBuffer()));
|
|
1101
|
-
return route.
|
|
1189
|
+
return route.fallback();
|
|
1102
1190
|
}
|
|
1103
1191
|
return refuse();
|
|
1104
1192
|
};
|
|
@@ -1115,11 +1203,6 @@ export class BrowserEngine {
|
|
|
1115
1203
|
}
|
|
1116
1204
|
}
|
|
1117
1205
|
}
|
|
1118
|
-
// The refresh broker. Registered after the write policy so it runs first
|
|
1119
|
-
// and hands every request on to it: a refresh it lets through is still
|
|
1120
|
-
// judged by the policy like any other request.
|
|
1121
|
-
if (this.refresh)
|
|
1122
|
-
await this.context.route("**/*", (route) => this.brokerRefresh(route));
|
|
1123
1206
|
// Popups / target=_blank: adopt same-origin pages as the active page (with
|
|
1124
1207
|
// oracles attached); close foreign-origin popups so exploration cannot
|
|
1125
1208
|
// silently escape the app under test.
|
|
@@ -1245,6 +1328,7 @@ export class BrowserEngine {
|
|
|
1245
1328
|
/** Dialogs (confirm/alert): dismiss in read-only mode, accept otherwise. Must be wired on every page we drive, including adopted popups. */
|
|
1246
1329
|
wireDialogHandler(page) {
|
|
1247
1330
|
page.on("dialog", (dialog) => {
|
|
1331
|
+
this.nativeDialogAt = Date.now();
|
|
1248
1332
|
const action = this.readOnly ? "dismiss" : "accept";
|
|
1249
1333
|
this.logAction({
|
|
1250
1334
|
action: `dialog:${action}`,
|
|
@@ -1254,6 +1338,8 @@ export class BrowserEngine {
|
|
|
1254
1338
|
void (this.readOnly ? dialog.dismiss() : dialog.accept()).catch(() => { });
|
|
1255
1339
|
});
|
|
1256
1340
|
}
|
|
1341
|
+
/** When the page last opened a native dialog (confirm, alert, prompt). */
|
|
1342
|
+
nativeDialogAt = 0;
|
|
1257
1343
|
requirePage() {
|
|
1258
1344
|
if (!this.page || !this.memory) {
|
|
1259
1345
|
throw new Error("Not attached. Call scout_attach first with the app URL and project path.");
|
|
@@ -1376,7 +1462,9 @@ export class BrowserEngine {
|
|
|
1376
1462
|
...framed.flatMap((g) => g.raws.map((raw) => ({ raw: raw, frame: g.frame, tag: g.tag }))),
|
|
1377
1463
|
];
|
|
1378
1464
|
const elements = all.map(({ raw: el, frame, tag }) => {
|
|
1379
|
-
|
|
1465
|
+
// A live region listed for what it says is known by its role, not its
|
|
1466
|
+
// text, so a new message reads as the same region saying something else.
|
|
1467
|
+
const baseKey = frameElementKey(el.liveOnly ? `live:${el.role}` : elementKey(el), tag);
|
|
1380
1468
|
const count = keyCounts.get(baseKey) ?? 0;
|
|
1381
1469
|
keyCounts.set(baseKey, count + 1);
|
|
1382
1470
|
const key = count === 0 ? baseKey : `${baseKey}~${count}`;
|
|
@@ -1385,7 +1473,8 @@ export class BrowserEngine {
|
|
|
1385
1473
|
...el,
|
|
1386
1474
|
...(tag ? { frame: tag } : {}),
|
|
1387
1475
|
// Judged on the real label, then masked: the policy must see what a click would press.
|
|
1388
|
-
|
|
1476
|
+
// A message is not a control: "Could not delete" in an alert is not a Delete button.
|
|
1477
|
+
destructive: el.liveOnly ? false : isDestructive(el.name, el.testid),
|
|
1389
1478
|
...(tag?.foreign ? { name: masksForeignName(el.tag, el.role) ? MASKED_NAME : capForeignName(el.name) } : {}),
|
|
1390
1479
|
...(tag?.foreign && el.href ? { href: stripForeignHref(el.href) } : {}),
|
|
1391
1480
|
ref,
|
|
@@ -1530,8 +1619,8 @@ export class BrowserEngine {
|
|
|
1530
1619
|
const page = this.requirePage();
|
|
1531
1620
|
const { elements, forms } = await this.collect();
|
|
1532
1621
|
const url = page.url();
|
|
1533
|
-
const fp = fingerprintState(url, elements);
|
|
1534
|
-
this.memory?.visitState(fp, url, normalizePath(url), elements.map((el) => el.key));
|
|
1622
|
+
const fp = fingerprintState(url, trackedElements(elements));
|
|
1623
|
+
this.memory?.visitState(fp, url, normalizePath(url), trackedElements(elements).map((el) => el.key), inertKeys(elements));
|
|
1535
1624
|
for (const f of forms)
|
|
1536
1625
|
this.memory?.recordForm(fp, f.key, f.guarded);
|
|
1537
1626
|
return { fp, elements, url };
|
|
@@ -1575,6 +1664,16 @@ export class BrowserEngine {
|
|
|
1575
1664
|
if (found.length > 0)
|
|
1576
1665
|
this.memory.addDiscoveredRoutes(found);
|
|
1577
1666
|
}
|
|
1667
|
+
/** What the main region holds besides controls (MAIN_REGION_SCRIPT). A failed read is logged and gives no line, not "EMPTY". */
|
|
1668
|
+
async readMainRegion(page) {
|
|
1669
|
+
try {
|
|
1670
|
+
return (await page.evaluate(MAIN_REGION_SCRIPT));
|
|
1671
|
+
}
|
|
1672
|
+
catch (err) {
|
|
1673
|
+
console.error(`[scenescout] main-region read failed: ${err instanceof Error ? err.message : String(err)}`);
|
|
1674
|
+
return null;
|
|
1675
|
+
}
|
|
1676
|
+
}
|
|
1578
1677
|
/** Every file input on the page. A failed probe is logged, not passed off as "none". */
|
|
1579
1678
|
async listFileInputs(page) {
|
|
1580
1679
|
try {
|
|
@@ -1609,9 +1708,9 @@ export class BrowserEngine {
|
|
|
1609
1708
|
const url = page.url();
|
|
1610
1709
|
this.snapshotUrl = url;
|
|
1611
1710
|
const route = normalizePath(url);
|
|
1612
|
-
const fp = fingerprintState(url, elements);
|
|
1711
|
+
const fp = fingerprintState(url, trackedElements(elements));
|
|
1613
1712
|
this.currentFingerprint = fp;
|
|
1614
|
-
const isNew = memory.visitState(fp, url, route, elements.map((el) => el.key));
|
|
1713
|
+
const isNew = memory.visitState(fp, url, route, trackedElements(elements).map((el) => el.key), inertKeys(elements));
|
|
1615
1714
|
for (const f of forms)
|
|
1616
1715
|
memory.recordForm(fp, f.key, f.guarded);
|
|
1617
1716
|
memory.recordRoleAccess(this.role, route, "reached");
|
|
@@ -1625,12 +1724,14 @@ export class BrowserEngine {
|
|
|
1625
1724
|
const line = (el) => {
|
|
1626
1725
|
const dup = el.key.match(/~(\d+)$/);
|
|
1627
1726
|
const flags = [
|
|
1727
|
+
...stateFlags(el.state),
|
|
1628
1728
|
el.testid ? `testid=${el.testid}` : null,
|
|
1629
1729
|
dup ? `copy#${Number(dup[1]) + 1}` : null,
|
|
1630
1730
|
el.disabled ? "disabled" : null,
|
|
1631
1731
|
el.destructive ? "DESTRUCTIVE" : null,
|
|
1632
1732
|
labelFlag(el),
|
|
1633
|
-
|
|
1733
|
+
// "exercised", not "done": it says an earlier action or run acted on it, and "done" read as the control's own state.
|
|
1734
|
+
memory.wasExercised(fp, el.key) ? "exercised" : null,
|
|
1634
1735
|
el.href ? `href=${el.href.slice(0, 60)}` : null,
|
|
1635
1736
|
].filter(Boolean);
|
|
1636
1737
|
return `${el.ref} ${el.role} "${displayName(el)}"${flags.length ? ` [${flags.join(", ")}]` : ""}${el.frame ? ` ⟨in ${frameLabel(el.frame)}⟩` : ""}`;
|
|
@@ -1638,7 +1739,10 @@ export class BrowserEngine {
|
|
|
1638
1739
|
// Diff mode: when re-snapshotting the same route, report only what
|
|
1639
1740
|
// changed — same idea as UI reconciliation, applied to agent context.
|
|
1640
1741
|
const prev = this.lastSnap?.route === route ? this.lastSnap : null;
|
|
1641
|
-
this.lastSnap = {
|
|
1742
|
+
this.lastSnap = {
|
|
1743
|
+
route,
|
|
1744
|
+
byKey: new Map(elements.map((el) => [el.key, { ref: el.ref, label: el.name, disabled: el.disabled, state: stateFlags(el.state) }])),
|
|
1745
|
+
};
|
|
1642
1746
|
let body;
|
|
1643
1747
|
if (!full && prev) {
|
|
1644
1748
|
const currentKeys = new Set(elements.map((el) => el.key));
|
|
@@ -1660,9 +1764,16 @@ export class BrowserEngine {
|
|
|
1660
1764
|
const old = prev.byKey.get(el.key);
|
|
1661
1765
|
return old !== undefined && old.disabled !== el.disabled;
|
|
1662
1766
|
});
|
|
1663
|
-
|
|
1767
|
+
// A filter pill that became the active one, a tab now selected, a
|
|
1768
|
+
// section now open: the same element, in a different state.
|
|
1769
|
+
const restated = elements.flatMap((el) => {
|
|
1770
|
+
const old = prev.byKey.get(el.key);
|
|
1771
|
+
const change = old ? stateChange(old.state, stateFlags(el.state)) : null;
|
|
1772
|
+
return change ? [{ el, change }] : [];
|
|
1773
|
+
});
|
|
1774
|
+
const changedKeys = new Set([...relabeled, ...retoggled, ...restated.map((r) => r.el)].map((el) => el.key));
|
|
1664
1775
|
const unchanged = elements.length - added.length - changedKeys.size;
|
|
1665
|
-
if (added.length === 0 && removed.length === 0 && relabeled.length === 0 && retoggled.length === 0) {
|
|
1776
|
+
if (added.length === 0 && removed.length === 0 && relabeled.length === 0 && retoggled.length === 0 && restated.length === 0) {
|
|
1666
1777
|
body = `No element changes since the last snapshot (${unchanged} interactables, refs unchanged).`;
|
|
1667
1778
|
}
|
|
1668
1779
|
else {
|
|
@@ -1671,13 +1782,17 @@ export class BrowserEngine {
|
|
|
1671
1782
|
[
|
|
1672
1783
|
...added.map((el) => `+ ${line(el)}`),
|
|
1673
1784
|
...removed.map(([key, v]) => `- ${v.ref} "${v.label}" (gone: ${key})`),
|
|
1674
|
-
|
|
1785
|
+
// A live region saying something new is the message itself, so it is shown in full like a new element.
|
|
1786
|
+
...relabeled.map((el) => el.liveOnly
|
|
1787
|
+
? `~ ${el.ref} ${el.role} "${displayName(el)}" (was ${prev.byKey.get(el.key)?.label ? `"${prev.byKey.get(el.key)?.label}"` : "empty"})`
|
|
1788
|
+
: `~ ${el.ref} relabeled → "${el.name}"`),
|
|
1675
1789
|
...retoggled.map((el) => `~ ${el.ref} "${el.name}" is now ${el.disabled ? "DISABLED" : "ENABLED"}`),
|
|
1790
|
+
...restated.map(({ el, change }) => `~ ${el.ref} "${displayName(el)}" ${change}`),
|
|
1676
1791
|
].join("\n");
|
|
1677
1792
|
}
|
|
1678
1793
|
}
|
|
1679
1794
|
else {
|
|
1680
|
-
const missingTestids = elements.filter((el) => !el.testid && !el.disabled).length;
|
|
1795
|
+
const missingTestids = trackedElements(elements).filter((el) => !el.testid && !el.disabled).length;
|
|
1681
1796
|
body =
|
|
1682
1797
|
`Interactables (${elements.length}${truncated ? "+ — TRUNCATED at 150, dense page" : ""}${missingTestids ? `, ${missingTestids} missing data-testid` : ""}):\n` +
|
|
1683
1798
|
elements.map(line).join("\n");
|
|
@@ -1690,10 +1805,12 @@ export class BrowserEngine {
|
|
|
1690
1805
|
const cov = memory.coverage();
|
|
1691
1806
|
const unvisited = this.unvisitedKnownRoutes();
|
|
1692
1807
|
const title = await page.title();
|
|
1808
|
+
const main = await this.readMainRegion(page);
|
|
1693
1809
|
return (`URL: ${url}\nTitle: ${title}\nState: ${fp} ${isNew ? "(NEW state)" : "(revisited)"}\n` +
|
|
1694
1810
|
`Coverage: ${cov.states} states known · ${cov.elementsExercised}/${cov.elementsTotal} elements exercised` +
|
|
1695
1811
|
(this.allKnownRoutes().length > 0 ? ` · routes ${this.allKnownRoutes().length - unvisited.length}/${this.allKnownRoutes().length} visited` : "") +
|
|
1696
1812
|
`\n` +
|
|
1813
|
+
(main ? `${mainRegionLine(main)}\n` : "") +
|
|
1697
1814
|
body +
|
|
1698
1815
|
(geometry.length > 0 ? `\nGEOMETRY issues:\n` + geometry.map((g) => ` ⚠ ${g}`).join("\n") : "") +
|
|
1699
1816
|
(brokenImages.length > 0 ? `\nBROKEN IMAGES:\n` + brokenImages.map((b) => ` ⚠ ${b}`).join("\n") : "") +
|
|
@@ -1712,7 +1829,7 @@ export class BrowserEngine {
|
|
|
1712
1829
|
: "") +
|
|
1713
1830
|
this.socketNotice() +
|
|
1714
1831
|
formatViolations(this.oracles.drain()) +
|
|
1715
|
-
(elements.length === 0
|
|
1832
|
+
(trackedElements(elements).length === 0
|
|
1716
1833
|
? hasVisibleFrame(frames)
|
|
1717
1834
|
? "\n⚠ No interactable elements in the page itself: what it shows is inside the frames listed above, which were not explored."
|
|
1718
1835
|
: "\n⚠ DEAD END: no interactable elements found on this page."
|
|
@@ -1745,6 +1862,7 @@ export class BrowserEngine {
|
|
|
1745
1862
|
if (!live) {
|
|
1746
1863
|
throw new Error(`Element ${ref} no longer exists in the DOM — take a new scout_snapshot.`);
|
|
1747
1864
|
}
|
|
1865
|
+
await this.beginInput();
|
|
1748
1866
|
if (el.testid && live.testid !== el.testid) {
|
|
1749
1867
|
this.refs.clear();
|
|
1750
1868
|
throw new Error(`Element under ${ref} changed (expected testid=${el.testid}, found ${live.testid ?? "none"}) — the DOM shifted; take a new scout_snapshot.`);
|
|
@@ -1770,7 +1888,31 @@ export class BrowserEngine {
|
|
|
1770
1888
|
}
|
|
1771
1889
|
return null;
|
|
1772
1890
|
}
|
|
1773
|
-
|
|
1891
|
+
/**
|
|
1892
|
+
* Re-rank a page error this click raised when it reads as a router
|
|
1893
|
+
* cancelling a route change (oracles.ts isRouteCancellation). The dialog
|
|
1894
|
+
* count is read only when there is such an error to judge.
|
|
1895
|
+
*/
|
|
1896
|
+
async rankRouteCancellations(click, url) {
|
|
1897
|
+
const since = this.actionStartedAt;
|
|
1898
|
+
if (!this.oracles.hasPageErrorSince(since))
|
|
1899
|
+
return;
|
|
1900
|
+
let dialogsNow = null;
|
|
1901
|
+
try {
|
|
1902
|
+
dialogsNow = (await this.requirePage().evaluate(OPEN_DIALOGS_SCRIPT));
|
|
1903
|
+
}
|
|
1904
|
+
catch {
|
|
1905
|
+
// A page mid-navigation: only a native dialog can count.
|
|
1906
|
+
}
|
|
1907
|
+
const ariaDialogOpened = dialogsNow !== null && click.dialogsBefore !== null && dialogsNow > click.dialogsBefore;
|
|
1908
|
+
this.oracles.downgradeRouteCancellations(since, {
|
|
1909
|
+
byClick: true,
|
|
1910
|
+
viaLink: click.viaLink,
|
|
1911
|
+
urlChanged: url !== click.urlBefore,
|
|
1912
|
+
dialogOpened: ariaDialogOpened || this.nativeDialogAt >= since,
|
|
1913
|
+
});
|
|
1914
|
+
}
|
|
1915
|
+
async afterAction(action, target, click) {
|
|
1774
1916
|
const page = this.requirePage();
|
|
1775
1917
|
await this.settle();
|
|
1776
1918
|
let url = page.url();
|
|
@@ -1797,6 +1939,8 @@ export class BrowserEngine {
|
|
|
1797
1939
|
this.logAction({ action, target, url, ...(frame ? { frame } : {}) });
|
|
1798
1940
|
await this.scanForInjections();
|
|
1799
1941
|
await this.scanForContradictions();
|
|
1942
|
+
if (click)
|
|
1943
|
+
await this.rankRouteCancellations(click, url);
|
|
1800
1944
|
const violations = this.oracles.drain();
|
|
1801
1945
|
const mutations = this.drainMutations() + this.drainBlocked() + this.drainCreated();
|
|
1802
1946
|
const navigated = this.snapshotUrl !== "" && url !== this.snapshotUrl;
|
|
@@ -2314,7 +2458,8 @@ export class BrowserEngine {
|
|
|
2314
2458
|
// Submit-shaped clicks that fire zero network requests are a smell
|
|
2315
2459
|
// (silent no-op forms): capture the count before to compare after.
|
|
2316
2460
|
const xhrBefore = this.xhrCount;
|
|
2317
|
-
const submitLike = el.role
|
|
2461
|
+
const submitLike = isSubmitLike(el.role, el.name, el.testid);
|
|
2462
|
+
const clickContext = { viaLink: el.role === "link", urlBefore: page.url(), dialogsBefore: this.claimBaseline?.dialogs ?? null };
|
|
2318
2463
|
const clickTarget = this.scopeOf(el).locator(`xpath=${el.xpath}`);
|
|
2319
2464
|
// Read before the click: what the form's fields hold when it goes. Only a
|
|
2320
2465
|
// button or an input can submit a form; nothing else is asked.
|
|
@@ -2323,7 +2468,7 @@ export class BrowserEngine {
|
|
|
2323
2468
|
const { forced } = await this.resilientClick(clickTarget, this.limits.actionMs, clicks);
|
|
2324
2469
|
this.memory.markExercised(this.currentFingerprint, el.key, clicks > 1 ? `click×${clicks}` : "click");
|
|
2325
2470
|
this.noteFormSubmit(formState, "click", form);
|
|
2326
|
-
const result = await this.afterAction(clicks > 1 ? `click×${clicks}` : "click", `${el.role} "${el.name}"
|
|
2471
|
+
const result = await this.afterAction(clicks > 1 ? `click×${clicks}` : "click", `${el.role} "${el.name}"`, clickContext);
|
|
2327
2472
|
// Impatient-user probe: a rapid multi-click that fires the SAME
|
|
2328
2473
|
// state-changing request more than once means the action is not guarded
|
|
2329
2474
|
// against double submission (button not disabled during flight, endpoint
|
|
@@ -2347,6 +2492,14 @@ export class BrowserEngine {
|
|
|
2347
2492
|
// beacon is not counted as xhr/fetch, so an aborted one looks exactly like
|
|
2348
2493
|
// "fired nothing" — and the note would blame the app for the tool's block.
|
|
2349
2494
|
if (submitLike && this.xhrCount === xhrBefore && this.lastActionBlocked === 0 && page.url() === this.snapshotUrl) {
|
|
2495
|
+
// A client-side router can move the page just after the request-based
|
|
2496
|
+
// settle, with no request for it to wait on: watch the URL before
|
|
2497
|
+
// calling the click silent, and look at the requests again after.
|
|
2498
|
+
const landed = await this.stableUrl();
|
|
2499
|
+
if (landed !== this.snapshotUrl)
|
|
2500
|
+
return result + `\nℹ The page then moved client-side to ${landed} (take a new snapshot).` + forcedNote;
|
|
2501
|
+
if (this.xhrCount !== xhrBefore)
|
|
2502
|
+
return result + forcedNote;
|
|
2350
2503
|
return (result +
|
|
2351
2504
|
`\nℹ NOTE: this submit-style click fired ZERO network requests and no navigation — if the UI showed success, the data may have been silently discarded (worth verifying; category: other/silent-failure).` +
|
|
2352
2505
|
forcedNote);
|
|
@@ -2634,7 +2787,8 @@ export class BrowserEngine {
|
|
|
2634
2787
|
async measureLayout(page, elements, url) {
|
|
2635
2788
|
const viewport = page.viewportSize() ?? { width: 1280, height: 900 };
|
|
2636
2789
|
const byDocument = new Map();
|
|
2637
|
-
|
|
2790
|
+
// A live region is a message, not a control: its box overlapping a control is not a collision.
|
|
2791
|
+
for (const el of trackedElements(elements)) {
|
|
2638
2792
|
const doc = el.frame ? el.key.slice(0, el.key.indexOf("|") + 1) : "";
|
|
2639
2793
|
byDocument.set(doc, [...(byDocument.get(doc) ?? []), el]);
|
|
2640
2794
|
}
|
|
@@ -2848,6 +3002,7 @@ export class BrowserEngine {
|
|
|
2848
3002
|
async pressNow(key) {
|
|
2849
3003
|
this.actionStartedAt = Date.now();
|
|
2850
3004
|
const page = this.requirePage();
|
|
3005
|
+
await this.beginInput();
|
|
2851
3006
|
const refusal = await this.vetFocusedActivation(key);
|
|
2852
3007
|
if (refusal)
|
|
2853
3008
|
return refusal;
|
|
@@ -3051,113 +3206,344 @@ export class BrowserEngine {
|
|
|
3051
3206
|
return { ok: true };
|
|
3052
3207
|
}
|
|
3053
3208
|
/**
|
|
3054
|
-
* The refresh broker's route handler
|
|
3055
|
-
*
|
|
3056
|
-
*
|
|
3057
|
-
*
|
|
3058
|
-
*
|
|
3059
|
-
*
|
|
3060
|
-
*
|
|
3061
|
-
*
|
|
3209
|
+
* The refresh broker's route handler, run after the write policy has let a
|
|
3210
|
+
* request out. brokerDecision (refresh.ts) says which requests are refresh
|
|
3211
|
+
* calls; every other request goes on untouched, and a static asset is
|
|
3212
|
+
* decided before the first await so it is never held back. A refresh waits
|
|
3213
|
+
* for the role's lock; holding it, the session re-reads the profile, and
|
|
3214
|
+
* when another session has rotated the token meanwhile it loads that
|
|
3215
|
+
* profile and sends the current token in place of the spent one. The lock
|
|
3216
|
+
* is kept until the rotation is written back. Whatever goes wrong here, the
|
|
3217
|
+
* request goes on as the page sent it, with a line in the action log: an
|
|
3218
|
+
* app whose requests fail is worse than the rare double refresh. Token
|
|
3219
|
+
* values are never logged.
|
|
3062
3220
|
*/
|
|
3063
3221
|
async brokerRefresh(route) {
|
|
3064
|
-
const broker = this.refresh;
|
|
3065
3222
|
const req = route.request();
|
|
3066
|
-
|
|
3067
|
-
|
|
3068
|
-
|
|
3069
|
-
|
|
3070
|
-
|
|
3071
|
-
|
|
3072
|
-
|
|
3073
|
-
|
|
3074
|
-
|
|
3223
|
+
const broker = this.refresh;
|
|
3224
|
+
const method = req.method();
|
|
3225
|
+
if (!broker || isStaticAsset(method, req.resourceType()))
|
|
3226
|
+
return this.passOn(route);
|
|
3227
|
+
const where = `${method} ${pathnameOf(req.url())}`;
|
|
3228
|
+
let decision;
|
|
3229
|
+
try {
|
|
3230
|
+
const tokens = [...broker.known, ...broker.spent];
|
|
3231
|
+
const learned = this.learnedEndpoints();
|
|
3232
|
+
const base = { method, url: req.url(), resourceType: req.resourceType(), body: req.postData() };
|
|
3233
|
+
decision = brokerDecision({ ...base, headers: req.headers() }, tokens, learned);
|
|
3234
|
+
// The Cookie header is read only for a request it could make a refresh call:
|
|
3235
|
+
// a refresh cookie scoped to "/" rides on every request the page makes.
|
|
3236
|
+
if (decision.kind === "pass" && decision.why === "no known token" && cookieMayCount(method, base.url, learned)) {
|
|
3237
|
+
decision = brokerDecision({ ...base, headers: await this.withCookieHeader(req) }, tokens, learned);
|
|
3238
|
+
}
|
|
3239
|
+
}
|
|
3240
|
+
catch (err) {
|
|
3241
|
+
return this.unbrokered(route, where, err);
|
|
3242
|
+
}
|
|
3243
|
+
if (decision.kind === "pass")
|
|
3244
|
+
return this.passOn(route);
|
|
3075
3245
|
let lock;
|
|
3076
3246
|
try {
|
|
3077
3247
|
lock = await acquireLock(lockPathFor(broker.file));
|
|
3078
3248
|
}
|
|
3079
3249
|
catch (err) {
|
|
3080
|
-
this.
|
|
3081
|
-
this.logAction({
|
|
3082
|
-
action: "refresh-broker:dropped",
|
|
3083
|
-
target: `${where} (${err instanceof Error ? err.message : String(err)})`,
|
|
3084
|
-
url: this.page?.url() ?? "",
|
|
3085
|
-
});
|
|
3086
|
-
await route.abort("blockedbyclient").catch(() => { });
|
|
3087
|
-
return;
|
|
3250
|
+
return this.unbrokered(route, where, err);
|
|
3088
3251
|
}
|
|
3252
|
+
this.brokeredRequests.add(req);
|
|
3253
|
+
const sent = decision.sent;
|
|
3089
3254
|
let handedOn = false;
|
|
3090
3255
|
try {
|
|
3091
3256
|
const read = this.readRoleProfile();
|
|
3092
3257
|
if (!read.ok)
|
|
3093
3258
|
throw new Error(read.why);
|
|
3094
3259
|
const plan = planRefresh(sent, refreshTokenSlots(read.state));
|
|
3095
|
-
|
|
3096
|
-
|
|
3097
|
-
|
|
3098
|
-
|
|
3099
|
-
|
|
3100
|
-
|
|
3101
|
-
|
|
3102
|
-
|
|
3103
|
-
|
|
3260
|
+
if (plan.kind === "unknown") {
|
|
3261
|
+
// The profile holds another sign-in now: send this one on, and leave the profile alone.
|
|
3262
|
+
this.logAction({ action: "refresh-broker", target: `${where} with ${sent.slot} (no longer in the profile; sent as is)`, url: this.page?.url() ?? "" });
|
|
3263
|
+
handedOn = true;
|
|
3264
|
+
lock.release();
|
|
3265
|
+
return this.passOn(route);
|
|
3266
|
+
}
|
|
3267
|
+
if (plan.kind === "send") {
|
|
3268
|
+
this.logAction({ action: "refresh-broker", target: `${where} with ${sent.slot}`, url: this.page?.url() ?? "" });
|
|
3269
|
+
handedOn = true;
|
|
3270
|
+
this.trackRefreshTask(this.writeBackAfter(this.pageAnswer(req), sent, lock));
|
|
3271
|
+
return this.passOn(route);
|
|
3272
|
+
}
|
|
3273
|
+
// Another session rotated the token while this one waited: load what it saved, and send the current token.
|
|
3274
|
+
const applied = await this.applyState(read.state);
|
|
3275
|
+
if (!applied.ok)
|
|
3276
|
+
throw new Error(applied.why);
|
|
3277
|
+
this.refreshCounts.swapped += 1;
|
|
3278
|
+
const all = await req.allHeaders();
|
|
3279
|
+
const cookieLine = Object.entries(all).find(([name]) => name.toLowerCase() === "cookie")?.[1] ?? "";
|
|
3280
|
+
const swapped = swapRequest({ url: req.url(), body: req.postData(), headers: all }, sent.value, plan.to.value);
|
|
3281
|
+
if (swapToken(cookieLine, sent.value, plan.to.value) === cookieLine) {
|
|
3282
|
+
// Not in the Cookie header the request shows: a route override carries a token
|
|
3283
|
+
// in its body, URL or headers, and a browser that adds cookies after this
|
|
3284
|
+
// handler (WebKit) takes them from the jar the profile was just loaded into.
|
|
3285
|
+
this.logAction({
|
|
3286
|
+
action: "refresh-broker",
|
|
3287
|
+
target: `${where} with ${plan.to.slot} (rotated by another session; loaded its profile)`,
|
|
3288
|
+
url: this.page?.url() ?? "",
|
|
3289
|
+
});
|
|
3290
|
+
handedOn = true;
|
|
3291
|
+
this.trackRefreshTask(this.writeBackAfter(this.pageAnswer(req), plan.to, lock));
|
|
3292
|
+
return this.passOn(route, {
|
|
3104
3293
|
...(swapped.url ? { url: swapped.url } : {}),
|
|
3105
3294
|
...(swapped.body !== undefined ? { postData: swapped.body } : {}),
|
|
3106
|
-
...(swapped.headers ? { headers: swapped.headers } : {}),
|
|
3107
|
-
};
|
|
3108
|
-
|
|
3109
|
-
|
|
3295
|
+
...(swapped.headers ? { headers: headersForResend(swapped.headers, null) } : {}),
|
|
3296
|
+
});
|
|
3297
|
+
}
|
|
3298
|
+
// In the Cookie header, which a browser keeps as it built it whatever a
|
|
3299
|
+
// route override says: the broker sends the request itself and answers
|
|
3300
|
+
// the page with the response. Only a script's request; anything else
|
|
3301
|
+
// goes on as the page sent it.
|
|
3302
|
+
handedOn = true;
|
|
3303
|
+
const type = req.resourceType();
|
|
3304
|
+
if (type !== "fetch" && type !== "xhr") {
|
|
3305
|
+
lock.release();
|
|
3306
|
+
return this.unbrokered(route, where, new Error(`the spent token is in its Cookie header, and a ${type} request is not one the broker answers itself`));
|
|
3307
|
+
}
|
|
3308
|
+
let response;
|
|
3309
|
+
try {
|
|
3310
|
+
response = await route.fetch({
|
|
3311
|
+
...(swapped.url ? { url: swapped.url } : {}),
|
|
3312
|
+
...(swapped.body !== undefined ? { postData: swapped.body } : {}),
|
|
3313
|
+
headers: headersForResend(swapped.headers ?? all, swapToken(cookieLine, sent.value, plan.to.value)),
|
|
3314
|
+
maxRedirects: 0,
|
|
3315
|
+
});
|
|
3316
|
+
}
|
|
3317
|
+
catch (err) {
|
|
3318
|
+
lock.release();
|
|
3319
|
+
return this.unbrokered(route, where, err);
|
|
3110
3320
|
}
|
|
3111
3321
|
this.logAction({
|
|
3112
3322
|
action: "refresh-broker",
|
|
3113
|
-
target: `${where} with ${
|
|
3323
|
+
target: `${where} with ${plan.to.slot} (rotated by another session; loaded its profile and sent the current cookie)`,
|
|
3114
3324
|
url: this.page?.url() ?? "",
|
|
3115
3325
|
});
|
|
3116
|
-
|
|
3117
|
-
|
|
3118
|
-
|
|
3119
|
-
|
|
3120
|
-
|
|
3326
|
+
// The response's cookies are already in the browser's jar, so the write-back does not wait on the page.
|
|
3327
|
+
this.trackRefreshTask(this.writeBackAfter(Promise.resolve(BrowserEngine.fetchedAnswer(response)), plan.to, lock));
|
|
3328
|
+
await route.fulfill({ response }).catch(() => {
|
|
3329
|
+
/* the page went away before its answer: the rotation is still saved */
|
|
3330
|
+
});
|
|
3331
|
+
}
|
|
3332
|
+
catch (err) {
|
|
3333
|
+
if (handedOn) {
|
|
3334
|
+
this.logAction({
|
|
3335
|
+
action: "refresh-broker:error",
|
|
3336
|
+
target: `${where} (${err instanceof Error ? err.message.split("\n")[0] : String(err)})`,
|
|
3337
|
+
url: this.page?.url() ?? "",
|
|
3338
|
+
});
|
|
3121
3339
|
return;
|
|
3122
3340
|
}
|
|
3123
|
-
|
|
3124
|
-
this.
|
|
3125
|
-
|
|
3341
|
+
lock.release();
|
|
3342
|
+
return this.unbrokered(route, where, err);
|
|
3343
|
+
}
|
|
3344
|
+
}
|
|
3345
|
+
/**
|
|
3346
|
+
* A request's headers with the Cookie header it will be sent with. Some
|
|
3347
|
+
* browsers (WebKit) add cookies only after the route handler has run, so
|
|
3348
|
+
* when the request shows none, the line is built from the browser's jar
|
|
3349
|
+
* for that URL, which is what will be sent.
|
|
3350
|
+
*/
|
|
3351
|
+
async withCookieHeader(req) {
|
|
3352
|
+
const headers = await req.allHeaders();
|
|
3353
|
+
if (Object.keys(headers).some((name) => name.toLowerCase() === "cookie") || !this.context)
|
|
3354
|
+
return headers;
|
|
3355
|
+
const jar = await this.context.cookies(req.url());
|
|
3356
|
+
return jar.length > 0 ? { ...headers, cookie: jar.map((c) => `${c.name}=${c.value}`).join("; ") } : headers;
|
|
3357
|
+
}
|
|
3358
|
+
/** Hand a request on as it is, or with overrides. A request the page has already cancelled rejects this; nothing is left to send then. */
|
|
3359
|
+
async passOn(route, overrides) {
|
|
3360
|
+
await route.fallback(overrides).catch(() => { });
|
|
3361
|
+
}
|
|
3362
|
+
/** The broker could not do its job for this request: it goes on as the page sent it, and the action log says why. */
|
|
3363
|
+
async unbrokered(route, where, err) {
|
|
3364
|
+
this.refreshCounts.failed += 1;
|
|
3365
|
+
// Unbrokered after all: if its response rotates the cookie, the rotation watcher saves it.
|
|
3366
|
+
this.brokeredRequests.delete(route.request());
|
|
3367
|
+
this.logAction({
|
|
3368
|
+
action: "refresh-broker:unbrokered",
|
|
3369
|
+
target: `${where} sent as the page sent it (${err instanceof Error ? err.message.split("\n")[0] : String(err)})`,
|
|
3370
|
+
url: this.page?.url() ?? "",
|
|
3371
|
+
});
|
|
3372
|
+
return this.passOn(route);
|
|
3373
|
+
}
|
|
3374
|
+
/** Track a write-back so close() waits for it to release the role's lock. */
|
|
3375
|
+
trackRefreshTask(task) {
|
|
3376
|
+
const tracked = task.finally(() => this.refreshTasks.delete(tracked));
|
|
3377
|
+
this.refreshTasks.add(tracked);
|
|
3378
|
+
}
|
|
3379
|
+
/** The page's response to a request, or null if none came in 30 seconds. */
|
|
3380
|
+
pageAnswer(req) {
|
|
3381
|
+
let timer;
|
|
3382
|
+
const answered = req
|
|
3383
|
+
.response()
|
|
3384
|
+
.then((res) => res ? { ok: res.ok(), status: res.status(), text: () => res.text(), setCookies: () => res.headerValues("set-cookie") } : null)
|
|
3385
|
+
.finally(() => clearTimeout(timer));
|
|
3386
|
+
return Promise.race([answered, new Promise((resolve) => (timer = setTimeout(() => resolve(null), 30000)))]);
|
|
3387
|
+
}
|
|
3388
|
+
/** A response the broker fetched itself, read the way the write-back reads one. */
|
|
3389
|
+
static fetchedAnswer(res) {
|
|
3390
|
+
return {
|
|
3391
|
+
ok: res.ok(),
|
|
3392
|
+
status: res.status(),
|
|
3393
|
+
text: () => res.text(),
|
|
3394
|
+
setCookies: async () => res
|
|
3395
|
+
.headersArray()
|
|
3396
|
+
.filter((h) => h.name.toLowerCase() === "set-cookie")
|
|
3397
|
+
.map((h) => h.value),
|
|
3398
|
+
};
|
|
3399
|
+
}
|
|
3400
|
+
/**
|
|
3401
|
+
* The endpoints learned to rotate the role's token: this session's, and
|
|
3402
|
+
* those any session of the role saved beside the profile, re-read at most
|
|
3403
|
+
* twice a second and only when the file changed.
|
|
3404
|
+
*/
|
|
3405
|
+
learnedEndpoints() {
|
|
3406
|
+
const broker = this.refresh;
|
|
3407
|
+
if (!broker)
|
|
3408
|
+
return new Set();
|
|
3409
|
+
const now = Date.now();
|
|
3410
|
+
if (now - broker.learnedCheckedAt < 500)
|
|
3411
|
+
return broker.learned;
|
|
3412
|
+
broker.learnedCheckedAt = now;
|
|
3413
|
+
const file = endpointsPathFor(broker.file);
|
|
3414
|
+
try {
|
|
3415
|
+
const mtimeMs = fs.statSync(file).mtimeMs;
|
|
3416
|
+
if (mtimeMs !== broker.learnedMtimeMs) {
|
|
3417
|
+
broker.learnedMtimeMs = mtimeMs;
|
|
3418
|
+
for (const key of readLearnedEndpoints(file))
|
|
3419
|
+
broker.learned.add(key);
|
|
3420
|
+
}
|
|
3126
3421
|
}
|
|
3127
3422
|
catch (err) {
|
|
3128
|
-
|
|
3129
|
-
|
|
3130
|
-
this.refreshCounts.failed += 1;
|
|
3423
|
+
// No file yet means nothing learned yet. Any other error keeps what this session already knows.
|
|
3424
|
+
if (err.code !== "ENOENT")
|
|
3131
3425
|
this.logAction({
|
|
3132
|
-
action: "refresh-broker:
|
|
3133
|
-
target:
|
|
3426
|
+
action: "refresh-broker:error",
|
|
3427
|
+
target: `could not read the learned endpoints (${err instanceof Error ? err.message.split("\n")[0] : String(err)})`,
|
|
3428
|
+
url: "",
|
|
3429
|
+
});
|
|
3430
|
+
}
|
|
3431
|
+
return broker.learned;
|
|
3432
|
+
}
|
|
3433
|
+
/**
|
|
3434
|
+
* A response that replaced the role's refresh cookie with a new token on a
|
|
3435
|
+
* request the broker did not take the lock for: that endpoint rotates the
|
|
3436
|
+
* token. It is learned, for this session at once and for every session of
|
|
3437
|
+
* the role through the file beside the profile, so the next call to it is
|
|
3438
|
+
* brokered; and the rotation is written back, so the other sessions do not
|
|
3439
|
+
* go on presenting the token this one spent. Only a POST, PUT or PATCH is
|
|
3440
|
+
* learned, and never a sign-in, sign-up or sign-out (learnableEndpoint).
|
|
3441
|
+
*/
|
|
3442
|
+
watchRotation(res) {
|
|
3443
|
+
const broker = this.refresh;
|
|
3444
|
+
if (!broker || !broker.known.some((t) => t.cookie !== undefined))
|
|
3445
|
+
return;
|
|
3446
|
+
const req = res.request();
|
|
3447
|
+
// A GET is never learned: a server that re-issues the cookie on its reads would otherwise put them all behind the lock.
|
|
3448
|
+
const method = req.method();
|
|
3449
|
+
if (method !== "POST" && method !== "PUT" && method !== "PATCH")
|
|
3450
|
+
return;
|
|
3451
|
+
if (this.brokeredRequests.has(req) || !res.ok() || !learnableEndpoint(req.url()))
|
|
3452
|
+
return;
|
|
3453
|
+
this.trackRefreshTask((async () => {
|
|
3454
|
+
const rotated = rotatedCookies(await res.headerValues("set-cookie"), broker.known);
|
|
3455
|
+
if (rotated.length > 0)
|
|
3456
|
+
await this.adoptRotation(endpointKey(req.method(), req.url()), rotated);
|
|
3457
|
+
})().catch((err) => {
|
|
3458
|
+
this.logAction({
|
|
3459
|
+
action: "refresh-broker:error",
|
|
3460
|
+
target: `${pathnameOf(req.url())}: ${err instanceof Error ? err.message.split("\n")[0] : String(err)}`,
|
|
3461
|
+
url: this.page?.url() ?? "",
|
|
3462
|
+
});
|
|
3463
|
+
}));
|
|
3464
|
+
}
|
|
3465
|
+
/** Learn an endpoint seen to rotate the token, and write the rotation back if the profile still holds the token it replaced. */
|
|
3466
|
+
async adoptRotation(key, rotated) {
|
|
3467
|
+
const broker = this.refresh;
|
|
3468
|
+
if (!broker)
|
|
3469
|
+
return;
|
|
3470
|
+
const isNew = !broker.learned.has(key);
|
|
3471
|
+
broker.learned.add(key);
|
|
3472
|
+
if (isNew)
|
|
3473
|
+
this.refreshCounts.learned += 1;
|
|
3474
|
+
const names = rotated.map((r) => r.slot).join(", ");
|
|
3475
|
+
let lock;
|
|
3476
|
+
try {
|
|
3477
|
+
lock = await acquireLock(lockPathFor(broker.file));
|
|
3478
|
+
}
|
|
3479
|
+
catch (err) {
|
|
3480
|
+
this.logAction({
|
|
3481
|
+
action: "refresh-broker:learned",
|
|
3482
|
+
target: `${key} rotates ${names}; not saved (${err instanceof Error ? err.message.split("\n")[0] : String(err)})`,
|
|
3483
|
+
url: this.page?.url() ?? "",
|
|
3484
|
+
});
|
|
3485
|
+
return;
|
|
3486
|
+
}
|
|
3487
|
+
try {
|
|
3488
|
+
const file = endpointsPathFor(broker.file);
|
|
3489
|
+
const list = readLearnedEndpoints(file);
|
|
3490
|
+
if (!list.includes(key))
|
|
3491
|
+
writeLearnedEndpoints(file, withLearnedEndpoint(list, key));
|
|
3492
|
+
const read = this.readRoleProfile();
|
|
3493
|
+
if (!read.ok)
|
|
3494
|
+
throw new Error(read.why);
|
|
3495
|
+
const current = refreshTokenSlots(read.state);
|
|
3496
|
+
// Only over the token this session spent: a profile another session moved on since is left as it is.
|
|
3497
|
+
const stillSpent = rotated.every((r) => current.some((c) => c.slot === r.slot && c.value === r.value));
|
|
3498
|
+
let now = undefined;
|
|
3499
|
+
// The jar takes the Set-Cookie as the response arrives; a short wait covers a browser that is a moment behind.
|
|
3500
|
+
for (let waited = 0; stillSpent && waited <= 1000; waited += 50) {
|
|
3501
|
+
const state = await this.context?.storageState().catch(() => undefined);
|
|
3502
|
+
if (state && rotated.every((r) => rotationStored(state, r))) {
|
|
3503
|
+
now = state;
|
|
3504
|
+
break;
|
|
3505
|
+
}
|
|
3506
|
+
await new Promise((r) => setTimeout(r, 50));
|
|
3507
|
+
}
|
|
3508
|
+
if (now) {
|
|
3509
|
+
const state = profileAfterRotation(now, read.state);
|
|
3510
|
+
writeProfile(broker.projectDir, broker.role, state);
|
|
3511
|
+
this.rememberTokens(refreshTokenSlots(state));
|
|
3512
|
+
this.logAction({
|
|
3513
|
+
action: "refresh-broker:learned",
|
|
3514
|
+
target: `${key} rotates ${names}; saved to the profile, and brokered from now on`,
|
|
3515
|
+
url: this.page?.url() ?? "",
|
|
3516
|
+
});
|
|
3517
|
+
}
|
|
3518
|
+
else {
|
|
3519
|
+
this.logAction({
|
|
3520
|
+
action: "refresh-broker:learned",
|
|
3521
|
+
target: `${key} rotates ${names}; brokered from now on (${stillSpent ? "the browser had not stored the new token" : "the profile already holds another token"}, so the profile was left as it is)`,
|
|
3134
3522
|
url: this.page?.url() ?? "",
|
|
3135
3523
|
});
|
|
3136
|
-
await route.abort("blockedbyclient").catch(() => { });
|
|
3137
3524
|
}
|
|
3138
3525
|
}
|
|
3526
|
+
finally {
|
|
3527
|
+
lock.release();
|
|
3528
|
+
}
|
|
3139
3529
|
}
|
|
3140
3530
|
/**
|
|
3141
3531
|
* Once the refresh is answered, write the rotation back over the profile
|
|
3142
3532
|
* and release the lock. The page's own state is saved once it holds the
|
|
3143
3533
|
* rotated token, so the next session gets the access token with it; a page
|
|
3144
3534
|
* that has not stored it in time has the token read from the response
|
|
3145
|
-
* instead. A refused or failed refresh changes nothing on disk
|
|
3535
|
+
* instead. A refused or failed refresh changes nothing on disk, and nor does
|
|
3536
|
+
* one whose response left a refresh cookie as it was (a server that does
|
|
3537
|
+
* not rotate refresh tokens).
|
|
3146
3538
|
*/
|
|
3147
|
-
async writeBackAfter(
|
|
3539
|
+
async writeBackAfter(answer, presented, lock) {
|
|
3148
3540
|
const broker = this.refresh;
|
|
3149
3541
|
try {
|
|
3150
|
-
|
|
3151
|
-
const res = await Promise.race([
|
|
3152
|
-
req.response().finally(() => clearTimeout(timer)),
|
|
3153
|
-
new Promise((resolve) => {
|
|
3154
|
-
timer = setTimeout(() => resolve(null), 30000);
|
|
3155
|
-
}),
|
|
3156
|
-
]);
|
|
3542
|
+
const res = await answer;
|
|
3157
3543
|
if (!broker)
|
|
3158
3544
|
return;
|
|
3159
|
-
if (!res || !res.ok
|
|
3160
|
-
const why = res ? `answered ${res.status
|
|
3545
|
+
if (!res || !res.ok) {
|
|
3546
|
+
const why = res ? `answered ${res.status}` : "had no response";
|
|
3161
3547
|
this.logAction({
|
|
3162
3548
|
action: "refresh-broker:not-saved",
|
|
3163
3549
|
target: `${presented.slot}: the refresh ${why}; the profile was left as it was`,
|
|
@@ -3166,7 +3552,9 @@ export class BrowserEngine {
|
|
|
3166
3552
|
return;
|
|
3167
3553
|
}
|
|
3168
3554
|
let state = null;
|
|
3169
|
-
|
|
3555
|
+
// A cookie is in the browser's jar by the time the response is; a token the page stores itself can take longer.
|
|
3556
|
+
const patienceMs = presented.cookie !== undefined ? 1000 : 5000;
|
|
3557
|
+
for (let waited = 0; waited <= patienceMs; waited += 50) {
|
|
3170
3558
|
const now = await this.context?.storageState().catch(() => null);
|
|
3171
3559
|
if (now && rotationStored(now, presented)) {
|
|
3172
3560
|
const onDisk = this.readRoleProfile();
|
|
@@ -3177,11 +3565,19 @@ export class BrowserEngine {
|
|
|
3177
3565
|
}
|
|
3178
3566
|
if (!state) {
|
|
3179
3567
|
const body = await res.text().catch(() => "");
|
|
3180
|
-
const rotated = rotatedFromResponse(body, await res.
|
|
3568
|
+
const rotated = rotatedFromResponse(body, await res.setCookies().catch(() => []));
|
|
3181
3569
|
const read = this.readRoleProfile();
|
|
3182
3570
|
if (rotated && read.ok)
|
|
3183
3571
|
state = swapProfileToken(read.state, presented.value, rotated);
|
|
3184
3572
|
}
|
|
3573
|
+
if (!state && presented.cookie !== undefined) {
|
|
3574
|
+
this.logAction({
|
|
3575
|
+
action: "refresh-broker:unrotated",
|
|
3576
|
+
target: `${presented.slot}: the response kept it as it was; nothing to save`,
|
|
3577
|
+
url: this.page?.url() ?? "",
|
|
3578
|
+
});
|
|
3579
|
+
return;
|
|
3580
|
+
}
|
|
3185
3581
|
if (!state) {
|
|
3186
3582
|
this.refreshCounts.failed += 1;
|
|
3187
3583
|
this.logAction({
|
|
@@ -3217,12 +3613,14 @@ export class BrowserEngine {
|
|
|
3217
3613
|
}
|
|
3218
3614
|
/** What the refresh broker did this session, or "" when it did nothing. Counts only. */
|
|
3219
3615
|
refreshSummary() {
|
|
3220
|
-
const { refreshed, swapped, failed } = this.refreshCounts;
|
|
3221
|
-
if (refreshed + swapped + failed === 0)
|
|
3616
|
+
const { refreshed, swapped, failed, learned } = this.refreshCounts;
|
|
3617
|
+
if (refreshed + swapped + failed + learned === 0)
|
|
3222
3618
|
return "";
|
|
3223
3619
|
const parts = [`refreshed role '${this.role}''s token ${refreshed} time${refreshed === 1 ? "" : "s"} under the role's lock`];
|
|
3224
3620
|
if (swapped > 0)
|
|
3225
3621
|
parts.push(`${swapped} of them after another session had rotated it`);
|
|
3622
|
+
if (learned > 0)
|
|
3623
|
+
parts.push(`learned ${learned} endpoint${learned === 1 ? "" : "s"} that rotate${learned === 1 ? "s" : ""} it`);
|
|
3226
3624
|
if (failed > 0)
|
|
3227
3625
|
parts.push(`${failed} refresh${failed === 1 ? "" : "es"} could not be brokered (see the action log)`);
|
|
3228
3626
|
return parts.join("; ");
|
|
@@ -3326,6 +3724,9 @@ export class BrowserEngine {
|
|
|
3326
3724
|
* harvested from their links, none is marked visited or attempted, and a
|
|
3327
3725
|
* failed load is not remembered, so the pages and what they link to stay on
|
|
3328
3726
|
* the unvisited list exactly as before. A check's re-test loads use it.
|
|
3727
|
+
*
|
|
3728
|
+
* `deadline` (epoch ms): no route is started once it has passed; those left
|
|
3729
|
+
* stay unvisited. The first run's time budget is the only caller.
|
|
3329
3730
|
*/
|
|
3330
3731
|
async crawl(paths, opts = {}) {
|
|
3331
3732
|
const page = this.requirePage();
|
|
@@ -3350,6 +3751,10 @@ export class BrowserEngine {
|
|
|
3350
3751
|
const queue = [...targets];
|
|
3351
3752
|
for (let i = 0; i < queue.length; i++) {
|
|
3352
3753
|
const path = queue[i];
|
|
3754
|
+
if (opts.deadline !== undefined && Date.now() >= opts.deadline) {
|
|
3755
|
+
summary.push(`… stopped at the time limit: ${queue.length - i} route(s) not started`);
|
|
3756
|
+
break;
|
|
3757
|
+
}
|
|
3353
3758
|
const url = `${this.baseUrl}${path.startsWith("/") ? "" : "/"}${path}`;
|
|
3354
3759
|
if (!this.isSameOrigin(url)) {
|
|
3355
3760
|
summary.push(`${path} — SKIPPED (off-origin)`);
|
|
@@ -3401,9 +3806,9 @@ export class BrowserEngine {
|
|
|
3401
3806
|
const { elements, forms } = await this.collect().finally(() => (this.harvestPaused = false));
|
|
3402
3807
|
const finalUrl = page.url();
|
|
3403
3808
|
const route = normalizePath(finalUrl);
|
|
3404
|
-
const fp = fingerprintState(finalUrl, elements);
|
|
3809
|
+
const fp = fingerprintState(finalUrl, trackedElements(elements));
|
|
3405
3810
|
if (!opts.measureOnly) {
|
|
3406
|
-
memory.visitState(fp, finalUrl, route, elements.map((el) => el.key));
|
|
3811
|
+
memory.visitState(fp, finalUrl, route, trackedElements(elements).map((el) => el.key), inertKeys(elements));
|
|
3407
3812
|
for (const f of forms)
|
|
3408
3813
|
memory.recordForm(fp, f.key, f.guarded);
|
|
3409
3814
|
memory.recordRoleAccess(this.role, route, "reached");
|
|
@@ -3440,7 +3845,7 @@ export class BrowserEngine {
|
|
|
3440
3845
|
url: finalUrl,
|
|
3441
3846
|
status: typeof status === "number" ? status : null,
|
|
3442
3847
|
loginRedirect,
|
|
3443
|
-
elements: elements.length,
|
|
3848
|
+
elements: trackedElements(elements).length,
|
|
3444
3849
|
unnamed: own.filter(missingName).map(describeControl),
|
|
3445
3850
|
placeholderOnly: own.filter(placeholderOnly).map(placeholderEvidence),
|
|
3446
3851
|
violations: violations.map(({ kind, severity, detail, url, embed }) => ({ kind, severity, detail, url, ...(embed ? { embed } : {}) })),
|
|
@@ -3449,13 +3854,17 @@ export class BrowserEngine {
|
|
|
3449
3854
|
design: inspected?.design ?? [],
|
|
3450
3855
|
...(inspected?.auditError ? { auditError: inspected.auditError } : {}),
|
|
3451
3856
|
});
|
|
3452
|
-
const deadEnd = elements.length === 0;
|
|
3857
|
+
const deadEnd = trackedElements(elements).length === 0;
|
|
3453
3858
|
// A field labelled only by its placeholder counts here too: to someone reading the crawl, it has no label.
|
|
3454
3859
|
// Counted over the page's own controls, as the check's lists are: another site's frame is not this app's to fix.
|
|
3455
3860
|
const unnamed = own.filter((el) => missingName(el) || placeholderOnly(el)).length;
|
|
3456
|
-
const missingTestid = elements.filter((el) => !el.testid && !el.disabled).length;
|
|
3861
|
+
const missingTestid = trackedElements(elements).filter((el) => !el.testid && !el.disabled).length;
|
|
3862
|
+
// What the page's main area holds besides controls: "41 el" alone cannot tell a page
|
|
3863
|
+
// of text from a main area that rendered nothing.
|
|
3864
|
+
const main = await this.readMainRegion(page);
|
|
3457
3865
|
const flags = [loginRedirect ? "AUTH-REDIRECT" : null, deadEnd ? "DEAD-END" : null, violations.length > 0 ? `${violations.length}⚠` : null].filter(Boolean);
|
|
3458
|
-
summary.push(`${path} — ${status} · ${elements.length} el` +
|
|
3866
|
+
summary.push(`${path} — ${status} · ${trackedElements(elements).length} el` +
|
|
3867
|
+
(main ? ` · ${mainRegionTag(main)}` : "") +
|
|
3459
3868
|
(missingTestid ? ` · ${missingTestid} no-testid` : "") +
|
|
3460
3869
|
(unnamed ? ` · ${unnamed} unnamed` : "") +
|
|
3461
3870
|
(flags.length ? ` · ${flags.join(" ")}` : ""));
|
|
@@ -3534,6 +3943,9 @@ export class BrowserEngine {
|
|
|
3534
3943
|
};
|
|
3535
3944
|
for (const [i, step] of steps.slice(0, 20).entries()) {
|
|
3536
3945
|
this.actionStartedAt = Date.now();
|
|
3946
|
+
// A navigation or a scroll is not input: navigate() moves the action mark itself, and what a scroll loads is the page's own doing.
|
|
3947
|
+
if (step.action !== "navigate" && step.action !== "scroll")
|
|
3948
|
+
await this.beginInput();
|
|
3537
3949
|
const desc = `${i + 1}. ${step.action} ${step.target ?? step.value ?? ""}`;
|
|
3538
3950
|
// Identity captured BEFORE the action — buttons that relabel themselves
|
|
3539
3951
|
// (Add to Cart → View Cart) are unmatchable in the post-action DOM.
|
|
@@ -3685,8 +4097,8 @@ export class BrowserEngine {
|
|
|
3685
4097
|
// this plan just reached, and it deserves coverage of its own)…
|
|
3686
4098
|
const { elements, forms } = await this.collect();
|
|
3687
4099
|
const url = page.url();
|
|
3688
|
-
const fp = fingerprintState(url, elements);
|
|
3689
|
-
this.memory.visitState(fp, url, normalizePath(url), elements.map((el) => el.key));
|
|
4100
|
+
const fp = fingerprintState(url, trackedElements(elements));
|
|
4101
|
+
this.memory.visitState(fp, url, normalizePath(url), trackedElements(elements).map((el) => el.key), inertKeys(elements));
|
|
3690
4102
|
for (const f of forms)
|
|
3691
4103
|
this.memory.recordForm(fp, f.key, f.guarded);
|
|
3692
4104
|
this.memory.recordRoleAccess(this.role, normalizePath(url), "reached");
|
|
@@ -3854,6 +4266,8 @@ export class BrowserEngine {
|
|
|
3854
4266
|
let failure = null;
|
|
3855
4267
|
let refusal = null;
|
|
3856
4268
|
this.actionStartedAt = Date.now();
|
|
4269
|
+
if (step.action === "click" || step.action === "type" || step.action === "select" || step.action === "press")
|
|
4270
|
+
await this.beginInput();
|
|
3857
4271
|
try {
|
|
3858
4272
|
const current = this.requirePage();
|
|
3859
4273
|
if (step.action === "navigate") {
|
|
@@ -4011,10 +4425,9 @@ export class BrowserEngine {
|
|
|
4011
4425
|
this.designAuditCount += 1;
|
|
4012
4426
|
if (this.memory)
|
|
4013
4427
|
this.memory.auditsThisRun += 1;
|
|
4014
|
-
// The census is built from previous audits
|
|
4015
|
-
//
|
|
4016
|
-
//
|
|
4017
|
-
// seen on several routes.
|
|
4428
|
+
// The census is built from previous audits and knows nothing until it has
|
|
4429
|
+
// seen several routes; the shell's landmarks (design.ts) cover those first
|
|
4430
|
+
// pages, so the shell is out of the score from the first audit.
|
|
4018
4431
|
const { report, score, signatures, defects } = analyzeDesign(payload, page.viewportSize() ?? { width: 1280, height: 900 }, this.memory?.designChromeKeys() ?? new Set());
|
|
4019
4432
|
const route = normalizePath(page.url());
|
|
4020
4433
|
this.memory?.recordDesignElements(route, signatures);
|