webhand 0.2.0 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.ja.md +104 -0
- package/README.md +70 -219
- package/assets/recorder.js +4 -0
- package/assets/snapshot.js +224 -4
- package/dist/agent.js +154 -13
- package/dist/auth.js +62 -0
- package/dist/bin.js +0 -0
- package/dist/browser.js +702 -23
- package/dist/cli.js +684 -86
- package/dist/collect.js +70 -0
- package/dist/compare.js +173 -0
- package/dist/daemon.js +125 -0
- package/dist/format.js +44 -0
- package/dist/knowledge.js +206 -0
- package/dist/learn.js +11 -5
- package/dist/metrics.js +64 -4
- package/dist/more.js +58 -0
- package/dist/policies/base.js +110 -14
- package/dist/policies/jev.js +245 -18
- package/dist/policies/scripted.js +2 -2
- package/dist/remote.js +165 -0
- package/dist/runbook.js +34 -19
- package/dist/session.js +261 -17
- package/dist/types.js +12 -0
- package/package.json +9 -6
- package/dist/py.js +0 -141
- package/dist/run.js +0 -80
- package/dist/tasks.js +0 -37
package/dist/browser.js
CHANGED
|
@@ -1,24 +1,192 @@
|
|
|
1
1
|
/** Playwright launches an isolated Chrome or attaches to the session Chrome; observation and input go through one CDP session. */
|
|
2
|
-
import { readFileSync } from "node:fs";
|
|
2
|
+
import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs";
|
|
3
3
|
import { isDeepStrictEqual } from "node:util";
|
|
4
4
|
import { chromium, } from "playwright-core";
|
|
5
|
-
import { sleep } from "./
|
|
5
|
+
import { sleep } from "./format.js";
|
|
6
|
+
import { home as sessionHome, tabPath } from "./session.js";
|
|
6
7
|
import { MutationUncertain, StalePage } from "./types.js";
|
|
7
8
|
export function asset(name) {
|
|
8
9
|
return readFileSync(new URL(`../assets/${name}`, import.meta.url), "utf8");
|
|
9
10
|
}
|
|
11
|
+
// A frame smaller than this on either side (a tracking pixel, an ad slot) is neither read nor counted as unseen.
|
|
12
|
+
const MIN_FRAME_SIDE = 20;
|
|
10
13
|
const SNAPSHOT = asset("snapshot.js").trim();
|
|
11
14
|
/** The snapshot expression for a view ({focus, skip} region patterns). The view never changes freshness. */
|
|
12
15
|
function readState(view = null) {
|
|
13
16
|
return `(${SNAPSHOT})(${JSON.stringify(view ?? {})})`;
|
|
14
17
|
}
|
|
15
18
|
export const READ_STATE = readState();
|
|
19
|
+
/**
|
|
20
|
+
* Signs of a bot check, for the run log. A marker is a sign, not proof: a normal page can embed a CAPTCHA
|
|
21
|
+
* widget. Only element presence is read, never text or field values.
|
|
22
|
+
*/
|
|
23
|
+
const CHALLENGES = [
|
|
24
|
+
[
|
|
25
|
+
"cloudflare",
|
|
26
|
+
'iframe[src*="challenges.cloudflare.com"], script[src*="challenges.cloudflare.com"], #challenge-form, .cf-turnstile',
|
|
27
|
+
],
|
|
28
|
+
["recaptcha", 'iframe[src*="recaptcha"], .g-recaptcha'],
|
|
29
|
+
["hcaptcha", 'iframe[src*="hcaptcha.com"], .h-captcha'],
|
|
30
|
+
["datadome", '[src*="captcha-delivery.com"]'],
|
|
31
|
+
["perimeterx", "#px-captcha"],
|
|
32
|
+
["arkose", 'iframe[src*="arkoselabs.com"], iframe[src*="funcaptcha"]'],
|
|
33
|
+
];
|
|
34
|
+
const PAGE_CHECK = `(() => {
|
|
35
|
+
const nav = performance.getEntriesByType("navigation")[0];
|
|
36
|
+
const found = ${JSON.stringify(CHALLENGES)}.filter(([, selector]) => document.querySelector(selector)).map(([name]) => name);
|
|
37
|
+
return {
|
|
38
|
+
http_status: nav && nav.responseStatus ? nav.responseStatus : null,
|
|
39
|
+
title: document.title.slice(0, 120),
|
|
40
|
+
challenge: found,
|
|
41
|
+
text_chars: document.body ? document.body.innerText.length : 0,
|
|
42
|
+
};
|
|
43
|
+
})()`;
|
|
44
|
+
// v2: the kind of page, from signs only (element presence and the document's HTTP status, never text or values).
|
|
45
|
+
const STATUS = `(() => {
|
|
46
|
+
const shown = e => { const r=e.getBoundingClientRect(); return r.width>0 && r.height>0 && getComputedStyle(e).visibility !== 'hidden'; };
|
|
47
|
+
const any = selector => [...document.querySelectorAll(selector)].some(shown);
|
|
48
|
+
const nav = performance.getEntriesByType('navigation')[0];
|
|
49
|
+
const http = nav && nav.responseStatus ? nav.responseStatus : null;
|
|
50
|
+
const signs = ${JSON.stringify(CHALLENGES)}.filter(([, selector]) => document.querySelector(selector)).map(([name]) => name);
|
|
51
|
+
const kind = http >= 400 ? 'error' : signs.length ? 'check' : any('input[type="password"]') ? 'sign_in'
|
|
52
|
+
: any('dialog[open],[role="dialog"],[role="alertdialog"],[aria-modal="true"]') ? 'dialog' : 'ready';
|
|
53
|
+
const count = selector => [...document.querySelectorAll(selector)].filter(shown).length;
|
|
54
|
+
const framed = [...document.querySelectorAll('iframe,frame')].filter(e => { const r=e.getBoundingClientRect(); return r.width>=${MIN_FRAME_SIDE} && r.height>=${MIN_FRAME_SIDE} && getComputedStyle(e).visibility !== 'hidden'; }).length;
|
|
55
|
+
const unseen = {frames: framed, canvases: count('canvas'), objects: count('object,embed')};
|
|
56
|
+
return {kind, ...(http ? {http_status: http} : {}), ...(signs.length ? {signs} : {}), unseen};
|
|
57
|
+
})()`;
|
|
58
|
+
const UNSEEN = `(() => {
|
|
59
|
+
const shown = e => { const r=e.getBoundingClientRect(); return r.width>0 && r.height>0; };
|
|
60
|
+
const count = selector => [...document.querySelectorAll(selector)].filter(shown).length;
|
|
61
|
+
const framed = [...document.querySelectorAll('iframe,frame')].filter(e => { const r=e.getBoundingClientRect(); return r.width>=${MIN_FRAME_SIDE} && r.height>=${MIN_FRAME_SIDE}; }).length;
|
|
62
|
+
return {frames: framed, canvases: count('canvas'), objects: count('object,embed')};
|
|
63
|
+
})()`;
|
|
64
|
+
/**
|
|
65
|
+
* Notes what asks for a new window: window.open and a click on a link with target=_blank. Nothing is stopped or
|
|
66
|
+
* changed; the browser does what it does. Installed once per document, and read with TAKE_WINDOWS.
|
|
67
|
+
*/
|
|
68
|
+
const TRACK_WINDOWS = `(() => {
|
|
69
|
+
if (window.__webhandWindows) return;
|
|
70
|
+
const asked = [];
|
|
71
|
+
window.__webhandWindows = asked;
|
|
72
|
+
const open = window.open;
|
|
73
|
+
window.open = function (url, ...rest) {
|
|
74
|
+
let where = String(url ?? '');
|
|
75
|
+
try { where = new URL(where, document.baseURI).href; } catch (e) {}
|
|
76
|
+
asked.push(where);
|
|
77
|
+
return open.call(this, url, ...rest);
|
|
78
|
+
};
|
|
79
|
+
document.addEventListener('click', event => {
|
|
80
|
+
const link = event.target instanceof Element ? event.target.closest('a[target],area[target]') : null;
|
|
81
|
+
if (link && link.target === '_blank') asked.push(link.href);
|
|
82
|
+
}, true);
|
|
83
|
+
})()`;
|
|
84
|
+
/**
|
|
85
|
+
* What Obscura lacks that snapshot.js reads or a page's own script uses. Added to the page, so snapshot.js stays the
|
|
86
|
+
* same for every engine. Installed once per document.
|
|
87
|
+
* - select.selectedOptions that can be spread (snapshot.js)
|
|
88
|
+
* - isContentEditable (snapshot.js: an editable div is a textbox)
|
|
89
|
+
* - a form's controls by name: form.guest, and a form by id: document.forms.id (page scripts; without it a handler that reads
|
|
90
|
+
* form.guest.value throws and the form is never submitted)
|
|
91
|
+
* Not covered: a cursor set by a stylesheet rule reads as 'auto' (only an inline style gives 'pointer'), so an element
|
|
92
|
+
* that is clickable only through a CSS cursor is not offered.
|
|
93
|
+
*/
|
|
94
|
+
const COMPAT = `(() => {
|
|
95
|
+
if (window.__webhandCompat) return;
|
|
96
|
+
window.__webhandCompat = true;
|
|
97
|
+
const define = (proto, name, descriptor) => {
|
|
98
|
+
try { if (!(name in proto)) Object.defineProperty(proto, name, { configurable: true, ...descriptor }); } catch (e) {}
|
|
99
|
+
};
|
|
100
|
+
try {
|
|
101
|
+
Object.defineProperty(HTMLSelectElement.prototype, 'selectedOptions', {
|
|
102
|
+
configurable: true,
|
|
103
|
+
get() { return [...this.options].filter(option => option.selected); },
|
|
104
|
+
});
|
|
105
|
+
} catch (e) {}
|
|
106
|
+
define(HTMLElement.prototype, 'isContentEditable', {
|
|
107
|
+
get() {
|
|
108
|
+
for (let node = this; node instanceof Element; node = node.parentElement) {
|
|
109
|
+
const value = node.getAttribute('contenteditable');
|
|
110
|
+
if (value !== null) return value !== 'false';
|
|
111
|
+
}
|
|
112
|
+
return false;
|
|
113
|
+
},
|
|
114
|
+
});
|
|
115
|
+
// A name that is not a property is looked up among the controls (or, for a collection, its named items).
|
|
116
|
+
const named = (proto, lookup) => {
|
|
117
|
+
try {
|
|
118
|
+
const parent = Object.getPrototypeOf(proto);
|
|
119
|
+
Object.setPrototypeOf(proto, new Proxy(parent, {
|
|
120
|
+
get(target, prop, receiver) {
|
|
121
|
+
if (typeof prop === 'string' && !(prop in target)) {
|
|
122
|
+
try { const hit = lookup(receiver, prop); if (hit) return hit; } catch (e) {}
|
|
123
|
+
}
|
|
124
|
+
return Reflect.get(target, prop, receiver);
|
|
125
|
+
},
|
|
126
|
+
}));
|
|
127
|
+
} catch (e) {}
|
|
128
|
+
};
|
|
129
|
+
{
|
|
130
|
+
named(HTMLFormElement.prototype, (form, name) => form.elements && form.elements[name]);
|
|
131
|
+
// document.forms is a NodeList here, which has no named items: wrap it so document.forms.id finds the form.
|
|
132
|
+
const forms = Object.getOwnPropertyDescriptor(Document.prototype, 'forms');
|
|
133
|
+
if (forms && forms.get) {
|
|
134
|
+
Object.defineProperty(Document.prototype, 'forms', {
|
|
135
|
+
configurable: true,
|
|
136
|
+
get() {
|
|
137
|
+
const list = forms.get.call(this);
|
|
138
|
+
return new Proxy(list, {
|
|
139
|
+
get(target, prop) {
|
|
140
|
+
if (typeof prop === 'string' && !(prop in target)) {
|
|
141
|
+
for (const form of target) if (form.id === prop || form.name === prop) return form;
|
|
142
|
+
}
|
|
143
|
+
const value = Reflect.get(target, prop, target);
|
|
144
|
+
return typeof value === 'function' ? value.bind(target) : value;
|
|
145
|
+
},
|
|
146
|
+
});
|
|
147
|
+
},
|
|
148
|
+
});
|
|
149
|
+
}
|
|
150
|
+
}
|
|
151
|
+
})()`;
|
|
152
|
+
const TAKE_WINDOWS = "(() => (window.__webhandWindows ?? []).splice(0))()";
|
|
16
153
|
const MARKER = `(() => { const state=${READ_STATE}; return state?.marker ?? null; })()`;
|
|
154
|
+
// The parts of the freshness keys in assets/snapshot.js (viewportKey, pageKey, guard, marker), by position.
|
|
155
|
+
const VIEWPORT_PARTS = ["navigation", "url", "scroll", "scroll", "viewport", "viewport"];
|
|
156
|
+
const PAGE_KEY_PARTS = [...VIEWPORT_PARTS, "form_fields"];
|
|
157
|
+
const GUARD_PARTS = [
|
|
158
|
+
"target_replaced",
|
|
159
|
+
"target_label",
|
|
160
|
+
"target_label",
|
|
161
|
+
...Array(9).fill("target_state"),
|
|
162
|
+
"target_link",
|
|
163
|
+
"scope_text",
|
|
164
|
+
"scope_kind",
|
|
165
|
+
];
|
|
166
|
+
// v2: what the freshness check leaves out. The target is hit-tested just before input whatever the scroll, and
|
|
167
|
+
// the text around a target outside a list item (a form, a dialog) can change without changing what it does.
|
|
168
|
+
// Without a target (Jev's DONE or BLOCKED), the page's own text is left out too: a ticking timer or a rotating
|
|
169
|
+
// banner would otherwise keep any answer from standing. The address, the elements, and field values still count.
|
|
170
|
+
const V2_IGNORED = new Set(["scroll", "viewport"]);
|
|
171
|
+
const V2_IGNORED_PAGE = new Set(["scroll", "viewport", "title", "text"]);
|
|
172
|
+
const UNCHANGING = new Set(["scroll", "wait"]);
|
|
173
|
+
const MARKER_PARTS = [...VIEWPORT_PARTS, "title", "text", "semantics", "form_fields"];
|
|
174
|
+
/** The names of the parts that differ between two freshness keys, each name once, in order. */
|
|
175
|
+
function differing(current, observed, parts) {
|
|
176
|
+
if (isDeepStrictEqual(current, observed))
|
|
177
|
+
return [];
|
|
178
|
+
if (!Array.isArray(current) || !Array.isArray(observed) || current.length !== observed.length)
|
|
179
|
+
return ["shape"];
|
|
180
|
+
const names = current.flatMap((value, i) => (isDeepStrictEqual(value, observed[i]) ? [] : [parts[i] ?? "other"]));
|
|
181
|
+
return [...new Set(names)];
|
|
182
|
+
}
|
|
17
183
|
// Read only once the document has loaded and the DOM has stopped changing, so content rendered by
|
|
18
184
|
// DOMContentLoaded handlers or after an XHR is observed. A page that never goes quiet is read at the cap.
|
|
19
185
|
const SETTLE_QUIET_MS = 150;
|
|
186
|
+
/** Tests of the fixtures shorten the quiet time (WEBHAND_SETTLE_QUIET_MS) to run faster; people never set it. */
|
|
187
|
+
const settleQuietMs = () => Number(process.env.WEBHAND_SETTLE_QUIET_MS) || SETTLE_QUIET_MS;
|
|
20
188
|
export const SETTLE_CAP_MS = 2000;
|
|
21
|
-
const
|
|
189
|
+
const settleScript = () => `new Promise(resolve => {
|
|
22
190
|
const start = performance.now();
|
|
23
191
|
let last = start;
|
|
24
192
|
const observer = new MutationObserver(() => { last = performance.now(); });
|
|
@@ -26,7 +194,7 @@ const SETTLE = `new Promise(resolve => {
|
|
|
26
194
|
const check = () => {
|
|
27
195
|
const now = performance.now();
|
|
28
196
|
if (now - start >= ${SETTLE_CAP_MS} ||
|
|
29
|
-
(document.readyState === 'complete' && now - last >= ${
|
|
197
|
+
(document.readyState === 'complete' && now - last >= ${settleQuietMs()})) {
|
|
30
198
|
observer.disconnect();
|
|
31
199
|
resolve();
|
|
32
200
|
} else setTimeout(check, 25);
|
|
@@ -34,20 +202,112 @@ const SETTLE = `new Promise(resolve => {
|
|
|
34
202
|
check();
|
|
35
203
|
})`;
|
|
36
204
|
// Node ids are code-owned references to observed elements, never model output.
|
|
205
|
+
// Where to click an observed element, or why not: gone, disabled, hidden, read-only, outside the viewport, or
|
|
206
|
+
// covered (with what covers it).
|
|
37
207
|
const RESOLVE_TARGET = `(action => {
|
|
38
208
|
const c=window.__browserLab, e=c?.nodes.get(action.node);
|
|
39
|
-
if (!e?.isConnected
|
|
40
|
-
|
|
41
|
-
if (
|
|
209
|
+
if (!e?.isConnected) return {reason:'gone'};
|
|
210
|
+
if (e.matches(':disabled') || c.closest(e,'[aria-disabled="true"],[inert]')) return {reason:'disabled'};
|
|
211
|
+
if (!e.checkVisibility({checkOpacity:true,checkVisibilityCSS:true})) return {reason:'hidden'};
|
|
212
|
+
if (action.kind==='fill' && (e.readOnly || e.getAttribute('aria-readonly')==='true')) return {reason:'read_only'};
|
|
42
213
|
// The center of the whole box first. An inline element that wraps can leave that center between its lines,
|
|
43
214
|
// so the center of each line box is tried next.
|
|
215
|
+
let covered=null;
|
|
44
216
|
for (const r of [e.getBoundingClientRect(), ...e.getClientRects()]) {
|
|
45
217
|
const x=r.x+r.width/2, y=r.y+r.height/2;
|
|
46
218
|
if (!r.width || !r.height || x<0 || y<0 || x>=innerWidth || y>=innerHeight) continue;
|
|
47
219
|
if (c.hit(e,x,y)) return {x,y};
|
|
220
|
+
covered ??= c.cover(x,y);
|
|
48
221
|
}
|
|
49
|
-
return
|
|
222
|
+
return covered ? {reason:'covered', cover:covered} : {reason:'outside'};
|
|
50
223
|
})`;
|
|
224
|
+
// Scrolls an observed element into the middle of the viewport. It never clicks or types.
|
|
225
|
+
const REVEAL = `(node => {
|
|
226
|
+
const e=window.__browserLab?.nodes.get(node);
|
|
227
|
+
if (!e?.isConnected) return false;
|
|
228
|
+
e.scrollIntoView({block: "center", inline: "nearest", behavior: "instant"});
|
|
229
|
+
return true;
|
|
230
|
+
})`;
|
|
231
|
+
// The text of the row, card, or list item each observed element is in, for comparing items. Page text only;
|
|
232
|
+
// field values are not part of innerText.
|
|
233
|
+
const CONTEXTS = `(nodes => {
|
|
234
|
+
const c=window.__browserLab, out={};
|
|
235
|
+
for (const node of nodes) {
|
|
236
|
+
const e=c?.nodes.get(node);
|
|
237
|
+
if (!e?.isConnected) continue;
|
|
238
|
+
const item=e.closest('tr,li,[role="row"],[role="listitem"],article,[role="article"]') || e.parentElement;
|
|
239
|
+
out[node]=String(item?.innerText || '').replace(/\\s+/g, ' ').trim().slice(0, 300);
|
|
240
|
+
}
|
|
241
|
+
return out;
|
|
242
|
+
})`;
|
|
243
|
+
// The items of lists and tables, header rows left out, in document order. An item is a table row, or one of three or more
|
|
244
|
+
// sibling blocks of the same tag and class (cards built from divs, list items, articles), outside navigation,
|
|
245
|
+
// headers and footers. Of nested items the outer one is kept (a card, not the bullet points inside it), unless
|
|
246
|
+
// it only wraps the inner ones: they hold nearly all of its text (a row of cards in a grid), or it is too long
|
|
247
|
+
// for one item (a section holding a whole list).
|
|
248
|
+
const ITEMS = `(() => {
|
|
249
|
+
const ROW='tr,[role="row"]', BLOCK=new Set(['DIV','LI','ARTICLE','SECTION','A','DD']);
|
|
250
|
+
const SKIP='nav,header,footer,aside,[role="navigation"],[role="banner"],[role="contentinfo"],[role="dialog"]';
|
|
251
|
+
const items=new Set(document.querySelectorAll(ROW));
|
|
252
|
+
for (const parent of document.querySelectorAll('body *')) {
|
|
253
|
+
if (parent.childElementCount<3) continue;
|
|
254
|
+
const groups=new Map();
|
|
255
|
+
for (const child of parent.children) {
|
|
256
|
+
if (!BLOCK.has(child.tagName) && !child.matches('[role="listitem"],[role="article"]')) continue;
|
|
257
|
+
const key=child.tagName+'|'+child.getAttribute('class')+'|'+child.getAttribute('role');
|
|
258
|
+
groups.set(key, [...(groups.get(key) || []), child]);
|
|
259
|
+
}
|
|
260
|
+
for (const group of groups.values()) {
|
|
261
|
+
if (group.length<3) continue;
|
|
262
|
+
for (const e of group) if ((e.textContent || '').trim()) items.add(e);
|
|
263
|
+
}
|
|
264
|
+
}
|
|
265
|
+
const length=e=>(e.textContent || '').replace(/\\s+/g, '').length;
|
|
266
|
+
const up=new Map(), kids=new Map();
|
|
267
|
+
for (const e of items) {
|
|
268
|
+
let a=e.parentElement;
|
|
269
|
+
while (a && !items.has(a)) a=a.parentElement;
|
|
270
|
+
up.set(e, a);
|
|
271
|
+
if (a) kids.set(a, [...(kids.get(a) || []), e]);
|
|
272
|
+
}
|
|
273
|
+
const wrapper=e=>{
|
|
274
|
+
const inner=kids.get(e) || [];
|
|
275
|
+
return inner.length>=2 && (length(e)>1500 || inner.reduce((n,d)=>n+length(d),0)>=0.85*length(e));
|
|
276
|
+
};
|
|
277
|
+
const kept=e=>{
|
|
278
|
+
if (e.closest(SKIP) || wrapper(e)) return false;
|
|
279
|
+
for (let a=up.get(e); a; a=up.get(a)) if (!a.closest(SKIP) && !wrapper(a)) return false;
|
|
280
|
+
return true;
|
|
281
|
+
};
|
|
282
|
+
const header=e=>e.querySelector('th,[role="columnheader"]') && !e.querySelector('td,[role="cell"],[role="gridcell"]');
|
|
283
|
+
return [...items].filter(e=>!header(e) && kept(e))
|
|
284
|
+
.sort((a,b)=>a.compareDocumentPosition(b)&Node.DOCUMENT_POSITION_FOLLOWING?-1:1);
|
|
285
|
+
})`;
|
|
286
|
+
const ROWS = `(() => {
|
|
287
|
+
const out=[];
|
|
288
|
+
for (const e of ${ITEMS}()) {
|
|
289
|
+
const r=e.getBoundingClientRect();
|
|
290
|
+
if (r.height<=0 || r.width<=0 || r.bottom<=0 || r.top>=innerHeight) continue;
|
|
291
|
+
const text=String(e.innerText || '').replace(/\\s+/g, ' ').trim().slice(0, 300);
|
|
292
|
+
if (text) out.push(text);
|
|
293
|
+
}
|
|
294
|
+
return out;
|
|
295
|
+
})()`;
|
|
296
|
+
// Where the main list ends: the bottom, in the viewport, of the last item of the longest list (items of the same
|
|
297
|
+
// tag and class under parents of the same tag and class, so a grid's rows count as one list). Null without a list
|
|
298
|
+
// of three or more items.
|
|
299
|
+
const LIST_END = `(() => {
|
|
300
|
+
const sig=e=>e.tagName+'|'+e.getAttribute('class');
|
|
301
|
+
const lists=new Map();
|
|
302
|
+
for (const e of ${ITEMS}()) {
|
|
303
|
+
if (!(e.getBoundingClientRect().height>0)) continue;
|
|
304
|
+
const key=sig(e)+'<'+(e.parentElement ? sig(e.parentElement) : '');
|
|
305
|
+
lists.set(key, [...(lists.get(key) || []), e]);
|
|
306
|
+
}
|
|
307
|
+
let longest=[];
|
|
308
|
+
for (const list of lists.values()) if (list.length>longest.length) longest=list;
|
|
309
|
+
return longest.length>=3 ? Math.round(longest.at(-1).getBoundingClientRect().bottom) : null;
|
|
310
|
+
})()`;
|
|
51
311
|
const SELECT_OPTION = `(action => {
|
|
52
312
|
const e=window.__browserLab.nodes.get(action.node);
|
|
53
313
|
if (e.tagName!=='SELECT' || ![...e.options].some(o=>o.value===action.value &&
|
|
@@ -57,13 +317,32 @@ const SELECT_OPTION = `(action => {
|
|
|
57
317
|
e.dispatchEvent(new Event('change',{bubbles:true}));
|
|
58
318
|
return true;
|
|
59
319
|
})`;
|
|
320
|
+
const MAX_TAB_TITLE = 60;
|
|
321
|
+
// v2: elements inside frames are read where they are, one snapshot per visible frame. A node id carries its
|
|
322
|
+
// frame: frame number times FRAME_BASE plus the id the frame's own snapshot gave. The main page is frame 0.
|
|
323
|
+
const FRAME_BASE = 1_000_000;
|
|
324
|
+
const MAX_FRAMES = 8;
|
|
325
|
+
const FRAME_TEXT_LIMIT = 1500;
|
|
326
|
+
const frameOf = (node) => (Number.isInteger(node) ? Math.floor(node / FRAME_BASE) : 0);
|
|
327
|
+
// Makes the ids a frame's snapshot gives start at its own base.
|
|
328
|
+
const seedFrame = (no) => `(() => { const c = (window.__browserLab ||= {ids: new WeakMap(), nodes: new Map(), next: 1}); ` +
|
|
329
|
+
`if (c.next < ${no * FRAME_BASE}) c.next = ${no * FRAME_BASE}; })()`;
|
|
330
|
+
/** A tab's title as the model sees it: one line, clipped. Page text, so never instructions. */
|
|
331
|
+
function tabTitle(title) {
|
|
332
|
+
return title.replace(/\s+/g, " ").trim().slice(0, MAX_TAB_TITLE);
|
|
333
|
+
}
|
|
60
334
|
export class Browser {
|
|
61
335
|
view;
|
|
62
336
|
attached;
|
|
337
|
+
keepsTab = true;
|
|
338
|
+
tracksWindows = false;
|
|
339
|
+
compat = false;
|
|
63
340
|
browser;
|
|
64
341
|
context;
|
|
65
342
|
page;
|
|
66
343
|
cdp;
|
|
344
|
+
frameNos = new Map();
|
|
345
|
+
framesByNo = new Map();
|
|
67
346
|
constructor(attached, view) {
|
|
68
347
|
this.attached = attached;
|
|
69
348
|
this.view = view;
|
|
@@ -72,22 +351,27 @@ export class Browser {
|
|
|
72
351
|
static async open(url = null, options = {}) {
|
|
73
352
|
const port = options.port ?? null;
|
|
74
353
|
const instance = new Browser(port !== null, options.view ?? null);
|
|
354
|
+
instance.keepsTab = options.rememberTab ?? true;
|
|
355
|
+
instance.tracksWindows = options.trackWindows ?? false;
|
|
356
|
+
instance.compat = options.compat ?? false;
|
|
75
357
|
try {
|
|
76
358
|
if (port !== null) {
|
|
77
359
|
instance.browser = await chromium.connectOverCDP(`http://127.0.0.1:${port}`);
|
|
78
360
|
instance.context = instance.browser.contexts()[0];
|
|
361
|
+
await instance.findPlaceholder();
|
|
79
362
|
if (url) {
|
|
80
363
|
// A saved profile may restore old tabs. The run owns exactly one fresh tab.
|
|
81
|
-
const stale =
|
|
364
|
+
const stale = instance.openTabs();
|
|
82
365
|
instance.page = await instance.context.newPage();
|
|
83
366
|
for (const page of stale)
|
|
84
367
|
await page.close();
|
|
85
368
|
await instance.page.goto(url, { waitUntil: "load" });
|
|
86
369
|
}
|
|
87
370
|
else {
|
|
88
|
-
instance.page = instance.
|
|
371
|
+
instance.page = await instance.rememberedTab();
|
|
89
372
|
}
|
|
90
373
|
instance.cdp = await instance.context.newCDPSession(instance.page);
|
|
374
|
+
await instance.rememberTab();
|
|
91
375
|
return instance;
|
|
92
376
|
}
|
|
93
377
|
// launch() uses a throwaway profile. Never point this at a signed-in profile.
|
|
@@ -104,6 +388,143 @@ export class Browser {
|
|
|
104
388
|
throw error;
|
|
105
389
|
}
|
|
106
390
|
}
|
|
391
|
+
/** Lightpanda lists a placeholder tab that is no page: nothing can attach to it and it hangs on close. It is never a tab. */
|
|
392
|
+
placeholder = null;
|
|
393
|
+
async findPlaceholder() {
|
|
394
|
+
for (const page of this.context.pages()) {
|
|
395
|
+
if ((await this.tabId(page)) === null)
|
|
396
|
+
this.placeholder = page;
|
|
397
|
+
}
|
|
398
|
+
}
|
|
399
|
+
openTabs() {
|
|
400
|
+
return this.context.pages().filter((page) => !page.isClosed() && page !== this.placeholder);
|
|
401
|
+
}
|
|
402
|
+
async tabId(page) {
|
|
403
|
+
try {
|
|
404
|
+
const session = await this.context.newCDPSession(page);
|
|
405
|
+
try {
|
|
406
|
+
const { targetInfo } = await session.send("Target.getTargetInfo");
|
|
407
|
+
return targetInfo.targetId;
|
|
408
|
+
}
|
|
409
|
+
finally {
|
|
410
|
+
await session.detach().catch(() => { });
|
|
411
|
+
}
|
|
412
|
+
}
|
|
413
|
+
catch {
|
|
414
|
+
return null;
|
|
415
|
+
}
|
|
416
|
+
}
|
|
417
|
+
/** The tab the last command left the run on; the first tab when none was kept or it is gone. */
|
|
418
|
+
async rememberedTab() {
|
|
419
|
+
const pages = this.openTabs();
|
|
420
|
+
const file = tabPath();
|
|
421
|
+
if (pages.length > 1 && existsSync(file)) {
|
|
422
|
+
const wanted = readFileSync(file, "utf8").trim();
|
|
423
|
+
for (const page of pages)
|
|
424
|
+
if ((await this.tabId(page)) === wanted)
|
|
425
|
+
return page;
|
|
426
|
+
}
|
|
427
|
+
return pages[0];
|
|
428
|
+
}
|
|
429
|
+
async rememberTab() {
|
|
430
|
+
if (!this.attached || !this.keepsTab)
|
|
431
|
+
return;
|
|
432
|
+
const id = await this.tabId(this.page);
|
|
433
|
+
if (id === null)
|
|
434
|
+
return;
|
|
435
|
+
mkdirSync(sessionHome(), { recursive: true });
|
|
436
|
+
writeFileSync(tabPath(), id, { encoding: "utf8", mode: 0o600 });
|
|
437
|
+
}
|
|
438
|
+
/** The other tabs, as controls Jev can choose by number. Offered only when more than one tab is open. */
|
|
439
|
+
async offerTabs(state) {
|
|
440
|
+
const open = this.openTabs();
|
|
441
|
+
if (open.length < 2)
|
|
442
|
+
return;
|
|
443
|
+
const current = open.indexOf(this.page);
|
|
444
|
+
const titles = await Promise.all(open.map((page) => page.title().catch(() => "")));
|
|
445
|
+
state.tabs = { count: open.length, current: current + 1 };
|
|
446
|
+
open.forEach((_, i) => {
|
|
447
|
+
if (i === current)
|
|
448
|
+
return;
|
|
449
|
+
const title = tabTitle(titles[i]);
|
|
450
|
+
const control = { id: `tab_${i + 1}`, kind: "tab", label: `Switch to tab ${i + 1}: ${title}`, tab: i + 1, title };
|
|
451
|
+
state.actions.splice(state.actions.length - 1, 0, control); // Before "wait", which stays last.
|
|
452
|
+
});
|
|
453
|
+
}
|
|
454
|
+
/** Makes another open tab the one the run observes and acts on. Nothing on any page changes. */
|
|
455
|
+
async switchTab(action, page) {
|
|
456
|
+
const open = this.openTabs();
|
|
457
|
+
const tabs = page.tabs;
|
|
458
|
+
if (!tabs || open.length !== tabs.count || open[tabs.current - 1] !== this.page) {
|
|
459
|
+
throw new StalePage("Tabs changed since this decision", ["tabs"]);
|
|
460
|
+
}
|
|
461
|
+
const target = open[action.tab - 1];
|
|
462
|
+
if (!target || target === this.page)
|
|
463
|
+
throw new StalePage("Tab is gone", ["tab_gone"]);
|
|
464
|
+
const title = tabTitle(await target.title().catch(() => ""));
|
|
465
|
+
if (title !== action.title)
|
|
466
|
+
throw new StalePage("Tab changed since this decision", ["tab_title"]);
|
|
467
|
+
try {
|
|
468
|
+
const next = await this.context.newCDPSession(target);
|
|
469
|
+
// Bringing a tab to the front only matters with a window. An engine without the method (Obscura) still switches.
|
|
470
|
+
await target.bringToFront().catch((error) => {
|
|
471
|
+
if (!/Unknown .*method|not (implemented|supported)|wasn't found/i.test(error.message))
|
|
472
|
+
throw error;
|
|
473
|
+
});
|
|
474
|
+
await this.cdp?.detach().catch(() => { });
|
|
475
|
+
this.page = target;
|
|
476
|
+
this.cdp = next;
|
|
477
|
+
}
|
|
478
|
+
catch {
|
|
479
|
+
throw new StalePage("Tab could not be switched to", ["tab_gone"]);
|
|
480
|
+
}
|
|
481
|
+
await this.rememberTab();
|
|
482
|
+
}
|
|
483
|
+
/**
|
|
484
|
+
* The session's cookies and localStorage, to be saved as a login. Nothing on any page changes. Playwright only knows
|
|
485
|
+
* the localStorage of pages it saw load, and each command is a new connection, so the sites of the open tabs are read
|
|
486
|
+
* directly. A site the session visited and left is not in it.
|
|
487
|
+
*/
|
|
488
|
+
async storageState() {
|
|
489
|
+
const state = await this.context.storageState();
|
|
490
|
+
for (const page of this.openTabs()) {
|
|
491
|
+
let origin;
|
|
492
|
+
try {
|
|
493
|
+
origin = new URL(page.url()).origin;
|
|
494
|
+
}
|
|
495
|
+
catch {
|
|
496
|
+
continue;
|
|
497
|
+
}
|
|
498
|
+
if (!/^https?:/.test(origin) || state.origins.some((known) => known.origin === origin))
|
|
499
|
+
continue;
|
|
500
|
+
try {
|
|
501
|
+
const items = await page.evaluate(() => Object.entries(localStorage).map(([name, value]) => ({ name, value: String(value) })));
|
|
502
|
+
if (items.length)
|
|
503
|
+
state.origins.push({ origin, localStorage: items });
|
|
504
|
+
}
|
|
505
|
+
catch {
|
|
506
|
+
// A page that refuses storage (an opaque origin) has none to keep.
|
|
507
|
+
}
|
|
508
|
+
}
|
|
509
|
+
return state;
|
|
510
|
+
}
|
|
511
|
+
/**
|
|
512
|
+
* Signs this browser in with a saved state and opens url. Cookies go in before the page loads. localStorage can only
|
|
513
|
+
* be written from a page of its own site, so it is set for the site of url and the page is loaded once more.
|
|
514
|
+
* Returns the sites whose localStorage could not be restored.
|
|
515
|
+
*/
|
|
516
|
+
async restore(state, url) {
|
|
517
|
+
if (state.cookies.length)
|
|
518
|
+
await this.context.addCookies(state.cookies);
|
|
519
|
+
await this.page.goto(url, { waitUntil: "load" });
|
|
520
|
+
const here = new URL(this.page.url()).origin;
|
|
521
|
+
const items = state.origins.find((origin) => origin.origin === here)?.localStorage ?? [];
|
|
522
|
+
if (items.length) {
|
|
523
|
+
await this.evaluate(`(() => { for (const [k, v] of ${JSON.stringify(items.map((item) => [item.name, item.value]))}) localStorage.setItem(k, v); })()`);
|
|
524
|
+
await this.page.goto(url, { waitUntil: "load" });
|
|
525
|
+
}
|
|
526
|
+
return state.origins.filter((origin) => origin.origin !== here && origin.localStorage.length).map((o) => o.origin);
|
|
527
|
+
}
|
|
107
528
|
get version() {
|
|
108
529
|
return this.browser.version();
|
|
109
530
|
}
|
|
@@ -130,12 +551,154 @@ export class Browser {
|
|
|
130
551
|
throw new StalePage("Document changed during evaluation");
|
|
131
552
|
return response.result?.value ?? null;
|
|
132
553
|
}
|
|
554
|
+
/** A tab that closed itself (a popup done with its work) leaves the run on the last tab still open. */
|
|
555
|
+
async leaveClosedTab() {
|
|
556
|
+
if (!this.page.isClosed())
|
|
557
|
+
return;
|
|
558
|
+
const open = this.openTabs();
|
|
559
|
+
if (open.length === 0)
|
|
560
|
+
throw new StalePage("Every tab is closed");
|
|
561
|
+
this.page = open[open.length - 1];
|
|
562
|
+
this.cdp = await this.context.newCDPSession(this.page);
|
|
563
|
+
await this.rememberTab();
|
|
564
|
+
}
|
|
565
|
+
/** The page's frames, the main one first. For tests. */
|
|
566
|
+
frames() {
|
|
567
|
+
return this.page.frames();
|
|
568
|
+
}
|
|
569
|
+
frameNo(frame) {
|
|
570
|
+
let no = this.frameNos.get(frame);
|
|
571
|
+
if (no === undefined) {
|
|
572
|
+
no = this.frameNos.size + 1;
|
|
573
|
+
this.frameNos.set(frame, no);
|
|
574
|
+
this.framesByNo.set(no, frame);
|
|
575
|
+
}
|
|
576
|
+
return no;
|
|
577
|
+
}
|
|
578
|
+
/** Evaluates in the frame the node belongs to. Null when that frame is gone. */
|
|
579
|
+
// biome-ignore lint/suspicious/noExplicitAny: page values are untyped JSON.
|
|
580
|
+
async evaluateFor(node, expression) {
|
|
581
|
+
const no = frameOf(node);
|
|
582
|
+
if (no === 0)
|
|
583
|
+
return this.evaluate(expression);
|
|
584
|
+
const frame = this.framesByNo.get(no);
|
|
585
|
+
if (!frame || frame.isDetached())
|
|
586
|
+
return null;
|
|
587
|
+
try {
|
|
588
|
+
return (await frame.evaluate(expression)) ?? null;
|
|
589
|
+
}
|
|
590
|
+
catch {
|
|
591
|
+
throw new StalePage("Document changed during evaluation");
|
|
592
|
+
}
|
|
593
|
+
}
|
|
594
|
+
/** Where a frame's own viewport starts, in the main page's viewport; null when it has no size. */
|
|
595
|
+
async frameOffset(frame) {
|
|
596
|
+
const element = await frame.frameElement().catch(() => null);
|
|
597
|
+
if (!element)
|
|
598
|
+
return null;
|
|
599
|
+
try {
|
|
600
|
+
const box = await element.boundingBox();
|
|
601
|
+
if (!box || box.width < MIN_FRAME_SIDE || box.height < MIN_FRAME_SIDE)
|
|
602
|
+
return null;
|
|
603
|
+
const [left, top] = await element.evaluate((e) => [e.clientLeft, e.clientTop]);
|
|
604
|
+
return { x: box.x + left, y: box.y + top };
|
|
605
|
+
}
|
|
606
|
+
catch {
|
|
607
|
+
return null;
|
|
608
|
+
}
|
|
609
|
+
finally {
|
|
610
|
+
await element.dispose().catch(() => { });
|
|
611
|
+
}
|
|
612
|
+
}
|
|
613
|
+
/** v2: the elements of the frames in the page, appended to the page's own, before its scroll and wait controls. */
|
|
614
|
+
async observeFrames(state) {
|
|
615
|
+
if (!this.view?.whole || this.compat)
|
|
616
|
+
return;
|
|
617
|
+
const main = this.page.mainFrame();
|
|
618
|
+
const frames = this.page.frames().filter((f) => f !== main && !f.isDetached());
|
|
619
|
+
const added = [];
|
|
620
|
+
const keys = {};
|
|
621
|
+
const markers = {};
|
|
622
|
+
const skipped = {};
|
|
623
|
+
const skip = (reason) => {
|
|
624
|
+
skipped[reason] = (skipped[reason] ?? 0) + 1;
|
|
625
|
+
};
|
|
626
|
+
let direct = 0;
|
|
627
|
+
for (const frame of frames) {
|
|
628
|
+
if (Object.keys(keys).length >= MAX_FRAMES) {
|
|
629
|
+
skip("over_limit");
|
|
630
|
+
continue;
|
|
631
|
+
}
|
|
632
|
+
try {
|
|
633
|
+
if (!(await this.frameOffset(frame))) {
|
|
634
|
+
skip("no_size");
|
|
635
|
+
continue;
|
|
636
|
+
}
|
|
637
|
+
const no = this.frameNo(frame);
|
|
638
|
+
await frame.evaluate(seedFrame(no));
|
|
639
|
+
const inner = (await frame.evaluate(readState(this.view)));
|
|
640
|
+
if (!inner) {
|
|
641
|
+
skip("no_page");
|
|
642
|
+
continue;
|
|
643
|
+
}
|
|
644
|
+
keys[String(no)] = inner.page_key;
|
|
645
|
+
markers[String(no)] = inner.marker;
|
|
646
|
+
Object.assign(state.guards, inner.guards);
|
|
647
|
+
for (const action of inner.actions) {
|
|
648
|
+
if (["scroll", "wait", "tab"].includes(action.kind))
|
|
649
|
+
continue;
|
|
650
|
+
added.push({ ...action, frame: no, region: `frame_${no}` });
|
|
651
|
+
}
|
|
652
|
+
const text = String(inner.text ?? "")
|
|
653
|
+
.trim()
|
|
654
|
+
.slice(0, FRAME_TEXT_LIMIT);
|
|
655
|
+
if (text)
|
|
656
|
+
state.text = `${state.text}\n[frame ${no}] ${text}`;
|
|
657
|
+
if (frame.parentFrame() === main)
|
|
658
|
+
direct++;
|
|
659
|
+
}
|
|
660
|
+
catch {
|
|
661
|
+
// A frame that changes or goes while it is read is left out; the page's own elements stand.
|
|
662
|
+
skip("error");
|
|
663
|
+
}
|
|
664
|
+
}
|
|
665
|
+
// For the run log (counts only): the frames read and why the others were not.
|
|
666
|
+
if (frames.length) {
|
|
667
|
+
state.frames_log = { found: frames.length, read: Object.keys(keys).length, elements: added.length, skipped };
|
|
668
|
+
}
|
|
669
|
+
if (!added.length && !Object.keys(keys).length)
|
|
670
|
+
return;
|
|
671
|
+
state.frame_keys = keys;
|
|
672
|
+
// What the frames showed, so that a change inside a frame counts as the page having changed (not in `marker`, which
|
|
673
|
+
// the freshness check compares with the main page alone).
|
|
674
|
+
state.frame_markers = markers;
|
|
675
|
+
const at = state.actions.findIndex((a) => ["scroll", "wait"].includes(a.kind));
|
|
676
|
+
state.actions.splice(at < 0 ? state.actions.length : at, 0, ...added);
|
|
677
|
+
let n = 0;
|
|
678
|
+
for (const action of state.actions)
|
|
679
|
+
if (/^e\d+$/.test(action.id))
|
|
680
|
+
action.id = `e${++n}`;
|
|
681
|
+
if (state.status?.unseen)
|
|
682
|
+
state.status.unseen.frames = Math.max(0, state.status.unseen.frames - direct);
|
|
683
|
+
}
|
|
133
684
|
async observe() {
|
|
685
|
+
await this.leaveClosedTab();
|
|
134
686
|
for (let i = 0; i < 10; i++) {
|
|
135
687
|
let state;
|
|
136
688
|
try {
|
|
137
|
-
await this.evaluate(
|
|
689
|
+
await this.evaluate(settleScript(), { awaitPromise: true });
|
|
690
|
+
if (this.compat)
|
|
691
|
+
await this.evaluate(COMPAT);
|
|
692
|
+
if (this.tracksWindows)
|
|
693
|
+
await this.evaluate(TRACK_WINDOWS);
|
|
138
694
|
state = await this.evaluate(readState(this.view));
|
|
695
|
+
// v2: what kind of page this is, for Jev and Claude.
|
|
696
|
+
if (state !== null && this.view?.whole)
|
|
697
|
+
state.status = await this.evaluate(STATUS);
|
|
698
|
+
if (state !== null)
|
|
699
|
+
await this.observeFrames(state);
|
|
700
|
+
if (state !== null)
|
|
701
|
+
await this.offerTabs(state);
|
|
139
702
|
}
|
|
140
703
|
catch (error) {
|
|
141
704
|
if (!(error instanceof StalePage))
|
|
@@ -149,31 +712,124 @@ export class Browser {
|
|
|
149
712
|
throw new StalePage("Page did not settle");
|
|
150
713
|
}
|
|
151
714
|
async fresh(page, action = null) {
|
|
152
|
-
|
|
715
|
+
return (await this.staleParts(page, action)).length === 0;
|
|
716
|
+
}
|
|
717
|
+
/** What differs from the observation: nothing when fresh. The parts are named for the run log only. */
|
|
718
|
+
async staleParts(page, action = null) {
|
|
719
|
+
// A compare or collect scan's own scroll: no decision to be stale (round gv1: 8 scans ended on "Page changed").
|
|
720
|
+
if (action?.scan)
|
|
721
|
+
return [];
|
|
722
|
+
const parts = await this.differingParts(page, action);
|
|
723
|
+
if (!this.view?.whole)
|
|
724
|
+
return parts;
|
|
725
|
+
// Scrolling and waiting change nothing on the page, and waiting is for the page to change: v2 never refuses
|
|
726
|
+
// them (in BU Bench round gv2, 22 of 41 refusals were these, on pages whose text kept changing).
|
|
727
|
+
if (action !== null && UNCHANGING.has(action.kind))
|
|
728
|
+
return [];
|
|
729
|
+
if (action === null)
|
|
730
|
+
return parts.filter((part) => !V2_IGNORED_PAGE.has(part));
|
|
731
|
+
const item = page.guards?.[String(action.node)]?.[14] === "item";
|
|
732
|
+
return parts.filter((part) => !V2_IGNORED.has(part) && (item || part !== "scope_text"));
|
|
733
|
+
}
|
|
734
|
+
async differingParts(page, action) {
|
|
735
|
+
// v2 checks typing against its field too, instead of against the whole viewport's text.
|
|
736
|
+
const targeted = ["click", "select", "reveal", ...(this.view?.whole ? ["fill"] : [])];
|
|
737
|
+
if (action !== null && targeted.includes(action.kind)) {
|
|
153
738
|
const node = action.node;
|
|
154
739
|
if (!Number.isInteger(node))
|
|
155
|
-
return
|
|
156
|
-
const current = await this.
|
|
740
|
+
return ["target_unknown"];
|
|
741
|
+
const current = await this.evaluateFor(node, "(() => { const c=window.__browserLab; " +
|
|
157
742
|
`return c ? [c.pageKey(),c.guard(c.nodes.get(${node}))] : null; })()`);
|
|
158
|
-
|
|
743
|
+
if (current === null)
|
|
744
|
+
return ["snapshot_gone"];
|
|
745
|
+
const guard = page.guards?.[String(node)] ?? null;
|
|
746
|
+
const key = frameOf(node) === 0 ? page.page_key : page.frame_keys?.[String(frameOf(node))];
|
|
747
|
+
return [
|
|
748
|
+
...differing(current[0], key, PAGE_KEY_PARTS),
|
|
749
|
+
...(current[1] === null && guard !== null ? ["target_gone"] : differing(current[1], guard, GUARD_PARTS)),
|
|
750
|
+
];
|
|
751
|
+
}
|
|
752
|
+
const marker = await this.evaluate(MARKER);
|
|
753
|
+
return marker === null && page.marker != null ? ["snapshot_gone"] : differing(marker, page.marker, MARKER_PARTS);
|
|
754
|
+
}
|
|
755
|
+
/** The text around observed elements, by node. Read-only. */
|
|
756
|
+
async contexts(nodes) {
|
|
757
|
+
const out = {};
|
|
758
|
+
for (const no of new Set(nodes.map(frameOf))) {
|
|
759
|
+
const own = nodes.filter((n) => frameOf(n) === no);
|
|
760
|
+
Object.assign(out, (await this.evaluateFor(own[0], `${CONTEXTS}(${JSON.stringify(own)})`)) ?? {});
|
|
761
|
+
}
|
|
762
|
+
return out;
|
|
763
|
+
}
|
|
764
|
+
/** Opens a page a scan has already visited, to come back to an item on it. Navigation only. */
|
|
765
|
+
async goto(url) {
|
|
766
|
+
await this.page.goto(url, { waitUntil: "load" });
|
|
767
|
+
}
|
|
768
|
+
/** The text of each list or table item in view, top to bottom of the document. */
|
|
769
|
+
async rows() {
|
|
770
|
+
return (await this.evaluate(ROWS)) ?? [];
|
|
771
|
+
}
|
|
772
|
+
/** The bottom of the main list's last item, in pixels from the top of the viewport; null without a list. */
|
|
773
|
+
async listEnd() {
|
|
774
|
+
return (await this.evaluate(LIST_END)) ?? null;
|
|
775
|
+
}
|
|
776
|
+
/**
|
|
777
|
+
* Where to click the action's element. When it is covered or outside the viewport, it is scrolled to the
|
|
778
|
+
* middle once and tried again: a sticky header or a fixed bar often covers an element near an edge. Otherwise
|
|
779
|
+
* StalePage, saying what covers it, so Jev can deal with that first. Nothing is clicked or typed here.
|
|
780
|
+
*/
|
|
781
|
+
async target(action) {
|
|
782
|
+
const no = frameOf(action.node);
|
|
783
|
+
const resolve = async () => {
|
|
784
|
+
const found = (await this.evaluateFor(action.node, `${RESOLVE_TARGET}(${JSON.stringify(action)})`));
|
|
785
|
+
if (found === null)
|
|
786
|
+
return { reason: "gone" };
|
|
787
|
+
if (no === 0 || "reason" in found)
|
|
788
|
+
return found;
|
|
789
|
+
// A point in a frame is clicked where the frame sits in the page's viewport.
|
|
790
|
+
const frame = this.framesByNo.get(no);
|
|
791
|
+
const offset = frame ? await this.frameOffset(frame) : null;
|
|
792
|
+
if (!offset)
|
|
793
|
+
return { reason: "hidden" };
|
|
794
|
+
const [w, h] = await this.evaluate("[innerWidth, innerHeight]");
|
|
795
|
+
const x = found.x + offset.x;
|
|
796
|
+
const y = found.y + offset.y;
|
|
797
|
+
return x < 0 || y < 0 || x >= w || y >= h ? { reason: "outside" } : { x, y };
|
|
798
|
+
};
|
|
799
|
+
let found = await resolve();
|
|
800
|
+
if ("reason" in found && (found.reason === "covered" || found.reason === "outside")) {
|
|
801
|
+
await this.evaluateFor(action.node, `${REVEAL}(${JSON.stringify(action.node)})`);
|
|
802
|
+
await sleep(100);
|
|
803
|
+
found = await resolve();
|
|
159
804
|
}
|
|
160
|
-
|
|
805
|
+
if (!("reason" in found))
|
|
806
|
+
return found;
|
|
807
|
+
if (found.reason === "covered") {
|
|
808
|
+
throw new StalePage(`Target is covered by ${found.cover}; deal with that first`, ["target_covered"]);
|
|
809
|
+
}
|
|
810
|
+
throw new StalePage(`Target is ${found.reason.replace("_", "-")}`, [`target_${found.reason}`]);
|
|
161
811
|
}
|
|
162
812
|
/** Execute once. StalePage means nothing happened; MutationUncertain means do not retry. */
|
|
163
813
|
async act(action, page, text = null) {
|
|
164
|
-
|
|
165
|
-
|
|
814
|
+
const changed = await this.staleParts(page, action);
|
|
815
|
+
if (changed.length)
|
|
816
|
+
throw new StalePage("Page changed since this decision", changed);
|
|
166
817
|
const kind = action.kind;
|
|
167
818
|
if (kind === "wait") {
|
|
168
819
|
await sleep(100);
|
|
169
820
|
return;
|
|
170
821
|
}
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
822
|
+
if (kind === "tab")
|
|
823
|
+
return await this.switchTab(action, page);
|
|
824
|
+
if (kind === "reveal") {
|
|
825
|
+
const found = await this.evaluateFor(action.node, `${REVEAL}(${JSON.stringify(action.node)})`);
|
|
826
|
+
if (!found)
|
|
827
|
+
throw new StalePage("Target changed", ["target_gone"]);
|
|
828
|
+
return;
|
|
176
829
|
}
|
|
830
|
+
let target = null;
|
|
831
|
+
if (kind !== "scroll")
|
|
832
|
+
target = await this.target(action);
|
|
177
833
|
try {
|
|
178
834
|
if (kind === "scroll") {
|
|
179
835
|
await this.call("Input.dispatchMouseEvent", {
|
|
@@ -206,6 +862,12 @@ export class Browser {
|
|
|
206
862
|
}
|
|
207
863
|
}
|
|
208
864
|
async select(action) {
|
|
865
|
+
if (frameOf(action.node) !== 0) {
|
|
866
|
+
const done = await this.evaluateFor(action.node, `${SELECT_OPTION}(${JSON.stringify(action)})`);
|
|
867
|
+
if (done !== true)
|
|
868
|
+
throw new MutationUncertain("Dropdown change was not confirmed");
|
|
869
|
+
return;
|
|
870
|
+
}
|
|
209
871
|
const response = await this.call("Runtime.evaluate", {
|
|
210
872
|
expression: `${SELECT_OPTION}(${JSON.stringify(action)})`,
|
|
211
873
|
returnByValue: true,
|
|
@@ -222,6 +884,23 @@ export class Browser {
|
|
|
222
884
|
await this.call("Input.dispatchKeyEvent", { type: "keyUp", ...key });
|
|
223
885
|
await this.call("Input.insertText", { text });
|
|
224
886
|
}
|
|
887
|
+
/** Read-only signs of why a page cannot be used: see PageCheck. */
|
|
888
|
+
async check() {
|
|
889
|
+
return await this.evaluate(PAGE_CHECK);
|
|
890
|
+
}
|
|
891
|
+
/** Frames, canvases, and plugin objects in the page: parts whose content webhand does not observe. */
|
|
892
|
+
async unseen() {
|
|
893
|
+
return (await this.evaluate(UNSEEN)) ?? {};
|
|
894
|
+
}
|
|
895
|
+
/** The new windows pages asked for since the last call (URLs). Only when trackWindows is on. */
|
|
896
|
+
async ignoredWindows() {
|
|
897
|
+
return this.tracksWindows ? ((await this.evaluate(TAKE_WINDOWS)) ?? []) : [];
|
|
898
|
+
}
|
|
899
|
+
/** A PNG of the viewport. Read-only; typed values show in it. */
|
|
900
|
+
async screenshot() {
|
|
901
|
+
const response = await this.call("Page.captureScreenshot", { format: "png" });
|
|
902
|
+
return Buffer.from(response.data, "base64");
|
|
903
|
+
}
|
|
225
904
|
/**
|
|
226
905
|
* An attached session Chrome stays alive; only session.stopChrome() ends it. Closing a browser that
|
|
227
906
|
* connectOverCDP attached to only disconnects: Chrome keeps its tabs and pages.
|