pawbrowse 0.5.2 → 0.6.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/extension/background.js +1306 -170
- package/extension/manifest.json +2 -2
- package/mcp/server.mjs +15 -5
- package/package.json +3 -2
package/extension/background.js
CHANGED
|
@@ -21,7 +21,8 @@ let reconnectTimer = null;
|
|
|
21
21
|
const attachedTabs = new Set(); // tabIds we currently hold a debugger on
|
|
22
22
|
const sessions = new Map(); // sessionId -> { activeTabId, createdTabs:Set, groupId, num, color }
|
|
23
23
|
const tabOwner = new Map(); // tabId -> sessionId (so sessions don't steal each other's tabs)
|
|
24
|
-
const chains = new Map();
|
|
24
|
+
const chains = new Map();
|
|
25
|
+
const refSeed = new Map(); // tabId -> next unused ref number (see SNAPSHOT: refs never repeat within a tab) // sessionId -> Promise (serialize commands within a session)
|
|
25
26
|
let sessionCounter = 0;
|
|
26
27
|
let colorCursor = 0;
|
|
27
28
|
// PawBrowse's own group identity — deliberately NOT Claude-in-Chrome's blue "Claude" group.
|
|
@@ -44,7 +45,7 @@ function persistState() {
|
|
|
44
45
|
try {
|
|
45
46
|
const ser = {};
|
|
46
47
|
for (const [k, v] of sessions) ser[k] = { activeTabId: v.activeTabId, createdTabs: [...v.createdTabs], groupId: v.groupId, num: v.num, color: v.color };
|
|
47
|
-
await chrome.storage.session.set({ pawbrowse_state: { sessions: ser, tabOwner: [...tabOwner], sessionCounter, colorCursor } });
|
|
48
|
+
await chrome.storage.session.set({ pawbrowse_state: { sessions: ser, tabOwner: [...tabOwner], sessionCounter, colorCursor, refSeed: [...refSeed] } });
|
|
48
49
|
} catch {}
|
|
49
50
|
}, 250);
|
|
50
51
|
persistTimer.unref?.();
|
|
@@ -59,6 +60,7 @@ const rehydrated = (async () => {
|
|
|
59
60
|
for (const [t, sess] of (st.tabOwner || [])) if (!tabOwner.has(t)) tabOwner.set(t, sess);
|
|
60
61
|
if (typeof st.sessionCounter === 'number') sessionCounter = Math.max(sessionCounter, st.sessionCounter);
|
|
61
62
|
if (typeof st.colorCursor === 'number') colorCursor = Math.max(colorCursor, st.colorCursor);
|
|
63
|
+
for (const [t, n] of (st.refSeed || [])) refSeed.set(t, Math.max(refSeed.get(t) || 1, n));
|
|
62
64
|
} catch {}
|
|
63
65
|
})();
|
|
64
66
|
|
|
@@ -117,12 +119,17 @@ async function connect() {
|
|
|
117
119
|
// navigate) stops issuing further CDP ops instead of leaking work into the next one.
|
|
118
120
|
const token = { cancelled: false };
|
|
119
121
|
const prev = chains.get(session) || Promise.resolve();
|
|
122
|
+
let settled = null;
|
|
120
123
|
const run = prev.then(async () => {
|
|
121
124
|
let timer;
|
|
122
125
|
try {
|
|
126
|
+
const work = handleCommand(msg.cmd, msg.args || {}, token, session);
|
|
127
|
+
// The next command in this session must not start while this one is still touching the tab,
|
|
128
|
+
// even after we've replied with a timeout (bounded, so a wedged page can't block forever).
|
|
129
|
+
settled = Promise.race([work.catch(() => {}), sleep(15000)]);
|
|
123
130
|
const result = await Promise.race([
|
|
124
|
-
|
|
125
|
-
new Promise((_, rej) => { timer = setTimeout(() => { token.cancelled = true; rej(new Error(
|
|
131
|
+
work,
|
|
132
|
+
new Promise((_, rej) => { timer = setTimeout(() => { token.cancelled = true; rej(new Error(`command timed out in extension after 25s${msg.cmd === 'act' || msg.cmd === 'navigate' ? ' — it may already have acted: observe before retrying' : ''}`)); }, 25000); }),
|
|
126
133
|
]);
|
|
127
134
|
clearTimeout(timer);
|
|
128
135
|
sock.send(JSON.stringify({ id: msg.id, ok: true, result }));
|
|
@@ -131,7 +138,7 @@ async function connect() {
|
|
|
131
138
|
try { sock.send(JSON.stringify({ id: msg.id, ok: false, error: String(e && e.message || e) })); } catch {}
|
|
132
139
|
}
|
|
133
140
|
});
|
|
134
|
-
chains.set(session, run.catch(() => {}));
|
|
141
|
+
chains.set(session, run.catch(() => {}).then(() => settled));
|
|
135
142
|
};
|
|
136
143
|
ws.onclose = () => { setBadge('off'); scheduleReconnect(); };
|
|
137
144
|
ws.onerror = () => { try { ws.close(); } catch {} };
|
|
@@ -151,21 +158,61 @@ function setBadge(state) {
|
|
|
151
158
|
|
|
152
159
|
/* ------------------------------- CDP helpers ------------------------------ */
|
|
153
160
|
|
|
154
|
-
|
|
161
|
+
// A CDP target is a tabId (the tab's top frame) or { tabId, sessionId?, frameId? }: sessionId
|
|
162
|
+
// addresses an out-of-process (cross-site) iframe attached via Target.setAutoAttach (flat sessions,
|
|
163
|
+
// Chrome 125+); frameId pins evaluation to a cross-origin frame living in that session's process.
|
|
164
|
+
const tabOf = (t) => (typeof t === 'object' ? t.tabId : t);
|
|
165
|
+
function sendCdp(target, method, params = {}) {
|
|
166
|
+
const dbg = typeof target === 'object' ? (target.sessionId ? { tabId: target.tabId, sessionId: target.sessionId } : { tabId: target.tabId }) : { tabId: target };
|
|
155
167
|
return new Promise((resolve, reject) => {
|
|
156
|
-
chrome.debugger.sendCommand(
|
|
168
|
+
chrome.debugger.sendCommand(dbg, method, params, (res) => {
|
|
157
169
|
const err = chrome.runtime.lastError;
|
|
158
170
|
if (err) reject(new Error(err.message)); else resolve(res);
|
|
159
171
|
});
|
|
160
172
|
});
|
|
161
173
|
}
|
|
162
174
|
|
|
163
|
-
|
|
164
|
-
|
|
175
|
+
// All of PawBrowse's page-side code runs in its own ISOLATED WORLD (like an extension content
|
|
176
|
+
// script / Playwright's utility world): it shares the DOM with the page but not its JS globals, so a
|
|
177
|
+
// page that monkey-patches Array.prototype / JSON / Element.prototype, or that squats on our cache
|
|
178
|
+
// name, can't blind or steer the snapshot. One world per tab's main frame; a navigation destroys it
|
|
179
|
+
// and the next call transparently recreates it.
|
|
180
|
+
const worlds = new Map(); // tabId -> Map(frameKey -> executionContextId)
|
|
181
|
+
const frameKey = (t) => (typeof t === 'object' ? `${t.sessionId || ''}|${t.frameId || ''}` : '|');
|
|
182
|
+
|
|
183
|
+
async function worldFor(target) {
|
|
184
|
+
const tabId = tabOf(target), key = frameKey(target);
|
|
185
|
+
let m = worlds.get(tabId);
|
|
186
|
+
if (!m) { m = new Map(); worlds.set(tabId, m); }
|
|
187
|
+
const cached = m.get(key);
|
|
188
|
+
if (cached != null) return cached;
|
|
189
|
+
let frameId = typeof target === 'object' ? target.frameId : null;
|
|
190
|
+
if (!frameId) ({ frameTree: { frame: { id: frameId } } } = await sendCdp(target, 'Page.getFrameTree'));
|
|
191
|
+
const { executionContextId } = await sendCdp(target, 'Page.createIsolatedWorld', { frameId, worldName: 'pawbrowse' });
|
|
192
|
+
m.set(key, executionContextId);
|
|
193
|
+
return executionContextId;
|
|
194
|
+
}
|
|
195
|
+
|
|
196
|
+
async function evaluate(target, expression, opts) {
|
|
197
|
+
let r;
|
|
198
|
+
for (let attempt = 0; ; attempt++) {
|
|
199
|
+
const contextId = await worldFor(target);
|
|
200
|
+
try {
|
|
201
|
+
r = await sendCdp(target, 'Runtime.evaluate', { expression, contextId, returnByValue: !(opts && opts.handle), awaitPromise: true });
|
|
202
|
+
break;
|
|
203
|
+
} catch (e) {
|
|
204
|
+
// The world died with its previous document (navigation/reload) BEFORE this call: make a fresh
|
|
205
|
+
// one and retry once. NOT when the context was destroyed DURING the call ("Execution context
|
|
206
|
+
// was destroyed") — the expression may have already acted (e.g. a select that navigated) and
|
|
207
|
+
// re-running it on the new page would act twice.
|
|
208
|
+
worlds.get(tabOf(target))?.delete(frameKey(target));
|
|
209
|
+
if (attempt >= 1 || !/Cannot find context/i.test(e.message)) throw e;
|
|
210
|
+
}
|
|
211
|
+
}
|
|
165
212
|
if (r && r.exceptionDetails) {
|
|
166
213
|
throw new Error(r.exceptionDetails.exception?.description || r.exceptionDetails.text || 'evaluation error');
|
|
167
214
|
}
|
|
168
|
-
return r.result.value;
|
|
215
|
+
return opts && opts.handle ? r.result : r.result.value;
|
|
169
216
|
}
|
|
170
217
|
|
|
171
218
|
// Attach to a SPECIFIC tab and keep it attached (many tabs can be attached at once, one per
|
|
@@ -193,6 +240,15 @@ async function attach(tabId) {
|
|
|
193
240
|
await sendCdp(tabId, 'Runtime.enable', {}).catch(() => {});
|
|
194
241
|
await sendCdp(tabId, 'Page.enable', {}).catch(() => {});
|
|
195
242
|
await sendCdp(tabId, 'DOM.enable', {}).catch(() => {});
|
|
243
|
+
// Lifecycle events (cheap: a handful per load) tell us when a new document is ready. The Network
|
|
244
|
+
// domain is NOT left on: on ad-heavy pages (hundreds of requests) it slows the whole browser, so
|
|
245
|
+
// it's switched on only around actions (netOn/netOff), where "did this click start a fetch?" matters.
|
|
246
|
+
await sendCdp(tabId, 'Page.setLifecycleEventsEnabled', { enabled: true }).catch(() => {});
|
|
247
|
+
try { const { frameTree } = await sendCdp(tabId, 'Page.getFrameTree'); tabWatch(tabId).mainFrame = frameTree.frame.id; } catch {}
|
|
248
|
+
// WebMCP (sites exposing their own agent tools; Chrome 146+ behind a flag / origin trial).
|
|
249
|
+
await sendCdp(tabId, 'WebMCP.enable', {}).catch(() => {});
|
|
250
|
+
// Cross-site iframes run in other renderer processes: attach to each as a flat child session.
|
|
251
|
+
await sendCdp(tabId, 'Target.setAutoAttach', AUTO_ATTACH).catch(() => {});
|
|
196
252
|
// Make the tab behave as focused even when it's a background tab, so focus/blur, rendering,
|
|
197
253
|
// and focus-dependent menus/dropdowns work while driving (the same approach Playwright uses for
|
|
198
254
|
// backgrounded pages). A hidden tab still throttles requestAnimationFrame, so our waits use
|
|
@@ -201,15 +257,71 @@ async function attach(tabId) {
|
|
|
201
257
|
}
|
|
202
258
|
|
|
203
259
|
function detach(tabId) {
|
|
204
|
-
return new Promise((resolve) => chrome.debugger.detach({ tabId }, () => { void chrome.runtime.lastError; attachedTabs.delete(tabId); resolve(); }));
|
|
260
|
+
return new Promise((resolve) => chrome.debugger.detach({ tabId }, () => { void chrome.runtime.lastError; attachedTabs.delete(tabId); forgetTab(tabId, false); resolve(); }));
|
|
261
|
+
}
|
|
262
|
+
|
|
263
|
+
// Everything we remember about a tab's CDP state. On detach (user cancelled the debugging bar,
|
|
264
|
+
// renderer crash) child sessions are gone without detachedFromTarget events: drop them so a
|
|
265
|
+
// re-attach rebuilds cleanly. On tab close, also drop what outlives an attachment.
|
|
266
|
+
function forgetTab(tabId, closed) {
|
|
267
|
+
worlds.delete(tabId); childSessions.delete(tabId); webTools.delete(tabId); shotScale.delete(tabId);
|
|
268
|
+
if (!closed) return;
|
|
269
|
+
watches.delete(tabId); frameIdx.delete(tabId); lastTable.delete(tabId); lastFull.delete(tabId);
|
|
270
|
+
openDialogs.delete(tabId); acting.delete(tabId); dialogLog.delete(tabId); refSeed.delete(tabId);
|
|
271
|
+
for (const k of [...refSeed.keys()]) if (String(k).startsWith(`${tabId}#`)) refSeed.delete(k);
|
|
272
|
+
}
|
|
273
|
+
chrome.debugger.onDetach.addListener((source) => { if (source.tabId != null) { attachedTabs.delete(source.tabId); forgetTab(source.tabId, false); } });
|
|
274
|
+
|
|
275
|
+
// JavaScript dialogs (alert/confirm/prompt/beforeunload) block the page's main thread, so every CDP
|
|
276
|
+
// call into the tab would hang until someone clicks the dialog. While PawBrowse is ACTING on a tab
|
|
277
|
+
// it answers them itself: alerts are accepted, confirm/prompt/beforeunload are DISMISSED unless the
|
|
278
|
+
// op opted in with dialog:"accept" (a destructive confirm is never auto-approved). What was shown is
|
|
279
|
+
// reported back. Dialogs that appear while we're idle belong to the user and are left alone; if one
|
|
280
|
+
// is still open when a command arrives we say so instead of hanging.
|
|
281
|
+
const acting = new Map(); // tabId -> { accept?: boolean, text?: string } while an act/navigate runs
|
|
282
|
+
const dialogLog = new Map(); // tabId -> [lines] reported in the next result
|
|
283
|
+
const openDialogs = new Map(); // tabId -> { type, message } left open (appeared while idle)
|
|
284
|
+
|
|
285
|
+
chrome.debugger.onEvent.addListener((source, method, params) => {
|
|
286
|
+
const tabId = source.tabId;
|
|
287
|
+
if (tabId == null) return;
|
|
288
|
+
if (method === 'Page.javascriptDialogClosed') { openDialogs.delete(tabId); return; }
|
|
289
|
+
if (method !== 'Page.javascriptDialogOpening') return;
|
|
290
|
+
const pol = acting.get(tabId);
|
|
291
|
+
const where = source.sessionId ? { tabId, sessionId: source.sessionId } : tabId; // dialogs can come from frames
|
|
292
|
+
if (!pol) { openDialogs.set(tabId, { type: params.type, message: params.message, where }); return; }
|
|
293
|
+
const accept = pol.accept != null ? pol.accept : params.type === 'alert';
|
|
294
|
+
const promptText = pol.text != null ? String(pol.text) : (params.defaultPrompt || '');
|
|
295
|
+
sendCdp(where, 'Page.handleJavaScriptDialog', { accept, promptText }).catch(() => {});
|
|
296
|
+
const lines = dialogLog.get(tabId) || [];
|
|
297
|
+
lines.push(`${params.type} "${String(params.message || '').replace(/\s+/g, ' ').slice(0, 200)}" → ${accept ? 'accepted' : 'dismissed'}${params.type === 'prompt' && accept ? ` with "${promptText}"` : ''}${!accept && params.type !== 'alert' ? ` (to accept, repeat ${pol.nav ? 'browser_navigate' : 'the op'} with dialog:"accept")` : ''}`);
|
|
298
|
+
dialogLog.set(tabId, lines);
|
|
299
|
+
});
|
|
300
|
+
|
|
301
|
+
// Run fn with this tab's dialogs auto-answered (see above).
|
|
302
|
+
async function whileActing(tabId, fn, policy) {
|
|
303
|
+
const mine = { ...(policy || {}) };
|
|
304
|
+
acting.set(tabId, mine);
|
|
305
|
+
try { return await fn(); } finally { if (acting.get(tabId) === mine) acting.delete(tabId); }
|
|
306
|
+
}
|
|
307
|
+
|
|
308
|
+
function takeDialogLog(tabId) {
|
|
309
|
+
const l = dialogLog.get(tabId); dialogLog.delete(tabId);
|
|
310
|
+
return l && l.length ? l.map((x) => ` dialog: ${x}`).join('\n') + '\n' : '';
|
|
205
311
|
}
|
|
206
312
|
|
|
207
|
-
|
|
313
|
+
// Refuse to talk to a tab frozen by a dialog the user hasn't answered (it would hang). The
|
|
314
|
+
// act op {op:"dialog"} answers it.
|
|
315
|
+
function assertNoOpenDialog(tabId) {
|
|
316
|
+
const d = openDialogs.get(tabId);
|
|
317
|
+
if (d) throw new Error(`the page is showing ${d.type === 'alert' ? 'an' : 'a'} ${d.type} dialog "${String(d.message || '').slice(0, 120)}" and is frozen until it's answered: run browser_act with [{op:"dialog",accept:true|false}] (or answer it in the browser)`);
|
|
318
|
+
}
|
|
208
319
|
|
|
209
320
|
// If a driven tab is closed (by the user or by us), forget it everywhere so a session doesn't
|
|
210
321
|
// keep pointing at a dead tab.
|
|
211
322
|
chrome.tabs.onRemoved.addListener((tabId) => {
|
|
212
323
|
attachedTabs.delete(tabId);
|
|
324
|
+
forgetTab(tabId, true);
|
|
213
325
|
const owner = tabOwner.get(tabId);
|
|
214
326
|
tabOwner.delete(tabId);
|
|
215
327
|
if (owner != null) {
|
|
@@ -303,35 +415,101 @@ async function endSession(session) {
|
|
|
303
415
|
* action re-resolves the exact element it was chosen from.
|
|
304
416
|
* -------------------------------------------------------------------------- */
|
|
305
417
|
|
|
306
|
-
|
|
418
|
+
// DOM activity tracker, kept apart from the ref cache (window.__pawbrowse) so creating it can't
|
|
419
|
+
// reset ref numbering. Installed by the first snapshot or wait in each document; shadow roots are
|
|
420
|
+
// added as the snapshot discovers them. Keeps recent mutation times so a wait can tell "this page
|
|
421
|
+
// is always animating" (don't wait for quiet that never comes) from "the action changed things".
|
|
422
|
+
const MO_INSTALL = `var M=window.__pawmo; if(!M){ M=window.__pawmo={last:performance.now(),times:[],roots:new WeakSet()};
|
|
423
|
+
M.mo=new MutationObserver(function(){ var t=performance.now(); M.last=t; M.times.push(t); if(M.times.length>64) M.times.shift(); });
|
|
424
|
+
M.watch=function(r){ if(!M.roots.has(r)){ M.roots.add(r); try{ M.mo.observe(r,{subtree:true,childList:true,attributes:true,characterData:true}); }catch(_){} } };
|
|
425
|
+
M.watch(document); }`;
|
|
426
|
+
const SNAPSHOT = `(function(seed, opts){
|
|
427
|
+
opts=opts||{};
|
|
307
428
|
try{
|
|
308
429
|
if(!document.body) return null;
|
|
309
|
-
|
|
310
|
-
|
|
430
|
+
// A NEW document continues numbering from where the tab's previous document stopped (seed), so a
|
|
431
|
+
// ref held over from the last page can never silently name a different element on this one.
|
|
432
|
+
var cache = window.__pawbrowse || (window.__pawbrowse = {ids:new WeakMap(), nodes:new Map(), next:Math.max(1,seed|0), byId:{}});
|
|
433
|
+
var fresh=new Set(); // node ids first allocated during this snapshot
|
|
434
|
+
function identity(e){
|
|
435
|
+
var id=cache.ids.get(e), holder=id!=null && cache.nodes.get(id);
|
|
436
|
+
// A new id if unseen — or if its old id now belongs to a different live element (a node that
|
|
437
|
+
// was detached, had its ref handed to a replacement, then came back).
|
|
438
|
+
if(id==null || (holder && holder!==e && holder.isConnected)){ id=cache.next++; cache.ids.set(e, id); fresh.add(id); }
|
|
439
|
+
cache.nodes.set(id,e); return id;
|
|
440
|
+
}
|
|
311
441
|
cache.nodes.forEach(function(e,id){ if(!e.isConnected) cache.nodes.delete(id); });
|
|
312
|
-
|
|
313
|
-
|
|
442
|
+
${MO_INSTALL}
|
|
443
|
+
// CLOSED shadow roots are invisible to page JS; the extension hands them to us via CDP
|
|
444
|
+
// (probeClosedRoots) keyed by host. sroot() = a host's shadow root, open or closed.
|
|
445
|
+
if(!cache.closed){ cache.closed=new WeakMap(); cache.probed=new WeakSet(); }
|
|
446
|
+
function sroot(n){ return n.shadowRoot || cache.closed.get(n) || null; }
|
|
447
|
+
cache.sroot=sroot;
|
|
448
|
+
var pendingHosts=[];
|
|
449
|
+
function safe(e){ return ['password','hidden'].indexOf(e.type)<0; }
|
|
450
|
+
// display:contents boxes (every <slot>, many design-system wrappers) have no box of their own, so
|
|
451
|
+
// checkVisibility() says false even though their children render: judge those by their parent.
|
|
452
|
+
function shown(e){ for(var g=0; e && g<32; g++){ if(e.checkVisibility({checkOpacity:true,checkVisibilityCSS:true})) return true; var v=e.ownerDocument.defaultView; if(!v || v.getComputedStyle(e).display!=='contents') return false; e=e.parentElement || (e.parentNode && e.parentNode.host); } return false; }
|
|
453
|
+
function visible(e){ return !!e && !e.closest('[aria-hidden="true"],[inert]') && shown(e); }
|
|
454
|
+
function sized(e){ var r=e.getBoundingClientRect(); return r.width>0 && r.height>0; }
|
|
455
|
+
function clean(s,n){ return String(s||'').replace(/\\s+/g,' ').trim().slice(0,n||120); }
|
|
314
456
|
function name(e,seen){
|
|
315
457
|
seen=seen||new Set();
|
|
316
|
-
if(!e||seen.has(e)) return '';
|
|
458
|
+
if(!e||seen.has(e)||e.nodeType!==1) return '';
|
|
317
459
|
seen.add(e);
|
|
318
|
-
var
|
|
319
|
-
|
|
460
|
+
var tag=e.tagName;
|
|
461
|
+
if(tag==='SCRIPT'||tag==='STYLE'||tag==='NOSCRIPT'||tag==='TEMPLATE') return '';
|
|
462
|
+
if(tag.toLowerCase()==='svg'){ var st=e.querySelector('title'); return e.getAttribute('aria-label')||(st?st.textContent:''); }
|
|
463
|
+
var rt=(e.getRootNode&&e.getRootNode())||document; var gid=function(id){try{return (rt.getElementById&&rt.getElementById(id))||e.ownerDocument.getElementById(id);}catch(_){return null;}};
|
|
464
|
+
// (A control that lists ITSELF in aria-labelledby contributes its own content, per accname.)
|
|
465
|
+
var ref=(e.getAttribute('aria-labelledby')||'').split(/\\s+/).filter(Boolean).map(function(id){ var t=gid(id); return t===e ? clean(e.textContent,80) : name(t,seen); }).filter(Boolean).join(' ');
|
|
320
466
|
if(ref) return ref;
|
|
321
467
|
if(e.getAttribute('aria-label')) return e.getAttribute('aria-label');
|
|
322
468
|
var labs=[].slice.call(e.labels||[]).map(function(l){return name(l,seen);}).filter(Boolean).join(' ');
|
|
323
469
|
if(labs) return labs;
|
|
324
470
|
if(['button','submit','reset'].indexOf(e.type)>=0 && e.value) return e.value;
|
|
325
471
|
if(e.getAttribute('alt')) return e.getAttribute('alt');
|
|
326
|
-
|
|
472
|
+
// Visible descendants only: display:none / hidden children (tooltips, menus) must not leak in.
|
|
473
|
+
// A rich-text editor's text is its VALUE, never its name (else typing into it renames it and
|
|
474
|
+
// its ref is refused on the next action).
|
|
475
|
+
var txt = (tag==='INPUT'||tag==='SELECT'||tag==='TEXTAREA'||e.isContentEditable) ? '' : [].map.call(e.childNodes,function(n){ return n.nodeType===3 ? n.textContent : (n.nodeType===1 && visible(n) ? name(n,seen) : ''); }).join(' ').trim();
|
|
327
476
|
if(txt) return txt;
|
|
328
|
-
return e.getAttribute('title')||e.getAttribute('placeholder')||'';
|
|
477
|
+
return e.getAttribute('title')||e.getAttribute('placeholder')||e.getAttribute('aria-placeholder')||'';
|
|
478
|
+
}
|
|
479
|
+
// A form field with no programmatic label: use a nearby <label> sibling (the common unassociated
|
|
480
|
+
// "<label>Name</label><div><input></div>" markup), then placeholder-ish hints, then its name attr.
|
|
481
|
+
function fieldLabel(e){
|
|
482
|
+
var n=name(e); if(n) return n;
|
|
483
|
+
if(!e.isContentEditable && !e.matches('input,textarea,select')) return '';
|
|
484
|
+
for(var s=e.parentElement,d=0; s && d<3; s=s.parentElement,d++){
|
|
485
|
+
if(['BODY','HTML','FORM'].indexOf(s.tagName)>=0) break;
|
|
486
|
+
for(var k=0;k<s.children.length;k++){ var c=s.children[k]; if(c.tagName==='LABEL' && !c.contains(e) && !c.control){ var t=name(c); if(t) return t; } }
|
|
487
|
+
}
|
|
488
|
+
return e.getAttribute('data-placeholder')||e.getAttribute('name')||(e.isContentEditable?'Rich text editor':'');
|
|
489
|
+
}
|
|
490
|
+
// Last resort for a nameless control (icon-font buttons): a test id, a meaningful id/name, or an
|
|
491
|
+
// icon class (fa-trash, bi-share, icon-close, material-icons text...) — shown as "icon:trash".
|
|
492
|
+
function hint(e){
|
|
493
|
+
var t=e.getAttribute('data-testid')||e.getAttribute('data-test')||e.getAttribute('data-qa')||e.getAttribute('data-cy')||e.getAttribute('data-action');
|
|
494
|
+
if(t) return 'testid:'+clean(t,40);
|
|
495
|
+
var els=[e].concat([].slice.call(e.querySelectorAll('i,span,svg,use,img')).slice(0,6));
|
|
496
|
+
for(var k=0;k<els.length;k++){
|
|
497
|
+
var cls=(els[k].getAttribute('class')||'')+' '+(els[k].getAttribute('href')||els[k].getAttribute('xlink:href')||'');
|
|
498
|
+
var re=/(?:^|[\\s#])(?:fa|bi|mdi|icon|glyphicon|lucide|ti|ri|octicon|ico)[-_]([a-z][a-z0-9-]{1,30})/ig, m;
|
|
499
|
+
while((m=re.exec(cls))){ if(!/^(solid|regular|light|thin|duotone|sharp|brands|lg|sm|xs|xl|fw|[0-9]+x|spin|pulse|icon|button|btn|wrapper|container|inner)$/i.test(m[1])) return 'icon:'+m[1].toLowerCase(); }
|
|
500
|
+
}
|
|
501
|
+
var id=e.id||e.getAttribute('name')||'';
|
|
502
|
+
if(id && /[a-z]{3}/i.test(id) && !/\\d{3}|[0-9a-f]{8}|^(ember|react|radix|mui|headlessui|:r)/i.test(id)) return 'id:'+clean(id,40);
|
|
503
|
+
return '';
|
|
329
504
|
}
|
|
330
505
|
var roles=['button','link','checkbox','radio','switch','tab','menuitem','menuitemradio','menuitemcheckbox','option','gridcell','combobox','textbox','searchbox','spinbutton','slider','treeitem'];
|
|
331
506
|
// Semantic controls + custom clickables: any contenteditable, an inline onclick, or a
|
|
332
507
|
// keyboard-focusable [tabindex] (framework buttons — React-Native-Web Pressables, design-system
|
|
333
508
|
// divs — often expose only these). cursor:pointer clickables are added separately in collect().
|
|
334
|
-
var selector='a[href],button,input,textarea,select,summary,[contenteditable]:not([contenteditable="false"]),[onclick],[tabindex]:not([tabindex="-1"]),'+roles.map(function(r){return '[role="'+r+'"]';}).join(',');
|
|
509
|
+
var selector='a[href],button,input,textarea,select,summary,[contenteditable]:not([contenteditable="false"]),[onclick],[tabindex]:not([tabindex="-1"]),[draggable="true"],'+roles.map(function(r){return '[role="'+r+'"]';}).join(',');
|
|
510
|
+
// Inputs whose value is SET (not typed): typing into these is unreliable, so type() routes them
|
|
511
|
+
// through a value setter. The hint tells the agent the expected format.
|
|
512
|
+
var SETTABLE={date:'YYYY-MM-DD',time:'HH:MM','datetime-local':'YYYY-MM-DDTHH:MM',month:'YYYY-MM',week:'YYYY-Www',color:'#rrggbb',range:''};
|
|
335
513
|
function role(e){
|
|
336
514
|
var explicit=e.getAttribute('role');
|
|
337
515
|
if(roles.indexOf(explicit)>=0) return explicit;
|
|
@@ -341,27 +519,62 @@ const SNAPSHOT = `(function(){
|
|
|
341
519
|
if(e.tagName==='TEXTAREA'||e.isContentEditable) return 'textbox';
|
|
342
520
|
if(e.tagName==='INPUT'){
|
|
343
521
|
if(['checkbox','radio'].indexOf(e.type)>=0) return e.type;
|
|
344
|
-
if(['button','submit','reset','image'].indexOf(e.type)>=0) return 'button';
|
|
522
|
+
if(['button','submit','reset','image','file'].indexOf(e.type)>=0) return 'button';
|
|
345
523
|
if(e.type==='search') return 'searchbox';
|
|
346
524
|
if(e.type==='number') return 'spinbutton';
|
|
347
|
-
if(
|
|
525
|
+
if(e.type==='range') return 'slider';
|
|
526
|
+
if(['text','email','url','tel'].indexOf(e.type)>=0 || SETTABLE.hasOwnProperty(e.type)) return 'textbox';
|
|
348
527
|
}
|
|
349
528
|
return null;
|
|
350
529
|
}
|
|
351
|
-
//
|
|
352
|
-
//
|
|
353
|
-
|
|
354
|
-
|
|
355
|
-
|
|
356
|
-
|
|
357
|
-
|
|
358
|
-
|
|
359
|
-
|
|
530
|
+
// Hit-test a frame-local point in the element's own root, descending into nested open shadow
|
|
531
|
+
// roots, so a covered control (overlay, modal backdrop, pointer-events:none) is flagged.
|
|
532
|
+
cache.hits=function(t, lx, ly){
|
|
533
|
+
try{
|
|
534
|
+
if(lx==null){ var hr=t.getBoundingClientRect(); lx=hr.x+hr.width/2; ly=hr.y+hr.height/2; }
|
|
535
|
+
var root=t.getRootNode(); if(!root.elementFromPoint) root=t.ownerDocument;
|
|
536
|
+
var f=root.elementFromPoint(lx,ly), g=0;
|
|
537
|
+
while(f && sroot(f) && g++<16){ var inner=sroot(f).elementFromPoint(lx,ly); if(!inner||inner===f) break; f=inner; }
|
|
538
|
+
return !!f && (t===f || t.contains(f) || (t.control && t.control===f));
|
|
539
|
+
}catch(_){ return true; }
|
|
540
|
+
};
|
|
541
|
+
var hits=cache.hits;
|
|
542
|
+
// The element to click for a control. Styled checkboxes/radios/file pickers usually hide the
|
|
543
|
+
// native input (opacity:0, 0x0, display:none, sr-only) and show a <label> or a card instead:
|
|
544
|
+
// the visible label is the interaction surface; an opacity-0 input stretched over a visible
|
|
545
|
+
// card is clicked directly. Returns null when the control has no usable surface.
|
|
546
|
+
cache.surface=function(e){
|
|
547
|
+
if(!e||!e.isConnected) return null;
|
|
548
|
+
var hidable=e.tagName==='INPUT' && ['checkbox','radio','file'].indexOf(e.type)>=0;
|
|
549
|
+
if(visible(e) && sized(e)){
|
|
550
|
+
// A sr-only (1px, clipped) native input is "visible" but not clickable: use its label.
|
|
551
|
+
if(!hidable || cache.hits(e)) return e;
|
|
552
|
+
}
|
|
553
|
+
if(!hidable) return null;
|
|
554
|
+
if(e.closest('[aria-hidden="true"],[inert]')) return null;
|
|
555
|
+
var labs=[].slice.call(e.labels||[]);
|
|
556
|
+
for(var i=0;i<labs.length;i++) if(visible(labs[i]) && sized(labs[i])) return labs[i];
|
|
557
|
+
if(visible(e) && sized(e)) return e; // no label to fall back to: report it as-is (covered)
|
|
558
|
+
if(sized(e) && e.checkVisibility({checkOpacity:false,checkVisibilityCSS:true}) && visible(e.parentElement)){
|
|
559
|
+
try{ if(e.ownerDocument.defaultView.getComputedStyle(e).pointerEvents!=='none') return e; }catch(_){}
|
|
560
|
+
}
|
|
561
|
+
return null;
|
|
562
|
+
};
|
|
563
|
+
// Identity-focused semantic guard: role + accessible name. Catches a target silently becoming a
|
|
564
|
+
// different control (relabel), while tolerating value/checked/expanded churn.
|
|
565
|
+
cache.guard=function(el){ if(!el) return ''; try{ return [role(el),clean(fieldLabel(el)||hint(el),200)].join(String.fromCharCode(1)); }catch(_){ return ''; } };
|
|
566
|
+
var unnamed=function(g){ return !g || g.charAt(g.length-1)===String.fromCharCode(1); }; // role only, no label: never rebind on it
|
|
567
|
+
// Walk the top document, OPEN shadow roots, and SAME-ORIGIN iframes. dx/dy translate each
|
|
568
|
+
// element's frame-local rect into top-level viewport coordinates (shadow roots share their
|
|
569
|
+
// frame's coords; iframes add their content-box offset). Cross-origin frames can't be read from
|
|
570
|
+
// here; they're reported so the agent knows content exists that it can't see.
|
|
571
|
+
var frames=[], remoteEls=[], textRoots=[];
|
|
360
572
|
function collect(){
|
|
361
573
|
var out=[], seen=new Set(), scanned=0;
|
|
362
574
|
function add(el, dx, dy, clk){ if(seen.has(el)) return; seen.add(el); out.push({el:el, dx:dx, dy:dy, clk:clk}); }
|
|
363
575
|
function walk(root, dx, dy, depth){
|
|
364
576
|
if(depth>12) return;
|
|
577
|
+
textRoots.push({root:root, dx:dx, dy:dy});
|
|
365
578
|
var els; try{ els=root.querySelectorAll(selector); }catch(_){ els=[]; }
|
|
366
579
|
for(var a=0;a<els.length;a++) add(els[a], dx, dy, false);
|
|
367
580
|
var all; try{ all=root.querySelectorAll('*'); }catch(_){ all=[]; }
|
|
@@ -381,100 +594,266 @@ const SNAPSHOT = `(function(){
|
|
|
381
594
|
}
|
|
382
595
|
}catch(_){}
|
|
383
596
|
}
|
|
384
|
-
|
|
385
|
-
if(
|
|
386
|
-
|
|
597
|
+
var sr=sroot(n);
|
|
598
|
+
if(sr){ M.watch(sr); walk(sr, dx, dy, depth+1); }
|
|
599
|
+
else if(n.localName.indexOf('-')>0 && !cache.probed.has(n) && pendingHosts.length<40 && sized(n)) pendingHosts.push(n); // custom element: may hide a closed root
|
|
600
|
+
if(n.tagName==='IFRAME' || n.tagName==='FRAME'){
|
|
601
|
+
var idoc=null; try{ idoc=n.contentDocument; }catch(_){}
|
|
387
602
|
if(idoc && idoc.body){
|
|
388
|
-
var ir=n.getBoundingClientRect()
|
|
389
|
-
walk(idoc,
|
|
390
|
-
|
|
391
|
-
|
|
603
|
+
var ir=n.getBoundingClientRect();
|
|
604
|
+
walk(idoc, dx+ir.left+n.clientLeft+(parseFloat(n.ownerDocument.defaultView.getComputedStyle(n).paddingLeft)||0),
|
|
605
|
+
dy+ir.top+n.clientTop+(parseFloat(n.ownerDocument.defaultView.getComputedStyle(n).paddingTop)||0), depth+1);
|
|
606
|
+
} else if(visible(n) && (function(){ var fr=n.getBoundingClientRect(); return fr.width>=10 && fr.height>=10; })()){ // 1x1 ad/tracking frames don't count
|
|
607
|
+
var src=''; try{ src=new URL(n.src, location.href).host; }catch(_){ src=n.getAttribute('src')||''; }
|
|
608
|
+
frames.push(clean(n.getAttribute('title')||n.getAttribute('aria-label')||n.name||src||'frame',60));
|
|
609
|
+
remoteEls.push(n);
|
|
392
610
|
}
|
|
393
|
-
}
|
|
611
|
+
}
|
|
394
612
|
}
|
|
395
613
|
}
|
|
396
614
|
walk(document, 0, 0, 0);
|
|
397
615
|
return out;
|
|
398
616
|
}
|
|
399
|
-
var
|
|
617
|
+
var scrollerCache=new Map(); // element -> does it clip its overflow? (one style read per ancestor per snapshot)
|
|
618
|
+
function clipper(a, view){ var v=scrollerCache.get(a); if(v===undefined){ var cs=view.getComputedStyle(a); v=!(cs.overflowX==='visible' && cs.overflowY==='visible'); scrollerCache.set(a,v); } return v; }
|
|
619
|
+
function clipped(t){
|
|
620
|
+
try{
|
|
621
|
+
var r=t.getBoundingClientRect(), cx=r.x+r.width/2, cy=r.y+r.height/2, view=t.ownerDocument.defaultView;
|
|
622
|
+
for(var a=t.parentElement, g=0; a && g<40; a=a.parentElement, g++){
|
|
623
|
+
if(!clipper(a, view)) continue;
|
|
624
|
+
var ar=a.getBoundingClientRect();
|
|
625
|
+
// How far outside the box it is (0 = inside), so the nearest hidden rows are kept first.
|
|
626
|
+
if(cx<ar.left||cx>ar.right||cy<ar.top||cy>ar.bottom) return 1+Math.max(ar.left-cx,cx-ar.right,ar.top-cy,cy-ar.bottom,0);
|
|
627
|
+
}
|
|
628
|
+
}catch(_){}
|
|
629
|
+
return 0;
|
|
630
|
+
}
|
|
631
|
+
var inView=[], offView=[], farOff=0, nodes=collect(), VH=innerHeight, VW=innerWidth;
|
|
400
632
|
for(var i=0;i<nodes.length;i++){
|
|
401
633
|
var e=nodes[i].el, ox=nodes[i].dx, oy=nodes[i].dy, clk=nodes[i].clk;
|
|
402
634
|
try{
|
|
403
|
-
if(!safe(e)
|
|
404
|
-
|
|
635
|
+
if(!safe(e)||e.matches(':disabled')||e.closest('[aria-disabled="true"],[inert]')) continue;
|
|
636
|
+
// Cheap early exit for controls screens away (huge pages have thousands): count, don't process.
|
|
637
|
+
var r0=e.getBoundingClientRect();
|
|
638
|
+
if(!opts.all && r0.width>0 && r0.height>0){ var y0=r0.y+r0.height/2+oy; if(y0<-VH||y0>=2*VH){ if(visible(e)) farOff++; continue; } }
|
|
639
|
+
var surf=cache.surface(e); if(!surf) continue;
|
|
640
|
+
var rname=role(e);
|
|
405
641
|
if(!rname){
|
|
406
642
|
// Role-less custom clickable (cursor:pointer, inline onclick, or focusable [tabindex]).
|
|
407
|
-
// Only accept it if it has a real label and isn't just a wrapper around an actual control
|
|
408
|
-
// so we don't flood the table with layout containers.
|
|
643
|
+
// Only accept it if it has a real label and isn't just a wrapper around an actual control
|
|
644
|
+
// (or a <label> standing in for one), so we don't flood the table with layout containers.
|
|
409
645
|
var ti=e.getAttribute('tabindex');
|
|
410
|
-
if(clk || e.hasAttribute('onclick') || (ti!==null && ti!=='-1')){
|
|
646
|
+
if(clk || e.hasAttribute('onclick') || (ti!==null && ti!=='-1') || e.getAttribute('draggable')==='true'){
|
|
647
|
+
if(e.tagName==='LABEL' && e.control) continue;
|
|
411
648
|
if(!(name(e)||'').trim() || e.querySelector(selector)) continue;
|
|
412
649
|
rname='button';
|
|
413
650
|
}
|
|
414
651
|
}
|
|
415
|
-
if(!rname
|
|
652
|
+
if(!rname) continue;
|
|
653
|
+
var r=surf.getBoundingClientRect(), lx=r.x+r.width/2, ly=r.y+r.height/2, x=lx+ox, y=ly+oy;
|
|
654
|
+
if(x<0||x>=VW) continue;
|
|
416
655
|
if(rname==='gridcell' && e.querySelector('button,[role="button"]')) continue;
|
|
417
|
-
var base={node:identity(e), role:rname, label:(
|
|
656
|
+
var base={node:identity(e), role:rname, label:clean(fieldLabel(e)||hint(e)||rname), x:Math.round(x), y:Math.round(y), w:Math.round(r.width), h:Math.round(r.height)};
|
|
418
657
|
var achecked=e.getAttribute('aria-checked');
|
|
419
658
|
if(['checkbox','radio'].indexOf(e.type)>=0) base.checked=!!e.checked;
|
|
420
659
|
else if(achecked!=null) base.checked=(achecked==='true');
|
|
421
660
|
var aexp=e.getAttribute('aria-expanded'); if(aexp!=null) base.expanded=(aexp==='true');
|
|
422
661
|
var asel=e.getAttribute('aria-selected'); if(asel!=null) base.selected=(asel==='true');
|
|
662
|
+
if(e.required || e.getAttribute('aria-required')==='true') base.required=true;
|
|
663
|
+
if(e.getAttribute('draggable')==='true') base.draggable=true;
|
|
664
|
+
// Only surface validation errors on fields the user (or agent) has put a value in, or that the
|
|
665
|
+
// page itself flags, so an untouched required form isn't a wall of warnings.
|
|
666
|
+
if(e.getAttribute('aria-invalid')==='true') base.invalid='invalid';
|
|
667
|
+
else if(e.validity && !e.validity.valid && e.value) base.invalid=clean(e.validationMessage,80)||'invalid';
|
|
668
|
+
var clip=clipped(surf);
|
|
669
|
+
if(clip){
|
|
670
|
+
base.off='scroll'; base.dist=clip; // inside an overflow box, scrolled out of it: acting scrolls it into view
|
|
671
|
+
} else if(y<0||y>=VH){
|
|
672
|
+
// Off-screen (scrolled away): keep the nearest ones so the agent knows they exist; acting on
|
|
673
|
+
// them scrolls them into view first.
|
|
674
|
+
if(!opts.all && (y<-VH/2||y>=1.5*VH)){ farOff++; continue; }
|
|
675
|
+
base.off=y<0?'up':'down'; base.dist=y<0?-y:y-VH;
|
|
676
|
+
} else if((function(){ var fv=surf.ownerDocument.defaultView; return fv!==window && (lx<0||ly<0||lx>=fv.innerWidth||ly>=fv.innerHeight); })()){
|
|
677
|
+
base.off='scroll'; // scrolled out of its (same-origin) iframe's viewport
|
|
678
|
+
} else if(!hits(surf, lx, ly)){
|
|
679
|
+
// Not hittable: either scrolled out of an overflow container (reachable — acting scrolls it
|
|
680
|
+
// in) or genuinely covered by an overlay/modal (needs dismissing first).
|
|
681
|
+
base.covered=true;
|
|
682
|
+
}
|
|
683
|
+
var list=base.off?offView:inView;
|
|
423
684
|
if(e.tagName==='SELECT'){
|
|
424
685
|
base.kind='select';
|
|
425
|
-
|
|
426
|
-
base.
|
|
427
|
-
|
|
686
|
+
if(e.multiple) base.fmt='multiple: select values:[...]';
|
|
687
|
+
base.value=clean([].map.call(e.selectedOptions,function(o){return o.label;}).join(', '),80);
|
|
688
|
+
base.options=[].filter.call(e.options,function(o){return !o.disabled && !(o.closest&&o.closest('optgroup[disabled]'));}).map(function(o){return clean(o.label,60);}).slice(0,40);
|
|
689
|
+
list.push(base);
|
|
690
|
+
} else if(e.tagName==='INPUT' && e.type==='file'){
|
|
691
|
+
base.kind='upload';
|
|
692
|
+
if(e.files && e.files.length) base.value=clean([].map.call(e.files,function(f){return f.name;}).join(', '),80);
|
|
693
|
+
if(e.accept) base.fmt=clean(e.accept,60);
|
|
694
|
+
list.push(base);
|
|
428
695
|
} else {
|
|
429
|
-
var
|
|
430
|
-
var
|
|
431
|
-
|
|
696
|
+
var settable=e.tagName==='INPUT' && SETTABLE.hasOwnProperty(e.type);
|
|
697
|
+
var editable=!e.readOnly && e.getAttribute('aria-readonly')!=='true' && (['textbox','searchbox','spinbutton'].indexOf(rname)>=0 || (rname==='slider' && e.tagName==='INPUT') || (rname==='combobox' && ['INPUT','TEXTAREA'].indexOf(e.tagName)>=0));
|
|
698
|
+
var value = (['checkbox','radio'].indexOf(e.type)>=0) ? '' : (('value' in e && e.tagName!=='BUTTON' && e.tagName!=='LI') ? String(e.value) : ((e.isContentEditable||rname==='combobox') ? e.innerText.trim() : ''));
|
|
699
|
+
if(e.tagName==='INPUT' && ['button','submit','reset','image'].indexOf(e.type)>=0) value=''; // its value IS its label
|
|
700
|
+
if(value) base.value=clean(value,80); // single line: page text must never forge extra table rows
|
|
701
|
+
if(settable){ base.fmt=e.type==='range' ? (e.min||'0')+'..'+(e.max||'100')+(e.step&&e.step!=='any'?' step '+e.step:'') : SETTABLE[e.type]; }
|
|
432
702
|
base.kind=editable?'fill':'click';
|
|
433
|
-
|
|
703
|
+
list.push(base);
|
|
434
704
|
// For an editable combobox, also offer a plain click to open its popup (not just type).
|
|
435
|
-
if(editable && rname==='combobox'){
|
|
705
|
+
if(editable && rname==='combobox'){ list.push({node:base.node, role:rname, label:'Open '+base.label, x:base.x, y:base.y, kind:'click', expanded:base.expanded, off:base.off, covered:base.covered}); }
|
|
436
706
|
}
|
|
437
707
|
}catch(_){ continue; }
|
|
438
708
|
}
|
|
439
|
-
|
|
440
|
-
//
|
|
441
|
-
|
|
709
|
+
var omitted=Math.max(0, inView.length-250); inView.splice(250);
|
|
710
|
+
// Keep the 25 off-screen controls NEAREST the visible area (not the first 25 in DOM order, which
|
|
711
|
+
// for a scrolled list are the rows furthest behind), then restore page order.
|
|
712
|
+
var OFFCAP=opts.all?400:25;
|
|
713
|
+
var offMore=Math.max(0, offView.length-OFFCAP)+farOff;
|
|
714
|
+
if(offView.length>OFFCAP){ offView.forEach(function(a,k){ a.ord=k; }); offView.sort(function(a,b){ return a.dist-b.dist; }); offView.splice(OFFCAP); offView.sort(function(a,b){ return a.ord-b.ord; }); }
|
|
715
|
+
var actions=inView.concat(offView);
|
|
716
|
+
// The nearest ancestor text that isn't just the label itself: which row/item a control is in.
|
|
717
|
+
function ctxOf(el, lab){
|
|
718
|
+
for(var p=el&&el.parentElement, g=0; p && g<6 && p.tagName!=='BODY'; p=p.parentElement, g++){
|
|
719
|
+
var tx=clean(p.innerText,400); if(!tx || tx===lab) continue;
|
|
720
|
+
var rest=clean(tx.split(lab).join(' '),40); if(rest) return rest;
|
|
721
|
+
}
|
|
722
|
+
return '';
|
|
723
|
+
}
|
|
724
|
+
// Identical labels ("Delete" per row, "Edit" per user) are ambiguous to the agent: tag each with
|
|
725
|
+
// the nearest ancestor text that tells them apart (e.g. the list row it lives in).
|
|
442
726
|
try{
|
|
443
|
-
var
|
|
444
|
-
|
|
445
|
-
|
|
446
|
-
if(
|
|
447
|
-
|
|
448
|
-
|
|
449
|
-
|
|
450
|
-
|
|
451
|
-
|
|
452
|
-
|
|
727
|
+
var byLabel={};
|
|
728
|
+
for(var u=0;u<actions.length;u++){ var key=actions[u].kind+'|'+actions[u].label; (byLabel[key]=byLabel[key]||[]).push(actions[u]); }
|
|
729
|
+
Object.keys(byLabel).forEach(function(key){
|
|
730
|
+
var grp=byLabel[key]; if(grp.length<2) return;
|
|
731
|
+
grp.forEach(function(a){ var cx=ctxOf(cache.nodes.get(a.node), a.label); if(cx) a.ctx=cx; });
|
|
732
|
+
});
|
|
733
|
+
}catch(_){}
|
|
734
|
+
var focus=null;
|
|
735
|
+
try{ var fe=document.activeElement; while(fe && fe.shadowRoot && fe.shadowRoot.activeElement) fe=fe.shadowRoot.activeElement;
|
|
736
|
+
for(var q=0; fe && fe.tagName==='IFRAME' && q<8; q++){ var fd=null; try{ fd=fe.contentDocument; }catch(_){} fe=fd?fd.activeElement:null; }
|
|
737
|
+
if(fe && fe.tagName!=='BODY' && fe.tagName!=='HTML' && cache.ids.has(fe)) focus=cache.ids.get(fe);
|
|
738
|
+
}catch(_){}
|
|
453
739
|
// Displayed id derives from the STABLE node id (not position), so a reused number can never
|
|
454
740
|
// remap to a different element across observations; duplicates (e.g. combobox Open) get a suffix.
|
|
455
741
|
// Guarded so a getter/DOM quirk while building ids can't blank the whole table.
|
|
742
|
+
// Resolved to CDP frame ids by the extension (cross-origin frames): biggest first, so a real
|
|
743
|
+
// embedded app/checkout wins over banner slots when there are many.
|
|
744
|
+
cache.pendingHosts=pendingHosts;
|
|
745
|
+
cache.remoteEls=remoteEls.map(function(el){ var r=el.getBoundingClientRect(); return {el:el, a:r.width*r.height}; }).sort(function(p,q){ return q.a-p.a; }).slice(0,12).map(function(p){ return p.el; });
|
|
746
|
+
// Keep refs stable across re-renders that REPLACE nodes: a brand-new element whose identity
|
|
747
|
+
// (role + label + row context) uniquely matches an element that vanished since the last snapshot
|
|
748
|
+
// takes over its node id, so "e12" keeps meaning "Done in Task 7" and the agent's refs survive.
|
|
456
749
|
try{
|
|
457
|
-
|
|
458
|
-
|
|
750
|
+
var gone={}, gcount={}, ncount={}, k2;
|
|
751
|
+
Object.keys(cache.byId||{}).forEach(function(id){
|
|
752
|
+
var nid=cache.byId[id]; if(cache.nodes.has(nid)) return; // still on the page
|
|
753
|
+
var fp=cache.fps&&cache.fps[id]; if(!fp||!cache.guards[id]||unnamed(cache.guards[id])) return;
|
|
754
|
+
k2=cache.guards[id]+'\u0002'+(fp.ctx||''); gone[k2]=nid; gcount[k2]=(gcount[k2]||0)+1;
|
|
755
|
+
});
|
|
756
|
+
var keyOf=function(a){ return cache.guard(cache.nodes.get(a.node))+'\u0002'+(a.ctx||''); };
|
|
757
|
+
actions.forEach(function(a){ if(fresh.has(a.node)){ k2=keyOf(a); ncount[k2]=(ncount[k2]||0)+1; } });
|
|
758
|
+
actions.forEach(function(a){
|
|
759
|
+
if(!fresh.has(a.node)) return;
|
|
760
|
+
k2=keyOf(a);
|
|
761
|
+
if(gcount[k2]!==1 || ncount[k2]!==1) return; // ambiguous: never guess
|
|
762
|
+
var el=cache.nodes.get(a.node), old=gone[k2];
|
|
763
|
+
cache.nodes.delete(a.node); cache.ids.set(el, old); cache.nodes.set(old, el);
|
|
764
|
+
actions.forEach(function(b){ if(b.node===a.node && b!==a) b.node=old; });
|
|
765
|
+
a.node=old; delete gcount[k2];
|
|
766
|
+
});
|
|
459
767
|
}catch(_){}
|
|
460
|
-
|
|
461
|
-
|
|
462
|
-
}
|
|
768
|
+
var focusId=null;
|
|
769
|
+
try{
|
|
770
|
+
cache.byId={}; cache.guards={}; cache.fps={}; var used={};
|
|
771
|
+
for(var j=0;j<actions.length;j++){ var bid='e'+actions[j].node, id=bid, kk=2; while(used[id]){ id=bid+'_'+kk; kk++; } used[id]=1; actions[j].id=id; cache.byId[id]=actions[j].node; cache.guards[id]=cache.guard(cache.nodes.get(actions[j].node)); cache.fps[id]={label:actions[j].label, ctx:actions[j].ctx||''}; if(focus!=null && actions[j].node===focus && !focusId) focusId=id; }
|
|
772
|
+
}catch(_){}
|
|
773
|
+
// Resolve a ref to its live element. Frameworks that re-render by REPLACING nodes (innerHTML
|
|
774
|
+
// templates, keyed lists) orphan every ref; re-find the replacement by the same identity the guard
|
|
775
|
+
// uses (role + label) plus its row context — only when exactly ONE element matches, never a guess.
|
|
776
|
+
cache.get=function(id){
|
|
777
|
+
var node=cache.byId[id]; if(node==null) return null;
|
|
778
|
+
var e=cache.nodes.get(node); if(e && e.isConnected) return e;
|
|
779
|
+
var fp=cache.fps && cache.fps[id], want=cache.guards && cache.guards[id]; if(!fp || !want || unnamed(want)) return null;
|
|
780
|
+
var cands=collect(), hit=null, n=0;
|
|
781
|
+
for(var k=0;k<cands.length && n<2;k++){
|
|
782
|
+
var el=cands[k].el;
|
|
783
|
+
try{
|
|
784
|
+
if(!el.isConnected || cache.guard(el)!==want || !cache.surface(el)) continue;
|
|
785
|
+
if(fp.ctx && ctxOf(el, fp.label)!==fp.ctx) continue;
|
|
786
|
+
hit=el; n++;
|
|
787
|
+
}catch(_){}
|
|
788
|
+
}
|
|
789
|
+
if(n!==1) return null;
|
|
790
|
+
cache.byId[id]=identity(hit);
|
|
791
|
+
return hit;
|
|
792
|
+
};
|
|
793
|
+
// Optional: the visible text in reading order (top-to-bottom, left-to-right), for prices, headings
|
|
794
|
+
// and results the controls table doesn't carry.
|
|
795
|
+
var vtext='';
|
|
796
|
+
if(opts.text){ try{
|
|
797
|
+
var frags=[], seenT=0;
|
|
798
|
+
textRoots.forEach(function(tr){
|
|
799
|
+
var doc=tr.root.ownerDocument||tr.root, w=doc.createTreeWalker(tr.root.body||tr.root, NodeFilter.SHOW_TEXT), rg=doc.createRange(), nd;
|
|
800
|
+
while((nd=w.nextNode()) && seenT<20000){ seenT++;
|
|
801
|
+
var v=nd.textContent.replace(/\\s+/g,' ').trim(), p=nd.parentElement;
|
|
802
|
+
if(!v||!p||p.closest('script,style,noscript,template')||!visible(p)) continue;
|
|
803
|
+
rg.selectNodeContents(nd); var q=rg.getBoundingClientRect(), ty=q.y+tr.dy, tx=q.x+tr.dx;
|
|
804
|
+
if(q.width<=0||q.height<=0||ty+q.height<0||ty>=VH||tx>=VW||tx+q.width<0) continue;
|
|
805
|
+
frags.push({t:v, x:tx, y:ty});
|
|
806
|
+
}
|
|
807
|
+
});
|
|
808
|
+
frags.sort(function(a,b){ return Math.round(a.y/6)-Math.round(b.y/6) || a.x-b.x; });
|
|
809
|
+
var lines=[], cur=null, lastY=-1e9;
|
|
810
|
+
frags.forEach(function(f){ if(Math.abs(f.y-lastY)>6){ if(cur) lines.push(cur); cur=f.t; lastY=f.y; } else cur+=' '+f.t; });
|
|
811
|
+
if(cur) lines.push(cur);
|
|
812
|
+
vtext=lines.join('\\n').slice(0,4000);
|
|
813
|
+
}catch(_){} }
|
|
814
|
+
return {url:location.href, title:document.title, vh:innerHeight, text:vtext, scrollY:Math.round(scrollY), scrollH:Math.round(document.documentElement.scrollHeight), omitted:omitted, offMore:offMore, frames:frames.slice(0,10), focus:focusId, next:cache.next, probe:pendingHosts.length, actions:actions};
|
|
815
|
+
}catch(_){ return {url:location.href, title:(document&&document.title)||'', scrollY:0, scrollH:0, omitted:0, actions:[]}; }
|
|
816
|
+
})`;
|
|
817
|
+
|
|
818
|
+
function formatRow(a) {
|
|
819
|
+
let flag = ' ';
|
|
820
|
+
if (typeof a.expanded === 'boolean') flag = a.expanded ? '▾' : '▸'; // open / closed
|
|
821
|
+
else if (typeof a.checked === 'boolean') flag = a.checked ? '✓' : '·';
|
|
822
|
+
else if (a.selected === true) flag = '◉';
|
|
823
|
+
let line = `${a.id.padEnd(4)} ${a.kind.padEnd(6)}${flag} "${a.label}"`;
|
|
824
|
+
if (a.ctx) line += ` in "${a.ctx}"`;
|
|
825
|
+
if (a.required) line += ' (required)';
|
|
826
|
+
if (a.value) line += ` ▸ "${a.value}"`;
|
|
827
|
+
if (a.fmt) line += ` fmt{${a.fmt}}`;
|
|
828
|
+
if (a.kind === 'select' && a.options && a.options.length) line += ` opts{${a.options.join(' | ')}}`;
|
|
829
|
+
if (a.invalid) line += ` ⚠ "${a.invalid}"`;
|
|
830
|
+
if (a.off === 'up') line += ' ↑ above view';
|
|
831
|
+
else if (a.off === 'down') line += ' ↓ below view';
|
|
832
|
+
else if (a.off === 'scroll') line += ' ↕ scrolled out of its box';
|
|
833
|
+
if (a.draggable) line += ' ⇄ draggable';
|
|
834
|
+
if (a.covered) line += ' ⊘ covered';
|
|
835
|
+
return line;
|
|
836
|
+
}
|
|
463
837
|
|
|
464
838
|
function formatTable(snap) {
|
|
465
839
|
if (!snap) return '(page not ready)';
|
|
466
840
|
const lines = [];
|
|
841
|
+
const fsnaps = snap.frameSnaps || [];
|
|
842
|
+
const count = snap.actions.length + fsnaps.reduce((n, f) => n + Math.min(60, f.snap.actions.length), 0);
|
|
467
843
|
lines.push(`${snap.title || '(untitled)'} — ${snap.url}`);
|
|
468
|
-
|
|
469
|
-
|
|
470
|
-
|
|
471
|
-
|
|
472
|
-
|
|
473
|
-
|
|
474
|
-
|
|
475
|
-
|
|
476
|
-
|
|
477
|
-
|
|
844
|
+
const more = (snap.omitted || 0) + (snap.offMore || 0);
|
|
845
|
+
lines.push(`scroll ${snap.scrollY}/${snap.scrollH} · ${count} controls${more ? ` (+${more} more; scroll to reveal)` : ''}${snap.focus ? ` · focus ${snap.focus}` : ''}`);
|
|
846
|
+
if (snap.frames && snap.frames.length > fsnaps.length) lines.push(`cross-origin frames (content not readable): ${snap.frames.map((f) => `"${f}"`).join(', ')}`);
|
|
847
|
+
for (const a of snap.actions) lines.push(formatRow(a));
|
|
848
|
+
for (const { f, off, snap: fs } of fsnaps) {
|
|
849
|
+
let host = f.url;
|
|
850
|
+
try { host = new URL(f.url).host || f.url; } catch {}
|
|
851
|
+
lines.push(`frame f${f.idx} "${String(host).slice(0, 60)}" (cross-origin):`);
|
|
852
|
+
for (const a of fs.actions.slice(0, 60)) {
|
|
853
|
+
const ty = off.y + a.y;
|
|
854
|
+
if (!a.off && (ty < 0 || ty >= (snap.vh || 1e9))) a.off = ty < 0 ? 'up' : 'down';
|
|
855
|
+
lines.push(formatRow({ ...a, id: `f${f.idx}.${a.id}` }));
|
|
856
|
+
}
|
|
478
857
|
}
|
|
479
858
|
return lines.join('\n');
|
|
480
859
|
}
|
|
@@ -482,25 +861,264 @@ function formatTable(snap) {
|
|
|
482
861
|
// A page signature to detect whether an action actually changed the page. Includes per-input
|
|
483
862
|
// value/checked/selectedIndex so fills, toggles, and selects register as changes (password
|
|
484
863
|
// values excluded).
|
|
485
|
-
const SIG = `JSON.stringify([location.href, document.title, [].map.call(document.querySelectorAll('input,textarea,select'),function(e){return e.type==='password'?'':(String(e.value)+'~'+(e.checked?1:0)+'~'+(e.selectedIndex==null?'':e.selectedIndex));}).join('|'), document.querySelectorAll('a,button,input,select,textarea,summary,[role]').length])`;
|
|
864
|
+
const SIG = `JSON.stringify([location.href, document.title, document.body ? document.body.textContent.length : 0, [].map.call(document.querySelectorAll('input,textarea,select'),function(e){return e.type==='password'?'':(String(e.value)+'~'+(e.checked?1:0)+'~'+(e.selectedIndex==null?'':e.selectedIndex));}).join('|'), document.querySelectorAll('a,button,input,select,textarea,summary,[role]').length])`;
|
|
486
865
|
|
|
487
866
|
// Retry through transient "document is navigating" states so a snapshot taken
|
|
488
867
|
// during a transition settles instead of failing.
|
|
489
|
-
|
|
868
|
+
// Readable page text. innerText stops at shadow roots and iframes, so it misses web-component
|
|
869
|
+
// content and same-origin frames. Mark every composed ancestor of a shadow host / iframe; unmarked
|
|
870
|
+
// subtrees use the (fast, layout-aware) innerText, marked ones are walked through their composed
|
|
871
|
+
// children (shadow root, slots' assigned nodes, iframe document).
|
|
872
|
+
const READ_TEXT = `(function(max){
|
|
873
|
+
var main=document.querySelector('main')||document.body; if(!main) return {title:document.title,url:location.href,text:''};
|
|
874
|
+
var mark=new Set(), C=window.__pawbrowse&&window.__pawbrowse.closed;
|
|
875
|
+
function sr(n){ return n.shadowRoot || (C&&C.get(n)) || null; }
|
|
876
|
+
function scan(root,d){ if(d>12) return; var all; try{ all=root.querySelectorAll('*'); }catch(_){ return; }
|
|
877
|
+
for(var i=0;i<all.length;i++){ var n=all[i], special=false;
|
|
878
|
+
if(sr(n)){ special=true; scan(sr(n),d+1); }
|
|
879
|
+
if(n.tagName==='SLOT') special=true;
|
|
880
|
+
if(n.tagName==='IFRAME'||n.tagName==='FRAME'){ var doc=null; try{ doc=n.contentDocument; }catch(_){} if(doc&&doc.body){ special=true; scan(doc,d+1); } }
|
|
881
|
+
if(special){ for(var p=n; p && !mark.has(p); ){ mark.add(p); p=p.parentNode; if(p && p.nodeType===11) p=p.host; } }
|
|
882
|
+
} }
|
|
883
|
+
scan(document,0);
|
|
884
|
+
var out=[], len=0;
|
|
885
|
+
function vis(e){ try{ if(e.checkVisibility({checkOpacity:true,checkVisibilityCSS:true})) return true; return getComputedStyle(e).display==='contents'; }catch(_){ return true; } }
|
|
886
|
+
function put(t){ if(t && len<max){ out.push(t); len+=t.length; } }
|
|
887
|
+
function kids(n){
|
|
888
|
+
if(sr(n)) return sr(n).childNodes;
|
|
889
|
+
if(n.tagName==='SLOT'){ var a=n.assignedNodes({flatten:true}); return a.length?a:n.childNodes; }
|
|
890
|
+
return n.childNodes;
|
|
891
|
+
}
|
|
892
|
+
function walk(n,d){
|
|
893
|
+
if(len>=max||d>60) return;
|
|
894
|
+
if(n.nodeType===3){ var v=n.textContent.replace(/\\s+/g,' ').trim(); if(v && n.parentElement && vis(n.parentElement)) put(v); return; }
|
|
895
|
+
if(n.nodeType!==1 && n.nodeType!==11) return;
|
|
896
|
+
if(n.nodeType===1){
|
|
897
|
+
if(/^(SCRIPT|STYLE|NOSCRIPT|TEMPLATE)$/.test(n.tagName) || !vis(n)) return;
|
|
898
|
+
if(n.tagName==='IFRAME'||n.tagName==='FRAME'){ var doc=null; try{ doc=n.contentDocument; }catch(_){} if(doc&&doc.body){ put('\\n'); walk(doc.body,d+1); put('\\n'); } return; }
|
|
899
|
+
if(!mark.has(n)){ put(n.innerText); if(getComputedStyle(n).display!=='inline') put('\\n'); return; }
|
|
900
|
+
}
|
|
901
|
+
var k=kids(n); for(var i=0;i<k.length;i++) walk(k[i],d+1);
|
|
902
|
+
if(n.nodeType===1 && getComputedStyle(n).display!=='inline') put('\\n');
|
|
903
|
+
}
|
|
904
|
+
walk(main,0);
|
|
905
|
+
var text=out.join(' ').replace(/[ \\t]*\\n[ \\t]*/g,'\\n').replace(/\\n{3,}/g,'\\n\\n').trim().slice(0,max);
|
|
906
|
+
return {title:document.title,url:location.href,text:text};
|
|
907
|
+
})`;
|
|
908
|
+
|
|
909
|
+
/* ------------------------------ Cross-origin frames ------------------------- *
|
|
910
|
+
* The page-side snapshot can't reach into cross-origin iframes (payment fields, embedded logins,
|
|
911
|
+
* widgets). CDP can: cross-SITE frames are separate targets we auto-attach to as flat child
|
|
912
|
+
* sessions; cross-origin but same-site frames share their parent's process and get their own
|
|
913
|
+
* isolated world via frameId. Each readable frame gets a stable index per tab (f1, f2...), its
|
|
914
|
+
* refs are shown as f1.e3, and clicks are translated by the frame's on-screen offset. */
|
|
915
|
+
const AUTO_ATTACH = { autoAttach: true, waitForDebuggerOnStart: false, flatten: true, filter: [{ type: 'iframe' }, { exclude: true }] };
|
|
916
|
+
const childSessions = new Map(); // tabId -> Map(sessionId -> { targetId, parent: sessionId|null })
|
|
917
|
+
const frameIdx = new Map(); // tabId -> { next, byKey: Map(key -> idx), byIdx: Map(idx -> frame) }
|
|
918
|
+
function kidsOf(tabId) { let m = childSessions.get(tabId); if (!m) { m = new Map(); childSessions.set(tabId, m); } return m; }
|
|
919
|
+
|
|
920
|
+
chrome.debugger.onEvent.addListener((source, method, params) => {
|
|
921
|
+
const tabId = source.tabId;
|
|
922
|
+
if (tabId == null) return;
|
|
923
|
+
if (method === 'Target.attachedToTarget') {
|
|
924
|
+
const info = params.targetInfo || {};
|
|
925
|
+
if (info.type !== 'iframe' || /^chrome-extension:/i.test(info.url || '')) return; // other extensions' frames aren't ours to drive
|
|
926
|
+
// Ad-heavy pages spawn hundreds of (mostly invisible) frames: just record them; a session is
|
|
927
|
+
// only set up (ensureSession) when one of its frames is actually visible and read.
|
|
928
|
+
kidsOf(tabId).set(params.sessionId, { targetId: info.targetId, parent: source.sessionId || null, ready: null });
|
|
929
|
+
} else if (method === 'Target.detachedFromTarget') {
|
|
930
|
+
kidsOf(tabId).delete(params.sessionId);
|
|
931
|
+
const m = worlds.get(tabId);
|
|
932
|
+
if (m) for (const k of [...m.keys()]) if (k.startsWith(`${params.sessionId}|`)) m.delete(k);
|
|
933
|
+
}
|
|
934
|
+
});
|
|
935
|
+
|
|
936
|
+
async function ensureSession(tabId, sessionId) {
|
|
937
|
+
const k = kidsOf(tabId).get(sessionId);
|
|
938
|
+
if (!k) return;
|
|
939
|
+
if (!k.ready) {
|
|
940
|
+
const t = { tabId, sessionId };
|
|
941
|
+
k.ready = Promise.all([['Page.enable', {}], ['DOM.enable', {}], ['Target.setAutoAttach', AUTO_ATTACH]].map(([m, p]) => sendCdp(t, m, p).catch(() => {})));
|
|
942
|
+
}
|
|
943
|
+
await k.ready;
|
|
944
|
+
}
|
|
945
|
+
|
|
946
|
+
// CDP frame ids of the visible cross-origin iframes a snapshot found in `target`'s document.
|
|
947
|
+
async function remoteFrameIds(target) {
|
|
948
|
+
const ctx = await worldFor(target);
|
|
949
|
+
const r = await sendCdp(target, 'Runtime.evaluate', { expression: 'window.__pawbrowse && window.__pawbrowse.remoteEls', contextId: ctx, objectGroup: 'pawframes' });
|
|
950
|
+
const ids = [];
|
|
951
|
+
try {
|
|
952
|
+
if (!r.result || !r.result.objectId) return ids;
|
|
953
|
+
const { result } = await sendCdp(target, 'Runtime.getProperties', { objectId: r.result.objectId, ownProperties: true });
|
|
954
|
+
for (const p of result) {
|
|
955
|
+
if (!/^\d+$/.test(p.name) || !p.value || !p.value.objectId) continue;
|
|
956
|
+
try { const { node } = await sendCdp(target, 'DOM.describeNode', { objectId: p.value.objectId }); if (node.frameId) ids.push(node.frameId); } catch {}
|
|
957
|
+
}
|
|
958
|
+
} finally { sendCdp(target, 'Runtime.releaseObjectGroup', { objectGroup: 'pawframes' }).catch(() => {}); }
|
|
959
|
+
return ids;
|
|
960
|
+
}
|
|
961
|
+
|
|
962
|
+
// Read the cross-origin frames under a snapshot, depth-first: only visible ones, never invisible
|
|
963
|
+
// subtrees. Cross-SITE frames are their own attached session; cross-origin same-site frames live
|
|
964
|
+
// in the parent's process and are addressed by frameId.
|
|
965
|
+
async function readFrames(tabId, parentTarget, parentSnap, out, depth = 0) {
|
|
966
|
+
if (!parentSnap || !parentSnap.frames || !parentSnap.frames.length || depth > 4 || out.length >= 12) return out;
|
|
967
|
+
let ids = [];
|
|
968
|
+
try { ids = await remoteFrameIds(parentTarget); } catch {}
|
|
969
|
+
let fi = frameIdx.get(tabId);
|
|
970
|
+
if (!fi) { fi = { next: 1, byKey: new Map(), byIdx: new Map() }; frameIdx.set(tabId, fi); }
|
|
971
|
+
for (const fid of ids) {
|
|
972
|
+
if (out.length >= 12) break;
|
|
973
|
+
const sid = [...kidsOf(tabId)].reverse().find(([, k]) => k.targetId === fid)?.[0]; // newest wins after a re-attach
|
|
974
|
+
const parentSession = typeof parentTarget === 'object' ? parentTarget.sessionId : undefined;
|
|
975
|
+
const f = sid
|
|
976
|
+
? { target: { tabId, sessionId: sid }, root: true, key: `s:${fid}` }
|
|
977
|
+
: { target: { tabId, sessionId: parentSession, frameId: fid }, root: false, key: `f:${fid}` };
|
|
978
|
+
try {
|
|
979
|
+
if (sid) await ensureSession(tabId, sid);
|
|
980
|
+
const off = await frameOffset(tabId, f);
|
|
981
|
+
if (off.w < 10 || off.h < 10) continue; // tracking pixels / collapsed frames
|
|
982
|
+
const fs = await snapshot(f.target, 1);
|
|
983
|
+
if (!fs || !fs.actions) continue;
|
|
984
|
+
if (!fi.byKey.has(f.key)) fi.byKey.set(f.key, fi.next++);
|
|
985
|
+
f.idx = fi.byKey.get(f.key); f.url = fs.url;
|
|
986
|
+
fi.byIdx.set(f.idx, f);
|
|
987
|
+
out.push({ f, off, snap: fs });
|
|
988
|
+
await readFrames(tabId, f.target, fs, out, depth + 1);
|
|
989
|
+
} catch {}
|
|
990
|
+
}
|
|
991
|
+
return out;
|
|
992
|
+
}
|
|
993
|
+
|
|
994
|
+
// Top-level viewport offset (and size) of a frame's content box: its owner <iframe>'s content quad
|
|
995
|
+
// in the parent's local root, plus that parent session's own offset, up to the top.
|
|
996
|
+
async function frameOffset(tabId, f) {
|
|
997
|
+
let x = 0, y = 0, w = 0, h = 0, first = true, topOwner = null;
|
|
998
|
+
let sess = f.target.sessionId || null, isRoot = !!f.root;
|
|
999
|
+
let fid = isRoot ? kidsOf(tabId).get(sess)?.targetId : f.target.frameId;
|
|
1000
|
+
for (let g = 0; g < 10 && fid; g++) {
|
|
1001
|
+
// A session's root frame is owned by an <iframe> in the PARENT session; an in-process frame's
|
|
1002
|
+
// owner is in its own session, whose box coords are relative to that session's root.
|
|
1003
|
+
const owner = isRoot ? (kidsOf(tabId).get(sess)?.parent ?? null) : sess;
|
|
1004
|
+
const t = owner ? { tabId, sessionId: owner } : tabId;
|
|
1005
|
+
const { backendNodeId } = await sendCdp(t, 'DOM.getFrameOwner', { frameId: fid });
|
|
1006
|
+
const { model } = await sendCdp(t, 'DOM.getBoxModel', { backendNodeId });
|
|
1007
|
+
const q = model.content;
|
|
1008
|
+
x += q[0]; y += q[1];
|
|
1009
|
+
if (first) { w = q[2] - q[0]; h = q[5] - q[1]; first = false; }
|
|
1010
|
+
if (!owner) { topOwner = backendNodeId; break; } // reached the top session: coordinates are now top-level
|
|
1011
|
+
sess = owner; isRoot = true; fid = kidsOf(tabId).get(owner)?.targetId;
|
|
1012
|
+
}
|
|
1013
|
+
return { x, y, w, h, topOwner };
|
|
1014
|
+
}
|
|
1015
|
+
|
|
1016
|
+
// Split "f2.e7" into its frame and the frame-local ref.
|
|
1017
|
+
function routeRef(tabId, ref) {
|
|
1018
|
+
const m = /^f(\d+)\.(.+)$/.exec(String(ref || ''));
|
|
1019
|
+
if (!m) return { target: tabId, ref, frame: null };
|
|
1020
|
+
const f = frameIdx.get(tabId)?.byIdx.get(Number(m[1]));
|
|
1021
|
+
if (!f) return { target: null, ref: m[2], frame: null };
|
|
1022
|
+
return { target: f.target, ref: m[2], frame: f };
|
|
1023
|
+
}
|
|
1024
|
+
|
|
1025
|
+
// Hand CLOSED shadow roots of the custom elements a snapshot flagged to our isolated world:
|
|
1026
|
+
// DOM.describeNode(pierce) exposes them to CDP even though page JS can't reach them. Each host is
|
|
1027
|
+
// probed once per document. Returns how many roots were found.
|
|
1028
|
+
async function probeClosedRoots(target) {
|
|
1029
|
+
const ctx = await worldFor(target);
|
|
1030
|
+
let found = 0;
|
|
1031
|
+
try {
|
|
1032
|
+
const r = await sendCdp(target, 'Runtime.evaluate', { expression: 'window.__pawbrowse && window.__pawbrowse.pendingHosts', contextId: ctx, objectGroup: 'pawshadow' });
|
|
1033
|
+
if (!r.result || !r.result.objectId) return 0;
|
|
1034
|
+
const { result } = await sendCdp(target, 'Runtime.getProperties', { objectId: r.result.objectId, ownProperties: true });
|
|
1035
|
+
for (const p of result) {
|
|
1036
|
+
if (!/^\d+$/.test(p.name) || !p.value || !p.value.objectId) continue;
|
|
1037
|
+
try {
|
|
1038
|
+
const { node } = await sendCdp(target, 'DOM.describeNode', { objectId: p.value.objectId, depth: 0, pierce: true });
|
|
1039
|
+
const root = (node.shadowRoots || []).find((x) => x.shadowRootType === 'closed');
|
|
1040
|
+
if (!root) continue;
|
|
1041
|
+
const { object } = await sendCdp(target, 'DOM.resolveNode', { backendNodeId: root.backendNodeId, executionContextId: ctx, objectGroup: 'pawshadow' });
|
|
1042
|
+
await sendCdp(target, 'Runtime.callFunctionOn', { objectId: p.value.objectId, functionDeclaration: 'function(r){ window.__pawbrowse.closed.set(this, r); }', arguments: [{ objectId: object.objectId }] });
|
|
1043
|
+
found++;
|
|
1044
|
+
} catch {}
|
|
1045
|
+
}
|
|
1046
|
+
await sendCdp(target, 'Runtime.evaluate', { expression: 'window.__pawbrowse.pendingHosts.forEach(function(h){ window.__pawbrowse.probed.add(h); })', contextId: ctx });
|
|
1047
|
+
} finally { sendCdp(target, 'Runtime.releaseObjectGroup', { objectGroup: 'pawshadow' }).catch(() => {}); }
|
|
1048
|
+
return found;
|
|
1049
|
+
}
|
|
1050
|
+
|
|
1051
|
+
async function snapshot(target, tries = 8, opts) {
|
|
1052
|
+
const O = JSON.stringify(opts || {});
|
|
1053
|
+
const sk = typeof target === 'object' ? `${target.tabId}#${frameKey(target)}` : target;
|
|
490
1054
|
let last;
|
|
491
|
-
for (let i = 0; i <
|
|
1055
|
+
for (let i = 0; i < tries; i++) {
|
|
492
1056
|
try {
|
|
493
|
-
|
|
494
|
-
|
|
1057
|
+
let snap = await evaluate(target, `${SNAPSHOT}(${refSeed.get(sk) || 1}, ${O})`);
|
|
1058
|
+
// Closed shadow roots found: re-snapshot with them (nested closed hosts: a few rounds).
|
|
1059
|
+
for (let k = 0; k < 3 && snap && snap.probe; k++) {
|
|
1060
|
+
if (!(await probeClosedRoots(target).catch(() => 0))) break;
|
|
1061
|
+
snap = await evaluate(target, `${SNAPSHOT}(${refSeed.get(sk) || 1}, ${O})`);
|
|
1062
|
+
}
|
|
1063
|
+
if (snap) { if (snap.next > (refSeed.get(sk) || 1)) { refSeed.set(sk, snap.next); persistState(); } return snap; }
|
|
495
1064
|
} catch (e) { last = e; }
|
|
496
|
-
await sleep(120);
|
|
1065
|
+
if (i < tries - 1) await sleep(120);
|
|
497
1066
|
}
|
|
498
1067
|
if (last) throw last;
|
|
499
1068
|
return null;
|
|
500
1069
|
}
|
|
501
1070
|
|
|
502
|
-
|
|
503
|
-
|
|
1071
|
+
// The last table each tab reported, so act() can tell the agent whether ANYTHING it can see changed
|
|
1072
|
+
// (scrolling a panel, opening a popover...) rather than only form values / URL / control count.
|
|
1073
|
+
const lastTable = new Map(); // tabId -> table text (without the header lines that hold ref numbers)
|
|
1074
|
+
|
|
1075
|
+
// Ref numbers and the focus marker are dropped: clicking a button that does nothing still focuses
|
|
1076
|
+
// it, and that alone must not count as "the page changed".
|
|
1077
|
+
function tableBody(t) { return String(t).split('\n').map((l) => l.replace(/^(f\d+\.)?e\d+(_\d+)?\s+/, '').replace(/ {2}· {2}focus e\S+$/, '')).join('\n'); }
|
|
1078
|
+
|
|
1079
|
+
// What the agent last saw, per tab (full table text), so act() can answer with just the rows that
|
|
1080
|
+
// changed. Same document + mostly-unchanged table => delta; otherwise the full table.
|
|
1081
|
+
const lastFull = new Map();
|
|
1082
|
+
const ROW_ID = /^((?:f\d+\.)?e\d+(?:_\d+)?)\s/;
|
|
1083
|
+
function deltaTable(prev, cur) {
|
|
1084
|
+
if (!prev) return null;
|
|
1085
|
+
const P = prev.split('\n'), C = cur.split('\n');
|
|
1086
|
+
if (P[0] !== C[0]) return null; // title/url changed: a different page, send it whole
|
|
1087
|
+
const prevRows = new Map();
|
|
1088
|
+
for (const l of P) { const m = ROW_ID.exec(l); if (m) prevRows.set(m[1], l); }
|
|
1089
|
+
const out = [], ids = new Set();
|
|
1090
|
+
let rows = 0, unchanged = 0;
|
|
1091
|
+
for (const l of C) {
|
|
1092
|
+
const m = ROW_ID.exec(l);
|
|
1093
|
+
if (!m) { out.push(l); continue; } // headers, frame and tool lines: always
|
|
1094
|
+
rows++; ids.add(m[1]);
|
|
1095
|
+
if (prevRows.get(m[1]) === l) unchanged++; else out.push(l);
|
|
1096
|
+
}
|
|
1097
|
+
if (rows < 12 || unchanged < rows * 0.5) return null; // small table or big change: whole is clearer
|
|
1098
|
+
const gone = [...prevRows.keys()].filter((id) => !ids.has(id));
|
|
1099
|
+
const at = out.findIndex((l) => l.startsWith('scroll ')) + 1 || 1;
|
|
1100
|
+
out.splice(at, 0, `(only changes shown: ${rows - unchanged} new/changed row(s); ${unchanged} unchanged row(s) omitted${gone.length ? `; gone: ${gone.slice(0, 40).join(', ')}${gone.length > 40 ? ' …' : ''}` : ''} — browser_observe for the full table)`);
|
|
1101
|
+
return out.join('\n');
|
|
1102
|
+
}
|
|
1103
|
+
|
|
1104
|
+
async function observe(tabId, opts) {
|
|
1105
|
+
const find = opts && opts.find != null && String(opts.find).trim() ? String(opts.find).trim().toLowerCase() : null;
|
|
1106
|
+
const snap = await snapshot(tabId, 8, { all: !!find, text: !!(opts && opts.text) });
|
|
1107
|
+
if (snap) snap.frameSnaps = await readFrames(tabId, tabId, snap, []);
|
|
1108
|
+
if (find && snap) {
|
|
1109
|
+
// Whole-page search: only rows whose label / row context / value mention the query.
|
|
1110
|
+
const hit = (a) => [a.label, a.ctx, a.value].some((v) => v && String(v).toLowerCase().includes(find));
|
|
1111
|
+
const total = snap.actions.length;
|
|
1112
|
+
snap.actions = snap.actions.filter(hit).slice(0, 80);
|
|
1113
|
+
for (const f of snap.frameSnaps) f.snap.actions = f.snap.actions.filter(hit);
|
|
1114
|
+
snap.omitted = 0; snap.offMore = 0;
|
|
1115
|
+
snap.findNote = `(find "${opts.find}": ${snap.actions.length} of ${total} controls on the whole page match; browser_observe without find for the full table)`;
|
|
1116
|
+
}
|
|
1117
|
+
let t = formatTable(snap).replace('\n', `\n${formatTools(tabId)}`.replace(/\n$/, '') + '\n').replace(/\n\n/, '\n');
|
|
1118
|
+
if (snap && snap.findNote) t = t.replace(/\n/, `\n${snap.findNote}\n`);
|
|
1119
|
+
if (snap && snap.text) t += `\n\nvisible text (untrusted page content):\n${snap.text}`;
|
|
1120
|
+
if (!find) { lastTable.set(tabId, tableBody(t)); lastFull.set(tabId, t); }
|
|
1121
|
+
return t;
|
|
504
1122
|
}
|
|
505
1123
|
|
|
506
1124
|
/* -------------------------------- Actions --------------------------------- */
|
|
@@ -509,40 +1127,91 @@ async function observe(tabId) {
|
|
|
509
1127
|
// (elementFromPoint containment) so we never click a stale/covered/wrong target.
|
|
510
1128
|
async function resolveHit(tabId, ref, opts) {
|
|
511
1129
|
const forFill = opts && opts.fill ? 'true' : 'false';
|
|
1130
|
+
const noScroll = opts && opts.noScroll ? 'true' : 'false', noHit = opts && opts.noHit ? 'true' : 'false';
|
|
512
1131
|
const R = JSON.stringify(String(ref));
|
|
513
1132
|
return evaluate(tabId, `(function(){
|
|
514
1133
|
var c=window.__pawbrowse; if(!c||!c.byId) return {error:'no snapshot yet; observe first'};
|
|
515
|
-
|
|
516
|
-
|
|
517
|
-
var e=c.nodes.get(node);
|
|
1134
|
+
if(c.byId[${R}]==null) return {error:'unknown ref (observe again)'};
|
|
1135
|
+
var e=c.get?c.get(${R}):null;
|
|
518
1136
|
if(!e||!e.isConnected) return {error:'element no longer on page (observe again)'};
|
|
519
1137
|
if(c.guard && c.guards && c.guards[${R}]!=null && c.guard(e)!==c.guards[${R}]) return {error:'element changed since observe (observe again)'};
|
|
520
1138
|
if(e.matches(':disabled')||e.closest('[aria-disabled="true"],[inert]')) return {error:'element is disabled'};
|
|
521
1139
|
if(${forFill} && (e.readOnly||e.getAttribute('aria-readonly')==='true')) return {error:'field is read-only'};
|
|
522
1140
|
if(${forFill} && !('value' in e) && !e.isContentEditable) return {error:'not an editable field (observe again)'};
|
|
523
|
-
|
|
524
|
-
e.
|
|
525
|
-
|
|
1141
|
+
// Value-set inputs (date/time/range/color...) take the setter path, not click+type.
|
|
1142
|
+
if(${forFill} && e.tagName==='INPUT' && ['date','time','datetime-local','month','week','color','range'].indexOf(e.type)>=0) return {set:true};
|
|
1143
|
+
// Click the control's visible SURFACE: a styled checkbox's <label>, or the element itself.
|
|
1144
|
+
var s=c.surface?c.surface(e):e;
|
|
1145
|
+
if(!s) return {error:'element not visible'};
|
|
1146
|
+
if(!${noScroll}) s.scrollIntoView({block:'center',inline:'center'});
|
|
1147
|
+
var r=s.getBoundingClientRect(); if(!r.width||!r.height) return {error:'element has no size'};
|
|
526
1148
|
// Frame-local center (in the element's own frame viewport)...
|
|
527
1149
|
var lx=r.x+r.width/2, ly=r.y+r.height/2;
|
|
528
1150
|
// ...plus the offset chain of any ancestor iframes, giving the TOP-LEVEL click point for CDP.
|
|
529
|
-
var dx=0, dy=0, w=(
|
|
1151
|
+
var dx=0, dy=0, w=(s.ownerDocument&&s.ownerDocument.defaultView), g=0;
|
|
530
1152
|
while(w && w.frameElement && g++<12){
|
|
531
1153
|
var fe=w.frameElement, fr=fe.getBoundingClientRect(), fcs=fe.ownerDocument.defaultView.getComputedStyle(fe);
|
|
532
|
-
dx+=fr.left+
|
|
533
|
-
dy+=fr.top+
|
|
1154
|
+
dx+=fr.left+fe.clientLeft+(parseFloat(fcs.paddingLeft)||0);
|
|
1155
|
+
dy+=fr.top+fe.clientTop+(parseFloat(fcs.paddingTop)||0);
|
|
534
1156
|
w=fe.ownerDocument.defaultView;
|
|
535
1157
|
}
|
|
536
1158
|
var x=Math.round(lx+dx), y=Math.round(ly+dy);
|
|
537
1159
|
if(x<0||y<0||x>=innerWidth||y>=innerHeight) return {error:'element off-screen after scroll'};
|
|
538
|
-
// Hit-test in the
|
|
539
|
-
// coords, so shadow-DOM and iframe elements
|
|
540
|
-
|
|
541
|
-
if(
|
|
1160
|
+
// Hit-test in the surface's OWN root (document / shadow root / iframe doc) with frame-local
|
|
1161
|
+
// coords, descending through nested open shadow roots, so shadow-DOM and iframe elements
|
|
1162
|
+
// aren't falsely reported as covered.
|
|
1163
|
+
if(${noHit}) return {x:x, y:y};
|
|
1164
|
+
var root=s.getRootNode(); if(!root||!root.elementFromPoint) root=s.ownerDocument;
|
|
1165
|
+
var f=root.elementFromPoint(lx,ly), k=0;
|
|
1166
|
+
var sroot=function(n){ return n.shadowRoot || (c.closed && c.closed.get(n)) || null; };
|
|
1167
|
+
while(f && sroot(f) && k++<16){ var inner=sroot(f).elementFromPoint(lx,ly); if(!inner||inner===f) break; f=inner; }
|
|
1168
|
+
if(!f || !(s===f || s.contains(f) || (s.control && s.control===f))) return {error:'element is covered by another element (dismiss the overlay/dialog first)'};
|
|
542
1169
|
return {x:x, y:y};
|
|
543
1170
|
})()`);
|
|
544
1171
|
}
|
|
545
1172
|
|
|
1173
|
+
// Set the value of a date/time/month/week/color/range input the way a user's picker would: via the
|
|
1174
|
+
// native value setter (so framework value-trackers see a real change), then input + change events.
|
|
1175
|
+
// The browser sanitizes invalid values to '' (or clamps ranges), so we read back and report that.
|
|
1176
|
+
async function setValue(tabId, ref, text) {
|
|
1177
|
+
const R = JSON.stringify(String(ref));
|
|
1178
|
+
const V = JSON.stringify(String(text ?? ''));
|
|
1179
|
+
return evaluate(tabId, `(function(){
|
|
1180
|
+
var c=window.__pawbrowse; var e=c&&c.get&&c.get(${R});
|
|
1181
|
+
if(!e||!e.isConnected) return {error:'element no longer on page (observe again)'};
|
|
1182
|
+
var val=${V};
|
|
1183
|
+
var setter=Object.getOwnPropertyDescriptor(HTMLInputElement.prototype,'value').set;
|
|
1184
|
+
try{ e.focus({preventScroll:true}); }catch(_){}
|
|
1185
|
+
setter.call(e,val);
|
|
1186
|
+
e.dispatchEvent(new Event('input',{bubbles:true,composed:true}));
|
|
1187
|
+
e.dispatchEvent(new Event('change',{bubbles:true}));
|
|
1188
|
+
if(val!=='' && e.value==='') return {error:'value "'+val+'" rejected by the '+e.type+' field (use the fmt{} format shown)'};
|
|
1189
|
+
return {value:e.value};
|
|
1190
|
+
})()`);
|
|
1191
|
+
}
|
|
1192
|
+
|
|
1193
|
+
// Attach local files to an <input type=file> (DOM.setFileInputFiles). Chrome only allows this for
|
|
1194
|
+
// an extension that the user has granted "Allow access to file URLs" in chrome://extensions.
|
|
1195
|
+
async function uploadFiles(tabId, ref, paths) {
|
|
1196
|
+
const R = JSON.stringify(String(ref));
|
|
1197
|
+
const h = await evaluate(tabId, `(function(){
|
|
1198
|
+
var c=window.__pawbrowse; var e=c&&c.get&&c.get(${R});
|
|
1199
|
+
if(e && c.guards && c.guards[${R}]!=null && c.guard(e)!==c.guards[${R}]) return null; // relabelled / reused input
|
|
1200
|
+
return (e&&e.isConnected&&e.tagName==='INPUT'&&e.type==='file'&&!e.disabled)?e:null;
|
|
1201
|
+
})()`, { handle: true });
|
|
1202
|
+
if (!h || !h.objectId) return { error: 'not a file-upload field, or it changed since observe (observe again)' };
|
|
1203
|
+
try {
|
|
1204
|
+
await sendCdp(tabId, 'DOM.setFileInputFiles', { files: paths, objectId: h.objectId }); // tabId: the frame target
|
|
1205
|
+
} catch (e) {
|
|
1206
|
+
return { error: /not allowed/i.test(e.message)
|
|
1207
|
+
? 'Chrome blocked the upload: enable "Allow access to file URLs" for PawBrowse in chrome://extensions'
|
|
1208
|
+
: e.message };
|
|
1209
|
+
} finally {
|
|
1210
|
+
sendCdp(tabId, 'Runtime.releaseObject', { objectId: h.objectId }).catch(() => {}); // (frame target, see above)
|
|
1211
|
+
}
|
|
1212
|
+
return { ok: true };
|
|
1213
|
+
}
|
|
1214
|
+
|
|
546
1215
|
// Click the most specific visible element matching text, for custom widgets/menus
|
|
547
1216
|
// (dropdowns, flair pickers) whose options aren't standard controls in the table.
|
|
548
1217
|
async function centerOfText(tabId, text) {
|
|
@@ -576,17 +1245,163 @@ async function centerOfText(tabId, text) {
|
|
|
576
1245
|
})()`);
|
|
577
1246
|
}
|
|
578
1247
|
|
|
579
|
-
async function clickAt(tabId, x, y) {
|
|
1248
|
+
async function clickAt(tabId, x, y, opts) {
|
|
1249
|
+
const button = (opts && opts.button) || 'left', count = Math.max(1, Math.min(3, Number(opts && opts.count) || 1));
|
|
580
1250
|
await sendCdp(tabId, 'Input.dispatchMouseEvent', { type: 'mouseMoved', x, y });
|
|
581
|
-
|
|
582
|
-
|
|
1251
|
+
// A double/triple click is successive press/release pairs with a rising clickCount.
|
|
1252
|
+
for (let n = 1; n <= count; n++) {
|
|
1253
|
+
await sendCdp(tabId, 'Input.dispatchMouseEvent', { type: 'mousePressed', x, y, button, clickCount: n });
|
|
1254
|
+
await sendCdp(tabId, 'Input.dispatchMouseEvent', { type: 'mouseReleased', x, y, button, clickCount: n });
|
|
1255
|
+
}
|
|
583
1256
|
}
|
|
584
1257
|
|
|
585
|
-
|
|
586
|
-
|
|
587
|
-
|
|
588
|
-
|
|
589
|
-
|
|
1258
|
+
/* ------------------------------ Waiting (settle) ---------------------------- *
|
|
1259
|
+
* Wait for what the page is ACTUALLY doing instead of a fixed delay: a navigation in progress (until
|
|
1260
|
+
* the new document's DOMContentLoaded), fetch/XHR requests started by the action, then a short
|
|
1261
|
+
* DOM-quiet window (MutationObserver). Each phase is capped, so long-polling, analytics beacons or a
|
|
1262
|
+
* ticking clock can't stall us, and a click that does nothing returns in a few tens of ms. */
|
|
1263
|
+
const watches = new Map(); // tabId -> { mainFrame, inflight: Map(reqId -> startedAt), navStart, navDone }
|
|
1264
|
+
function tabWatch(tabId) {
|
|
1265
|
+
let w = watches.get(tabId);
|
|
1266
|
+
if (!w) { w = { mainFrame: null, inflight: new Map(), navStart: 0, navDone: 0, committed: true, navReq: null }; watches.set(tabId, w); }
|
|
1267
|
+
return w;
|
|
1268
|
+
}
|
|
1269
|
+
const TRACKED = new Set(['Fetch', 'XHR', 'Document']);
|
|
1270
|
+
chrome.debugger.onEvent.addListener((source, method, params) => {
|
|
1271
|
+
if (source.tabId == null) return;
|
|
1272
|
+
const w = watches.get(source.tabId);
|
|
1273
|
+
if (!w) return;
|
|
1274
|
+
// A request can finish on a different session than it started on (an iframe's document request
|
|
1275
|
+
// is announced by the parent, completed by the child): accept completions from any session.
|
|
1276
|
+
if (method === 'Network.loadingFinished' || method === 'Network.loadingFailed') {
|
|
1277
|
+
w.inflight.delete(params.requestId);
|
|
1278
|
+
if (method === 'Network.loadingFailed' && params.requestId === w.navReq && !w.committed) w.navDone = Date.now(); // blocked/aborted navigation
|
|
1279
|
+
return;
|
|
1280
|
+
}
|
|
1281
|
+
if (source.sessionId) return; // everything else: the top-level target only
|
|
1282
|
+
const now = Date.now();
|
|
1283
|
+
const begin = (loaderId) => { w.navStart = now; w.navDone = 0; w.committed = false; w.navReq = loaderId || null; w.netIdle = 0; };
|
|
1284
|
+
switch (method) {
|
|
1285
|
+
case 'Network.requestWillBeSent':
|
|
1286
|
+
// Documents: only the main frame's (subframe documents finish on other sessions / may never).
|
|
1287
|
+
if (TRACKED.has(params.type) && (params.type !== 'Document' || params.frameId === w.mainFrame)) w.inflight.set(params.requestId, now);
|
|
1288
|
+
if (params.type === 'Document' && params.frameId === w.mainFrame && params.requestId === params.loaderId && w.navReq !== params.requestId) {
|
|
1289
|
+
if (w.navDone >= w.navStart) begin(params.requestId); else w.navReq = params.requestId;
|
|
1290
|
+
}
|
|
1291
|
+
break;
|
|
1292
|
+
case 'Network.responseReceived':
|
|
1293
|
+
// A 204/205 navigation never replaces the document: it's over as soon as the response lands.
|
|
1294
|
+
if (params.requestId === w.navReq && (params.response.status === 204 || params.response.status === 205)) w.navDone = now;
|
|
1295
|
+
break;
|
|
1296
|
+
case 'Page.frameRequestedNavigation': case 'Page.frameStartedNavigating':
|
|
1297
|
+
if (params.frameId === w.mainFrame && params.navigationType !== 'sameDocument') begin(params.loaderId);
|
|
1298
|
+
break;
|
|
1299
|
+
case 'Page.frameNavigated':
|
|
1300
|
+
if (!params.frame.parentId) { w.mainFrame = params.frame.id; w.committed = true; if (params.frame.loaderId) w.navReq = params.frame.loaderId; }
|
|
1301
|
+
break;
|
|
1302
|
+
// The OLD document can still fire load/stop events after a navigation starts: only events that
|
|
1303
|
+
// follow the new document's commit (frameNavigated) mean "arrived".
|
|
1304
|
+
case 'Page.domContentEventFired': case 'Page.loadEventFired':
|
|
1305
|
+
if (w.committed) w.navDone = now;
|
|
1306
|
+
break;
|
|
1307
|
+
case 'Page.frameStoppedLoading':
|
|
1308
|
+
if (params.frameId === w.mainFrame && w.committed) w.navDone = now;
|
|
1309
|
+
break;
|
|
1310
|
+
case 'Page.lifecycleEvent':
|
|
1311
|
+
// Tied to the NEW document's loader, so the old page's late events can't end a wait.
|
|
1312
|
+
if (params.frameId === w.mainFrame && params.loaderId && params.loaderId === w.navReq) {
|
|
1313
|
+
if (params.name === 'DOMContentLoaded' || params.name === 'load') { w.committed = true; w.navDone = now; }
|
|
1314
|
+
if (params.name === 'networkAlmostIdle' || params.name === 'networkIdle') w.netIdle = now;
|
|
1315
|
+
}
|
|
1316
|
+
break;
|
|
1317
|
+
case 'Page.downloadWillBegin': case 'Page.navigatedWithinDocument':
|
|
1318
|
+
w.navDone = now;
|
|
1319
|
+
break;
|
|
1320
|
+
}
|
|
1321
|
+
});
|
|
1322
|
+
|
|
1323
|
+
// -> [ms since last DOM change, was the DOM already busy in the 600ms before `since` (epoch ms)?]
|
|
1324
|
+
const quietExpr = (since) => `(function(){ ${MO_INSTALL}
|
|
1325
|
+
var now=performance.now(), s=${Number(since) || 0}-Date.now()+now, b={};
|
|
1326
|
+
M.times.forEach(function(t){ if(t<s && t>=s-600) b[Math.floor((s-t)/100)]=1; });
|
|
1327
|
+
return [now-M.last, Object.keys(b).length>=4];
|
|
1328
|
+
})()`;
|
|
1329
|
+
|
|
1330
|
+
// Network tracking only while an action is being watched (see attach()).
|
|
1331
|
+
async function netOn(tabId) {
|
|
1332
|
+
const w = tabWatch(tabId);
|
|
1333
|
+
if (w.net) return;
|
|
1334
|
+
w.net = true;
|
|
1335
|
+
await sendCdp(tabId, 'Network.enable', { maxTotalBufferSize: 0, maxResourceBufferSize: 0 }).catch(() => { w.net = false; });
|
|
1336
|
+
}
|
|
1337
|
+
async function netOff(tabId) {
|
|
1338
|
+
const w = tabWatch(tabId);
|
|
1339
|
+
if (!w.net) return;
|
|
1340
|
+
w.net = false; w.inflight.clear();
|
|
1341
|
+
await sendCdp(tabId, 'Network.disable').catch(() => {});
|
|
1342
|
+
}
|
|
1343
|
+
|
|
1344
|
+
async function settle(tabId, capMs, since, opts) {
|
|
1345
|
+
const start = since || Date.now();
|
|
1346
|
+
const w = tabWatch(tabId);
|
|
1347
|
+
const deadline = start + (capMs || 3000);
|
|
1348
|
+
// After typing, search boxes commonly DEBOUNCE (wait ~150-300ms of no typing, then fetch): there's
|
|
1349
|
+
// no signal to follow during that gap, so give a typed field a grace window for a request or a
|
|
1350
|
+
// re-render to begin before calling it idle.
|
|
1351
|
+
const grace = start + ((opts && opts.grace) || 0);
|
|
1352
|
+
// Let the action's handlers run first (event loop turn + a frame).
|
|
1353
|
+
await sleep(25);
|
|
1354
|
+
let idleSince = 0;
|
|
1355
|
+
// Which requests are the action's (vs. background beacons/polling that never stop): those that
|
|
1356
|
+
// start within 500ms of the action — or of the new page's DOMContentLoaded — plus requests that
|
|
1357
|
+
// start right after one of those finishes (a fetch chain). Anything else is ignored.
|
|
1358
|
+
const rel = new Set();
|
|
1359
|
+
let windowEnd = start + 500, sawNav = false;
|
|
1360
|
+
for (;;) {
|
|
1361
|
+
const now = Date.now();
|
|
1362
|
+
const navigating = w.navStart >= start - 50 && w.navDone < w.navStart;
|
|
1363
|
+
// A navigation gets a longer allowance: the next page must actually arrive.
|
|
1364
|
+
if (now > (navigating ? Math.max(deadline, start + 10000) : deadline)) break;
|
|
1365
|
+
if (navigating) {
|
|
1366
|
+
// Once the new document has committed, stop network tracking: its (possibly hundreds of)
|
|
1367
|
+
// subresource requests would only slow the browser down. Lifecycle events take over.
|
|
1368
|
+
if (w.committed && w.net) await netOff(tabId);
|
|
1369
|
+
sawNav = true; await sleep(30); continue;
|
|
1370
|
+
}
|
|
1371
|
+
if (sawNav) {
|
|
1372
|
+
sawNav = false; windowEnd = Math.max(windowEnd, w.navDone + 500);
|
|
1373
|
+
if (!w.net) {
|
|
1374
|
+
// New page: let its initial data requests settle (Chrome's networkAlmostIdle), capped.
|
|
1375
|
+
// Ends early once the document is fully loaded and the DOM has been quiet for 300ms.
|
|
1376
|
+
const until = Math.min(deadline, w.navDone + 800);
|
|
1377
|
+
while (!(w.netIdle >= w.navStart) && Date.now() < until) {
|
|
1378
|
+
try {
|
|
1379
|
+
const [q, , rs] = await evaluate(tabId, `(function(){ var r=${quietExpr(start)}; return [r[0], r[1], document.readyState]; })()`);
|
|
1380
|
+
if (rs === 'complete' && q >= 300) break;
|
|
1381
|
+
} catch {}
|
|
1382
|
+
await sleep(40);
|
|
1383
|
+
}
|
|
1384
|
+
}
|
|
1385
|
+
}
|
|
1386
|
+
let busy = false;
|
|
1387
|
+
for (const [id, t] of w.inflight) {
|
|
1388
|
+
if (now - t > 15000) { w.inflight.delete(id); continue; } // leaked / long-poll: forget it
|
|
1389
|
+
if (!rel.has(id) && t >= start - 50 && t <= windowEnd) rel.add(id);
|
|
1390
|
+
}
|
|
1391
|
+
for (const id of rel) {
|
|
1392
|
+
if (!w.inflight.has(id)) { rel.delete(id); windowEnd = Math.max(windowEnd, now + 150); continue; } // finished: allow a follow-up
|
|
1393
|
+
if (now - w.inflight.get(id) < 8000) busy = true;
|
|
1394
|
+
}
|
|
1395
|
+
if (busy) { idleSince = 0; await sleep(30); continue; }
|
|
1396
|
+
if (now < grace) { await sleep(30); continue; } // debounce window: a request may still be coming
|
|
1397
|
+
if (!idleSince) idleSince = now;
|
|
1398
|
+
let quiet = 1e9, ambient = false;
|
|
1399
|
+
try { [quiet, ambient] = await evaluate(tabId, quietExpr(start)); } catch { await sleep(30); continue; } // document swapping
|
|
1400
|
+
// DOM still changing: wait for 60ms of quiet, capped at 600ms after the network went idle, or
|
|
1401
|
+
// 120ms on a page that was already constantly mutating (clocks, tickers, carousels) before us.
|
|
1402
|
+
if (quiet < 60 && now - idleSince < (ambient ? 120 : 600)) { await sleep(Math.max(10, 60 - quiet)); continue; }
|
|
1403
|
+
break;
|
|
1404
|
+
}
|
|
590
1405
|
}
|
|
591
1406
|
|
|
592
1407
|
// After typing into a combobox, wait for its autocomplete options to actually render
|
|
@@ -600,7 +1415,7 @@ async function waitForOptions(tabId, ref, ms) {
|
|
|
600
1415
|
await evaluate(tabId, `new Promise(function(res){
|
|
601
1416
|
var done=false; function fin(){ if(done) return; done=true; try{clearInterval(iv);}catch(_){} res(1); }
|
|
602
1417
|
setTimeout(fin, ${cap});
|
|
603
|
-
var c=window.__pawbrowse; var
|
|
1418
|
+
var c=window.__pawbrowse; var e=(c&&c.get)?c.get(${R}):null;
|
|
604
1419
|
if(!e || (e.getAttribute('role')||'').toLowerCase()!=='combobox'){ return fin(); }
|
|
605
1420
|
var ids=(e.getAttribute('aria-controls')||e.getAttribute('aria-owns')||'').split(/\\s+/).filter(Boolean);
|
|
606
1421
|
var iv=setInterval(function(){
|
|
@@ -624,15 +1439,202 @@ const KEYMAP = {
|
|
|
624
1439
|
ArrowUp: { key: 'ArrowUp', code: 'ArrowUp', windowsVirtualKeyCode: 38 },
|
|
625
1440
|
ArrowLeft: { key: 'ArrowLeft', code: 'ArrowLeft', windowsVirtualKeyCode: 37 },
|
|
626
1441
|
ArrowRight: { key: 'ArrowRight', code: 'ArrowRight', windowsVirtualKeyCode: 39 },
|
|
1442
|
+
Space: { key: ' ', code: 'Space', windowsVirtualKeyCode: 32, text: ' ' },
|
|
1443
|
+
Delete: { key: 'Delete', code: 'Delete', windowsVirtualKeyCode: 46 },
|
|
1444
|
+
Home: { key: 'Home', code: 'Home', windowsVirtualKeyCode: 36 },
|
|
1445
|
+
End: { key: 'End', code: 'End', windowsVirtualKeyCode: 35 },
|
|
1446
|
+
PageUp: { key: 'PageUp', code: 'PageUp', windowsVirtualKeyCode: 33 },
|
|
1447
|
+
PageDown: { key: 'PageDown', code: 'PageDown', windowsVirtualKeyCode: 34 },
|
|
627
1448
|
};
|
|
628
1449
|
|
|
1450
|
+
// "Shift+Tab", "Mod+A" (Cmd on macOS, Ctrl elsewhere), "Control+Enter", "F5", "a", "?" ...
|
|
1451
|
+
const MODS = { Alt: 1, Option: 1, Control: 2, Ctrl: 2, Meta: 4, Cmd: 4, Command: 4, Shift: 8, Mod: IS_MAC ? 4 : 2 };
|
|
1452
|
+
// macOS editing shortcuts aren't bound to Cmd+key for synthetic events: they need the command name.
|
|
1453
|
+
const MAC_COMMANDS = { a: 'selectAll', c: 'copy', v: 'paste', x: 'cut', z: 'undo' };
|
|
1454
|
+
function keyDef(name) {
|
|
1455
|
+
if (KEYMAP[name]) return { ...KEYMAP[name] };
|
|
1456
|
+
if (name.length === 1) {
|
|
1457
|
+
const up = name.toUpperCase();
|
|
1458
|
+
const alnum = /[a-z0-9]/i.test(name);
|
|
1459
|
+
return { key: name, code: /[a-z]/i.test(name) ? `Key${up}` : /[0-9]/.test(name) ? `Digit${name}` : '', windowsVirtualKeyCode: alnum ? up.charCodeAt(0) : name.charCodeAt(0), text: name };
|
|
1460
|
+
}
|
|
1461
|
+
const f = /^F(\d{1,2})$/.exec(name);
|
|
1462
|
+
if (f) return { key: name, code: name, windowsVirtualKeyCode: 111 + Number(f[1]) };
|
|
1463
|
+
return null;
|
|
1464
|
+
}
|
|
1465
|
+
async function pressKey(tabId, combo) {
|
|
1466
|
+
const parts = String(combo).split(/\+(?!$)/); // "Control++" -> ["Control", "+"]
|
|
1467
|
+
let mods = 0;
|
|
1468
|
+
for (const m of parts.slice(0, -1)) { if (!(m in MODS)) return `unknown modifier "${m}"`; mods |= MODS[m]; }
|
|
1469
|
+
const def = keyDef(parts[parts.length - 1]);
|
|
1470
|
+
if (!def) return `key "${combo}" not supported`;
|
|
1471
|
+
if ((mods & 8) && def.text && def.text.length === 1) { def.key = def.text = def.text.toUpperCase(); }
|
|
1472
|
+
if (mods & 6) delete def.text; // Ctrl/Cmd chords are shortcuts, not text
|
|
1473
|
+
const commands = IS_MAC && (mods & 4) && MAC_COMMANDS[String(def.key).toLowerCase()] ? [(mods & 8) && def.key.toLowerCase() === 'z' ? 'redo' : MAC_COMMANDS[def.key.toLowerCase()]] : undefined;
|
|
1474
|
+
await sendCdp(tabId, 'Input.dispatchKeyEvent', { type: def.text ? 'keyDown' : 'rawKeyDown', ...def, modifiers: mods, commands });
|
|
1475
|
+
await sendCdp(tabId, 'Input.dispatchKeyEvent', { type: 'keyUp', ...def, text: undefined, modifiers: mods });
|
|
1476
|
+
return null;
|
|
1477
|
+
}
|
|
1478
|
+
|
|
1479
|
+
/* ---------------------------------- WebMCP --------------------------------- *
|
|
1480
|
+
* Pages that implement WebMCP (navigator.modelContext.registerTool / <form toolname>) describe
|
|
1481
|
+
* their own actions with JSON schemas. Calling one is a single deterministic step instead of a
|
|
1482
|
+
* dozen clicks, so observe lists them first and op "tool" invokes them via the WebMCP CDP domain. */
|
|
1483
|
+
const webTools = new Map(); // tabId -> Map(name -> { tool, where })
|
|
1484
|
+
const toolCalls = new Map(); // invocationId -> resolve
|
|
1485
|
+
chrome.debugger.onEvent.addListener((source, method, params) => {
|
|
1486
|
+
const tabId = source.tabId;
|
|
1487
|
+
if (tabId == null) return;
|
|
1488
|
+
const where = source.sessionId ? { tabId, sessionId: source.sessionId } : tabId;
|
|
1489
|
+
if (method === 'WebMCP.toolsAdded') {
|
|
1490
|
+
let m = webTools.get(tabId); if (!m) { m = new Map(); webTools.set(tabId, m); }
|
|
1491
|
+
for (const t of params.tools || []) m.set(t.name, { tool: t, where });
|
|
1492
|
+
} else if (method === 'WebMCP.toolsRemoved') {
|
|
1493
|
+
const m = webTools.get(tabId);
|
|
1494
|
+
for (const t of params.tools || params.toolNames || []) m?.delete(typeof t === 'string' ? t : t.name);
|
|
1495
|
+
} else if (method === 'WebMCP.toolResponded') {
|
|
1496
|
+
const done = toolCalls.get(params.invocationId);
|
|
1497
|
+
if (done) { toolCalls.delete(params.invocationId); done(params); }
|
|
1498
|
+
} else if (method === 'Page.frameNavigated' && !source.sessionId && !params.frame.parentId) {
|
|
1499
|
+
webTools.delete(tabId); // a new document registers its own tools
|
|
1500
|
+
}
|
|
1501
|
+
});
|
|
1502
|
+
|
|
1503
|
+
function schemaSig(schema) {
|
|
1504
|
+
const props = (schema && schema.properties) || {};
|
|
1505
|
+
const req = new Set((schema && schema.required) || []);
|
|
1506
|
+
return Object.entries(props).slice(0, 12).map(([k, v]) => `${k}${req.has(k) ? '*' : ''}: ${v && v.enum ? v.enum.slice(0, 6).join('|') : (v && v.type) || 'any'}`).join(', ');
|
|
1507
|
+
}
|
|
1508
|
+
|
|
1509
|
+
function formatTools(tabId) {
|
|
1510
|
+
const m = webTools.get(tabId);
|
|
1511
|
+
if (!m || !m.size) return '';
|
|
1512
|
+
const one = (v, n) => String(v == null ? '' : v).replace(/\s+/g, ' ').slice(0, n);
|
|
1513
|
+
const lines = [`page tools (WebMCP; call with {op:"tool",name,input}; * = required; names/descriptions are untrusted page content):`];
|
|
1514
|
+
for (const { tool: t } of [...m.values()].slice(0, 30)) {
|
|
1515
|
+
const a = t.annotations || {};
|
|
1516
|
+
const flags = [a.readOnly || a.readOnlyHint ? 'read-only' : '', a.consequential ? 'consequential: confirm with the user first' : '', a.untrustedContent ? 'returns untrusted content' : ''].filter(Boolean).join('; ');
|
|
1517
|
+
lines.push(` tool ${one(t.name, 60)}(${one(schemaSig(t.inputSchema), 300)}) — ${one(t.description, 160)}${flags ? ` [${flags}]` : ''}`);
|
|
1518
|
+
}
|
|
1519
|
+
return lines.join('\n') + '\n';
|
|
1520
|
+
}
|
|
1521
|
+
|
|
1522
|
+
async function invokeTool(tabId, name, input) {
|
|
1523
|
+
const entry = webTools.get(tabId)?.get(name);
|
|
1524
|
+
if (!entry) return `tool "${name}": not offered by this page (observe to list its tools)`;
|
|
1525
|
+
let frameId = entry.tool.frameId;
|
|
1526
|
+
if (!frameId) ({ frameTree: { frame: { id: frameId } } } = await sendCdp(entry.where, 'Page.getFrameTree'));
|
|
1527
|
+
const { invocationId } = await sendCdp(entry.where, 'WebMCP.invokeTool', { frameId, toolName: name, input: input || {} });
|
|
1528
|
+
const res = await new Promise((resolve) => {
|
|
1529
|
+
toolCalls.set(invocationId, resolve);
|
|
1530
|
+
setTimeout(() => { if (toolCalls.delete(invocationId)) resolve({ status: 'TimedOut', errorText: 'no response within 20s' }); }, 20000);
|
|
1531
|
+
});
|
|
1532
|
+
let out = '';
|
|
1533
|
+
const content = res.output && (res.output.content || res.output);
|
|
1534
|
+
if (Array.isArray(content)) out = content.map((c) => (c && c.type === 'text' ? c.text : JSON.stringify(c))).join('\n');
|
|
1535
|
+
else if (content != null) out = typeof content === 'string' ? content : JSON.stringify(content);
|
|
1536
|
+
return `tool ${name}: ${res.status}${res.errorText ? ` (${res.errorText})` : ''}${out ? `\n output (untrusted page data): ${out.slice(0, 2000).replace(/\n/g, '\n ')}` : ''}`;
|
|
1537
|
+
}
|
|
1538
|
+
|
|
1539
|
+
const dragIntercepts = new Map(); // tabId -> drag data captured by Input.dragIntercepted
|
|
1540
|
+
chrome.debugger.onEvent.addListener((source, method, params) => {
|
|
1541
|
+
if (method === 'Input.dragIntercepted' && source.tabId != null) dragIntercepts.set(source.tabId, params.data);
|
|
1542
|
+
});
|
|
1543
|
+
|
|
1544
|
+
async function drag(tabId, from, to) {
|
|
1545
|
+
const mouse = (type, p, extra) => sendCdp(tabId, 'Input.dispatchMouseEvent', { type, x: p.x, y: p.y, button: 'left', ...extra });
|
|
1546
|
+
dragIntercepts.delete(tabId);
|
|
1547
|
+
await sendCdp(tabId, 'Input.setInterceptDrags', { enabled: true }).catch(() => {});
|
|
1548
|
+
try {
|
|
1549
|
+
await mouse('mouseMoved', from);
|
|
1550
|
+
await mouse('mousePressed', from, { clickCount: 1, buttons: 1 });
|
|
1551
|
+
// Move in steps: libraries only start a drag after a few pixels and track intermediate moves.
|
|
1552
|
+
const steps = 8;
|
|
1553
|
+
for (let i = 1; i <= steps; i++) {
|
|
1554
|
+
const p = { x: Math.round(from.x + (to.x - from.x) * i / steps), y: Math.round(from.y + (to.y - from.y) * i / steps) };
|
|
1555
|
+
await mouse('mouseMoved', p, { buttons: 1 });
|
|
1556
|
+
const data = dragIntercepts.get(tabId);
|
|
1557
|
+
if (data) {
|
|
1558
|
+
// Native HTML5 drag started: deliver it to the drop target and finish there.
|
|
1559
|
+
for (const type of ['dragEnter', 'dragOver', 'drop']) await sendCdp(tabId, 'Input.dispatchDragEvent', { type, x: to.x, y: to.y, data });
|
|
1560
|
+
await mouse('mouseReleased', to, { clickCount: 1 });
|
|
1561
|
+
return ' (html5 drop)';
|
|
1562
|
+
}
|
|
1563
|
+
await sleep(16);
|
|
1564
|
+
}
|
|
1565
|
+
await mouse('mouseReleased', to, { clickCount: 1 });
|
|
1566
|
+
return '';
|
|
1567
|
+
} finally {
|
|
1568
|
+
dragIntercepts.delete(tabId);
|
|
1569
|
+
sendCdp(tabId, 'Input.setInterceptDrags', { enabled: false }).catch(() => {});
|
|
1570
|
+
}
|
|
1571
|
+
}
|
|
1572
|
+
|
|
629
1573
|
async function runOp(tabId, op) {
|
|
1574
|
+
const pol = acting.get(tabId);
|
|
1575
|
+
if (pol) { pol.accept = op.dialog === 'accept' ? true : op.dialog === 'dismiss' ? false : undefined; pol.text = op.dialog_text; }
|
|
1576
|
+
// Route the ref to its frame: T is where page-side code runs, R the frame-local ref; top() turns
|
|
1577
|
+
// frame-local coordinates into top-level ones for input (which the browser routes to the frame).
|
|
1578
|
+
const rt = op.ref != null ? routeRef(tabId, op.ref) : { target: tabId, ref: op.ref, frame: null };
|
|
1579
|
+
if (op.ref != null && !rt.target) return `${op.ref}: unknown frame (observe again)`;
|
|
1580
|
+
const T = rt.target, REF = rt.ref;
|
|
1581
|
+
const top = async (x, y) => {
|
|
1582
|
+
if (!rt.frame) return { x, y };
|
|
1583
|
+
const off = await frameOffset(tabId, rt.frame);
|
|
1584
|
+
const p = { x: Math.round(x + off.x), y: Math.round(y + off.y) };
|
|
1585
|
+
// The frame's own hit-test can't see the PARENT page: make sure the point really lands in this
|
|
1586
|
+
// frame and not on a cookie banner / modal of the page around it.
|
|
1587
|
+
try {
|
|
1588
|
+
const hit = await sendCdp(tabId, 'DOM.getNodeForLocation', { x: p.x, y: p.y, includeUserAgentShadowDOM: false });
|
|
1589
|
+
const main = tabWatch(tabId).mainFrame;
|
|
1590
|
+
if (off.topOwner != null && hit.backendNodeId !== off.topOwner && (!hit.frameId || hit.frameId === main)) p.covered = true;
|
|
1591
|
+
} catch {}
|
|
1592
|
+
return p;
|
|
1593
|
+
};
|
|
630
1594
|
switch (op.op) {
|
|
1595
|
+
case 'dialog': {
|
|
1596
|
+
const d = openDialogs.get(tabId);
|
|
1597
|
+
if (!d) return 'dialog: no dialog is open';
|
|
1598
|
+
await sendCdp(d.where || tabId, 'Page.handleJavaScriptDialog', { accept: !!op.accept, promptText: op.text != null ? String(op.text) : '' });
|
|
1599
|
+
openDialogs.delete(tabId);
|
|
1600
|
+
return `dialog: ${d.type} "${String(d.message || '').slice(0, 120)}" → ${op.accept ? 'accepted' : 'dismissed'}`;
|
|
1601
|
+
}
|
|
631
1602
|
case 'click': {
|
|
632
|
-
const r = await resolveHit(
|
|
1603
|
+
const r = await resolveHit(T, REF);
|
|
633
1604
|
if (r.error) return `${op.ref}: ${r.error}`;
|
|
634
|
-
await
|
|
635
|
-
return
|
|
1605
|
+
const p = await top(r.x, r.y);
|
|
1606
|
+
if (p.covered) return `${op.ref}: element is covered by another element of the page around its frame (dismiss the overlay first)`;
|
|
1607
|
+
const button = ['right', 'middle'].includes(op.button) ? op.button : 'left';
|
|
1608
|
+
await clickAt(tabId, p.x, p.y, { button, count: op.count });
|
|
1609
|
+
return `${op.count > 1 ? `${op.count}x ` : ''}${button !== 'left' ? `${button}-` : ''}click ${op.ref}`;
|
|
1610
|
+
}
|
|
1611
|
+
case 'hover': {
|
|
1612
|
+
const r = await resolveHit(T, REF);
|
|
1613
|
+
if (r.error) return `${op.ref}: ${r.error}`;
|
|
1614
|
+
const p = await top(r.x, r.y);
|
|
1615
|
+
await sendCdp(tabId, 'Input.dispatchMouseEvent', { type: 'mouseMoved', x: p.x, y: p.y });
|
|
1616
|
+
return `hover ${op.ref}`;
|
|
1617
|
+
}
|
|
1618
|
+
case 'drag': {
|
|
1619
|
+
// Drag ref onto to:"eN" (or by dx/dy pixels). Pointer-driven widgets (sliders, sortable
|
|
1620
|
+
// lists) get a real press-move-release; native HTML5 drag-and-drop is intercepted by CDP and
|
|
1621
|
+
// replayed as dragEnter/dragOver/drop on the target.
|
|
1622
|
+
const a = await resolveHit(T, REF);
|
|
1623
|
+
if (a.error) return `${op.ref}: ${a.error}`;
|
|
1624
|
+
const from = await top(a.x, a.y);
|
|
1625
|
+
let to;
|
|
1626
|
+
if (op.to_text) {
|
|
1627
|
+
const c = await centerOfText(tabId, op.to_text);
|
|
1628
|
+
if (!c) return `drag: drop target "${op.to_text}" not found`;
|
|
1629
|
+
to = c;
|
|
1630
|
+
} else if (op.to) {
|
|
1631
|
+
const rt2 = routeRef(tabId, op.to);
|
|
1632
|
+
if (!rt2.target) return `${op.to}: unknown frame (observe again)`;
|
|
1633
|
+
const b = await resolveHit(rt2.target, rt2.ref, { noScroll: true, noHit: true });
|
|
1634
|
+
if (b.error) return `${op.to}: ${b.error}`;
|
|
1635
|
+
to = rt2.frame ? await (async () => { const off = await frameOffset(tabId, rt2.frame); return { x: Math.round(b.x + off.x), y: Math.round(b.y + off.y) }; })() : { x: b.x, y: b.y };
|
|
1636
|
+
} else to = { x: from.x + Number(op.dx || 0), y: from.y + Number(op.dy || 0) };
|
|
1637
|
+
return drag(tabId, from, to).then((how) => `drag ${op.ref} → ${op.to || (op.to_text ? `"${op.to_text}"` : `${op.dx || 0},${op.dy || 0}`)}${how}`);
|
|
636
1638
|
}
|
|
637
1639
|
case 'click_text': {
|
|
638
1640
|
const c = await centerOfText(tabId, op.text);
|
|
@@ -641,9 +1643,15 @@ async function runOp(tabId, op) {
|
|
|
641
1643
|
return `click_text "${op.text}"`;
|
|
642
1644
|
}
|
|
643
1645
|
case 'type': {
|
|
644
|
-
const r = await resolveHit(
|
|
1646
|
+
const r = await resolveHit(T, REF, { fill: true });
|
|
645
1647
|
if (r.error) return `${op.ref}: ${r.error}`;
|
|
646
|
-
|
|
1648
|
+
if (r.set) {
|
|
1649
|
+
const sv = await setValue(T, REF, op.text);
|
|
1650
|
+
return sv.error ? `${op.ref}: ${sv.error}` : `type ${op.ref} (set to "${sv.value}")`;
|
|
1651
|
+
}
|
|
1652
|
+
const p = await top(r.x, r.y);
|
|
1653
|
+
if (p.covered) return `${op.ref}: element is covered by another element of the page around its frame (dismiss the overlay first)`;
|
|
1654
|
+
await clickAt(tabId, p.x, p.y); // focus the field with a trusted click
|
|
647
1655
|
// Select-all then insert — robust for React/controlled inputs.
|
|
648
1656
|
await sendCdp(tabId, 'Input.dispatchKeyEvent', { type: 'keyDown', key: 'a', code: 'KeyA', modifiers: IS_MAC ? 4 : 2, commands: ['selectAll'] });
|
|
649
1657
|
await sendCdp(tabId, 'Input.dispatchKeyEvent', { type: 'keyUp', key: 'a', code: 'KeyA', modifiers: IS_MAC ? 4 : 2 });
|
|
@@ -655,23 +1663,25 @@ async function runOp(tabId, op) {
|
|
|
655
1663
|
} else {
|
|
656
1664
|
await sendCdp(tabId, 'Input.insertText', { text: txt });
|
|
657
1665
|
}
|
|
658
|
-
await waitForOptions(
|
|
1666
|
+
await waitForOptions(T, REF, 250); // let autocomplete suggestions render
|
|
659
1667
|
return `type ${op.ref}`;
|
|
660
1668
|
}
|
|
661
1669
|
case 'select': {
|
|
662
|
-
const R = JSON.stringify(String(
|
|
663
|
-
const
|
|
1670
|
+
const R = JSON.stringify(String(REF));
|
|
1671
|
+
const VS = JSON.stringify([].concat(op.values ?? op.value ?? '').map(String));
|
|
664
1672
|
try {
|
|
665
|
-
const res = await evaluate(
|
|
666
|
-
var c=window.__pawbrowse; var
|
|
667
|
-
var e=node!=null?c.nodes.get(node):null;
|
|
1673
|
+
const res = await evaluate(T, `(function(){
|
|
1674
|
+
var c=window.__pawbrowse; var e=(c&&c.get)?c.get(${R}):null;
|
|
668
1675
|
if(!e||!e.isConnected) return 'unknown ref (observe again)';
|
|
1676
|
+
if(c.guards && c.guards[${R}]!=null && c.guard(e)!==c.guards[${R}]) return 'element changed since observe (observe again)';
|
|
669
1677
|
if(e.tagName!=='SELECT') return 'not a dropdown';
|
|
670
1678
|
if(e.matches(':disabled')||e.closest('[aria-disabled="true"],[inert]')) return 'dropdown is disabled';
|
|
671
1679
|
if(!e.checkVisibility({checkOpacity:true,checkVisibilityCSS:true})) return 'dropdown not visible';
|
|
672
|
-
var
|
|
673
|
-
|
|
674
|
-
if(
|
|
1680
|
+
var vals=${VS}, m=0, ok=function(o){ return !o.disabled && !(o.closest&&o.closest('optgroup[disabled]')); };
|
|
1681
|
+
var hit=function(o){ return vals.some(function(v){ return o.value===v||o.label===v||o.text===v; }); };
|
|
1682
|
+
if(e.multiple){ for(var i=0;i<e.options.length;i++){ var o=e.options[i]; var want=ok(o)&&hit(o); if(want) m++; o.selected=want; } }
|
|
1683
|
+
else for(var j=0;j<e.options.length;j++){ if(ok(e.options[j]) && hit(e.options[j])){ e.selectedIndex=j; m=1; break; } }
|
|
1684
|
+
if(m<vals.length) return m ? 'some options not found' : 'option not found';
|
|
675
1685
|
e.dispatchEvent(new Event('input',{bubbles:true})); e.dispatchEvent(new Event('change',{bubbles:true}));
|
|
676
1686
|
return 'ok';
|
|
677
1687
|
})()`);
|
|
@@ -681,19 +1691,60 @@ async function runOp(tabId, op) {
|
|
|
681
1691
|
return `select ${op.ref}: may have applied and navigated the page; observe again before retrying`;
|
|
682
1692
|
}
|
|
683
1693
|
}
|
|
1694
|
+
case 'upload': {
|
|
1695
|
+
const paths = [].concat(op.paths ?? op.path ?? []).map(String).filter(Boolean);
|
|
1696
|
+
if (!paths.length) return `${op.ref}: upload needs paths:["/absolute/file"]`;
|
|
1697
|
+
const u = await uploadFiles(T, REF, paths);
|
|
1698
|
+
return u.error ? `${op.ref}: ${u.error}` : `upload ${op.ref} (${paths.length} file${paths.length > 1 ? 's' : ''})`;
|
|
1699
|
+
}
|
|
1700
|
+
case 'click_xy': {
|
|
1701
|
+
// Coordinates from the last browser_screenshot image (converted to CSS px).
|
|
1702
|
+
const k = shotScale.get(tabId) || 1;
|
|
1703
|
+
const x = Math.round(Number(op.x) * k), y = Math.round(Number(op.y) * k);
|
|
1704
|
+
if (!Number.isFinite(x) || !Number.isFinite(y)) return 'click_xy needs numeric x and y';
|
|
1705
|
+
const button = ['right', 'middle'].includes(op.button) ? op.button : 'left';
|
|
1706
|
+
await clickAt(tabId, x, y, { button, count: op.count });
|
|
1707
|
+
return `click_xy ${op.x},${op.y}`;
|
|
1708
|
+
}
|
|
1709
|
+
case 'tool': {
|
|
1710
|
+
return invokeTool(tabId, String(op.name || ''), op.input);
|
|
1711
|
+
}
|
|
684
1712
|
case 'key': {
|
|
685
|
-
const
|
|
686
|
-
|
|
687
|
-
await sendCdp(tabId, 'Input.dispatchKeyEvent', { type: 'keyDown', ...k });
|
|
688
|
-
await sendCdp(tabId, 'Input.dispatchKeyEvent', { type: 'keyUp', ...k });
|
|
689
|
-
return `key ${op.key}`;
|
|
1713
|
+
const err = await pressKey(tabId, op.key);
|
|
1714
|
+
return err || `key ${op.key}`;
|
|
690
1715
|
}
|
|
691
1716
|
case 'scroll': {
|
|
692
1717
|
const dy = Number(op.dy ?? 600);
|
|
693
|
-
// Real wheel event so overflow containers, virtualized lists, and infinite scroll fire.
|
|
1718
|
+
// Real wheel event so overflow containers, virtualized lists, and infinite scroll fire. With a
|
|
1719
|
+
// ref, the wheel goes to THAT element (a side panel, a dropdown list, a chat pane) instead of
|
|
1720
|
+
// the middle of the page, so the right box scrolls.
|
|
694
1721
|
let cx = 400, cy = 400;
|
|
695
|
-
try {
|
|
1722
|
+
try {
|
|
1723
|
+
const R = JSON.stringify(String(REF || ''));
|
|
1724
|
+
const c = await evaluate(T, `(function(){
|
|
1725
|
+
var c=window.__pawbrowse, e=${R}&&c&&c.get&&c.get(${R});
|
|
1726
|
+
if(e&&e.isConnected){ var s=c.surface?c.surface(e):e; if(s){ var r=s.getBoundingClientRect(), x=r.x+r.width/2, y=r.y+r.height/2, w=s.ownerDocument.defaultView;
|
|
1727
|
+
while(w&&w.frameElement){ var fr=w.frameElement.getBoundingClientRect(); x+=fr.left+w.frameElement.clientLeft; y+=fr.top+w.frameElement.clientTop; w=w.frameElement.ownerDocument.defaultView; }
|
|
1728
|
+
if(x>=0&&y>=0&&x<innerWidth&&y<innerHeight) return [Math.round(x),Math.round(y)]; } }
|
|
1729
|
+
return [Math.round(innerWidth/2),Math.round(innerHeight/2)];
|
|
1730
|
+
})()`);
|
|
1731
|
+
if (Array.isArray(c)) ({ x: cx, y: cy } = await top(c[0], c[1]));
|
|
1732
|
+
} catch {}
|
|
1733
|
+
// Scroll offsets of the page and every scroll container under the point, as one signature.
|
|
1734
|
+
const probe = rt.frame ? null : `(function(){ var s=[scrollX,scrollY], e=document.elementFromPoint(${cx},${cy}); for(var g=0;e&&g<60;g++){ if(e.scrollHeight>e.clientHeight||e.scrollWidth>e.clientWidth) s.push(e.scrollTop,e.scrollLeft); e=e.parentElement||(e.getRootNode&&e.getRootNode().host); } return s.join(','); })()`;
|
|
1735
|
+
const before = probe ? await evaluate(tabId, probe).catch(() => null) : null;
|
|
696
1736
|
await sendCdp(tabId, 'Input.dispatchMouseEvent', { type: 'mouseWheel', x: cx, y: cy, deltaX: 0, deltaY: dy });
|
|
1737
|
+
// Wheel scrolling is often ANIMATED (smooth scrolling, e.g. on Linux): wait until the offsets
|
|
1738
|
+
// stop moving, so the table we return shows where the page actually ended up.
|
|
1739
|
+
if (probe) {
|
|
1740
|
+
let last = before, stable = 0;
|
|
1741
|
+
for (const end = Date.now() + 1500; Date.now() < end;) {
|
|
1742
|
+
await sleep(30);
|
|
1743
|
+
const now = await evaluate(tabId, probe).catch(() => null);
|
|
1744
|
+
if (now !== last) { last = now; stable = 0; continue; }
|
|
1745
|
+
if (++stable >= 3 && (now !== before || Date.now() > end - 1200)) break; // settled (or nothing scrolls here)
|
|
1746
|
+
}
|
|
1747
|
+
}
|
|
697
1748
|
return `scroll ${dy}`;
|
|
698
1749
|
}
|
|
699
1750
|
case 'wait': {
|
|
@@ -705,6 +1756,50 @@ async function runOp(tabId, op) {
|
|
|
705
1756
|
}
|
|
706
1757
|
}
|
|
707
1758
|
|
|
1759
|
+
/* -------------------------------- Screenshots ------------------------------ *
|
|
1760
|
+
* For what the element table can't express — canvas apps (Docs/Sheets/Figma/maps), charts, visual
|
|
1761
|
+
* state. The image is in CSS pixels (so it maps 1:1 to click_xy on any display density), capped at
|
|
1762
|
+
* 1568px on the long side, and, where OffscreenCanvas exists, labelled with the table's refs. */
|
|
1763
|
+
const shotScale = new Map(); // tabId -> CSS px per image px of the last screenshot
|
|
1764
|
+
|
|
1765
|
+
async function screenshot(tabId, opts) {
|
|
1766
|
+
const [dpr, vw, vh] = await evaluate(tabId, '[devicePixelRatio, innerWidth, innerHeight]');
|
|
1767
|
+
const { cssVisualViewport: vv } = await sendCdp(tabId, 'Page.getLayoutMetrics');
|
|
1768
|
+
const fit = Math.min(1, 1568 / Math.max(vw, vh));
|
|
1769
|
+
const shot = await sendCdp(tabId, 'Page.captureScreenshot', {
|
|
1770
|
+
format: 'jpeg', quality: 70,
|
|
1771
|
+
clip: { x: vv.pageX, y: vv.pageY, width: vw, height: vh, scale: fit / dpr },
|
|
1772
|
+
});
|
|
1773
|
+
shotScale.set(tabId, 1 / fit);
|
|
1774
|
+
let data = shot.data, marked = 0;
|
|
1775
|
+
if (opts && opts.marks !== false && typeof OffscreenCanvas !== 'undefined' && typeof createImageBitmap !== 'undefined') {
|
|
1776
|
+
try {
|
|
1777
|
+
const snap = await snapshot(tabId, 1);
|
|
1778
|
+
const frames = snap ? await readFrames(tabId, tabId, snap, []) : [];
|
|
1779
|
+
// Label at each control's top-left corner (not its centre, which would hide its text).
|
|
1780
|
+
const rows = (snap ? snap.actions.filter((a) => !a.off) : []).map((a) => ({ id: a.id, x: a.x - (a.w || 0) / 2, y: a.y - (a.h || 0) / 2 }));
|
|
1781
|
+
for (const { f, off, snap: fs } of frames) for (const a of fs.actions) if (!a.off) rows.push({ id: `f${f.idx}.${a.id}`, x: a.x - (a.w || 0) / 2 + off.x, y: a.y - (a.h || 0) / 2 + off.y });
|
|
1782
|
+
const bytes = Uint8Array.from(atob(data), (c) => c.charCodeAt(0));
|
|
1783
|
+
const bmp = await createImageBitmap(new Blob([bytes], { type: 'image/jpeg' }));
|
|
1784
|
+
const cv = new OffscreenCanvas(bmp.width, bmp.height), g = cv.getContext('2d');
|
|
1785
|
+
g.drawImage(bmp, 0, 0);
|
|
1786
|
+
g.font = 'bold 11px sans-serif'; g.textBaseline = 'top';
|
|
1787
|
+
for (const r of rows) {
|
|
1788
|
+
const w = g.measureText(r.id).width + 4, x = Math.max(0, r.x * fit - 2), y = Math.max(0, r.y * fit - 9);
|
|
1789
|
+
g.fillStyle = 'rgba(220,38,38,.85)'; g.fillRect(x, y, w, 13);
|
|
1790
|
+
g.fillStyle = '#fff'; g.fillText(r.id, x + 2, y + 1);
|
|
1791
|
+
marked++;
|
|
1792
|
+
}
|
|
1793
|
+
const out = await cv.convertToBlob({ type: 'image/jpeg', quality: 0.7 });
|
|
1794
|
+
const buf = new Uint8Array(await out.arrayBuffer());
|
|
1795
|
+
let bin = ''; for (let i = 0; i < buf.length; i += 0x8000) bin += String.fromCharCode.apply(null, buf.subarray(i, i + 0x8000));
|
|
1796
|
+
data = btoa(bin);
|
|
1797
|
+
} catch { /* unlabelled screenshot is still useful */ }
|
|
1798
|
+
}
|
|
1799
|
+
const w = Math.round(vw * fit), h = Math.round(vh * fit);
|
|
1800
|
+
return { data, mimeType: 'image/jpeg', width: w, height: h, note: `screenshot ${w}x${h} of the visible viewport${marked ? `, ${marked} controls labelled with their refs` : ''}. For things not in the element table (canvas apps, maps, charts), act with {op:"click_xy",x,y} using THIS image's pixel coordinates.` };
|
|
1801
|
+
}
|
|
1802
|
+
|
|
708
1803
|
/* ------------------------------ Command router ---------------------------- */
|
|
709
1804
|
|
|
710
1805
|
async function handleCommand(cmd, args, token, session) {
|
|
@@ -738,58 +1833,98 @@ async function handleCommand(cmd, args, token, session) {
|
|
|
738
1833
|
if (!/^[a-z][a-z0-9+.-]*:/i.test(url)) url = 'https://' + url; // bare domain -> https
|
|
739
1834
|
if (!/^https?:\/\//i.test(url)) throw new Error(`navigate only supports http(s) URLs (refusing "${url.split(':')[0]}:")`);
|
|
740
1835
|
await attach(tabId);
|
|
741
|
-
|
|
742
|
-
|
|
743
|
-
|
|
744
|
-
|
|
745
|
-
|
|
746
|
-
|
|
747
|
-
|
|
1836
|
+
// beforeunload ("Leave site? Changes may not be saved") is dismissed unless dialog:"accept".
|
|
1837
|
+
return whileActing(tabId, async () => {
|
|
1838
|
+
const t0 = Date.now();
|
|
1839
|
+
const w = tabWatch(tabId);
|
|
1840
|
+
w.navStart = t0; w.navDone = 0; w.committed = false; w.navReq = null; // the old document must not count as "arrived"
|
|
1841
|
+
const nav = await sendCdp(tabId, 'Page.navigate', { url });
|
|
1842
|
+
if (nav && nav.loaderId && !w.committed) w.navReq = nav.loaderId;
|
|
1843
|
+
worlds.delete(tabId); // the old document's world is going away with it
|
|
1844
|
+
if (nav && nav.errorText) {
|
|
1845
|
+
// Cancelled by a "leave site?" prompt we dismissed: not a failure — say so, show where we are.
|
|
1846
|
+
const dl0 = takeDialogLog(tabId);
|
|
1847
|
+
if (dl0 && /beforeunload/.test(dl0)) return `${dl0}(navigation cancelled: the page asked to keep unsaved changes)\n\n${await observe(tabId)}`;
|
|
1848
|
+
throw new Error(`navigation failed: ${nav.errorText}`);
|
|
748
1849
|
}
|
|
749
|
-
|
|
750
|
-
|
|
1850
|
+
if (nav && !nav.loaderId) w.navDone = Date.now(); // same-document (fragment) navigation
|
|
1851
|
+
await settle(tabId, 5000, t0);
|
|
1852
|
+
const dl = takeDialogLog(tabId);
|
|
1853
|
+
return (dl ? dl + '\n' : '') + await observe(tabId);
|
|
1854
|
+
}, { nav: true, accept: args.dialog === 'accept' ? true : undefined });
|
|
751
1855
|
}
|
|
752
1856
|
case 'observe': {
|
|
753
1857
|
const tabId = await resolveTabId(session, args, 'inspect');
|
|
754
1858
|
await attach(tabId);
|
|
755
|
-
|
|
1859
|
+
assertNoOpenDialog(tabId);
|
|
1860
|
+
return observe(tabId, args);
|
|
756
1861
|
}
|
|
757
1862
|
case 'read': {
|
|
758
1863
|
const tabId = await resolveTabId(session, args, 'inspect');
|
|
759
1864
|
await attach(tabId);
|
|
1865
|
+
assertNoOpenDialog(tabId);
|
|
760
1866
|
const max = Math.min(Number(args.max_chars) || 12000, 50000);
|
|
761
|
-
const
|
|
762
|
-
|
|
1867
|
+
const r = await evaluate(tabId, `${READ_TEXT}(${max})`);
|
|
1868
|
+
let text = r.text;
|
|
1869
|
+
// Cross-origin frames the page-side reader can't enter (embedded docs, widgets, checkouts).
|
|
1870
|
+
try {
|
|
1871
|
+
for (const { f } of await readFrames(tabId, tabId, await snapshot(tabId, 1), [])) {
|
|
1872
|
+
if (text.length >= max) break;
|
|
1873
|
+
try {
|
|
1874
|
+
const fr = await evaluate(f.target, `${READ_TEXT}(${max})`);
|
|
1875
|
+
if (fr && fr.text) text += `\n\n[frame f${f.idx}: ${fr.url}]\n${fr.text}`;
|
|
1876
|
+
} catch {}
|
|
1877
|
+
}
|
|
1878
|
+
} catch {}
|
|
1879
|
+
return `${r.title} — ${r.url}\n\n${text.slice(0, max)}`;
|
|
763
1880
|
}
|
|
764
1881
|
case 'act': {
|
|
765
1882
|
const ops = args.ops || [];
|
|
766
1883
|
if (ops.length > 50) throw new Error('too many ops in one call (max 50); split into smaller batches');
|
|
767
1884
|
const tabId = await resolveTabId(session, args, 'inspect');
|
|
768
1885
|
await attach(tabId);
|
|
769
|
-
|
|
770
|
-
|
|
771
|
-
|
|
772
|
-
|
|
773
|
-
|
|
774
|
-
|
|
775
|
-
await
|
|
776
|
-
|
|
777
|
-
|
|
778
|
-
|
|
779
|
-
|
|
780
|
-
|
|
781
|
-
|
|
782
|
-
|
|
783
|
-
|
|
784
|
-
|
|
785
|
-
|
|
786
|
-
|
|
787
|
-
|
|
788
|
-
|
|
1886
|
+
return whileActing(tabId, async () => {
|
|
1887
|
+
const logLines = [];
|
|
1888
|
+
// Answer a dialog left open from before FIRST: until then the page can't run anything.
|
|
1889
|
+
let i = 0;
|
|
1890
|
+
for (; i < ops.length && ops[i].op === 'dialog'; i++) logLines.push(' ' + await runOp(tabId, ops[i]));
|
|
1891
|
+
assertNoOpenDialog(tabId);
|
|
1892
|
+
const before = await evaluate(tabId, SIG).catch(() => null);
|
|
1893
|
+
await netOn(tabId);
|
|
1894
|
+
try {
|
|
1895
|
+
for (; i < ops.length; i++) {
|
|
1896
|
+
const op = ops[i];
|
|
1897
|
+
if (aborted()) { logLines.push(' (aborted: command timed out; remaining ops not run)'); break; }
|
|
1898
|
+
const t0 = Date.now();
|
|
1899
|
+
try { logLines.push(' ' + await runOp(tabId, op)); }
|
|
1900
|
+
catch (e) { logLines.push(` ${op.op} ${op.ref || ''}: ERROR ${e.message}`); }
|
|
1901
|
+
if (op.op !== 'wait') await settle(tabId, i === ops.length - 1 ? 4000 : 2500, t0, { grace: op.op === 'type' ? 400 : 0 });
|
|
1902
|
+
}
|
|
1903
|
+
} finally { await netOff(tabId); }
|
|
1904
|
+
const after = await evaluate(tabId, SIG).catch(() => null);
|
|
1905
|
+
const seen = lastTable.get(tabId), seenFull = lastFull.get(tabId);
|
|
1906
|
+
// The ops already executed; a failed post-action read (page navigating) must NOT make
|
|
1907
|
+
// the caller think they failed and retry them.
|
|
1908
|
+
let table;
|
|
1909
|
+
try { table = await observe(tabId); } catch { table = null; }
|
|
1910
|
+
const changed = before == null || after == null || before !== after || table == null || seen == null || tableBody(table) !== seen || dialogLog.has(tabId);
|
|
1911
|
+
const note = changed ? 'page changed' : 'page did NOT change (if you expected an effect, the action may not have worked — try a different target)';
|
|
1912
|
+
if (table == null) {
|
|
1913
|
+
return `ran ${ops.length} op(s) [${note}]:\n${logLines.join('\n')}\n${takeDialogLog(tabId)}\n(ops executed; the page is navigating and could not be read yet — call browser_observe next. Do NOT re-run these ops.)`;
|
|
1914
|
+
}
|
|
1915
|
+
return `ran ${ops.length} op(s) [${note}]:\n${logLines.join('\n')}\n${takeDialogLog(tabId)}\n${(changed && deltaTable(seenFull, table)) || table}`;
|
|
1916
|
+
});
|
|
1917
|
+
}
|
|
1918
|
+
case 'screenshot': {
|
|
1919
|
+
const tabId = await resolveTabId(session, args, 'inspect');
|
|
1920
|
+
await attach(tabId);
|
|
1921
|
+
assertNoOpenDialog(tabId);
|
|
1922
|
+
return screenshot(tabId, args);
|
|
789
1923
|
}
|
|
790
1924
|
case 'assert': {
|
|
791
1925
|
const tabId = await resolveTabId(session, args, 'inspect');
|
|
792
1926
|
await attach(tabId);
|
|
1927
|
+
assertNoOpenDialog(tabId);
|
|
793
1928
|
if (args.contains != null) {
|
|
794
1929
|
const ok = await evaluate(tabId, `!!(document.body && document.body.innerText && document.body.innerText.indexOf(${JSON.stringify(args.contains)})>=0)`);
|
|
795
1930
|
return { pass: !!ok, kind: 'contains', value: args.contains };
|
|
@@ -799,7 +1934,8 @@ async function handleCommand(cmd, args, token, session) {
|
|
|
799
1934
|
return { pass: String(u).indexOf(args.url_includes) >= 0, kind: 'url_includes', url: u };
|
|
800
1935
|
}
|
|
801
1936
|
if (args.ref_visible != null) {
|
|
802
|
-
const
|
|
1937
|
+
const rt = routeRef(tabId, args.ref_visible);
|
|
1938
|
+
const r = rt.target ? await resolveHit(rt.target, rt.ref, { noScroll: true }) : { error: 'unknown frame (observe again)' };
|
|
803
1939
|
return { pass: !r.error, kind: 'ref_visible', ref: args.ref_visible, note: r.error };
|
|
804
1940
|
}
|
|
805
1941
|
return { pass: false, error: 'provide one of: contains, url_includes, ref_visible' };
|