pawbrowse 0.5.2 → 0.6.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -21,7 +21,8 @@ let reconnectTimer = null;
21
21
  const attachedTabs = new Set(); // tabIds we currently hold a debugger on
22
22
  const sessions = new Map(); // sessionId -> { activeTabId, createdTabs:Set, groupId, num, color }
23
23
  const tabOwner = new Map(); // tabId -> sessionId (so sessions don't steal each other's tabs)
24
- const chains = new Map(); // sessionId -> Promise (serialize commands within a session)
24
+ const chains = new Map();
25
+ const refSeed = new Map(); // tabId -> next unused ref number (see SNAPSHOT: refs never repeat within a tab) // sessionId -> Promise (serialize commands within a session)
25
26
  let sessionCounter = 0;
26
27
  let colorCursor = 0;
27
28
  // PawBrowse's own group identity — deliberately NOT Claude-in-Chrome's blue "Claude" group.
@@ -44,7 +45,7 @@ function persistState() {
44
45
  try {
45
46
  const ser = {};
46
47
  for (const [k, v] of sessions) ser[k] = { activeTabId: v.activeTabId, createdTabs: [...v.createdTabs], groupId: v.groupId, num: v.num, color: v.color };
47
- await chrome.storage.session.set({ pawbrowse_state: { sessions: ser, tabOwner: [...tabOwner], sessionCounter, colorCursor } });
48
+ await chrome.storage.session.set({ pawbrowse_state: { sessions: ser, tabOwner: [...tabOwner], sessionCounter, colorCursor, refSeed: [...refSeed] } });
48
49
  } catch {}
49
50
  }, 250);
50
51
  persistTimer.unref?.();
@@ -59,6 +60,7 @@ const rehydrated = (async () => {
59
60
  for (const [t, sess] of (st.tabOwner || [])) if (!tabOwner.has(t)) tabOwner.set(t, sess);
60
61
  if (typeof st.sessionCounter === 'number') sessionCounter = Math.max(sessionCounter, st.sessionCounter);
61
62
  if (typeof st.colorCursor === 'number') colorCursor = Math.max(colorCursor, st.colorCursor);
63
+ for (const [t, n] of (st.refSeed || [])) refSeed.set(t, Math.max(refSeed.get(t) || 1, n));
62
64
  } catch {}
63
65
  })();
64
66
 
@@ -117,12 +119,17 @@ async function connect() {
117
119
  // navigate) stops issuing further CDP ops instead of leaking work into the next one.
118
120
  const token = { cancelled: false };
119
121
  const prev = chains.get(session) || Promise.resolve();
122
+ let settled = null;
120
123
  const run = prev.then(async () => {
121
124
  let timer;
122
125
  try {
126
+ const work = handleCommand(msg.cmd, msg.args || {}, token, session);
127
+ // The next command in this session must not start while this one is still touching the tab,
128
+ // even after we've replied with a timeout (bounded, so a wedged page can't block forever).
129
+ settled = Promise.race([work.catch(() => {}), sleep(15000)]);
123
130
  const result = await Promise.race([
124
- handleCommand(msg.cmd, msg.args || {}, token, session),
125
- new Promise((_, rej) => { timer = setTimeout(() => { token.cancelled = true; rej(new Error('command timed out in extension after 25s')); }, 25000); }),
131
+ work,
132
+ new Promise((_, rej) => { timer = setTimeout(() => { token.cancelled = true; rej(new Error(`command timed out in extension after 25s${msg.cmd === 'act' || msg.cmd === 'navigate' ? ' — it may already have acted: observe before retrying' : ''}`)); }, 25000); }),
126
133
  ]);
127
134
  clearTimeout(timer);
128
135
  sock.send(JSON.stringify({ id: msg.id, ok: true, result }));
@@ -131,7 +138,7 @@ async function connect() {
131
138
  try { sock.send(JSON.stringify({ id: msg.id, ok: false, error: String(e && e.message || e) })); } catch {}
132
139
  }
133
140
  });
134
- chains.set(session, run.catch(() => {}));
141
+ chains.set(session, run.catch(() => {}).then(() => settled));
135
142
  };
136
143
  ws.onclose = () => { setBadge('off'); scheduleReconnect(); };
137
144
  ws.onerror = () => { try { ws.close(); } catch {} };
@@ -151,21 +158,61 @@ function setBadge(state) {
151
158
 
152
159
  /* ------------------------------- CDP helpers ------------------------------ */
153
160
 
154
- function sendCdp(tabId, method, params = {}) {
161
+ // A CDP target is a tabId (the tab's top frame) or { tabId, sessionId?, frameId? }: sessionId
162
+ // addresses an out-of-process (cross-site) iframe attached via Target.setAutoAttach (flat sessions,
163
+ // Chrome 125+); frameId pins evaluation to a cross-origin frame living in that session's process.
164
+ const tabOf = (t) => (typeof t === 'object' ? t.tabId : t);
165
+ function sendCdp(target, method, params = {}) {
166
+ const dbg = typeof target === 'object' ? (target.sessionId ? { tabId: target.tabId, sessionId: target.sessionId } : { tabId: target.tabId }) : { tabId: target };
155
167
  return new Promise((resolve, reject) => {
156
- chrome.debugger.sendCommand({ tabId }, method, params, (res) => {
168
+ chrome.debugger.sendCommand(dbg, method, params, (res) => {
157
169
  const err = chrome.runtime.lastError;
158
170
  if (err) reject(new Error(err.message)); else resolve(res);
159
171
  });
160
172
  });
161
173
  }
162
174
 
163
- async function evaluate(tabId, expression) {
164
- const r = await sendCdp(tabId, 'Runtime.evaluate', { expression, returnByValue: true, awaitPromise: true });
175
+ // All of PawBrowse's page-side code runs in its own ISOLATED WORLD (like an extension content
176
+ // script / Playwright's utility world): it shares the DOM with the page but not its JS globals, so a
177
+ // page that monkey-patches Array.prototype / JSON / Element.prototype, or that squats on our cache
178
+ // name, can't blind or steer the snapshot. One world per tab's main frame; a navigation destroys it
179
+ // and the next call transparently recreates it.
180
+ const worlds = new Map(); // tabId -> Map(frameKey -> executionContextId)
181
+ const frameKey = (t) => (typeof t === 'object' ? `${t.sessionId || ''}|${t.frameId || ''}` : '|');
182
+
183
+ async function worldFor(target) {
184
+ const tabId = tabOf(target), key = frameKey(target);
185
+ let m = worlds.get(tabId);
186
+ if (!m) { m = new Map(); worlds.set(tabId, m); }
187
+ const cached = m.get(key);
188
+ if (cached != null) return cached;
189
+ let frameId = typeof target === 'object' ? target.frameId : null;
190
+ if (!frameId) ({ frameTree: { frame: { id: frameId } } } = await sendCdp(target, 'Page.getFrameTree'));
191
+ const { executionContextId } = await sendCdp(target, 'Page.createIsolatedWorld', { frameId, worldName: 'pawbrowse' });
192
+ m.set(key, executionContextId);
193
+ return executionContextId;
194
+ }
195
+
196
+ async function evaluate(target, expression, opts) {
197
+ let r;
198
+ for (let attempt = 0; ; attempt++) {
199
+ const contextId = await worldFor(target);
200
+ try {
201
+ r = await sendCdp(target, 'Runtime.evaluate', { expression, contextId, returnByValue: !(opts && opts.handle), awaitPromise: true });
202
+ break;
203
+ } catch (e) {
204
+ // The world died with its previous document (navigation/reload) BEFORE this call: make a fresh
205
+ // one and retry once. NOT when the context was destroyed DURING the call ("Execution context
206
+ // was destroyed") — the expression may have already acted (e.g. a select that navigated) and
207
+ // re-running it on the new page would act twice.
208
+ worlds.get(tabOf(target))?.delete(frameKey(target));
209
+ if (attempt >= 1 || !/Cannot find context/i.test(e.message)) throw e;
210
+ }
211
+ }
165
212
  if (r && r.exceptionDetails) {
166
213
  throw new Error(r.exceptionDetails.exception?.description || r.exceptionDetails.text || 'evaluation error');
167
214
  }
168
- return r.result.value;
215
+ return opts && opts.handle ? r.result : r.result.value;
169
216
  }
170
217
 
171
218
  // Attach to a SPECIFIC tab and keep it attached (many tabs can be attached at once, one per
@@ -193,6 +240,15 @@ async function attach(tabId) {
193
240
  await sendCdp(tabId, 'Runtime.enable', {}).catch(() => {});
194
241
  await sendCdp(tabId, 'Page.enable', {}).catch(() => {});
195
242
  await sendCdp(tabId, 'DOM.enable', {}).catch(() => {});
243
+ // Lifecycle events (cheap: a handful per load) tell us when a new document is ready. The Network
244
+ // domain is NOT left on: on ad-heavy pages (hundreds of requests) it slows the whole browser, so
245
+ // it's switched on only around actions (netOn/netOff), where "did this click start a fetch?" matters.
246
+ await sendCdp(tabId, 'Page.setLifecycleEventsEnabled', { enabled: true }).catch(() => {});
247
+ try { const { frameTree } = await sendCdp(tabId, 'Page.getFrameTree'); tabWatch(tabId).mainFrame = frameTree.frame.id; } catch {}
248
+ // WebMCP (sites exposing their own agent tools; Chrome 146+ behind a flag / origin trial).
249
+ await sendCdp(tabId, 'WebMCP.enable', {}).catch(() => {});
250
+ // Cross-site iframes run in other renderer processes: attach to each as a flat child session.
251
+ await sendCdp(tabId, 'Target.setAutoAttach', AUTO_ATTACH).catch(() => {});
196
252
  // Make the tab behave as focused even when it's a background tab, so focus/blur, rendering,
197
253
  // and focus-dependent menus/dropdowns work while driving (the same approach Playwright uses for
198
254
  // backgrounded pages). A hidden tab still throttles requestAnimationFrame, so our waits use
@@ -201,15 +257,71 @@ async function attach(tabId) {
201
257
  }
202
258
 
203
259
  function detach(tabId) {
204
- return new Promise((resolve) => chrome.debugger.detach({ tabId }, () => { void chrome.runtime.lastError; attachedTabs.delete(tabId); resolve(); }));
260
+ return new Promise((resolve) => chrome.debugger.detach({ tabId }, () => { void chrome.runtime.lastError; attachedTabs.delete(tabId); forgetTab(tabId, false); resolve(); }));
261
+ }
262
+
263
+ // Everything we remember about a tab's CDP state. On detach (user cancelled the debugging bar,
264
+ // renderer crash) child sessions are gone without detachedFromTarget events: drop them so a
265
+ // re-attach rebuilds cleanly. On tab close, also drop what outlives an attachment.
266
+ function forgetTab(tabId, closed) {
267
+ worlds.delete(tabId); childSessions.delete(tabId); webTools.delete(tabId); shotScale.delete(tabId);
268
+ if (!closed) return;
269
+ watches.delete(tabId); frameIdx.delete(tabId); lastTable.delete(tabId); lastFull.delete(tabId);
270
+ openDialogs.delete(tabId); acting.delete(tabId); dialogLog.delete(tabId); refSeed.delete(tabId);
271
+ for (const k of [...refSeed.keys()]) if (String(k).startsWith(`${tabId}#`)) refSeed.delete(k);
272
+ }
273
+ chrome.debugger.onDetach.addListener((source) => { if (source.tabId != null) { attachedTabs.delete(source.tabId); forgetTab(source.tabId, false); } });
274
+
275
+ // JavaScript dialogs (alert/confirm/prompt/beforeunload) block the page's main thread, so every CDP
276
+ // call into the tab would hang until someone clicks the dialog. While PawBrowse is ACTING on a tab
277
+ // it answers them itself: alerts are accepted, confirm/prompt/beforeunload are DISMISSED unless the
278
+ // op opted in with dialog:"accept" (a destructive confirm is never auto-approved). What was shown is
279
+ // reported back. Dialogs that appear while we're idle belong to the user and are left alone; if one
280
+ // is still open when a command arrives we say so instead of hanging.
281
+ const acting = new Map(); // tabId -> { accept?: boolean, text?: string } while an act/navigate runs
282
+ const dialogLog = new Map(); // tabId -> [lines] reported in the next result
283
+ const openDialogs = new Map(); // tabId -> { type, message } left open (appeared while idle)
284
+
285
+ chrome.debugger.onEvent.addListener((source, method, params) => {
286
+ const tabId = source.tabId;
287
+ if (tabId == null) return;
288
+ if (method === 'Page.javascriptDialogClosed') { openDialogs.delete(tabId); return; }
289
+ if (method !== 'Page.javascriptDialogOpening') return;
290
+ const pol = acting.get(tabId);
291
+ const where = source.sessionId ? { tabId, sessionId: source.sessionId } : tabId; // dialogs can come from frames
292
+ if (!pol) { openDialogs.set(tabId, { type: params.type, message: params.message, where }); return; }
293
+ const accept = pol.accept != null ? pol.accept : params.type === 'alert';
294
+ const promptText = pol.text != null ? String(pol.text) : (params.defaultPrompt || '');
295
+ sendCdp(where, 'Page.handleJavaScriptDialog', { accept, promptText }).catch(() => {});
296
+ const lines = dialogLog.get(tabId) || [];
297
+ lines.push(`${params.type} "${String(params.message || '').replace(/\s+/g, ' ').slice(0, 200)}" → ${accept ? 'accepted' : 'dismissed'}${params.type === 'prompt' && accept ? ` with "${promptText}"` : ''}${!accept && params.type !== 'alert' ? ` (to accept, repeat ${pol.nav ? 'browser_navigate' : 'the op'} with dialog:"accept")` : ''}`);
298
+ dialogLog.set(tabId, lines);
299
+ });
300
+
301
+ // Run fn with this tab's dialogs auto-answered (see above).
302
+ async function whileActing(tabId, fn, policy) {
303
+ const mine = { ...(policy || {}) };
304
+ acting.set(tabId, mine);
305
+ try { return await fn(); } finally { if (acting.get(tabId) === mine) acting.delete(tabId); }
306
+ }
307
+
308
+ function takeDialogLog(tabId) {
309
+ const l = dialogLog.get(tabId); dialogLog.delete(tabId);
310
+ return l && l.length ? l.map((x) => ` dialog: ${x}`).join('\n') + '\n' : '';
205
311
  }
206
312
 
207
- chrome.debugger.onDetach.addListener((source) => { if (source.tabId != null) attachedTabs.delete(source.tabId); });
313
+ // Refuse to talk to a tab frozen by a dialog the user hasn't answered (it would hang). The
314
+ // act op {op:"dialog"} answers it.
315
+ function assertNoOpenDialog(tabId) {
316
+ const d = openDialogs.get(tabId);
317
+ if (d) throw new Error(`the page is showing ${d.type === 'alert' ? 'an' : 'a'} ${d.type} dialog "${String(d.message || '').slice(0, 120)}" and is frozen until it's answered: run browser_act with [{op:"dialog",accept:true|false}] (or answer it in the browser)`);
318
+ }
208
319
 
209
320
  // If a driven tab is closed (by the user or by us), forget it everywhere so a session doesn't
210
321
  // keep pointing at a dead tab.
211
322
  chrome.tabs.onRemoved.addListener((tabId) => {
212
323
  attachedTabs.delete(tabId);
324
+ forgetTab(tabId, true);
213
325
  const owner = tabOwner.get(tabId);
214
326
  tabOwner.delete(tabId);
215
327
  if (owner != null) {
@@ -303,35 +415,101 @@ async function endSession(session) {
303
415
  * action re-resolves the exact element it was chosen from.
304
416
  * -------------------------------------------------------------------------- */
305
417
 
306
- const SNAPSHOT = `(function(){
418
+ // DOM activity tracker, kept apart from the ref cache (window.__pawbrowse) so creating it can't
419
+ // reset ref numbering. Installed by the first snapshot or wait in each document; shadow roots are
420
+ // added as the snapshot discovers them. Keeps recent mutation times so a wait can tell "this page
421
+ // is always animating" (don't wait for quiet that never comes) from "the action changed things".
422
+ const MO_INSTALL = `var M=window.__pawmo; if(!M){ M=window.__pawmo={last:performance.now(),times:[],roots:new WeakSet()};
423
+ M.mo=new MutationObserver(function(){ var t=performance.now(); M.last=t; M.times.push(t); if(M.times.length>64) M.times.shift(); });
424
+ M.watch=function(r){ if(!M.roots.has(r)){ M.roots.add(r); try{ M.mo.observe(r,{subtree:true,childList:true,attributes:true,characterData:true}); }catch(_){} } };
425
+ M.watch(document); }`;
426
+ const SNAPSHOT = `(function(seed, opts){
427
+ opts=opts||{};
307
428
  try{
308
429
  if(!document.body) return null;
309
- var cache = window.__pawbrowse || (window.__pawbrowse = {ids:new WeakMap(), nodes:new Map(), next:1, byId:{}});
310
- function identity(e){ if(!cache.ids.has(e)) cache.ids.set(e, cache.next++); var id=cache.ids.get(e); cache.nodes.set(id,e); return id; }
430
+ // A NEW document continues numbering from where the tab's previous document stopped (seed), so a
431
+ // ref held over from the last page can never silently name a different element on this one.
432
+ var cache = window.__pawbrowse || (window.__pawbrowse = {ids:new WeakMap(), nodes:new Map(), next:Math.max(1,seed|0), byId:{}});
433
+ var fresh=new Set(); // node ids first allocated during this snapshot
434
+ function identity(e){
435
+ var id=cache.ids.get(e), holder=id!=null && cache.nodes.get(id);
436
+ // A new id if unseen — or if its old id now belongs to a different live element (a node that
437
+ // was detached, had its ref handed to a replacement, then came back).
438
+ if(id==null || (holder && holder!==e && holder.isConnected)){ id=cache.next++; cache.ids.set(e, id); fresh.add(id); }
439
+ cache.nodes.set(id,e); return id;
440
+ }
311
441
  cache.nodes.forEach(function(e,id){ if(!e.isConnected) cache.nodes.delete(id); });
312
- function safe(e){ return ['password','file','hidden'].indexOf(e.type)<0; }
313
- function visible(e){ return !e.closest('[aria-hidden="true"],[inert]') && e.checkVisibility({checkOpacity:true,checkVisibilityCSS:true}); }
442
+ ${MO_INSTALL}
443
+ // CLOSED shadow roots are invisible to page JS; the extension hands them to us via CDP
444
+ // (probeClosedRoots) keyed by host. sroot() = a host's shadow root, open or closed.
445
+ if(!cache.closed){ cache.closed=new WeakMap(); cache.probed=new WeakSet(); }
446
+ function sroot(n){ return n.shadowRoot || cache.closed.get(n) || null; }
447
+ cache.sroot=sroot;
448
+ var pendingHosts=[];
449
+ function safe(e){ return ['password','hidden'].indexOf(e.type)<0; }
450
+ // display:contents boxes (every <slot>, many design-system wrappers) have no box of their own, so
451
+ // checkVisibility() says false even though their children render: judge those by their parent.
452
+ function shown(e){ for(var g=0; e && g<32; g++){ if(e.checkVisibility({checkOpacity:true,checkVisibilityCSS:true})) return true; var v=e.ownerDocument.defaultView; if(!v || v.getComputedStyle(e).display!=='contents') return false; e=e.parentElement || (e.parentNode && e.parentNode.host); } return false; }
453
+ function visible(e){ return !!e && !e.closest('[aria-hidden="true"],[inert]') && shown(e); }
454
+ function sized(e){ var r=e.getBoundingClientRect(); return r.width>0 && r.height>0; }
455
+ function clean(s,n){ return String(s||'').replace(/\\s+/g,' ').trim().slice(0,n||120); }
314
456
  function name(e,seen){
315
457
  seen=seen||new Set();
316
- if(!e||seen.has(e)) return '';
458
+ if(!e||seen.has(e)||e.nodeType!==1) return '';
317
459
  seen.add(e);
318
- var rt=(e.getRootNode&&e.getRootNode())||document; var gid=function(id){try{return rt.getElementById?rt.getElementById(id):document.getElementById(id);}catch(_){return null;}};
319
- var ref=(e.getAttribute('aria-labelledby')||'').split(/\\s+/).map(function(id){return name(gid(id),seen);}).filter(Boolean).join(' ');
460
+ var tag=e.tagName;
461
+ if(tag==='SCRIPT'||tag==='STYLE'||tag==='NOSCRIPT'||tag==='TEMPLATE') return '';
462
+ if(tag.toLowerCase()==='svg'){ var st=e.querySelector('title'); return e.getAttribute('aria-label')||(st?st.textContent:''); }
463
+ var rt=(e.getRootNode&&e.getRootNode())||document; var gid=function(id){try{return (rt.getElementById&&rt.getElementById(id))||e.ownerDocument.getElementById(id);}catch(_){return null;}};
464
+ // (A control that lists ITSELF in aria-labelledby contributes its own content, per accname.)
465
+ var ref=(e.getAttribute('aria-labelledby')||'').split(/\\s+/).filter(Boolean).map(function(id){ var t=gid(id); return t===e ? clean(e.textContent,80) : name(t,seen); }).filter(Boolean).join(' ');
320
466
  if(ref) return ref;
321
467
  if(e.getAttribute('aria-label')) return e.getAttribute('aria-label');
322
468
  var labs=[].slice.call(e.labels||[]).map(function(l){return name(l,seen);}).filter(Boolean).join(' ');
323
469
  if(labs) return labs;
324
470
  if(['button','submit','reset'].indexOf(e.type)>=0 && e.value) return e.value;
325
471
  if(e.getAttribute('alt')) return e.getAttribute('alt');
326
- var txt = e.tagName==='INPUT' ? '' : [].map.call(e.childNodes,function(n){ return n.nodeType===3 ? n.textContent : (n.nodeType===1 && n.getAttribute('aria-hidden')!=='true' ? name(n,seen) : ''); }).join(' ').trim();
472
+ // Visible descendants only: display:none / hidden children (tooltips, menus) must not leak in.
473
+ // A rich-text editor's text is its VALUE, never its name (else typing into it renames it and
474
+ // its ref is refused on the next action).
475
+ var txt = (tag==='INPUT'||tag==='SELECT'||tag==='TEXTAREA'||e.isContentEditable) ? '' : [].map.call(e.childNodes,function(n){ return n.nodeType===3 ? n.textContent : (n.nodeType===1 && visible(n) ? name(n,seen) : ''); }).join(' ').trim();
327
476
  if(txt) return txt;
328
- return e.getAttribute('title')||e.getAttribute('placeholder')||'';
477
+ return e.getAttribute('title')||e.getAttribute('placeholder')||e.getAttribute('aria-placeholder')||'';
478
+ }
479
+ // A form field with no programmatic label: use a nearby <label> sibling (the common unassociated
480
+ // "<label>Name</label><div><input></div>" markup), then placeholder-ish hints, then its name attr.
481
+ function fieldLabel(e){
482
+ var n=name(e); if(n) return n;
483
+ if(!e.isContentEditable && !e.matches('input,textarea,select')) return '';
484
+ for(var s=e.parentElement,d=0; s && d<3; s=s.parentElement,d++){
485
+ if(['BODY','HTML','FORM'].indexOf(s.tagName)>=0) break;
486
+ for(var k=0;k<s.children.length;k++){ var c=s.children[k]; if(c.tagName==='LABEL' && !c.contains(e) && !c.control){ var t=name(c); if(t) return t; } }
487
+ }
488
+ return e.getAttribute('data-placeholder')||e.getAttribute('name')||(e.isContentEditable?'Rich text editor':'');
489
+ }
490
+ // Last resort for a nameless control (icon-font buttons): a test id, a meaningful id/name, or an
491
+ // icon class (fa-trash, bi-share, icon-close, material-icons text...) — shown as "icon:trash".
492
+ function hint(e){
493
+ var t=e.getAttribute('data-testid')||e.getAttribute('data-test')||e.getAttribute('data-qa')||e.getAttribute('data-cy')||e.getAttribute('data-action');
494
+ if(t) return 'testid:'+clean(t,40);
495
+ var els=[e].concat([].slice.call(e.querySelectorAll('i,span,svg,use,img')).slice(0,6));
496
+ for(var k=0;k<els.length;k++){
497
+ var cls=(els[k].getAttribute('class')||'')+' '+(els[k].getAttribute('href')||els[k].getAttribute('xlink:href')||'');
498
+ var re=/(?:^|[\\s#])(?:fa|bi|mdi|icon|glyphicon|lucide|ti|ri|octicon|ico)[-_]([a-z][a-z0-9-]{1,30})/ig, m;
499
+ while((m=re.exec(cls))){ if(!/^(solid|regular|light|thin|duotone|sharp|brands|lg|sm|xs|xl|fw|[0-9]+x|spin|pulse|icon|button|btn|wrapper|container|inner)$/i.test(m[1])) return 'icon:'+m[1].toLowerCase(); }
500
+ }
501
+ var id=e.id||e.getAttribute('name')||'';
502
+ if(id && /[a-z]{3}/i.test(id) && !/\\d{3}|[0-9a-f]{8}|^(ember|react|radix|mui|headlessui|:r)/i.test(id)) return 'id:'+clean(id,40);
503
+ return '';
329
504
  }
330
505
  var roles=['button','link','checkbox','radio','switch','tab','menuitem','menuitemradio','menuitemcheckbox','option','gridcell','combobox','textbox','searchbox','spinbutton','slider','treeitem'];
331
506
  // Semantic controls + custom clickables: any contenteditable, an inline onclick, or a
332
507
  // keyboard-focusable [tabindex] (framework buttons — React-Native-Web Pressables, design-system
333
508
  // divs — often expose only these). cursor:pointer clickables are added separately in collect().
334
- var selector='a[href],button,input,textarea,select,summary,[contenteditable]:not([contenteditable="false"]),[onclick],[tabindex]:not([tabindex="-1"]),'+roles.map(function(r){return '[role="'+r+'"]';}).join(',');
509
+ var selector='a[href],button,input,textarea,select,summary,[contenteditable]:not([contenteditable="false"]),[onclick],[tabindex]:not([tabindex="-1"]),[draggable="true"],'+roles.map(function(r){return '[role="'+r+'"]';}).join(',');
510
+ // Inputs whose value is SET (not typed): typing into these is unreliable, so type() routes them
511
+ // through a value setter. The hint tells the agent the expected format.
512
+ var SETTABLE={date:'YYYY-MM-DD',time:'HH:MM','datetime-local':'YYYY-MM-DDTHH:MM',month:'YYYY-MM',week:'YYYY-Www',color:'#rrggbb',range:''};
335
513
  function role(e){
336
514
  var explicit=e.getAttribute('role');
337
515
  if(roles.indexOf(explicit)>=0) return explicit;
@@ -341,27 +519,62 @@ const SNAPSHOT = `(function(){
341
519
  if(e.tagName==='TEXTAREA'||e.isContentEditable) return 'textbox';
342
520
  if(e.tagName==='INPUT'){
343
521
  if(['checkbox','radio'].indexOf(e.type)>=0) return e.type;
344
- if(['button','submit','reset','image'].indexOf(e.type)>=0) return 'button';
522
+ if(['button','submit','reset','image','file'].indexOf(e.type)>=0) return 'button';
345
523
  if(e.type==='search') return 'searchbox';
346
524
  if(e.type==='number') return 'spinbutton';
347
- if(['text','email','url','tel'].indexOf(e.type)>=0) return 'textbox';
525
+ if(e.type==='range') return 'slider';
526
+ if(['text','email','url','tel'].indexOf(e.type)>=0 || SETTABLE.hasOwnProperty(e.type)) return 'textbox';
348
527
  }
349
528
  return null;
350
529
  }
351
- // Semantic guard: a stable fingerprint of an element's MEANING (role/name/value/state).
352
- // Compared at action time so a silently-relabeled or changed target is rejected.
353
- // Identity-focused: role + accessible name. Catches a target silently becoming a different
354
- // control (relabel), while tolerating benign value/checked/expanded churn and same-element
355
- // multi-op batches.
356
- cache.guard=function(el){ if(!el) return ''; try{ return [role(el),(name(el)||'').replace(/\\s+/g,' ').trim()].join(String.fromCharCode(1)); }catch(_){ return ''; } };
357
- // Collect actionable elements across the top document, OPEN shadow roots, and SAME-ORIGIN
358
- // iframes. dx/dy translate each element's frame-local rect into top-level viewport coordinates
359
- // (shadow roots share the frame's coords so dx/dy carry through unchanged; iframes add offset).
530
+ // Hit-test a frame-local point in the element's own root, descending into nested open shadow
531
+ // roots, so a covered control (overlay, modal backdrop, pointer-events:none) is flagged.
532
+ cache.hits=function(t, lx, ly){
533
+ try{
534
+ if(lx==null){ var hr=t.getBoundingClientRect(); lx=hr.x+hr.width/2; ly=hr.y+hr.height/2; }
535
+ var root=t.getRootNode(); if(!root.elementFromPoint) root=t.ownerDocument;
536
+ var f=root.elementFromPoint(lx,ly), g=0;
537
+ while(f && sroot(f) && g++<16){ var inner=sroot(f).elementFromPoint(lx,ly); if(!inner||inner===f) break; f=inner; }
538
+ return !!f && (t===f || t.contains(f) || (t.control && t.control===f));
539
+ }catch(_){ return true; }
540
+ };
541
+ var hits=cache.hits;
542
+ // The element to click for a control. Styled checkboxes/radios/file pickers usually hide the
543
+ // native input (opacity:0, 0x0, display:none, sr-only) and show a <label> or a card instead:
544
+ // the visible label is the interaction surface; an opacity-0 input stretched over a visible
545
+ // card is clicked directly. Returns null when the control has no usable surface.
546
+ cache.surface=function(e){
547
+ if(!e||!e.isConnected) return null;
548
+ var hidable=e.tagName==='INPUT' && ['checkbox','radio','file'].indexOf(e.type)>=0;
549
+ if(visible(e) && sized(e)){
550
+ // A sr-only (1px, clipped) native input is "visible" but not clickable: use its label.
551
+ if(!hidable || cache.hits(e)) return e;
552
+ }
553
+ if(!hidable) return null;
554
+ if(e.closest('[aria-hidden="true"],[inert]')) return null;
555
+ var labs=[].slice.call(e.labels||[]);
556
+ for(var i=0;i<labs.length;i++) if(visible(labs[i]) && sized(labs[i])) return labs[i];
557
+ if(visible(e) && sized(e)) return e; // no label to fall back to: report it as-is (covered)
558
+ if(sized(e) && e.checkVisibility({checkOpacity:false,checkVisibilityCSS:true}) && visible(e.parentElement)){
559
+ try{ if(e.ownerDocument.defaultView.getComputedStyle(e).pointerEvents!=='none') return e; }catch(_){}
560
+ }
561
+ return null;
562
+ };
563
+ // Identity-focused semantic guard: role + accessible name. Catches a target silently becoming a
564
+ // different control (relabel), while tolerating value/checked/expanded churn.
565
+ cache.guard=function(el){ if(!el) return ''; try{ return [role(el),clean(fieldLabel(el)||hint(el),200)].join(String.fromCharCode(1)); }catch(_){ return ''; } };
566
+ var unnamed=function(g){ return !g || g.charAt(g.length-1)===String.fromCharCode(1); }; // role only, no label: never rebind on it
567
+ // Walk the top document, OPEN shadow roots, and SAME-ORIGIN iframes. dx/dy translate each
568
+ // element's frame-local rect into top-level viewport coordinates (shadow roots share their
569
+ // frame's coords; iframes add their content-box offset). Cross-origin frames can't be read from
570
+ // here; they're reported so the agent knows content exists that it can't see.
571
+ var frames=[], remoteEls=[], textRoots=[];
360
572
  function collect(){
361
573
  var out=[], seen=new Set(), scanned=0;
362
574
  function add(el, dx, dy, clk){ if(seen.has(el)) return; seen.add(el); out.push({el:el, dx:dx, dy:dy, clk:clk}); }
363
575
  function walk(root, dx, dy, depth){
364
576
  if(depth>12) return;
577
+ textRoots.push({root:root, dx:dx, dy:dy});
365
578
  var els; try{ els=root.querySelectorAll(selector); }catch(_){ els=[]; }
366
579
  for(var a=0;a<els.length;a++) add(els[a], dx, dy, false);
367
580
  var all; try{ all=root.querySelectorAll('*'); }catch(_){ all=[]; }
@@ -381,100 +594,266 @@ const SNAPSHOT = `(function(){
381
594
  }
382
595
  }catch(_){}
383
596
  }
384
- if(n.shadowRoot) walk(n.shadowRoot, dx, dy, depth+1);
385
- if(n.tagName==='IFRAME'){ try{
386
- var idoc=n.contentDocument;
597
+ var sr=sroot(n);
598
+ if(sr){ M.watch(sr); walk(sr, dx, dy, depth+1); }
599
+ else if(n.localName.indexOf('-')>0 && !cache.probed.has(n) && pendingHosts.length<40 && sized(n)) pendingHosts.push(n); // custom element: may hide a closed root
600
+ if(n.tagName==='IFRAME' || n.tagName==='FRAME'){
601
+ var idoc=null; try{ idoc=n.contentDocument; }catch(_){}
387
602
  if(idoc && idoc.body){
388
- var ir=n.getBoundingClientRect(), cs=(n.ownerDocument.defaultView||window).getComputedStyle(n);
389
- walk(idoc,
390
- dx+ir.left+(parseFloat(cs.borderLeftWidth)||0)+(parseFloat(cs.paddingLeft)||0),
391
- dy+ir.top+(parseFloat(cs.borderTopWidth)||0)+(parseFloat(cs.paddingTop)||0), depth+1);
603
+ var ir=n.getBoundingClientRect();
604
+ walk(idoc, dx+ir.left+n.clientLeft+(parseFloat(n.ownerDocument.defaultView.getComputedStyle(n).paddingLeft)||0),
605
+ dy+ir.top+n.clientTop+(parseFloat(n.ownerDocument.defaultView.getComputedStyle(n).paddingTop)||0), depth+1);
606
+ } else if(visible(n) && (function(){ var fr=n.getBoundingClientRect(); return fr.width>=10 && fr.height>=10; })()){ // 1x1 ad/tracking frames don't count
607
+ var src=''; try{ src=new URL(n.src, location.href).host; }catch(_){ src=n.getAttribute('src')||''; }
608
+ frames.push(clean(n.getAttribute('title')||n.getAttribute('aria-label')||n.name||src||'frame',60));
609
+ remoteEls.push(n);
392
610
  }
393
- }catch(_){} }
611
+ }
394
612
  }
395
613
  }
396
614
  walk(document, 0, 0, 0);
397
615
  return out;
398
616
  }
399
- var actions=[], nodes=collect();
617
+ var scrollerCache=new Map(); // element -> does it clip its overflow? (one style read per ancestor per snapshot)
618
+ function clipper(a, view){ var v=scrollerCache.get(a); if(v===undefined){ var cs=view.getComputedStyle(a); v=!(cs.overflowX==='visible' && cs.overflowY==='visible'); scrollerCache.set(a,v); } return v; }
619
+ function clipped(t){
620
+ try{
621
+ var r=t.getBoundingClientRect(), cx=r.x+r.width/2, cy=r.y+r.height/2, view=t.ownerDocument.defaultView;
622
+ for(var a=t.parentElement, g=0; a && g<40; a=a.parentElement, g++){
623
+ if(!clipper(a, view)) continue;
624
+ var ar=a.getBoundingClientRect();
625
+ // How far outside the box it is (0 = inside), so the nearest hidden rows are kept first.
626
+ if(cx<ar.left||cx>ar.right||cy<ar.top||cy>ar.bottom) return 1+Math.max(ar.left-cx,cx-ar.right,ar.top-cy,cy-ar.bottom,0);
627
+ }
628
+ }catch(_){}
629
+ return 0;
630
+ }
631
+ var inView=[], offView=[], farOff=0, nodes=collect(), VH=innerHeight, VW=innerWidth;
400
632
  for(var i=0;i<nodes.length;i++){
401
633
  var e=nodes[i].el, ox=nodes[i].dx, oy=nodes[i].dy, clk=nodes[i].clk;
402
634
  try{
403
- if(!safe(e)||!visible(e)||e.matches(':disabled')||e.closest('[aria-disabled="true"]')) continue;
404
- var r=e.getBoundingClientRect(), x=r.x+r.width/2+ox, y=r.y+r.height/2+oy, rname=role(e);
635
+ if(!safe(e)||e.matches(':disabled')||e.closest('[aria-disabled="true"],[inert]')) continue;
636
+ // Cheap early exit for controls screens away (huge pages have thousands): count, don't process.
637
+ var r0=e.getBoundingClientRect();
638
+ if(!opts.all && r0.width>0 && r0.height>0){ var y0=r0.y+r0.height/2+oy; if(y0<-VH||y0>=2*VH){ if(visible(e)) farOff++; continue; } }
639
+ var surf=cache.surface(e); if(!surf) continue;
640
+ var rname=role(e);
405
641
  if(!rname){
406
642
  // Role-less custom clickable (cursor:pointer, inline onclick, or focusable [tabindex]).
407
- // Only accept it if it has a real label and isn't just a wrapper around an actual control,
408
- // so we don't flood the table with layout containers.
643
+ // Only accept it if it has a real label and isn't just a wrapper around an actual control
644
+ // (or a <label> standing in for one), so we don't flood the table with layout containers.
409
645
  var ti=e.getAttribute('tabindex');
410
- if(clk || e.hasAttribute('onclick') || (ti!==null && ti!=='-1')){
646
+ if(clk || e.hasAttribute('onclick') || (ti!==null && ti!=='-1') || e.getAttribute('draggable')==='true'){
647
+ if(e.tagName==='LABEL' && e.control) continue;
411
648
  if(!(name(e)||'').trim() || e.querySelector(selector)) continue;
412
649
  rname='button';
413
650
  }
414
651
  }
415
- if(!rname||r.width<=0||r.height<=0||x<0||y<0||x>=innerWidth||y>=innerHeight) continue;
652
+ if(!rname) continue;
653
+ var r=surf.getBoundingClientRect(), lx=r.x+r.width/2, ly=r.y+r.height/2, x=lx+ox, y=ly+oy;
654
+ if(x<0||x>=VW) continue;
416
655
  if(rname==='gridcell' && e.querySelector('button,[role="button"]')) continue;
417
- var base={node:identity(e), role:rname, label:(name(e)||rname).replace(/\\s+/g,' ').trim().slice(0,120), x:Math.round(x), y:Math.round(y)};
656
+ var base={node:identity(e), role:rname, label:clean(fieldLabel(e)||hint(e)||rname), x:Math.round(x), y:Math.round(y), w:Math.round(r.width), h:Math.round(r.height)};
418
657
  var achecked=e.getAttribute('aria-checked');
419
658
  if(['checkbox','radio'].indexOf(e.type)>=0) base.checked=!!e.checked;
420
659
  else if(achecked!=null) base.checked=(achecked==='true');
421
660
  var aexp=e.getAttribute('aria-expanded'); if(aexp!=null) base.expanded=(aexp==='true');
422
661
  var asel=e.getAttribute('aria-selected'); if(asel!=null) base.selected=(asel==='true');
662
+ if(e.required || e.getAttribute('aria-required')==='true') base.required=true;
663
+ if(e.getAttribute('draggable')==='true') base.draggable=true;
664
+ // Only surface validation errors on fields the user (or agent) has put a value in, or that the
665
+ // page itself flags, so an untouched required form isn't a wall of warnings.
666
+ if(e.getAttribute('aria-invalid')==='true') base.invalid='invalid';
667
+ else if(e.validity && !e.validity.valid && e.value) base.invalid=clean(e.validationMessage,80)||'invalid';
668
+ var clip=clipped(surf);
669
+ if(clip){
670
+ base.off='scroll'; base.dist=clip; // inside an overflow box, scrolled out of it: acting scrolls it into view
671
+ } else if(y<0||y>=VH){
672
+ // Off-screen (scrolled away): keep the nearest ones so the agent knows they exist; acting on
673
+ // them scrolls them into view first.
674
+ if(!opts.all && (y<-VH/2||y>=1.5*VH)){ farOff++; continue; }
675
+ base.off=y<0?'up':'down'; base.dist=y<0?-y:y-VH;
676
+ } else if((function(){ var fv=surf.ownerDocument.defaultView; return fv!==window && (lx<0||ly<0||lx>=fv.innerWidth||ly>=fv.innerHeight); })()){
677
+ base.off='scroll'; // scrolled out of its (same-origin) iframe's viewport
678
+ } else if(!hits(surf, lx, ly)){
679
+ // Not hittable: either scrolled out of an overflow container (reachable — acting scrolls it
680
+ // in) or genuinely covered by an overlay/modal (needs dismissing first).
681
+ base.covered=true;
682
+ }
683
+ var list=base.off?offView:inView;
423
684
  if(e.tagName==='SELECT'){
424
685
  base.kind='select';
425
- base.value=[].map.call(e.selectedOptions,function(o){return o.label;}).join(', ');
426
- base.options=[].filter.call(e.options,function(o){return !o.disabled && !(o.closest&&o.closest('optgroup[disabled]'));}).map(function(o){return o.label;}).slice(0,40);
427
- actions.push(base);
686
+ if(e.multiple) base.fmt='multiple: select values:[...]';
687
+ base.value=clean([].map.call(e.selectedOptions,function(o){return o.label;}).join(', '),80);
688
+ base.options=[].filter.call(e.options,function(o){return !o.disabled && !(o.closest&&o.closest('optgroup[disabled]'));}).map(function(o){return clean(o.label,60);}).slice(0,40);
689
+ list.push(base);
690
+ } else if(e.tagName==='INPUT' && e.type==='file'){
691
+ base.kind='upload';
692
+ if(e.files && e.files.length) base.value=clean([].map.call(e.files,function(f){return f.name;}).join(', '),80);
693
+ if(e.accept) base.fmt=clean(e.accept,60);
694
+ list.push(base);
428
695
  } else {
429
- var editable=!e.readOnly && e.getAttribute('aria-readonly')!=='true' && (['textbox','searchbox','spinbutton'].indexOf(rname)>=0 || (rname==='combobox' && ['INPUT','TEXTAREA'].indexOf(e.tagName)>=0));
430
- var value = (['checkbox','radio'].indexOf(e.type)>=0) ? '' : (('value' in e) ? String(e.value) : ((e.isContentEditable||rname==='combobox') ? e.innerText.trim() : ''));
431
- if(value) base.value=value.slice(0,80);
696
+ var settable=e.tagName==='INPUT' && SETTABLE.hasOwnProperty(e.type);
697
+ var editable=!e.readOnly && e.getAttribute('aria-readonly')!=='true' && (['textbox','searchbox','spinbutton'].indexOf(rname)>=0 || (rname==='slider' && e.tagName==='INPUT') || (rname==='combobox' && ['INPUT','TEXTAREA'].indexOf(e.tagName)>=0));
698
+ var value = (['checkbox','radio'].indexOf(e.type)>=0) ? '' : (('value' in e && e.tagName!=='BUTTON' && e.tagName!=='LI') ? String(e.value) : ((e.isContentEditable||rname==='combobox') ? e.innerText.trim() : ''));
699
+ if(e.tagName==='INPUT' && ['button','submit','reset','image'].indexOf(e.type)>=0) value=''; // its value IS its label
700
+ if(value) base.value=clean(value,80); // single line: page text must never forge extra table rows
701
+ if(settable){ base.fmt=e.type==='range' ? (e.min||'0')+'..'+(e.max||'100')+(e.step&&e.step!=='any'?' step '+e.step:'') : SETTABLE[e.type]; }
432
702
  base.kind=editable?'fill':'click';
433
- actions.push(base);
703
+ list.push(base);
434
704
  // For an editable combobox, also offer a plain click to open its popup (not just type).
435
- if(editable && rname==='combobox'){ actions.push({node:base.node, role:rname, label:'Open '+base.label, x:base.x, y:base.y, kind:'click', expanded:base.expanded}); }
705
+ if(editable && rname==='combobox'){ list.push({node:base.node, role:rname, label:'Open '+base.label, x:base.x, y:base.y, kind:'click', expanded:base.expanded, off:base.off, covered:base.covered}); }
436
706
  }
437
707
  }catch(_){ continue; }
438
708
  }
439
- // Page text is a best-effort extra — a failure here must NOT discard the element table
440
- // we already computed above.
441
- var text='';
709
+ var omitted=Math.max(0, inView.length-250); inView.splice(250);
710
+ // Keep the 25 off-screen controls NEAREST the visible area (not the first 25 in DOM order, which
711
+ // for a scrolled list are the rows furthest behind), then restore page order.
712
+ var OFFCAP=opts.all?400:25;
713
+ var offMore=Math.max(0, offView.length-OFFCAP)+farOff;
714
+ if(offView.length>OFFCAP){ offView.forEach(function(a,k){ a.ord=k; }); offView.sort(function(a,b){ return a.dist-b.dist; }); offView.splice(OFFCAP); offView.sort(function(a,b){ return a.ord-b.ord; }); }
715
+ var actions=inView.concat(offView);
716
+ // The nearest ancestor text that isn't just the label itself: which row/item a control is in.
717
+ function ctxOf(el, lab){
718
+ for(var p=el&&el.parentElement, g=0; p && g<6 && p.tagName!=='BODY'; p=p.parentElement, g++){
719
+ var tx=clean(p.innerText,400); if(!tx || tx===lab) continue;
720
+ var rest=clean(tx.split(lab).join(' '),40); if(rest) return rest;
721
+ }
722
+ return '';
723
+ }
724
+ // Identical labels ("Delete" per row, "Edit" per user) are ambiguous to the agent: tag each with
725
+ // the nearest ancestor text that tells them apart (e.g. the list row it lives in).
442
726
  try{
443
- var words=[], walker=document.createTreeWalker(document.body,NodeFilter.SHOW_TEXT), range=document.createRange(), node, length=0;
444
- while((node=walker.nextNode()) && length<4000){
445
- var v=node.textContent.trim(), p=node.parentElement;
446
- if(!v||!p||p.closest('script,style,noscript,template')||!visible(p)) continue;
447
- range.selectNodeContents(node); var tr=range.getBoundingClientRect();
448
- if(tr.width>0&&tr.height>0&&tr.bottom>0&&tr.top<innerHeight&&tr.right>0&&tr.left<innerWidth){ words.push(v); length+=v.length; }
449
- }
450
- text=words.join('\\n').slice(0,4000);
451
- }catch(_){ text=''; }
452
- var omitted=Math.max(0, actions.length-250); actions.splice(250);
727
+ var byLabel={};
728
+ for(var u=0;u<actions.length;u++){ var key=actions[u].kind+'|'+actions[u].label; (byLabel[key]=byLabel[key]||[]).push(actions[u]); }
729
+ Object.keys(byLabel).forEach(function(key){
730
+ var grp=byLabel[key]; if(grp.length<2) return;
731
+ grp.forEach(function(a){ var cx=ctxOf(cache.nodes.get(a.node), a.label); if(cx) a.ctx=cx; });
732
+ });
733
+ }catch(_){}
734
+ var focus=null;
735
+ try{ var fe=document.activeElement; while(fe && fe.shadowRoot && fe.shadowRoot.activeElement) fe=fe.shadowRoot.activeElement;
736
+ for(var q=0; fe && fe.tagName==='IFRAME' && q<8; q++){ var fd=null; try{ fd=fe.contentDocument; }catch(_){} fe=fd?fd.activeElement:null; }
737
+ if(fe && fe.tagName!=='BODY' && fe.tagName!=='HTML' && cache.ids.has(fe)) focus=cache.ids.get(fe);
738
+ }catch(_){}
453
739
  // Displayed id derives from the STABLE node id (not position), so a reused number can never
454
740
  // remap to a different element across observations; duplicates (e.g. combobox Open) get a suffix.
455
741
  // Guarded so a getter/DOM quirk while building ids can't blank the whole table.
742
+ // Resolved to CDP frame ids by the extension (cross-origin frames): biggest first, so a real
743
+ // embedded app/checkout wins over banner slots when there are many.
744
+ cache.pendingHosts=pendingHosts;
745
+ cache.remoteEls=remoteEls.map(function(el){ var r=el.getBoundingClientRect(); return {el:el, a:r.width*r.height}; }).sort(function(p,q){ return q.a-p.a; }).slice(0,12).map(function(p){ return p.el; });
746
+ // Keep refs stable across re-renders that REPLACE nodes: a brand-new element whose identity
747
+ // (role + label + row context) uniquely matches an element that vanished since the last snapshot
748
+ // takes over its node id, so "e12" keeps meaning "Done in Task 7" and the agent's refs survive.
456
749
  try{
457
- cache.byId={}; cache.guards={}; var used={};
458
- for(var j=0;j<actions.length;j++){ var bid='e'+actions[j].node, id=bid, kk=2; while(used[id]){ id=bid+'_'+kk; kk++; } used[id]=1; actions[j].id=id; cache.byId[id]=actions[j].node; cache.guards[id]=cache.guard(cache.nodes.get(actions[j].node)); }
750
+ var gone={}, gcount={}, ncount={}, k2;
751
+ Object.keys(cache.byId||{}).forEach(function(id){
752
+ var nid=cache.byId[id]; if(cache.nodes.has(nid)) return; // still on the page
753
+ var fp=cache.fps&&cache.fps[id]; if(!fp||!cache.guards[id]||unnamed(cache.guards[id])) return;
754
+ k2=cache.guards[id]+'\u0002'+(fp.ctx||''); gone[k2]=nid; gcount[k2]=(gcount[k2]||0)+1;
755
+ });
756
+ var keyOf=function(a){ return cache.guard(cache.nodes.get(a.node))+'\u0002'+(a.ctx||''); };
757
+ actions.forEach(function(a){ if(fresh.has(a.node)){ k2=keyOf(a); ncount[k2]=(ncount[k2]||0)+1; } });
758
+ actions.forEach(function(a){
759
+ if(!fresh.has(a.node)) return;
760
+ k2=keyOf(a);
761
+ if(gcount[k2]!==1 || ncount[k2]!==1) return; // ambiguous: never guess
762
+ var el=cache.nodes.get(a.node), old=gone[k2];
763
+ cache.nodes.delete(a.node); cache.ids.set(el, old); cache.nodes.set(old, el);
764
+ actions.forEach(function(b){ if(b.node===a.node && b!==a) b.node=old; });
765
+ a.node=old; delete gcount[k2];
766
+ });
459
767
  }catch(_){}
460
- return {url:location.href, title:document.title, scrollY:Math.round(scrollY), scrollH:Math.round(document.documentElement.scrollHeight), text:text, omitted:omitted, actions:actions};
461
- }catch(_){ return {url:location.href, title:(document&&document.title)||'', scrollY:0, scrollH:0, text:'', omitted:0, actions:[]}; }
462
- })()`;
768
+ var focusId=null;
769
+ try{
770
+ cache.byId={}; cache.guards={}; cache.fps={}; var used={};
771
+ for(var j=0;j<actions.length;j++){ var bid='e'+actions[j].node, id=bid, kk=2; while(used[id]){ id=bid+'_'+kk; kk++; } used[id]=1; actions[j].id=id; cache.byId[id]=actions[j].node; cache.guards[id]=cache.guard(cache.nodes.get(actions[j].node)); cache.fps[id]={label:actions[j].label, ctx:actions[j].ctx||''}; if(focus!=null && actions[j].node===focus && !focusId) focusId=id; }
772
+ }catch(_){}
773
+ // Resolve a ref to its live element. Frameworks that re-render by REPLACING nodes (innerHTML
774
+ // templates, keyed lists) orphan every ref; re-find the replacement by the same identity the guard
775
+ // uses (role + label) plus its row context — only when exactly ONE element matches, never a guess.
776
+ cache.get=function(id){
777
+ var node=cache.byId[id]; if(node==null) return null;
778
+ var e=cache.nodes.get(node); if(e && e.isConnected) return e;
779
+ var fp=cache.fps && cache.fps[id], want=cache.guards && cache.guards[id]; if(!fp || !want || unnamed(want)) return null;
780
+ var cands=collect(), hit=null, n=0;
781
+ for(var k=0;k<cands.length && n<2;k++){
782
+ var el=cands[k].el;
783
+ try{
784
+ if(!el.isConnected || cache.guard(el)!==want || !cache.surface(el)) continue;
785
+ if(fp.ctx && ctxOf(el, fp.label)!==fp.ctx) continue;
786
+ hit=el; n++;
787
+ }catch(_){}
788
+ }
789
+ if(n!==1) return null;
790
+ cache.byId[id]=identity(hit);
791
+ return hit;
792
+ };
793
+ // Optional: the visible text in reading order (top-to-bottom, left-to-right), for prices, headings
794
+ // and results the controls table doesn't carry.
795
+ var vtext='';
796
+ if(opts.text){ try{
797
+ var frags=[], seenT=0;
798
+ textRoots.forEach(function(tr){
799
+ var doc=tr.root.ownerDocument||tr.root, w=doc.createTreeWalker(tr.root.body||tr.root, NodeFilter.SHOW_TEXT), rg=doc.createRange(), nd;
800
+ while((nd=w.nextNode()) && seenT<20000){ seenT++;
801
+ var v=nd.textContent.replace(/\\s+/g,' ').trim(), p=nd.parentElement;
802
+ if(!v||!p||p.closest('script,style,noscript,template')||!visible(p)) continue;
803
+ rg.selectNodeContents(nd); var q=rg.getBoundingClientRect(), ty=q.y+tr.dy, tx=q.x+tr.dx;
804
+ if(q.width<=0||q.height<=0||ty+q.height<0||ty>=VH||tx>=VW||tx+q.width<0) continue;
805
+ frags.push({t:v, x:tx, y:ty});
806
+ }
807
+ });
808
+ frags.sort(function(a,b){ return Math.round(a.y/6)-Math.round(b.y/6) || a.x-b.x; });
809
+ var lines=[], cur=null, lastY=-1e9;
810
+ frags.forEach(function(f){ if(Math.abs(f.y-lastY)>6){ if(cur) lines.push(cur); cur=f.t; lastY=f.y; } else cur+=' '+f.t; });
811
+ if(cur) lines.push(cur);
812
+ vtext=lines.join('\\n').slice(0,4000);
813
+ }catch(_){} }
814
+ return {url:location.href, title:document.title, vh:innerHeight, text:vtext, scrollY:Math.round(scrollY), scrollH:Math.round(document.documentElement.scrollHeight), omitted:omitted, offMore:offMore, frames:frames.slice(0,10), focus:focusId, next:cache.next, probe:pendingHosts.length, actions:actions};
815
+ }catch(_){ return {url:location.href, title:(document&&document.title)||'', scrollY:0, scrollH:0, omitted:0, actions:[]}; }
816
+ })`;
817
+
818
+ function formatRow(a) {
819
+ let flag = ' ';
820
+ if (typeof a.expanded === 'boolean') flag = a.expanded ? '▾' : '▸'; // open / closed
821
+ else if (typeof a.checked === 'boolean') flag = a.checked ? '✓' : '·';
822
+ else if (a.selected === true) flag = '◉';
823
+ let line = `${a.id.padEnd(4)} ${a.kind.padEnd(6)}${flag} "${a.label}"`;
824
+ if (a.ctx) line += ` in "${a.ctx}"`;
825
+ if (a.required) line += ' (required)';
826
+ if (a.value) line += ` ▸ "${a.value}"`;
827
+ if (a.fmt) line += ` fmt{${a.fmt}}`;
828
+ if (a.kind === 'select' && a.options && a.options.length) line += ` opts{${a.options.join(' | ')}}`;
829
+ if (a.invalid) line += ` ⚠ "${a.invalid}"`;
830
+ if (a.off === 'up') line += ' ↑ above view';
831
+ else if (a.off === 'down') line += ' ↓ below view';
832
+ else if (a.off === 'scroll') line += ' ↕ scrolled out of its box';
833
+ if (a.draggable) line += ' ⇄ draggable';
834
+ if (a.covered) line += ' ⊘ covered';
835
+ return line;
836
+ }
463
837
 
464
838
  function formatTable(snap) {
465
839
  if (!snap) return '(page not ready)';
466
840
  const lines = [];
841
+ const fsnaps = snap.frameSnaps || [];
842
+ const count = snap.actions.length + fsnaps.reduce((n, f) => n + Math.min(60, f.snap.actions.length), 0);
467
843
  lines.push(`${snap.title || '(untitled)'} — ${snap.url}`);
468
- lines.push(`scroll ${snap.scrollY}/${snap.scrollH} · ${snap.actions.length} controls${snap.omitted ? ` (+${snap.omitted} more; scroll to reveal)` : ''}`);
469
- for (const a of snap.actions) {
470
- let flag = ' ';
471
- if (typeof a.expanded === 'boolean') flag = a.expanded ? '▾' : '▸'; // open / closed
472
- else if (typeof a.checked === 'boolean') flag = a.checked ? '✓' : '·';
473
- else if (a.selected === true) flag = '◉';
474
- let line = `${a.id.padEnd(4)} ${a.kind.padEnd(6)}${flag} "${a.label}"`;
475
- if (a.value) line += ` ▸ "${a.value}"`;
476
- if (a.kind === 'select' && a.options && a.options.length) line += ` opts{${a.options.join(' | ')}}`;
477
- lines.push(line);
844
+ const more = (snap.omitted || 0) + (snap.offMore || 0);
845
+ lines.push(`scroll ${snap.scrollY}/${snap.scrollH} · ${count} controls${more ? ` (+${more} more; scroll to reveal)` : ''}${snap.focus ? ` · focus ${snap.focus}` : ''}`);
846
+ if (snap.frames && snap.frames.length > fsnaps.length) lines.push(`cross-origin frames (content not readable): ${snap.frames.map((f) => `"${f}"`).join(', ')}`);
847
+ for (const a of snap.actions) lines.push(formatRow(a));
848
+ for (const { f, off, snap: fs } of fsnaps) {
849
+ let host = f.url;
850
+ try { host = new URL(f.url).host || f.url; } catch {}
851
+ lines.push(`frame f${f.idx} "${String(host).slice(0, 60)}" (cross-origin):`);
852
+ for (const a of fs.actions.slice(0, 60)) {
853
+ const ty = off.y + a.y;
854
+ if (!a.off && (ty < 0 || ty >= (snap.vh || 1e9))) a.off = ty < 0 ? 'up' : 'down';
855
+ lines.push(formatRow({ ...a, id: `f${f.idx}.${a.id}` }));
856
+ }
478
857
  }
479
858
  return lines.join('\n');
480
859
  }
@@ -482,25 +861,264 @@ function formatTable(snap) {
482
861
  // A page signature to detect whether an action actually changed the page. Includes per-input
483
862
  // value/checked/selectedIndex so fills, toggles, and selects register as changes (password
484
863
  // values excluded).
485
- const SIG = `JSON.stringify([location.href, document.title, [].map.call(document.querySelectorAll('input,textarea,select'),function(e){return e.type==='password'?'':(String(e.value)+'~'+(e.checked?1:0)+'~'+(e.selectedIndex==null?'':e.selectedIndex));}).join('|'), document.querySelectorAll('a,button,input,select,textarea,summary,[role]').length])`;
864
+ const SIG = `JSON.stringify([location.href, document.title, document.body ? document.body.textContent.length : 0, [].map.call(document.querySelectorAll('input,textarea,select'),function(e){return e.type==='password'?'':(String(e.value)+'~'+(e.checked?1:0)+'~'+(e.selectedIndex==null?'':e.selectedIndex));}).join('|'), document.querySelectorAll('a,button,input,select,textarea,summary,[role]').length])`;
486
865
 
487
866
  // Retry through transient "document is navigating" states so a snapshot taken
488
867
  // during a transition settles instead of failing.
489
- async function snapshot(tabId) {
868
+ // Readable page text. innerText stops at shadow roots and iframes, so it misses web-component
869
+ // content and same-origin frames. Mark every composed ancestor of a shadow host / iframe; unmarked
870
+ // subtrees use the (fast, layout-aware) innerText, marked ones are walked through their composed
871
+ // children (shadow root, slots' assigned nodes, iframe document).
872
+ const READ_TEXT = `(function(max){
873
+ var main=document.querySelector('main')||document.body; if(!main) return {title:document.title,url:location.href,text:''};
874
+ var mark=new Set(), C=window.__pawbrowse&&window.__pawbrowse.closed;
875
+ function sr(n){ return n.shadowRoot || (C&&C.get(n)) || null; }
876
+ function scan(root,d){ if(d>12) return; var all; try{ all=root.querySelectorAll('*'); }catch(_){ return; }
877
+ for(var i=0;i<all.length;i++){ var n=all[i], special=false;
878
+ if(sr(n)){ special=true; scan(sr(n),d+1); }
879
+ if(n.tagName==='SLOT') special=true;
880
+ if(n.tagName==='IFRAME'||n.tagName==='FRAME'){ var doc=null; try{ doc=n.contentDocument; }catch(_){} if(doc&&doc.body){ special=true; scan(doc,d+1); } }
881
+ if(special){ for(var p=n; p && !mark.has(p); ){ mark.add(p); p=p.parentNode; if(p && p.nodeType===11) p=p.host; } }
882
+ } }
883
+ scan(document,0);
884
+ var out=[], len=0;
885
+ function vis(e){ try{ if(e.checkVisibility({checkOpacity:true,checkVisibilityCSS:true})) return true; return getComputedStyle(e).display==='contents'; }catch(_){ return true; } }
886
+ function put(t){ if(t && len<max){ out.push(t); len+=t.length; } }
887
+ function kids(n){
888
+ if(sr(n)) return sr(n).childNodes;
889
+ if(n.tagName==='SLOT'){ var a=n.assignedNodes({flatten:true}); return a.length?a:n.childNodes; }
890
+ return n.childNodes;
891
+ }
892
+ function walk(n,d){
893
+ if(len>=max||d>60) return;
894
+ if(n.nodeType===3){ var v=n.textContent.replace(/\\s+/g,' ').trim(); if(v && n.parentElement && vis(n.parentElement)) put(v); return; }
895
+ if(n.nodeType!==1 && n.nodeType!==11) return;
896
+ if(n.nodeType===1){
897
+ if(/^(SCRIPT|STYLE|NOSCRIPT|TEMPLATE)$/.test(n.tagName) || !vis(n)) return;
898
+ if(n.tagName==='IFRAME'||n.tagName==='FRAME'){ var doc=null; try{ doc=n.contentDocument; }catch(_){} if(doc&&doc.body){ put('\\n'); walk(doc.body,d+1); put('\\n'); } return; }
899
+ if(!mark.has(n)){ put(n.innerText); if(getComputedStyle(n).display!=='inline') put('\\n'); return; }
900
+ }
901
+ var k=kids(n); for(var i=0;i<k.length;i++) walk(k[i],d+1);
902
+ if(n.nodeType===1 && getComputedStyle(n).display!=='inline') put('\\n');
903
+ }
904
+ walk(main,0);
905
+ var text=out.join(' ').replace(/[ \\t]*\\n[ \\t]*/g,'\\n').replace(/\\n{3,}/g,'\\n\\n').trim().slice(0,max);
906
+ return {title:document.title,url:location.href,text:text};
907
+ })`;
908
+
909
+ /* ------------------------------ Cross-origin frames ------------------------- *
910
+ * The page-side snapshot can't reach into cross-origin iframes (payment fields, embedded logins,
911
+ * widgets). CDP can: cross-SITE frames are separate targets we auto-attach to as flat child
912
+ * sessions; cross-origin but same-site frames share their parent's process and get their own
913
+ * isolated world via frameId. Each readable frame gets a stable index per tab (f1, f2...), its
914
+ * refs are shown as f1.e3, and clicks are translated by the frame's on-screen offset. */
915
+ const AUTO_ATTACH = { autoAttach: true, waitForDebuggerOnStart: false, flatten: true, filter: [{ type: 'iframe' }, { exclude: true }] };
916
+ const childSessions = new Map(); // tabId -> Map(sessionId -> { targetId, parent: sessionId|null })
917
+ const frameIdx = new Map(); // tabId -> { next, byKey: Map(key -> idx), byIdx: Map(idx -> frame) }
918
+ function kidsOf(tabId) { let m = childSessions.get(tabId); if (!m) { m = new Map(); childSessions.set(tabId, m); } return m; }
919
+
920
+ chrome.debugger.onEvent.addListener((source, method, params) => {
921
+ const tabId = source.tabId;
922
+ if (tabId == null) return;
923
+ if (method === 'Target.attachedToTarget') {
924
+ const info = params.targetInfo || {};
925
+ if (info.type !== 'iframe' || /^chrome-extension:/i.test(info.url || '')) return; // other extensions' frames aren't ours to drive
926
+ // Ad-heavy pages spawn hundreds of (mostly invisible) frames: just record them; a session is
927
+ // only set up (ensureSession) when one of its frames is actually visible and read.
928
+ kidsOf(tabId).set(params.sessionId, { targetId: info.targetId, parent: source.sessionId || null, ready: null });
929
+ } else if (method === 'Target.detachedFromTarget') {
930
+ kidsOf(tabId).delete(params.sessionId);
931
+ const m = worlds.get(tabId);
932
+ if (m) for (const k of [...m.keys()]) if (k.startsWith(`${params.sessionId}|`)) m.delete(k);
933
+ }
934
+ });
935
+
936
+ async function ensureSession(tabId, sessionId) {
937
+ const k = kidsOf(tabId).get(sessionId);
938
+ if (!k) return;
939
+ if (!k.ready) {
940
+ const t = { tabId, sessionId };
941
+ k.ready = Promise.all([['Page.enable', {}], ['DOM.enable', {}], ['Target.setAutoAttach', AUTO_ATTACH]].map(([m, p]) => sendCdp(t, m, p).catch(() => {})));
942
+ }
943
+ await k.ready;
944
+ }
945
+
946
+ // CDP frame ids of the visible cross-origin iframes a snapshot found in `target`'s document.
947
+ async function remoteFrameIds(target) {
948
+ const ctx = await worldFor(target);
949
+ const r = await sendCdp(target, 'Runtime.evaluate', { expression: 'window.__pawbrowse && window.__pawbrowse.remoteEls', contextId: ctx, objectGroup: 'pawframes' });
950
+ const ids = [];
951
+ try {
952
+ if (!r.result || !r.result.objectId) return ids;
953
+ const { result } = await sendCdp(target, 'Runtime.getProperties', { objectId: r.result.objectId, ownProperties: true });
954
+ for (const p of result) {
955
+ if (!/^\d+$/.test(p.name) || !p.value || !p.value.objectId) continue;
956
+ try { const { node } = await sendCdp(target, 'DOM.describeNode', { objectId: p.value.objectId }); if (node.frameId) ids.push(node.frameId); } catch {}
957
+ }
958
+ } finally { sendCdp(target, 'Runtime.releaseObjectGroup', { objectGroup: 'pawframes' }).catch(() => {}); }
959
+ return ids;
960
+ }
961
+
962
+ // Read the cross-origin frames under a snapshot, depth-first: only visible ones, never invisible
963
+ // subtrees. Cross-SITE frames are their own attached session; cross-origin same-site frames live
964
+ // in the parent's process and are addressed by frameId.
965
+ async function readFrames(tabId, parentTarget, parentSnap, out, depth = 0) {
966
+ if (!parentSnap || !parentSnap.frames || !parentSnap.frames.length || depth > 4 || out.length >= 12) return out;
967
+ let ids = [];
968
+ try { ids = await remoteFrameIds(parentTarget); } catch {}
969
+ let fi = frameIdx.get(tabId);
970
+ if (!fi) { fi = { next: 1, byKey: new Map(), byIdx: new Map() }; frameIdx.set(tabId, fi); }
971
+ for (const fid of ids) {
972
+ if (out.length >= 12) break;
973
+ const sid = [...kidsOf(tabId)].reverse().find(([, k]) => k.targetId === fid)?.[0]; // newest wins after a re-attach
974
+ const parentSession = typeof parentTarget === 'object' ? parentTarget.sessionId : undefined;
975
+ const f = sid
976
+ ? { target: { tabId, sessionId: sid }, root: true, key: `s:${fid}` }
977
+ : { target: { tabId, sessionId: parentSession, frameId: fid }, root: false, key: `f:${fid}` };
978
+ try {
979
+ if (sid) await ensureSession(tabId, sid);
980
+ const off = await frameOffset(tabId, f);
981
+ if (off.w < 10 || off.h < 10) continue; // tracking pixels / collapsed frames
982
+ const fs = await snapshot(f.target, 1);
983
+ if (!fs || !fs.actions) continue;
984
+ if (!fi.byKey.has(f.key)) fi.byKey.set(f.key, fi.next++);
985
+ f.idx = fi.byKey.get(f.key); f.url = fs.url;
986
+ fi.byIdx.set(f.idx, f);
987
+ out.push({ f, off, snap: fs });
988
+ await readFrames(tabId, f.target, fs, out, depth + 1);
989
+ } catch {}
990
+ }
991
+ return out;
992
+ }
993
+
994
+ // Top-level viewport offset (and size) of a frame's content box: its owner <iframe>'s content quad
995
+ // in the parent's local root, plus that parent session's own offset, up to the top.
996
+ async function frameOffset(tabId, f) {
997
+ let x = 0, y = 0, w = 0, h = 0, first = true, topOwner = null;
998
+ let sess = f.target.sessionId || null, isRoot = !!f.root;
999
+ let fid = isRoot ? kidsOf(tabId).get(sess)?.targetId : f.target.frameId;
1000
+ for (let g = 0; g < 10 && fid; g++) {
1001
+ // A session's root frame is owned by an <iframe> in the PARENT session; an in-process frame's
1002
+ // owner is in its own session, whose box coords are relative to that session's root.
1003
+ const owner = isRoot ? (kidsOf(tabId).get(sess)?.parent ?? null) : sess;
1004
+ const t = owner ? { tabId, sessionId: owner } : tabId;
1005
+ const { backendNodeId } = await sendCdp(t, 'DOM.getFrameOwner', { frameId: fid });
1006
+ const { model } = await sendCdp(t, 'DOM.getBoxModel', { backendNodeId });
1007
+ const q = model.content;
1008
+ x += q[0]; y += q[1];
1009
+ if (first) { w = q[2] - q[0]; h = q[5] - q[1]; first = false; }
1010
+ if (!owner) { topOwner = backendNodeId; break; } // reached the top session: coordinates are now top-level
1011
+ sess = owner; isRoot = true; fid = kidsOf(tabId).get(owner)?.targetId;
1012
+ }
1013
+ return { x, y, w, h, topOwner };
1014
+ }
1015
+
1016
+ // Split "f2.e7" into its frame and the frame-local ref.
1017
+ function routeRef(tabId, ref) {
1018
+ const m = /^f(\d+)\.(.+)$/.exec(String(ref || ''));
1019
+ if (!m) return { target: tabId, ref, frame: null };
1020
+ const f = frameIdx.get(tabId)?.byIdx.get(Number(m[1]));
1021
+ if (!f) return { target: null, ref: m[2], frame: null };
1022
+ return { target: f.target, ref: m[2], frame: f };
1023
+ }
1024
+
1025
+ // Hand CLOSED shadow roots of the custom elements a snapshot flagged to our isolated world:
1026
+ // DOM.describeNode(pierce) exposes them to CDP even though page JS can't reach them. Each host is
1027
+ // probed once per document. Returns how many roots were found.
1028
+ async function probeClosedRoots(target) {
1029
+ const ctx = await worldFor(target);
1030
+ let found = 0;
1031
+ try {
1032
+ const r = await sendCdp(target, 'Runtime.evaluate', { expression: 'window.__pawbrowse && window.__pawbrowse.pendingHosts', contextId: ctx, objectGroup: 'pawshadow' });
1033
+ if (!r.result || !r.result.objectId) return 0;
1034
+ const { result } = await sendCdp(target, 'Runtime.getProperties', { objectId: r.result.objectId, ownProperties: true });
1035
+ for (const p of result) {
1036
+ if (!/^\d+$/.test(p.name) || !p.value || !p.value.objectId) continue;
1037
+ try {
1038
+ const { node } = await sendCdp(target, 'DOM.describeNode', { objectId: p.value.objectId, depth: 0, pierce: true });
1039
+ const root = (node.shadowRoots || []).find((x) => x.shadowRootType === 'closed');
1040
+ if (!root) continue;
1041
+ const { object } = await sendCdp(target, 'DOM.resolveNode', { backendNodeId: root.backendNodeId, executionContextId: ctx, objectGroup: 'pawshadow' });
1042
+ await sendCdp(target, 'Runtime.callFunctionOn', { objectId: p.value.objectId, functionDeclaration: 'function(r){ window.__pawbrowse.closed.set(this, r); }', arguments: [{ objectId: object.objectId }] });
1043
+ found++;
1044
+ } catch {}
1045
+ }
1046
+ await sendCdp(target, 'Runtime.evaluate', { expression: 'window.__pawbrowse.pendingHosts.forEach(function(h){ window.__pawbrowse.probed.add(h); })', contextId: ctx });
1047
+ } finally { sendCdp(target, 'Runtime.releaseObjectGroup', { objectGroup: 'pawshadow' }).catch(() => {}); }
1048
+ return found;
1049
+ }
1050
+
1051
+ async function snapshot(target, tries = 8, opts) {
1052
+ const O = JSON.stringify(opts || {});
1053
+ const sk = typeof target === 'object' ? `${target.tabId}#${frameKey(target)}` : target;
490
1054
  let last;
491
- for (let i = 0; i < 8; i++) {
1055
+ for (let i = 0; i < tries; i++) {
492
1056
  try {
493
- const snap = await evaluate(tabId, SNAPSHOT);
494
- if (snap) return snap;
1057
+ let snap = await evaluate(target, `${SNAPSHOT}(${refSeed.get(sk) || 1}, ${O})`);
1058
+ // Closed shadow roots found: re-snapshot with them (nested closed hosts: a few rounds).
1059
+ for (let k = 0; k < 3 && snap && snap.probe; k++) {
1060
+ if (!(await probeClosedRoots(target).catch(() => 0))) break;
1061
+ snap = await evaluate(target, `${SNAPSHOT}(${refSeed.get(sk) || 1}, ${O})`);
1062
+ }
1063
+ if (snap) { if (snap.next > (refSeed.get(sk) || 1)) { refSeed.set(sk, snap.next); persistState(); } return snap; }
495
1064
  } catch (e) { last = e; }
496
- await sleep(120);
1065
+ if (i < tries - 1) await sleep(120);
497
1066
  }
498
1067
  if (last) throw last;
499
1068
  return null;
500
1069
  }
501
1070
 
502
- async function observe(tabId) {
503
- return formatTable(await snapshot(tabId));
1071
+ // The last table each tab reported, so act() can tell the agent whether ANYTHING it can see changed
1072
+ // (scrolling a panel, opening a popover...) rather than only form values / URL / control count.
1073
+ const lastTable = new Map(); // tabId -> table text (without the header lines that hold ref numbers)
1074
+
1075
+ // Ref numbers and the focus marker are dropped: clicking a button that does nothing still focuses
1076
+ // it, and that alone must not count as "the page changed".
1077
+ function tableBody(t) { return String(t).split('\n').map((l) => l.replace(/^(f\d+\.)?e\d+(_\d+)?\s+/, '').replace(/ {2}· {2}focus e\S+$/, '')).join('\n'); }
1078
+
1079
+ // What the agent last saw, per tab (full table text), so act() can answer with just the rows that
1080
+ // changed. Same document + mostly-unchanged table => delta; otherwise the full table.
1081
+ const lastFull = new Map();
1082
+ const ROW_ID = /^((?:f\d+\.)?e\d+(?:_\d+)?)\s/;
1083
+ function deltaTable(prev, cur) {
1084
+ if (!prev) return null;
1085
+ const P = prev.split('\n'), C = cur.split('\n');
1086
+ if (P[0] !== C[0]) return null; // title/url changed: a different page, send it whole
1087
+ const prevRows = new Map();
1088
+ for (const l of P) { const m = ROW_ID.exec(l); if (m) prevRows.set(m[1], l); }
1089
+ const out = [], ids = new Set();
1090
+ let rows = 0, unchanged = 0;
1091
+ for (const l of C) {
1092
+ const m = ROW_ID.exec(l);
1093
+ if (!m) { out.push(l); continue; } // headers, frame and tool lines: always
1094
+ rows++; ids.add(m[1]);
1095
+ if (prevRows.get(m[1]) === l) unchanged++; else out.push(l);
1096
+ }
1097
+ if (rows < 12 || unchanged < rows * 0.5) return null; // small table or big change: whole is clearer
1098
+ const gone = [...prevRows.keys()].filter((id) => !ids.has(id));
1099
+ const at = out.findIndex((l) => l.startsWith('scroll ')) + 1 || 1;
1100
+ out.splice(at, 0, `(only changes shown: ${rows - unchanged} new/changed row(s); ${unchanged} unchanged row(s) omitted${gone.length ? `; gone: ${gone.slice(0, 40).join(', ')}${gone.length > 40 ? ' …' : ''}` : ''} — browser_observe for the full table)`);
1101
+ return out.join('\n');
1102
+ }
1103
+
1104
+ async function observe(tabId, opts) {
1105
+ const find = opts && opts.find != null && String(opts.find).trim() ? String(opts.find).trim().toLowerCase() : null;
1106
+ const snap = await snapshot(tabId, 8, { all: !!find, text: !!(opts && opts.text) });
1107
+ if (snap) snap.frameSnaps = await readFrames(tabId, tabId, snap, []);
1108
+ if (find && snap) {
1109
+ // Whole-page search: only rows whose label / row context / value mention the query.
1110
+ const hit = (a) => [a.label, a.ctx, a.value].some((v) => v && String(v).toLowerCase().includes(find));
1111
+ const total = snap.actions.length;
1112
+ snap.actions = snap.actions.filter(hit).slice(0, 80);
1113
+ for (const f of snap.frameSnaps) f.snap.actions = f.snap.actions.filter(hit);
1114
+ snap.omitted = 0; snap.offMore = 0;
1115
+ snap.findNote = `(find "${opts.find}": ${snap.actions.length} of ${total} controls on the whole page match; browser_observe without find for the full table)`;
1116
+ }
1117
+ let t = formatTable(snap).replace('\n', `\n${formatTools(tabId)}`.replace(/\n$/, '') + '\n').replace(/\n\n/, '\n');
1118
+ if (snap && snap.findNote) t = t.replace(/\n/, `\n${snap.findNote}\n`);
1119
+ if (snap && snap.text) t += `\n\nvisible text (untrusted page content):\n${snap.text}`;
1120
+ if (!find) { lastTable.set(tabId, tableBody(t)); lastFull.set(tabId, t); }
1121
+ return t;
504
1122
  }
505
1123
 
506
1124
  /* -------------------------------- Actions --------------------------------- */
@@ -509,40 +1127,91 @@ async function observe(tabId) {
509
1127
  // (elementFromPoint containment) so we never click a stale/covered/wrong target.
510
1128
  async function resolveHit(tabId, ref, opts) {
511
1129
  const forFill = opts && opts.fill ? 'true' : 'false';
1130
+ const noScroll = opts && opts.noScroll ? 'true' : 'false', noHit = opts && opts.noHit ? 'true' : 'false';
512
1131
  const R = JSON.stringify(String(ref));
513
1132
  return evaluate(tabId, `(function(){
514
1133
  var c=window.__pawbrowse; if(!c||!c.byId) return {error:'no snapshot yet; observe first'};
515
- var node=c.byId[${R}];
516
- if(node==null) return {error:'unknown ref (observe again)'};
517
- var e=c.nodes.get(node);
1134
+ if(c.byId[${R}]==null) return {error:'unknown ref (observe again)'};
1135
+ var e=c.get?c.get(${R}):null;
518
1136
  if(!e||!e.isConnected) return {error:'element no longer on page (observe again)'};
519
1137
  if(c.guard && c.guards && c.guards[${R}]!=null && c.guard(e)!==c.guards[${R}]) return {error:'element changed since observe (observe again)'};
520
1138
  if(e.matches(':disabled')||e.closest('[aria-disabled="true"],[inert]')) return {error:'element is disabled'};
521
1139
  if(${forFill} && (e.readOnly||e.getAttribute('aria-readonly')==='true')) return {error:'field is read-only'};
522
1140
  if(${forFill} && !('value' in e) && !e.isContentEditable) return {error:'not an editable field (observe again)'};
523
- if(!e.checkVisibility({checkOpacity:true,checkVisibilityCSS:true})) return {error:'element not visible'};
524
- e.scrollIntoView({block:'center',inline:'center'});
525
- var r=e.getBoundingClientRect(); if(!r.width||!r.height) return {error:'element has no size'};
1141
+ // Value-set inputs (date/time/range/color...) take the setter path, not click+type.
1142
+ if(${forFill} && e.tagName==='INPUT' && ['date','time','datetime-local','month','week','color','range'].indexOf(e.type)>=0) return {set:true};
1143
+ // Click the control's visible SURFACE: a styled checkbox's <label>, or the element itself.
1144
+ var s=c.surface?c.surface(e):e;
1145
+ if(!s) return {error:'element not visible'};
1146
+ if(!${noScroll}) s.scrollIntoView({block:'center',inline:'center'});
1147
+ var r=s.getBoundingClientRect(); if(!r.width||!r.height) return {error:'element has no size'};
526
1148
  // Frame-local center (in the element's own frame viewport)...
527
1149
  var lx=r.x+r.width/2, ly=r.y+r.height/2;
528
1150
  // ...plus the offset chain of any ancestor iframes, giving the TOP-LEVEL click point for CDP.
529
- var dx=0, dy=0, w=(e.ownerDocument&&e.ownerDocument.defaultView), g=0;
1151
+ var dx=0, dy=0, w=(s.ownerDocument&&s.ownerDocument.defaultView), g=0;
530
1152
  while(w && w.frameElement && g++<12){
531
1153
  var fe=w.frameElement, fr=fe.getBoundingClientRect(), fcs=fe.ownerDocument.defaultView.getComputedStyle(fe);
532
- dx+=fr.left+(parseFloat(fcs.borderLeftWidth)||0)+(parseFloat(fcs.paddingLeft)||0);
533
- dy+=fr.top+(parseFloat(fcs.borderTopWidth)||0)+(parseFloat(fcs.paddingTop)||0);
1154
+ dx+=fr.left+fe.clientLeft+(parseFloat(fcs.paddingLeft)||0);
1155
+ dy+=fr.top+fe.clientTop+(parseFloat(fcs.paddingTop)||0);
534
1156
  w=fe.ownerDocument.defaultView;
535
1157
  }
536
1158
  var x=Math.round(lx+dx), y=Math.round(ly+dy);
537
1159
  if(x<0||y<0||x>=innerWidth||y>=innerHeight) return {error:'element off-screen after scroll'};
538
- // Hit-test in the element's OWN root (document / shadow root / iframe doc) using frame-local
539
- // coords, so shadow-DOM and iframe elements aren't falsely reported as covered.
540
- var root=e.getRootNode(); var efp=(root&&root.elementFromPoint)?root.elementFromPoint(lx,ly):document.elementFromPoint(lx,ly);
541
- if(!e.contains(efp)) return {error:'element is covered by another element'};
1160
+ // Hit-test in the surface's OWN root (document / shadow root / iframe doc) with frame-local
1161
+ // coords, descending through nested open shadow roots, so shadow-DOM and iframe elements
1162
+ // aren't falsely reported as covered.
1163
+ if(${noHit}) return {x:x, y:y};
1164
+ var root=s.getRootNode(); if(!root||!root.elementFromPoint) root=s.ownerDocument;
1165
+ var f=root.elementFromPoint(lx,ly), k=0;
1166
+ var sroot=function(n){ return n.shadowRoot || (c.closed && c.closed.get(n)) || null; };
1167
+ while(f && sroot(f) && k++<16){ var inner=sroot(f).elementFromPoint(lx,ly); if(!inner||inner===f) break; f=inner; }
1168
+ if(!f || !(s===f || s.contains(f) || (s.control && s.control===f))) return {error:'element is covered by another element (dismiss the overlay/dialog first)'};
542
1169
  return {x:x, y:y};
543
1170
  })()`);
544
1171
  }
545
1172
 
1173
+ // Set the value of a date/time/month/week/color/range input the way a user's picker would: via the
1174
+ // native value setter (so framework value-trackers see a real change), then input + change events.
1175
+ // The browser sanitizes invalid values to '' (or clamps ranges), so we read back and report that.
1176
+ async function setValue(tabId, ref, text) {
1177
+ const R = JSON.stringify(String(ref));
1178
+ const V = JSON.stringify(String(text ?? ''));
1179
+ return evaluate(tabId, `(function(){
1180
+ var c=window.__pawbrowse; var e=c&&c.get&&c.get(${R});
1181
+ if(!e||!e.isConnected) return {error:'element no longer on page (observe again)'};
1182
+ var val=${V};
1183
+ var setter=Object.getOwnPropertyDescriptor(HTMLInputElement.prototype,'value').set;
1184
+ try{ e.focus({preventScroll:true}); }catch(_){}
1185
+ setter.call(e,val);
1186
+ e.dispatchEvent(new Event('input',{bubbles:true,composed:true}));
1187
+ e.dispatchEvent(new Event('change',{bubbles:true}));
1188
+ if(val!=='' && e.value==='') return {error:'value "'+val+'" rejected by the '+e.type+' field (use the fmt{} format shown)'};
1189
+ return {value:e.value};
1190
+ })()`);
1191
+ }
1192
+
1193
+ // Attach local files to an <input type=file> (DOM.setFileInputFiles). Chrome only allows this for
1194
+ // an extension that the user has granted "Allow access to file URLs" in chrome://extensions.
1195
+ async function uploadFiles(tabId, ref, paths) {
1196
+ const R = JSON.stringify(String(ref));
1197
+ const h = await evaluate(tabId, `(function(){
1198
+ var c=window.__pawbrowse; var e=c&&c.get&&c.get(${R});
1199
+ if(e && c.guards && c.guards[${R}]!=null && c.guard(e)!==c.guards[${R}]) return null; // relabelled / reused input
1200
+ return (e&&e.isConnected&&e.tagName==='INPUT'&&e.type==='file'&&!e.disabled)?e:null;
1201
+ })()`, { handle: true });
1202
+ if (!h || !h.objectId) return { error: 'not a file-upload field, or it changed since observe (observe again)' };
1203
+ try {
1204
+ await sendCdp(tabId, 'DOM.setFileInputFiles', { files: paths, objectId: h.objectId }); // tabId: the frame target
1205
+ } catch (e) {
1206
+ return { error: /not allowed/i.test(e.message)
1207
+ ? 'Chrome blocked the upload: enable "Allow access to file URLs" for PawBrowse in chrome://extensions'
1208
+ : e.message };
1209
+ } finally {
1210
+ sendCdp(tabId, 'Runtime.releaseObject', { objectId: h.objectId }).catch(() => {}); // (frame target, see above)
1211
+ }
1212
+ return { ok: true };
1213
+ }
1214
+
546
1215
  // Click the most specific visible element matching text, for custom widgets/menus
547
1216
  // (dropdowns, flair pickers) whose options aren't standard controls in the table.
548
1217
  async function centerOfText(tabId, text) {
@@ -576,17 +1245,163 @@ async function centerOfText(tabId, text) {
576
1245
  })()`);
577
1246
  }
578
1247
 
579
- async function clickAt(tabId, x, y) {
1248
+ async function clickAt(tabId, x, y, opts) {
1249
+ const button = (opts && opts.button) || 'left', count = Math.max(1, Math.min(3, Number(opts && opts.count) || 1));
580
1250
  await sendCdp(tabId, 'Input.dispatchMouseEvent', { type: 'mouseMoved', x, y });
581
- await sendCdp(tabId, 'Input.dispatchMouseEvent', { type: 'mousePressed', x, y, button: 'left', clickCount: 1 });
582
- await sendCdp(tabId, 'Input.dispatchMouseEvent', { type: 'mouseReleased', x, y, button: 'left', clickCount: 1 });
1251
+ // A double/triple click is successive press/release pairs with a rising clickCount.
1252
+ for (let n = 1; n <= count; n++) {
1253
+ await sendCdp(tabId, 'Input.dispatchMouseEvent', { type: 'mousePressed', x, y, button, clickCount: n });
1254
+ await sendCdp(tabId, 'Input.dispatchMouseEvent', { type: 'mouseReleased', x, y, button, clickCount: n });
1255
+ }
583
1256
  }
584
1257
 
585
- // Wait for the page to settle: two animation frames, or up to ms, whichever first.
586
- async function settle(tabId, ms) {
587
- try {
588
- await evaluate(tabId, `new Promise(function(res){var f=0;function step(){if(++f>=2)return res(1);requestAnimationFrame(step);}requestAnimationFrame(step);setTimeout(function(){res(1);}, ${Number(ms) || 300});})`);
589
- } catch { await sleep(Number(ms) || 300); }
1258
+ /* ------------------------------ Waiting (settle) ---------------------------- *
1259
+ * Wait for what the page is ACTUALLY doing instead of a fixed delay: a navigation in progress (until
1260
+ * the new document's DOMContentLoaded), fetch/XHR requests started by the action, then a short
1261
+ * DOM-quiet window (MutationObserver). Each phase is capped, so long-polling, analytics beacons or a
1262
+ * ticking clock can't stall us, and a click that does nothing returns in a few tens of ms. */
1263
+ const watches = new Map(); // tabId -> { mainFrame, inflight: Map(reqId -> startedAt), navStart, navDone }
1264
+ function tabWatch(tabId) {
1265
+ let w = watches.get(tabId);
1266
+ if (!w) { w = { mainFrame: null, inflight: new Map(), navStart: 0, navDone: 0, committed: true, navReq: null }; watches.set(tabId, w); }
1267
+ return w;
1268
+ }
1269
+ const TRACKED = new Set(['Fetch', 'XHR', 'Document']);
1270
+ chrome.debugger.onEvent.addListener((source, method, params) => {
1271
+ if (source.tabId == null) return;
1272
+ const w = watches.get(source.tabId);
1273
+ if (!w) return;
1274
+ // A request can finish on a different session than it started on (an iframe's document request
1275
+ // is announced by the parent, completed by the child): accept completions from any session.
1276
+ if (method === 'Network.loadingFinished' || method === 'Network.loadingFailed') {
1277
+ w.inflight.delete(params.requestId);
1278
+ if (method === 'Network.loadingFailed' && params.requestId === w.navReq && !w.committed) w.navDone = Date.now(); // blocked/aborted navigation
1279
+ return;
1280
+ }
1281
+ if (source.sessionId) return; // everything else: the top-level target only
1282
+ const now = Date.now();
1283
+ const begin = (loaderId) => { w.navStart = now; w.navDone = 0; w.committed = false; w.navReq = loaderId || null; w.netIdle = 0; };
1284
+ switch (method) {
1285
+ case 'Network.requestWillBeSent':
1286
+ // Documents: only the main frame's (subframe documents finish on other sessions / may never).
1287
+ if (TRACKED.has(params.type) && (params.type !== 'Document' || params.frameId === w.mainFrame)) w.inflight.set(params.requestId, now);
1288
+ if (params.type === 'Document' && params.frameId === w.mainFrame && params.requestId === params.loaderId && w.navReq !== params.requestId) {
1289
+ if (w.navDone >= w.navStart) begin(params.requestId); else w.navReq = params.requestId;
1290
+ }
1291
+ break;
1292
+ case 'Network.responseReceived':
1293
+ // A 204/205 navigation never replaces the document: it's over as soon as the response lands.
1294
+ if (params.requestId === w.navReq && (params.response.status === 204 || params.response.status === 205)) w.navDone = now;
1295
+ break;
1296
+ case 'Page.frameRequestedNavigation': case 'Page.frameStartedNavigating':
1297
+ if (params.frameId === w.mainFrame && params.navigationType !== 'sameDocument') begin(params.loaderId);
1298
+ break;
1299
+ case 'Page.frameNavigated':
1300
+ if (!params.frame.parentId) { w.mainFrame = params.frame.id; w.committed = true; if (params.frame.loaderId) w.navReq = params.frame.loaderId; }
1301
+ break;
1302
+ // The OLD document can still fire load/stop events after a navigation starts: only events that
1303
+ // follow the new document's commit (frameNavigated) mean "arrived".
1304
+ case 'Page.domContentEventFired': case 'Page.loadEventFired':
1305
+ if (w.committed) w.navDone = now;
1306
+ break;
1307
+ case 'Page.frameStoppedLoading':
1308
+ if (params.frameId === w.mainFrame && w.committed) w.navDone = now;
1309
+ break;
1310
+ case 'Page.lifecycleEvent':
1311
+ // Tied to the NEW document's loader, so the old page's late events can't end a wait.
1312
+ if (params.frameId === w.mainFrame && params.loaderId && params.loaderId === w.navReq) {
1313
+ if (params.name === 'DOMContentLoaded' || params.name === 'load') { w.committed = true; w.navDone = now; }
1314
+ if (params.name === 'networkAlmostIdle' || params.name === 'networkIdle') w.netIdle = now;
1315
+ }
1316
+ break;
1317
+ case 'Page.downloadWillBegin': case 'Page.navigatedWithinDocument':
1318
+ w.navDone = now;
1319
+ break;
1320
+ }
1321
+ });
1322
+
1323
+ // -> [ms since last DOM change, was the DOM already busy in the 600ms before `since` (epoch ms)?]
1324
+ const quietExpr = (since) => `(function(){ ${MO_INSTALL}
1325
+ var now=performance.now(), s=${Number(since) || 0}-Date.now()+now, b={};
1326
+ M.times.forEach(function(t){ if(t<s && t>=s-600) b[Math.floor((s-t)/100)]=1; });
1327
+ return [now-M.last, Object.keys(b).length>=4];
1328
+ })()`;
1329
+
1330
+ // Network tracking only while an action is being watched (see attach()).
1331
+ async function netOn(tabId) {
1332
+ const w = tabWatch(tabId);
1333
+ if (w.net) return;
1334
+ w.net = true;
1335
+ await sendCdp(tabId, 'Network.enable', { maxTotalBufferSize: 0, maxResourceBufferSize: 0 }).catch(() => { w.net = false; });
1336
+ }
1337
+ async function netOff(tabId) {
1338
+ const w = tabWatch(tabId);
1339
+ if (!w.net) return;
1340
+ w.net = false; w.inflight.clear();
1341
+ await sendCdp(tabId, 'Network.disable').catch(() => {});
1342
+ }
1343
+
1344
+ async function settle(tabId, capMs, since, opts) {
1345
+ const start = since || Date.now();
1346
+ const w = tabWatch(tabId);
1347
+ const deadline = start + (capMs || 3000);
1348
+ // After typing, search boxes commonly DEBOUNCE (wait ~150-300ms of no typing, then fetch): there's
1349
+ // no signal to follow during that gap, so give a typed field a grace window for a request or a
1350
+ // re-render to begin before calling it idle.
1351
+ const grace = start + ((opts && opts.grace) || 0);
1352
+ // Let the action's handlers run first (event loop turn + a frame).
1353
+ await sleep(25);
1354
+ let idleSince = 0;
1355
+ // Which requests are the action's (vs. background beacons/polling that never stop): those that
1356
+ // start within 500ms of the action — or of the new page's DOMContentLoaded — plus requests that
1357
+ // start right after one of those finishes (a fetch chain). Anything else is ignored.
1358
+ const rel = new Set();
1359
+ let windowEnd = start + 500, sawNav = false;
1360
+ for (;;) {
1361
+ const now = Date.now();
1362
+ const navigating = w.navStart >= start - 50 && w.navDone < w.navStart;
1363
+ // A navigation gets a longer allowance: the next page must actually arrive.
1364
+ if (now > (navigating ? Math.max(deadline, start + 10000) : deadline)) break;
1365
+ if (navigating) {
1366
+ // Once the new document has committed, stop network tracking: its (possibly hundreds of)
1367
+ // subresource requests would only slow the browser down. Lifecycle events take over.
1368
+ if (w.committed && w.net) await netOff(tabId);
1369
+ sawNav = true; await sleep(30); continue;
1370
+ }
1371
+ if (sawNav) {
1372
+ sawNav = false; windowEnd = Math.max(windowEnd, w.navDone + 500);
1373
+ if (!w.net) {
1374
+ // New page: let its initial data requests settle (Chrome's networkAlmostIdle), capped.
1375
+ // Ends early once the document is fully loaded and the DOM has been quiet for 300ms.
1376
+ const until = Math.min(deadline, w.navDone + 800);
1377
+ while (!(w.netIdle >= w.navStart) && Date.now() < until) {
1378
+ try {
1379
+ const [q, , rs] = await evaluate(tabId, `(function(){ var r=${quietExpr(start)}; return [r[0], r[1], document.readyState]; })()`);
1380
+ if (rs === 'complete' && q >= 300) break;
1381
+ } catch {}
1382
+ await sleep(40);
1383
+ }
1384
+ }
1385
+ }
1386
+ let busy = false;
1387
+ for (const [id, t] of w.inflight) {
1388
+ if (now - t > 15000) { w.inflight.delete(id); continue; } // leaked / long-poll: forget it
1389
+ if (!rel.has(id) && t >= start - 50 && t <= windowEnd) rel.add(id);
1390
+ }
1391
+ for (const id of rel) {
1392
+ if (!w.inflight.has(id)) { rel.delete(id); windowEnd = Math.max(windowEnd, now + 150); continue; } // finished: allow a follow-up
1393
+ if (now - w.inflight.get(id) < 8000) busy = true;
1394
+ }
1395
+ if (busy) { idleSince = 0; await sleep(30); continue; }
1396
+ if (now < grace) { await sleep(30); continue; } // debounce window: a request may still be coming
1397
+ if (!idleSince) idleSince = now;
1398
+ let quiet = 1e9, ambient = false;
1399
+ try { [quiet, ambient] = await evaluate(tabId, quietExpr(start)); } catch { await sleep(30); continue; } // document swapping
1400
+ // DOM still changing: wait for 60ms of quiet, capped at 600ms after the network went idle, or
1401
+ // 120ms on a page that was already constantly mutating (clocks, tickers, carousels) before us.
1402
+ if (quiet < 60 && now - idleSince < (ambient ? 120 : 600)) { await sleep(Math.max(10, 60 - quiet)); continue; }
1403
+ break;
1404
+ }
590
1405
  }
591
1406
 
592
1407
  // After typing into a combobox, wait for its autocomplete options to actually render
@@ -600,7 +1415,7 @@ async function waitForOptions(tabId, ref, ms) {
600
1415
  await evaluate(tabId, `new Promise(function(res){
601
1416
  var done=false; function fin(){ if(done) return; done=true; try{clearInterval(iv);}catch(_){} res(1); }
602
1417
  setTimeout(fin, ${cap});
603
- var c=window.__pawbrowse; var node=(c&&c.byId)?c.byId[${R}]:null; var e=node!=null?c.nodes.get(node):null;
1418
+ var c=window.__pawbrowse; var e=(c&&c.get)?c.get(${R}):null;
604
1419
  if(!e || (e.getAttribute('role')||'').toLowerCase()!=='combobox'){ return fin(); }
605
1420
  var ids=(e.getAttribute('aria-controls')||e.getAttribute('aria-owns')||'').split(/\\s+/).filter(Boolean);
606
1421
  var iv=setInterval(function(){
@@ -624,15 +1439,202 @@ const KEYMAP = {
624
1439
  ArrowUp: { key: 'ArrowUp', code: 'ArrowUp', windowsVirtualKeyCode: 38 },
625
1440
  ArrowLeft: { key: 'ArrowLeft', code: 'ArrowLeft', windowsVirtualKeyCode: 37 },
626
1441
  ArrowRight: { key: 'ArrowRight', code: 'ArrowRight', windowsVirtualKeyCode: 39 },
1442
+ Space: { key: ' ', code: 'Space', windowsVirtualKeyCode: 32, text: ' ' },
1443
+ Delete: { key: 'Delete', code: 'Delete', windowsVirtualKeyCode: 46 },
1444
+ Home: { key: 'Home', code: 'Home', windowsVirtualKeyCode: 36 },
1445
+ End: { key: 'End', code: 'End', windowsVirtualKeyCode: 35 },
1446
+ PageUp: { key: 'PageUp', code: 'PageUp', windowsVirtualKeyCode: 33 },
1447
+ PageDown: { key: 'PageDown', code: 'PageDown', windowsVirtualKeyCode: 34 },
627
1448
  };
628
1449
 
1450
+ // "Shift+Tab", "Mod+A" (Cmd on macOS, Ctrl elsewhere), "Control+Enter", "F5", "a", "?" ...
1451
+ const MODS = { Alt: 1, Option: 1, Control: 2, Ctrl: 2, Meta: 4, Cmd: 4, Command: 4, Shift: 8, Mod: IS_MAC ? 4 : 2 };
1452
+ // macOS editing shortcuts aren't bound to Cmd+key for synthetic events: they need the command name.
1453
+ const MAC_COMMANDS = { a: 'selectAll', c: 'copy', v: 'paste', x: 'cut', z: 'undo' };
1454
+ function keyDef(name) {
1455
+ if (KEYMAP[name]) return { ...KEYMAP[name] };
1456
+ if (name.length === 1) {
1457
+ const up = name.toUpperCase();
1458
+ const alnum = /[a-z0-9]/i.test(name);
1459
+ return { key: name, code: /[a-z]/i.test(name) ? `Key${up}` : /[0-9]/.test(name) ? `Digit${name}` : '', windowsVirtualKeyCode: alnum ? up.charCodeAt(0) : name.charCodeAt(0), text: name };
1460
+ }
1461
+ const f = /^F(\d{1,2})$/.exec(name);
1462
+ if (f) return { key: name, code: name, windowsVirtualKeyCode: 111 + Number(f[1]) };
1463
+ return null;
1464
+ }
1465
+ async function pressKey(tabId, combo) {
1466
+ const parts = String(combo).split(/\+(?!$)/); // "Control++" -> ["Control", "+"]
1467
+ let mods = 0;
1468
+ for (const m of parts.slice(0, -1)) { if (!(m in MODS)) return `unknown modifier "${m}"`; mods |= MODS[m]; }
1469
+ const def = keyDef(parts[parts.length - 1]);
1470
+ if (!def) return `key "${combo}" not supported`;
1471
+ if ((mods & 8) && def.text && def.text.length === 1) { def.key = def.text = def.text.toUpperCase(); }
1472
+ if (mods & 6) delete def.text; // Ctrl/Cmd chords are shortcuts, not text
1473
+ const commands = IS_MAC && (mods & 4) && MAC_COMMANDS[String(def.key).toLowerCase()] ? [(mods & 8) && def.key.toLowerCase() === 'z' ? 'redo' : MAC_COMMANDS[def.key.toLowerCase()]] : undefined;
1474
+ await sendCdp(tabId, 'Input.dispatchKeyEvent', { type: def.text ? 'keyDown' : 'rawKeyDown', ...def, modifiers: mods, commands });
1475
+ await sendCdp(tabId, 'Input.dispatchKeyEvent', { type: 'keyUp', ...def, text: undefined, modifiers: mods });
1476
+ return null;
1477
+ }
1478
+
1479
+ /* ---------------------------------- WebMCP --------------------------------- *
1480
+ * Pages that implement WebMCP (navigator.modelContext.registerTool / <form toolname>) describe
1481
+ * their own actions with JSON schemas. Calling one is a single deterministic step instead of a
1482
+ * dozen clicks, so observe lists them first and op "tool" invokes them via the WebMCP CDP domain. */
1483
+ const webTools = new Map(); // tabId -> Map(name -> { tool, where })
1484
+ const toolCalls = new Map(); // invocationId -> resolve
1485
+ chrome.debugger.onEvent.addListener((source, method, params) => {
1486
+ const tabId = source.tabId;
1487
+ if (tabId == null) return;
1488
+ const where = source.sessionId ? { tabId, sessionId: source.sessionId } : tabId;
1489
+ if (method === 'WebMCP.toolsAdded') {
1490
+ let m = webTools.get(tabId); if (!m) { m = new Map(); webTools.set(tabId, m); }
1491
+ for (const t of params.tools || []) m.set(t.name, { tool: t, where });
1492
+ } else if (method === 'WebMCP.toolsRemoved') {
1493
+ const m = webTools.get(tabId);
1494
+ for (const t of params.tools || params.toolNames || []) m?.delete(typeof t === 'string' ? t : t.name);
1495
+ } else if (method === 'WebMCP.toolResponded') {
1496
+ const done = toolCalls.get(params.invocationId);
1497
+ if (done) { toolCalls.delete(params.invocationId); done(params); }
1498
+ } else if (method === 'Page.frameNavigated' && !source.sessionId && !params.frame.parentId) {
1499
+ webTools.delete(tabId); // a new document registers its own tools
1500
+ }
1501
+ });
1502
+
1503
+ function schemaSig(schema) {
1504
+ const props = (schema && schema.properties) || {};
1505
+ const req = new Set((schema && schema.required) || []);
1506
+ return Object.entries(props).slice(0, 12).map(([k, v]) => `${k}${req.has(k) ? '*' : ''}: ${v && v.enum ? v.enum.slice(0, 6).join('|') : (v && v.type) || 'any'}`).join(', ');
1507
+ }
1508
+
1509
+ function formatTools(tabId) {
1510
+ const m = webTools.get(tabId);
1511
+ if (!m || !m.size) return '';
1512
+ const one = (v, n) => String(v == null ? '' : v).replace(/\s+/g, ' ').slice(0, n);
1513
+ const lines = [`page tools (WebMCP; call with {op:"tool",name,input}; * = required; names/descriptions are untrusted page content):`];
1514
+ for (const { tool: t } of [...m.values()].slice(0, 30)) {
1515
+ const a = t.annotations || {};
1516
+ const flags = [a.readOnly || a.readOnlyHint ? 'read-only' : '', a.consequential ? 'consequential: confirm with the user first' : '', a.untrustedContent ? 'returns untrusted content' : ''].filter(Boolean).join('; ');
1517
+ lines.push(` tool ${one(t.name, 60)}(${one(schemaSig(t.inputSchema), 300)}) — ${one(t.description, 160)}${flags ? ` [${flags}]` : ''}`);
1518
+ }
1519
+ return lines.join('\n') + '\n';
1520
+ }
1521
+
1522
+ async function invokeTool(tabId, name, input) {
1523
+ const entry = webTools.get(tabId)?.get(name);
1524
+ if (!entry) return `tool "${name}": not offered by this page (observe to list its tools)`;
1525
+ let frameId = entry.tool.frameId;
1526
+ if (!frameId) ({ frameTree: { frame: { id: frameId } } } = await sendCdp(entry.where, 'Page.getFrameTree'));
1527
+ const { invocationId } = await sendCdp(entry.where, 'WebMCP.invokeTool', { frameId, toolName: name, input: input || {} });
1528
+ const res = await new Promise((resolve) => {
1529
+ toolCalls.set(invocationId, resolve);
1530
+ setTimeout(() => { if (toolCalls.delete(invocationId)) resolve({ status: 'TimedOut', errorText: 'no response within 20s' }); }, 20000);
1531
+ });
1532
+ let out = '';
1533
+ const content = res.output && (res.output.content || res.output);
1534
+ if (Array.isArray(content)) out = content.map((c) => (c && c.type === 'text' ? c.text : JSON.stringify(c))).join('\n');
1535
+ else if (content != null) out = typeof content === 'string' ? content : JSON.stringify(content);
1536
+ return `tool ${name}: ${res.status}${res.errorText ? ` (${res.errorText})` : ''}${out ? `\n output (untrusted page data): ${out.slice(0, 2000).replace(/\n/g, '\n ')}` : ''}`;
1537
+ }
1538
+
1539
+ const dragIntercepts = new Map(); // tabId -> drag data captured by Input.dragIntercepted
1540
+ chrome.debugger.onEvent.addListener((source, method, params) => {
1541
+ if (method === 'Input.dragIntercepted' && source.tabId != null) dragIntercepts.set(source.tabId, params.data);
1542
+ });
1543
+
1544
+ async function drag(tabId, from, to) {
1545
+ const mouse = (type, p, extra) => sendCdp(tabId, 'Input.dispatchMouseEvent', { type, x: p.x, y: p.y, button: 'left', ...extra });
1546
+ dragIntercepts.delete(tabId);
1547
+ await sendCdp(tabId, 'Input.setInterceptDrags', { enabled: true }).catch(() => {});
1548
+ try {
1549
+ await mouse('mouseMoved', from);
1550
+ await mouse('mousePressed', from, { clickCount: 1, buttons: 1 });
1551
+ // Move in steps: libraries only start a drag after a few pixels and track intermediate moves.
1552
+ const steps = 8;
1553
+ for (let i = 1; i <= steps; i++) {
1554
+ const p = { x: Math.round(from.x + (to.x - from.x) * i / steps), y: Math.round(from.y + (to.y - from.y) * i / steps) };
1555
+ await mouse('mouseMoved', p, { buttons: 1 });
1556
+ const data = dragIntercepts.get(tabId);
1557
+ if (data) {
1558
+ // Native HTML5 drag started: deliver it to the drop target and finish there.
1559
+ for (const type of ['dragEnter', 'dragOver', 'drop']) await sendCdp(tabId, 'Input.dispatchDragEvent', { type, x: to.x, y: to.y, data });
1560
+ await mouse('mouseReleased', to, { clickCount: 1 });
1561
+ return ' (html5 drop)';
1562
+ }
1563
+ await sleep(16);
1564
+ }
1565
+ await mouse('mouseReleased', to, { clickCount: 1 });
1566
+ return '';
1567
+ } finally {
1568
+ dragIntercepts.delete(tabId);
1569
+ sendCdp(tabId, 'Input.setInterceptDrags', { enabled: false }).catch(() => {});
1570
+ }
1571
+ }
1572
+
629
1573
  async function runOp(tabId, op) {
1574
+ const pol = acting.get(tabId);
1575
+ if (pol) { pol.accept = op.dialog === 'accept' ? true : op.dialog === 'dismiss' ? false : undefined; pol.text = op.dialog_text; }
1576
+ // Route the ref to its frame: T is where page-side code runs, R the frame-local ref; top() turns
1577
+ // frame-local coordinates into top-level ones for input (which the browser routes to the frame).
1578
+ const rt = op.ref != null ? routeRef(tabId, op.ref) : { target: tabId, ref: op.ref, frame: null };
1579
+ if (op.ref != null && !rt.target) return `${op.ref}: unknown frame (observe again)`;
1580
+ const T = rt.target, REF = rt.ref;
1581
+ const top = async (x, y) => {
1582
+ if (!rt.frame) return { x, y };
1583
+ const off = await frameOffset(tabId, rt.frame);
1584
+ const p = { x: Math.round(x + off.x), y: Math.round(y + off.y) };
1585
+ // The frame's own hit-test can't see the PARENT page: make sure the point really lands in this
1586
+ // frame and not on a cookie banner / modal of the page around it.
1587
+ try {
1588
+ const hit = await sendCdp(tabId, 'DOM.getNodeForLocation', { x: p.x, y: p.y, includeUserAgentShadowDOM: false });
1589
+ const main = tabWatch(tabId).mainFrame;
1590
+ if (off.topOwner != null && hit.backendNodeId !== off.topOwner && (!hit.frameId || hit.frameId === main)) p.covered = true;
1591
+ } catch {}
1592
+ return p;
1593
+ };
630
1594
  switch (op.op) {
1595
+ case 'dialog': {
1596
+ const d = openDialogs.get(tabId);
1597
+ if (!d) return 'dialog: no dialog is open';
1598
+ await sendCdp(d.where || tabId, 'Page.handleJavaScriptDialog', { accept: !!op.accept, promptText: op.text != null ? String(op.text) : '' });
1599
+ openDialogs.delete(tabId);
1600
+ return `dialog: ${d.type} "${String(d.message || '').slice(0, 120)}" → ${op.accept ? 'accepted' : 'dismissed'}`;
1601
+ }
631
1602
  case 'click': {
632
- const r = await resolveHit(tabId, op.ref);
1603
+ const r = await resolveHit(T, REF);
633
1604
  if (r.error) return `${op.ref}: ${r.error}`;
634
- await clickAt(tabId, r.x, r.y);
635
- return `click ${op.ref}`;
1605
+ const p = await top(r.x, r.y);
1606
+ if (p.covered) return `${op.ref}: element is covered by another element of the page around its frame (dismiss the overlay first)`;
1607
+ const button = ['right', 'middle'].includes(op.button) ? op.button : 'left';
1608
+ await clickAt(tabId, p.x, p.y, { button, count: op.count });
1609
+ return `${op.count > 1 ? `${op.count}x ` : ''}${button !== 'left' ? `${button}-` : ''}click ${op.ref}`;
1610
+ }
1611
+ case 'hover': {
1612
+ const r = await resolveHit(T, REF);
1613
+ if (r.error) return `${op.ref}: ${r.error}`;
1614
+ const p = await top(r.x, r.y);
1615
+ await sendCdp(tabId, 'Input.dispatchMouseEvent', { type: 'mouseMoved', x: p.x, y: p.y });
1616
+ return `hover ${op.ref}`;
1617
+ }
1618
+ case 'drag': {
1619
+ // Drag ref onto to:"eN" (or by dx/dy pixels). Pointer-driven widgets (sliders, sortable
1620
+ // lists) get a real press-move-release; native HTML5 drag-and-drop is intercepted by CDP and
1621
+ // replayed as dragEnter/dragOver/drop on the target.
1622
+ const a = await resolveHit(T, REF);
1623
+ if (a.error) return `${op.ref}: ${a.error}`;
1624
+ const from = await top(a.x, a.y);
1625
+ let to;
1626
+ if (op.to_text) {
1627
+ const c = await centerOfText(tabId, op.to_text);
1628
+ if (!c) return `drag: drop target "${op.to_text}" not found`;
1629
+ to = c;
1630
+ } else if (op.to) {
1631
+ const rt2 = routeRef(tabId, op.to);
1632
+ if (!rt2.target) return `${op.to}: unknown frame (observe again)`;
1633
+ const b = await resolveHit(rt2.target, rt2.ref, { noScroll: true, noHit: true });
1634
+ if (b.error) return `${op.to}: ${b.error}`;
1635
+ to = rt2.frame ? await (async () => { const off = await frameOffset(tabId, rt2.frame); return { x: Math.round(b.x + off.x), y: Math.round(b.y + off.y) }; })() : { x: b.x, y: b.y };
1636
+ } else to = { x: from.x + Number(op.dx || 0), y: from.y + Number(op.dy || 0) };
1637
+ return drag(tabId, from, to).then((how) => `drag ${op.ref} → ${op.to || (op.to_text ? `"${op.to_text}"` : `${op.dx || 0},${op.dy || 0}`)}${how}`);
636
1638
  }
637
1639
  case 'click_text': {
638
1640
  const c = await centerOfText(tabId, op.text);
@@ -641,9 +1643,15 @@ async function runOp(tabId, op) {
641
1643
  return `click_text "${op.text}"`;
642
1644
  }
643
1645
  case 'type': {
644
- const r = await resolveHit(tabId, op.ref, { fill: true });
1646
+ const r = await resolveHit(T, REF, { fill: true });
645
1647
  if (r.error) return `${op.ref}: ${r.error}`;
646
- await clickAt(tabId, r.x, r.y); // focus the field with a trusted click
1648
+ if (r.set) {
1649
+ const sv = await setValue(T, REF, op.text);
1650
+ return sv.error ? `${op.ref}: ${sv.error}` : `type ${op.ref} (set to "${sv.value}")`;
1651
+ }
1652
+ const p = await top(r.x, r.y);
1653
+ if (p.covered) return `${op.ref}: element is covered by another element of the page around its frame (dismiss the overlay first)`;
1654
+ await clickAt(tabId, p.x, p.y); // focus the field with a trusted click
647
1655
  // Select-all then insert — robust for React/controlled inputs.
648
1656
  await sendCdp(tabId, 'Input.dispatchKeyEvent', { type: 'keyDown', key: 'a', code: 'KeyA', modifiers: IS_MAC ? 4 : 2, commands: ['selectAll'] });
649
1657
  await sendCdp(tabId, 'Input.dispatchKeyEvent', { type: 'keyUp', key: 'a', code: 'KeyA', modifiers: IS_MAC ? 4 : 2 });
@@ -655,23 +1663,25 @@ async function runOp(tabId, op) {
655
1663
  } else {
656
1664
  await sendCdp(tabId, 'Input.insertText', { text: txt });
657
1665
  }
658
- await waitForOptions(tabId, op.ref, 250); // let autocomplete suggestions render
1666
+ await waitForOptions(T, REF, 250); // let autocomplete suggestions render
659
1667
  return `type ${op.ref}`;
660
1668
  }
661
1669
  case 'select': {
662
- const R = JSON.stringify(String(op.ref));
663
- const V = JSON.stringify(String(op.value ?? ''));
1670
+ const R = JSON.stringify(String(REF));
1671
+ const VS = JSON.stringify([].concat(op.values ?? op.value ?? '').map(String));
664
1672
  try {
665
- const res = await evaluate(tabId, `(function(){
666
- var c=window.__pawbrowse; var node=(c&&c.byId)?c.byId[${R}]:null;
667
- var e=node!=null?c.nodes.get(node):null;
1673
+ const res = await evaluate(T, `(function(){
1674
+ var c=window.__pawbrowse; var e=(c&&c.get)?c.get(${R}):null;
668
1675
  if(!e||!e.isConnected) return 'unknown ref (observe again)';
1676
+ if(c.guards && c.guards[${R}]!=null && c.guard(e)!==c.guards[${R}]) return 'element changed since observe (observe again)';
669
1677
  if(e.tagName!=='SELECT') return 'not a dropdown';
670
1678
  if(e.matches(':disabled')||e.closest('[aria-disabled="true"],[inert]')) return 'dropdown is disabled';
671
1679
  if(!e.checkVisibility({checkOpacity:true,checkVisibilityCSS:true})) return 'dropdown not visible';
672
- var val=${V}, m=false;
673
- for(var i=0;i<e.options.length;i++){var o=e.options[i]; if(!o.disabled && !(o.closest&&o.closest('optgroup[disabled]')) && (o.value===val||o.label===val||o.text===val)){e.selectedIndex=i;m=true;break;}}
674
- if(!m) return 'option not found';
1680
+ var vals=${VS}, m=0, ok=function(o){ return !o.disabled && !(o.closest&&o.closest('optgroup[disabled]')); };
1681
+ var hit=function(o){ return vals.some(function(v){ return o.value===v||o.label===v||o.text===v; }); };
1682
+ if(e.multiple){ for(var i=0;i<e.options.length;i++){ var o=e.options[i]; var want=ok(o)&&hit(o); if(want) m++; o.selected=want; } }
1683
+ else for(var j=0;j<e.options.length;j++){ if(ok(e.options[j]) && hit(e.options[j])){ e.selectedIndex=j; m=1; break; } }
1684
+ if(m<vals.length) return m ? 'some options not found' : 'option not found';
675
1685
  e.dispatchEvent(new Event('input',{bubbles:true})); e.dispatchEvent(new Event('change',{bubbles:true}));
676
1686
  return 'ok';
677
1687
  })()`);
@@ -681,19 +1691,60 @@ async function runOp(tabId, op) {
681
1691
  return `select ${op.ref}: may have applied and navigated the page; observe again before retrying`;
682
1692
  }
683
1693
  }
1694
+ case 'upload': {
1695
+ const paths = [].concat(op.paths ?? op.path ?? []).map(String).filter(Boolean);
1696
+ if (!paths.length) return `${op.ref}: upload needs paths:["/absolute/file"]`;
1697
+ const u = await uploadFiles(T, REF, paths);
1698
+ return u.error ? `${op.ref}: ${u.error}` : `upload ${op.ref} (${paths.length} file${paths.length > 1 ? 's' : ''})`;
1699
+ }
1700
+ case 'click_xy': {
1701
+ // Coordinates from the last browser_screenshot image (converted to CSS px).
1702
+ const k = shotScale.get(tabId) || 1;
1703
+ const x = Math.round(Number(op.x) * k), y = Math.round(Number(op.y) * k);
1704
+ if (!Number.isFinite(x) || !Number.isFinite(y)) return 'click_xy needs numeric x and y';
1705
+ const button = ['right', 'middle'].includes(op.button) ? op.button : 'left';
1706
+ await clickAt(tabId, x, y, { button, count: op.count });
1707
+ return `click_xy ${op.x},${op.y}`;
1708
+ }
1709
+ case 'tool': {
1710
+ return invokeTool(tabId, String(op.name || ''), op.input);
1711
+ }
684
1712
  case 'key': {
685
- const k = KEYMAP[op.key];
686
- if (!k) return `key "${op.key}" not supported`;
687
- await sendCdp(tabId, 'Input.dispatchKeyEvent', { type: 'keyDown', ...k });
688
- await sendCdp(tabId, 'Input.dispatchKeyEvent', { type: 'keyUp', ...k });
689
- return `key ${op.key}`;
1713
+ const err = await pressKey(tabId, op.key);
1714
+ return err || `key ${op.key}`;
690
1715
  }
691
1716
  case 'scroll': {
692
1717
  const dy = Number(op.dy ?? 600);
693
- // Real wheel event so overflow containers, virtualized lists, and infinite scroll fire.
1718
+ // Real wheel event so overflow containers, virtualized lists, and infinite scroll fire. With a
1719
+ // ref, the wheel goes to THAT element (a side panel, a dropdown list, a chat pane) instead of
1720
+ // the middle of the page, so the right box scrolls.
694
1721
  let cx = 400, cy = 400;
695
- try { const c = await evaluate(tabId, '[Math.round(innerWidth/2),Math.round(innerHeight/2)]'); if (Array.isArray(c)) { cx = c[0]; cy = c[1]; } } catch {}
1722
+ try {
1723
+ const R = JSON.stringify(String(REF || ''));
1724
+ const c = await evaluate(T, `(function(){
1725
+ var c=window.__pawbrowse, e=${R}&&c&&c.get&&c.get(${R});
1726
+ if(e&&e.isConnected){ var s=c.surface?c.surface(e):e; if(s){ var r=s.getBoundingClientRect(), x=r.x+r.width/2, y=r.y+r.height/2, w=s.ownerDocument.defaultView;
1727
+ while(w&&w.frameElement){ var fr=w.frameElement.getBoundingClientRect(); x+=fr.left+w.frameElement.clientLeft; y+=fr.top+w.frameElement.clientTop; w=w.frameElement.ownerDocument.defaultView; }
1728
+ if(x>=0&&y>=0&&x<innerWidth&&y<innerHeight) return [Math.round(x),Math.round(y)]; } }
1729
+ return [Math.round(innerWidth/2),Math.round(innerHeight/2)];
1730
+ })()`);
1731
+ if (Array.isArray(c)) ({ x: cx, y: cy } = await top(c[0], c[1]));
1732
+ } catch {}
1733
+ // Scroll offsets of the page and every scroll container under the point, as one signature.
1734
+ const probe = rt.frame ? null : `(function(){ var s=[scrollX,scrollY], e=document.elementFromPoint(${cx},${cy}); for(var g=0;e&&g<60;g++){ if(e.scrollHeight>e.clientHeight||e.scrollWidth>e.clientWidth) s.push(e.scrollTop,e.scrollLeft); e=e.parentElement||(e.getRootNode&&e.getRootNode().host); } return s.join(','); })()`;
1735
+ const before = probe ? await evaluate(tabId, probe).catch(() => null) : null;
696
1736
  await sendCdp(tabId, 'Input.dispatchMouseEvent', { type: 'mouseWheel', x: cx, y: cy, deltaX: 0, deltaY: dy });
1737
+ // Wheel scrolling is often ANIMATED (smooth scrolling, e.g. on Linux): wait until the offsets
1738
+ // stop moving, so the table we return shows where the page actually ended up.
1739
+ if (probe) {
1740
+ let last = before, stable = 0;
1741
+ for (const end = Date.now() + 1500; Date.now() < end;) {
1742
+ await sleep(30);
1743
+ const now = await evaluate(tabId, probe).catch(() => null);
1744
+ if (now !== last) { last = now; stable = 0; continue; }
1745
+ if (++stable >= 3 && (now !== before || Date.now() > end - 1200)) break; // settled (or nothing scrolls here)
1746
+ }
1747
+ }
697
1748
  return `scroll ${dy}`;
698
1749
  }
699
1750
  case 'wait': {
@@ -705,6 +1756,50 @@ async function runOp(tabId, op) {
705
1756
  }
706
1757
  }
707
1758
 
1759
+ /* -------------------------------- Screenshots ------------------------------ *
1760
+ * For what the element table can't express — canvas apps (Docs/Sheets/Figma/maps), charts, visual
1761
+ * state. The image is in CSS pixels (so it maps 1:1 to click_xy on any display density), capped at
1762
+ * 1568px on the long side, and, where OffscreenCanvas exists, labelled with the table's refs. */
1763
+ const shotScale = new Map(); // tabId -> CSS px per image px of the last screenshot
1764
+
1765
+ async function screenshot(tabId, opts) {
1766
+ const [dpr, vw, vh] = await evaluate(tabId, '[devicePixelRatio, innerWidth, innerHeight]');
1767
+ const { cssVisualViewport: vv } = await sendCdp(tabId, 'Page.getLayoutMetrics');
1768
+ const fit = Math.min(1, 1568 / Math.max(vw, vh));
1769
+ const shot = await sendCdp(tabId, 'Page.captureScreenshot', {
1770
+ format: 'jpeg', quality: 70,
1771
+ clip: { x: vv.pageX, y: vv.pageY, width: vw, height: vh, scale: fit / dpr },
1772
+ });
1773
+ shotScale.set(tabId, 1 / fit);
1774
+ let data = shot.data, marked = 0;
1775
+ if (opts && opts.marks !== false && typeof OffscreenCanvas !== 'undefined' && typeof createImageBitmap !== 'undefined') {
1776
+ try {
1777
+ const snap = await snapshot(tabId, 1);
1778
+ const frames = snap ? await readFrames(tabId, tabId, snap, []) : [];
1779
+ // Label at each control's top-left corner (not its centre, which would hide its text).
1780
+ const rows = (snap ? snap.actions.filter((a) => !a.off) : []).map((a) => ({ id: a.id, x: a.x - (a.w || 0) / 2, y: a.y - (a.h || 0) / 2 }));
1781
+ for (const { f, off, snap: fs } of frames) for (const a of fs.actions) if (!a.off) rows.push({ id: `f${f.idx}.${a.id}`, x: a.x - (a.w || 0) / 2 + off.x, y: a.y - (a.h || 0) / 2 + off.y });
1782
+ const bytes = Uint8Array.from(atob(data), (c) => c.charCodeAt(0));
1783
+ const bmp = await createImageBitmap(new Blob([bytes], { type: 'image/jpeg' }));
1784
+ const cv = new OffscreenCanvas(bmp.width, bmp.height), g = cv.getContext('2d');
1785
+ g.drawImage(bmp, 0, 0);
1786
+ g.font = 'bold 11px sans-serif'; g.textBaseline = 'top';
1787
+ for (const r of rows) {
1788
+ const w = g.measureText(r.id).width + 4, x = Math.max(0, r.x * fit - 2), y = Math.max(0, r.y * fit - 9);
1789
+ g.fillStyle = 'rgba(220,38,38,.85)'; g.fillRect(x, y, w, 13);
1790
+ g.fillStyle = '#fff'; g.fillText(r.id, x + 2, y + 1);
1791
+ marked++;
1792
+ }
1793
+ const out = await cv.convertToBlob({ type: 'image/jpeg', quality: 0.7 });
1794
+ const buf = new Uint8Array(await out.arrayBuffer());
1795
+ let bin = ''; for (let i = 0; i < buf.length; i += 0x8000) bin += String.fromCharCode.apply(null, buf.subarray(i, i + 0x8000));
1796
+ data = btoa(bin);
1797
+ } catch { /* unlabelled screenshot is still useful */ }
1798
+ }
1799
+ const w = Math.round(vw * fit), h = Math.round(vh * fit);
1800
+ return { data, mimeType: 'image/jpeg', width: w, height: h, note: `screenshot ${w}x${h} of the visible viewport${marked ? `, ${marked} controls labelled with their refs` : ''}. For things not in the element table (canvas apps, maps, charts), act with {op:"click_xy",x,y} using THIS image's pixel coordinates.` };
1801
+ }
1802
+
708
1803
  /* ------------------------------ Command router ---------------------------- */
709
1804
 
710
1805
  async function handleCommand(cmd, args, token, session) {
@@ -738,58 +1833,98 @@ async function handleCommand(cmd, args, token, session) {
738
1833
  if (!/^[a-z][a-z0-9+.-]*:/i.test(url)) url = 'https://' + url; // bare domain -> https
739
1834
  if (!/^https?:\/\//i.test(url)) throw new Error(`navigate only supports http(s) URLs (refusing "${url.split(':')[0]}:")`);
740
1835
  await attach(tabId);
741
- await sendCdp(tabId, 'Page.navigate', { url });
742
- await sleep(350); // let the new document commit before polling, so we don't read the old page
743
- for (let i = 0; i < 75; i++) {
744
- if (aborted()) break;
745
- const rs = await evaluate(tabId, 'document.readyState').catch(() => null);
746
- if (rs === 'complete') break;
747
- await sleep(200);
1836
+ // beforeunload ("Leave site? Changes may not be saved") is dismissed unless dialog:"accept".
1837
+ return whileActing(tabId, async () => {
1838
+ const t0 = Date.now();
1839
+ const w = tabWatch(tabId);
1840
+ w.navStart = t0; w.navDone = 0; w.committed = false; w.navReq = null; // the old document must not count as "arrived"
1841
+ const nav = await sendCdp(tabId, 'Page.navigate', { url });
1842
+ if (nav && nav.loaderId && !w.committed) w.navReq = nav.loaderId;
1843
+ worlds.delete(tabId); // the old document's world is going away with it
1844
+ if (nav && nav.errorText) {
1845
+ // Cancelled by a "leave site?" prompt we dismissed: not a failure — say so, show where we are.
1846
+ const dl0 = takeDialogLog(tabId);
1847
+ if (dl0 && /beforeunload/.test(dl0)) return `${dl0}(navigation cancelled: the page asked to keep unsaved changes)\n\n${await observe(tabId)}`;
1848
+ throw new Error(`navigation failed: ${nav.errorText}`);
748
1849
  }
749
- await settle(tabId, 400);
750
- return observe(tabId);
1850
+ if (nav && !nav.loaderId) w.navDone = Date.now(); // same-document (fragment) navigation
1851
+ await settle(tabId, 5000, t0);
1852
+ const dl = takeDialogLog(tabId);
1853
+ return (dl ? dl + '\n' : '') + await observe(tabId);
1854
+ }, { nav: true, accept: args.dialog === 'accept' ? true : undefined });
751
1855
  }
752
1856
  case 'observe': {
753
1857
  const tabId = await resolveTabId(session, args, 'inspect');
754
1858
  await attach(tabId);
755
- return observe(tabId);
1859
+ assertNoOpenDialog(tabId);
1860
+ return observe(tabId, args);
756
1861
  }
757
1862
  case 'read': {
758
1863
  const tabId = await resolveTabId(session, args, 'inspect');
759
1864
  await attach(tabId);
1865
+ assertNoOpenDialog(tabId);
760
1866
  const max = Math.min(Number(args.max_chars) || 12000, 50000);
761
- const text = await evaluate(tabId, `(function(){var el=document.querySelector('main')||document.body;var t=(el.innerText||'').replace(/\\n{3,}/g,'\\n\\n');return t.slice(0, ${max});})()`);
762
- return `${await evaluate(tabId, 'document.title')} — ${await evaluate(tabId, 'location.href')}\n\n${text}`;
1867
+ const r = await evaluate(tabId, `${READ_TEXT}(${max})`);
1868
+ let text = r.text;
1869
+ // Cross-origin frames the page-side reader can't enter (embedded docs, widgets, checkouts).
1870
+ try {
1871
+ for (const { f } of await readFrames(tabId, tabId, await snapshot(tabId, 1), [])) {
1872
+ if (text.length >= max) break;
1873
+ try {
1874
+ const fr = await evaluate(f.target, `${READ_TEXT}(${max})`);
1875
+ if (fr && fr.text) text += `\n\n[frame f${f.idx}: ${fr.url}]\n${fr.text}`;
1876
+ } catch {}
1877
+ }
1878
+ } catch {}
1879
+ return `${r.title} — ${r.url}\n\n${text.slice(0, max)}`;
763
1880
  }
764
1881
  case 'act': {
765
1882
  const ops = args.ops || [];
766
1883
  if (ops.length > 50) throw new Error('too many ops in one call (max 50); split into smaller batches');
767
1884
  const tabId = await resolveTabId(session, args, 'inspect');
768
1885
  await attach(tabId);
769
- const before = await evaluate(tabId, SIG).catch(() => null);
770
- const logLines = [];
771
- for (const op of ops) {
772
- if (aborted()) { logLines.push(' (aborted: command timed out; remaining ops not run)'); break; }
773
- try { logLines.push(' ' + await runOp(tabId, op)); }
774
- catch (e) { logLines.push(` ${op.op} ${op.ref || ''}: ERROR ${e.message}`); }
775
- await settle(tabId, 250);
776
- }
777
- await settle(tabId, 450);
778
- const after = await evaluate(tabId, SIG).catch(() => null);
779
- const changed = before == null || after == null || before !== after;
780
- const note = changed ? 'page changed' : 'page did NOT change (if you expected an effect, the action may not have worked — try a different target)';
781
- // The ops already executed; a failed post-action read (page navigating) must NOT make
782
- // the caller think they failed and retry them.
783
- let table;
784
- try { table = await observe(tabId); }
785
- catch {
786
- return `ran ${ops.length} op(s) [${note}]:\n${logLines.join('\n')}\n\n(ops executed; the page is navigating and could not be read yet — call browser_observe next. Do NOT re-run these ops.)`;
787
- }
788
- return `ran ${ops.length} op(s) [${note}]:\n${logLines.join('\n')}\n\n${table}`;
1886
+ return whileActing(tabId, async () => {
1887
+ const logLines = [];
1888
+ // Answer a dialog left open from before FIRST: until then the page can't run anything.
1889
+ let i = 0;
1890
+ for (; i < ops.length && ops[i].op === 'dialog'; i++) logLines.push(' ' + await runOp(tabId, ops[i]));
1891
+ assertNoOpenDialog(tabId);
1892
+ const before = await evaluate(tabId, SIG).catch(() => null);
1893
+ await netOn(tabId);
1894
+ try {
1895
+ for (; i < ops.length; i++) {
1896
+ const op = ops[i];
1897
+ if (aborted()) { logLines.push(' (aborted: command timed out; remaining ops not run)'); break; }
1898
+ const t0 = Date.now();
1899
+ try { logLines.push(' ' + await runOp(tabId, op)); }
1900
+ catch (e) { logLines.push(` ${op.op} ${op.ref || ''}: ERROR ${e.message}`); }
1901
+ if (op.op !== 'wait') await settle(tabId, i === ops.length - 1 ? 4000 : 2500, t0, { grace: op.op === 'type' ? 400 : 0 });
1902
+ }
1903
+ } finally { await netOff(tabId); }
1904
+ const after = await evaluate(tabId, SIG).catch(() => null);
1905
+ const seen = lastTable.get(tabId), seenFull = lastFull.get(tabId);
1906
+ // The ops already executed; a failed post-action read (page navigating) must NOT make
1907
+ // the caller think they failed and retry them.
1908
+ let table;
1909
+ try { table = await observe(tabId); } catch { table = null; }
1910
+ const changed = before == null || after == null || before !== after || table == null || seen == null || tableBody(table) !== seen || dialogLog.has(tabId);
1911
+ const note = changed ? 'page changed' : 'page did NOT change (if you expected an effect, the action may not have worked — try a different target)';
1912
+ if (table == null) {
1913
+ return `ran ${ops.length} op(s) [${note}]:\n${logLines.join('\n')}\n${takeDialogLog(tabId)}\n(ops executed; the page is navigating and could not be read yet — call browser_observe next. Do NOT re-run these ops.)`;
1914
+ }
1915
+ return `ran ${ops.length} op(s) [${note}]:\n${logLines.join('\n')}\n${takeDialogLog(tabId)}\n${(changed && deltaTable(seenFull, table)) || table}`;
1916
+ });
1917
+ }
1918
+ case 'screenshot': {
1919
+ const tabId = await resolveTabId(session, args, 'inspect');
1920
+ await attach(tabId);
1921
+ assertNoOpenDialog(tabId);
1922
+ return screenshot(tabId, args);
789
1923
  }
790
1924
  case 'assert': {
791
1925
  const tabId = await resolveTabId(session, args, 'inspect');
792
1926
  await attach(tabId);
1927
+ assertNoOpenDialog(tabId);
793
1928
  if (args.contains != null) {
794
1929
  const ok = await evaluate(tabId, `!!(document.body && document.body.innerText && document.body.innerText.indexOf(${JSON.stringify(args.contains)})>=0)`);
795
1930
  return { pass: !!ok, kind: 'contains', value: args.contains };
@@ -799,7 +1934,8 @@ async function handleCommand(cmd, args, token, session) {
799
1934
  return { pass: String(u).indexOf(args.url_includes) >= 0, kind: 'url_includes', url: u };
800
1935
  }
801
1936
  if (args.ref_visible != null) {
802
- const r = await resolveHit(tabId, args.ref_visible);
1937
+ const rt = routeRef(tabId, args.ref_visible);
1938
+ const r = rt.target ? await resolveHit(rt.target, rt.ref, { noScroll: true }) : { error: 'unknown frame (observe again)' };
803
1939
  return { pass: !r.error, kind: 'ref_visible', ref: args.ref_visible, note: r.error };
804
1940
  }
805
1941
  return { pass: false, error: 'provide one of: contains, url_includes, ref_visible' };