teamshare-bridge 0.21.24 → 0.21.26
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cli/commands/browse.js +315 -137
- package/dist/cli/commands/browse.js.map +1 -1
- package/dist/lib/browser/cdp.d.ts +29 -2
- package/dist/lib/browser/cdp.js +133 -6
- package/dist/lib/browser/cdp.js.map +1 -1
- package/dist/lib/browser/human-input-guard.d.ts +59 -0
- package/dist/lib/browser/human-input-guard.js +330 -0
- package/dist/lib/browser/human-input-guard.js.map +1 -0
- package/dist/lib/browser/personal-fields.d.ts +78 -0
- package/dist/lib/browser/personal-fields.js +267 -0
- package/dist/lib/browser/personal-fields.js.map +1 -0
- package/package.json +3 -2
|
@@ -119,6 +119,7 @@ const SYSTEM_PROMPT = `You are a browser automation agent. A screenshot and a gr
|
|
|
119
119
|
Your task: identify unanswered questions and answer them by clicking the correct elements.
|
|
120
120
|
|
|
121
121
|
RULES (STRICT):
|
|
122
|
+
0. BE FAST AND DECISIVE. Answer in a single pass with minimal reasoning — do NOT deliberate at length, re-check repeatedly, or restate the page. Emit the final JSON array as your primary output. Speed matters more than exhaustive deliberation.
|
|
122
123
|
1. NEVER click Next, Submit, Back, Save, Continue, or any navigation. ONLY answer questions on the current screen.
|
|
123
124
|
2. NEVER ask for help. If uncertain, pick the BEST GUESS. Always decide — never wait.
|
|
124
125
|
3. Handle MULTIPLE unanswered questions in a SINGLE response.
|
|
@@ -257,6 +258,133 @@ RESPONSE FORMAT RULES (CRITICAL — follow exactly):
|
|
|
257
258
|
- RIGHT: {"text":"say \\"hello\\""}
|
|
258
259
|
- NEVER put raw newlines or unescaped quotes inside the text string.
|
|
259
260
|
- If you need to explain your reasoning, use the "reason" field, NOT the "text" field.`;
|
|
261
|
+
/**
|
|
262
|
+
* Enable the Page domain and install the per-tab instrumentation the loop
|
|
263
|
+
* relies on: the stealth console wrapper, the Page.frameNavigated listener and
|
|
264
|
+
* the debounced MutationObserver. Extracted so every (re)attach - including a
|
|
265
|
+
* tab switch/close - re-runs it on the NEW tab. All of it must be idempotent
|
|
266
|
+
* (the injected scripts already early-return when their Symbol key exists).
|
|
267
|
+
*/
|
|
268
|
+
async function instrumentPage(page) {
|
|
269
|
+
try {
|
|
270
|
+
await cdpSend(page, 'Page.enable', {});
|
|
271
|
+
console.log('[browse] Page domain enabled');
|
|
272
|
+
}
|
|
273
|
+
catch (err) {
|
|
274
|
+
console.error(`[browse] Page.enable failed: ${(0, config_1.msg)(err)}`);
|
|
275
|
+
}
|
|
276
|
+
// Stealth: install stack trace sanitization to hide CDP frames (global, once)
|
|
277
|
+
installStackTraceStealth();
|
|
278
|
+
// Stealth: suppress console serialization leak caused by CDP's Runtime.enable.
|
|
279
|
+
// Anti-bot probe: Object.defineProperty(obj,'x',{get(){window.__detected=1}});
|
|
280
|
+
// console.debug(obj); if(window.__detected) → CDP is serializing args
|
|
281
|
+
// Countermeasure: wrap console methods to prevent deep serialization.
|
|
282
|
+
try {
|
|
283
|
+
await cdpEval(page, `
|
|
284
|
+
(() => {
|
|
285
|
+
const CK = Symbol('__ts_cp');
|
|
286
|
+
if (window[CK]) return;
|
|
287
|
+
const wrap = (fn) => function(...args) {
|
|
288
|
+
return fn.apply(console, args.map(a => {
|
|
289
|
+
if (a && typeof a === 'object' && typeof a !== 'function') {
|
|
290
|
+
try { return Object.assign(Array.isArray(a) ? [] : {}, a); } catch { return a; }
|
|
291
|
+
}
|
|
292
|
+
return a;
|
|
293
|
+
}));
|
|
294
|
+
};
|
|
295
|
+
console.debug = wrap(console.debug);
|
|
296
|
+
console.log = wrap(console.log);
|
|
297
|
+
console.warn = wrap(console.warn);
|
|
298
|
+
console.error = wrap(console.error);
|
|
299
|
+
window[CK] = true;
|
|
300
|
+
})()
|
|
301
|
+
`);
|
|
302
|
+
}
|
|
303
|
+
catch { /* non-critical */ }
|
|
304
|
+
// Register CDP event: Page.frameNavigated (URL/navigation changes)
|
|
305
|
+
onCdpEvent(page, 'Page.frameNavigated', () => {
|
|
306
|
+
page._cdpNavigated = Date.now();
|
|
307
|
+
});
|
|
308
|
+
// Inject MutationObserver: watches DOM for significant content changes
|
|
309
|
+
// Uses same fingerprint comparison as Node.js side to prevent false positives.
|
|
310
|
+
// Filters out: timers, animations, scripts, styles, notifications, toasts.
|
|
311
|
+
// Debounces mutations (500ms) to batch rapid changes.
|
|
312
|
+
// Stealth: uses Symbol-keyed property (invisible to Object.keys / anti-bot scans).
|
|
313
|
+
try {
|
|
314
|
+
await cdpEval(page, `
|
|
315
|
+
(() => {
|
|
316
|
+
const KEY = Symbol('__ts_obs');
|
|
317
|
+
if (window[KEY]) return 'already attached';
|
|
318
|
+
const IGNORE = /timer|countdown|elapsed|clock|animation|animate|transition|spinner|progress|loading|toast|notification|alert|banner|snackbar|cookie|consent|modal|overlay|popup|tooltip|badge|tag|avatar|icon|svg|img|video|audio|canvas|font|stylesheet|script|meta|link|head/i;
|
|
319
|
+
let timer = null;
|
|
320
|
+
let lastText = '';
|
|
321
|
+
function getText() {
|
|
322
|
+
const els = document.querySelectorAll('input[type="radio"],input[type="checkbox"],input[type="text"],input[type="email"],input[type="password"],textarea,label,button');
|
|
323
|
+
return [...els].map(e => {
|
|
324
|
+
const t = (e.innerText || e.textContent || '').trim().replace(/\\s+/g, ' ').slice(0, 200);
|
|
325
|
+
return e.type + '|' + t + '|' + (e.checked || false);
|
|
326
|
+
}).join('\\n');
|
|
327
|
+
}
|
|
328
|
+
function check() {
|
|
329
|
+
try {
|
|
330
|
+
const curr = getText();
|
|
331
|
+
if (curr === lastText) return;
|
|
332
|
+
const prev = lastText;
|
|
333
|
+
lastText = curr;
|
|
334
|
+
const prevGroups = prev ? prev.split('\\n').filter(Boolean) : [];
|
|
335
|
+
const currGroups = curr.split('\\n').filter(Boolean);
|
|
336
|
+
const prevSet = new Set(prevGroups);
|
|
337
|
+
const currSet = new Set(currGroups);
|
|
338
|
+
let removed = 0, changed = 0;
|
|
339
|
+
for (const g of prevGroups) {
|
|
340
|
+
if (!currSet.has(g)) {
|
|
341
|
+
if (prevGroups.indexOf(g) < currGroups.length) changed++;
|
|
342
|
+
else removed++;
|
|
343
|
+
}
|
|
344
|
+
}
|
|
345
|
+
const added = currGroups.filter(g => !prevSet.has(g)).length;
|
|
346
|
+
const score = added * 10 + removed * 10 + changed * 8;
|
|
347
|
+
if (score >= 15) {
|
|
348
|
+
window[KEY].changed = Date.now();
|
|
349
|
+
}
|
|
350
|
+
} catch {}
|
|
351
|
+
}
|
|
352
|
+
const obs = new MutationObserver((mutations) => {
|
|
353
|
+
let hasRelevant = false;
|
|
354
|
+
for (const m of mutations) {
|
|
355
|
+
if (m.target instanceof Element) {
|
|
356
|
+
const tag = m.target.tagName;
|
|
357
|
+
if (tag === 'SCRIPT' || tag === 'STYLE' || tag === 'HEAD') continue;
|
|
358
|
+
const cls = m.target.className || '';
|
|
359
|
+
if (typeof cls === 'string' && IGNORE.test(cls)) continue;
|
|
360
|
+
const id = m.target.id || '';
|
|
361
|
+
if (IGNORE.test(id)) continue;
|
|
362
|
+
if (m.type === 'characterData' && m.target.parentElement) {
|
|
363
|
+
const ptag = m.target.parentElement.tagName;
|
|
364
|
+
if (ptag === 'SCRIPT' || ptag === 'STYLE') continue;
|
|
365
|
+
}
|
|
366
|
+
}
|
|
367
|
+
hasRelevant = true;
|
|
368
|
+
}
|
|
369
|
+
if (hasRelevant) {
|
|
370
|
+
clearTimeout(timer);
|
|
371
|
+
timer = setTimeout(check, 500);
|
|
372
|
+
}
|
|
373
|
+
});
|
|
374
|
+
obs.observe(document.body || document.documentElement, {
|
|
375
|
+
childList: true, subtree: true, characterData: true
|
|
376
|
+
});
|
|
377
|
+
window[KEY] = { observer: obs, changed: 0 };
|
|
378
|
+
lastText = getText();
|
|
379
|
+
return 'ok';
|
|
380
|
+
})()
|
|
381
|
+
`);
|
|
382
|
+
console.log('[browse] MutationObserver injected (stealth, debounced, filtered)');
|
|
383
|
+
}
|
|
384
|
+
catch (err) {
|
|
385
|
+
console.error(`[browse] MutationObserver injection failed: ${(0, config_1.msg)(err)}`);
|
|
386
|
+
}
|
|
387
|
+
}
|
|
260
388
|
async function cmdBrowse(flags) {
|
|
261
389
|
const autoAnswer = flags.get('auto-answer') === 'true';
|
|
262
390
|
if (!autoAnswer) {
|
|
@@ -316,9 +444,18 @@ async function cmdBrowse(flags) {
|
|
|
316
444
|
const maxActions = continuous ? Number.MAX_SAFE_INTEGER : parseInt(flags.get('max-actions') ?? '50', 10);
|
|
317
445
|
const intervalMs = parseInt(flags.get('interval') ?? '2000', 10);
|
|
318
446
|
const modelSpec = flags.get('model') ?? models_1.DEFAULT_VISION_MODEL;
|
|
319
|
-
|
|
447
|
+
// Target: a decisive answer in well under a minute. 60s is the abandon
|
|
448
|
+
// threshold (overridable); the fast abort + short retry below makes giving up
|
|
449
|
+
// cheap, so we never wait 5 minutes for a stale decision.
|
|
450
|
+
const timeoutSec = Math.min(Math.max(parseInt(flags.get('timeout') ?? '60', 10), 20), 600);
|
|
320
451
|
const timeoutMs = timeoutSec * 1000;
|
|
321
|
-
const quality = Math.min(Math.max(parseInt(flags.get('quality') ?? '
|
|
452
|
+
const quality = Math.min(Math.max(parseInt(flags.get('quality') ?? '70', 10), 50), 100);
|
|
453
|
+
/**
|
|
454
|
+
* Downscale the screenshot when the viewport is bigger than this many px on
|
|
455
|
+
* its longest side (fewer image tokens => faster vision response on big
|
|
456
|
+
* pages; a 1440px page produced 100KB+ images that blew the timeout).
|
|
457
|
+
*/
|
|
458
|
+
const SCREENSHOT_MAX_EDGE = 1024;
|
|
322
459
|
const maxConsecutiveFailures = parseInt(flags.get('max-consecutive-failures') ?? '0', 10); // 0 = unlimited
|
|
323
460
|
const charDelayMs = Math.min(Math.max(parseInt(flags.get('char-delay') ?? '150', 10), 5), 200);
|
|
324
461
|
const optimize = flags.has('optimize');
|
|
@@ -400,125 +537,9 @@ async function cmdBrowse(flags) {
|
|
|
400
537
|
process.exitCode = 1;
|
|
401
538
|
return;
|
|
402
539
|
}
|
|
403
|
-
// Enable Page
|
|
404
|
-
|
|
405
|
-
|
|
406
|
-
console.log('[browse] Page domain enabled');
|
|
407
|
-
}
|
|
408
|
-
catch (err) {
|
|
409
|
-
console.error(`[browse] Page.enable failed: ${(0, config_1.msg)(err)}`);
|
|
410
|
-
}
|
|
411
|
-
// Stealth: install stack trace sanitization to hide CDP frames
|
|
412
|
-
installStackTraceStealth();
|
|
413
|
-
// Stealth: suppress console serialization leak caused by CDP's Runtime.enable.
|
|
414
|
-
// Anti-bot probe: Object.defineProperty(obj,'x',{get(){window.__detected=1}});
|
|
415
|
-
// console.debug(obj); if(window.__detected) → CDP is serializing args
|
|
416
|
-
// Countermeasure: wrap console methods to prevent deep serialization.
|
|
417
|
-
try {
|
|
418
|
-
await cdpEval(page, `
|
|
419
|
-
(() => {
|
|
420
|
-
const CK = Symbol('__ts_cp');
|
|
421
|
-
if (window[CK]) return;
|
|
422
|
-
const wrap = (fn) => function(...args) {
|
|
423
|
-
return fn.apply(console, args.map(a => {
|
|
424
|
-
if (a && typeof a === 'object' && typeof a !== 'function') {
|
|
425
|
-
try { return Object.assign(Array.isArray(a) ? [] : {}, a); } catch { return a; }
|
|
426
|
-
}
|
|
427
|
-
return a;
|
|
428
|
-
}));
|
|
429
|
-
};
|
|
430
|
-
console.debug = wrap(console.debug);
|
|
431
|
-
console.log = wrap(console.log);
|
|
432
|
-
console.warn = wrap(console.warn);
|
|
433
|
-
console.error = wrap(console.error);
|
|
434
|
-
window[CK] = true;
|
|
435
|
-
})()
|
|
436
|
-
`);
|
|
437
|
-
}
|
|
438
|
-
catch { /* non-critical */ }
|
|
439
|
-
// Register CDP event: Page.frameNavigated (URL/navigation changes)
|
|
440
|
-
onCdpEvent(page, 'Page.frameNavigated', () => {
|
|
441
|
-
page._cdpNavigated = Date.now();
|
|
442
|
-
});
|
|
443
|
-
// Inject MutationObserver: watches DOM for significant content changes
|
|
444
|
-
// Uses same fingerprint comparison as Node.js side to prevent false positives.
|
|
445
|
-
// Filters out: timers, animations, scripts, styles, notifications, toasts.
|
|
446
|
-
// Debounces mutations (500ms) to batch rapid changes.
|
|
447
|
-
// Stealth: uses Symbol-keyed property (invisible to Object.keys / anti-bot scans).
|
|
448
|
-
try {
|
|
449
|
-
await cdpEval(page, `
|
|
450
|
-
(() => {
|
|
451
|
-
const KEY = Symbol('__ts_obs');
|
|
452
|
-
if (window[KEY]) return 'already attached';
|
|
453
|
-
const IGNORE = /timer|countdown|elapsed|clock|animation|animate|transition|spinner|progress|loading|toast|notification|alert|banner|snackbar|cookie|consent|modal|overlay|popup|tooltip|badge|tag|avatar|icon|svg|img|video|audio|canvas|font|stylesheet|script|meta|link|head/i;
|
|
454
|
-
let timer = null;
|
|
455
|
-
let lastText = '';
|
|
456
|
-
function getText() {
|
|
457
|
-
const els = document.querySelectorAll('input[type="radio"],input[type="checkbox"],input[type="text"],input[type="email"],input[type="password"],textarea,label,button');
|
|
458
|
-
return [...els].map(e => {
|
|
459
|
-
const t = (e.innerText || e.textContent || '').trim().replace(/\\s+/g, ' ').slice(0, 200);
|
|
460
|
-
return e.type + '|' + t + '|' + (e.checked || false);
|
|
461
|
-
}).join('\\n');
|
|
462
|
-
}
|
|
463
|
-
function check() {
|
|
464
|
-
try {
|
|
465
|
-
const curr = getText();
|
|
466
|
-
if (curr === lastText) return;
|
|
467
|
-
const prev = lastText;
|
|
468
|
-
lastText = curr;
|
|
469
|
-
const prevGroups = prev ? prev.split('\\n').filter(Boolean) : [];
|
|
470
|
-
const currGroups = curr.split('\\n').filter(Boolean);
|
|
471
|
-
const prevSet = new Set(prevGroups);
|
|
472
|
-
const currSet = new Set(currGroups);
|
|
473
|
-
let removed = 0, changed = 0;
|
|
474
|
-
for (const g of prevGroups) {
|
|
475
|
-
if (!currSet.has(g)) {
|
|
476
|
-
if (prevGroups.indexOf(g) < currGroups.length) changed++;
|
|
477
|
-
else removed++;
|
|
478
|
-
}
|
|
479
|
-
}
|
|
480
|
-
const added = currGroups.filter(g => !prevSet.has(g)).length;
|
|
481
|
-
const score = added * 10 + removed * 10 + changed * 8;
|
|
482
|
-
if (score >= 15) {
|
|
483
|
-
window[KEY].changed = Date.now();
|
|
484
|
-
}
|
|
485
|
-
} catch {}
|
|
486
|
-
}
|
|
487
|
-
const obs = new MutationObserver((mutations) => {
|
|
488
|
-
let hasRelevant = false;
|
|
489
|
-
for (const m of mutations) {
|
|
490
|
-
if (m.target instanceof Element) {
|
|
491
|
-
const tag = m.target.tagName;
|
|
492
|
-
if (tag === 'SCRIPT' || tag === 'STYLE' || tag === 'HEAD') continue;
|
|
493
|
-
const cls = m.target.className || '';
|
|
494
|
-
if (typeof cls === 'string' && IGNORE.test(cls)) continue;
|
|
495
|
-
const id = m.target.id || '';
|
|
496
|
-
if (IGNORE.test(id)) continue;
|
|
497
|
-
if (m.type === 'characterData' && m.target.parentElement) {
|
|
498
|
-
const ptag = m.target.parentElement.tagName;
|
|
499
|
-
if (ptag === 'SCRIPT' || ptag === 'STYLE') continue;
|
|
500
|
-
}
|
|
501
|
-
}
|
|
502
|
-
hasRelevant = true;
|
|
503
|
-
}
|
|
504
|
-
if (hasRelevant) {
|
|
505
|
-
clearTimeout(timer);
|
|
506
|
-
timer = setTimeout(check, 500);
|
|
507
|
-
}
|
|
508
|
-
});
|
|
509
|
-
obs.observe(document.body || document.documentElement, {
|
|
510
|
-
childList: true, subtree: true, characterData: true
|
|
511
|
-
});
|
|
512
|
-
window[KEY] = { observer: obs, changed: 0 };
|
|
513
|
-
lastText = getText();
|
|
514
|
-
return 'ok';
|
|
515
|
-
})()
|
|
516
|
-
`);
|
|
517
|
-
console.log('[browse] MutationObserver injected (stealth, debounced, filtered)');
|
|
518
|
-
}
|
|
519
|
-
catch (err) {
|
|
520
|
-
console.error(`[browse] MutationObserver injection failed: ${(0, config_1.msg)(err)}`);
|
|
521
|
-
}
|
|
540
|
+
// Enable Page + install per-tab instrumentation. Re-run on every (re)attach
|
|
541
|
+
// so a tab switch/close leaves the loop fully instrumented.
|
|
542
|
+
await instrumentPage(page);
|
|
522
543
|
// Agent loop
|
|
523
544
|
let actionCount = 0;
|
|
524
545
|
let llmCallCount = 0;
|
|
@@ -530,6 +551,81 @@ async function cmdBrowse(flags) {
|
|
|
530
551
|
/** A 401/403 from the model API: terminal, retrying cannot succeed. */
|
|
531
552
|
let fatalAuthError = false;
|
|
532
553
|
const maxRetries = 5;
|
|
554
|
+
/** How often to re-check which tab is active (ms). */
|
|
555
|
+
const TAB_CHECK_MS = 2000;
|
|
556
|
+
let lastTabCheck = 0;
|
|
557
|
+
let consecutiveScreenshotFailures = 0;
|
|
558
|
+
let reattachFailures = 0;
|
|
559
|
+
// Latency observability: rolling average of vision-call latency plus
|
|
560
|
+
// abort/timeout counts, so the operator can see how slow the model is.
|
|
561
|
+
const llmStats = { calls: 0, ok: 0, screenAborts: 0, timeouts: 0, totalMs: 0 };
|
|
562
|
+
const logLlmStats = () => {
|
|
563
|
+
if (llmStats.calls === 0 || llmStats.calls % 5 !== 0)
|
|
564
|
+
return;
|
|
565
|
+
const avg = Math.round(llmStats.totalMs / llmStats.calls / 1000);
|
|
566
|
+
console.log(`[browse] LLM latency: avg ${avg}s over ${llmStats.calls} call(s) — ${llmStats.ok} ok, ${llmStats.screenAborts} screen-abort, ${llmStats.timeouts} timeout`);
|
|
567
|
+
};
|
|
568
|
+
/**
|
|
569
|
+
* (Re)attach to a tab. A closed or switched tab used to leave the loop
|
|
570
|
+
* hammering a dead target ("CDP timeout: Page.captureScreenshot") forever;
|
|
571
|
+
* now we detect it and follow the tab the human is actually on.
|
|
572
|
+
*/
|
|
573
|
+
async function reattach(reason, preferTargetId) {
|
|
574
|
+
console.log(`[browse] ${reason} - re-attaching...`);
|
|
575
|
+
try {
|
|
576
|
+
page.ws.close();
|
|
577
|
+
}
|
|
578
|
+
catch {
|
|
579
|
+
/* already closed */
|
|
580
|
+
}
|
|
581
|
+
page = await connectToPage(port, preferTargetId);
|
|
582
|
+
await instrumentPage(page);
|
|
583
|
+
// A different tab means different content: reset change trackers.
|
|
584
|
+
lastFingerprint = null;
|
|
585
|
+
lastScreenshot = null;
|
|
586
|
+
codeConfirmedOptimal = false;
|
|
587
|
+
consecutiveFailures = 0;
|
|
588
|
+
consecutiveScreenshotFailures = 0;
|
|
589
|
+
console.log(`[browse] Re-attached to tab: ${page.title} (${page.targetId.slice(0, 8)})`);
|
|
590
|
+
}
|
|
591
|
+
/**
|
|
592
|
+
* Throttled liveness check: is the bound tab still alive and the active one?
|
|
593
|
+
* Follows the human's active tab (and recovers after it is closed). `force`
|
|
594
|
+
* skips the throttle (used from the screenshot-failure path).
|
|
595
|
+
*/
|
|
596
|
+
async function refreshTabIfNeeded(force = false) {
|
|
597
|
+
if (!force && Date.now() - lastTabCheck < TAB_CHECK_MS)
|
|
598
|
+
return false;
|
|
599
|
+
lastTabCheck = Date.now();
|
|
600
|
+
try {
|
|
601
|
+
if (page.disconnected) {
|
|
602
|
+
// The bound tab is gone: move straight to whatever tab is active now.
|
|
603
|
+
const active = await (0, cdp_1.pickActiveTab)(port).catch(() => null);
|
|
604
|
+
await reattach('Bound tab closed', active?.id);
|
|
605
|
+
reattachFailures = 0;
|
|
606
|
+
return true;
|
|
607
|
+
}
|
|
608
|
+
const active = await (0, cdp_1.pickActiveTab)(port, page.targetId);
|
|
609
|
+
if (!active)
|
|
610
|
+
return false;
|
|
611
|
+
if (active.id !== page.targetId) {
|
|
612
|
+
await reattach(`Active tab changed to "${active.title}"`, active.id);
|
|
613
|
+
reattachFailures = 0;
|
|
614
|
+
return true;
|
|
615
|
+
}
|
|
616
|
+
return false;
|
|
617
|
+
}
|
|
618
|
+
catch (err) {
|
|
619
|
+
reattachFailures++;
|
|
620
|
+
console.error(`[browse] Re-attach attempt ${reattachFailures} failed: ${(0, config_1.msg)(err)}`);
|
|
621
|
+
if (reattachFailures >= 6) {
|
|
622
|
+
console.error('[browse] Could not re-attach to any tab - stopping.');
|
|
623
|
+
stopped = true;
|
|
624
|
+
process.exitCode = 1;
|
|
625
|
+
}
|
|
626
|
+
return false;
|
|
627
|
+
}
|
|
628
|
+
}
|
|
533
629
|
process.on('SIGINT', () => {
|
|
534
630
|
console.log('\n[browse] Stopped by user');
|
|
535
631
|
stopped = true;
|
|
@@ -538,17 +634,26 @@ async function cmdBrowse(flags) {
|
|
|
538
634
|
console.log(`[browse] Agent started. ${continuous ? 'Running until Ctrl+C.' : `Max ${maxActions} actions.`} Press Ctrl+C to stop.\n`);
|
|
539
635
|
while (!stopped && actionCount < maxActions) {
|
|
540
636
|
try {
|
|
637
|
+
// Follow the human's active tab: re-attach if the bound tab was closed or
|
|
638
|
+
// the user switched to another one.
|
|
639
|
+
await refreshTabIfNeeded();
|
|
541
640
|
// Take screenshot and extract elements in parallel
|
|
542
641
|
const [screenshot, elements, pageContext] = await Promise.all([
|
|
543
|
-
captureScreenshot(page, quality),
|
|
642
|
+
captureScreenshot(page, quality, SCREENSHOT_MAX_EDGE),
|
|
544
643
|
extractElements(page),
|
|
545
644
|
extractPageContext(page),
|
|
546
645
|
]);
|
|
547
646
|
if (!screenshot) {
|
|
548
|
-
|
|
647
|
+
consecutiveScreenshotFailures++;
|
|
648
|
+
console.error(`[browse] Failed to capture screenshot (${consecutiveScreenshotFailures}), retrying...`);
|
|
649
|
+
// Two failures in a row generally means the target is gone.
|
|
650
|
+
if (consecutiveScreenshotFailures >= 2) {
|
|
651
|
+
await refreshTabIfNeeded(true);
|
|
652
|
+
}
|
|
549
653
|
await sleep(intervalMs);
|
|
550
654
|
continue;
|
|
551
655
|
}
|
|
656
|
+
consecutiveScreenshotFailures = 0;
|
|
552
657
|
// Detect screen change via question fingerprint comparison
|
|
553
658
|
// Threshold 15: requires ≥1 new/removed question (10) + text change (8) = 18,
|
|
554
659
|
// or 2 text changes (16). Score of 5 (single option change) is too sensitive.
|
|
@@ -568,29 +673,47 @@ async function cmdBrowse(flags) {
|
|
|
568
673
|
const actionLabel = continuous ? `${actionCount + 1}/∞` : `${actionCount + 1}/${maxActions}`;
|
|
569
674
|
console.log(`[browse] [${actionLabel}] Analyzing screenshot...`);
|
|
570
675
|
let result = null;
|
|
676
|
+
/** The page changed under the LLM: re-analyze immediately, no backoff. */
|
|
677
|
+
let screenChangedDuringLlm = false;
|
|
571
678
|
for (let attempt = 1; attempt <= maxRetries; attempt++) {
|
|
572
679
|
result = await analyzeScreenshot(baseUrl, apiKey, model, screenshot, elements, pageContext, timeoutMs, attempt, maxRetries, page, lastFingerprint);
|
|
573
680
|
llmCallCount++;
|
|
681
|
+
llmStats.calls++;
|
|
682
|
+
llmStats.totalMs += result.durationMs;
|
|
574
683
|
if (llmCallCount > 50 && llmCallCount % 25 === 0) {
|
|
575
684
|
console.log(`[browse] Warning: ${llmCallCount} LLM calls made — may hit rate limits`);
|
|
576
685
|
}
|
|
577
|
-
if (result.actions.length > 0)
|
|
686
|
+
if (result.actions.length > 0) {
|
|
687
|
+
llmStats.ok++;
|
|
688
|
+
logLlmStats();
|
|
578
689
|
break; // success
|
|
690
|
+
}
|
|
579
691
|
// Auth failure is terminal: retrying cannot help.
|
|
580
692
|
if (result.error === 'auth_error') {
|
|
581
693
|
fatalAuthError = true;
|
|
582
694
|
break;
|
|
583
695
|
}
|
|
584
|
-
//
|
|
696
|
+
// The screen changed while the LLM was thinking: its answer is stale.
|
|
697
|
+
// Bail out of the retry loop and re-analyze the NEW screen right away
|
|
698
|
+
// (speed matters: a 30s backoff here livelocked on dynamic pages).
|
|
699
|
+
if (result.error === 'screen_changed') {
|
|
700
|
+
screenChangedDuringLlm = true;
|
|
701
|
+
llmStats.screenAborts++;
|
|
702
|
+
logLlmStats();
|
|
703
|
+
break;
|
|
704
|
+
}
|
|
705
|
+
// Genuine timeout → SHORT backoff and retry (was 30/60/120/240/300s).
|
|
585
706
|
if (result.error === 'timeout') {
|
|
586
|
-
|
|
707
|
+
llmStats.timeouts++;
|
|
708
|
+
logLlmStats();
|
|
709
|
+
const backoff = Math.min(3 * attempt, 12); // 3, 6, 9, 12, 12
|
|
587
710
|
console.log(`[browse] LLM timed out — retrying in ${backoff}s (attempt ${attempt}/${maxRetries})...`);
|
|
588
711
|
await sleep(backoff * 1000);
|
|
589
712
|
continue;
|
|
590
713
|
}
|
|
591
|
-
// Other errors → short backoff
|
|
714
|
+
// Other errors → very short backoff
|
|
592
715
|
if (attempt < maxRetries) {
|
|
593
|
-
const backoff =
|
|
716
|
+
const backoff = 2;
|
|
594
717
|
console.log(`[browse] ${result.error === 'empty' ? 'Empty response' : 'Parse failed'} (attempt ${attempt}/${maxRetries}), retrying in ${backoff}s...`);
|
|
595
718
|
await sleep(backoff * 1000);
|
|
596
719
|
}
|
|
@@ -601,6 +724,10 @@ async function cmdBrowse(flags) {
|
|
|
601
724
|
process.exitCode = 1;
|
|
602
725
|
break;
|
|
603
726
|
}
|
|
727
|
+
if (screenChangedDuringLlm) {
|
|
728
|
+
// No sleep: go straight back to the top, re-screenshot + re-analyze.
|
|
729
|
+
continue;
|
|
730
|
+
}
|
|
604
731
|
if (!result || result.actions.length === 0) {
|
|
605
732
|
consecutiveFailures++;
|
|
606
733
|
console.log(`[browse] No valid response after ${maxRetries} attempts (${consecutiveFailures} consecutive failures)`);
|
|
@@ -897,11 +1024,11 @@ Do NOT simplify variable names or change the function signature. Only optimize t
|
|
|
897
1024
|
* Connect directly to a page's WebSocket from /json/list.
|
|
898
1025
|
* This avoids the Target.attachToTarget complexity.
|
|
899
1026
|
*/
|
|
900
|
-
async function connectToPage(port) {
|
|
1027
|
+
async function connectToPage(port, preferTargetId) {
|
|
901
1028
|
// Shared picker (cdp.ts): prefers a real page, accepts about:blank/new-tab,
|
|
902
1029
|
// and opens a blank tab if none exists - so a Chrome launched on the NTP is
|
|
903
1030
|
// still attachable. (Was a local chrome:// filter that could never match.)
|
|
904
|
-
const pageTarget = await (0, cdp_1.resolveAttachablePage)(port);
|
|
1031
|
+
const pageTarget = await (0, cdp_1.resolveAttachablePage)(port, preferTargetId);
|
|
905
1032
|
const wsUrl = pageTarget.webSocketDebuggerUrl;
|
|
906
1033
|
return new Promise((resolve, reject) => {
|
|
907
1034
|
const ws = new ws_1.default(wsUrl);
|
|
@@ -958,6 +1085,8 @@ async function connectToPage(port) {
|
|
|
958
1085
|
}
|
|
959
1086
|
});
|
|
960
1087
|
ws.on('close', () => {
|
|
1088
|
+
// The endpoint is gone: flag it so the agent loop can re-attach.
|
|
1089
|
+
page.disconnected = true;
|
|
961
1090
|
// Reject all pending
|
|
962
1091
|
for (const [id, p] of page.pending) {
|
|
963
1092
|
p.reject(new Error('WebSocket closed'));
|
|
@@ -1022,9 +1151,32 @@ function cdpSend(page, method, params) {
|
|
|
1022
1151
|
page.ws.send(JSON.stringify({ id, method, params }));
|
|
1023
1152
|
});
|
|
1024
1153
|
}
|
|
1025
|
-
|
|
1154
|
+
/**
|
|
1155
|
+
* Capture the visible viewport as JPEG. When the viewport is larger than
|
|
1156
|
+
* `maxEdge` on either side the capture is scaled down (`clip.scale`) so the
|
|
1157
|
+
* vision call stays fast: a big page produced 100KB+ images that exceeded the
|
|
1158
|
+
* model timeout. Clicking is unaffected — actions use element coordinates
|
|
1159
|
+
* (CSS px), not image pixels.
|
|
1160
|
+
*/
|
|
1161
|
+
async function captureScreenshot(page, quality, maxEdge = 0) {
|
|
1026
1162
|
try {
|
|
1027
|
-
const
|
|
1163
|
+
const params = { format: 'jpeg', quality };
|
|
1164
|
+
if (maxEdge > 0) {
|
|
1165
|
+
try {
|
|
1166
|
+
const metrics = await cdpSend(page, 'Page.getLayoutMetrics', {});
|
|
1167
|
+
const vp = metrics?.cssLayoutViewport ?? metrics?.layoutViewport;
|
|
1168
|
+
const w = Number(vp?.clientWidth ?? 0);
|
|
1169
|
+
const h = Number(vp?.clientHeight ?? 0);
|
|
1170
|
+
if (w > 0 && h > 0 && (w > maxEdge || h > maxEdge)) {
|
|
1171
|
+
const scale = Math.min(maxEdge / w, maxEdge / h);
|
|
1172
|
+
params.clip = { x: 0, y: 0, width: w, height: h, scale };
|
|
1173
|
+
}
|
|
1174
|
+
}
|
|
1175
|
+
catch {
|
|
1176
|
+
/* metrics unavailable - capture unscaled */
|
|
1177
|
+
}
|
|
1178
|
+
}
|
|
1179
|
+
const result = await cdpSend(page, 'Page.captureScreenshot', params);
|
|
1028
1180
|
return result?.data ?? null;
|
|
1029
1181
|
}
|
|
1030
1182
|
catch (err) {
|
|
@@ -2362,11 +2514,19 @@ async function analyzeScreenshot(baseUrl, apiKey, model, screenshotBase64, eleme
|
|
|
2362
2514
|
},
|
|
2363
2515
|
],
|
|
2364
2516
|
temperature: 0.1,
|
|
2365
|
-
max_tokens:
|
|
2517
|
+
max_tokens: 2048,
|
|
2366
2518
|
stream: false,
|
|
2367
2519
|
};
|
|
2368
2520
|
const controller = new AbortController();
|
|
2369
|
-
|
|
2521
|
+
// Distinguish a genuine timeout from the heartbeat cancelling a stale request
|
|
2522
|
+
// because the page changed: they need opposite handling (backoff vs re-analyze
|
|
2523
|
+
// now). The old code reported both as "timed out (300s)" and backed off 30s+,
|
|
2524
|
+
// which livelocked on dynamic pages.
|
|
2525
|
+
let abortReason = null;
|
|
2526
|
+
const timeout = setTimeout(() => {
|
|
2527
|
+
abortReason = 'timeout';
|
|
2528
|
+
controller.abort();
|
|
2529
|
+
}, timeoutMs);
|
|
2370
2530
|
// Heartbeat: check for screen changes every 5s, abort LLM if page changed
|
|
2371
2531
|
const heartbeat = setInterval(async () => {
|
|
2372
2532
|
const elapsed = Math.round((Date.now() - startTime) / 1000);
|
|
@@ -2412,6 +2572,7 @@ async function analyzeScreenshot(baseUrl, apiKey, model, screenshotBase64, eleme
|
|
|
2412
2572
|
const changeScore = calculateChangeScore(lastFingerprint, freshFingerprint);
|
|
2413
2573
|
if (changeScore >= 15) {
|
|
2414
2574
|
console.log(`[browse] Fingerprint confirms change (score: ${changeScore}) — aborting LLM (${elapsed}s)`);
|
|
2575
|
+
abortReason = 'screen_change';
|
|
2415
2576
|
controller.abort();
|
|
2416
2577
|
return;
|
|
2417
2578
|
}
|
|
@@ -2478,6 +2639,13 @@ async function analyzeScreenshot(baseUrl, apiKey, model, screenshotBase64, eleme
|
|
|
2478
2639
|
}
|
|
2479
2640
|
catch (err) {
|
|
2480
2641
|
if (err instanceof Error && err.name === 'AbortError') {
|
|
2642
|
+
const elapsed = Math.round((Date.now() - startTime) / 1000);
|
|
2643
|
+
if (abortReason === 'screen_change') {
|
|
2644
|
+
// The page changed while we were waiting: the answer is already stale.
|
|
2645
|
+
// Signal the caller to re-analyze right away (no backoff).
|
|
2646
|
+
console.log(`[browse] LLM aborted at ${elapsed}s — screen changed, re-analyzing now`);
|
|
2647
|
+
return { actions: [], attempt, durationMs: Date.now() - startTime, error: 'screen_changed' };
|
|
2648
|
+
}
|
|
2481
2649
|
console.error(`[browse] LLM request timed out (${Math.round(timeoutMs / 1000)}s)`);
|
|
2482
2650
|
return { actions: [], attempt, durationMs: Date.now() - startTime, error: 'timeout' };
|
|
2483
2651
|
}
|
|
@@ -2509,11 +2677,15 @@ async function analyzeScreenshotWithPrompt(baseUrl, apiKey, model, screenshotBas
|
|
|
2509
2677
|
},
|
|
2510
2678
|
],
|
|
2511
2679
|
temperature: 0.1,
|
|
2512
|
-
max_tokens:
|
|
2680
|
+
max_tokens: 2048,
|
|
2513
2681
|
stream: false,
|
|
2514
2682
|
};
|
|
2515
2683
|
const controller = new AbortController();
|
|
2516
|
-
|
|
2684
|
+
let optAbortReason = null;
|
|
2685
|
+
const timeout = setTimeout(() => {
|
|
2686
|
+
optAbortReason = 'timeout';
|
|
2687
|
+
controller.abort();
|
|
2688
|
+
}, timeoutMs);
|
|
2517
2689
|
// Heartbeat: check for screen changes every 5s, abort if page changed
|
|
2518
2690
|
const heartbeat = setInterval(async () => {
|
|
2519
2691
|
const elapsed = Math.round((Date.now() - startTime) / 1000);
|
|
@@ -2530,6 +2702,7 @@ async function analyzeScreenshotWithPrompt(baseUrl, apiKey, model, screenshotBas
|
|
|
2530
2702
|
const flagTs = flagResult?.result?.value;
|
|
2531
2703
|
if (flagTs && flagTs > startTime) {
|
|
2532
2704
|
console.log(`[browse] DOM mutation detected in opt pass (${elapsed}s) — aborting`);
|
|
2705
|
+
optAbortReason = 'screen_change';
|
|
2533
2706
|
controller.abort();
|
|
2534
2707
|
return;
|
|
2535
2708
|
}
|
|
@@ -2573,7 +2746,12 @@ async function analyzeScreenshotWithPrompt(baseUrl, apiKey, model, screenshotBas
|
|
|
2573
2746
|
}
|
|
2574
2747
|
catch (err) {
|
|
2575
2748
|
if (err instanceof Error && err.name === 'AbortError') {
|
|
2576
|
-
return {
|
|
2749
|
+
return {
|
|
2750
|
+
actions: [],
|
|
2751
|
+
attempt,
|
|
2752
|
+
durationMs: Date.now() - startTime,
|
|
2753
|
+
error: optAbortReason === 'screen_change' ? 'screen_changed' : 'timeout',
|
|
2754
|
+
};
|
|
2577
2755
|
}
|
|
2578
2756
|
return { actions: [], attempt, durationMs: Date.now() - startTime, error: 'http_error' };
|
|
2579
2757
|
}
|