teamshare-bridge 0.21.24 → 0.21.27

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -59,6 +59,8 @@ const child_process_1 = require("child_process");
59
59
  const cdp_1 = require("../../lib/browser/cdp");
60
60
  const launch_1 = require("../../lib/browser/launch");
61
61
  const session_1 = require("../../lib/browser/session");
62
+ const personal_fields_1 = require("../../lib/browser/personal-fields");
63
+ const human_input_guard_1 = require("../../lib/browser/human-input-guard");
62
64
  /** Create a fingerprint from the current element list. */
63
65
  function createFingerprint(elements) {
64
66
  const groups = new Map();
@@ -119,6 +121,7 @@ const SYSTEM_PROMPT = `You are a browser automation agent. A screenshot and a gr
119
121
  Your task: identify unanswered questions and answer them by clicking the correct elements.
120
122
 
121
123
  RULES (STRICT):
124
+ 0. BE FAST AND DECISIVE. Answer in a single pass with minimal reasoning — do NOT deliberate at length, re-check repeatedly, or restate the page. Emit the final JSON array as your primary output. Speed matters more than exhaustive deliberation.
122
125
  1. NEVER click Next, Submit, Back, Save, Continue, or any navigation. ONLY answer questions on the current screen.
123
126
  2. NEVER ask for help. If uncertain, pick the BEST GUESS. Always decide — never wait.
124
127
  3. Handle MULTIPLE unanswered questions in a SINGLE response.
@@ -238,10 +241,10 @@ CORRECTION MODE:
238
241
  - If you previously typed wrong code, use write_code to replace with correct code
239
242
  - You may be asked the same question multiple times — each time, check if the current answer is correct before acting
240
243
 
241
- PERSONAL QUESTIONS — SKIP THEM:
242
- - NEVER type answers for questions asking: name, email, phone, address, SSN, password, credit card, date of birth, or any personal information
243
- - If a question asks for personal info, do NOT click or type — skip it entirely
244
- - If ALL remaining questions are personal, return done
244
+ PERSONAL QUESTIONS — NEVER ANSWER THEM:
245
+ - Questions about the person operating the browser (name, email, phone, address, SSN, passport, password, card, bank, date of birth, salary, work authorization, personal profile links) are OFF LIMITS. Read the PERSONAL QUESTIONS addendum appended below and follow it exactly.
246
+ - If a question asks for personal info, do NOT click or type — leave it empty.
247
+ - A personal field left empty is NOT an unanswered question. If ALL remaining questions are personal, return done immediately.
245
248
 
246
249
  RESPONSE FORMAT RULES (CRITICAL — follow exactly):
247
250
  - Return ONLY a valid JSON array. No explanations, no markdown fences, no extra text.
@@ -257,6 +260,133 @@ RESPONSE FORMAT RULES (CRITICAL — follow exactly):
257
260
  - RIGHT: {"text":"say \\"hello\\""}
258
261
  - NEVER put raw newlines or unescaped quotes inside the text string.
259
262
  - If you need to explain your reasoning, use the "reason" field, NOT the "text" field.`;
263
+ /**
264
+ * Enable the Page domain and install the per-tab instrumentation the loop
265
+ * relies on: the stealth console wrapper, the Page.frameNavigated listener and
266
+ * the debounced MutationObserver. Extracted so every (re)attach - including a
267
+ * tab switch/close - re-runs it on the NEW tab. All of it must be idempotent
268
+ * (the injected scripts already early-return when their Symbol key exists).
269
+ */
270
+ async function instrumentPage(page) {
271
+ try {
272
+ await cdpSend(page, 'Page.enable', {});
273
+ console.log('[browse] Page domain enabled');
274
+ }
275
+ catch (err) {
276
+ console.error(`[browse] Page.enable failed: ${(0, config_1.msg)(err)}`);
277
+ }
278
+ // Stealth: install stack trace sanitization to hide CDP frames (global, once)
279
+ installStackTraceStealth();
280
+ // Stealth: suppress console serialization leak caused by CDP's Runtime.enable.
281
+ // Anti-bot probe: Object.defineProperty(obj,'x',{get(){window.__detected=1}});
282
+ // console.debug(obj); if(window.__detected) → CDP is serializing args
283
+ // Countermeasure: wrap console methods to prevent deep serialization.
284
+ try {
285
+ await cdpEval(page, `
286
+ (() => {
287
+ const CK = Symbol('__ts_cp');
288
+ if (window[CK]) return;
289
+ const wrap = (fn) => function(...args) {
290
+ return fn.apply(console, args.map(a => {
291
+ if (a && typeof a === 'object' && typeof a !== 'function') {
292
+ try { return Object.assign(Array.isArray(a) ? [] : {}, a); } catch { return a; }
293
+ }
294
+ return a;
295
+ }));
296
+ };
297
+ console.debug = wrap(console.debug);
298
+ console.log = wrap(console.log);
299
+ console.warn = wrap(console.warn);
300
+ console.error = wrap(console.error);
301
+ window[CK] = true;
302
+ })()
303
+ `);
304
+ }
305
+ catch { /* non-critical */ }
306
+ // Register CDP event: Page.frameNavigated (URL/navigation changes)
307
+ onCdpEvent(page, 'Page.frameNavigated', () => {
308
+ page._cdpNavigated = Date.now();
309
+ });
310
+ // Inject MutationObserver: watches DOM for significant content changes
311
+ // Uses same fingerprint comparison as Node.js side to prevent false positives.
312
+ // Filters out: timers, animations, scripts, styles, notifications, toasts.
313
+ // Debounces mutations (500ms) to batch rapid changes.
314
+ // Stealth: uses Symbol-keyed property (invisible to Object.keys / anti-bot scans).
315
+ try {
316
+ await cdpEval(page, `
317
+ (() => {
318
+ const KEY = Symbol('__ts_obs');
319
+ if (window[KEY]) return 'already attached';
320
+ const IGNORE = /timer|countdown|elapsed|clock|animation|animate|transition|spinner|progress|loading|toast|notification|alert|banner|snackbar|cookie|consent|modal|overlay|popup|tooltip|badge|tag|avatar|icon|svg|img|video|audio|canvas|font|stylesheet|script|meta|link|head/i;
321
+ let timer = null;
322
+ let lastText = '';
323
+ function getText() {
324
+ const els = document.querySelectorAll('input[type="radio"],input[type="checkbox"],input[type="text"],input[type="email"],input[type="password"],textarea,label,button');
325
+ return [...els].map(e => {
326
+ const t = (e.innerText || e.textContent || '').trim().replace(/\\s+/g, ' ').slice(0, 200);
327
+ return e.type + '|' + t + '|' + (e.checked || false);
328
+ }).join('\\n');
329
+ }
330
+ function check() {
331
+ try {
332
+ const curr = getText();
333
+ if (curr === lastText) return;
334
+ const prev = lastText;
335
+ lastText = curr;
336
+ const prevGroups = prev ? prev.split('\\n').filter(Boolean) : [];
337
+ const currGroups = curr.split('\\n').filter(Boolean);
338
+ const prevSet = new Set(prevGroups);
339
+ const currSet = new Set(currGroups);
340
+ let removed = 0, changed = 0;
341
+ for (const g of prevGroups) {
342
+ if (!currSet.has(g)) {
343
+ if (prevGroups.indexOf(g) < currGroups.length) changed++;
344
+ else removed++;
345
+ }
346
+ }
347
+ const added = currGroups.filter(g => !prevSet.has(g)).length;
348
+ const score = added * 10 + removed * 10 + changed * 8;
349
+ if (score >= 15) {
350
+ window[KEY].changed = Date.now();
351
+ }
352
+ } catch {}
353
+ }
354
+ const obs = new MutationObserver((mutations) => {
355
+ let hasRelevant = false;
356
+ for (const m of mutations) {
357
+ if (m.target instanceof Element) {
358
+ const tag = m.target.tagName;
359
+ if (tag === 'SCRIPT' || tag === 'STYLE' || tag === 'HEAD') continue;
360
+ const cls = m.target.className || '';
361
+ if (typeof cls === 'string' && IGNORE.test(cls)) continue;
362
+ const id = m.target.id || '';
363
+ if (IGNORE.test(id)) continue;
364
+ if (m.type === 'characterData' && m.target.parentElement) {
365
+ const ptag = m.target.parentElement.tagName;
366
+ if (ptag === 'SCRIPT' || ptag === 'STYLE') continue;
367
+ }
368
+ }
369
+ hasRelevant = true;
370
+ }
371
+ if (hasRelevant) {
372
+ clearTimeout(timer);
373
+ timer = setTimeout(check, 500);
374
+ }
375
+ });
376
+ obs.observe(document.body || document.documentElement, {
377
+ childList: true, subtree: true, characterData: true
378
+ });
379
+ window[KEY] = { observer: obs, changed: 0 };
380
+ lastText = getText();
381
+ return 'ok';
382
+ })()
383
+ `);
384
+ console.log('[browse] MutationObserver injected (stealth, debounced, filtered)');
385
+ }
386
+ catch (err) {
387
+ console.error(`[browse] MutationObserver injection failed: ${(0, config_1.msg)(err)}`);
388
+ }
389
+ }
260
390
  async function cmdBrowse(flags) {
261
391
  const autoAnswer = flags.get('auto-answer') === 'true';
262
392
  if (!autoAnswer) {
@@ -316,15 +446,33 @@ async function cmdBrowse(flags) {
316
446
  const maxActions = continuous ? Number.MAX_SAFE_INTEGER : parseInt(flags.get('max-actions') ?? '50', 10);
317
447
  const intervalMs = parseInt(flags.get('interval') ?? '2000', 10);
318
448
  const modelSpec = flags.get('model') ?? models_1.DEFAULT_VISION_MODEL;
319
- const timeoutSec = Math.min(Math.max(parseInt(flags.get('timeout') ?? '300', 10), 30), 600);
449
+ // Target: a decisive answer in well under a minute. 60s is the abandon
450
+ // threshold (overridable); the fast abort + short retry below makes giving up
451
+ // cheap, so we never wait 5 minutes for a stale decision.
452
+ const timeoutSec = Math.min(Math.max(parseInt(flags.get('timeout') ?? '60', 10), 20), 600);
320
453
  const timeoutMs = timeoutSec * 1000;
321
- const quality = Math.min(Math.max(parseInt(flags.get('quality') ?? '78', 10), 50), 100);
454
+ const quality = Math.min(Math.max(parseInt(flags.get('quality') ?? '70', 10), 50), 100);
455
+ /**
456
+ * Downscale the screenshot when the viewport is bigger than this many px on
457
+ * its longest side (fewer image tokens => faster vision response on big
458
+ * pages; a 1440px page produced 100KB+ images that blew the timeout).
459
+ */
460
+ const SCREENSHOT_MAX_EDGE = 1024;
322
461
  const maxConsecutiveFailures = parseInt(flags.get('max-consecutive-failures') ?? '0', 10); // 0 = unlimited
323
462
  const charDelayMs = Math.min(Math.max(parseInt(flags.get('char-delay') ?? '150', 10), 5), 200);
324
463
  const optimize = flags.has('optimize');
325
464
  const stealth = flags.has('stealth');
326
465
  const hybrid = flags.has('hybrid');
327
466
  const autoLaunch = flags.has('auto-launch');
467
+ // Field safety (see personal-fields.ts / human-input-guard.ts):
468
+ // --stop-on-user-input opt-in: stop typing and skip the field for the run
469
+ // as soon as the human types into it
470
+ // --personal-questions "allow" answers personal questions; credentials and
471
+ // card/bank fields stay blocked either way
472
+ const stopOnUserInput = flags.has('stop-on-user-input') && flags.get('stop-on-user-input') !== 'false';
473
+ const personalMode = (flags.get('personal-questions') ?? 'skip').toLowerCase();
474
+ fieldSafety.stopOnUserInput = stopOnUserInput;
475
+ fieldSafety.allowPersonal = personalMode === 'allow';
328
476
  // The auto-answer browser is ALWAYS visible (2026-09): a headless window is
329
477
  // the "profile seems mismatched" case and hides the pages the loop works on.
330
478
  // `--headless` is accepted for backward compatibility but ignored.
@@ -356,6 +504,8 @@ async function cmdBrowse(flags) {
356
504
  }
357
505
  console.log(`[browse] Char delay: ${charDelayMs}ms`);
358
506
  console.log(`[browse] Screenshot interval: ${intervalMs}ms`);
507
+ console.log(`[browse] Personal questions: ${fieldSafety.allowPersonal ? 'answered (credentials still blocked)' : 'never answered (left for you)'}`);
508
+ console.log(`[browse] Stop typing when you type in the field: ${fieldSafety.stopOnUserInput ? 'on' : 'off'}`);
359
509
  if (!stealth) {
360
510
  console.log(`[browse] Max consecutive failures: ${maxConsecutiveFailures === 0 ? 'unlimited' : maxConsecutiveFailures}`);
361
511
  }
@@ -400,125 +550,9 @@ async function cmdBrowse(flags) {
400
550
  process.exitCode = 1;
401
551
  return;
402
552
  }
403
- // Enable Page domain (required for captureScreenshot)
404
- try {
405
- await cdpSend(page, 'Page.enable', {});
406
- console.log('[browse] Page domain enabled');
407
- }
408
- catch (err) {
409
- console.error(`[browse] Page.enable failed: ${(0, config_1.msg)(err)}`);
410
- }
411
- // Stealth: install stack trace sanitization to hide CDP frames
412
- installStackTraceStealth();
413
- // Stealth: suppress console serialization leak caused by CDP's Runtime.enable.
414
- // Anti-bot probe: Object.defineProperty(obj,'x',{get(){window.__detected=1}});
415
- // console.debug(obj); if(window.__detected) → CDP is serializing args
416
- // Countermeasure: wrap console methods to prevent deep serialization.
417
- try {
418
- await cdpEval(page, `
419
- (() => {
420
- const CK = Symbol('__ts_cp');
421
- if (window[CK]) return;
422
- const wrap = (fn) => function(...args) {
423
- return fn.apply(console, args.map(a => {
424
- if (a && typeof a === 'object' && typeof a !== 'function') {
425
- try { return Object.assign(Array.isArray(a) ? [] : {}, a); } catch { return a; }
426
- }
427
- return a;
428
- }));
429
- };
430
- console.debug = wrap(console.debug);
431
- console.log = wrap(console.log);
432
- console.warn = wrap(console.warn);
433
- console.error = wrap(console.error);
434
- window[CK] = true;
435
- })()
436
- `);
437
- }
438
- catch { /* non-critical */ }
439
- // Register CDP event: Page.frameNavigated (URL/navigation changes)
440
- onCdpEvent(page, 'Page.frameNavigated', () => {
441
- page._cdpNavigated = Date.now();
442
- });
443
- // Inject MutationObserver: watches DOM for significant content changes
444
- // Uses same fingerprint comparison as Node.js side to prevent false positives.
445
- // Filters out: timers, animations, scripts, styles, notifications, toasts.
446
- // Debounces mutations (500ms) to batch rapid changes.
447
- // Stealth: uses Symbol-keyed property (invisible to Object.keys / anti-bot scans).
448
- try {
449
- await cdpEval(page, `
450
- (() => {
451
- const KEY = Symbol('__ts_obs');
452
- if (window[KEY]) return 'already attached';
453
- const IGNORE = /timer|countdown|elapsed|clock|animation|animate|transition|spinner|progress|loading|toast|notification|alert|banner|snackbar|cookie|consent|modal|overlay|popup|tooltip|badge|tag|avatar|icon|svg|img|video|audio|canvas|font|stylesheet|script|meta|link|head/i;
454
- let timer = null;
455
- let lastText = '';
456
- function getText() {
457
- const els = document.querySelectorAll('input[type="radio"],input[type="checkbox"],input[type="text"],input[type="email"],input[type="password"],textarea,label,button');
458
- return [...els].map(e => {
459
- const t = (e.innerText || e.textContent || '').trim().replace(/\\s+/g, ' ').slice(0, 200);
460
- return e.type + '|' + t + '|' + (e.checked || false);
461
- }).join('\\n');
462
- }
463
- function check() {
464
- try {
465
- const curr = getText();
466
- if (curr === lastText) return;
467
- const prev = lastText;
468
- lastText = curr;
469
- const prevGroups = prev ? prev.split('\\n').filter(Boolean) : [];
470
- const currGroups = curr.split('\\n').filter(Boolean);
471
- const prevSet = new Set(prevGroups);
472
- const currSet = new Set(currGroups);
473
- let removed = 0, changed = 0;
474
- for (const g of prevGroups) {
475
- if (!currSet.has(g)) {
476
- if (prevGroups.indexOf(g) < currGroups.length) changed++;
477
- else removed++;
478
- }
479
- }
480
- const added = currGroups.filter(g => !prevSet.has(g)).length;
481
- const score = added * 10 + removed * 10 + changed * 8;
482
- if (score >= 15) {
483
- window[KEY].changed = Date.now();
484
- }
485
- } catch {}
486
- }
487
- const obs = new MutationObserver((mutations) => {
488
- let hasRelevant = false;
489
- for (const m of mutations) {
490
- if (m.target instanceof Element) {
491
- const tag = m.target.tagName;
492
- if (tag === 'SCRIPT' || tag === 'STYLE' || tag === 'HEAD') continue;
493
- const cls = m.target.className || '';
494
- if (typeof cls === 'string' && IGNORE.test(cls)) continue;
495
- const id = m.target.id || '';
496
- if (IGNORE.test(id)) continue;
497
- if (m.type === 'characterData' && m.target.parentElement) {
498
- const ptag = m.target.parentElement.tagName;
499
- if (ptag === 'SCRIPT' || ptag === 'STYLE') continue;
500
- }
501
- }
502
- hasRelevant = true;
503
- }
504
- if (hasRelevant) {
505
- clearTimeout(timer);
506
- timer = setTimeout(check, 500);
507
- }
508
- });
509
- obs.observe(document.body || document.documentElement, {
510
- childList: true, subtree: true, characterData: true
511
- });
512
- window[KEY] = { observer: obs, changed: 0 };
513
- lastText = getText();
514
- return 'ok';
515
- })()
516
- `);
517
- console.log('[browse] MutationObserver injected (stealth, debounced, filtered)');
518
- }
519
- catch (err) {
520
- console.error(`[browse] MutationObserver injection failed: ${(0, config_1.msg)(err)}`);
521
- }
553
+ // Enable Page + install per-tab instrumentation. Re-run on every (re)attach
554
+ // so a tab switch/close leaves the loop fully instrumented.
555
+ await instrumentPage(page);
522
556
  // Agent loop
523
557
  let actionCount = 0;
524
558
  let llmCallCount = 0;
@@ -530,6 +564,81 @@ async function cmdBrowse(flags) {
530
564
  /** A 401/403 from the model API: terminal, retrying cannot succeed. */
531
565
  let fatalAuthError = false;
532
566
  const maxRetries = 5;
567
+ /** How often to re-check which tab is active (ms). */
568
+ const TAB_CHECK_MS = 2000;
569
+ let lastTabCheck = 0;
570
+ let consecutiveScreenshotFailures = 0;
571
+ let reattachFailures = 0;
572
+ // Latency observability: rolling average of vision-call latency plus
573
+ // abort/timeout counts, so the operator can see how slow the model is.
574
+ const llmStats = { calls: 0, ok: 0, screenAborts: 0, timeouts: 0, totalMs: 0 };
575
+ const logLlmStats = () => {
576
+ if (llmStats.calls === 0 || llmStats.calls % 5 !== 0)
577
+ return;
578
+ const avg = Math.round(llmStats.totalMs / llmStats.calls / 1000);
579
+ console.log(`[browse] LLM latency: avg ${avg}s over ${llmStats.calls} call(s) — ${llmStats.ok} ok, ${llmStats.screenAborts} screen-abort, ${llmStats.timeouts} timeout`);
580
+ };
581
+ /**
582
+ * (Re)attach to a tab. A closed or switched tab used to leave the loop
583
+ * hammering a dead target ("CDP timeout: Page.captureScreenshot") forever;
584
+ * now we detect it and follow the tab the human is actually on.
585
+ */
586
+ async function reattach(reason, preferTargetId) {
587
+ console.log(`[browse] ${reason} - re-attaching...`);
588
+ try {
589
+ page.ws.close();
590
+ }
591
+ catch {
592
+ /* already closed */
593
+ }
594
+ page = await connectToPage(port, preferTargetId);
595
+ await instrumentPage(page);
596
+ // A different tab means different content: reset change trackers.
597
+ lastFingerprint = null;
598
+ lastScreenshot = null;
599
+ codeConfirmedOptimal = false;
600
+ consecutiveFailures = 0;
601
+ consecutiveScreenshotFailures = 0;
602
+ console.log(`[browse] Re-attached to tab: ${page.title} (${page.targetId.slice(0, 8)})`);
603
+ }
604
+ /**
605
+ * Throttled liveness check: is the bound tab still alive and the active one?
606
+ * Follows the human's active tab (and recovers after it is closed). `force`
607
+ * skips the throttle (used from the screenshot-failure path).
608
+ */
609
+ async function refreshTabIfNeeded(force = false) {
610
+ if (!force && Date.now() - lastTabCheck < TAB_CHECK_MS)
611
+ return false;
612
+ lastTabCheck = Date.now();
613
+ try {
614
+ if (page.disconnected) {
615
+ // The bound tab is gone: move straight to whatever tab is active now.
616
+ const active = await (0, cdp_1.pickActiveTab)(port).catch(() => null);
617
+ await reattach('Bound tab closed', active?.id);
618
+ reattachFailures = 0;
619
+ return true;
620
+ }
621
+ const active = await (0, cdp_1.pickActiveTab)(port, page.targetId);
622
+ if (!active)
623
+ return false;
624
+ if (active.id !== page.targetId) {
625
+ await reattach(`Active tab changed to "${active.title}"`, active.id);
626
+ reattachFailures = 0;
627
+ return true;
628
+ }
629
+ return false;
630
+ }
631
+ catch (err) {
632
+ reattachFailures++;
633
+ console.error(`[browse] Re-attach attempt ${reattachFailures} failed: ${(0, config_1.msg)(err)}`);
634
+ if (reattachFailures >= 6) {
635
+ console.error('[browse] Could not re-attach to any tab - stopping.');
636
+ stopped = true;
637
+ process.exitCode = 1;
638
+ }
639
+ return false;
640
+ }
641
+ }
533
642
  process.on('SIGINT', () => {
534
643
  console.log('\n[browse] Stopped by user');
535
644
  stopped = true;
@@ -538,17 +647,26 @@ async function cmdBrowse(flags) {
538
647
  console.log(`[browse] Agent started. ${continuous ? 'Running until Ctrl+C.' : `Max ${maxActions} actions.`} Press Ctrl+C to stop.\n`);
539
648
  while (!stopped && actionCount < maxActions) {
540
649
  try {
650
+ // Follow the human's active tab: re-attach if the bound tab was closed or
651
+ // the user switched to another one.
652
+ await refreshTabIfNeeded();
541
653
  // Take screenshot and extract elements in parallel
542
654
  const [screenshot, elements, pageContext] = await Promise.all([
543
- captureScreenshot(page, quality),
655
+ captureScreenshot(page, quality, SCREENSHOT_MAX_EDGE),
544
656
  extractElements(page),
545
657
  extractPageContext(page),
546
658
  ]);
547
659
  if (!screenshot) {
548
- console.error('[browse] Failed to capture screenshot, retrying...');
660
+ consecutiveScreenshotFailures++;
661
+ console.error(`[browse] Failed to capture screenshot (${consecutiveScreenshotFailures}), retrying...`);
662
+ // Two failures in a row generally means the target is gone.
663
+ if (consecutiveScreenshotFailures >= 2) {
664
+ await refreshTabIfNeeded(true);
665
+ }
549
666
  await sleep(intervalMs);
550
667
  continue;
551
668
  }
669
+ consecutiveScreenshotFailures = 0;
552
670
  // Detect screen change via question fingerprint comparison
553
671
  // Threshold 15: requires ≥1 new/removed question (10) + text change (8) = 18,
554
672
  // or 2 text changes (16). Score of 5 (single option change) is too sensitive.
@@ -568,29 +686,47 @@ async function cmdBrowse(flags) {
568
686
  const actionLabel = continuous ? `${actionCount + 1}/∞` : `${actionCount + 1}/${maxActions}`;
569
687
  console.log(`[browse] [${actionLabel}] Analyzing screenshot...`);
570
688
  let result = null;
689
+ /** The page changed under the LLM: re-analyze immediately, no backoff. */
690
+ let screenChangedDuringLlm = false;
571
691
  for (let attempt = 1; attempt <= maxRetries; attempt++) {
572
692
  result = await analyzeScreenshot(baseUrl, apiKey, model, screenshot, elements, pageContext, timeoutMs, attempt, maxRetries, page, lastFingerprint);
573
693
  llmCallCount++;
694
+ llmStats.calls++;
695
+ llmStats.totalMs += result.durationMs;
574
696
  if (llmCallCount > 50 && llmCallCount % 25 === 0) {
575
697
  console.log(`[browse] Warning: ${llmCallCount} LLM calls made — may hit rate limits`);
576
698
  }
577
- if (result.actions.length > 0)
699
+ if (result.actions.length > 0) {
700
+ llmStats.ok++;
701
+ logLlmStats();
578
702
  break; // success
703
+ }
579
704
  // Auth failure is terminal: retrying cannot help.
580
705
  if (result.error === 'auth_error') {
581
706
  fatalAuthError = true;
582
707
  break;
583
708
  }
584
- // Timeout exponential backoff, keep retrying (never count as failure)
709
+ // The screen changed while the LLM was thinking: its answer is stale.
710
+ // Bail out of the retry loop and re-analyze the NEW screen right away
711
+ // (speed matters: a 30s backoff here livelocked on dynamic pages).
712
+ if (result.error === 'screen_changed') {
713
+ screenChangedDuringLlm = true;
714
+ llmStats.screenAborts++;
715
+ logLlmStats();
716
+ break;
717
+ }
718
+ // Genuine timeout → SHORT backoff and retry (was 30/60/120/240/300s).
585
719
  if (result.error === 'timeout') {
586
- const backoff = Math.min(30 * Math.pow(2, attempt - 1), 300); // 30, 60, 120, 240, 300
720
+ llmStats.timeouts++;
721
+ logLlmStats();
722
+ const backoff = Math.min(3 * attempt, 12); // 3, 6, 9, 12, 12
587
723
  console.log(`[browse] LLM timed out — retrying in ${backoff}s (attempt ${attempt}/${maxRetries})...`);
588
724
  await sleep(backoff * 1000);
589
725
  continue;
590
726
  }
591
- // Other errors → short backoff
727
+ // Other errors → very short backoff
592
728
  if (attempt < maxRetries) {
593
- const backoff = attempt === 1 ? 3 : 5;
729
+ const backoff = 2;
594
730
  console.log(`[browse] ${result.error === 'empty' ? 'Empty response' : 'Parse failed'} (attempt ${attempt}/${maxRetries}), retrying in ${backoff}s...`);
595
731
  await sleep(backoff * 1000);
596
732
  }
@@ -601,6 +737,10 @@ async function cmdBrowse(flags) {
601
737
  process.exitCode = 1;
602
738
  break;
603
739
  }
740
+ if (screenChangedDuringLlm) {
741
+ // No sleep: go straight back to the top, re-screenshot + re-analyze.
742
+ continue;
743
+ }
604
744
  if (!result || result.actions.length === 0) {
605
745
  consecutiveFailures++;
606
746
  console.log(`[browse] No valid response after ${maxRetries} attempts (${consecutiveFailures} consecutive failures)`);
@@ -699,6 +839,18 @@ async function cmdBrowse(flags) {
699
839
  console.log(`[browse] Skipping ${act.action} — code already confirmed optimal`);
700
840
  continue;
701
841
  }
842
+ // Field safety: never type into a personal field, a credential field, or
843
+ // a field the human already answered themselves.
844
+ if (act.action === 'write_code' || act.action === 'type') {
845
+ const targetEl = act.elementIndex !== undefined
846
+ ? elements.find((e) => e.i === act.elementIndex)
847
+ : undefined;
848
+ const gated = fieldGate(targetEl);
849
+ if (gated) {
850
+ console.log(`[browse] Skipping element ${act.elementIndex} - ${gated}`);
851
+ continue;
852
+ }
853
+ }
702
854
  // Ask LLM to decide if action is redundant
703
855
  const redundancy = await isActionRedundant(baseUrl, apiKey, model, page, elements, act, timeoutMs);
704
856
  if (redundancy.decision === 'skip') {
@@ -890,6 +1042,14 @@ Do NOT simplify variable names or change the function signature. Only optimize t
890
1042
  console.log(`[browse] Reached max actions (${maxActions})`);
891
1043
  }
892
1044
  console.log(`[browse] Agent finished. ${actionCount} actions performed.`);
1045
+ const skipped = personalSkipSummary();
1046
+ if (skipped.personal > 0 || skipped.human > 0) {
1047
+ console.log(`[browse] Left for you: ${skipped.personal} personal question(s) never answered` +
1048
+ (skipped.human > 0 ? `, ${skipped.human} field(s) you answered yourself` : ''));
1049
+ for (const entry of skippedFields.values()) {
1050
+ console.log(` - ${entry.note}`);
1051
+ }
1052
+ }
893
1053
  page.ws.close();
894
1054
  }
895
1055
  // ── CDP Helpers ──────────────────────────────────────────────────────────────
@@ -897,11 +1057,11 @@ Do NOT simplify variable names or change the function signature. Only optimize t
897
1057
  * Connect directly to a page's WebSocket from /json/list.
898
1058
  * This avoids the Target.attachToTarget complexity.
899
1059
  */
900
- async function connectToPage(port) {
1060
+ async function connectToPage(port, preferTargetId) {
901
1061
  // Shared picker (cdp.ts): prefers a real page, accepts about:blank/new-tab,
902
1062
  // and opens a blank tab if none exists - so a Chrome launched on the NTP is
903
1063
  // still attachable. (Was a local chrome:// filter that could never match.)
904
- const pageTarget = await (0, cdp_1.resolveAttachablePage)(port);
1064
+ const pageTarget = await (0, cdp_1.resolveAttachablePage)(port, preferTargetId);
905
1065
  const wsUrl = pageTarget.webSocketDebuggerUrl;
906
1066
  return new Promise((resolve, reject) => {
907
1067
  const ws = new ws_1.default(wsUrl);
@@ -958,6 +1118,8 @@ async function connectToPage(port) {
958
1118
  }
959
1119
  });
960
1120
  ws.on('close', () => {
1121
+ // The endpoint is gone: flag it so the agent loop can re-attach.
1122
+ page.disconnected = true;
961
1123
  // Reject all pending
962
1124
  for (const [id, p] of page.pending) {
963
1125
  p.reject(new Error('WebSocket closed'));
@@ -1022,9 +1184,32 @@ function cdpSend(page, method, params) {
1022
1184
  page.ws.send(JSON.stringify({ id, method, params }));
1023
1185
  });
1024
1186
  }
1025
- async function captureScreenshot(page, quality) {
1187
+ /**
1188
+ * Capture the visible viewport as JPEG. When the viewport is larger than
1189
+ * `maxEdge` on either side the capture is scaled down (`clip.scale`) so the
1190
+ * vision call stays fast: a big page produced 100KB+ images that exceeded the
1191
+ * model timeout. Clicking is unaffected — actions use element coordinates
1192
+ * (CSS px), not image pixels.
1193
+ */
1194
+ async function captureScreenshot(page, quality, maxEdge = 0) {
1026
1195
  try {
1027
- const result = await cdpSend(page, 'Page.captureScreenshot', { format: 'jpeg', quality });
1196
+ const params = { format: 'jpeg', quality };
1197
+ if (maxEdge > 0) {
1198
+ try {
1199
+ const metrics = await cdpSend(page, 'Page.getLayoutMetrics', {});
1200
+ const vp = metrics?.cssLayoutViewport ?? metrics?.layoutViewport;
1201
+ const w = Number(vp?.clientWidth ?? 0);
1202
+ const h = Number(vp?.clientHeight ?? 0);
1203
+ if (w > 0 && h > 0 && (w > maxEdge || h > maxEdge)) {
1204
+ const scale = Math.min(maxEdge / w, maxEdge / h);
1205
+ params.clip = { x: 0, y: 0, width: w, height: h, scale };
1206
+ }
1207
+ }
1208
+ catch {
1209
+ /* metrics unavailable - capture unscaled */
1210
+ }
1211
+ }
1212
+ const result = await cdpSend(page, 'Page.captureScreenshot', params);
1028
1213
  return result?.data ?? null;
1029
1214
  }
1030
1215
  catch (err) {
@@ -1032,6 +1217,124 @@ async function captureScreenshot(page, quality) {
1032
1217
  return null;
1033
1218
  }
1034
1219
  }
1220
+ /**
1221
+ * Field-safety addendum appended to every system prompt (main loop, custom /
1222
+ * optimization passes, stealth loop). The loop may not answer questions that
1223
+ * ask about the human behind the browser.
1224
+ */
1225
+ const PERSONAL_SAFETY_RULES = `
1226
+ PERSONAL QUESTIONS - NEVER ANSWER THEM:
1227
+ - Some questions ask about YOU (the human operating the browser), not about the
1228
+ task. NEVER answer them with an invented value.
1229
+ - Never type: names, email addresses, phone numbers, postal addresses, dates of
1230
+ birth or age, government IDs (SSN/NIN/passport/driving licence/tax id),
1231
+ passwords, one-time codes, card or bank details, salary or compensation,
1232
+ notice periods, work-authorization/visa/sponsorship answers, or personal
1233
+ profile links (LinkedIn, GitHub, portfolio, personal website).
1234
+ - Fields marked "[NEVER ANSWER: ...]" or "[PERSONAL: ...]" in the element list
1235
+ MUST be left untouched - do not click them, do not type into them.
1236
+ - Leaving a personal field EMPTY is correct and expected. A personal field is
1237
+ NOT an unanswered question: never treat it as one, and never delay "done"
1238
+ because of it.
1239
+ - If every remaining question is personal, return done.
1240
+ - Demographics and free-text personal questions (gender, marital status, "tell
1241
+ us about yourself") are the human's to answer: prefer to skip them too.
1242
+ `;
1243
+ const fieldSafety = {
1244
+ stopOnUserInput: false,
1245
+ allowPersonal: false,
1246
+ };
1247
+ /** fieldKey -> why the field must not be typed into for the rest of the run. */
1248
+ const skippedFields = new Map();
1249
+ /**
1250
+ * Stealth mode has no DOM, so fields are identified by the focused OS control
1251
+ * (AutomationId|ClassName|Name) instead of a field key.
1252
+ */
1253
+ const skippedOsFields = new Map();
1254
+ function signalsOf(el) {
1255
+ return {
1256
+ tag: el.tag,
1257
+ type: el.type,
1258
+ name: el.name,
1259
+ id: el.id,
1260
+ autocomplete: el.autocomplete,
1261
+ placeholder: el.placeholder,
1262
+ ariaLabel: el.ariaLabel,
1263
+ label: el.label,
1264
+ questionText: el.questionText,
1265
+ text: el.text,
1266
+ };
1267
+ }
1268
+ function classificationOf(el) {
1269
+ return {
1270
+ tier: el.personalTier ?? null,
1271
+ category: el.personalCategory ?? null,
1272
+ label: el.personalLabel ?? '',
1273
+ reason: el.personalReason ?? '',
1274
+ };
1275
+ }
1276
+ /**
1277
+ * Classify every typable element and stamp its stable field key. Elements are
1278
+ * annotated in place so the gate below never re-parses the DOM.
1279
+ */
1280
+ function annotateFieldSafety(elements) {
1281
+ for (const el of elements) {
1282
+ const signals = signalsOf(el);
1283
+ el.fieldKey = (0, personal_fields_1.fieldKeyOf)(signals);
1284
+ if (!el.typable)
1285
+ continue;
1286
+ const cls = (0, personal_fields_1.classifyField)(signals);
1287
+ el.personalTier = cls.tier;
1288
+ el.personalCategory = cls.category;
1289
+ el.personalLabel = cls.label;
1290
+ el.personalReason = cls.reason;
1291
+ }
1292
+ }
1293
+ function shortLabel(el) {
1294
+ const raw = el.label || el.questionText || el.text || el.placeholder || el.name || '';
1295
+ return raw.replace(/\s+/g, ' ').trim().slice(0, 60);
1296
+ }
1297
+ /**
1298
+ * Returns a human-readable reason when this element must not be typed into,
1299
+ * or null when it is fine to type.
1300
+ */
1301
+ function fieldGate(el) {
1302
+ if (!el || !el.typable)
1303
+ return null;
1304
+ const key = el.fieldKey ?? (0, personal_fields_1.fieldKeyOf)(signalsOf(el));
1305
+ const already = skippedFields.get(key);
1306
+ if (already)
1307
+ return already.note;
1308
+ const cls = classificationOf(el);
1309
+ if (cls.tier === null)
1310
+ return null;
1311
+ if (!(0, personal_fields_1.isBlocked)(cls, { allowPersonal: fieldSafety.allowPersonal }))
1312
+ return null;
1313
+ const note = `left to you (${cls.category}: "${shortLabel(el) || cls.label}")`;
1314
+ skippedFields.set(key, { kind: 'personal', note });
1315
+ console.log(`[browse] Not answering a personal question - ${note}`);
1316
+ return note;
1317
+ }
1318
+ /** Record a field the human took over mid-typing; it is never typed again. */
1319
+ function noteHumanTookOver(el, reason) {
1320
+ const key = el?.fieldKey ?? (el ? (0, personal_fields_1.fieldKeyOf)(signalsOf(el)) : '');
1321
+ if (!key)
1322
+ return;
1323
+ const note = `you answered this one yourself (${reason})`;
1324
+ skippedFields.set(key, { kind: 'human', note });
1325
+ console.log(`[browse] Stopped typing - ${note}`);
1326
+ }
1327
+ function personalSkipSummary() {
1328
+ let personal = 0;
1329
+ let human = 0;
1330
+ for (const entry of skippedFields.values()) {
1331
+ if (entry.kind === 'personal')
1332
+ personal++;
1333
+ else
1334
+ human++;
1335
+ }
1336
+ return { personal, human };
1337
+ }
1035
1338
  /**
1036
1339
  * Extract all interactive elements from the DOM with their exact text,
1037
1340
  * bounding boxes, state, and question group assignments.
@@ -1072,6 +1375,29 @@ async function extractElements(page) {
1072
1375
  text,
1073
1376
  tag: el.tagName.toLowerCase(),
1074
1377
  type: el.type || el.getAttribute('role') || '',
1378
+ typable: el.tagName === 'INPUT' || el.tagName === 'TEXTAREA' || el.isContentEditable === true,
1379
+ name: el.getAttribute('name') || '',
1380
+ id: el.getAttribute('id') || '',
1381
+ autocomplete: el.getAttribute('autocomplete') || '',
1382
+ placeholder: el.getAttribute('placeholder') || '',
1383
+ ariaLabel: el.getAttribute('aria-label') || '',
1384
+ label: (() => {
1385
+ try {
1386
+ const id = el.getAttribute('id');
1387
+ if (id) {
1388
+ const forLabel = document.querySelector('label[for="' + CSS.escape(id) + '"]');
1389
+ if (forLabel && forLabel.innerText) return forLabel.innerText.trim().replace(/\\s+/g, ' ').slice(0, 200);
1390
+ }
1391
+ const wrap = el.closest ? el.closest('label') : null;
1392
+ if (wrap && wrap.innerText) return wrap.innerText.trim().replace(/\\s+/g, ' ').slice(0, 200);
1393
+ const labelled = el.getAttribute('aria-labelledby');
1394
+ if (labelled) {
1395
+ const t = document.getElementById(labelled);
1396
+ if (t && t.innerText) return t.innerText.trim().replace(/\\s+/g, ' ').slice(0, 200);
1397
+ }
1398
+ } catch (err) {}
1399
+ return '';
1400
+ })(),
1075
1401
  checked: el.checked || el.getAttribute('aria-checked') === 'true' || false,
1076
1402
  x: Math.round(rect.x + rect.width / 2),
1077
1403
  y: Math.round(rect.y + rect.height / 2),
@@ -1184,6 +1510,13 @@ async function extractElements(page) {
1184
1510
  y: item.y,
1185
1511
  group: item.groupId,
1186
1512
  inViewport: item.inViewport,
1513
+ typable: item.typable,
1514
+ name: item.name,
1515
+ id: item.id,
1516
+ autocomplete: item.autocomplete,
1517
+ placeholder: item.placeholder,
1518
+ ariaLabel: item.ariaLabel,
1519
+ label: item.label,
1187
1520
  });
1188
1521
  });
1189
1522
 
@@ -1192,7 +1525,9 @@ async function extractElements(page) {
1192
1525
  `;
1193
1526
  try {
1194
1527
  const result = await cdpEval(page, js);
1195
- return result?.result?.value || [];
1528
+ const elements = result?.result?.value || [];
1529
+ annotateFieldSafety(elements);
1530
+ return elements;
1196
1531
  }
1197
1532
  catch {
1198
1533
  return [];
@@ -1241,6 +1576,11 @@ async function executeAction(page, act, elements, charDelayMs) {
1241
1576
  console.log(`[browse] Element index ${act.elementIndex} not found, will retry`);
1242
1577
  return false;
1243
1578
  }
1579
+ const gated = fieldGate(el);
1580
+ if (gated) {
1581
+ console.log(`[browse] Skipping element ${el.i} - ${gated}`);
1582
+ return true;
1583
+ }
1244
1584
  // 1. Detect editor type
1245
1585
  const editor = await detectEditor(page);
1246
1586
  const isCodeEditor = editor !== null;
@@ -1255,12 +1595,19 @@ async function executeAction(page, act, elements, charDelayMs) {
1255
1595
  // 3. Normalize indentation for code editors (LLM often adds extra indent)
1256
1596
  const text = isCodeEditor ? normalizeCodeIndent(rawText) : rawText;
1257
1597
  console.log(`[browse] Typing ${text.length} characters...`);
1598
+ const guard = new human_input_guard_1.TypingGuard({
1599
+ enabled: fieldSafety.stopOnUserInput,
1600
+ page,
1601
+ label: shortLabel(el),
1602
+ });
1258
1603
  if (isCodeEditor) {
1259
- await insertIntoCodeEditor(page, editor.selector, text, charDelayMs);
1604
+ await insertIntoCodeEditor(page, editor.selector, text, charDelayMs, guard);
1260
1605
  }
1261
1606
  else {
1262
- await insertIntoGenericInput(page, el, text, charDelayMs);
1607
+ await insertIntoGenericInput(page, el, text, charDelayMs, guard);
1263
1608
  }
1609
+ if (guard.stopped)
1610
+ noteHumanTookOver(el, guard.describeStop());
1264
1611
  return true;
1265
1612
  }
1266
1613
  case 'write_code': {
@@ -1271,6 +1618,11 @@ async function executeAction(page, act, elements, charDelayMs) {
1271
1618
  console.log(`[browse] Element index ${act.elementIndex} not found, will retry`);
1272
1619
  return false;
1273
1620
  }
1621
+ const gated = fieldGate(el);
1622
+ if (gated) {
1623
+ console.log(`[browse] Skipping element ${el.i} - ${gated}`);
1624
+ return true;
1625
+ }
1274
1626
  // 1. Detect editor type
1275
1627
  const editor = await detectEditor(page);
1276
1628
  const isCodeEditor = editor !== null;
@@ -1283,13 +1635,20 @@ async function executeAction(page, act, elements, charDelayMs) {
1283
1635
  .replace(/\\"/g, '"')
1284
1636
  .replace(/\\\\/g, '\\');
1285
1637
  console.log(`[browse] Writing code (${text.length} chars)...`);
1638
+ const guard = new human_input_guard_1.TypingGuard({
1639
+ enabled: fieldSafety.stopOnUserInput,
1640
+ page,
1641
+ label: shortLabel(el),
1642
+ });
1286
1643
  // 3. write_code inserts EXACTLY as-is — no normalizeCodeIndent
1287
1644
  if (isCodeEditor) {
1288
- await insertIntoCodeEditor(page, editor.selector, text, charDelayMs);
1645
+ await insertIntoCodeEditor(page, editor.selector, text, charDelayMs, guard);
1289
1646
  }
1290
1647
  else {
1291
- await insertIntoGenericInput(page, el, text, charDelayMs);
1648
+ await insertIntoGenericInput(page, el, text, charDelayMs, guard);
1292
1649
  }
1650
+ if (guard.stopped)
1651
+ noteHumanTookOver(el, guard.describeStop());
1293
1652
  return true;
1294
1653
  }
1295
1654
  case 'scroll': {
@@ -1363,6 +1722,11 @@ async function executeActionHybrid(page, act, elements, charDelayMs) {
1363
1722
  console.log(`[browse] Element index ${act.elementIndex} not found, will retry`);
1364
1723
  return false;
1365
1724
  }
1725
+ const gated = fieldGate(el);
1726
+ if (gated) {
1727
+ console.log(`[browse] Skipping element ${el.i} - ${gated}`);
1728
+ return true;
1729
+ }
1366
1730
  const editor = await detectEditor(page);
1367
1731
  const isCodeEditor = editor !== null;
1368
1732
  if (editor)
@@ -1381,7 +1745,16 @@ async function executeActionHybrid(page, act, elements, charDelayMs) {
1381
1745
  pressKeyOS('ctrl+a');
1382
1746
  await sleep(randomInt(50, 100));
1383
1747
  }
1384
- await typeHumanLike(text, charDelayMs);
1748
+ const guard = new human_input_guard_1.TypingGuard({
1749
+ enabled: fieldSafety.stopOnUserInput,
1750
+ page,
1751
+ label: shortLabel(el),
1752
+ });
1753
+ if (guard)
1754
+ await guard.arm({ isCodeEditor });
1755
+ await typeHumanLike(text, charDelayMs, guard);
1756
+ if (guard.stopped)
1757
+ noteHumanTookOver(el, guard.describeStop());
1385
1758
  return true;
1386
1759
  }
1387
1760
  case 'write_code': {
@@ -1392,6 +1765,11 @@ async function executeActionHybrid(page, act, elements, charDelayMs) {
1392
1765
  console.log(`[browse] Element index ${act.elementIndex} not found, will retry`);
1393
1766
  return false;
1394
1767
  }
1768
+ const gated = fieldGate(el);
1769
+ if (gated) {
1770
+ console.log(`[browse] Skipping element ${el.i} - ${gated}`);
1771
+ return true;
1772
+ }
1395
1773
  const editor = await detectEditor(page);
1396
1774
  const isCodeEditor = editor !== null;
1397
1775
  if (editor)
@@ -1409,7 +1787,16 @@ async function executeActionHybrid(page, act, elements, charDelayMs) {
1409
1787
  pressKeyOS('ctrl+a');
1410
1788
  await sleep(randomInt(50, 100));
1411
1789
  }
1412
- await typeHumanLike(text, charDelayMs);
1790
+ const guard = new human_input_guard_1.TypingGuard({
1791
+ enabled: fieldSafety.stopOnUserInput,
1792
+ page,
1793
+ label: shortLabel(el),
1794
+ });
1795
+ if (guard)
1796
+ await guard.arm({ isCodeEditor });
1797
+ await typeHumanLike(text, charDelayMs, guard);
1798
+ if (guard.stopped)
1799
+ noteHumanTookOver(el, guard.describeStop());
1413
1800
  return true;
1414
1801
  }
1415
1802
  case 'scroll': {
@@ -1596,13 +1983,21 @@ function hasUnansweredQuestions(elements) {
1596
1983
  const types = els.map((e) => e.type);
1597
1984
  const isRadio = types.some((t) => t === 'radio');
1598
1985
  const isCheckbox = types.some((t) => t === 'checkbox');
1599
- const hasInput = els.some((e) => e.tag === 'input' || e.tag === 'textarea');
1986
+ // Personal fields are the human's to answer: they are never "unanswered"
1987
+ // for the agent, so a `done` from the model with them left empty sticks
1988
+ // instead of triggering the "you must answer everything" retry loop.
1989
+ const inputs = els.filter((e) => (e.tag === 'input' || e.tag === 'textarea') &&
1990
+ e.type !== 'radio' &&
1991
+ e.type !== 'checkbox' &&
1992
+ e.typable !== false);
1993
+ const openInputs = inputs.filter((e) => !(0, personal_fields_1.excludesFromCompleteness)(classificationOf(e)));
1994
+ const hasInput = openInputs.length > 0;
1600
1995
  const checkedCount = els.filter((e) => e.checked).length;
1601
1996
  if (isRadio && checkedCount === 0)
1602
1997
  return true;
1603
1998
  if (isCheckbox && checkedCount < els.length)
1604
1999
  return true;
1605
- if (hasInput && !els.some((e) => e.text && e.tag === 'input'))
2000
+ if (hasInput && !openInputs.some((e) => e.text && e.tag === 'input'))
1606
2001
  return true;
1607
2002
  }
1608
2003
  return false;
@@ -1827,7 +2222,7 @@ async function removeTrailingBrackets(page) {
1827
2222
  console.log(`[browse] Removed ${extra} extra trailing bracket(s)`);
1828
2223
  }
1829
2224
  /** Insert text into ANY code editor — character-by-character typing, no pasting. */
1830
- async function insertIntoCodeEditor(page, selector, text, charDelayMs) {
2225
+ async function insertIntoCodeEditor(page, selector, text, charDelayMs, guard) {
1831
2226
  // Focus the editor
1832
2227
  await focusSelector(page, selector);
1833
2228
  await sleep(100);
@@ -1879,8 +2274,12 @@ async function insertIntoCodeEditor(page, selector, text, charDelayMs) {
1879
2274
  // Type character by character — NO pasting
1880
2275
  const est = Math.round(text.length * charDelayMs / 1000);
1881
2276
  console.log(`[browse] Typing ${text.length} chars (~${est}s est.)...`);
2277
+ if (guard)
2278
+ await guard.arm({ selector, isCodeEditor: true });
1882
2279
  for (let i = 0; i < text.length; i++) {
1883
2280
  const char = text[i];
2281
+ if (guard)
2282
+ await guard.markOwn(char, char === '\n' ? 'Enter' : char === '\t' ? 'Tab' : undefined);
1884
2283
  if (char === '\n') {
1885
2284
  await cdpSend(page, 'Input.dispatchKeyEvent', {
1886
2285
  type: 'keyDown', key: 'Enter', code: 'Enter', text: '\r',
@@ -1918,7 +2317,16 @@ async function insertIntoCodeEditor(page, selector, text, charDelayMs) {
1918
2317
  await cdpSend(page, 'Input.dispatchKeyEvent', upParams);
1919
2318
  }
1920
2319
  await sleep(variableDelay(charDelayMs, char, i, text.length));
2320
+ if (guard) {
2321
+ await guard.tick(text.slice(0, i + 1));
2322
+ if (guard.stopped) {
2323
+ console.log(`[browse] Typing stopped after ${i + 1}/${text.length} chars`);
2324
+ break;
2325
+ }
2326
+ }
1921
2327
  }
2328
+ if (guard)
2329
+ await guard.dispose();
1922
2330
  // Post-typing: detect and remove extra trailing brackets
1923
2331
  await removeTrailingBrackets(page);
1924
2332
  // Re-enable auto-formatting
@@ -1939,7 +2347,7 @@ async function insertIntoCodeEditor(page, selector, text, charDelayMs) {
1939
2347
  console.log(`[browse] Typing complete (${text.length} chars)`);
1940
2348
  }
1941
2349
  /** Insert text into a generic input — click focus + char-by-char typing. */
1942
- async function insertIntoGenericInput(page, el, text, charDelayMs) {
2350
+ async function insertIntoGenericInput(page, el, text, charDelayMs, guard) {
1943
2351
  await cdpSend(page, 'Input.dispatchMouseEvent', {
1944
2352
  type: 'mousePressed', x: el.x, y: el.y, button: 'left', clickCount: 1,
1945
2353
  });
@@ -1949,8 +2357,12 @@ async function insertIntoGenericInput(page, el, text, charDelayMs) {
1949
2357
  await sleep(200);
1950
2358
  await clearEditor(page);
1951
2359
  await sleep(200);
2360
+ if (guard)
2361
+ await guard.arm({ isCodeEditor: false });
1952
2362
  for (let i = 0; i < text.length; i++) {
1953
2363
  const char = text[i];
2364
+ if (guard)
2365
+ await guard.markOwn(char, char === '\n' ? 'Enter' : char === '\t' ? 'Tab' : undefined);
1954
2366
  if (char === '\n') {
1955
2367
  await cdpSend(page, 'Input.dispatchKeyEvent', {
1956
2368
  type: 'keyDown', key: 'Enter', code: 'Enter', text: '\r',
@@ -1988,7 +2400,16 @@ async function insertIntoGenericInput(page, el, text, charDelayMs) {
1988
2400
  await cdpSend(page, 'Input.dispatchKeyEvent', upParams);
1989
2401
  }
1990
2402
  await sleep(variableDelay(charDelayMs, char, i, text.length));
2403
+ if (guard) {
2404
+ await guard.tick(text.slice(0, i + 1));
2405
+ if (guard.stopped) {
2406
+ console.log(`[browse] Typing stopped after ${i + 1}/${text.length} chars`);
2407
+ break;
2408
+ }
2409
+ }
1991
2410
  }
2411
+ if (guard)
2412
+ await guard.dispose();
1992
2413
  }
1993
2414
  /**
1994
2415
  * Extract page context for the LLM: title, URL, headings, errors, editor content.
@@ -2340,7 +2761,11 @@ async function analyzeScreenshot(baseUrl, apiKey, model, screenshotBase64, eleme
2340
2761
  for (const el of els) {
2341
2762
  const checked = el.checked ? ' [CHECKED]' : '';
2342
2763
  const visibility = el.inViewport ? '' : ' [OFF-SCREEN]';
2343
- lines.push(` ${el.i}: [${el.type || el.tag}] "${el.text}"${checked}${visibility}`);
2764
+ const skipNote = el.fieldKey && skippedFields.has(el.fieldKey)
2765
+ ? ` [SKIPPED: ${skippedFields.get(el.fieldKey).note}]`
2766
+ : '';
2767
+ const personal = el.typable ? (0, personal_fields_1.promptMarker)(classificationOf(el)) : '';
2768
+ lines.push(` ${el.i}: [${el.type || el.tag}] "${el.text}"${checked}${visibility}${personal}${skipNote}`);
2344
2769
  }
2345
2770
  lines.push('');
2346
2771
  }
@@ -2352,7 +2777,7 @@ async function analyzeScreenshot(baseUrl, apiKey, model, screenshotBase64, eleme
2352
2777
  const body = {
2353
2778
  model,
2354
2779
  messages: [
2355
- { role: 'system', content: SYSTEM_PROMPT },
2780
+ { role: 'system', content: SYSTEM_PROMPT + PERSONAL_SAFETY_RULES },
2356
2781
  {
2357
2782
  role: 'user',
2358
2783
  content: [
@@ -2362,11 +2787,19 @@ async function analyzeScreenshot(baseUrl, apiKey, model, screenshotBase64, eleme
2362
2787
  },
2363
2788
  ],
2364
2789
  temperature: 0.1,
2365
- max_tokens: 16384,
2790
+ max_tokens: 2048,
2366
2791
  stream: false,
2367
2792
  };
2368
2793
  const controller = new AbortController();
2369
- const timeout = setTimeout(() => controller.abort(), timeoutMs);
2794
+ // Distinguish a genuine timeout from the heartbeat cancelling a stale request
2795
+ // because the page changed: they need opposite handling (backoff vs re-analyze
2796
+ // now). The old code reported both as "timed out (300s)" and backed off 30s+,
2797
+ // which livelocked on dynamic pages.
2798
+ let abortReason = null;
2799
+ const timeout = setTimeout(() => {
2800
+ abortReason = 'timeout';
2801
+ controller.abort();
2802
+ }, timeoutMs);
2370
2803
  // Heartbeat: check for screen changes every 5s, abort LLM if page changed
2371
2804
  const heartbeat = setInterval(async () => {
2372
2805
  const elapsed = Math.round((Date.now() - startTime) / 1000);
@@ -2412,6 +2845,7 @@ async function analyzeScreenshot(baseUrl, apiKey, model, screenshotBase64, eleme
2412
2845
  const changeScore = calculateChangeScore(lastFingerprint, freshFingerprint);
2413
2846
  if (changeScore >= 15) {
2414
2847
  console.log(`[browse] Fingerprint confirms change (score: ${changeScore}) — aborting LLM (${elapsed}s)`);
2848
+ abortReason = 'screen_change';
2415
2849
  controller.abort();
2416
2850
  return;
2417
2851
  }
@@ -2478,6 +2912,13 @@ async function analyzeScreenshot(baseUrl, apiKey, model, screenshotBase64, eleme
2478
2912
  }
2479
2913
  catch (err) {
2480
2914
  if (err instanceof Error && err.name === 'AbortError') {
2915
+ const elapsed = Math.round((Date.now() - startTime) / 1000);
2916
+ if (abortReason === 'screen_change') {
2917
+ // The page changed while we were waiting: the answer is already stale.
2918
+ // Signal the caller to re-analyze right away (no backoff).
2919
+ console.log(`[browse] LLM aborted at ${elapsed}s — screen changed, re-analyzing now`);
2920
+ return { actions: [], attempt, durationMs: Date.now() - startTime, error: 'screen_changed' };
2921
+ }
2481
2922
  console.error(`[browse] LLM request timed out (${Math.round(timeoutMs / 1000)}s)`);
2482
2923
  return { actions: [], attempt, durationMs: Date.now() - startTime, error: 'timeout' };
2483
2924
  }
@@ -2499,7 +2940,7 @@ async function analyzeScreenshotWithPrompt(baseUrl, apiKey, model, screenshotBas
2499
2940
  const body = {
2500
2941
  model,
2501
2942
  messages: [
2502
- { role: 'system', content: customSystemPrompt },
2943
+ { role: 'system', content: customSystemPrompt + PERSONAL_SAFETY_RULES },
2503
2944
  {
2504
2945
  role: 'user',
2505
2946
  content: [
@@ -2509,11 +2950,15 @@ async function analyzeScreenshotWithPrompt(baseUrl, apiKey, model, screenshotBas
2509
2950
  },
2510
2951
  ],
2511
2952
  temperature: 0.1,
2512
- max_tokens: 16384,
2953
+ max_tokens: 2048,
2513
2954
  stream: false,
2514
2955
  };
2515
2956
  const controller = new AbortController();
2516
- const timeout = setTimeout(() => controller.abort(), timeoutMs);
2957
+ let optAbortReason = null;
2958
+ const timeout = setTimeout(() => {
2959
+ optAbortReason = 'timeout';
2960
+ controller.abort();
2961
+ }, timeoutMs);
2517
2962
  // Heartbeat: check for screen changes every 5s, abort if page changed
2518
2963
  const heartbeat = setInterval(async () => {
2519
2964
  const elapsed = Math.round((Date.now() - startTime) / 1000);
@@ -2530,6 +2975,7 @@ async function analyzeScreenshotWithPrompt(baseUrl, apiKey, model, screenshotBas
2530
2975
  const flagTs = flagResult?.result?.value;
2531
2976
  if (flagTs && flagTs > startTime) {
2532
2977
  console.log(`[browse] DOM mutation detected in opt pass (${elapsed}s) — aborting`);
2978
+ optAbortReason = 'screen_change';
2533
2979
  controller.abort();
2534
2980
  return;
2535
2981
  }
@@ -2573,7 +3019,12 @@ async function analyzeScreenshotWithPrompt(baseUrl, apiKey, model, screenshotBas
2573
3019
  }
2574
3020
  catch (err) {
2575
3021
  if (err instanceof Error && err.name === 'AbortError') {
2576
- return { actions: [], attempt, durationMs: Date.now() - startTime, error: 'timeout' };
3022
+ return {
3023
+ actions: [],
3024
+ attempt,
3025
+ durationMs: Date.now() - startTime,
3026
+ error: optAbortReason === 'screen_change' ? 'screen_changed' : 'timeout',
3027
+ };
2577
3028
  }
2578
3029
  return { actions: [], attempt, durationMs: Date.now() - startTime, error: 'http_error' };
2579
3030
  }
@@ -2767,19 +3218,24 @@ function getAdjacentKey(char) {
2767
3218
  * Type text with human-like patterns: variable speed, occasional typos,
2768
3219
  * backspaces, natural pauses. Uses OS-level keyboard (no CDP).
2769
3220
  */
2770
- async function typeHumanLike(text, baseDelayMs = 120) {
3221
+ async function typeHumanLike(text, baseDelayMs = 120, guard) {
3222
+ const send = async (ch, keyName) => {
3223
+ if (guard)
3224
+ await guard.markOwn(ch, keyName);
3225
+ typeCharOS(ch);
3226
+ };
2771
3227
  for (let i = 0; i < text.length; i++) {
2772
3228
  const char = text[i];
2773
3229
  // 1. Occasional typo (2% chance) — then backspace to correct
2774
3230
  if (char !== '\n' && char !== '\t' && Math.random() < 0.02) {
2775
3231
  const wrong = getAdjacentKey(char);
2776
- typeCharOS(wrong);
3232
+ await send(wrong);
2777
3233
  await sleep(randomInt(40, 120));
2778
- typeCharOS('\b'); // backspace
3234
+ await send('\b', 'Backspace'); // backspace
2779
3235
  await sleep(randomInt(25, 70));
2780
3236
  }
2781
3237
  // 2. Type the correct character
2782
- typeCharOS(char);
3238
+ await send(char, char === '\n' ? 'Enter' : char === '\t' ? 'Tab' : undefined);
2783
3239
  // 3. Variable delay based on character type (human rhythm)
2784
3240
  let delay = baseDelayMs + randomInt(-Math.floor(baseDelayMs * 0.3), Math.floor(baseDelayMs * 0.4));
2785
3241
  if (char === ' ')
@@ -2801,7 +3257,16 @@ async function typeHumanLike(text, baseDelayMs = 120) {
2801
3257
  await sleep(randomInt(200, 600));
2802
3258
  }
2803
3259
  await sleep(Math.max(delay, 30));
3260
+ if (guard) {
3261
+ await guard.tick(text.slice(0, i + 1));
3262
+ if (guard.stopped) {
3263
+ console.log(`[browse] Typing stopped after ${i + 1}/${text.length} chars`);
3264
+ break;
3265
+ }
3266
+ }
2804
3267
  }
3268
+ if (guard)
3269
+ await guard.dispose();
2805
3270
  }
2806
3271
  /**
2807
3272
  * Wait for the screen to change by comparing screenshots.
@@ -2864,7 +3329,7 @@ async function callStealthLLM(baseUrl, apiKey, model, screenshotPath, timeoutMs,
2864
3329
  body: JSON.stringify({
2865
3330
  model,
2866
3331
  messages: [
2867
- { role: 'system', content: systemPrompt },
3332
+ { role: 'system', content: systemPrompt + PERSONAL_SAFETY_RULES },
2868
3333
  {
2869
3334
  role: 'user',
2870
3335
  content: [
@@ -2961,9 +3426,41 @@ async function runStealthLoop(baseUrl, apiKey, model, maxActions, intervalMs, ti
2961
3426
  }
2962
3427
  case 'type': {
2963
3428
  if (result.text && result.text.length > 0) {
3429
+ const guard = new human_input_guard_1.TypingGuard({
3430
+ enabled: fieldSafety.stopOnUserInput,
3431
+ osLevel: true,
3432
+ label: result.text.slice(0, 40),
3433
+ });
3434
+ // Stealth has no DOM: use the focused control's own name as the
3435
+ // label for both the personal classifier and the human-skip list.
3436
+ const control = await guard.currentOsControl().catch(() => null);
3437
+ if (control) {
3438
+ if (control.signature && skippedOsFields.has(control.signature)) {
3439
+ console.log(`[stealth] Skipping field - ${skippedOsFields.get(control.signature)}`);
3440
+ actionCount++;
3441
+ break;
3442
+ }
3443
+ if (control.name && fieldSafety.allowPersonal === false) {
3444
+ const cls = (0, personal_fields_1.classifyField)({ label: control.name, ariaLabel: control.name });
3445
+ if ((0, personal_fields_1.isBlocked)(cls, { allowPersonal: fieldSafety.allowPersonal })) {
3446
+ skippedOsFields.set(control.signature, `left to you (${cls.category}: "${control.name.slice(0, 60)}")`);
3447
+ console.log(`[stealth] Not answering a personal question - left to you (${cls.category}: "${control.name.slice(0, 60)}")`);
3448
+ actionCount++;
3449
+ break;
3450
+ }
3451
+ }
3452
+ }
2964
3453
  console.log(`[stealth] Typing ${result.text.length} chars...`);
2965
3454
  await sleep(randomInt(300, 700)); // natural pause before typing
2966
- await typeHumanLike(result.text, charDelayMs);
3455
+ await guard.arm({ isCodeEditor: false });
3456
+ await typeHumanLike(result.text, charDelayMs, guard);
3457
+ if (guard.stopped) {
3458
+ const after = await guard.currentOsControl().catch(() => null);
3459
+ const sig = after?.signature || control?.signature;
3460
+ if (sig)
3461
+ skippedOsFields.set(sig, `you answered this one yourself (${guard.describeStop()})`);
3462
+ console.log(`[stealth] Stopped typing - you answered this one yourself (${guard.describeStop()})`);
3463
+ }
2967
3464
  actionCount++;
2968
3465
  }
2969
3466
  else {