teamshare-bridge 0.21.29 → 0.21.30

package/dist/cli/commands/browse.js

@@ -61,6 +61,7 @@ const launch_1 = require("../../lib/browser/launch");
 const session_1 = require("../../lib/browser/session");
 const personal_fields_1 = require("../../lib/browser/personal-fields");
 const human_input_guard_1 = require("../../lib/browser/human-input-guard");
+const screen_change_1 = require("../../lib/browser/screen-change");
 /** Create a fingerprint from the current element list. */
 function createFingerprint(elements) {
     const groups = new Map();
@@ -116,6 +117,34 @@ function calculateChangeScore(prev, curr) {
     }
     return score;
 }
+/**
+ * True when the QUESTION SET changed - a new/removed question, different
+ * question text, or different option text. This is the signal to abort a stale
+ * LLM call / discard its answer. It deliberately ignores checked-state-only
+ * changes (a human answering) and low-level noise (timers, animations) because
+ * those never alter question text or the group set - so a countdown can't
+ * trigger a false "screen changed".
+ */
+function questionSetChanged(prev, curr) {
+    const prevMap = new Map(prev.map((fp) => [fp.groupId, fp]));
+    const currMap = new Map(curr.map((fp) => [fp.groupId, fp]));
+    for (const gid of currMap.keys())
+        if (!prevMap.has(gid))
+            return true;
+    for (const gid of prevMap.keys())
+        if (!currMap.has(gid))
+            return true;
+    for (const [gid, c] of currMap) {
+        const p = prevMap.get(gid);
+        if (!p)
+            continue;
+        if (p.questionText !== c.questionText)
+            return true;
+        if (p.options.join('|') !== c.options.join('|'))
+            return true;
+    }
+    return false;
+}
 /** Fast path (easy objective questions): small reasoning budget. */
 const FAST_MAX_TOKENS = 2048;
 /** Deep path (hard coding questions): enough tokens to emit a full solution. */
@@ -327,90 +356,55 @@ async function instrumentPage(page) {
     `);
     }
     catch { /* non-critical */ }
-    // Register CDP event: Page.frameNavigated (URL/navigation changes)
-    onCdpEvent(page, 'Page.frameNavigated', () => {
-        page._cdpNavigated = Date.now();
-    });
-    // Inject MutationObserver: watches DOM for significant content changes
-    // Uses same fingerprint comparison as Node.js side to prevent false positives.
-    // Filters out: timers, animations, scripts, styles, notifications, toasts.
-    // Debounces mutations (500ms) to batch rapid changes.
-    // Stealth: uses Symbol-keyed property (invisible to Object.keys / anti-bot scans).
+    // Register CDP event: Page.frameNavigated (URL/navigation changes). Bind
+    // ONCE per page object - instrumentPage also runs on re-attach.
+    //
+    // The MutationObserver is DOCUMENT-scoped: a full navigation wipes it. It used
+    // to be injected only here (attach time, frequently on about:blank), so after
+    // the human navigated to the real page the heartbeat's DOM-change path was
+    // dead for the rest of the run - a question swap during a 50-120s vision call
+    // went undetected and its stale answer was applied to the next screen. We now
+    // re-inject whenever the MAIN frame finishes navigating; subframe/iframe
+    // navigations (ads, widgets) are ignored.
+    if (!page._frameNavBound) {
+        page._frameNavBound = true;
+        onCdpEvent(page, 'Page.frameNavigated', (params) => {
+            if (!(0, screen_change_1.isMainFrameNavigation)(params))
+                return;
+            page._cdpNavigated = Date.now();
+            scheduleObserverReinject(page);
+        });
+    }
+    await reinjectObserver(page);
+}
+/**
+ * Re-inject the document-scoped MutationObserver into the page's CURRENT
+ * document. Idempotent: the injected script early-returns when its Symbol key
+ * already exists. Must run again after every (main-frame) navigation.
+ */
+async function reinjectObserver(page) {
     try {
-        await cdpEval(page, `
-      (() => {
-        const KEY = Symbol('__ts_obs');
-        if (window[KEY]) return 'already attached';
-        const IGNORE = /timer|countdown|elapsed|clock|animation|animate|transition|spinner|progress|loading|toast|notification|alert|banner|snackbar|cookie|consent|modal|overlay|popup|tooltip|badge|tag|avatar|icon|svg|img|video|audio|canvas|font|stylesheet|script|meta|link|head/i;
-        let timer = null;
-        let lastText = '';
-        function getText() {
-          const els = document.querySelectorAll('input[type="radio"],input[type="checkbox"],input[type="text"],input[type="email"],input[type="password"],textarea,label,button');
-          return [...els].map(e => {
-            const t = (e.innerText || e.textContent || '').trim().replace(/\\s+/g, ' ').slice(0, 200);
-            return e.type + '|' + t + '|' + (e.checked || false);
-          }).join('\\n');
-        }
-        function check() {
-          try {
-            const curr = getText();
-            if (curr === lastText) return;
-            const prev = lastText;
-            lastText = curr;
-            const prevGroups = prev ? prev.split('\\n').filter(Boolean) : [];
-            const currGroups = curr.split('\\n').filter(Boolean);
-            const prevSet = new Set(prevGroups);
-            const currSet = new Set(currGroups);
-            let removed = 0, changed = 0;
-            for (const g of prevGroups) {
-              if (!currSet.has(g)) {
-                if (prevGroups.indexOf(g) < currGroups.length) changed++;
-                else removed++;
-              }
-            }
-            const added = currGroups.filter(g => !prevSet.has(g)).length;
-            const score = added * 10 + removed * 10 + changed * 8;
-            if (score >= 15) {
-              window[KEY].changed = Date.now();
-            }
-          } catch {}
-        }
-        const obs = new MutationObserver((mutations) => {
-          let hasRelevant = false;
-          for (const m of mutations) {
-            if (m.target instanceof Element) {
-              const tag = m.target.tagName;
-              if (tag === 'SCRIPT' || tag === 'STYLE' || tag === 'HEAD') continue;
-              const cls = m.target.className || '';
-              if (typeof cls === 'string' && IGNORE.test(cls)) continue;
-              const id = m.target.id || '';
-              if (IGNORE.test(id)) continue;
-              if (m.type === 'characterData' && m.target.parentElement) {
-                const ptag = m.target.parentElement.tagName;
-                if (ptag === 'SCRIPT' || ptag === 'STYLE') continue;
-              }
-            }
-            hasRelevant = true;
-          }
-          if (hasRelevant) {
-            clearTimeout(timer);
-            timer = setTimeout(check, 500);
-          }
-        });
-        obs.observe(document.body || document.documentElement, {
-          childList: true, subtree: true, characterData: true
-        });
-        window[KEY] = { observer: obs, changed: 0 };
-        lastText = getText();
-        return 'ok';
-      })()
-    `);
+        await cdpEval(page, (0, screen_change_1.buildObserverScript)());
         console.log('[browse] MutationObserver injected (stealth, debounced, filtered)');
     }
     catch (err) {
         console.error(`[browse] MutationObserver injection failed: ${(0, config_1.msg)(err)}`);
     }
 }
+/**
+ * Debounced observer re-inject after a main-frame navigation: `frameNavigated`
+ * can fire before the new document is ready, and rapid redirects would
+ * otherwise inject several times.
+ */
+function scheduleObserverReinject(page) {
+    if (page._reinjectTimer)
+        clearTimeout(page._reinjectTimer);
+    page._reinjectTimer = setTimeout(() => {
+        page._reinjectTimer = undefined;
+        if (!page.disconnected)
+            void reinjectObserver(page);
+    }, 600);
+}
 async function cmdBrowse(flags) {
     const autoAnswer = flags.get('auto-answer') === 'true';
     if (!autoAnswer) {
@@ -475,13 +469,18 @@ async function cmdBrowse(flags) {
     // the fast retry below abandons a slow easy answer at ~45s and escalates.
     const timeoutSec = Math.min(Math.max(parseInt(flags.get('timeout') ?? '300', 10), 30), 600);
     const timeoutMs = timeoutSec * 1000;
-    const quality = Math.min(Math.max(parseInt(flags.get('quality') ?? '70', 10), 50), 100);
+    const quality = Math.min(Math.max(parseInt(flags.get('quality') ?? '82', 10), 50), 100);
     /**
-     * Downscale the FAST-path screenshot (attempt 1) to keep easy questions
-     * quick. The deep retry re-captures at full resolution so code/text stays
-     * legible. Clicks are unaffected either way (actions use element coords).
+     * Screenshot detail: the viewport is captured at native resolution and only
+     * scaled down when BOTH sides exceed this many px. Default 1600 keeps
+     * question/option text crisp for the vision model; pass `--max-edge 0` for
+     * native full resolution, or a smaller value for speed. Clicks are unaffected
+     * (actions use element coordinates, not image pixels).
      */
-    const SCREENSHOT_MAX_EDGE = 1024;
+    const maxEdgeRaw = parseInt(flags.get('max-edge') ?? '1600', 10);
+    const SCREENSHOT_MAX_EDGE = Number.isFinite(maxEdgeRaw)
+        ? Math.max(0, Math.min(maxEdgeRaw, 4096))
+        : 1600;
     const maxConsecutiveFailures = parseInt(flags.get('max-consecutive-failures') ?? '0', 10); // 0 = unlimited
     const charDelayMs = Math.min(Math.max(parseInt(flags.get('char-delay') ?? '150', 10), 5), 200);
     const optimize = flags.has('optimize');
@@ -711,7 +710,7 @@ async function cmdBrowse(flags) {
             const currentFingerprint = createFingerprint(elements);
             if (lastFingerprint) {
                 const changeScore = calculateChangeScore(lastFingerprint, currentFingerprint);
-                if (changeScore >= 15) {
+                if (questionSetChanged(lastFingerprint, currentFingerprint) || changeScore >= 15) {
                     console.log(`[browse] Screen changed (score: ${changeScore}) — resetting state`);
                     codeConfirmedOptimal = false;
                     consecutiveFailures = 0;
@@ -731,20 +730,17 @@ async function cmdBrowse(flags) {
             /** The page changed under the LLM: re-analyze immediately, no backoff. */
             let screenChangedDuringLlm = false;
             for (let attempt = 1; attempt <= maxRetries; attempt++) {
-                // Fast-first ladder: attempt 1 uses a small budget + downscaled shot so
-                // easy objective questions answer quickly. If it fails, attempts 2+
-                // switch to the DEEP budget (full timeout + tokens, full-res capture)
-                // for hard coding questions. `max_tokens`/timeout are ceilings, so the
+                // Fast-first ladder: attempt 1 uses a small token budget; attempts 2+
+                // switch to the DEEP budget (full timeout + tokens) for hard coding
+                // questions. Both use the SAME full-detail screenshot (maxEdge/quality)
+                // so text is always legible; `max_tokens`/timeout are ceilings, so the
                 // deep path never slows a question that answers on the fast path.
                 const deep = attempt > 1;
                 const attemptTimeoutMs = deep ? timeoutMs : Math.min(FAST_ATTEMPT_TIMEOUT_MS, timeoutMs);
                 if (deep && attempt === 2) {
                     console.log('[browse] Fast attempt did not answer — escalating to the deep budget (more time + tokens)');
                 }
-                const attemptShot = deep
-                    ? ((await captureScreenshot(page, quality)) ?? screenshot)
-                    : screenshot;
-                result = await analyzeScreenshot(baseUrl, apiKey, model, attemptShot, elements, pageContext, attemptTimeoutMs, attempt, maxRetries, page, lastFingerprint, { maxTokens: deep ? DEEP_MAX_TOKENS : FAST_MAX_TOKENS, deep });
+                result = await analyzeScreenshot(baseUrl, apiKey, model, screenshot, elements, pageContext, attemptTimeoutMs, attempt, maxRetries, page, lastFingerprint, { maxTokens: deep ? DEEP_MAX_TOKENS : FAST_MAX_TOKENS, deep });
                 llmCallCount++;
                 llmStats.calls++;
                 llmStats.totalMs += result.durationMs;
@@ -822,10 +818,12 @@ async function cmdBrowse(flags) {
                     const freshFingerprint = createFingerprint(freshElements);
                     if (lastFingerprint) {
                         const changeScore = calculateChangeScore(lastFingerprint, freshFingerprint);
-                        // Only discard a (costly) LLM answer for a MATERIAL change. The old
-                        // threshold of 15 threw away whole 40-170s calls on score-20 noise.
-                        if (changeScore >= 40) {
-                            console.log(`[browse] Screen changed materially during LLM call (score: ${changeScore}) — skipping actions, re-analyzing`);
+                        // Discard a (costly) LLM answer when the QUESTION SET changed under
+                        // it (its answer is stale), or on a material whole-screen change.
+                        // A pure question-text swap scores only 8 on the generic scale, so
+                        // the targeted check catches what the score alone missed.
+                        if (questionSetChanged(lastFingerprint, freshFingerprint) || changeScore >= 40) {
+                            console.log(`[browse] Question set changed during LLM call (score: ${changeScore}) — skipping actions, re-analyzing`);
                             lastFingerprint = freshFingerprint;
                             lastScreenshot = freshScreenshot;
                             await sleep(intervalMs);
@@ -2625,6 +2623,26 @@ async function extractPageContext(page) {
           break;
         }
       }
+      // FULL PAGE CONTENT (2026-09): the model must be fully informed from the
+      // PAGE TEXT, not just the screenshot or the element list. Prefer the main
+      // content region, else the body; collapse whitespace and cap the size.
+      try {
+        const root =
+          document.querySelector('main, article, [role="main"]') || document.body;
+        let text = (root && (root.innerText || root.textContent)) || '';
+        text = text
+          .replace(/[ \\t\\u00a0]+/g, ' ')
+          .replace(/\\n\\s*\\n+/g, '\\n')
+          .trim();
+        if (text) {
+          const CAP = 12000;
+          ctx.push('Page content:');
+          ctx.push(text.slice(0, CAP));
+          if (text.length > CAP) {
+            ctx.push('... (page content truncated at ' + CAP + ' chars)');
+          }
+        }
+      } catch (e) { /* best-effort */ }
       return ctx.join('\\n');
     })()
   `;
@@ -2868,9 +2886,9 @@ function extractModelText(json) {
  * Non-streaming never resolves until the provider finishes the whole answer, so
  * a long reasoning response looked like a 150s+ "hang" with zero feedback. With
  * streaming we see the first token, log its latency, and can catch a genuinely
- * stalled request: a `first-token` watchdog (nothing on the wire) and an `idle`
- * watchdog (no new token). `: PROCESSING` keepalive comments are ignored and do
- * NOT count as progress (so a stuck model still trips the watchdog).
+ * stalled request (no bytes on the wire at all). `: PROCESSING` keepalives are
+ * NOT tokens but DO prove the connection is alive, so they reset the watchdog;
+ * only real wire silence trips it.
  *
  * The caller's `signal` (screen-change / overall timeout) still aborts this.
  */
@@ -2885,13 +2903,21 @@ async function streamChatCompletion(opts) {
     let sawToken = false;
     let watchdog = null;
     const start = Date.now();
-    const arm = (ms) => {
+    /**
+     * Any bytes on the wire (including `: PROCESSING` keepalive comments) prove
+     * the connection is alive, so they RESET the watchdog. This provider sends
+     * keepalives every few seconds and only emits real tokens much later - the
+     * old watchdog ignored comments and killed a healthy request at 60s
+     * ("LLM stalled (no token received)" while the model was actively working).
+     * The stall now means genuine wire silence: no bytes at all.
+     */
+    const arm = () => {
         if (watchdog)
             clearTimeout(watchdog);
         watchdog = setTimeout(() => {
             stalled = sawToken ? 'idle' : 'first-token';
             ac.abort();
-        }, ms);
+        }, sawToken ? opts.idleTimeoutMs : opts.firstTokenTimeoutMs);
     };
     const cleanup = () => {
         if (watchdog)
@@ -2899,7 +2925,7 @@ async function streamChatCompletion(opts) {
         watchdog = null;
         opts.signal.removeEventListener('abort', forwardAbort);
     };
-    arm(opts.firstTokenTimeoutMs);
+    arm();
     try {
         const res = await fetch(opts.url, {
             method: 'POST',
@@ -2933,6 +2959,9 @@ async function streamChatCompletion(opts) {
             const { done, value } = await reader.read();
             if (done)
                 break;
+            // Any bytes at all (comment keepalives included) = the connection is
+            // alive, so reset the stall watchdog.
+            arm();
             buffer += decoder.decode(value, { stream: true });
             let nl;
             while ((nl = buffer.indexOf('\n')) >= 0) {
@@ -2957,28 +2986,29 @@ async function streamChatCompletion(opts) {
                 const delta = j?.choices?.[0]?.delta;
                 const dc = delta?.content;
                 const dr = delta?.reasoning_content;
-                let progressed = false;
                 if (typeof dc === 'string' && dc) {
                     content += dc;
-                    progressed = true;
+                    if (!sawToken) {
+                        sawToken = true;
+                        opts.onFirstToken?.(Date.now() - start);
+                    }
                 }
                 else if (Array.isArray(dc)) {
                     const t = dc.map((x) => x?.text ?? '').join('');
                     if (t) {
                         content += t;
-                        progressed = true;
+                        if (!sawToken) {
+                            sawToken = true;
+                            opts.onFirstToken?.(Date.now() - start);
+                        }
                     }
                 }
                 if (typeof dr === 'string' && dr) {
                     reasoning += dr;
-                    progressed = true;
-                }
-                if (progressed) {
                     if (!sawToken) {
                         sawToken = true;
                         opts.onFirstToken?.(Date.now() - start);
                     }
-                    arm(opts.idleTimeoutMs);
                 }
             }
         }
@@ -3092,6 +3122,7 @@ async function analyzeScreenshot(baseUrl, apiKey, model, screenshotBase64, eleme
         controller.abort();
     }, timeoutMs);
     // Heartbeat: check for screen changes every 5s, abort LLM if page changed
+    let heartbeatTicks = 0;
     const heartbeat = setInterval(async () => {
         const elapsed = Math.round((Date.now() - startTime) / 1000);
         // Check for screen change via CDP navigation event or MutationObserver flag
@@ -3128,14 +3159,26 @@ async function analyzeScreenshot(baseUrl, apiKey, model, screenshotBase64, eleme
             }
             catch { /* page disconnected */ }
         }
-        if (screenChanged && lastFingerprint && page) {
+        // Fallback (2026-09): the in-page observer is document-scoped (re-injected on
+        // navigation, but a same-document SPA swap may not have ticked it yet) and a
+        // continuously-mutating page can starve its debounce. So every ~10s verify the
+        // fingerprint directly even when no flag fired - a question swap during a
+        // 50-120s vision call must never slip through unnoticed.
+        heartbeatTicks++;
+        const periodicCheck = !screenChanged && heartbeatTicks % 2 === 0;
+        if ((screenChanged || periodicCheck) && lastFingerprint && page) {
             // Verify with fingerprint comparison before aborting
             try {
                 const freshElements = await extractElements(page);
                 const freshFingerprint = createFingerprint(freshElements);
                 const changeScore = calculateChangeScore(lastFingerprint, freshFingerprint);
-                if (changeScore >= 15) {
-                    console.log(`[browse] Fingerprint confirms change (score: ${changeScore}) — aborting LLM (${elapsed}s)`);
+                // Abort on a real question change even when the aggregate score is below
+                // the generic threshold (a pure question-text swap scores only 8), while
+                // still ignoring checked-only / timer noise.
+                if (questionSetChanged(lastFingerprint, freshFingerprint) || changeScore >= 15) {
+                    console.log(screenChanged
+                        ? `[browse] Fingerprint confirms change (score: ${changeScore}) — aborting LLM (${elapsed}s)`
+                        : `[browse] Periodic fingerprint found change (score: ${changeScore}) — aborting LLM (${elapsed}s)`);
                     abortReason = 'screen_change';
                     controller.abort();
                     return;
@@ -3244,9 +3287,17 @@ async function analyzeScreenshotWithPrompt(baseUrl, apiKey, model, screenshotBas
         optAbortReason = 'timeout';
         controller.abort();
     }, timeoutMs);
-    // Heartbeat: check for screen changes every 5s, abort if page changed
+    // Heartbeat: check for screen changes every 5s, abort if page changed.
+    // The observer flag lives in the current document, so any navigation (which
+    // wipes it) must also abort - check `_cdpNavigated` too, as in the main call.
     const heartbeat = setInterval(async () => {
         const elapsed = Math.round((Date.now() - startTime) / 1000);
+        if (page?._cdpNavigated && page._cdpNavigated > startTime) {
+            console.log(`[browse] CDP navigation detected in opt pass (${elapsed}s) — aborting`);
+            optAbortReason = 'screen_change';
+            controller.abort();
+            return;
+        }
         try {
             const flagResult = await cdpEval(page, `
           (() => {