teamshare-bridge 0.21.29 → 0.21.31
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cli/commands/browse.js +216 -126
- package/dist/cli/commands/browse.js.map +1 -1
- package/dist/cli/commands/research.js +3 -0
- package/dist/cli/commands/research.js.map +1 -1
- package/dist/lib/browser/screen-change.d.ts +43 -0
- package/dist/lib/browser/screen-change.js +122 -0
- package/dist/lib/browser/screen-change.js.map +1 -0
- package/package.json +1 -1
|
@@ -61,6 +61,7 @@ const launch_1 = require("../../lib/browser/launch");
|
|
|
61
61
|
const session_1 = require("../../lib/browser/session");
|
|
62
62
|
const personal_fields_1 = require("../../lib/browser/personal-fields");
|
|
63
63
|
const human_input_guard_1 = require("../../lib/browser/human-input-guard");
|
|
64
|
+
const screen_change_1 = require("../../lib/browser/screen-change");
|
|
64
65
|
/** Create a fingerprint from the current element list. */
|
|
65
66
|
function createFingerprint(elements) {
|
|
66
67
|
const groups = new Map();
|
|
@@ -116,6 +117,34 @@ function calculateChangeScore(prev, curr) {
|
|
|
116
117
|
}
|
|
117
118
|
return score;
|
|
118
119
|
}
|
|
120
|
+
/**
|
|
121
|
+
* True when the QUESTION SET changed - a new/removed question, different
|
|
122
|
+
* question text, or different option text. This is the signal to abort a stale
|
|
123
|
+
* LLM call / discard its answer. It deliberately ignores checked-state-only
|
|
124
|
+
* changes (a human answering) and low-level noise (timers, animations) because
|
|
125
|
+
* those never alter question text or the group set - so a countdown can't
|
|
126
|
+
* trigger a false "screen changed".
|
|
127
|
+
*/
|
|
128
|
+
function questionSetChanged(prev, curr) {
|
|
129
|
+
const prevMap = new Map(prev.map((fp) => [fp.groupId, fp]));
|
|
130
|
+
const currMap = new Map(curr.map((fp) => [fp.groupId, fp]));
|
|
131
|
+
for (const gid of currMap.keys())
|
|
132
|
+
if (!prevMap.has(gid))
|
|
133
|
+
return true;
|
|
134
|
+
for (const gid of prevMap.keys())
|
|
135
|
+
if (!currMap.has(gid))
|
|
136
|
+
return true;
|
|
137
|
+
for (const [gid, c] of currMap) {
|
|
138
|
+
const p = prevMap.get(gid);
|
|
139
|
+
if (!p)
|
|
140
|
+
continue;
|
|
141
|
+
if (p.questionText !== c.questionText)
|
|
142
|
+
return true;
|
|
143
|
+
if (p.options.join('|') !== c.options.join('|'))
|
|
144
|
+
return true;
|
|
145
|
+
}
|
|
146
|
+
return false;
|
|
147
|
+
}
|
|
119
148
|
/** Fast path (easy objective questions): small reasoning budget. */
|
|
120
149
|
const FAST_MAX_TOKENS = 2048;
|
|
121
150
|
/** Deep path (hard coding questions): enough tokens to emit a full solution. */
|
|
@@ -327,90 +356,55 @@ async function instrumentPage(page) {
|
|
|
327
356
|
`);
|
|
328
357
|
}
|
|
329
358
|
catch { /* non-critical */ }
|
|
330
|
-
// Register CDP event: Page.frameNavigated (URL/navigation changes)
|
|
331
|
-
|
|
332
|
-
|
|
333
|
-
|
|
334
|
-
//
|
|
335
|
-
//
|
|
336
|
-
//
|
|
337
|
-
//
|
|
338
|
-
//
|
|
359
|
+
// Register CDP event: Page.frameNavigated (URL/navigation changes). Bind
|
|
360
|
+
// ONCE per page object - instrumentPage also runs on re-attach.
|
|
361
|
+
//
|
|
362
|
+
// The MutationObserver is DOCUMENT-scoped: a full navigation wipes it. It used
|
|
363
|
+
// to be injected only here (attach time, frequently on about:blank), so after
|
|
364
|
+
// the human navigated to the real page the heartbeat's DOM-change path was
|
|
365
|
+
// dead for the rest of the run - a question swap during a 50-120s vision call
|
|
366
|
+
// went undetected and its stale answer was applied to the next screen. We now
|
|
367
|
+
// re-inject whenever the MAIN frame finishes navigating; subframe/iframe
|
|
368
|
+
// navigations (ads, widgets) are ignored.
|
|
369
|
+
if (!page._frameNavBound) {
|
|
370
|
+
page._frameNavBound = true;
|
|
371
|
+
onCdpEvent(page, 'Page.frameNavigated', (params) => {
|
|
372
|
+
if (!(0, screen_change_1.isMainFrameNavigation)(params))
|
|
373
|
+
return;
|
|
374
|
+
page._cdpNavigated = Date.now();
|
|
375
|
+
scheduleObserverReinject(page);
|
|
376
|
+
});
|
|
377
|
+
}
|
|
378
|
+
await reinjectObserver(page);
|
|
379
|
+
}
|
|
380
|
+
/**
|
|
381
|
+
* Re-inject the document-scoped MutationObserver into the page's CURRENT
|
|
382
|
+
* document. Idempotent: the injected script early-returns when its Symbol key
|
|
383
|
+
* already exists. Must run again after every (main-frame) navigation.
|
|
384
|
+
*/
|
|
385
|
+
async function reinjectObserver(page) {
|
|
339
386
|
try {
|
|
340
|
-
await cdpEval(page,
|
|
341
|
-
(() => {
|
|
342
|
-
const KEY = Symbol('__ts_obs');
|
|
343
|
-
if (window[KEY]) return 'already attached';
|
|
344
|
-
const IGNORE = /timer|countdown|elapsed|clock|animation|animate|transition|spinner|progress|loading|toast|notification|alert|banner|snackbar|cookie|consent|modal|overlay|popup|tooltip|badge|tag|avatar|icon|svg|img|video|audio|canvas|font|stylesheet|script|meta|link|head/i;
|
|
345
|
-
let timer = null;
|
|
346
|
-
let lastText = '';
|
|
347
|
-
function getText() {
|
|
348
|
-
const els = document.querySelectorAll('input[type="radio"],input[type="checkbox"],input[type="text"],input[type="email"],input[type="password"],textarea,label,button');
|
|
349
|
-
return [...els].map(e => {
|
|
350
|
-
const t = (e.innerText || e.textContent || '').trim().replace(/\\s+/g, ' ').slice(0, 200);
|
|
351
|
-
return e.type + '|' + t + '|' + (e.checked || false);
|
|
352
|
-
}).join('\\n');
|
|
353
|
-
}
|
|
354
|
-
function check() {
|
|
355
|
-
try {
|
|
356
|
-
const curr = getText();
|
|
357
|
-
if (curr === lastText) return;
|
|
358
|
-
const prev = lastText;
|
|
359
|
-
lastText = curr;
|
|
360
|
-
const prevGroups = prev ? prev.split('\\n').filter(Boolean) : [];
|
|
361
|
-
const currGroups = curr.split('\\n').filter(Boolean);
|
|
362
|
-
const prevSet = new Set(prevGroups);
|
|
363
|
-
const currSet = new Set(currGroups);
|
|
364
|
-
let removed = 0, changed = 0;
|
|
365
|
-
for (const g of prevGroups) {
|
|
366
|
-
if (!currSet.has(g)) {
|
|
367
|
-
if (prevGroups.indexOf(g) < currGroups.length) changed++;
|
|
368
|
-
else removed++;
|
|
369
|
-
}
|
|
370
|
-
}
|
|
371
|
-
const added = currGroups.filter(g => !prevSet.has(g)).length;
|
|
372
|
-
const score = added * 10 + removed * 10 + changed * 8;
|
|
373
|
-
if (score >= 15) {
|
|
374
|
-
window[KEY].changed = Date.now();
|
|
375
|
-
}
|
|
376
|
-
} catch {}
|
|
377
|
-
}
|
|
378
|
-
const obs = new MutationObserver((mutations) => {
|
|
379
|
-
let hasRelevant = false;
|
|
380
|
-
for (const m of mutations) {
|
|
381
|
-
if (m.target instanceof Element) {
|
|
382
|
-
const tag = m.target.tagName;
|
|
383
|
-
if (tag === 'SCRIPT' || tag === 'STYLE' || tag === 'HEAD') continue;
|
|
384
|
-
const cls = m.target.className || '';
|
|
385
|
-
if (typeof cls === 'string' && IGNORE.test(cls)) continue;
|
|
386
|
-
const id = m.target.id || '';
|
|
387
|
-
if (IGNORE.test(id)) continue;
|
|
388
|
-
if (m.type === 'characterData' && m.target.parentElement) {
|
|
389
|
-
const ptag = m.target.parentElement.tagName;
|
|
390
|
-
if (ptag === 'SCRIPT' || ptag === 'STYLE') continue;
|
|
391
|
-
}
|
|
392
|
-
}
|
|
393
|
-
hasRelevant = true;
|
|
394
|
-
}
|
|
395
|
-
if (hasRelevant) {
|
|
396
|
-
clearTimeout(timer);
|
|
397
|
-
timer = setTimeout(check, 500);
|
|
398
|
-
}
|
|
399
|
-
});
|
|
400
|
-
obs.observe(document.body || document.documentElement, {
|
|
401
|
-
childList: true, subtree: true, characterData: true
|
|
402
|
-
});
|
|
403
|
-
window[KEY] = { observer: obs, changed: 0 };
|
|
404
|
-
lastText = getText();
|
|
405
|
-
return 'ok';
|
|
406
|
-
})()
|
|
407
|
-
`);
|
|
387
|
+
await cdpEval(page, (0, screen_change_1.buildObserverScript)());
|
|
408
388
|
console.log('[browse] MutationObserver injected (stealth, debounced, filtered)');
|
|
409
389
|
}
|
|
410
390
|
catch (err) {
|
|
411
391
|
console.error(`[browse] MutationObserver injection failed: ${(0, config_1.msg)(err)}`);
|
|
412
392
|
}
|
|
413
393
|
}
|
|
394
|
+
/**
|
|
395
|
+
* Debounced observer re-inject after a main-frame navigation: `frameNavigated`
|
|
396
|
+
* can fire before the new document is ready, and rapid redirects would
|
|
397
|
+
* otherwise inject several times.
|
|
398
|
+
*/
|
|
399
|
+
function scheduleObserverReinject(page) {
|
|
400
|
+
if (page._reinjectTimer)
|
|
401
|
+
clearTimeout(page._reinjectTimer);
|
|
402
|
+
page._reinjectTimer = setTimeout(() => {
|
|
403
|
+
page._reinjectTimer = undefined;
|
|
404
|
+
if (!page.disconnected)
|
|
405
|
+
void reinjectObserver(page);
|
|
406
|
+
}, 600);
|
|
407
|
+
}
|
|
414
408
|
async function cmdBrowse(flags) {
|
|
415
409
|
const autoAnswer = flags.get('auto-answer') === 'true';
|
|
416
410
|
if (!autoAnswer) {
|
|
@@ -475,13 +469,18 @@ async function cmdBrowse(flags) {
|
|
|
475
469
|
// the fast retry below abandons a slow easy answer at ~45s and escalates.
|
|
476
470
|
const timeoutSec = Math.min(Math.max(parseInt(flags.get('timeout') ?? '300', 10), 30), 600);
|
|
477
471
|
const timeoutMs = timeoutSec * 1000;
|
|
478
|
-
const quality = Math.min(Math.max(parseInt(flags.get('quality') ?? '
|
|
472
|
+
const quality = Math.min(Math.max(parseInt(flags.get('quality') ?? '82', 10), 50), 100);
|
|
479
473
|
/**
|
|
480
|
-
*
|
|
481
|
-
*
|
|
482
|
-
*
|
|
474
|
+
* Screenshot detail: the viewport is captured at native resolution and only
|
|
475
|
+
* scaled down when BOTH sides exceed this many px. Default 1600 keeps
|
|
476
|
+
* question/option text crisp for the vision model; pass `--max-edge 0` for
|
|
477
|
+
* native full resolution, or a smaller value for speed. Clicks are unaffected
|
|
478
|
+
* (actions use element coordinates, not image pixels).
|
|
483
479
|
*/
|
|
484
|
-
const
|
|
480
|
+
const maxEdgeRaw = parseInt(flags.get('max-edge') ?? '1600', 10);
|
|
481
|
+
const SCREENSHOT_MAX_EDGE = Number.isFinite(maxEdgeRaw)
|
|
482
|
+
? Math.max(0, Math.min(maxEdgeRaw, 4096))
|
|
483
|
+
: 1600;
|
|
485
484
|
const maxConsecutiveFailures = parseInt(flags.get('max-consecutive-failures') ?? '0', 10); // 0 = unlimited
|
|
486
485
|
const charDelayMs = Math.min(Math.max(parseInt(flags.get('char-delay') ?? '150', 10), 5), 200);
|
|
487
486
|
const optimize = flags.has('optimize');
|
|
@@ -711,7 +710,7 @@ async function cmdBrowse(flags) {
|
|
|
711
710
|
const currentFingerprint = createFingerprint(elements);
|
|
712
711
|
if (lastFingerprint) {
|
|
713
712
|
const changeScore = calculateChangeScore(lastFingerprint, currentFingerprint);
|
|
714
|
-
if (changeScore >= 15) {
|
|
713
|
+
if (questionSetChanged(lastFingerprint, currentFingerprint) || changeScore >= 15) {
|
|
715
714
|
console.log(`[browse] Screen changed (score: ${changeScore}) — resetting state`);
|
|
716
715
|
codeConfirmedOptimal = false;
|
|
717
716
|
consecutiveFailures = 0;
|
|
@@ -731,20 +730,17 @@ async function cmdBrowse(flags) {
|
|
|
731
730
|
/** The page changed under the LLM: re-analyze immediately, no backoff. */
|
|
732
731
|
let screenChangedDuringLlm = false;
|
|
733
732
|
for (let attempt = 1; attempt <= maxRetries; attempt++) {
|
|
734
|
-
// Fast-first ladder: attempt 1 uses a small budget +
|
|
735
|
-
//
|
|
736
|
-
//
|
|
737
|
-
//
|
|
733
|
+
// Fast-first ladder: attempt 1 uses a small token budget; attempts 2+
|
|
734
|
+
// switch to the DEEP budget (full timeout + tokens) for hard coding
|
|
735
|
+
// questions. Both use the SAME full-detail screenshot (maxEdge/quality)
|
|
736
|
+
// so text is always legible; `max_tokens`/timeout are ceilings, so the
|
|
738
737
|
// deep path never slows a question that answers on the fast path.
|
|
739
738
|
const deep = attempt > 1;
|
|
740
739
|
const attemptTimeoutMs = deep ? timeoutMs : Math.min(FAST_ATTEMPT_TIMEOUT_MS, timeoutMs);
|
|
741
740
|
if (deep && attempt === 2) {
|
|
742
741
|
console.log('[browse] Fast attempt did not answer — escalating to the deep budget (more time + tokens)');
|
|
743
742
|
}
|
|
744
|
-
|
|
745
|
-
? ((await captureScreenshot(page, quality)) ?? screenshot)
|
|
746
|
-
: screenshot;
|
|
747
|
-
result = await analyzeScreenshot(baseUrl, apiKey, model, attemptShot, elements, pageContext, attemptTimeoutMs, attempt, maxRetries, page, lastFingerprint, { maxTokens: deep ? DEEP_MAX_TOKENS : FAST_MAX_TOKENS, deep });
|
|
743
|
+
result = await analyzeScreenshot(baseUrl, apiKey, model, screenshot, elements, pageContext, attemptTimeoutMs, attempt, maxRetries, page, lastFingerprint, { maxTokens: deep ? DEEP_MAX_TOKENS : FAST_MAX_TOKENS, deep });
|
|
748
744
|
llmCallCount++;
|
|
749
745
|
llmStats.calls++;
|
|
750
746
|
llmStats.totalMs += result.durationMs;
|
|
@@ -822,10 +818,12 @@ async function cmdBrowse(flags) {
|
|
|
822
818
|
const freshFingerprint = createFingerprint(freshElements);
|
|
823
819
|
if (lastFingerprint) {
|
|
824
820
|
const changeScore = calculateChangeScore(lastFingerprint, freshFingerprint);
|
|
825
|
-
//
|
|
826
|
-
//
|
|
827
|
-
|
|
828
|
-
|
|
821
|
+
// Discard a (costly) LLM answer when the QUESTION SET changed under
|
|
822
|
+
// it (its answer is stale), or on a material whole-screen change.
|
|
823
|
+
// A pure question-text swap scores only 8 on the generic scale, so
|
|
824
|
+
// the targeted check catches what the score alone missed.
|
|
825
|
+
if (questionSetChanged(lastFingerprint, freshFingerprint) || changeScore >= 40) {
|
|
826
|
+
console.log(`[browse] Question set changed during LLM call (score: ${changeScore}) — skipping actions, re-analyzing`);
|
|
829
827
|
lastFingerprint = freshFingerprint;
|
|
830
828
|
lastScreenshot = freshScreenshot;
|
|
831
829
|
await sleep(intervalMs);
|
|
@@ -844,6 +842,10 @@ async function cmdBrowse(flags) {
|
|
|
844
842
|
// Process all actions
|
|
845
843
|
let executedAny = false;
|
|
846
844
|
let wroteCode = false;
|
|
845
|
+
// Set when an action advanced the page (navigation / question-set change);
|
|
846
|
+
// the rest of this action batch is then DROPPED so we don't answer the
|
|
847
|
+
// next question with answers derived from the previous screen.
|
|
848
|
+
let screenAdvancedDuringActions = false;
|
|
847
849
|
for (const act of result.actions) {
|
|
848
850
|
if (act.action === 'done') {
|
|
849
851
|
// Validate: are there actually unanswered questions? (The heuristic is
|
|
@@ -856,6 +858,7 @@ async function cmdBrowse(flags) {
|
|
|
856
858
|
// Fast budget: this is a short "act now" nudge, not a hard problem.
|
|
857
859
|
{ maxTokens: FAST_MAX_TOKENS, deep: false });
|
|
858
860
|
const followActions = followUp.actions.filter((a) => a.action !== 'done');
|
|
861
|
+
const followSaidDone = followUp.actions.some((a) => a.action === 'done');
|
|
859
862
|
if (followActions.length > 0) {
|
|
860
863
|
// Execute the follow-up actions here — the old code reassigned
|
|
861
864
|
// `result` inside the for-of, so these were silently discarded and
|
|
@@ -869,6 +872,14 @@ async function cmdBrowse(flags) {
|
|
|
869
872
|
}
|
|
870
873
|
continue; // re-analyze the screen after acting
|
|
871
874
|
}
|
|
875
|
+
if (!followSaidDone) {
|
|
876
|
+
// The follow-up was ABORTED (screen changed) / timed out / failed to
|
|
877
|
+
// parse - that is NOT a confirmation of "done". Treating it as one
|
|
878
|
+
// silently advanced the loop with questions still open.
|
|
879
|
+
console.log(`[browse] Follow-up did not answer (${followUp.error ?? 'no actions'}) — re-analyzing instead of trusting "done"`);
|
|
880
|
+
await sleep(intervalMs);
|
|
881
|
+
continue;
|
|
882
|
+
}
|
|
872
883
|
console.log(`[browse] LLM confirms done after verification`);
|
|
873
884
|
}
|
|
874
885
|
else if (hasUnansweredQuestions(elements)) {
|
|
@@ -1032,34 +1043,53 @@ async function cmdBrowse(flags) {
|
|
|
1032
1043
|
if (act.action === 'write_code' || act.action === 'type') {
|
|
1033
1044
|
wroteCode = true;
|
|
1034
1045
|
}
|
|
1035
|
-
// Verify a click
|
|
1036
|
-
//
|
|
1037
|
-
//
|
|
1038
|
-
//
|
|
1046
|
+
// Verify a click. A successful answer either (a) flips the clicked
|
|
1047
|
+
// control's own state (aria/checked) or (b) advances the page (the
|
|
1048
|
+
// question set changes / a navigation happens). Only when NEITHER is
|
|
1049
|
+
// true did the click genuinely miss - only then do we fall back to a
|
|
1050
|
+
// real DOM click. The old check used the whole-screen score < 15, which
|
|
1051
|
+
// a radio/checkbox selection (score ~2) always failed - so a working
|
|
1052
|
+
// click was "retried" onto the NEXT screen at stale coordinates.
|
|
1039
1053
|
if (act.action === 'click' && act.elementIndex !== undefined) {
|
|
1040
1054
|
const idx = act.elementIndex;
|
|
1055
|
+
const clickedEl = elements.find((e) => e.i === idx);
|
|
1056
|
+
const clickedAt = Date.now();
|
|
1041
1057
|
await sleep(400);
|
|
1042
1058
|
try {
|
|
1043
1059
|
const after = await extractElements(page);
|
|
1044
|
-
const
|
|
1045
|
-
|
|
1060
|
+
const afterFp = createFingerprint(after);
|
|
1061
|
+
const beforeFp = createFingerprint(elements);
|
|
1062
|
+
const score = calculateChangeScore(beforeFp, afterFp);
|
|
1063
|
+
const advanced = questionSetChanged(beforeFp, afterFp) ||
|
|
1064
|
+
score >= 15 ||
|
|
1065
|
+
(page._cdpNavigated ?? 0) > clickedAt;
|
|
1066
|
+
// The same control, re-found by text/type, now selected?
|
|
1067
|
+
const sameControl = clickedEl
|
|
1068
|
+
? after.find((e) => e.text === clickedEl.text && e.type === clickedEl.type)
|
|
1069
|
+
: undefined;
|
|
1070
|
+
const selectionChanged = !!clickedEl && !!sameControl && sameControl.checked !== clickedEl.checked;
|
|
1071
|
+
if (advanced || selectionChanged) {
|
|
1072
|
+
// Worked (answer selected, or the page moved on). Do NOT re-click.
|
|
1073
|
+
clickFailures.delete(idx);
|
|
1074
|
+
if (advanced) {
|
|
1075
|
+
screenAdvancedDuringActions = true;
|
|
1076
|
+
console.log('[browse] Screen advanced after the click — dropping the rest of this batch and re-analyzing');
|
|
1077
|
+
}
|
|
1078
|
+
}
|
|
1079
|
+
else {
|
|
1046
1080
|
const n = (clickFailures.get(idx) ?? 0) + 1;
|
|
1047
1081
|
clickFailures.set(idx, n);
|
|
1048
|
-
console.log(`[browse] Click on element ${idx}
|
|
1049
|
-
|
|
1050
|
-
|
|
1051
|
-
const ok = await domClickAt(page, el.x, el.y);
|
|
1082
|
+
console.log(`[browse] Click on element ${idx} had no effect (${n}/3)`);
|
|
1083
|
+
if (clickedEl && n <= 2) {
|
|
1084
|
+
const ok = await domClickAt(page, clickedEl.x, clickedEl.y);
|
|
1052
1085
|
console.log(`[browse] DOM-click fallback on element ${idx}: ${ok ? 'dispatched' : 'failed'}`);
|
|
1053
1086
|
}
|
|
1054
|
-
else if (
|
|
1087
|
+
else if (clickedEl && n >= 3) {
|
|
1055
1088
|
clickCooldown.set(idx, Date.now() + 30_000);
|
|
1056
1089
|
clickFailures.delete(idx);
|
|
1057
1090
|
console.log(`[browse] Element ${idx} not responding — skipping for 30s`);
|
|
1058
1091
|
}
|
|
1059
1092
|
}
|
|
1060
|
-
else {
|
|
1061
|
-
clickFailures.delete(idx);
|
|
1062
|
-
}
|
|
1063
1093
|
}
|
|
1064
1094
|
catch {
|
|
1065
1095
|
/* verification is best-effort */
|
|
@@ -1069,7 +1099,14 @@ async function cmdBrowse(flags) {
|
|
|
1069
1099
|
if (act.action === 'click' || act.action === 'type' || act.action === 'write_code') {
|
|
1070
1100
|
await sleep(ANSWER_PAUSE_MS);
|
|
1071
1101
|
}
|
|
1102
|
+
// The screen advanced: stop applying the rest of this (now stale) batch.
|
|
1103
|
+
if (screenAdvancedDuringActions)
|
|
1104
|
+
break;
|
|
1072
1105
|
}
|
|
1106
|
+
// Go straight back to re-screenshot + re-analyze the NEW screen (skip the
|
|
1107
|
+
// optimize/cooldown passes, which belong to the old screen).
|
|
1108
|
+
if (screenAdvancedDuringActions)
|
|
1109
|
+
continue;
|
|
1073
1110
|
// Optimization pass: if --optimize and code was written, ask LLM to optimize it
|
|
1074
1111
|
if (optimize && wroteCode) {
|
|
1075
1112
|
console.log('[browse] Running optimization pass...');
|
|
@@ -2625,6 +2662,26 @@ async function extractPageContext(page) {
|
|
|
2625
2662
|
break;
|
|
2626
2663
|
}
|
|
2627
2664
|
}
|
|
2665
|
+
// FULL PAGE CONTENT (2026-09): the model must be fully informed from the
|
|
2666
|
+
// PAGE TEXT, not just the screenshot or the element list. Prefer the main
|
|
2667
|
+
// content region, else the body; collapse whitespace and cap the size.
|
|
2668
|
+
try {
|
|
2669
|
+
const root =
|
|
2670
|
+
document.querySelector('main, article, [role="main"]') || document.body;
|
|
2671
|
+
let text = (root && (root.innerText || root.textContent)) || '';
|
|
2672
|
+
text = text
|
|
2673
|
+
.replace(/[ \\t\\u00a0]+/g, ' ')
|
|
2674
|
+
.replace(/\\n\\s*\\n+/g, '\\n')
|
|
2675
|
+
.trim();
|
|
2676
|
+
if (text) {
|
|
2677
|
+
const CAP = 12000;
|
|
2678
|
+
ctx.push('Page content:');
|
|
2679
|
+
ctx.push(text.slice(0, CAP));
|
|
2680
|
+
if (text.length > CAP) {
|
|
2681
|
+
ctx.push('... (page content truncated at ' + CAP + ' chars)');
|
|
2682
|
+
}
|
|
2683
|
+
}
|
|
2684
|
+
} catch (e) { /* best-effort */ }
|
|
2628
2685
|
return ctx.join('\\n');
|
|
2629
2686
|
})()
|
|
2630
2687
|
`;
|
|
@@ -2868,9 +2925,9 @@ function extractModelText(json) {
|
|
|
2868
2925
|
* Non-streaming never resolves until the provider finishes the whole answer, so
|
|
2869
2926
|
* a long reasoning response looked like a 150s+ "hang" with zero feedback. With
|
|
2870
2927
|
* streaming we see the first token, log its latency, and can catch a genuinely
|
|
2871
|
-
* stalled request
|
|
2872
|
-
*
|
|
2873
|
-
*
|
|
2928
|
+
* stalled request (no bytes on the wire at all). `: PROCESSING` keepalives are
|
|
2929
|
+
* NOT tokens but DO prove the connection is alive, so they reset the watchdog;
|
|
2930
|
+
* only real wire silence trips it.
|
|
2874
2931
|
*
|
|
2875
2932
|
* The caller's `signal` (screen-change / overall timeout) still aborts this.
|
|
2876
2933
|
*/
|
|
@@ -2885,13 +2942,21 @@ async function streamChatCompletion(opts) {
|
|
|
2885
2942
|
let sawToken = false;
|
|
2886
2943
|
let watchdog = null;
|
|
2887
2944
|
const start = Date.now();
|
|
2888
|
-
|
|
2945
|
+
/**
|
|
2946
|
+
* Any bytes on the wire (including `: PROCESSING` keepalive comments) prove
|
|
2947
|
+
* the connection is alive, so they RESET the watchdog. This provider sends
|
|
2948
|
+
* keepalives every few seconds and only emits real tokens much later - the
|
|
2949
|
+
* old watchdog ignored comments and killed a healthy request at 60s
|
|
2950
|
+
* ("LLM stalled (no token received)" while the model was actively working).
|
|
2951
|
+
* The stall now means genuine wire silence: no bytes at all.
|
|
2952
|
+
*/
|
|
2953
|
+
const arm = () => {
|
|
2889
2954
|
if (watchdog)
|
|
2890
2955
|
clearTimeout(watchdog);
|
|
2891
2956
|
watchdog = setTimeout(() => {
|
|
2892
2957
|
stalled = sawToken ? 'idle' : 'first-token';
|
|
2893
2958
|
ac.abort();
|
|
2894
|
-
},
|
|
2959
|
+
}, sawToken ? opts.idleTimeoutMs : opts.firstTokenTimeoutMs);
|
|
2895
2960
|
};
|
|
2896
2961
|
const cleanup = () => {
|
|
2897
2962
|
if (watchdog)
|
|
@@ -2899,7 +2964,7 @@ async function streamChatCompletion(opts) {
|
|
|
2899
2964
|
watchdog = null;
|
|
2900
2965
|
opts.signal.removeEventListener('abort', forwardAbort);
|
|
2901
2966
|
};
|
|
2902
|
-
arm(
|
|
2967
|
+
arm();
|
|
2903
2968
|
try {
|
|
2904
2969
|
const res = await fetch(opts.url, {
|
|
2905
2970
|
method: 'POST',
|
|
@@ -2933,6 +2998,9 @@ async function streamChatCompletion(opts) {
|
|
|
2933
2998
|
const { done, value } = await reader.read();
|
|
2934
2999
|
if (done)
|
|
2935
3000
|
break;
|
|
3001
|
+
// Any bytes at all (comment keepalives included) = the connection is
|
|
3002
|
+
// alive, so reset the stall watchdog.
|
|
3003
|
+
arm();
|
|
2936
3004
|
buffer += decoder.decode(value, { stream: true });
|
|
2937
3005
|
let nl;
|
|
2938
3006
|
while ((nl = buffer.indexOf('\n')) >= 0) {
|
|
@@ -2957,28 +3025,29 @@ async function streamChatCompletion(opts) {
|
|
|
2957
3025
|
const delta = j?.choices?.[0]?.delta;
|
|
2958
3026
|
const dc = delta?.content;
|
|
2959
3027
|
const dr = delta?.reasoning_content;
|
|
2960
|
-
let progressed = false;
|
|
2961
3028
|
if (typeof dc === 'string' && dc) {
|
|
2962
3029
|
content += dc;
|
|
2963
|
-
|
|
3030
|
+
if (!sawToken) {
|
|
3031
|
+
sawToken = true;
|
|
3032
|
+
opts.onFirstToken?.(Date.now() - start);
|
|
3033
|
+
}
|
|
2964
3034
|
}
|
|
2965
3035
|
else if (Array.isArray(dc)) {
|
|
2966
3036
|
const t = dc.map((x) => x?.text ?? '').join('');
|
|
2967
3037
|
if (t) {
|
|
2968
3038
|
content += t;
|
|
2969
|
-
|
|
3039
|
+
if (!sawToken) {
|
|
3040
|
+
sawToken = true;
|
|
3041
|
+
opts.onFirstToken?.(Date.now() - start);
|
|
3042
|
+
}
|
|
2970
3043
|
}
|
|
2971
3044
|
}
|
|
2972
3045
|
if (typeof dr === 'string' && dr) {
|
|
2973
3046
|
reasoning += dr;
|
|
2974
|
-
progressed = true;
|
|
2975
|
-
}
|
|
2976
|
-
if (progressed) {
|
|
2977
3047
|
if (!sawToken) {
|
|
2978
3048
|
sawToken = true;
|
|
2979
3049
|
opts.onFirstToken?.(Date.now() - start);
|
|
2980
3050
|
}
|
|
2981
|
-
arm(opts.idleTimeoutMs);
|
|
2982
3051
|
}
|
|
2983
3052
|
}
|
|
2984
3053
|
}
|
|
@@ -3092,6 +3161,7 @@ async function analyzeScreenshot(baseUrl, apiKey, model, screenshotBase64, eleme
|
|
|
3092
3161
|
controller.abort();
|
|
3093
3162
|
}, timeoutMs);
|
|
3094
3163
|
// Heartbeat: check for screen changes every 5s, abort LLM if page changed
|
|
3164
|
+
let heartbeatTicks = 0;
|
|
3095
3165
|
const heartbeat = setInterval(async () => {
|
|
3096
3166
|
const elapsed = Math.round((Date.now() - startTime) / 1000);
|
|
3097
3167
|
// Check for screen change via CDP navigation event or MutationObserver flag
|
|
@@ -3128,14 +3198,26 @@ async function analyzeScreenshot(baseUrl, apiKey, model, screenshotBase64, eleme
|
|
|
3128
3198
|
}
|
|
3129
3199
|
catch { /* page disconnected */ }
|
|
3130
3200
|
}
|
|
3131
|
-
|
|
3201
|
+
// Fallback (2026-09): the in-page observer is document-scoped (re-injected on
|
|
3202
|
+
// navigation, but a same-document SPA swap may not have ticked it yet) and a
|
|
3203
|
+
// continuously-mutating page can starve its debounce. So every ~10s verify the
|
|
3204
|
+
// fingerprint directly even when no flag fired - a question swap during a
|
|
3205
|
+
// 50-120s vision call must never slip through unnoticed.
|
|
3206
|
+
heartbeatTicks++;
|
|
3207
|
+
const periodicCheck = !screenChanged && heartbeatTicks % 2 === 0;
|
|
3208
|
+
if ((screenChanged || periodicCheck) && lastFingerprint && page) {
|
|
3132
3209
|
// Verify with fingerprint comparison before aborting
|
|
3133
3210
|
try {
|
|
3134
3211
|
const freshElements = await extractElements(page);
|
|
3135
3212
|
const freshFingerprint = createFingerprint(freshElements);
|
|
3136
3213
|
const changeScore = calculateChangeScore(lastFingerprint, freshFingerprint);
|
|
3137
|
-
|
|
3138
|
-
|
|
3214
|
+
// Abort on a real question change even when the aggregate score is below
|
|
3215
|
+
// the generic threshold (a pure question-text swap scores only 8), while
|
|
3216
|
+
// still ignoring checked-only / timer noise.
|
|
3217
|
+
if (questionSetChanged(lastFingerprint, freshFingerprint) || changeScore >= 15) {
|
|
3218
|
+
console.log(screenChanged
|
|
3219
|
+
? `[browse] Fingerprint confirms change (score: ${changeScore}) — aborting LLM (${elapsed}s)`
|
|
3220
|
+
: `[browse] Periodic fingerprint found change (score: ${changeScore}) — aborting LLM (${elapsed}s)`);
|
|
3139
3221
|
abortReason = 'screen_change';
|
|
3140
3222
|
controller.abort();
|
|
3141
3223
|
return;
|
|
@@ -3244,9 +3326,17 @@ async function analyzeScreenshotWithPrompt(baseUrl, apiKey, model, screenshotBas
|
|
|
3244
3326
|
optAbortReason = 'timeout';
|
|
3245
3327
|
controller.abort();
|
|
3246
3328
|
}, timeoutMs);
|
|
3247
|
-
// Heartbeat: check for screen changes every 5s, abort if page changed
|
|
3329
|
+
// Heartbeat: check for screen changes every 5s, abort if page changed.
|
|
3330
|
+
// The observer flag lives in the current document, so any navigation (which
|
|
3331
|
+
// wipes it) must also abort - check `_cdpNavigated` too, as in the main call.
|
|
3248
3332
|
const heartbeat = setInterval(async () => {
|
|
3249
3333
|
const elapsed = Math.round((Date.now() - startTime) / 1000);
|
|
3334
|
+
if (page?._cdpNavigated && page._cdpNavigated > startTime) {
|
|
3335
|
+
console.log(`[browse] CDP navigation detected in opt pass (${elapsed}s) — aborting`);
|
|
3336
|
+
optAbortReason = 'screen_change';
|
|
3337
|
+
controller.abort();
|
|
3338
|
+
return;
|
|
3339
|
+
}
|
|
3250
3340
|
try {
|
|
3251
3341
|
const flagResult = await cdpEval(page, `
|
|
3252
3342
|
(() => {
|