teamshare-bridge 0.21.24 → 0.21.27
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cli/commands/browse.js +658 -161
- package/dist/cli/commands/browse.js.map +1 -1
- package/dist/cli/commands/research.js +5 -0
- package/dist/cli/commands/research.js.map +1 -1
- package/dist/cli/index.js +6 -0
- package/dist/cli/index.js.map +1 -1
- package/dist/lib/browser/cdp.d.ts +29 -2
- package/dist/lib/browser/cdp.js +133 -6
- package/dist/lib/browser/cdp.js.map +1 -1
- package/dist/lib/browser/human-input-guard.d.ts +72 -0
- package/dist/lib/browser/human-input-guard.js +344 -0
- package/dist/lib/browser/human-input-guard.js.map +1 -0
- package/dist/lib/browser/personal-fields.d.ts +78 -0
- package/dist/lib/browser/personal-fields.js +267 -0
- package/dist/lib/browser/personal-fields.js.map +1 -0
- package/package.json +3 -2
|
@@ -59,6 +59,8 @@ const child_process_1 = require("child_process");
|
|
|
59
59
|
const cdp_1 = require("../../lib/browser/cdp");
|
|
60
60
|
const launch_1 = require("../../lib/browser/launch");
|
|
61
61
|
const session_1 = require("../../lib/browser/session");
|
|
62
|
+
const personal_fields_1 = require("../../lib/browser/personal-fields");
|
|
63
|
+
const human_input_guard_1 = require("../../lib/browser/human-input-guard");
|
|
62
64
|
/** Create a fingerprint from the current element list. */
|
|
63
65
|
function createFingerprint(elements) {
|
|
64
66
|
const groups = new Map();
|
|
@@ -119,6 +121,7 @@ const SYSTEM_PROMPT = `You are a browser automation agent. A screenshot and a gr
|
|
|
119
121
|
Your task: identify unanswered questions and answer them by clicking the correct elements.
|
|
120
122
|
|
|
121
123
|
RULES (STRICT):
|
|
124
|
+
0. BE FAST AND DECISIVE. Answer in a single pass with minimal reasoning — do NOT deliberate at length, re-check repeatedly, or restate the page. Emit the final JSON array as your primary output. Speed matters more than exhaustive deliberation.
|
|
122
125
|
1. NEVER click Next, Submit, Back, Save, Continue, or any navigation. ONLY answer questions on the current screen.
|
|
123
126
|
2. NEVER ask for help. If uncertain, pick the BEST GUESS. Always decide — never wait.
|
|
124
127
|
3. Handle MULTIPLE unanswered questions in a SINGLE response.
|
|
@@ -238,10 +241,10 @@ CORRECTION MODE:
|
|
|
238
241
|
- If you previously typed wrong code, use write_code to replace with correct code
|
|
239
242
|
- You may be asked the same question multiple times — each time, check if the current answer is correct before acting
|
|
240
243
|
|
|
241
|
-
PERSONAL QUESTIONS —
|
|
242
|
-
-
|
|
243
|
-
- If a question asks for personal info, do NOT click or type —
|
|
244
|
-
- If ALL remaining questions are personal, return done
|
|
244
|
+
PERSONAL QUESTIONS — NEVER ANSWER THEM:
|
|
245
|
+
- Questions about the person operating the browser (name, email, phone, address, SSN, passport, password, card, bank, date of birth, salary, work authorization, personal profile links) are OFF LIMITS. Read the PERSONAL QUESTIONS addendum appended below and follow it exactly.
|
|
246
|
+
- If a question asks for personal info, do NOT click or type — leave it empty.
|
|
247
|
+
- A personal field left empty is NOT an unanswered question. If ALL remaining questions are personal, return done immediately.
|
|
245
248
|
|
|
246
249
|
RESPONSE FORMAT RULES (CRITICAL — follow exactly):
|
|
247
250
|
- Return ONLY a valid JSON array. No explanations, no markdown fences, no extra text.
|
|
@@ -257,6 +260,133 @@ RESPONSE FORMAT RULES (CRITICAL — follow exactly):
|
|
|
257
260
|
- RIGHT: {"text":"say \\"hello\\""}
|
|
258
261
|
- NEVER put raw newlines or unescaped quotes inside the text string.
|
|
259
262
|
- If you need to explain your reasoning, use the "reason" field, NOT the "text" field.`;
|
|
263
|
+
/**
|
|
264
|
+
* Enable the Page domain and install the per-tab instrumentation the loop
|
|
265
|
+
* relies on: the stealth console wrapper, the Page.frameNavigated listener and
|
|
266
|
+
* the debounced MutationObserver. Extracted so every (re)attach - including a
|
|
267
|
+
* tab switch/close - re-runs it on the NEW tab. All of it must be idempotent
|
|
268
|
+
* (the injected scripts already early-return when their Symbol key exists).
|
|
269
|
+
*/
|
|
270
|
+
async function instrumentPage(page) {
|
|
271
|
+
try {
|
|
272
|
+
await cdpSend(page, 'Page.enable', {});
|
|
273
|
+
console.log('[browse] Page domain enabled');
|
|
274
|
+
}
|
|
275
|
+
catch (err) {
|
|
276
|
+
console.error(`[browse] Page.enable failed: ${(0, config_1.msg)(err)}`);
|
|
277
|
+
}
|
|
278
|
+
// Stealth: install stack trace sanitization to hide CDP frames (global, once)
|
|
279
|
+
installStackTraceStealth();
|
|
280
|
+
// Stealth: suppress console serialization leak caused by CDP's Runtime.enable.
|
|
281
|
+
// Anti-bot probe: Object.defineProperty(obj,'x',{get(){window.__detected=1}});
|
|
282
|
+
// console.debug(obj); if(window.__detected) → CDP is serializing args
|
|
283
|
+
// Countermeasure: wrap console methods to prevent deep serialization.
|
|
284
|
+
try {
|
|
285
|
+
await cdpEval(page, `
|
|
286
|
+
(() => {
|
|
287
|
+
const CK = Symbol('__ts_cp');
|
|
288
|
+
if (window[CK]) return;
|
|
289
|
+
const wrap = (fn) => function(...args) {
|
|
290
|
+
return fn.apply(console, args.map(a => {
|
|
291
|
+
if (a && typeof a === 'object' && typeof a !== 'function') {
|
|
292
|
+
try { return Object.assign(Array.isArray(a) ? [] : {}, a); } catch { return a; }
|
|
293
|
+
}
|
|
294
|
+
return a;
|
|
295
|
+
}));
|
|
296
|
+
};
|
|
297
|
+
console.debug = wrap(console.debug);
|
|
298
|
+
console.log = wrap(console.log);
|
|
299
|
+
console.warn = wrap(console.warn);
|
|
300
|
+
console.error = wrap(console.error);
|
|
301
|
+
window[CK] = true;
|
|
302
|
+
})()
|
|
303
|
+
`);
|
|
304
|
+
}
|
|
305
|
+
catch { /* non-critical */ }
|
|
306
|
+
// Register CDP event: Page.frameNavigated (URL/navigation changes)
|
|
307
|
+
onCdpEvent(page, 'Page.frameNavigated', () => {
|
|
308
|
+
page._cdpNavigated = Date.now();
|
|
309
|
+
});
|
|
310
|
+
// Inject MutationObserver: watches DOM for significant content changes
|
|
311
|
+
// Uses same fingerprint comparison as Node.js side to prevent false positives.
|
|
312
|
+
// Filters out: timers, animations, scripts, styles, notifications, toasts.
|
|
313
|
+
// Debounces mutations (500ms) to batch rapid changes.
|
|
314
|
+
// Stealth: uses Symbol-keyed property (invisible to Object.keys / anti-bot scans).
|
|
315
|
+
try {
|
|
316
|
+
await cdpEval(page, `
|
|
317
|
+
(() => {
|
|
318
|
+
const KEY = Symbol('__ts_obs');
|
|
319
|
+
if (window[KEY]) return 'already attached';
|
|
320
|
+
const IGNORE = /timer|countdown|elapsed|clock|animation|animate|transition|spinner|progress|loading|toast|notification|alert|banner|snackbar|cookie|consent|modal|overlay|popup|tooltip|badge|tag|avatar|icon|svg|img|video|audio|canvas|font|stylesheet|script|meta|link|head/i;
|
|
321
|
+
let timer = null;
|
|
322
|
+
let lastText = '';
|
|
323
|
+
function getText() {
|
|
324
|
+
const els = document.querySelectorAll('input[type="radio"],input[type="checkbox"],input[type="text"],input[type="email"],input[type="password"],textarea,label,button');
|
|
325
|
+
return [...els].map(e => {
|
|
326
|
+
const t = (e.innerText || e.textContent || '').trim().replace(/\\s+/g, ' ').slice(0, 200);
|
|
327
|
+
return e.type + '|' + t + '|' + (e.checked || false);
|
|
328
|
+
}).join('\\n');
|
|
329
|
+
}
|
|
330
|
+
function check() {
|
|
331
|
+
try {
|
|
332
|
+
const curr = getText();
|
|
333
|
+
if (curr === lastText) return;
|
|
334
|
+
const prev = lastText;
|
|
335
|
+
lastText = curr;
|
|
336
|
+
const prevGroups = prev ? prev.split('\\n').filter(Boolean) : [];
|
|
337
|
+
const currGroups = curr.split('\\n').filter(Boolean);
|
|
338
|
+
const prevSet = new Set(prevGroups);
|
|
339
|
+
const currSet = new Set(currGroups);
|
|
340
|
+
let removed = 0, changed = 0;
|
|
341
|
+
for (const g of prevGroups) {
|
|
342
|
+
if (!currSet.has(g)) {
|
|
343
|
+
if (prevGroups.indexOf(g) < currGroups.length) changed++;
|
|
344
|
+
else removed++;
|
|
345
|
+
}
|
|
346
|
+
}
|
|
347
|
+
const added = currGroups.filter(g => !prevSet.has(g)).length;
|
|
348
|
+
const score = added * 10 + removed * 10 + changed * 8;
|
|
349
|
+
if (score >= 15) {
|
|
350
|
+
window[KEY].changed = Date.now();
|
|
351
|
+
}
|
|
352
|
+
} catch {}
|
|
353
|
+
}
|
|
354
|
+
const obs = new MutationObserver((mutations) => {
|
|
355
|
+
let hasRelevant = false;
|
|
356
|
+
for (const m of mutations) {
|
|
357
|
+
if (m.target instanceof Element) {
|
|
358
|
+
const tag = m.target.tagName;
|
|
359
|
+
if (tag === 'SCRIPT' || tag === 'STYLE' || tag === 'HEAD') continue;
|
|
360
|
+
const cls = m.target.className || '';
|
|
361
|
+
if (typeof cls === 'string' && IGNORE.test(cls)) continue;
|
|
362
|
+
const id = m.target.id || '';
|
|
363
|
+
if (IGNORE.test(id)) continue;
|
|
364
|
+
if (m.type === 'characterData' && m.target.parentElement) {
|
|
365
|
+
const ptag = m.target.parentElement.tagName;
|
|
366
|
+
if (ptag === 'SCRIPT' || ptag === 'STYLE') continue;
|
|
367
|
+
}
|
|
368
|
+
}
|
|
369
|
+
hasRelevant = true;
|
|
370
|
+
}
|
|
371
|
+
if (hasRelevant) {
|
|
372
|
+
clearTimeout(timer);
|
|
373
|
+
timer = setTimeout(check, 500);
|
|
374
|
+
}
|
|
375
|
+
});
|
|
376
|
+
obs.observe(document.body || document.documentElement, {
|
|
377
|
+
childList: true, subtree: true, characterData: true
|
|
378
|
+
});
|
|
379
|
+
window[KEY] = { observer: obs, changed: 0 };
|
|
380
|
+
lastText = getText();
|
|
381
|
+
return 'ok';
|
|
382
|
+
})()
|
|
383
|
+
`);
|
|
384
|
+
console.log('[browse] MutationObserver injected (stealth, debounced, filtered)');
|
|
385
|
+
}
|
|
386
|
+
catch (err) {
|
|
387
|
+
console.error(`[browse] MutationObserver injection failed: ${(0, config_1.msg)(err)}`);
|
|
388
|
+
}
|
|
389
|
+
}
|
|
260
390
|
async function cmdBrowse(flags) {
|
|
261
391
|
const autoAnswer = flags.get('auto-answer') === 'true';
|
|
262
392
|
if (!autoAnswer) {
|
|
@@ -316,15 +446,33 @@ async function cmdBrowse(flags) {
|
|
|
316
446
|
const maxActions = continuous ? Number.MAX_SAFE_INTEGER : parseInt(flags.get('max-actions') ?? '50', 10);
|
|
317
447
|
const intervalMs = parseInt(flags.get('interval') ?? '2000', 10);
|
|
318
448
|
const modelSpec = flags.get('model') ?? models_1.DEFAULT_VISION_MODEL;
|
|
319
|
-
|
|
449
|
+
// Target: a decisive answer in well under a minute. 60s is the abandon
|
|
450
|
+
// threshold (overridable); the fast abort + short retry below makes giving up
|
|
451
|
+
// cheap, so we never wait 5 minutes for a stale decision.
|
|
452
|
+
const timeoutSec = Math.min(Math.max(parseInt(flags.get('timeout') ?? '60', 10), 20), 600);
|
|
320
453
|
const timeoutMs = timeoutSec * 1000;
|
|
321
|
-
const quality = Math.min(Math.max(parseInt(flags.get('quality') ?? '
|
|
454
|
+
const quality = Math.min(Math.max(parseInt(flags.get('quality') ?? '70', 10), 50), 100);
|
|
455
|
+
/**
|
|
456
|
+
* Downscale the screenshot when the viewport is bigger than this many px on
|
|
457
|
+
* its longest side (fewer image tokens => faster vision response on big
|
|
458
|
+
* pages; a 1440px page produced 100KB+ images that blew the timeout).
|
|
459
|
+
*/
|
|
460
|
+
const SCREENSHOT_MAX_EDGE = 1024;
|
|
322
461
|
const maxConsecutiveFailures = parseInt(flags.get('max-consecutive-failures') ?? '0', 10); // 0 = unlimited
|
|
323
462
|
const charDelayMs = Math.min(Math.max(parseInt(flags.get('char-delay') ?? '150', 10), 5), 200);
|
|
324
463
|
const optimize = flags.has('optimize');
|
|
325
464
|
const stealth = flags.has('stealth');
|
|
326
465
|
const hybrid = flags.has('hybrid');
|
|
327
466
|
const autoLaunch = flags.has('auto-launch');
|
|
467
|
+
// Field safety (see personal-fields.ts / human-input-guard.ts):
|
|
468
|
+
// --stop-on-user-input opt-in: stop typing and skip the field for the run
|
|
469
|
+
// as soon as the human types into it
|
|
470
|
+
// --personal-questions "allow" answers personal questions; credentials and
|
|
471
|
+
// card/bank fields stay blocked either way
|
|
472
|
+
const stopOnUserInput = flags.has('stop-on-user-input') && flags.get('stop-on-user-input') !== 'false';
|
|
473
|
+
const personalMode = (flags.get('personal-questions') ?? 'skip').toLowerCase();
|
|
474
|
+
fieldSafety.stopOnUserInput = stopOnUserInput;
|
|
475
|
+
fieldSafety.allowPersonal = personalMode === 'allow';
|
|
328
476
|
// The auto-answer browser is ALWAYS visible (2026-09): a headless window is
|
|
329
477
|
// the "profile seems mismatched" case and hides the pages the loop works on.
|
|
330
478
|
// `--headless` is accepted for backward compatibility but ignored.
|
|
@@ -356,6 +504,8 @@ async function cmdBrowse(flags) {
|
|
|
356
504
|
}
|
|
357
505
|
console.log(`[browse] Char delay: ${charDelayMs}ms`);
|
|
358
506
|
console.log(`[browse] Screenshot interval: ${intervalMs}ms`);
|
|
507
|
+
console.log(`[browse] Personal questions: ${fieldSafety.allowPersonal ? 'answered (credentials still blocked)' : 'never answered (left for you)'}`);
|
|
508
|
+
console.log(`[browse] Stop typing when you type in the field: ${fieldSafety.stopOnUserInput ? 'on' : 'off'}`);
|
|
359
509
|
if (!stealth) {
|
|
360
510
|
console.log(`[browse] Max consecutive failures: ${maxConsecutiveFailures === 0 ? 'unlimited' : maxConsecutiveFailures}`);
|
|
361
511
|
}
|
|
@@ -400,125 +550,9 @@ async function cmdBrowse(flags) {
|
|
|
400
550
|
process.exitCode = 1;
|
|
401
551
|
return;
|
|
402
552
|
}
|
|
403
|
-
// Enable Page
|
|
404
|
-
|
|
405
|
-
|
|
406
|
-
console.log('[browse] Page domain enabled');
|
|
407
|
-
}
|
|
408
|
-
catch (err) {
|
|
409
|
-
console.error(`[browse] Page.enable failed: ${(0, config_1.msg)(err)}`);
|
|
410
|
-
}
|
|
411
|
-
// Stealth: install stack trace sanitization to hide CDP frames
|
|
412
|
-
installStackTraceStealth();
|
|
413
|
-
// Stealth: suppress console serialization leak caused by CDP's Runtime.enable.
|
|
414
|
-
// Anti-bot probe: Object.defineProperty(obj,'x',{get(){window.__detected=1}});
|
|
415
|
-
// console.debug(obj); if(window.__detected) → CDP is serializing args
|
|
416
|
-
// Countermeasure: wrap console methods to prevent deep serialization.
|
|
417
|
-
try {
|
|
418
|
-
await cdpEval(page, `
|
|
419
|
-
(() => {
|
|
420
|
-
const CK = Symbol('__ts_cp');
|
|
421
|
-
if (window[CK]) return;
|
|
422
|
-
const wrap = (fn) => function(...args) {
|
|
423
|
-
return fn.apply(console, args.map(a => {
|
|
424
|
-
if (a && typeof a === 'object' && typeof a !== 'function') {
|
|
425
|
-
try { return Object.assign(Array.isArray(a) ? [] : {}, a); } catch { return a; }
|
|
426
|
-
}
|
|
427
|
-
return a;
|
|
428
|
-
}));
|
|
429
|
-
};
|
|
430
|
-
console.debug = wrap(console.debug);
|
|
431
|
-
console.log = wrap(console.log);
|
|
432
|
-
console.warn = wrap(console.warn);
|
|
433
|
-
console.error = wrap(console.error);
|
|
434
|
-
window[CK] = true;
|
|
435
|
-
})()
|
|
436
|
-
`);
|
|
437
|
-
}
|
|
438
|
-
catch { /* non-critical */ }
|
|
439
|
-
// Register CDP event: Page.frameNavigated (URL/navigation changes)
|
|
440
|
-
onCdpEvent(page, 'Page.frameNavigated', () => {
|
|
441
|
-
page._cdpNavigated = Date.now();
|
|
442
|
-
});
|
|
443
|
-
// Inject MutationObserver: watches DOM for significant content changes
|
|
444
|
-
// Uses same fingerprint comparison as Node.js side to prevent false positives.
|
|
445
|
-
// Filters out: timers, animations, scripts, styles, notifications, toasts.
|
|
446
|
-
// Debounces mutations (500ms) to batch rapid changes.
|
|
447
|
-
// Stealth: uses Symbol-keyed property (invisible to Object.keys / anti-bot scans).
|
|
448
|
-
try {
|
|
449
|
-
await cdpEval(page, `
|
|
450
|
-
(() => {
|
|
451
|
-
const KEY = Symbol('__ts_obs');
|
|
452
|
-
if (window[KEY]) return 'already attached';
|
|
453
|
-
const IGNORE = /timer|countdown|elapsed|clock|animation|animate|transition|spinner|progress|loading|toast|notification|alert|banner|snackbar|cookie|consent|modal|overlay|popup|tooltip|badge|tag|avatar|icon|svg|img|video|audio|canvas|font|stylesheet|script|meta|link|head/i;
|
|
454
|
-
let timer = null;
|
|
455
|
-
let lastText = '';
|
|
456
|
-
function getText() {
|
|
457
|
-
const els = document.querySelectorAll('input[type="radio"],input[type="checkbox"],input[type="text"],input[type="email"],input[type="password"],textarea,label,button');
|
|
458
|
-
return [...els].map(e => {
|
|
459
|
-
const t = (e.innerText || e.textContent || '').trim().replace(/\\s+/g, ' ').slice(0, 200);
|
|
460
|
-
return e.type + '|' + t + '|' + (e.checked || false);
|
|
461
|
-
}).join('\\n');
|
|
462
|
-
}
|
|
463
|
-
function check() {
|
|
464
|
-
try {
|
|
465
|
-
const curr = getText();
|
|
466
|
-
if (curr === lastText) return;
|
|
467
|
-
const prev = lastText;
|
|
468
|
-
lastText = curr;
|
|
469
|
-
const prevGroups = prev ? prev.split('\\n').filter(Boolean) : [];
|
|
470
|
-
const currGroups = curr.split('\\n').filter(Boolean);
|
|
471
|
-
const prevSet = new Set(prevGroups);
|
|
472
|
-
const currSet = new Set(currGroups);
|
|
473
|
-
let removed = 0, changed = 0;
|
|
474
|
-
for (const g of prevGroups) {
|
|
475
|
-
if (!currSet.has(g)) {
|
|
476
|
-
if (prevGroups.indexOf(g) < currGroups.length) changed++;
|
|
477
|
-
else removed++;
|
|
478
|
-
}
|
|
479
|
-
}
|
|
480
|
-
const added = currGroups.filter(g => !prevSet.has(g)).length;
|
|
481
|
-
const score = added * 10 + removed * 10 + changed * 8;
|
|
482
|
-
if (score >= 15) {
|
|
483
|
-
window[KEY].changed = Date.now();
|
|
484
|
-
}
|
|
485
|
-
} catch {}
|
|
486
|
-
}
|
|
487
|
-
const obs = new MutationObserver((mutations) => {
|
|
488
|
-
let hasRelevant = false;
|
|
489
|
-
for (const m of mutations) {
|
|
490
|
-
if (m.target instanceof Element) {
|
|
491
|
-
const tag = m.target.tagName;
|
|
492
|
-
if (tag === 'SCRIPT' || tag === 'STYLE' || tag === 'HEAD') continue;
|
|
493
|
-
const cls = m.target.className || '';
|
|
494
|
-
if (typeof cls === 'string' && IGNORE.test(cls)) continue;
|
|
495
|
-
const id = m.target.id || '';
|
|
496
|
-
if (IGNORE.test(id)) continue;
|
|
497
|
-
if (m.type === 'characterData' && m.target.parentElement) {
|
|
498
|
-
const ptag = m.target.parentElement.tagName;
|
|
499
|
-
if (ptag === 'SCRIPT' || ptag === 'STYLE') continue;
|
|
500
|
-
}
|
|
501
|
-
}
|
|
502
|
-
hasRelevant = true;
|
|
503
|
-
}
|
|
504
|
-
if (hasRelevant) {
|
|
505
|
-
clearTimeout(timer);
|
|
506
|
-
timer = setTimeout(check, 500);
|
|
507
|
-
}
|
|
508
|
-
});
|
|
509
|
-
obs.observe(document.body || document.documentElement, {
|
|
510
|
-
childList: true, subtree: true, characterData: true
|
|
511
|
-
});
|
|
512
|
-
window[KEY] = { observer: obs, changed: 0 };
|
|
513
|
-
lastText = getText();
|
|
514
|
-
return 'ok';
|
|
515
|
-
})()
|
|
516
|
-
`);
|
|
517
|
-
console.log('[browse] MutationObserver injected (stealth, debounced, filtered)');
|
|
518
|
-
}
|
|
519
|
-
catch (err) {
|
|
520
|
-
console.error(`[browse] MutationObserver injection failed: ${(0, config_1.msg)(err)}`);
|
|
521
|
-
}
|
|
553
|
+
// Enable Page + install per-tab instrumentation. Re-run on every (re)attach
|
|
554
|
+
// so a tab switch/close leaves the loop fully instrumented.
|
|
555
|
+
await instrumentPage(page);
|
|
522
556
|
// Agent loop
|
|
523
557
|
let actionCount = 0;
|
|
524
558
|
let llmCallCount = 0;
|
|
@@ -530,6 +564,81 @@ async function cmdBrowse(flags) {
|
|
|
530
564
|
/** A 401/403 from the model API: terminal, retrying cannot succeed. */
|
|
531
565
|
let fatalAuthError = false;
|
|
532
566
|
const maxRetries = 5;
|
|
567
|
+
/** How often to re-check which tab is active (ms). */
|
|
568
|
+
const TAB_CHECK_MS = 2000;
|
|
569
|
+
let lastTabCheck = 0;
|
|
570
|
+
let consecutiveScreenshotFailures = 0;
|
|
571
|
+
let reattachFailures = 0;
|
|
572
|
+
// Latency observability: rolling average of vision-call latency plus
|
|
573
|
+
// abort/timeout counts, so the operator can see how slow the model is.
|
|
574
|
+
const llmStats = { calls: 0, ok: 0, screenAborts: 0, timeouts: 0, totalMs: 0 };
|
|
575
|
+
const logLlmStats = () => {
|
|
576
|
+
if (llmStats.calls === 0 || llmStats.calls % 5 !== 0)
|
|
577
|
+
return;
|
|
578
|
+
const avg = Math.round(llmStats.totalMs / llmStats.calls / 1000);
|
|
579
|
+
console.log(`[browse] LLM latency: avg ${avg}s over ${llmStats.calls} call(s) — ${llmStats.ok} ok, ${llmStats.screenAborts} screen-abort, ${llmStats.timeouts} timeout`);
|
|
580
|
+
};
|
|
581
|
+
/**
|
|
582
|
+
* (Re)attach to a tab. A closed or switched tab used to leave the loop
|
|
583
|
+
* hammering a dead target ("CDP timeout: Page.captureScreenshot") forever;
|
|
584
|
+
* now we detect it and follow the tab the human is actually on.
|
|
585
|
+
*/
|
|
586
|
+
async function reattach(reason, preferTargetId) {
|
|
587
|
+
console.log(`[browse] ${reason} - re-attaching...`);
|
|
588
|
+
try {
|
|
589
|
+
page.ws.close();
|
|
590
|
+
}
|
|
591
|
+
catch {
|
|
592
|
+
/* already closed */
|
|
593
|
+
}
|
|
594
|
+
page = await connectToPage(port, preferTargetId);
|
|
595
|
+
await instrumentPage(page);
|
|
596
|
+
// A different tab means different content: reset change trackers.
|
|
597
|
+
lastFingerprint = null;
|
|
598
|
+
lastScreenshot = null;
|
|
599
|
+
codeConfirmedOptimal = false;
|
|
600
|
+
consecutiveFailures = 0;
|
|
601
|
+
consecutiveScreenshotFailures = 0;
|
|
602
|
+
console.log(`[browse] Re-attached to tab: ${page.title} (${page.targetId.slice(0, 8)})`);
|
|
603
|
+
}
|
|
604
|
+
/**
|
|
605
|
+
* Throttled liveness check: is the bound tab still alive and the active one?
|
|
606
|
+
* Follows the human's active tab (and recovers after it is closed). `force`
|
|
607
|
+
* skips the throttle (used from the screenshot-failure path).
|
|
608
|
+
*/
|
|
609
|
+
async function refreshTabIfNeeded(force = false) {
|
|
610
|
+
if (!force && Date.now() - lastTabCheck < TAB_CHECK_MS)
|
|
611
|
+
return false;
|
|
612
|
+
lastTabCheck = Date.now();
|
|
613
|
+
try {
|
|
614
|
+
if (page.disconnected) {
|
|
615
|
+
// The bound tab is gone: move straight to whatever tab is active now.
|
|
616
|
+
const active = await (0, cdp_1.pickActiveTab)(port).catch(() => null);
|
|
617
|
+
await reattach('Bound tab closed', active?.id);
|
|
618
|
+
reattachFailures = 0;
|
|
619
|
+
return true;
|
|
620
|
+
}
|
|
621
|
+
const active = await (0, cdp_1.pickActiveTab)(port, page.targetId);
|
|
622
|
+
if (!active)
|
|
623
|
+
return false;
|
|
624
|
+
if (active.id !== page.targetId) {
|
|
625
|
+
await reattach(`Active tab changed to "${active.title}"`, active.id);
|
|
626
|
+
reattachFailures = 0;
|
|
627
|
+
return true;
|
|
628
|
+
}
|
|
629
|
+
return false;
|
|
630
|
+
}
|
|
631
|
+
catch (err) {
|
|
632
|
+
reattachFailures++;
|
|
633
|
+
console.error(`[browse] Re-attach attempt ${reattachFailures} failed: ${(0, config_1.msg)(err)}`);
|
|
634
|
+
if (reattachFailures >= 6) {
|
|
635
|
+
console.error('[browse] Could not re-attach to any tab - stopping.');
|
|
636
|
+
stopped = true;
|
|
637
|
+
process.exitCode = 1;
|
|
638
|
+
}
|
|
639
|
+
return false;
|
|
640
|
+
}
|
|
641
|
+
}
|
|
533
642
|
process.on('SIGINT', () => {
|
|
534
643
|
console.log('\n[browse] Stopped by user');
|
|
535
644
|
stopped = true;
|
|
@@ -538,17 +647,26 @@ async function cmdBrowse(flags) {
|
|
|
538
647
|
console.log(`[browse] Agent started. ${continuous ? 'Running until Ctrl+C.' : `Max ${maxActions} actions.`} Press Ctrl+C to stop.\n`);
|
|
539
648
|
while (!stopped && actionCount < maxActions) {
|
|
540
649
|
try {
|
|
650
|
+
// Follow the human's active tab: re-attach if the bound tab was closed or
|
|
651
|
+
// the user switched to another one.
|
|
652
|
+
await refreshTabIfNeeded();
|
|
541
653
|
// Take screenshot and extract elements in parallel
|
|
542
654
|
const [screenshot, elements, pageContext] = await Promise.all([
|
|
543
|
-
captureScreenshot(page, quality),
|
|
655
|
+
captureScreenshot(page, quality, SCREENSHOT_MAX_EDGE),
|
|
544
656
|
extractElements(page),
|
|
545
657
|
extractPageContext(page),
|
|
546
658
|
]);
|
|
547
659
|
if (!screenshot) {
|
|
548
|
-
|
|
660
|
+
consecutiveScreenshotFailures++;
|
|
661
|
+
console.error(`[browse] Failed to capture screenshot (${consecutiveScreenshotFailures}), retrying...`);
|
|
662
|
+
// Two failures in a row generally means the target is gone.
|
|
663
|
+
if (consecutiveScreenshotFailures >= 2) {
|
|
664
|
+
await refreshTabIfNeeded(true);
|
|
665
|
+
}
|
|
549
666
|
await sleep(intervalMs);
|
|
550
667
|
continue;
|
|
551
668
|
}
|
|
669
|
+
consecutiveScreenshotFailures = 0;
|
|
552
670
|
// Detect screen change via question fingerprint comparison
|
|
553
671
|
// Threshold 15: requires ≥1 new/removed question (10) + text change (8) = 18,
|
|
554
672
|
// or 2 text changes (16). Score of 5 (single option change) is too sensitive.
|
|
@@ -568,29 +686,47 @@ async function cmdBrowse(flags) {
|
|
|
568
686
|
const actionLabel = continuous ? `${actionCount + 1}/∞` : `${actionCount + 1}/${maxActions}`;
|
|
569
687
|
console.log(`[browse] [${actionLabel}] Analyzing screenshot...`);
|
|
570
688
|
let result = null;
|
|
689
|
+
/** The page changed under the LLM: re-analyze immediately, no backoff. */
|
|
690
|
+
let screenChangedDuringLlm = false;
|
|
571
691
|
for (let attempt = 1; attempt <= maxRetries; attempt++) {
|
|
572
692
|
result = await analyzeScreenshot(baseUrl, apiKey, model, screenshot, elements, pageContext, timeoutMs, attempt, maxRetries, page, lastFingerprint);
|
|
573
693
|
llmCallCount++;
|
|
694
|
+
llmStats.calls++;
|
|
695
|
+
llmStats.totalMs += result.durationMs;
|
|
574
696
|
if (llmCallCount > 50 && llmCallCount % 25 === 0) {
|
|
575
697
|
console.log(`[browse] Warning: ${llmCallCount} LLM calls made — may hit rate limits`);
|
|
576
698
|
}
|
|
577
|
-
if (result.actions.length > 0)
|
|
699
|
+
if (result.actions.length > 0) {
|
|
700
|
+
llmStats.ok++;
|
|
701
|
+
logLlmStats();
|
|
578
702
|
break; // success
|
|
703
|
+
}
|
|
579
704
|
// Auth failure is terminal: retrying cannot help.
|
|
580
705
|
if (result.error === 'auth_error') {
|
|
581
706
|
fatalAuthError = true;
|
|
582
707
|
break;
|
|
583
708
|
}
|
|
584
|
-
//
|
|
709
|
+
// The screen changed while the LLM was thinking: its answer is stale.
|
|
710
|
+
// Bail out of the retry loop and re-analyze the NEW screen right away
|
|
711
|
+
// (speed matters: a 30s backoff here livelocked on dynamic pages).
|
|
712
|
+
if (result.error === 'screen_changed') {
|
|
713
|
+
screenChangedDuringLlm = true;
|
|
714
|
+
llmStats.screenAborts++;
|
|
715
|
+
logLlmStats();
|
|
716
|
+
break;
|
|
717
|
+
}
|
|
718
|
+
// Genuine timeout → SHORT backoff and retry (was 30/60/120/240/300s).
|
|
585
719
|
if (result.error === 'timeout') {
|
|
586
|
-
|
|
720
|
+
llmStats.timeouts++;
|
|
721
|
+
logLlmStats();
|
|
722
|
+
const backoff = Math.min(3 * attempt, 12); // 3, 6, 9, 12, 12
|
|
587
723
|
console.log(`[browse] LLM timed out — retrying in ${backoff}s (attempt ${attempt}/${maxRetries})...`);
|
|
588
724
|
await sleep(backoff * 1000);
|
|
589
725
|
continue;
|
|
590
726
|
}
|
|
591
|
-
// Other errors → short backoff
|
|
727
|
+
// Other errors → very short backoff
|
|
592
728
|
if (attempt < maxRetries) {
|
|
593
|
-
const backoff =
|
|
729
|
+
const backoff = 2;
|
|
594
730
|
console.log(`[browse] ${result.error === 'empty' ? 'Empty response' : 'Parse failed'} (attempt ${attempt}/${maxRetries}), retrying in ${backoff}s...`);
|
|
595
731
|
await sleep(backoff * 1000);
|
|
596
732
|
}
|
|
@@ -601,6 +737,10 @@ async function cmdBrowse(flags) {
|
|
|
601
737
|
process.exitCode = 1;
|
|
602
738
|
break;
|
|
603
739
|
}
|
|
740
|
+
if (screenChangedDuringLlm) {
|
|
741
|
+
// No sleep: go straight back to the top, re-screenshot + re-analyze.
|
|
742
|
+
continue;
|
|
743
|
+
}
|
|
604
744
|
if (!result || result.actions.length === 0) {
|
|
605
745
|
consecutiveFailures++;
|
|
606
746
|
console.log(`[browse] No valid response after ${maxRetries} attempts (${consecutiveFailures} consecutive failures)`);
|
|
@@ -699,6 +839,18 @@ async function cmdBrowse(flags) {
|
|
|
699
839
|
console.log(`[browse] Skipping ${act.action} — code already confirmed optimal`);
|
|
700
840
|
continue;
|
|
701
841
|
}
|
|
842
|
+
// Field safety: never type into a personal field, a credential field, or
|
|
843
|
+
// a field the human already answered themselves.
|
|
844
|
+
if (act.action === 'write_code' || act.action === 'type') {
|
|
845
|
+
const targetEl = act.elementIndex !== undefined
|
|
846
|
+
? elements.find((e) => e.i === act.elementIndex)
|
|
847
|
+
: undefined;
|
|
848
|
+
const gated = fieldGate(targetEl);
|
|
849
|
+
if (gated) {
|
|
850
|
+
console.log(`[browse] Skipping element ${act.elementIndex} - ${gated}`);
|
|
851
|
+
continue;
|
|
852
|
+
}
|
|
853
|
+
}
|
|
702
854
|
// Ask LLM to decide if action is redundant
|
|
703
855
|
const redundancy = await isActionRedundant(baseUrl, apiKey, model, page, elements, act, timeoutMs);
|
|
704
856
|
if (redundancy.decision === 'skip') {
|
|
@@ -890,6 +1042,14 @@ Do NOT simplify variable names or change the function signature. Only optimize t
|
|
|
890
1042
|
console.log(`[browse] Reached max actions (${maxActions})`);
|
|
891
1043
|
}
|
|
892
1044
|
console.log(`[browse] Agent finished. ${actionCount} actions performed.`);
|
|
1045
|
+
const skipped = personalSkipSummary();
|
|
1046
|
+
if (skipped.personal > 0 || skipped.human > 0) {
|
|
1047
|
+
console.log(`[browse] Left for you: ${skipped.personal} personal question(s) never answered` +
|
|
1048
|
+
(skipped.human > 0 ? `, ${skipped.human} field(s) you answered yourself` : ''));
|
|
1049
|
+
for (const entry of skippedFields.values()) {
|
|
1050
|
+
console.log(` - ${entry.note}`);
|
|
1051
|
+
}
|
|
1052
|
+
}
|
|
893
1053
|
page.ws.close();
|
|
894
1054
|
}
|
|
895
1055
|
// ── CDP Helpers ──────────────────────────────────────────────────────────────
|
|
@@ -897,11 +1057,11 @@ Do NOT simplify variable names or change the function signature. Only optimize t
|
|
|
897
1057
|
* Connect directly to a page's WebSocket from /json/list.
|
|
898
1058
|
* This avoids the Target.attachToTarget complexity.
|
|
899
1059
|
*/
|
|
900
|
-
async function connectToPage(port) {
|
|
1060
|
+
async function connectToPage(port, preferTargetId) {
|
|
901
1061
|
// Shared picker (cdp.ts): prefers a real page, accepts about:blank/new-tab,
|
|
902
1062
|
// and opens a blank tab if none exists - so a Chrome launched on the NTP is
|
|
903
1063
|
// still attachable. (Was a local chrome:// filter that could never match.)
|
|
904
|
-
const pageTarget = await (0, cdp_1.resolveAttachablePage)(port);
|
|
1064
|
+
const pageTarget = await (0, cdp_1.resolveAttachablePage)(port, preferTargetId);
|
|
905
1065
|
const wsUrl = pageTarget.webSocketDebuggerUrl;
|
|
906
1066
|
return new Promise((resolve, reject) => {
|
|
907
1067
|
const ws = new ws_1.default(wsUrl);
|
|
@@ -958,6 +1118,8 @@ async function connectToPage(port) {
|
|
|
958
1118
|
}
|
|
959
1119
|
});
|
|
960
1120
|
ws.on('close', () => {
|
|
1121
|
+
// The endpoint is gone: flag it so the agent loop can re-attach.
|
|
1122
|
+
page.disconnected = true;
|
|
961
1123
|
// Reject all pending
|
|
962
1124
|
for (const [id, p] of page.pending) {
|
|
963
1125
|
p.reject(new Error('WebSocket closed'));
|
|
@@ -1022,9 +1184,32 @@ function cdpSend(page, method, params) {
|
|
|
1022
1184
|
page.ws.send(JSON.stringify({ id, method, params }));
|
|
1023
1185
|
});
|
|
1024
1186
|
}
|
|
1025
|
-
|
|
1187
|
+
/**
|
|
1188
|
+
* Capture the visible viewport as JPEG. When the viewport is larger than
|
|
1189
|
+
* `maxEdge` on either side the capture is scaled down (`clip.scale`) so the
|
|
1190
|
+
* vision call stays fast: a big page produced 100KB+ images that exceeded the
|
|
1191
|
+
* model timeout. Clicking is unaffected — actions use element coordinates
|
|
1192
|
+
* (CSS px), not image pixels.
|
|
1193
|
+
*/
|
|
1194
|
+
async function captureScreenshot(page, quality, maxEdge = 0) {
|
|
1026
1195
|
try {
|
|
1027
|
-
const
|
|
1196
|
+
const params = { format: 'jpeg', quality };
|
|
1197
|
+
if (maxEdge > 0) {
|
|
1198
|
+
try {
|
|
1199
|
+
const metrics = await cdpSend(page, 'Page.getLayoutMetrics', {});
|
|
1200
|
+
const vp = metrics?.cssLayoutViewport ?? metrics?.layoutViewport;
|
|
1201
|
+
const w = Number(vp?.clientWidth ?? 0);
|
|
1202
|
+
const h = Number(vp?.clientHeight ?? 0);
|
|
1203
|
+
if (w > 0 && h > 0 && (w > maxEdge || h > maxEdge)) {
|
|
1204
|
+
const scale = Math.min(maxEdge / w, maxEdge / h);
|
|
1205
|
+
params.clip = { x: 0, y: 0, width: w, height: h, scale };
|
|
1206
|
+
}
|
|
1207
|
+
}
|
|
1208
|
+
catch {
|
|
1209
|
+
/* metrics unavailable - capture unscaled */
|
|
1210
|
+
}
|
|
1211
|
+
}
|
|
1212
|
+
const result = await cdpSend(page, 'Page.captureScreenshot', params);
|
|
1028
1213
|
return result?.data ?? null;
|
|
1029
1214
|
}
|
|
1030
1215
|
catch (err) {
|
|
@@ -1032,6 +1217,124 @@ async function captureScreenshot(page, quality) {
|
|
|
1032
1217
|
return null;
|
|
1033
1218
|
}
|
|
1034
1219
|
}
|
|
1220
|
+
/**
|
|
1221
|
+
* Field-safety addendum appended to every system prompt (main loop, custom /
|
|
1222
|
+
* optimization passes, stealth loop). The loop may not answer questions that
|
|
1223
|
+
* ask about the human behind the browser.
|
|
1224
|
+
*/
|
|
1225
|
+
const PERSONAL_SAFETY_RULES = `
|
|
1226
|
+
PERSONAL QUESTIONS - NEVER ANSWER THEM:
|
|
1227
|
+
- Some questions ask about YOU (the human operating the browser), not about the
|
|
1228
|
+
task. NEVER answer them with an invented value.
|
|
1229
|
+
- Never type: names, email addresses, phone numbers, postal addresses, dates of
|
|
1230
|
+
birth or age, government IDs (SSN/NIN/passport/driving licence/tax id),
|
|
1231
|
+
passwords, one-time codes, card or bank details, salary or compensation,
|
|
1232
|
+
notice periods, work-authorization/visa/sponsorship answers, or personal
|
|
1233
|
+
profile links (LinkedIn, GitHub, portfolio, personal website).
|
|
1234
|
+
- Fields marked "[NEVER ANSWER: ...]" or "[PERSONAL: ...]" in the element list
|
|
1235
|
+
MUST be left untouched - do not click them, do not type into them.
|
|
1236
|
+
- Leaving a personal field EMPTY is correct and expected. A personal field is
|
|
1237
|
+
NOT an unanswered question: never treat it as one, and never delay "done"
|
|
1238
|
+
because of it.
|
|
1239
|
+
- If every remaining question is personal, return done.
|
|
1240
|
+
- Demographics and free-text personal questions (gender, marital status, "tell
|
|
1241
|
+
us about yourself") are the human's to answer: prefer to skip them too.
|
|
1242
|
+
`;
|
|
1243
|
+
const fieldSafety = {
|
|
1244
|
+
stopOnUserInput: false,
|
|
1245
|
+
allowPersonal: false,
|
|
1246
|
+
};
|
|
1247
|
+
/** fieldKey -> why the field must not be typed into for the rest of the run. */
|
|
1248
|
+
const skippedFields = new Map();
|
|
1249
|
+
/**
|
|
1250
|
+
* Stealth mode has no DOM, so fields are identified by the focused OS control
|
|
1251
|
+
* (AutomationId|ClassName|Name) instead of a field key.
|
|
1252
|
+
*/
|
|
1253
|
+
const skippedOsFields = new Map();
|
|
1254
|
+
function signalsOf(el) {
|
|
1255
|
+
return {
|
|
1256
|
+
tag: el.tag,
|
|
1257
|
+
type: el.type,
|
|
1258
|
+
name: el.name,
|
|
1259
|
+
id: el.id,
|
|
1260
|
+
autocomplete: el.autocomplete,
|
|
1261
|
+
placeholder: el.placeholder,
|
|
1262
|
+
ariaLabel: el.ariaLabel,
|
|
1263
|
+
label: el.label,
|
|
1264
|
+
questionText: el.questionText,
|
|
1265
|
+
text: el.text,
|
|
1266
|
+
};
|
|
1267
|
+
}
|
|
1268
|
+
function classificationOf(el) {
|
|
1269
|
+
return {
|
|
1270
|
+
tier: el.personalTier ?? null,
|
|
1271
|
+
category: el.personalCategory ?? null,
|
|
1272
|
+
label: el.personalLabel ?? '',
|
|
1273
|
+
reason: el.personalReason ?? '',
|
|
1274
|
+
};
|
|
1275
|
+
}
|
|
1276
|
+
/**
|
|
1277
|
+
* Classify every typable element and stamp its stable field key. Elements are
|
|
1278
|
+
* annotated in place so the gate below never re-parses the DOM.
|
|
1279
|
+
*/
|
|
1280
|
+
function annotateFieldSafety(elements) {
|
|
1281
|
+
for (const el of elements) {
|
|
1282
|
+
const signals = signalsOf(el);
|
|
1283
|
+
el.fieldKey = (0, personal_fields_1.fieldKeyOf)(signals);
|
|
1284
|
+
if (!el.typable)
|
|
1285
|
+
continue;
|
|
1286
|
+
const cls = (0, personal_fields_1.classifyField)(signals);
|
|
1287
|
+
el.personalTier = cls.tier;
|
|
1288
|
+
el.personalCategory = cls.category;
|
|
1289
|
+
el.personalLabel = cls.label;
|
|
1290
|
+
el.personalReason = cls.reason;
|
|
1291
|
+
}
|
|
1292
|
+
}
|
|
1293
|
+
function shortLabel(el) {
|
|
1294
|
+
const raw = el.label || el.questionText || el.text || el.placeholder || el.name || '';
|
|
1295
|
+
return raw.replace(/\s+/g, ' ').trim().slice(0, 60);
|
|
1296
|
+
}
|
|
1297
|
+
/**
|
|
1298
|
+
* Returns a human-readable reason when this element must not be typed into,
|
|
1299
|
+
* or null when it is fine to type.
|
|
1300
|
+
*/
|
|
1301
|
+
function fieldGate(el) {
|
|
1302
|
+
if (!el || !el.typable)
|
|
1303
|
+
return null;
|
|
1304
|
+
const key = el.fieldKey ?? (0, personal_fields_1.fieldKeyOf)(signalsOf(el));
|
|
1305
|
+
const already = skippedFields.get(key);
|
|
1306
|
+
if (already)
|
|
1307
|
+
return already.note;
|
|
1308
|
+
const cls = classificationOf(el);
|
|
1309
|
+
if (cls.tier === null)
|
|
1310
|
+
return null;
|
|
1311
|
+
if (!(0, personal_fields_1.isBlocked)(cls, { allowPersonal: fieldSafety.allowPersonal }))
|
|
1312
|
+
return null;
|
|
1313
|
+
const note = `left to you (${cls.category}: "${shortLabel(el) || cls.label}")`;
|
|
1314
|
+
skippedFields.set(key, { kind: 'personal', note });
|
|
1315
|
+
console.log(`[browse] Not answering a personal question - ${note}`);
|
|
1316
|
+
return note;
|
|
1317
|
+
}
|
|
1318
|
+
/** Record a field the human took over mid-typing; it is never typed again. */
|
|
1319
|
+
function noteHumanTookOver(el, reason) {
|
|
1320
|
+
const key = el?.fieldKey ?? (el ? (0, personal_fields_1.fieldKeyOf)(signalsOf(el)) : '');
|
|
1321
|
+
if (!key)
|
|
1322
|
+
return;
|
|
1323
|
+
const note = `you answered this one yourself (${reason})`;
|
|
1324
|
+
skippedFields.set(key, { kind: 'human', note });
|
|
1325
|
+
console.log(`[browse] Stopped typing - ${note}`);
|
|
1326
|
+
}
|
|
1327
|
+
function personalSkipSummary() {
|
|
1328
|
+
let personal = 0;
|
|
1329
|
+
let human = 0;
|
|
1330
|
+
for (const entry of skippedFields.values()) {
|
|
1331
|
+
if (entry.kind === 'personal')
|
|
1332
|
+
personal++;
|
|
1333
|
+
else
|
|
1334
|
+
human++;
|
|
1335
|
+
}
|
|
1336
|
+
return { personal, human };
|
|
1337
|
+
}
|
|
1035
1338
|
/**
|
|
1036
1339
|
* Extract all interactive elements from the DOM with their exact text,
|
|
1037
1340
|
* bounding boxes, state, and question group assignments.
|
|
@@ -1072,6 +1375,29 @@ async function extractElements(page) {
|
|
|
1072
1375
|
text,
|
|
1073
1376
|
tag: el.tagName.toLowerCase(),
|
|
1074
1377
|
type: el.type || el.getAttribute('role') || '',
|
|
1378
|
+
typable: el.tagName === 'INPUT' || el.tagName === 'TEXTAREA' || el.isContentEditable === true,
|
|
1379
|
+
name: el.getAttribute('name') || '',
|
|
1380
|
+
id: el.getAttribute('id') || '',
|
|
1381
|
+
autocomplete: el.getAttribute('autocomplete') || '',
|
|
1382
|
+
placeholder: el.getAttribute('placeholder') || '',
|
|
1383
|
+
ariaLabel: el.getAttribute('aria-label') || '',
|
|
1384
|
+
label: (() => {
|
|
1385
|
+
try {
|
|
1386
|
+
const id = el.getAttribute('id');
|
|
1387
|
+
if (id) {
|
|
1388
|
+
const forLabel = document.querySelector('label[for="' + CSS.escape(id) + '"]');
|
|
1389
|
+
if (forLabel && forLabel.innerText) return forLabel.innerText.trim().replace(/\\s+/g, ' ').slice(0, 200);
|
|
1390
|
+
}
|
|
1391
|
+
const wrap = el.closest ? el.closest('label') : null;
|
|
1392
|
+
if (wrap && wrap.innerText) return wrap.innerText.trim().replace(/\\s+/g, ' ').slice(0, 200);
|
|
1393
|
+
const labelled = el.getAttribute('aria-labelledby');
|
|
1394
|
+
if (labelled) {
|
|
1395
|
+
const t = document.getElementById(labelled);
|
|
1396
|
+
if (t && t.innerText) return t.innerText.trim().replace(/\\s+/g, ' ').slice(0, 200);
|
|
1397
|
+
}
|
|
1398
|
+
} catch (err) {}
|
|
1399
|
+
return '';
|
|
1400
|
+
})(),
|
|
1075
1401
|
checked: el.checked || el.getAttribute('aria-checked') === 'true' || false,
|
|
1076
1402
|
x: Math.round(rect.x + rect.width / 2),
|
|
1077
1403
|
y: Math.round(rect.y + rect.height / 2),
|
|
@@ -1184,6 +1510,13 @@ async function extractElements(page) {
|
|
|
1184
1510
|
y: item.y,
|
|
1185
1511
|
group: item.groupId,
|
|
1186
1512
|
inViewport: item.inViewport,
|
|
1513
|
+
typable: item.typable,
|
|
1514
|
+
name: item.name,
|
|
1515
|
+
id: item.id,
|
|
1516
|
+
autocomplete: item.autocomplete,
|
|
1517
|
+
placeholder: item.placeholder,
|
|
1518
|
+
ariaLabel: item.ariaLabel,
|
|
1519
|
+
label: item.label,
|
|
1187
1520
|
});
|
|
1188
1521
|
});
|
|
1189
1522
|
|
|
@@ -1192,7 +1525,9 @@ async function extractElements(page) {
|
|
|
1192
1525
|
`;
|
|
1193
1526
|
try {
|
|
1194
1527
|
const result = await cdpEval(page, js);
|
|
1195
|
-
|
|
1528
|
+
const elements = result?.result?.value || [];
|
|
1529
|
+
annotateFieldSafety(elements);
|
|
1530
|
+
return elements;
|
|
1196
1531
|
}
|
|
1197
1532
|
catch {
|
|
1198
1533
|
return [];
|
|
@@ -1241,6 +1576,11 @@ async function executeAction(page, act, elements, charDelayMs) {
|
|
|
1241
1576
|
console.log(`[browse] Element index ${act.elementIndex} not found, will retry`);
|
|
1242
1577
|
return false;
|
|
1243
1578
|
}
|
|
1579
|
+
const gated = fieldGate(el);
|
|
1580
|
+
if (gated) {
|
|
1581
|
+
console.log(`[browse] Skipping element ${el.i} - ${gated}`);
|
|
1582
|
+
return true;
|
|
1583
|
+
}
|
|
1244
1584
|
// 1. Detect editor type
|
|
1245
1585
|
const editor = await detectEditor(page);
|
|
1246
1586
|
const isCodeEditor = editor !== null;
|
|
@@ -1255,12 +1595,19 @@ async function executeAction(page, act, elements, charDelayMs) {
|
|
|
1255
1595
|
// 3. Normalize indentation for code editors (LLM often adds extra indent)
|
|
1256
1596
|
const text = isCodeEditor ? normalizeCodeIndent(rawText) : rawText;
|
|
1257
1597
|
console.log(`[browse] Typing ${text.length} characters...`);
|
|
1598
|
+
const guard = new human_input_guard_1.TypingGuard({
|
|
1599
|
+
enabled: fieldSafety.stopOnUserInput,
|
|
1600
|
+
page,
|
|
1601
|
+
label: shortLabel(el),
|
|
1602
|
+
});
|
|
1258
1603
|
if (isCodeEditor) {
|
|
1259
|
-
await insertIntoCodeEditor(page, editor.selector, text, charDelayMs);
|
|
1604
|
+
await insertIntoCodeEditor(page, editor.selector, text, charDelayMs, guard);
|
|
1260
1605
|
}
|
|
1261
1606
|
else {
|
|
1262
|
-
await insertIntoGenericInput(page, el, text, charDelayMs);
|
|
1607
|
+
await insertIntoGenericInput(page, el, text, charDelayMs, guard);
|
|
1263
1608
|
}
|
|
1609
|
+
if (guard.stopped)
|
|
1610
|
+
noteHumanTookOver(el, guard.describeStop());
|
|
1264
1611
|
return true;
|
|
1265
1612
|
}
|
|
1266
1613
|
case 'write_code': {
|
|
@@ -1271,6 +1618,11 @@ async function executeAction(page, act, elements, charDelayMs) {
|
|
|
1271
1618
|
console.log(`[browse] Element index ${act.elementIndex} not found, will retry`);
|
|
1272
1619
|
return false;
|
|
1273
1620
|
}
|
|
1621
|
+
const gated = fieldGate(el);
|
|
1622
|
+
if (gated) {
|
|
1623
|
+
console.log(`[browse] Skipping element ${el.i} - ${gated}`);
|
|
1624
|
+
return true;
|
|
1625
|
+
}
|
|
1274
1626
|
// 1. Detect editor type
|
|
1275
1627
|
const editor = await detectEditor(page);
|
|
1276
1628
|
const isCodeEditor = editor !== null;
|
|
@@ -1283,13 +1635,20 @@ async function executeAction(page, act, elements, charDelayMs) {
|
|
|
1283
1635
|
.replace(/\\"/g, '"')
|
|
1284
1636
|
.replace(/\\\\/g, '\\');
|
|
1285
1637
|
console.log(`[browse] Writing code (${text.length} chars)...`);
|
|
1638
|
+
const guard = new human_input_guard_1.TypingGuard({
|
|
1639
|
+
enabled: fieldSafety.stopOnUserInput,
|
|
1640
|
+
page,
|
|
1641
|
+
label: shortLabel(el),
|
|
1642
|
+
});
|
|
1286
1643
|
// 3. write_code inserts EXACTLY as-is — no normalizeCodeIndent
|
|
1287
1644
|
if (isCodeEditor) {
|
|
1288
|
-
await insertIntoCodeEditor(page, editor.selector, text, charDelayMs);
|
|
1645
|
+
await insertIntoCodeEditor(page, editor.selector, text, charDelayMs, guard);
|
|
1289
1646
|
}
|
|
1290
1647
|
else {
|
|
1291
|
-
await insertIntoGenericInput(page, el, text, charDelayMs);
|
|
1648
|
+
await insertIntoGenericInput(page, el, text, charDelayMs, guard);
|
|
1292
1649
|
}
|
|
1650
|
+
if (guard.stopped)
|
|
1651
|
+
noteHumanTookOver(el, guard.describeStop());
|
|
1293
1652
|
return true;
|
|
1294
1653
|
}
|
|
1295
1654
|
case 'scroll': {
|
|
@@ -1363,6 +1722,11 @@ async function executeActionHybrid(page, act, elements, charDelayMs) {
|
|
|
1363
1722
|
console.log(`[browse] Element index ${act.elementIndex} not found, will retry`);
|
|
1364
1723
|
return false;
|
|
1365
1724
|
}
|
|
1725
|
+
const gated = fieldGate(el);
|
|
1726
|
+
if (gated) {
|
|
1727
|
+
console.log(`[browse] Skipping element ${el.i} - ${gated}`);
|
|
1728
|
+
return true;
|
|
1729
|
+
}
|
|
1366
1730
|
const editor = await detectEditor(page);
|
|
1367
1731
|
const isCodeEditor = editor !== null;
|
|
1368
1732
|
if (editor)
|
|
@@ -1381,7 +1745,16 @@ async function executeActionHybrid(page, act, elements, charDelayMs) {
|
|
|
1381
1745
|
pressKeyOS('ctrl+a');
|
|
1382
1746
|
await sleep(randomInt(50, 100));
|
|
1383
1747
|
}
|
|
1384
|
-
|
|
1748
|
+
const guard = new human_input_guard_1.TypingGuard({
|
|
1749
|
+
enabled: fieldSafety.stopOnUserInput,
|
|
1750
|
+
page,
|
|
1751
|
+
label: shortLabel(el),
|
|
1752
|
+
});
|
|
1753
|
+
if (guard)
|
|
1754
|
+
await guard.arm({ isCodeEditor });
|
|
1755
|
+
await typeHumanLike(text, charDelayMs, guard);
|
|
1756
|
+
if (guard.stopped)
|
|
1757
|
+
noteHumanTookOver(el, guard.describeStop());
|
|
1385
1758
|
return true;
|
|
1386
1759
|
}
|
|
1387
1760
|
case 'write_code': {
|
|
@@ -1392,6 +1765,11 @@ async function executeActionHybrid(page, act, elements, charDelayMs) {
|
|
|
1392
1765
|
console.log(`[browse] Element index ${act.elementIndex} not found, will retry`);
|
|
1393
1766
|
return false;
|
|
1394
1767
|
}
|
|
1768
|
+
const gated = fieldGate(el);
|
|
1769
|
+
if (gated) {
|
|
1770
|
+
console.log(`[browse] Skipping element ${el.i} - ${gated}`);
|
|
1771
|
+
return true;
|
|
1772
|
+
}
|
|
1395
1773
|
const editor = await detectEditor(page);
|
|
1396
1774
|
const isCodeEditor = editor !== null;
|
|
1397
1775
|
if (editor)
|
|
@@ -1409,7 +1787,16 @@ async function executeActionHybrid(page, act, elements, charDelayMs) {
|
|
|
1409
1787
|
pressKeyOS('ctrl+a');
|
|
1410
1788
|
await sleep(randomInt(50, 100));
|
|
1411
1789
|
}
|
|
1412
|
-
|
|
1790
|
+
const guard = new human_input_guard_1.TypingGuard({
|
|
1791
|
+
enabled: fieldSafety.stopOnUserInput,
|
|
1792
|
+
page,
|
|
1793
|
+
label: shortLabel(el),
|
|
1794
|
+
});
|
|
1795
|
+
if (guard)
|
|
1796
|
+
await guard.arm({ isCodeEditor });
|
|
1797
|
+
await typeHumanLike(text, charDelayMs, guard);
|
|
1798
|
+
if (guard.stopped)
|
|
1799
|
+
noteHumanTookOver(el, guard.describeStop());
|
|
1413
1800
|
return true;
|
|
1414
1801
|
}
|
|
1415
1802
|
case 'scroll': {
|
|
@@ -1596,13 +1983,21 @@ function hasUnansweredQuestions(elements) {
|
|
|
1596
1983
|
const types = els.map((e) => e.type);
|
|
1597
1984
|
const isRadio = types.some((t) => t === 'radio');
|
|
1598
1985
|
const isCheckbox = types.some((t) => t === 'checkbox');
|
|
1599
|
-
|
|
1986
|
+
// Personal fields are the human's to answer: they are never "unanswered"
|
|
1987
|
+
// for the agent, so a `done` from the model with them left empty sticks
|
|
1988
|
+
// instead of triggering the "you must answer everything" retry loop.
|
|
1989
|
+
const inputs = els.filter((e) => (e.tag === 'input' || e.tag === 'textarea') &&
|
|
1990
|
+
e.type !== 'radio' &&
|
|
1991
|
+
e.type !== 'checkbox' &&
|
|
1992
|
+
e.typable !== false);
|
|
1993
|
+
const openInputs = inputs.filter((e) => !(0, personal_fields_1.excludesFromCompleteness)(classificationOf(e)));
|
|
1994
|
+
const hasInput = openInputs.length > 0;
|
|
1600
1995
|
const checkedCount = els.filter((e) => e.checked).length;
|
|
1601
1996
|
if (isRadio && checkedCount === 0)
|
|
1602
1997
|
return true;
|
|
1603
1998
|
if (isCheckbox && checkedCount < els.length)
|
|
1604
1999
|
return true;
|
|
1605
|
-
if (hasInput && !
|
|
2000
|
+
if (hasInput && !openInputs.some((e) => e.text && e.tag === 'input'))
|
|
1606
2001
|
return true;
|
|
1607
2002
|
}
|
|
1608
2003
|
return false;
|
|
@@ -1827,7 +2222,7 @@ async function removeTrailingBrackets(page) {
|
|
|
1827
2222
|
console.log(`[browse] Removed ${extra} extra trailing bracket(s)`);
|
|
1828
2223
|
}
|
|
1829
2224
|
/** Insert text into ANY code editor — character-by-character typing, no pasting. */
|
|
1830
|
-
async function insertIntoCodeEditor(page, selector, text, charDelayMs) {
|
|
2225
|
+
async function insertIntoCodeEditor(page, selector, text, charDelayMs, guard) {
|
|
1831
2226
|
// Focus the editor
|
|
1832
2227
|
await focusSelector(page, selector);
|
|
1833
2228
|
await sleep(100);
|
|
@@ -1879,8 +2274,12 @@ async function insertIntoCodeEditor(page, selector, text, charDelayMs) {
|
|
|
1879
2274
|
// Type character by character — NO pasting
|
|
1880
2275
|
const est = Math.round(text.length * charDelayMs / 1000);
|
|
1881
2276
|
console.log(`[browse] Typing ${text.length} chars (~${est}s est.)...`);
|
|
2277
|
+
if (guard)
|
|
2278
|
+
await guard.arm({ selector, isCodeEditor: true });
|
|
1882
2279
|
for (let i = 0; i < text.length; i++) {
|
|
1883
2280
|
const char = text[i];
|
|
2281
|
+
if (guard)
|
|
2282
|
+
await guard.markOwn(char, char === '\n' ? 'Enter' : char === '\t' ? 'Tab' : undefined);
|
|
1884
2283
|
if (char === '\n') {
|
|
1885
2284
|
await cdpSend(page, 'Input.dispatchKeyEvent', {
|
|
1886
2285
|
type: 'keyDown', key: 'Enter', code: 'Enter', text: '\r',
|
|
@@ -1918,7 +2317,16 @@ async function insertIntoCodeEditor(page, selector, text, charDelayMs) {
|
|
|
1918
2317
|
await cdpSend(page, 'Input.dispatchKeyEvent', upParams);
|
|
1919
2318
|
}
|
|
1920
2319
|
await sleep(variableDelay(charDelayMs, char, i, text.length));
|
|
2320
|
+
if (guard) {
|
|
2321
|
+
await guard.tick(text.slice(0, i + 1));
|
|
2322
|
+
if (guard.stopped) {
|
|
2323
|
+
console.log(`[browse] Typing stopped after ${i + 1}/${text.length} chars`);
|
|
2324
|
+
break;
|
|
2325
|
+
}
|
|
2326
|
+
}
|
|
1921
2327
|
}
|
|
2328
|
+
if (guard)
|
|
2329
|
+
await guard.dispose();
|
|
1922
2330
|
// Post-typing: detect and remove extra trailing brackets
|
|
1923
2331
|
await removeTrailingBrackets(page);
|
|
1924
2332
|
// Re-enable auto-formatting
|
|
@@ -1939,7 +2347,7 @@ async function insertIntoCodeEditor(page, selector, text, charDelayMs) {
|
|
|
1939
2347
|
console.log(`[browse] Typing complete (${text.length} chars)`);
|
|
1940
2348
|
}
|
|
1941
2349
|
/** Insert text into a generic input — click focus + char-by-char typing. */
|
|
1942
|
-
async function insertIntoGenericInput(page, el, text, charDelayMs) {
|
|
2350
|
+
async function insertIntoGenericInput(page, el, text, charDelayMs, guard) {
|
|
1943
2351
|
await cdpSend(page, 'Input.dispatchMouseEvent', {
|
|
1944
2352
|
type: 'mousePressed', x: el.x, y: el.y, button: 'left', clickCount: 1,
|
|
1945
2353
|
});
|
|
@@ -1949,8 +2357,12 @@ async function insertIntoGenericInput(page, el, text, charDelayMs) {
|
|
|
1949
2357
|
await sleep(200);
|
|
1950
2358
|
await clearEditor(page);
|
|
1951
2359
|
await sleep(200);
|
|
2360
|
+
if (guard)
|
|
2361
|
+
await guard.arm({ isCodeEditor: false });
|
|
1952
2362
|
for (let i = 0; i < text.length; i++) {
|
|
1953
2363
|
const char = text[i];
|
|
2364
|
+
if (guard)
|
|
2365
|
+
await guard.markOwn(char, char === '\n' ? 'Enter' : char === '\t' ? 'Tab' : undefined);
|
|
1954
2366
|
if (char === '\n') {
|
|
1955
2367
|
await cdpSend(page, 'Input.dispatchKeyEvent', {
|
|
1956
2368
|
type: 'keyDown', key: 'Enter', code: 'Enter', text: '\r',
|
|
@@ -1988,7 +2400,16 @@ async function insertIntoGenericInput(page, el, text, charDelayMs) {
|
|
|
1988
2400
|
await cdpSend(page, 'Input.dispatchKeyEvent', upParams);
|
|
1989
2401
|
}
|
|
1990
2402
|
await sleep(variableDelay(charDelayMs, char, i, text.length));
|
|
2403
|
+
if (guard) {
|
|
2404
|
+
await guard.tick(text.slice(0, i + 1));
|
|
2405
|
+
if (guard.stopped) {
|
|
2406
|
+
console.log(`[browse] Typing stopped after ${i + 1}/${text.length} chars`);
|
|
2407
|
+
break;
|
|
2408
|
+
}
|
|
2409
|
+
}
|
|
1991
2410
|
}
|
|
2411
|
+
if (guard)
|
|
2412
|
+
await guard.dispose();
|
|
1992
2413
|
}
|
|
1993
2414
|
/**
|
|
1994
2415
|
* Extract page context for the LLM: title, URL, headings, errors, editor content.
|
|
@@ -2340,7 +2761,11 @@ async function analyzeScreenshot(baseUrl, apiKey, model, screenshotBase64, eleme
|
|
|
2340
2761
|
for (const el of els) {
|
|
2341
2762
|
const checked = el.checked ? ' [CHECKED]' : '';
|
|
2342
2763
|
const visibility = el.inViewport ? '' : ' [OFF-SCREEN]';
|
|
2343
|
-
|
|
2764
|
+
const skipNote = el.fieldKey && skippedFields.has(el.fieldKey)
|
|
2765
|
+
? ` [SKIPPED: ${skippedFields.get(el.fieldKey).note}]`
|
|
2766
|
+
: '';
|
|
2767
|
+
const personal = el.typable ? (0, personal_fields_1.promptMarker)(classificationOf(el)) : '';
|
|
2768
|
+
lines.push(` ${el.i}: [${el.type || el.tag}] "${el.text}"${checked}${visibility}${personal}${skipNote}`);
|
|
2344
2769
|
}
|
|
2345
2770
|
lines.push('');
|
|
2346
2771
|
}
|
|
@@ -2352,7 +2777,7 @@ async function analyzeScreenshot(baseUrl, apiKey, model, screenshotBase64, eleme
|
|
|
2352
2777
|
const body = {
|
|
2353
2778
|
model,
|
|
2354
2779
|
messages: [
|
|
2355
|
-
{ role: 'system', content: SYSTEM_PROMPT },
|
|
2780
|
+
{ role: 'system', content: SYSTEM_PROMPT + PERSONAL_SAFETY_RULES },
|
|
2356
2781
|
{
|
|
2357
2782
|
role: 'user',
|
|
2358
2783
|
content: [
|
|
@@ -2362,11 +2787,19 @@ async function analyzeScreenshot(baseUrl, apiKey, model, screenshotBase64, eleme
|
|
|
2362
2787
|
},
|
|
2363
2788
|
],
|
|
2364
2789
|
temperature: 0.1,
|
|
2365
|
-
max_tokens:
|
|
2790
|
+
max_tokens: 2048,
|
|
2366
2791
|
stream: false,
|
|
2367
2792
|
};
|
|
2368
2793
|
const controller = new AbortController();
|
|
2369
|
-
|
|
2794
|
+
// Distinguish a genuine timeout from the heartbeat cancelling a stale request
|
|
2795
|
+
// because the page changed: they need opposite handling (backoff vs re-analyze
|
|
2796
|
+
// now). The old code reported both as "timed out (300s)" and backed off 30s+,
|
|
2797
|
+
// which livelocked on dynamic pages.
|
|
2798
|
+
let abortReason = null;
|
|
2799
|
+
const timeout = setTimeout(() => {
|
|
2800
|
+
abortReason = 'timeout';
|
|
2801
|
+
controller.abort();
|
|
2802
|
+
}, timeoutMs);
|
|
2370
2803
|
// Heartbeat: check for screen changes every 5s, abort LLM if page changed
|
|
2371
2804
|
const heartbeat = setInterval(async () => {
|
|
2372
2805
|
const elapsed = Math.round((Date.now() - startTime) / 1000);
|
|
@@ -2412,6 +2845,7 @@ async function analyzeScreenshot(baseUrl, apiKey, model, screenshotBase64, eleme
|
|
|
2412
2845
|
const changeScore = calculateChangeScore(lastFingerprint, freshFingerprint);
|
|
2413
2846
|
if (changeScore >= 15) {
|
|
2414
2847
|
console.log(`[browse] Fingerprint confirms change (score: ${changeScore}) — aborting LLM (${elapsed}s)`);
|
|
2848
|
+
abortReason = 'screen_change';
|
|
2415
2849
|
controller.abort();
|
|
2416
2850
|
return;
|
|
2417
2851
|
}
|
|
@@ -2478,6 +2912,13 @@ async function analyzeScreenshot(baseUrl, apiKey, model, screenshotBase64, eleme
|
|
|
2478
2912
|
}
|
|
2479
2913
|
catch (err) {
|
|
2480
2914
|
if (err instanceof Error && err.name === 'AbortError') {
|
|
2915
|
+
const elapsed = Math.round((Date.now() - startTime) / 1000);
|
|
2916
|
+
if (abortReason === 'screen_change') {
|
|
2917
|
+
// The page changed while we were waiting: the answer is already stale.
|
|
2918
|
+
// Signal the caller to re-analyze right away (no backoff).
|
|
2919
|
+
console.log(`[browse] LLM aborted at ${elapsed}s — screen changed, re-analyzing now`);
|
|
2920
|
+
return { actions: [], attempt, durationMs: Date.now() - startTime, error: 'screen_changed' };
|
|
2921
|
+
}
|
|
2481
2922
|
console.error(`[browse] LLM request timed out (${Math.round(timeoutMs / 1000)}s)`);
|
|
2482
2923
|
return { actions: [], attempt, durationMs: Date.now() - startTime, error: 'timeout' };
|
|
2483
2924
|
}
|
|
@@ -2499,7 +2940,7 @@ async function analyzeScreenshotWithPrompt(baseUrl, apiKey, model, screenshotBas
|
|
|
2499
2940
|
const body = {
|
|
2500
2941
|
model,
|
|
2501
2942
|
messages: [
|
|
2502
|
-
{ role: 'system', content: customSystemPrompt },
|
|
2943
|
+
{ role: 'system', content: customSystemPrompt + PERSONAL_SAFETY_RULES },
|
|
2503
2944
|
{
|
|
2504
2945
|
role: 'user',
|
|
2505
2946
|
content: [
|
|
@@ -2509,11 +2950,15 @@ async function analyzeScreenshotWithPrompt(baseUrl, apiKey, model, screenshotBas
|
|
|
2509
2950
|
},
|
|
2510
2951
|
],
|
|
2511
2952
|
temperature: 0.1,
|
|
2512
|
-
max_tokens:
|
|
2953
|
+
max_tokens: 2048,
|
|
2513
2954
|
stream: false,
|
|
2514
2955
|
};
|
|
2515
2956
|
const controller = new AbortController();
|
|
2516
|
-
|
|
2957
|
+
let optAbortReason = null;
|
|
2958
|
+
const timeout = setTimeout(() => {
|
|
2959
|
+
optAbortReason = 'timeout';
|
|
2960
|
+
controller.abort();
|
|
2961
|
+
}, timeoutMs);
|
|
2517
2962
|
// Heartbeat: check for screen changes every 5s, abort if page changed
|
|
2518
2963
|
const heartbeat = setInterval(async () => {
|
|
2519
2964
|
const elapsed = Math.round((Date.now() - startTime) / 1000);
|
|
@@ -2530,6 +2975,7 @@ async function analyzeScreenshotWithPrompt(baseUrl, apiKey, model, screenshotBas
|
|
|
2530
2975
|
const flagTs = flagResult?.result?.value;
|
|
2531
2976
|
if (flagTs && flagTs > startTime) {
|
|
2532
2977
|
console.log(`[browse] DOM mutation detected in opt pass (${elapsed}s) — aborting`);
|
|
2978
|
+
optAbortReason = 'screen_change';
|
|
2533
2979
|
controller.abort();
|
|
2534
2980
|
return;
|
|
2535
2981
|
}
|
|
@@ -2573,7 +3019,12 @@ async function analyzeScreenshotWithPrompt(baseUrl, apiKey, model, screenshotBas
|
|
|
2573
3019
|
}
|
|
2574
3020
|
catch (err) {
|
|
2575
3021
|
if (err instanceof Error && err.name === 'AbortError') {
|
|
2576
|
-
return {
|
|
3022
|
+
return {
|
|
3023
|
+
actions: [],
|
|
3024
|
+
attempt,
|
|
3025
|
+
durationMs: Date.now() - startTime,
|
|
3026
|
+
error: optAbortReason === 'screen_change' ? 'screen_changed' : 'timeout',
|
|
3027
|
+
};
|
|
2577
3028
|
}
|
|
2578
3029
|
return { actions: [], attempt, durationMs: Date.now() - startTime, error: 'http_error' };
|
|
2579
3030
|
}
|
|
@@ -2767,19 +3218,24 @@ function getAdjacentKey(char) {
|
|
|
2767
3218
|
* Type text with human-like patterns: variable speed, occasional typos,
|
|
2768
3219
|
* backspaces, natural pauses. Uses OS-level keyboard (no CDP).
|
|
2769
3220
|
*/
|
|
2770
|
-
async function typeHumanLike(text, baseDelayMs = 120) {
|
|
3221
|
+
async function typeHumanLike(text, baseDelayMs = 120, guard) {
|
|
3222
|
+
const send = async (ch, keyName) => {
|
|
3223
|
+
if (guard)
|
|
3224
|
+
await guard.markOwn(ch, keyName);
|
|
3225
|
+
typeCharOS(ch);
|
|
3226
|
+
};
|
|
2771
3227
|
for (let i = 0; i < text.length; i++) {
|
|
2772
3228
|
const char = text[i];
|
|
2773
3229
|
// 1. Occasional typo (2% chance) — then backspace to correct
|
|
2774
3230
|
if (char !== '\n' && char !== '\t' && Math.random() < 0.02) {
|
|
2775
3231
|
const wrong = getAdjacentKey(char);
|
|
2776
|
-
|
|
3232
|
+
await send(wrong);
|
|
2777
3233
|
await sleep(randomInt(40, 120));
|
|
2778
|
-
|
|
3234
|
+
await send('\b', 'Backspace'); // backspace
|
|
2779
3235
|
await sleep(randomInt(25, 70));
|
|
2780
3236
|
}
|
|
2781
3237
|
// 2. Type the correct character
|
|
2782
|
-
|
|
3238
|
+
await send(char, char === '\n' ? 'Enter' : char === '\t' ? 'Tab' : undefined);
|
|
2783
3239
|
// 3. Variable delay based on character type (human rhythm)
|
|
2784
3240
|
let delay = baseDelayMs + randomInt(-Math.floor(baseDelayMs * 0.3), Math.floor(baseDelayMs * 0.4));
|
|
2785
3241
|
if (char === ' ')
|
|
@@ -2801,7 +3257,16 @@ async function typeHumanLike(text, baseDelayMs = 120) {
|
|
|
2801
3257
|
await sleep(randomInt(200, 600));
|
|
2802
3258
|
}
|
|
2803
3259
|
await sleep(Math.max(delay, 30));
|
|
3260
|
+
if (guard) {
|
|
3261
|
+
await guard.tick(text.slice(0, i + 1));
|
|
3262
|
+
if (guard.stopped) {
|
|
3263
|
+
console.log(`[browse] Typing stopped after ${i + 1}/${text.length} chars`);
|
|
3264
|
+
break;
|
|
3265
|
+
}
|
|
3266
|
+
}
|
|
2804
3267
|
}
|
|
3268
|
+
if (guard)
|
|
3269
|
+
await guard.dispose();
|
|
2805
3270
|
}
|
|
2806
3271
|
/**
|
|
2807
3272
|
* Wait for the screen to change by comparing screenshots.
|
|
@@ -2864,7 +3329,7 @@ async function callStealthLLM(baseUrl, apiKey, model, screenshotPath, timeoutMs,
|
|
|
2864
3329
|
body: JSON.stringify({
|
|
2865
3330
|
model,
|
|
2866
3331
|
messages: [
|
|
2867
|
-
{ role: 'system', content: systemPrompt },
|
|
3332
|
+
{ role: 'system', content: systemPrompt + PERSONAL_SAFETY_RULES },
|
|
2868
3333
|
{
|
|
2869
3334
|
role: 'user',
|
|
2870
3335
|
content: [
|
|
@@ -2961,9 +3426,41 @@ async function runStealthLoop(baseUrl, apiKey, model, maxActions, intervalMs, ti
|
|
|
2961
3426
|
}
|
|
2962
3427
|
case 'type': {
|
|
2963
3428
|
if (result.text && result.text.length > 0) {
|
|
3429
|
+
const guard = new human_input_guard_1.TypingGuard({
|
|
3430
|
+
enabled: fieldSafety.stopOnUserInput,
|
|
3431
|
+
osLevel: true,
|
|
3432
|
+
label: result.text.slice(0, 40),
|
|
3433
|
+
});
|
|
3434
|
+
// Stealth has no DOM: use the focused control's own name as the
|
|
3435
|
+
// label for both the personal classifier and the human-skip list.
|
|
3436
|
+
const control = await guard.currentOsControl().catch(() => null);
|
|
3437
|
+
if (control) {
|
|
3438
|
+
if (control.signature && skippedOsFields.has(control.signature)) {
|
|
3439
|
+
console.log(`[stealth] Skipping field - ${skippedOsFields.get(control.signature)}`);
|
|
3440
|
+
actionCount++;
|
|
3441
|
+
break;
|
|
3442
|
+
}
|
|
3443
|
+
if (control.name && fieldSafety.allowPersonal === false) {
|
|
3444
|
+
const cls = (0, personal_fields_1.classifyField)({ label: control.name, ariaLabel: control.name });
|
|
3445
|
+
if ((0, personal_fields_1.isBlocked)(cls, { allowPersonal: fieldSafety.allowPersonal })) {
|
|
3446
|
+
skippedOsFields.set(control.signature, `left to you (${cls.category}: "${control.name.slice(0, 60)}")`);
|
|
3447
|
+
console.log(`[stealth] Not answering a personal question - left to you (${cls.category}: "${control.name.slice(0, 60)}")`);
|
|
3448
|
+
actionCount++;
|
|
3449
|
+
break;
|
|
3450
|
+
}
|
|
3451
|
+
}
|
|
3452
|
+
}
|
|
2964
3453
|
console.log(`[stealth] Typing ${result.text.length} chars...`);
|
|
2965
3454
|
await sleep(randomInt(300, 700)); // natural pause before typing
|
|
2966
|
-
await
|
|
3455
|
+
await guard.arm({ isCodeEditor: false });
|
|
3456
|
+
await typeHumanLike(result.text, charDelayMs, guard);
|
|
3457
|
+
if (guard.stopped) {
|
|
3458
|
+
const after = await guard.currentOsControl().catch(() => null);
|
|
3459
|
+
const sig = after?.signature || control?.signature;
|
|
3460
|
+
if (sig)
|
|
3461
|
+
skippedOsFields.set(sig, `you answered this one yourself (${guard.describeStop()})`);
|
|
3462
|
+
console.log(`[stealth] Stopped typing - you answered this one yourself (${guard.describeStop()})`);
|
|
3463
|
+
}
|
|
2967
3464
|
actionCount++;
|
|
2968
3465
|
}
|
|
2969
3466
|
else {
|