teamshare-bridge 0.21.26 → 0.21.27
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cli/commands/browse.js +343 -24
- package/dist/cli/commands/browse.js.map +1 -1
- package/dist/cli/commands/research.js +5 -0
- package/dist/cli/commands/research.js.map +1 -1
- package/dist/cli/index.js +6 -0
- package/dist/cli/index.js.map +1 -1
- package/dist/lib/browser/human-input-guard.d.ts +15 -2
- package/dist/lib/browser/human-input-guard.js +18 -4
- package/dist/lib/browser/human-input-guard.js.map +1 -1
- package/package.json +2 -2
|
@@ -59,6 +59,8 @@ const child_process_1 = require("child_process");
|
|
|
59
59
|
const cdp_1 = require("../../lib/browser/cdp");
|
|
60
60
|
const launch_1 = require("../../lib/browser/launch");
|
|
61
61
|
const session_1 = require("../../lib/browser/session");
|
|
62
|
+
const personal_fields_1 = require("../../lib/browser/personal-fields");
|
|
63
|
+
const human_input_guard_1 = require("../../lib/browser/human-input-guard");
|
|
62
64
|
/** Create a fingerprint from the current element list. */
|
|
63
65
|
function createFingerprint(elements) {
|
|
64
66
|
const groups = new Map();
|
|
@@ -239,10 +241,10 @@ CORRECTION MODE:
|
|
|
239
241
|
- If you previously typed wrong code, use write_code to replace with correct code
|
|
240
242
|
- You may be asked the same question multiple times — each time, check if the current answer is correct before acting
|
|
241
243
|
|
|
242
|
-
PERSONAL QUESTIONS —
|
|
243
|
-
-
|
|
244
|
-
- If a question asks for personal info, do NOT click or type —
|
|
245
|
-
- If ALL remaining questions are personal, return done
|
|
244
|
+
PERSONAL QUESTIONS — NEVER ANSWER THEM:
|
|
245
|
+
- Questions about the person operating the browser (name, email, phone, address, SSN, passport, password, card, bank, date of birth, salary, work authorization, personal profile links) are OFF LIMITS. Read the PERSONAL QUESTIONS addendum appended below and follow it exactly.
|
|
246
|
+
- If a question asks for personal info, do NOT click or type — leave it empty.
|
|
247
|
+
- A personal field left empty is NOT an unanswered question. If ALL remaining questions are personal, return done immediately.
|
|
246
248
|
|
|
247
249
|
RESPONSE FORMAT RULES (CRITICAL — follow exactly):
|
|
248
250
|
- Return ONLY a valid JSON array. No explanations, no markdown fences, no extra text.
|
|
@@ -462,6 +464,15 @@ async function cmdBrowse(flags) {
|
|
|
462
464
|
const stealth = flags.has('stealth');
|
|
463
465
|
const hybrid = flags.has('hybrid');
|
|
464
466
|
const autoLaunch = flags.has('auto-launch');
|
|
467
|
+
// Field safety (see personal-fields.ts / human-input-guard.ts):
|
|
468
|
+
// --stop-on-user-input opt-in: stop typing and skip the field for the run
|
|
469
|
+
// as soon as the human types into it
|
|
470
|
+
// --personal-questions "allow" answers personal questions; credentials and
|
|
471
|
+
// card/bank fields stay blocked either way
|
|
472
|
+
const stopOnUserInput = flags.has('stop-on-user-input') && flags.get('stop-on-user-input') !== 'false';
|
|
473
|
+
const personalMode = (flags.get('personal-questions') ?? 'skip').toLowerCase();
|
|
474
|
+
fieldSafety.stopOnUserInput = stopOnUserInput;
|
|
475
|
+
fieldSafety.allowPersonal = personalMode === 'allow';
|
|
465
476
|
// The auto-answer browser is ALWAYS visible (2026-09): a headless window is
|
|
466
477
|
// the "profile seems mismatched" case and hides the pages the loop works on.
|
|
467
478
|
// `--headless` is accepted for backward compatibility but ignored.
|
|
@@ -493,6 +504,8 @@ async function cmdBrowse(flags) {
|
|
|
493
504
|
}
|
|
494
505
|
console.log(`[browse] Char delay: ${charDelayMs}ms`);
|
|
495
506
|
console.log(`[browse] Screenshot interval: ${intervalMs}ms`);
|
|
507
|
+
console.log(`[browse] Personal questions: ${fieldSafety.allowPersonal ? 'answered (credentials still blocked)' : 'never answered (left for you)'}`);
|
|
508
|
+
console.log(`[browse] Stop typing when you type in the field: ${fieldSafety.stopOnUserInput ? 'on' : 'off'}`);
|
|
496
509
|
if (!stealth) {
|
|
497
510
|
console.log(`[browse] Max consecutive failures: ${maxConsecutiveFailures === 0 ? 'unlimited' : maxConsecutiveFailures}`);
|
|
498
511
|
}
|
|
@@ -826,6 +839,18 @@ async function cmdBrowse(flags) {
|
|
|
826
839
|
console.log(`[browse] Skipping ${act.action} — code already confirmed optimal`);
|
|
827
840
|
continue;
|
|
828
841
|
}
|
|
842
|
+
// Field safety: never type into a personal field, a credential field, or
|
|
843
|
+
// a field the human already answered themselves.
|
|
844
|
+
if (act.action === 'write_code' || act.action === 'type') {
|
|
845
|
+
const targetEl = act.elementIndex !== undefined
|
|
846
|
+
? elements.find((e) => e.i === act.elementIndex)
|
|
847
|
+
: undefined;
|
|
848
|
+
const gated = fieldGate(targetEl);
|
|
849
|
+
if (gated) {
|
|
850
|
+
console.log(`[browse] Skipping element ${act.elementIndex} - ${gated}`);
|
|
851
|
+
continue;
|
|
852
|
+
}
|
|
853
|
+
}
|
|
829
854
|
// Ask LLM to decide if action is redundant
|
|
830
855
|
const redundancy = await isActionRedundant(baseUrl, apiKey, model, page, elements, act, timeoutMs);
|
|
831
856
|
if (redundancy.decision === 'skip') {
|
|
@@ -1017,6 +1042,14 @@ Do NOT simplify variable names or change the function signature. Only optimize t
|
|
|
1017
1042
|
console.log(`[browse] Reached max actions (${maxActions})`);
|
|
1018
1043
|
}
|
|
1019
1044
|
console.log(`[browse] Agent finished. ${actionCount} actions performed.`);
|
|
1045
|
+
const skipped = personalSkipSummary();
|
|
1046
|
+
if (skipped.personal > 0 || skipped.human > 0) {
|
|
1047
|
+
console.log(`[browse] Left for you: ${skipped.personal} personal question(s) never answered` +
|
|
1048
|
+
(skipped.human > 0 ? `, ${skipped.human} field(s) you answered yourself` : ''));
|
|
1049
|
+
for (const entry of skippedFields.values()) {
|
|
1050
|
+
console.log(` - ${entry.note}`);
|
|
1051
|
+
}
|
|
1052
|
+
}
|
|
1020
1053
|
page.ws.close();
|
|
1021
1054
|
}
|
|
1022
1055
|
// ── CDP Helpers ──────────────────────────────────────────────────────────────
|
|
@@ -1184,6 +1217,124 @@ async function captureScreenshot(page, quality, maxEdge = 0) {
|
|
|
1184
1217
|
return null;
|
|
1185
1218
|
}
|
|
1186
1219
|
}
|
|
1220
|
+
/**
|
|
1221
|
+
* Field-safety addendum appended to every system prompt (main loop, custom /
|
|
1222
|
+
* optimization passes, stealth loop). The loop may not answer questions that
|
|
1223
|
+
* ask about the human behind the browser.
|
|
1224
|
+
*/
|
|
1225
|
+
const PERSONAL_SAFETY_RULES = `
|
|
1226
|
+
PERSONAL QUESTIONS - NEVER ANSWER THEM:
|
|
1227
|
+
- Some questions ask about YOU (the human operating the browser), not about the
|
|
1228
|
+
task. NEVER answer them with an invented value.
|
|
1229
|
+
- Never type: names, email addresses, phone numbers, postal addresses, dates of
|
|
1230
|
+
birth or age, government IDs (SSN/NIN/passport/driving licence/tax id),
|
|
1231
|
+
passwords, one-time codes, card or bank details, salary or compensation,
|
|
1232
|
+
notice periods, work-authorization/visa/sponsorship answers, or personal
|
|
1233
|
+
profile links (LinkedIn, GitHub, portfolio, personal website).
|
|
1234
|
+
- Fields marked "[NEVER ANSWER: ...]" or "[PERSONAL: ...]" in the element list
|
|
1235
|
+
MUST be left untouched - do not click them, do not type into them.
|
|
1236
|
+
- Leaving a personal field EMPTY is correct and expected. A personal field is
|
|
1237
|
+
NOT an unanswered question: never treat it as one, and never delay "done"
|
|
1238
|
+
because of it.
|
|
1239
|
+
- If every remaining question is personal, return done.
|
|
1240
|
+
- Demographics and free-text personal questions (gender, marital status, "tell
|
|
1241
|
+
us about yourself") are the human's to answer: prefer to skip them too.
|
|
1242
|
+
`;
|
|
1243
|
+
const fieldSafety = {
|
|
1244
|
+
stopOnUserInput: false,
|
|
1245
|
+
allowPersonal: false,
|
|
1246
|
+
};
|
|
1247
|
+
/** fieldKey -> why the field must not be typed into for the rest of the run. */
|
|
1248
|
+
const skippedFields = new Map();
|
|
1249
|
+
/**
|
|
1250
|
+
* Stealth mode has no DOM, so fields are identified by the focused OS control
|
|
1251
|
+
* (AutomationId|ClassName|Name) instead of a field key.
|
|
1252
|
+
*/
|
|
1253
|
+
const skippedOsFields = new Map();
|
|
1254
|
+
function signalsOf(el) {
|
|
1255
|
+
return {
|
|
1256
|
+
tag: el.tag,
|
|
1257
|
+
type: el.type,
|
|
1258
|
+
name: el.name,
|
|
1259
|
+
id: el.id,
|
|
1260
|
+
autocomplete: el.autocomplete,
|
|
1261
|
+
placeholder: el.placeholder,
|
|
1262
|
+
ariaLabel: el.ariaLabel,
|
|
1263
|
+
label: el.label,
|
|
1264
|
+
questionText: el.questionText,
|
|
1265
|
+
text: el.text,
|
|
1266
|
+
};
|
|
1267
|
+
}
|
|
1268
|
+
function classificationOf(el) {
|
|
1269
|
+
return {
|
|
1270
|
+
tier: el.personalTier ?? null,
|
|
1271
|
+
category: el.personalCategory ?? null,
|
|
1272
|
+
label: el.personalLabel ?? '',
|
|
1273
|
+
reason: el.personalReason ?? '',
|
|
1274
|
+
};
|
|
1275
|
+
}
|
|
1276
|
+
/**
|
|
1277
|
+
* Classify every typable element and stamp its stable field key. Elements are
|
|
1278
|
+
* annotated in place so the gate below never re-parses the DOM.
|
|
1279
|
+
*/
|
|
1280
|
+
function annotateFieldSafety(elements) {
|
|
1281
|
+
for (const el of elements) {
|
|
1282
|
+
const signals = signalsOf(el);
|
|
1283
|
+
el.fieldKey = (0, personal_fields_1.fieldKeyOf)(signals);
|
|
1284
|
+
if (!el.typable)
|
|
1285
|
+
continue;
|
|
1286
|
+
const cls = (0, personal_fields_1.classifyField)(signals);
|
|
1287
|
+
el.personalTier = cls.tier;
|
|
1288
|
+
el.personalCategory = cls.category;
|
|
1289
|
+
el.personalLabel = cls.label;
|
|
1290
|
+
el.personalReason = cls.reason;
|
|
1291
|
+
}
|
|
1292
|
+
}
|
|
1293
|
+
function shortLabel(el) {
|
|
1294
|
+
const raw = el.label || el.questionText || el.text || el.placeholder || el.name || '';
|
|
1295
|
+
return raw.replace(/\s+/g, ' ').trim().slice(0, 60);
|
|
1296
|
+
}
|
|
1297
|
+
/**
|
|
1298
|
+
* Returns a human-readable reason when this element must not be typed into,
|
|
1299
|
+
* or null when it is fine to type.
|
|
1300
|
+
*/
|
|
1301
|
+
function fieldGate(el) {
|
|
1302
|
+
if (!el || !el.typable)
|
|
1303
|
+
return null;
|
|
1304
|
+
const key = el.fieldKey ?? (0, personal_fields_1.fieldKeyOf)(signalsOf(el));
|
|
1305
|
+
const already = skippedFields.get(key);
|
|
1306
|
+
if (already)
|
|
1307
|
+
return already.note;
|
|
1308
|
+
const cls = classificationOf(el);
|
|
1309
|
+
if (cls.tier === null)
|
|
1310
|
+
return null;
|
|
1311
|
+
if (!(0, personal_fields_1.isBlocked)(cls, { allowPersonal: fieldSafety.allowPersonal }))
|
|
1312
|
+
return null;
|
|
1313
|
+
const note = `left to you (${cls.category}: "${shortLabel(el) || cls.label}")`;
|
|
1314
|
+
skippedFields.set(key, { kind: 'personal', note });
|
|
1315
|
+
console.log(`[browse] Not answering a personal question - ${note}`);
|
|
1316
|
+
return note;
|
|
1317
|
+
}
|
|
1318
|
+
/** Record a field the human took over mid-typing; it is never typed again. */
|
|
1319
|
+
function noteHumanTookOver(el, reason) {
|
|
1320
|
+
const key = el?.fieldKey ?? (el ? (0, personal_fields_1.fieldKeyOf)(signalsOf(el)) : '');
|
|
1321
|
+
if (!key)
|
|
1322
|
+
return;
|
|
1323
|
+
const note = `you answered this one yourself (${reason})`;
|
|
1324
|
+
skippedFields.set(key, { kind: 'human', note });
|
|
1325
|
+
console.log(`[browse] Stopped typing - ${note}`);
|
|
1326
|
+
}
|
|
1327
|
+
function personalSkipSummary() {
|
|
1328
|
+
let personal = 0;
|
|
1329
|
+
let human = 0;
|
|
1330
|
+
for (const entry of skippedFields.values()) {
|
|
1331
|
+
if (entry.kind === 'personal')
|
|
1332
|
+
personal++;
|
|
1333
|
+
else
|
|
1334
|
+
human++;
|
|
1335
|
+
}
|
|
1336
|
+
return { personal, human };
|
|
1337
|
+
}
|
|
1187
1338
|
/**
|
|
1188
1339
|
* Extract all interactive elements from the DOM with their exact text,
|
|
1189
1340
|
* bounding boxes, state, and question group assignments.
|
|
@@ -1224,6 +1375,29 @@ async function extractElements(page) {
|
|
|
1224
1375
|
text,
|
|
1225
1376
|
tag: el.tagName.toLowerCase(),
|
|
1226
1377
|
type: el.type || el.getAttribute('role') || '',
|
|
1378
|
+
typable: el.tagName === 'INPUT' || el.tagName === 'TEXTAREA' || el.isContentEditable === true,
|
|
1379
|
+
name: el.getAttribute('name') || '',
|
|
1380
|
+
id: el.getAttribute('id') || '',
|
|
1381
|
+
autocomplete: el.getAttribute('autocomplete') || '',
|
|
1382
|
+
placeholder: el.getAttribute('placeholder') || '',
|
|
1383
|
+
ariaLabel: el.getAttribute('aria-label') || '',
|
|
1384
|
+
label: (() => {
|
|
1385
|
+
try {
|
|
1386
|
+
const id = el.getAttribute('id');
|
|
1387
|
+
if (id) {
|
|
1388
|
+
const forLabel = document.querySelector('label[for="' + CSS.escape(id) + '"]');
|
|
1389
|
+
if (forLabel && forLabel.innerText) return forLabel.innerText.trim().replace(/\\s+/g, ' ').slice(0, 200);
|
|
1390
|
+
}
|
|
1391
|
+
const wrap = el.closest ? el.closest('label') : null;
|
|
1392
|
+
if (wrap && wrap.innerText) return wrap.innerText.trim().replace(/\\s+/g, ' ').slice(0, 200);
|
|
1393
|
+
const labelled = el.getAttribute('aria-labelledby');
|
|
1394
|
+
if (labelled) {
|
|
1395
|
+
const t = document.getElementById(labelled);
|
|
1396
|
+
if (t && t.innerText) return t.innerText.trim().replace(/\\s+/g, ' ').slice(0, 200);
|
|
1397
|
+
}
|
|
1398
|
+
} catch (err) {}
|
|
1399
|
+
return '';
|
|
1400
|
+
})(),
|
|
1227
1401
|
checked: el.checked || el.getAttribute('aria-checked') === 'true' || false,
|
|
1228
1402
|
x: Math.round(rect.x + rect.width / 2),
|
|
1229
1403
|
y: Math.round(rect.y + rect.height / 2),
|
|
@@ -1336,6 +1510,13 @@ async function extractElements(page) {
|
|
|
1336
1510
|
y: item.y,
|
|
1337
1511
|
group: item.groupId,
|
|
1338
1512
|
inViewport: item.inViewport,
|
|
1513
|
+
typable: item.typable,
|
|
1514
|
+
name: item.name,
|
|
1515
|
+
id: item.id,
|
|
1516
|
+
autocomplete: item.autocomplete,
|
|
1517
|
+
placeholder: item.placeholder,
|
|
1518
|
+
ariaLabel: item.ariaLabel,
|
|
1519
|
+
label: item.label,
|
|
1339
1520
|
});
|
|
1340
1521
|
});
|
|
1341
1522
|
|
|
@@ -1344,7 +1525,9 @@ async function extractElements(page) {
|
|
|
1344
1525
|
`;
|
|
1345
1526
|
try {
|
|
1346
1527
|
const result = await cdpEval(page, js);
|
|
1347
|
-
|
|
1528
|
+
const elements = result?.result?.value || [];
|
|
1529
|
+
annotateFieldSafety(elements);
|
|
1530
|
+
return elements;
|
|
1348
1531
|
}
|
|
1349
1532
|
catch {
|
|
1350
1533
|
return [];
|
|
@@ -1393,6 +1576,11 @@ async function executeAction(page, act, elements, charDelayMs) {
|
|
|
1393
1576
|
console.log(`[browse] Element index ${act.elementIndex} not found, will retry`);
|
|
1394
1577
|
return false;
|
|
1395
1578
|
}
|
|
1579
|
+
const gated = fieldGate(el);
|
|
1580
|
+
if (gated) {
|
|
1581
|
+
console.log(`[browse] Skipping element ${el.i} - ${gated}`);
|
|
1582
|
+
return true;
|
|
1583
|
+
}
|
|
1396
1584
|
// 1. Detect editor type
|
|
1397
1585
|
const editor = await detectEditor(page);
|
|
1398
1586
|
const isCodeEditor = editor !== null;
|
|
@@ -1407,12 +1595,19 @@ async function executeAction(page, act, elements, charDelayMs) {
|
|
|
1407
1595
|
// 3. Normalize indentation for code editors (LLM often adds extra indent)
|
|
1408
1596
|
const text = isCodeEditor ? normalizeCodeIndent(rawText) : rawText;
|
|
1409
1597
|
console.log(`[browse] Typing ${text.length} characters...`);
|
|
1598
|
+
const guard = new human_input_guard_1.TypingGuard({
|
|
1599
|
+
enabled: fieldSafety.stopOnUserInput,
|
|
1600
|
+
page,
|
|
1601
|
+
label: shortLabel(el),
|
|
1602
|
+
});
|
|
1410
1603
|
if (isCodeEditor) {
|
|
1411
|
-
await insertIntoCodeEditor(page, editor.selector, text, charDelayMs);
|
|
1604
|
+
await insertIntoCodeEditor(page, editor.selector, text, charDelayMs, guard);
|
|
1412
1605
|
}
|
|
1413
1606
|
else {
|
|
1414
|
-
await insertIntoGenericInput(page, el, text, charDelayMs);
|
|
1607
|
+
await insertIntoGenericInput(page, el, text, charDelayMs, guard);
|
|
1415
1608
|
}
|
|
1609
|
+
if (guard.stopped)
|
|
1610
|
+
noteHumanTookOver(el, guard.describeStop());
|
|
1416
1611
|
return true;
|
|
1417
1612
|
}
|
|
1418
1613
|
case 'write_code': {
|
|
@@ -1423,6 +1618,11 @@ async function executeAction(page, act, elements, charDelayMs) {
|
|
|
1423
1618
|
console.log(`[browse] Element index ${act.elementIndex} not found, will retry`);
|
|
1424
1619
|
return false;
|
|
1425
1620
|
}
|
|
1621
|
+
const gated = fieldGate(el);
|
|
1622
|
+
if (gated) {
|
|
1623
|
+
console.log(`[browse] Skipping element ${el.i} - ${gated}`);
|
|
1624
|
+
return true;
|
|
1625
|
+
}
|
|
1426
1626
|
// 1. Detect editor type
|
|
1427
1627
|
const editor = await detectEditor(page);
|
|
1428
1628
|
const isCodeEditor = editor !== null;
|
|
@@ -1435,13 +1635,20 @@ async function executeAction(page, act, elements, charDelayMs) {
|
|
|
1435
1635
|
.replace(/\\"/g, '"')
|
|
1436
1636
|
.replace(/\\\\/g, '\\');
|
|
1437
1637
|
console.log(`[browse] Writing code (${text.length} chars)...`);
|
|
1638
|
+
const guard = new human_input_guard_1.TypingGuard({
|
|
1639
|
+
enabled: fieldSafety.stopOnUserInput,
|
|
1640
|
+
page,
|
|
1641
|
+
label: shortLabel(el),
|
|
1642
|
+
});
|
|
1438
1643
|
// 3. write_code inserts EXACTLY as-is — no normalizeCodeIndent
|
|
1439
1644
|
if (isCodeEditor) {
|
|
1440
|
-
await insertIntoCodeEditor(page, editor.selector, text, charDelayMs);
|
|
1645
|
+
await insertIntoCodeEditor(page, editor.selector, text, charDelayMs, guard);
|
|
1441
1646
|
}
|
|
1442
1647
|
else {
|
|
1443
|
-
await insertIntoGenericInput(page, el, text, charDelayMs);
|
|
1648
|
+
await insertIntoGenericInput(page, el, text, charDelayMs, guard);
|
|
1444
1649
|
}
|
|
1650
|
+
if (guard.stopped)
|
|
1651
|
+
noteHumanTookOver(el, guard.describeStop());
|
|
1445
1652
|
return true;
|
|
1446
1653
|
}
|
|
1447
1654
|
case 'scroll': {
|
|
@@ -1515,6 +1722,11 @@ async function executeActionHybrid(page, act, elements, charDelayMs) {
|
|
|
1515
1722
|
console.log(`[browse] Element index ${act.elementIndex} not found, will retry`);
|
|
1516
1723
|
return false;
|
|
1517
1724
|
}
|
|
1725
|
+
const gated = fieldGate(el);
|
|
1726
|
+
if (gated) {
|
|
1727
|
+
console.log(`[browse] Skipping element ${el.i} - ${gated}`);
|
|
1728
|
+
return true;
|
|
1729
|
+
}
|
|
1518
1730
|
const editor = await detectEditor(page);
|
|
1519
1731
|
const isCodeEditor = editor !== null;
|
|
1520
1732
|
if (editor)
|
|
@@ -1533,7 +1745,16 @@ async function executeActionHybrid(page, act, elements, charDelayMs) {
|
|
|
1533
1745
|
pressKeyOS('ctrl+a');
|
|
1534
1746
|
await sleep(randomInt(50, 100));
|
|
1535
1747
|
}
|
|
1536
|
-
|
|
1748
|
+
const guard = new human_input_guard_1.TypingGuard({
|
|
1749
|
+
enabled: fieldSafety.stopOnUserInput,
|
|
1750
|
+
page,
|
|
1751
|
+
label: shortLabel(el),
|
|
1752
|
+
});
|
|
1753
|
+
if (guard)
|
|
1754
|
+
await guard.arm({ isCodeEditor });
|
|
1755
|
+
await typeHumanLike(text, charDelayMs, guard);
|
|
1756
|
+
if (guard.stopped)
|
|
1757
|
+
noteHumanTookOver(el, guard.describeStop());
|
|
1537
1758
|
return true;
|
|
1538
1759
|
}
|
|
1539
1760
|
case 'write_code': {
|
|
@@ -1544,6 +1765,11 @@ async function executeActionHybrid(page, act, elements, charDelayMs) {
|
|
|
1544
1765
|
console.log(`[browse] Element index ${act.elementIndex} not found, will retry`);
|
|
1545
1766
|
return false;
|
|
1546
1767
|
}
|
|
1768
|
+
const gated = fieldGate(el);
|
|
1769
|
+
if (gated) {
|
|
1770
|
+
console.log(`[browse] Skipping element ${el.i} - ${gated}`);
|
|
1771
|
+
return true;
|
|
1772
|
+
}
|
|
1547
1773
|
const editor = await detectEditor(page);
|
|
1548
1774
|
const isCodeEditor = editor !== null;
|
|
1549
1775
|
if (editor)
|
|
@@ -1561,7 +1787,16 @@ async function executeActionHybrid(page, act, elements, charDelayMs) {
|
|
|
1561
1787
|
pressKeyOS('ctrl+a');
|
|
1562
1788
|
await sleep(randomInt(50, 100));
|
|
1563
1789
|
}
|
|
1564
|
-
|
|
1790
|
+
const guard = new human_input_guard_1.TypingGuard({
|
|
1791
|
+
enabled: fieldSafety.stopOnUserInput,
|
|
1792
|
+
page,
|
|
1793
|
+
label: shortLabel(el),
|
|
1794
|
+
});
|
|
1795
|
+
if (guard)
|
|
1796
|
+
await guard.arm({ isCodeEditor });
|
|
1797
|
+
await typeHumanLike(text, charDelayMs, guard);
|
|
1798
|
+
if (guard.stopped)
|
|
1799
|
+
noteHumanTookOver(el, guard.describeStop());
|
|
1565
1800
|
return true;
|
|
1566
1801
|
}
|
|
1567
1802
|
case 'scroll': {
|
|
@@ -1748,13 +1983,21 @@ function hasUnansweredQuestions(elements) {
|
|
|
1748
1983
|
const types = els.map((e) => e.type);
|
|
1749
1984
|
const isRadio = types.some((t) => t === 'radio');
|
|
1750
1985
|
const isCheckbox = types.some((t) => t === 'checkbox');
|
|
1751
|
-
|
|
1986
|
+
// Personal fields are the human's to answer: they are never "unanswered"
|
|
1987
|
+
// for the agent, so a `done` from the model with them left empty sticks
|
|
1988
|
+
// instead of triggering the "you must answer everything" retry loop.
|
|
1989
|
+
const inputs = els.filter((e) => (e.tag === 'input' || e.tag === 'textarea') &&
|
|
1990
|
+
e.type !== 'radio' &&
|
|
1991
|
+
e.type !== 'checkbox' &&
|
|
1992
|
+
e.typable !== false);
|
|
1993
|
+
const openInputs = inputs.filter((e) => !(0, personal_fields_1.excludesFromCompleteness)(classificationOf(e)));
|
|
1994
|
+
const hasInput = openInputs.length > 0;
|
|
1752
1995
|
const checkedCount = els.filter((e) => e.checked).length;
|
|
1753
1996
|
if (isRadio && checkedCount === 0)
|
|
1754
1997
|
return true;
|
|
1755
1998
|
if (isCheckbox && checkedCount < els.length)
|
|
1756
1999
|
return true;
|
|
1757
|
-
if (hasInput && !
|
|
2000
|
+
if (hasInput && !openInputs.some((e) => e.text && e.tag === 'input'))
|
|
1758
2001
|
return true;
|
|
1759
2002
|
}
|
|
1760
2003
|
return false;
|
|
@@ -1979,7 +2222,7 @@ async function removeTrailingBrackets(page) {
|
|
|
1979
2222
|
console.log(`[browse] Removed ${extra} extra trailing bracket(s)`);
|
|
1980
2223
|
}
|
|
1981
2224
|
/** Insert text into ANY code editor — character-by-character typing, no pasting. */
|
|
1982
|
-
async function insertIntoCodeEditor(page, selector, text, charDelayMs) {
|
|
2225
|
+
async function insertIntoCodeEditor(page, selector, text, charDelayMs, guard) {
|
|
1983
2226
|
// Focus the editor
|
|
1984
2227
|
await focusSelector(page, selector);
|
|
1985
2228
|
await sleep(100);
|
|
@@ -2031,8 +2274,12 @@ async function insertIntoCodeEditor(page, selector, text, charDelayMs) {
|
|
|
2031
2274
|
// Type character by character — NO pasting
|
|
2032
2275
|
const est = Math.round(text.length * charDelayMs / 1000);
|
|
2033
2276
|
console.log(`[browse] Typing ${text.length} chars (~${est}s est.)...`);
|
|
2277
|
+
if (guard)
|
|
2278
|
+
await guard.arm({ selector, isCodeEditor: true });
|
|
2034
2279
|
for (let i = 0; i < text.length; i++) {
|
|
2035
2280
|
const char = text[i];
|
|
2281
|
+
if (guard)
|
|
2282
|
+
await guard.markOwn(char, char === '\n' ? 'Enter' : char === '\t' ? 'Tab' : undefined);
|
|
2036
2283
|
if (char === '\n') {
|
|
2037
2284
|
await cdpSend(page, 'Input.dispatchKeyEvent', {
|
|
2038
2285
|
type: 'keyDown', key: 'Enter', code: 'Enter', text: '\r',
|
|
@@ -2070,7 +2317,16 @@ async function insertIntoCodeEditor(page, selector, text, charDelayMs) {
|
|
|
2070
2317
|
await cdpSend(page, 'Input.dispatchKeyEvent', upParams);
|
|
2071
2318
|
}
|
|
2072
2319
|
await sleep(variableDelay(charDelayMs, char, i, text.length));
|
|
2320
|
+
if (guard) {
|
|
2321
|
+
await guard.tick(text.slice(0, i + 1));
|
|
2322
|
+
if (guard.stopped) {
|
|
2323
|
+
console.log(`[browse] Typing stopped after ${i + 1}/${text.length} chars`);
|
|
2324
|
+
break;
|
|
2325
|
+
}
|
|
2326
|
+
}
|
|
2073
2327
|
}
|
|
2328
|
+
if (guard)
|
|
2329
|
+
await guard.dispose();
|
|
2074
2330
|
// Post-typing: detect and remove extra trailing brackets
|
|
2075
2331
|
await removeTrailingBrackets(page);
|
|
2076
2332
|
// Re-enable auto-formatting
|
|
@@ -2091,7 +2347,7 @@ async function insertIntoCodeEditor(page, selector, text, charDelayMs) {
|
|
|
2091
2347
|
console.log(`[browse] Typing complete (${text.length} chars)`);
|
|
2092
2348
|
}
|
|
2093
2349
|
/** Insert text into a generic input — click focus + char-by-char typing. */
|
|
2094
|
-
async function insertIntoGenericInput(page, el, text, charDelayMs) {
|
|
2350
|
+
async function insertIntoGenericInput(page, el, text, charDelayMs, guard) {
|
|
2095
2351
|
await cdpSend(page, 'Input.dispatchMouseEvent', {
|
|
2096
2352
|
type: 'mousePressed', x: el.x, y: el.y, button: 'left', clickCount: 1,
|
|
2097
2353
|
});
|
|
@@ -2101,8 +2357,12 @@ async function insertIntoGenericInput(page, el, text, charDelayMs) {
|
|
|
2101
2357
|
await sleep(200);
|
|
2102
2358
|
await clearEditor(page);
|
|
2103
2359
|
await sleep(200);
|
|
2360
|
+
if (guard)
|
|
2361
|
+
await guard.arm({ isCodeEditor: false });
|
|
2104
2362
|
for (let i = 0; i < text.length; i++) {
|
|
2105
2363
|
const char = text[i];
|
|
2364
|
+
if (guard)
|
|
2365
|
+
await guard.markOwn(char, char === '\n' ? 'Enter' : char === '\t' ? 'Tab' : undefined);
|
|
2106
2366
|
if (char === '\n') {
|
|
2107
2367
|
await cdpSend(page, 'Input.dispatchKeyEvent', {
|
|
2108
2368
|
type: 'keyDown', key: 'Enter', code: 'Enter', text: '\r',
|
|
@@ -2140,7 +2400,16 @@ async function insertIntoGenericInput(page, el, text, charDelayMs) {
|
|
|
2140
2400
|
await cdpSend(page, 'Input.dispatchKeyEvent', upParams);
|
|
2141
2401
|
}
|
|
2142
2402
|
await sleep(variableDelay(charDelayMs, char, i, text.length));
|
|
2403
|
+
if (guard) {
|
|
2404
|
+
await guard.tick(text.slice(0, i + 1));
|
|
2405
|
+
if (guard.stopped) {
|
|
2406
|
+
console.log(`[browse] Typing stopped after ${i + 1}/${text.length} chars`);
|
|
2407
|
+
break;
|
|
2408
|
+
}
|
|
2409
|
+
}
|
|
2143
2410
|
}
|
|
2411
|
+
if (guard)
|
|
2412
|
+
await guard.dispose();
|
|
2144
2413
|
}
|
|
2145
2414
|
/**
|
|
2146
2415
|
* Extract page context for the LLM: title, URL, headings, errors, editor content.
|
|
@@ -2492,7 +2761,11 @@ async function analyzeScreenshot(baseUrl, apiKey, model, screenshotBase64, eleme
|
|
|
2492
2761
|
for (const el of els) {
|
|
2493
2762
|
const checked = el.checked ? ' [CHECKED]' : '';
|
|
2494
2763
|
const visibility = el.inViewport ? '' : ' [OFF-SCREEN]';
|
|
2495
|
-
|
|
2764
|
+
const skipNote = el.fieldKey && skippedFields.has(el.fieldKey)
|
|
2765
|
+
? ` [SKIPPED: ${skippedFields.get(el.fieldKey).note}]`
|
|
2766
|
+
: '';
|
|
2767
|
+
const personal = el.typable ? (0, personal_fields_1.promptMarker)(classificationOf(el)) : '';
|
|
2768
|
+
lines.push(` ${el.i}: [${el.type || el.tag}] "${el.text}"${checked}${visibility}${personal}${skipNote}`);
|
|
2496
2769
|
}
|
|
2497
2770
|
lines.push('');
|
|
2498
2771
|
}
|
|
@@ -2504,7 +2777,7 @@ async function analyzeScreenshot(baseUrl, apiKey, model, screenshotBase64, eleme
|
|
|
2504
2777
|
const body = {
|
|
2505
2778
|
model,
|
|
2506
2779
|
messages: [
|
|
2507
|
-
{ role: 'system', content: SYSTEM_PROMPT },
|
|
2780
|
+
{ role: 'system', content: SYSTEM_PROMPT + PERSONAL_SAFETY_RULES },
|
|
2508
2781
|
{
|
|
2509
2782
|
role: 'user',
|
|
2510
2783
|
content: [
|
|
@@ -2667,7 +2940,7 @@ async function analyzeScreenshotWithPrompt(baseUrl, apiKey, model, screenshotBas
|
|
|
2667
2940
|
const body = {
|
|
2668
2941
|
model,
|
|
2669
2942
|
messages: [
|
|
2670
|
-
{ role: 'system', content: customSystemPrompt },
|
|
2943
|
+
{ role: 'system', content: customSystemPrompt + PERSONAL_SAFETY_RULES },
|
|
2671
2944
|
{
|
|
2672
2945
|
role: 'user',
|
|
2673
2946
|
content: [
|
|
@@ -2945,19 +3218,24 @@ function getAdjacentKey(char) {
|
|
|
2945
3218
|
* Type text with human-like patterns: variable speed, occasional typos,
|
|
2946
3219
|
* backspaces, natural pauses. Uses OS-level keyboard (no CDP).
|
|
2947
3220
|
*/
|
|
2948
|
-
async function typeHumanLike(text, baseDelayMs = 120) {
|
|
3221
|
+
async function typeHumanLike(text, baseDelayMs = 120, guard) {
|
|
3222
|
+
const send = async (ch, keyName) => {
|
|
3223
|
+
if (guard)
|
|
3224
|
+
await guard.markOwn(ch, keyName);
|
|
3225
|
+
typeCharOS(ch);
|
|
3226
|
+
};
|
|
2949
3227
|
for (let i = 0; i < text.length; i++) {
|
|
2950
3228
|
const char = text[i];
|
|
2951
3229
|
// 1. Occasional typo (2% chance) — then backspace to correct
|
|
2952
3230
|
if (char !== '\n' && char !== '\t' && Math.random() < 0.02) {
|
|
2953
3231
|
const wrong = getAdjacentKey(char);
|
|
2954
|
-
|
|
3232
|
+
await send(wrong);
|
|
2955
3233
|
await sleep(randomInt(40, 120));
|
|
2956
|
-
|
|
3234
|
+
await send('\b', 'Backspace'); // backspace
|
|
2957
3235
|
await sleep(randomInt(25, 70));
|
|
2958
3236
|
}
|
|
2959
3237
|
// 2. Type the correct character
|
|
2960
|
-
|
|
3238
|
+
await send(char, char === '\n' ? 'Enter' : char === '\t' ? 'Tab' : undefined);
|
|
2961
3239
|
// 3. Variable delay based on character type (human rhythm)
|
|
2962
3240
|
let delay = baseDelayMs + randomInt(-Math.floor(baseDelayMs * 0.3), Math.floor(baseDelayMs * 0.4));
|
|
2963
3241
|
if (char === ' ')
|
|
@@ -2979,7 +3257,16 @@ async function typeHumanLike(text, baseDelayMs = 120) {
|
|
|
2979
3257
|
await sleep(randomInt(200, 600));
|
|
2980
3258
|
}
|
|
2981
3259
|
await sleep(Math.max(delay, 30));
|
|
3260
|
+
if (guard) {
|
|
3261
|
+
await guard.tick(text.slice(0, i + 1));
|
|
3262
|
+
if (guard.stopped) {
|
|
3263
|
+
console.log(`[browse] Typing stopped after ${i + 1}/${text.length} chars`);
|
|
3264
|
+
break;
|
|
3265
|
+
}
|
|
3266
|
+
}
|
|
2982
3267
|
}
|
|
3268
|
+
if (guard)
|
|
3269
|
+
await guard.dispose();
|
|
2983
3270
|
}
|
|
2984
3271
|
/**
|
|
2985
3272
|
* Wait for the screen to change by comparing screenshots.
|
|
@@ -3042,7 +3329,7 @@ async function callStealthLLM(baseUrl, apiKey, model, screenshotPath, timeoutMs,
|
|
|
3042
3329
|
body: JSON.stringify({
|
|
3043
3330
|
model,
|
|
3044
3331
|
messages: [
|
|
3045
|
-
{ role: 'system', content: systemPrompt },
|
|
3332
|
+
{ role: 'system', content: systemPrompt + PERSONAL_SAFETY_RULES },
|
|
3046
3333
|
{
|
|
3047
3334
|
role: 'user',
|
|
3048
3335
|
content: [
|
|
@@ -3139,9 +3426,41 @@ async function runStealthLoop(baseUrl, apiKey, model, maxActions, intervalMs, ti
|
|
|
3139
3426
|
}
|
|
3140
3427
|
case 'type': {
|
|
3141
3428
|
if (result.text && result.text.length > 0) {
|
|
3429
|
+
const guard = new human_input_guard_1.TypingGuard({
|
|
3430
|
+
enabled: fieldSafety.stopOnUserInput,
|
|
3431
|
+
osLevel: true,
|
|
3432
|
+
label: result.text.slice(0, 40),
|
|
3433
|
+
});
|
|
3434
|
+
// Stealth has no DOM: use the focused control's own name as the
|
|
3435
|
+
// label for both the personal classifier and the human-skip list.
|
|
3436
|
+
const control = await guard.currentOsControl().catch(() => null);
|
|
3437
|
+
if (control) {
|
|
3438
|
+
if (control.signature && skippedOsFields.has(control.signature)) {
|
|
3439
|
+
console.log(`[stealth] Skipping field - ${skippedOsFields.get(control.signature)}`);
|
|
3440
|
+
actionCount++;
|
|
3441
|
+
break;
|
|
3442
|
+
}
|
|
3443
|
+
if (control.name && fieldSafety.allowPersonal === false) {
|
|
3444
|
+
const cls = (0, personal_fields_1.classifyField)({ label: control.name, ariaLabel: control.name });
|
|
3445
|
+
if ((0, personal_fields_1.isBlocked)(cls, { allowPersonal: fieldSafety.allowPersonal })) {
|
|
3446
|
+
skippedOsFields.set(control.signature, `left to you (${cls.category}: "${control.name.slice(0, 60)}")`);
|
|
3447
|
+
console.log(`[stealth] Not answering a personal question - left to you (${cls.category}: "${control.name.slice(0, 60)}")`);
|
|
3448
|
+
actionCount++;
|
|
3449
|
+
break;
|
|
3450
|
+
}
|
|
3451
|
+
}
|
|
3452
|
+
}
|
|
3142
3453
|
console.log(`[stealth] Typing ${result.text.length} chars...`);
|
|
3143
3454
|
await sleep(randomInt(300, 700)); // natural pause before typing
|
|
3144
|
-
await
|
|
3455
|
+
await guard.arm({ isCodeEditor: false });
|
|
3456
|
+
await typeHumanLike(result.text, charDelayMs, guard);
|
|
3457
|
+
if (guard.stopped) {
|
|
3458
|
+
const after = await guard.currentOsControl().catch(() => null);
|
|
3459
|
+
const sig = after?.signature || control?.signature;
|
|
3460
|
+
if (sig)
|
|
3461
|
+
skippedOsFields.set(sig, `you answered this one yourself (${guard.describeStop()})`);
|
|
3462
|
+
console.log(`[stealth] Stopped typing - you answered this one yourself (${guard.describeStop()})`);
|
|
3463
|
+
}
|
|
3145
3464
|
actionCount++;
|
|
3146
3465
|
}
|
|
3147
3466
|
else {
|