teamshare-bridge 0.19.17 → 0.19.18
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cli/commands/browse.js +1748 -176
- package/dist/cli/commands/browse.js.map +1 -1
- package/dist/cli/index.js +8 -1
- package/dist/cli/index.js.map +1 -1
- package/package.json +1 -1
|
@@ -1,4 +1,37 @@
|
|
|
1
1
|
"use strict";
|
|
2
|
+
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
|
|
3
|
+
if (k2 === undefined) k2 = k;
|
|
4
|
+
var desc = Object.getOwnPropertyDescriptor(m, k);
|
|
5
|
+
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
|
|
6
|
+
desc = { enumerable: true, get: function() { return m[k]; } };
|
|
7
|
+
}
|
|
8
|
+
Object.defineProperty(o, k2, desc);
|
|
9
|
+
}) : (function(o, m, k, k2) {
|
|
10
|
+
if (k2 === undefined) k2 = k;
|
|
11
|
+
o[k2] = m[k];
|
|
12
|
+
}));
|
|
13
|
+
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
|
|
14
|
+
Object.defineProperty(o, "default", { enumerable: true, value: v });
|
|
15
|
+
}) : function(o, v) {
|
|
16
|
+
o["default"] = v;
|
|
17
|
+
});
|
|
18
|
+
var __importStar = (this && this.__importStar) || (function () {
|
|
19
|
+
var ownKeys = function(o) {
|
|
20
|
+
ownKeys = Object.getOwnPropertyNames || function (o) {
|
|
21
|
+
var ar = [];
|
|
22
|
+
for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
|
|
23
|
+
return ar;
|
|
24
|
+
};
|
|
25
|
+
return ownKeys(o);
|
|
26
|
+
};
|
|
27
|
+
return function (mod) {
|
|
28
|
+
if (mod && mod.__esModule) return mod;
|
|
29
|
+
var result = {};
|
|
30
|
+
if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
|
|
31
|
+
__setModuleDefault(result, mod);
|
|
32
|
+
return result;
|
|
33
|
+
};
|
|
34
|
+
})();
|
|
2
35
|
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
3
36
|
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
4
37
|
};
|
|
@@ -12,11 +45,70 @@ exports.cmdBrowse = cmdBrowse;
|
|
|
12
45
|
* answers questions/forms on screen.
|
|
13
46
|
*
|
|
14
47
|
* Usage:
|
|
15
|
-
* teamshare-agent browse --auto-answer [--port 9222] [--model mimo-v2.5] [--max-actions 50]
|
|
48
|
+
* teamshare-agent browse --auto-answer [--port 9222] [--model mimo-v2.5] [--max-actions 50] [--continuous]
|
|
16
49
|
*/
|
|
17
50
|
const ws_1 = __importDefault(require("ws"));
|
|
18
51
|
const models_1 = require("../../lib/llm/models");
|
|
19
52
|
const config_1 = require("../../lib/config");
|
|
53
|
+
const fs = __importStar(require("fs"));
|
|
54
|
+
const os = __importStar(require("os"));
|
|
55
|
+
const path = __importStar(require("path"));
|
|
56
|
+
const child_process_1 = require("child_process");
|
|
57
|
+
/** Create a fingerprint from the current element list. */
|
|
58
|
+
function createFingerprint(elements) {
|
|
59
|
+
const groups = new Map();
|
|
60
|
+
for (const el of elements) {
|
|
61
|
+
if (!groups.has(el.group))
|
|
62
|
+
groups.set(el.group, []);
|
|
63
|
+
groups.get(el.group).push(el);
|
|
64
|
+
}
|
|
65
|
+
const fp = [];
|
|
66
|
+
for (const [gid, els] of groups) {
|
|
67
|
+
fp.push({
|
|
68
|
+
groupId: gid,
|
|
69
|
+
questionText: els[0]?.questionText || '',
|
|
70
|
+
options: els.map((e) => e.text || ''),
|
|
71
|
+
optionTypes: els.map((e) => e.type || e.tag),
|
|
72
|
+
checkedStates: els.map((e) => e.checked),
|
|
73
|
+
elementCount: els.length,
|
|
74
|
+
});
|
|
75
|
+
}
|
|
76
|
+
return fp;
|
|
77
|
+
}
|
|
78
|
+
/** Weighted comparison of two fingerprints. Returns change score. */
|
|
79
|
+
function calculateChangeScore(prev, curr) {
|
|
80
|
+
let score = 0;
|
|
81
|
+
const prevMap = new Map(prev.map((fp) => [fp.groupId, fp]));
|
|
82
|
+
const currMap = new Map(curr.map((fp) => [fp.groupId, fp]));
|
|
83
|
+
// New questions appeared (weight: 10)
|
|
84
|
+
for (const [gid] of currMap) {
|
|
85
|
+
if (!prevMap.has(gid))
|
|
86
|
+
score += 10;
|
|
87
|
+
}
|
|
88
|
+
// Questions removed (weight: 10)
|
|
89
|
+
for (const [gid] of prevMap) {
|
|
90
|
+
if (!currMap.has(gid))
|
|
91
|
+
score += 10;
|
|
92
|
+
}
|
|
93
|
+
// Compare matching questions
|
|
94
|
+
for (const [gid, currFp] of currMap) {
|
|
95
|
+
const prevFp = prevMap.get(gid);
|
|
96
|
+
if (!prevFp)
|
|
97
|
+
continue;
|
|
98
|
+
// Question text changed (weight: 8)
|
|
99
|
+
if (prevFp.questionText !== currFp.questionText)
|
|
100
|
+
score += 8;
|
|
101
|
+
// Options changed (weight: 5)
|
|
102
|
+
if (prevFp.options.join('|') !== currFp.options.join('|'))
|
|
103
|
+
score += 5;
|
|
104
|
+
// Checked states changed (weight: 2 — user answering, not screen change)
|
|
105
|
+
for (let i = 0; i < Math.min(prevFp.checkedStates.length, currFp.checkedStates.length); i++) {
|
|
106
|
+
if (prevFp.checkedStates[i] !== currFp.checkedStates[i])
|
|
107
|
+
score += 2;
|
|
108
|
+
}
|
|
109
|
+
}
|
|
110
|
+
return score;
|
|
111
|
+
}
|
|
20
112
|
const SYSTEM_PROMPT = `You are a browser automation agent. A screenshot and a grouped list of interactive elements are attached.
|
|
21
113
|
|
|
22
114
|
Your task: identify unanswered questions and answer them by clicking the correct elements.
|
|
@@ -25,7 +117,16 @@ RULES (STRICT):
|
|
|
25
117
|
1. NEVER click Next, Submit, Back, Save, Continue, or any navigation. ONLY answer questions on the current screen.
|
|
26
118
|
2. NEVER ask for help. If uncertain, pick the BEST GUESS. Always decide — never wait.
|
|
27
119
|
3. Handle MULTIPLE unanswered questions in a SINGLE response.
|
|
28
|
-
4.
|
|
120
|
+
4. NEVER return done unless you have VERIFIED every single question is answered. See VERIFICATION CHECKLIST below.
|
|
121
|
+
5. NEVER answer personal questions — see PERSONAL QUESTIONS section below.
|
|
122
|
+
|
|
123
|
+
VERIFICATION CHECKLIST (MANDATORY before returning done):
|
|
124
|
+
Before returning done, you MUST check EVERY question group:
|
|
125
|
+
- Radio questions: exactly ONE option must be selected (checked). If none selected → you MUST pick an answer.
|
|
126
|
+
- Checkbox questions: ALL correct options must be checked. If any correct option is missing → you MUST check it.
|
|
127
|
+
- Text input questions: the input must have a non-empty value. If empty → you MUST type an answer.
|
|
128
|
+
If ANY question fails this check, you MUST answer it — NEVER return done with unanswered questions.
|
|
129
|
+
If you return done while questions are unanswered, the task will FAIL. You will be retried.
|
|
29
130
|
|
|
30
131
|
QUESTION TYPES AND STATUS:
|
|
31
132
|
- radio (single answer): Only ONE option can be correct.
|
|
@@ -57,15 +158,86 @@ COMBINED EXAMPLE:
|
|
|
57
158
|
{"action":"type","elementIndex":9,"text":"42","reason":"Q3: fill answer"}]
|
|
58
159
|
|
|
59
160
|
CODE TYPE EXAMPLE (typing code into an editor — note escaped newlines and quotes):
|
|
60
|
-
[{"action":"type","elementIndex":5,"text":"function twoSum(nums, target) {\\n const map = new Map();\\n for (let i = 0; i < nums.length; i++) {\\n const complement = target - nums[i];\\n if (map.has(complement)) return [map.get(complement), i];\\n map.set(nums[i], i);\\n }\\n return [];\\n}","reason":"Type the solution function"}]
|
|
161
|
+
[{"action":"type","elementIndex":5,"text":"function twoSum(nums: number[], target: number): number[] {\\n const map = new Map<number, number>();\\n for (let i = 0; i < nums.length; i++) {\\n const complement = target - nums[i];\\n if (map.has(complement)) {\\n return [map.get(complement), i];\\n }\\n map.set(nums[i], i);\\n }\\n return [];\\n}","reason":"Type the solution function"}]
|
|
162
|
+
|
|
163
|
+
WRITE_CODE EXAMPLE (large code blocks — preferred for code > 100 chars):
|
|
164
|
+
[{"action":"write_code","elementIndex":5,"code":"function twoSum(nums: number[], target: number): number[] {\\n const map = new Map<number, number>();\\n for (let i = 0; i < nums.length; i++) {\\n const complement = target - nums[i];\\n if (map.has(complement)) {\\n return [map.get(complement), i];\\n }\\n map.set(nums[i], i);\\n }\\n return [];\\n}","reason":"Type the solution function"}]
|
|
165
|
+
|
|
166
|
+
WHEN TO USE write_code vs type:
|
|
167
|
+
- "write_code" for code blocks, functions, classes, multi-line snippets (> 100 chars)
|
|
168
|
+
- "type" for short answers: numbers, single words, short strings (< 100 chars)
|
|
169
|
+
- "write_code" preserves exact indentation and newlines — no formatting changes
|
|
170
|
+
|
|
171
|
+
CODE FORMATTING RULES (CRITICAL):
|
|
172
|
+
- Use 2-space indentation. Each nesting level adds exactly 2 more spaces.
|
|
173
|
+
- Function body = 2 spaces, loop body = 4 spaces, if-body = 6 spaces.
|
|
174
|
+
- NEVER use tabs or 4/8-space indentation.
|
|
175
|
+
- Keep code compact: single-line statements when appropriate.
|
|
176
|
+
|
|
177
|
+
CODE OPTIMIZATION RULES (CRITICAL — for coding problems):
|
|
178
|
+
1. ALWAYS write the most time-efficient solution. Correctness alone is NOT enough.
|
|
179
|
+
2. Prefer O(n) or O(n log n) over O(n²). Never write O(n²) if an O(n) approach exists.
|
|
180
|
+
3. Use HashMap/Set for O(1) lookups instead of nested loops for searching/matching.
|
|
181
|
+
4. Consider space-time tradeoffs — a HashMap is worth the O(n) space for O(n) time.
|
|
182
|
+
5. Avoid unnecessary re-computation — cache results in variables.
|
|
183
|
+
6. Use sliding window, two pointers, or binary search where applicable.
|
|
184
|
+
7. Pre-compute prefix sums, frequency maps, or lookup tables when helpful.
|
|
185
|
+
8. Choose the right data structure:
|
|
186
|
+
- Deque for sliding window problems
|
|
187
|
+
- Heap/priority queue for top-K problems
|
|
188
|
+
- Trie for prefix/string matching problems
|
|
189
|
+
- Union-Find for connected components
|
|
190
|
+
- BFS for shortest path, DFS for exploration
|
|
191
|
+
9. NEVER write brute force if an optimized approach exists.
|
|
192
|
+
10. If the problem has constraints (e.g., n <= 10^5), ensure your solution runs within limits.
|
|
193
|
+
|
|
194
|
+
OPTIMIZATION EXAMPLE:
|
|
195
|
+
Problem: "Given an array of integers, return indices of two numbers that add up to target."
|
|
196
|
+
|
|
197
|
+
NAIVE (WRONG for efficiency — O(n²) time):
|
|
198
|
+
for (let i = 0; i < nums.length; i++) {
|
|
199
|
+
for (let j = i + 1; j < nums.length; j++) {
|
|
200
|
+
if (nums[i] + nums[j] === target) return [i, j];
|
|
201
|
+
}
|
|
202
|
+
}
|
|
203
|
+
|
|
204
|
+
OPTIMIZED (CORRECT — O(n) time, O(n) space):
|
|
205
|
+
const map = new Map();
|
|
206
|
+
for (let i = 0; i < nums.length; i++) {
|
|
207
|
+
const complement = target - nums[i];
|
|
208
|
+
if (map.has(complement)) return [map.get(complement), i];
|
|
209
|
+
map.set(nums[i], i);
|
|
210
|
+
}
|
|
211
|
+
return [];
|
|
61
212
|
|
|
62
213
|
Available actions:
|
|
63
214
|
- click: {action:"click", elementIndex:number} — click to check/uncheck/select
|
|
64
|
-
- type: {action:"type", elementIndex:number, text:string} — type
|
|
215
|
+
- type: {action:"type", elementIndex:number, text:string} — type short input (< 100 chars)
|
|
216
|
+
- write_code: {action:"write_code", elementIndex:number, code:string} — write code block (> 100 chars), preserves exact indentation
|
|
65
217
|
- scroll: {action:"scroll", delta:number} — scroll up (-) or down (+)
|
|
66
218
|
- press: {action:"press", key:string} — press Enter, Tab, etc.
|
|
67
219
|
- done: {action:"done", summary:string} — all visible questions answered correctly
|
|
68
220
|
|
|
221
|
+
ERROR CORRECTION (CRITICAL):
|
|
222
|
+
- Check <page_context> for "Errors on screen" after each action
|
|
223
|
+
- If errors are visible (wrong answer, validation error, "Incorrect", "Wrong Answer", "Error"), FIX THEM immediately
|
|
224
|
+
- For text inputs with errors: clear and retype the correct answer
|
|
225
|
+
- For code editors with errors: retype the correct code using write_code
|
|
226
|
+
- For radio/checkbox with errors: click the correct option
|
|
227
|
+
- NEVER return "done" if errors are visible on screen
|
|
228
|
+
- Keep running until ALL questions are answered correctly with NO errors
|
|
229
|
+
|
|
230
|
+
CORRECTION MODE:
|
|
231
|
+
- The <page_context> block shows the current editor content — compare it to the expected answer
|
|
232
|
+
- If editor content is WRONG or INCOMPLETE, type the CORRECT code to replace it
|
|
233
|
+
- If you previously typed wrong code, use write_code to replace with correct code
|
|
234
|
+
- You may be asked the same question multiple times — each time, check if the current answer is correct before acting
|
|
235
|
+
|
|
236
|
+
PERSONAL QUESTIONS — SKIP THEM:
|
|
237
|
+
- NEVER type answers for questions asking: name, email, phone, address, SSN, password, credit card, date of birth, or any personal information
|
|
238
|
+
- If a question asks for personal info, do NOT click or type — skip it entirely
|
|
239
|
+
- If ALL remaining questions are personal, return done
|
|
240
|
+
|
|
69
241
|
RESPONSE FORMAT RULES (CRITICAL — follow exactly):
|
|
70
242
|
- Return ONLY a valid JSON array. No explanations, no markdown fences, no extra text.
|
|
71
243
|
- The response MUST be parseable by JSON.parse(). Test mentally: is your output valid JSON?
|
|
@@ -121,11 +293,18 @@ async function cmdBrowse(flags) {
|
|
|
121
293
|
return;
|
|
122
294
|
}
|
|
123
295
|
const port = parseInt(flags.get('port') ?? '9222', 10);
|
|
124
|
-
const
|
|
296
|
+
const continuous = flags.has('continuous');
|
|
297
|
+
const maxActions = continuous ? Number.MAX_SAFE_INTEGER : parseInt(flags.get('max-actions') ?? '50', 10);
|
|
125
298
|
const intervalMs = parseInt(flags.get('interval') ?? '2000', 10);
|
|
126
299
|
const modelSpec = flags.get('model') ?? 'mimo:mimo-v2.5';
|
|
127
|
-
const timeoutSec = Math.min(Math.max(parseInt(flags.get('timeout') ?? '
|
|
300
|
+
const timeoutSec = Math.min(Math.max(parseInt(flags.get('timeout') ?? '300', 10), 30), 600);
|
|
128
301
|
const timeoutMs = timeoutSec * 1000;
|
|
302
|
+
const quality = Math.min(Math.max(parseInt(flags.get('quality') ?? '92', 10), 50), 100);
|
|
303
|
+
const maxConsecutiveFailures = parseInt(flags.get('max-consecutive-failures') ?? '0', 10); // 0 = unlimited
|
|
304
|
+
const charDelayMs = Math.min(Math.max(parseInt(flags.get('char-delay') ?? '150', 10), 5), 200);
|
|
305
|
+
const optimize = flags.has('optimize');
|
|
306
|
+
const stealth = flags.has('stealth');
|
|
307
|
+
const hybrid = flags.has('hybrid');
|
|
129
308
|
// Resolve LLM endpoint
|
|
130
309
|
const resolved = (0, models_1.resolveModel)(modelSpec, process.env.LLM_BASE_URL);
|
|
131
310
|
const apiKey = (0, models_1.resolveApiKeyForProvider)(modelSpec);
|
|
@@ -138,9 +317,21 @@ async function cmdBrowse(flags) {
|
|
|
138
317
|
const model = resolved.model;
|
|
139
318
|
console.log(`[browse] Model: ${modelSpec} -> ${model} @ ${baseUrl}`);
|
|
140
319
|
console.log(`[browse] Chrome port: ${port}`);
|
|
141
|
-
console.log(`[browse]
|
|
320
|
+
console.log(`[browse] Mode: ${stealth ? 'STEALTH (OS-level, no CDP)' : hybrid ? 'HYBRID (CDP reads + OS-level writes)' : continuous ? 'continuous (runs until Ctrl+C)' : `capped at ${maxActions} actions`}`);
|
|
142
321
|
console.log(`[browse] LLM timeout: ${timeoutSec}s`);
|
|
322
|
+
if (!stealth) {
|
|
323
|
+
console.log(`[browse] Screenshot quality: ${quality}`);
|
|
324
|
+
}
|
|
325
|
+
console.log(`[browse] Char delay: ${charDelayMs}ms`);
|
|
143
326
|
console.log(`[browse] Screenshot interval: ${intervalMs}ms`);
|
|
327
|
+
if (!stealth) {
|
|
328
|
+
console.log(`[browse] Max consecutive failures: ${maxConsecutiveFailures === 0 ? 'unlimited' : maxConsecutiveFailures}`);
|
|
329
|
+
}
|
|
330
|
+
// ── Stealth mode: bypass CDP entirely ──
|
|
331
|
+
if (stealth) {
|
|
332
|
+
await runStealthLoop(baseUrl, apiKey, model, maxActions, intervalMs, timeoutMs, charDelayMs, continuous);
|
|
333
|
+
return;
|
|
334
|
+
}
|
|
144
335
|
// Connect directly to the page's WebSocket (not browser-level)
|
|
145
336
|
let page;
|
|
146
337
|
try {
|
|
@@ -163,79 +354,344 @@ async function cmdBrowse(flags) {
|
|
|
163
354
|
}
|
|
164
355
|
// Agent loop
|
|
165
356
|
let actionCount = 0;
|
|
357
|
+
let llmCallCount = 0;
|
|
166
358
|
let stopped = false;
|
|
167
359
|
let consecutiveFailures = 0;
|
|
168
|
-
|
|
169
|
-
|
|
360
|
+
let lastScreenshot = null;
|
|
361
|
+
let lastFingerprint = null;
|
|
362
|
+
let codeConfirmedOptimal = false;
|
|
363
|
+
const maxRetries = 5;
|
|
170
364
|
process.on('SIGINT', () => {
|
|
171
365
|
console.log('\n[browse] Stopped by user');
|
|
172
366
|
stopped = true;
|
|
173
367
|
});
|
|
174
368
|
process.on('SIGTERM', () => { stopped = true; });
|
|
175
|
-
console.log(
|
|
369
|
+
console.log(`[browse] Agent started. ${continuous ? 'Running until Ctrl+C.' : `Max ${maxActions} actions.`} Press Ctrl+C to stop.\n`);
|
|
176
370
|
while (!stopped && actionCount < maxActions) {
|
|
177
371
|
try {
|
|
178
372
|
// Take screenshot and extract elements in parallel
|
|
179
|
-
const [screenshot, elements] = await Promise.all([
|
|
180
|
-
captureScreenshot(page),
|
|
373
|
+
const [screenshot, elements, pageContext] = await Promise.all([
|
|
374
|
+
captureScreenshot(page, quality),
|
|
181
375
|
extractElements(page),
|
|
376
|
+
extractPageContext(page),
|
|
182
377
|
]);
|
|
183
378
|
if (!screenshot) {
|
|
184
379
|
console.error('[browse] Failed to capture screenshot, retrying...');
|
|
185
380
|
await sleep(intervalMs);
|
|
186
381
|
continue;
|
|
187
382
|
}
|
|
383
|
+
// Detect screen change via question fingerprint comparison
|
|
384
|
+
const currentFingerprint = createFingerprint(elements);
|
|
385
|
+
if (lastFingerprint) {
|
|
386
|
+
const changeScore = calculateChangeScore(lastFingerprint, currentFingerprint);
|
|
387
|
+
if (changeScore >= 5) {
|
|
388
|
+
console.log(`[browse] Screen changed (score: ${changeScore}) — resetting state`);
|
|
389
|
+
codeConfirmedOptimal = false;
|
|
390
|
+
consecutiveFailures = 0;
|
|
391
|
+
}
|
|
392
|
+
}
|
|
393
|
+
lastFingerprint = currentFingerprint;
|
|
394
|
+
lastScreenshot = screenshot;
|
|
188
395
|
console.log(`[browse] Found ${elements.length} interactive elements`);
|
|
189
396
|
// Send to vision LLM with retry logic
|
|
190
|
-
|
|
397
|
+
const actionLabel = continuous ? `${actionCount + 1}/∞` : `${actionCount + 1}/${maxActions}`;
|
|
398
|
+
console.log(`[browse] [${actionLabel}] Analyzing screenshot...`);
|
|
191
399
|
let result = null;
|
|
192
400
|
for (let attempt = 1; attempt <= maxRetries; attempt++) {
|
|
193
|
-
result = await analyzeScreenshot(baseUrl, apiKey, model, screenshot, elements, timeoutMs, attempt, maxRetries);
|
|
401
|
+
result = await analyzeScreenshot(baseUrl, apiKey, model, screenshot, elements, pageContext, timeoutMs, attempt, maxRetries);
|
|
402
|
+
llmCallCount++;
|
|
403
|
+
if (llmCallCount > 50 && llmCallCount % 25 === 0) {
|
|
404
|
+
console.log(`[browse] Warning: ${llmCallCount} LLM calls made — may hit rate limits`);
|
|
405
|
+
}
|
|
194
406
|
if (result.actions.length > 0)
|
|
195
407
|
break; // success
|
|
408
|
+
// Timeout → exponential backoff, keep retrying (never count as failure)
|
|
409
|
+
if (result.error === 'timeout') {
|
|
410
|
+
const backoff = Math.min(30 * Math.pow(2, attempt - 1), 300); // 30, 60, 120, 240, 300
|
|
411
|
+
console.log(`[browse] LLM timed out — retrying in ${backoff}s (attempt ${attempt}/${maxRetries})...`);
|
|
412
|
+
await sleep(backoff * 1000);
|
|
413
|
+
continue;
|
|
414
|
+
}
|
|
415
|
+
// Other errors → short backoff
|
|
196
416
|
if (attempt < maxRetries) {
|
|
197
417
|
const backoff = attempt === 1 ? 3 : 5;
|
|
198
|
-
console.log(`[browse] ${result.error === '
|
|
418
|
+
console.log(`[browse] ${result.error === 'empty' ? 'Empty response' : 'Parse failed'} (attempt ${attempt}/${maxRetries}), retrying in ${backoff}s...`);
|
|
199
419
|
await sleep(backoff * 1000);
|
|
200
420
|
}
|
|
201
421
|
}
|
|
202
422
|
if (!result || result.actions.length === 0) {
|
|
203
423
|
consecutiveFailures++;
|
|
204
424
|
console.log(`[browse] No valid response after ${maxRetries} attempts (${consecutiveFailures} consecutive failures)`);
|
|
205
|
-
if (consecutiveFailures >= maxConsecutiveFailures) {
|
|
425
|
+
if (maxConsecutiveFailures > 0 && consecutiveFailures >= maxConsecutiveFailures) {
|
|
206
426
|
console.error(`[browse] Too many consecutive failures (${consecutiveFailures}), stopping agent`);
|
|
207
427
|
break;
|
|
208
428
|
}
|
|
209
|
-
if (consecutiveFailures >= 5) {
|
|
210
|
-
console.log(`[browse] Warning: ${consecutiveFailures} consecutive failures —
|
|
429
|
+
if (consecutiveFailures >= 5 && consecutiveFailures % 5 === 0) {
|
|
430
|
+
console.log(`[browse] Warning: ${consecutiveFailures} consecutive failures — still running`);
|
|
211
431
|
}
|
|
212
432
|
await sleep(intervalMs);
|
|
213
433
|
continue;
|
|
214
434
|
}
|
|
215
435
|
// Success — reset consecutive failure counter
|
|
216
436
|
consecutiveFailures = 0;
|
|
217
|
-
//
|
|
218
|
-
|
|
437
|
+
// Pre-execution check: detect if screen changed during LLM call
|
|
438
|
+
const preActions = result.actions.filter((a) => a.action !== 'done');
|
|
439
|
+
if (preActions.length > 0) {
|
|
440
|
+
const [freshScreenshot, freshElements] = await Promise.all([
|
|
441
|
+
captureScreenshot(page, quality),
|
|
442
|
+
extractElements(page),
|
|
443
|
+
]);
|
|
444
|
+
if (freshScreenshot) {
|
|
445
|
+
const freshFingerprint = createFingerprint(freshElements);
|
|
446
|
+
if (lastFingerprint) {
|
|
447
|
+
const changeScore = calculateChangeScore(lastFingerprint, freshFingerprint);
|
|
448
|
+
if (changeScore >= 5) {
|
|
449
|
+
console.log(`[browse] Screen changed during LLM call (score: ${changeScore}) — skipping actions, re-analyzing`);
|
|
450
|
+
lastFingerprint = freshFingerprint;
|
|
451
|
+
lastScreenshot = freshScreenshot;
|
|
452
|
+
await sleep(intervalMs);
|
|
453
|
+
continue;
|
|
454
|
+
}
|
|
455
|
+
}
|
|
456
|
+
}
|
|
457
|
+
}
|
|
458
|
+
// Batch multiple write_code actions — use last one (complete solution)
|
|
459
|
+
const writeCodeActions = result.actions.filter((a) => a.action === 'write_code');
|
|
460
|
+
if (writeCodeActions.length > 1) {
|
|
461
|
+
result.actions = result.actions.filter((a) => a.action !== 'write_code');
|
|
462
|
+
result.actions.push(writeCodeActions[writeCodeActions.length - 1]);
|
|
463
|
+
console.log(`[browse] ${writeCodeActions.length} write_code actions — using last one (complete solution)`);
|
|
464
|
+
}
|
|
465
|
+
// Process all actions
|
|
466
|
+
let executedAny = false;
|
|
467
|
+
let wroteCode = false;
|
|
219
468
|
for (const act of result.actions) {
|
|
220
469
|
if (act.action === 'done') {
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
470
|
+
// Validate: are there actually unanswered questions?
|
|
471
|
+
if (hasUnansweredQuestions(elements)) {
|
|
472
|
+
console.log(`[browse] LLM says done but unanswered questions remain — retrying...`);
|
|
473
|
+
// Force retry with stronger prompt
|
|
474
|
+
result = await analyzeScreenshot(baseUrl, apiKey, model, screenshot, elements, pageContext, timeoutMs, 1, 1);
|
|
475
|
+
if (result.actions.length > 0 && result.actions[0].action !== 'done') {
|
|
476
|
+
// LLM returned actual actions — re-process from the start of this loop
|
|
477
|
+
executedAny = false;
|
|
478
|
+
wroteCode = false;
|
|
479
|
+
// Fall through to process new actions below
|
|
480
|
+
}
|
|
481
|
+
else {
|
|
482
|
+
// LLM still says done — trust it this time and continue
|
|
483
|
+
console.log(`[browse] LLM confirms done after verification`);
|
|
484
|
+
if (continuous && !executedAny) {
|
|
485
|
+
const changed = await waitForScreenChange(page, lastScreenshot ?? screenshot, intervalMs, quality, () => stopped);
|
|
486
|
+
if (!changed)
|
|
487
|
+
break;
|
|
488
|
+
executedAny = false;
|
|
489
|
+
}
|
|
490
|
+
}
|
|
491
|
+
continue;
|
|
492
|
+
}
|
|
493
|
+
// All questions genuinely answered
|
|
494
|
+
if (continuous && !executedAny) {
|
|
495
|
+
console.log(`[browse] LLM reports done: ${act.summary || 'All questions answered'}`);
|
|
496
|
+
// Log correctness indicators from page context
|
|
497
|
+
const correctIndicators = pageContext.match(/correct|accepted|✓|passed|success|well done/gi);
|
|
498
|
+
const wrongIndicators = pageContext.match(/incorrect|wrong|✗|failed|error|try again/gi);
|
|
499
|
+
if (wrongIndicators) {
|
|
500
|
+
console.log(`[browse] Errors detected on screen: ${wrongIndicators.join(', ')} — should be fixed`);
|
|
501
|
+
}
|
|
502
|
+
else if (correctIndicators) {
|
|
503
|
+
console.log(`[browse] Answer verified correct: ${correctIndicators.join(', ')}`);
|
|
504
|
+
}
|
|
505
|
+
const changed = await waitForScreenChange(page, lastScreenshot ?? screenshot, intervalMs, quality, () => stopped);
|
|
506
|
+
if (!changed)
|
|
507
|
+
break;
|
|
508
|
+
executedAny = false;
|
|
509
|
+
}
|
|
510
|
+
else {
|
|
511
|
+
console.log(`[browse] LLM reports done: ${act.summary || 'All questions answered'} — continuing to verify...`);
|
|
512
|
+
}
|
|
513
|
+
continue;
|
|
514
|
+
}
|
|
515
|
+
// Skip if code was already confirmed optimal (prevents re-typing after optimization pass)
|
|
516
|
+
if ((act.action === 'write_code' || act.action === 'type') && codeConfirmedOptimal) {
|
|
517
|
+
console.log(`[browse] Skipping ${act.action} — code already confirmed optimal`);
|
|
518
|
+
continue;
|
|
519
|
+
}
|
|
520
|
+
// Ask LLM to decide if action is redundant
|
|
521
|
+
const redundancy = await isActionRedundant(baseUrl, apiKey, model, page, elements, act, timeoutMs);
|
|
522
|
+
if (redundancy.decision === 'skip') {
|
|
523
|
+
continue;
|
|
224
524
|
}
|
|
225
|
-
|
|
226
|
-
|
|
525
|
+
if (redundancy.decision === 'partial' && (act.action === 'write_code' || act.action === 'type')) {
|
|
526
|
+
// Partial update — only retype the different part
|
|
527
|
+
const targetCode = act.action === 'write_code' ? act.code : act.text;
|
|
528
|
+
if (targetCode) {
|
|
529
|
+
console.log(`[browse] Partial update — only retying changed portion...`);
|
|
530
|
+
const currentContent = await readEditorContent(page);
|
|
531
|
+
const normCurrent = normalizeCode(currentContent);
|
|
532
|
+
const normTarget = normalizeCode(targetCode);
|
|
533
|
+
const commonLen = findCommonPrefix(normCurrent, normTarget);
|
|
534
|
+
// Find position in original code that corresponds to end of common prefix
|
|
535
|
+
const suffix = targetCode.slice(Math.min(commonLen, targetCode.length));
|
|
536
|
+
if (suffix.length > 0) {
|
|
537
|
+
// Position cursor at end, type only the suffix
|
|
538
|
+
await cdpSend(page, 'Input.dispatchKeyEvent', {
|
|
539
|
+
type: 'keyDown', key: 'End', code: 'End', modifiers: 2,
|
|
540
|
+
windowsVirtualKeyCode: 35, nativeVirtualKeyCode: 35,
|
|
541
|
+
});
|
|
542
|
+
await cdpSend(page, 'Input.dispatchKeyEvent', {
|
|
543
|
+
type: 'keyUp', key: 'End', code: 'End', modifiers: 2,
|
|
544
|
+
windowsVirtualKeyCode: 35, nativeVirtualKeyCode: 35,
|
|
545
|
+
});
|
|
546
|
+
await sleep(100);
|
|
547
|
+
// Select from cursor to end and overwrite
|
|
548
|
+
await cdpSend(page, 'Input.dispatchKeyEvent', {
|
|
549
|
+
type: 'keyDown', key: 'Shift', code: 'ShiftRight', modifiers: 2,
|
|
550
|
+
windowsVirtualKeyCode: 16, nativeVirtualKeyCode: 16,
|
|
551
|
+
});
|
|
552
|
+
// Move to end of document
|
|
553
|
+
for (let i = 0; i < 50; i++) {
|
|
554
|
+
await cdpSend(page, 'Input.dispatchKeyEvent', {
|
|
555
|
+
type: 'keyDown', key: 'ArrowRight', code: 'ArrowRight',
|
|
556
|
+
windowsVirtualKeyCode: 39, nativeVirtualKeyCode: 39,
|
|
557
|
+
});
|
|
558
|
+
}
|
|
559
|
+
await cdpSend(page, 'Input.dispatchKeyEvent', {
|
|
560
|
+
type: 'keyUp', key: 'Shift', code: 'ShiftRight', modifiers: 2,
|
|
561
|
+
windowsVirtualKeyCode: 16, nativeVirtualKeyCode: 16,
|
|
562
|
+
});
|
|
563
|
+
// Delete selection
|
|
564
|
+
await cdpSend(page, 'Input.dispatchKeyEvent', {
|
|
565
|
+
type: 'keyDown', key: 'Backspace', code: 'Backspace',
|
|
566
|
+
windowsVirtualKeyCode: 8, nativeVirtualKeyCode: 8,
|
|
567
|
+
});
|
|
568
|
+
await cdpSend(page, 'Input.dispatchKeyEvent', {
|
|
569
|
+
type: 'keyUp', key: 'Backspace', code: 'Backspace',
|
|
570
|
+
windowsVirtualKeyCode: 8, nativeVirtualKeyCode: 8,
|
|
571
|
+
});
|
|
572
|
+
await sleep(100);
|
|
573
|
+
// Type the suffix
|
|
574
|
+
for (let i = 0; i < suffix.length; i++) {
|
|
575
|
+
const char = suffix[i];
|
|
576
|
+
if (char === '\n') {
|
|
577
|
+
await cdpSend(page, 'Input.dispatchKeyEvent', {
|
|
578
|
+
type: 'keyDown', key: 'Enter', code: 'Enter', text: '\r',
|
|
579
|
+
windowsVirtualKeyCode: 13, nativeVirtualKeyCode: 13,
|
|
580
|
+
});
|
|
581
|
+
await cdpSend(page, 'Input.dispatchKeyEvent', {
|
|
582
|
+
type: 'keyUp', key: 'Enter', code: 'Enter',
|
|
583
|
+
windowsVirtualKeyCode: 13, nativeVirtualKeyCode: 13,
|
|
584
|
+
});
|
|
585
|
+
}
|
|
586
|
+
else {
|
|
587
|
+
const k = charToKey(char);
|
|
588
|
+
const mods = k.shift ? 2 : undefined;
|
|
589
|
+
const downParams = { type: 'keyDown', key: k.key, code: k.code, text: char };
|
|
590
|
+
const upParams = { type: 'keyUp', key: k.key, code: k.code };
|
|
591
|
+
if (mods !== undefined) {
|
|
592
|
+
downParams.modifiers = mods;
|
|
593
|
+
upParams.modifiers = mods;
|
|
594
|
+
}
|
|
595
|
+
await cdpSend(page, 'Input.dispatchKeyEvent', downParams);
|
|
596
|
+
await cdpSend(page, 'Input.dispatchKeyEvent', upParams);
|
|
597
|
+
}
|
|
598
|
+
await sleep(variableDelay(charDelayMs, char, i, suffix.length));
|
|
599
|
+
}
|
|
600
|
+
console.log(`[browse] Partial update complete (${suffix.length} chars typed)`);
|
|
601
|
+
}
|
|
602
|
+
actionCount++;
|
|
603
|
+
executedAny = true;
|
|
604
|
+
wroteCode = true;
|
|
605
|
+
}
|
|
606
|
+
continue;
|
|
607
|
+
}
|
|
608
|
+
// Full update — execute the action normally
|
|
609
|
+
if (act.action === 'type') {
|
|
610
|
+
console.log(`[browse] Action: type — ${act.text?.slice(0, 120)}${(act.text?.length ?? 0) > 120 ? '...' : ''}`);
|
|
611
|
+
}
|
|
612
|
+
else if (act.action === 'write_code') {
|
|
613
|
+
console.log(`[browse] Action: write_code — ${act.code?.slice(0, 120)}${(act.code?.length ?? 0) > 120 ? '...' : ''}`);
|
|
614
|
+
}
|
|
615
|
+
else {
|
|
616
|
+
console.log(`[browse] Action: ${act.action} - ${act.reasoning || ''}`);
|
|
617
|
+
}
|
|
618
|
+
await (hybrid ? executeActionHybrid : executeAction)(page, act, elements, charDelayMs);
|
|
227
619
|
actionCount++;
|
|
228
|
-
|
|
229
|
-
if (act.action === '
|
|
620
|
+
executedAny = true;
|
|
621
|
+
if (act.action === 'write_code' || act.action === 'type') {
|
|
622
|
+
wroteCode = true;
|
|
623
|
+
}
|
|
624
|
+
// Natural delay between actions
|
|
625
|
+
if (act.action === 'click' || act.action === 'type' || act.action === 'write_code') {
|
|
230
626
|
await sleep(2000);
|
|
231
627
|
}
|
|
232
628
|
}
|
|
233
|
-
//
|
|
234
|
-
if (
|
|
235
|
-
console.log('[browse]
|
|
236
|
-
await
|
|
629
|
+
// Optimization pass: if --optimize and code was written, ask LLM to optimize it
|
|
630
|
+
if (optimize && wroteCode) {
|
|
631
|
+
console.log('[browse] Running optimization pass...');
|
|
632
|
+
const [newScreenshot, newElements, newPageContext] = await Promise.all([
|
|
633
|
+
captureScreenshot(page, quality),
|
|
634
|
+
extractElements(page),
|
|
635
|
+
extractPageContext(page),
|
|
636
|
+
]);
|
|
637
|
+
if (newScreenshot) {
|
|
638
|
+
const optResult = await analyzeScreenshotWithPrompt(baseUrl, apiKey, model, newScreenshot, newElements, newPageContext, timeoutMs, 1, 1, `You are a code optimization expert. The code in the editor is CORRECT but may not be EFFICIENT.
|
|
639
|
+
|
|
640
|
+
Review the code and check:
|
|
641
|
+
1. Is it the most time-efficient approach? (O(n log n) or better?)
|
|
642
|
+
2. Are there better data structures to use?
|
|
643
|
+
3. Can nested loops be replaced with HashMap/Set lookups?
|
|
644
|
+
4. Can the solution be optimized with sliding window, two pointers, or binary search?
|
|
645
|
+
|
|
646
|
+
If the code is ALREADY optimal (top-tier efficiency), return: [{"action":"done","summary":"Code is already optimal"}]
|
|
647
|
+
|
|
648
|
+
If you can optimize it, return the FULL optimized code using write_code. Focus ONLY on performance — do not change correctness.
|
|
649
|
+
Do NOT simplify variable names or change the function signature. Only optimize the algorithm.`);
|
|
650
|
+
if (optResult.actions.length > 0 && optResult.actions[0].action !== 'done') {
|
|
651
|
+
const optAct = optResult.actions[0];
|
|
652
|
+
if ((optAct.action === 'write_code' || optAct.action === 'type') && optAct.code) {
|
|
653
|
+
console.log(`[browse] Optimization found — rewriting code...`);
|
|
654
|
+
await (hybrid ? executeActionHybrid : executeAction)(page, { action: 'click', elementIndex: optAct.elementIndex ?? result.actions[0].elementIndex }, newElements, charDelayMs);
|
|
655
|
+
await sleep(500);
|
|
656
|
+
await (hybrid ? executeActionHybrid : executeAction)(page, optAct, newElements, charDelayMs);
|
|
657
|
+
actionCount++;
|
|
658
|
+
await sleep(2000);
|
|
659
|
+
}
|
|
660
|
+
else {
|
|
661
|
+
console.log('[browse] No actionable optimization returned — keeping original');
|
|
662
|
+
}
|
|
663
|
+
}
|
|
664
|
+
else {
|
|
665
|
+
console.log('[browse] Code is already optimal — no changes needed');
|
|
666
|
+
codeConfirmedOptimal = true;
|
|
667
|
+
}
|
|
668
|
+
}
|
|
669
|
+
}
|
|
670
|
+
// Cooldown after code was written — give user time to see result and navigate
|
|
671
|
+
if (wroteCode) {
|
|
672
|
+
console.log('[browse] Code written — cooling down 30s before next check...');
|
|
673
|
+
await sleep(30000);
|
|
674
|
+
// After cooldown, check for test results on screen
|
|
675
|
+
const [, postElements, postContext] = await Promise.all([
|
|
676
|
+
captureScreenshot(page, quality),
|
|
677
|
+
extractElements(page),
|
|
678
|
+
extractPageContext(page),
|
|
679
|
+
]);
|
|
680
|
+
if (postContext) {
|
|
681
|
+
const failedIndicators = postContext.match(/wrong answer|time limit exceeded|runtime error|compile error|undefined is not|cannot read property|segmentation fault|n\s*\/\s*m/gi);
|
|
682
|
+
const passedIndicators = postContext.match(/accepted|all test cases passed|✓ passed|runtime:\s*\d+\s*ms/gi);
|
|
683
|
+
if (failedIndicators) {
|
|
684
|
+
console.log(`[browse] Test FAILED: ${failedIndicators.join(', ')} — will retry this question`);
|
|
685
|
+
codeConfirmedOptimal = false; // allow re-typing with fix
|
|
686
|
+
// Don't wait for screen change — loop again to fix the error
|
|
687
|
+
}
|
|
688
|
+
else if (passedIndicators) {
|
|
689
|
+
console.log(`[browse] Test PASSED: ${passedIndicators.join(', ')}`);
|
|
690
|
+
}
|
|
691
|
+
}
|
|
237
692
|
}
|
|
238
693
|
else {
|
|
694
|
+
// Wait between LLM calls
|
|
239
695
|
await sleep(2000);
|
|
240
696
|
}
|
|
241
697
|
}
|
|
@@ -244,7 +700,7 @@ async function cmdBrowse(flags) {
|
|
|
244
700
|
await sleep(intervalMs);
|
|
245
701
|
}
|
|
246
702
|
}
|
|
247
|
-
if (actionCount >= maxActions) {
|
|
703
|
+
if (actionCount >= maxActions && !continuous) {
|
|
248
704
|
console.log(`[browse] Reached max actions (${maxActions})`);
|
|
249
705
|
}
|
|
250
706
|
console.log(`[browse] Agent finished. ${actionCount} actions performed.`);
|
|
@@ -330,9 +786,9 @@ function cdpSend(page, method, params) {
|
|
|
330
786
|
page.ws.send(JSON.stringify({ id, method, params }));
|
|
331
787
|
});
|
|
332
788
|
}
|
|
333
|
-
async function captureScreenshot(page) {
|
|
789
|
+
async function captureScreenshot(page, quality) {
|
|
334
790
|
try {
|
|
335
|
-
const result = await cdpSend(page, 'Page.captureScreenshot', { format: 'jpeg', quality
|
|
791
|
+
const result = await cdpSend(page, 'Page.captureScreenshot', { format: 'jpeg', quality });
|
|
336
792
|
return result?.data ?? null;
|
|
337
793
|
}
|
|
338
794
|
catch (err) {
|
|
@@ -383,6 +839,7 @@ async function extractElements(page) {
|
|
|
383
839
|
checked: el.checked || el.getAttribute('aria-checked') === 'true' || false,
|
|
384
840
|
x: Math.round(rect.x + rect.width / 2),
|
|
385
841
|
y: Math.round(rect.y + rect.height / 2),
|
|
842
|
+
inViewport: rect.bottom >= 0 && rect.top <= window.innerHeight && rect.right >= 0 && rect.left <= window.innerWidth,
|
|
386
843
|
groupId: null,
|
|
387
844
|
});
|
|
388
845
|
});
|
|
@@ -490,6 +947,7 @@ async function extractElements(page) {
|
|
|
490
947
|
x: item.x,
|
|
491
948
|
y: item.y,
|
|
492
949
|
group: item.groupId,
|
|
950
|
+
inViewport: item.inViewport,
|
|
493
951
|
});
|
|
494
952
|
});
|
|
495
953
|
|
|
@@ -507,17 +965,29 @@ async function extractElements(page) {
|
|
|
507
965
|
return [];
|
|
508
966
|
}
|
|
509
967
|
}
|
|
510
|
-
async function executeAction(page, act, elements) {
|
|
968
|
+
async function executeAction(page, act, elements, charDelayMs) {
|
|
511
969
|
try {
|
|
512
970
|
switch (act.action) {
|
|
513
971
|
case 'click': {
|
|
514
972
|
if (act.elementIndex === undefined)
|
|
515
973
|
return false;
|
|
516
|
-
|
|
974
|
+
let el = elements.find((e) => e.i === act.elementIndex);
|
|
517
975
|
if (!el) {
|
|
518
976
|
console.log(`[browse] Element index ${act.elementIndex} not found, will retry`);
|
|
519
977
|
return false;
|
|
520
978
|
}
|
|
979
|
+
// Scroll off-screen element into view before clicking
|
|
980
|
+
if (!el.inViewport) {
|
|
981
|
+
console.log(`[browse] Element ${el.i} off-screen — scrolling to "${el.text?.slice(0, 30)}"...`);
|
|
982
|
+
await scrollElementIntoView(page, el);
|
|
983
|
+
// Re-extract elements to get updated coordinates
|
|
984
|
+
const updatedElements = await extractElements(page);
|
|
985
|
+
const updatedEl = updatedElements.find((e) => e.text === el.text && e.type === el.type && e.group === el.group);
|
|
986
|
+
if (updatedEl) {
|
|
987
|
+
el = updatedEl;
|
|
988
|
+
console.log(`[browse] Scrolled — element now at index ${el.i}, position (${el.x},${el.y})`);
|
|
989
|
+
}
|
|
990
|
+
}
|
|
521
991
|
console.log(`[browse] Clicking [${el.i}] ${el.tag} "${el.text.slice(0, 60)}" at (${el.x},${el.y})`);
|
|
522
992
|
await cdpSend(page, 'Input.dispatchMouseEvent', {
|
|
523
993
|
type: 'mousePressed', x: el.x, y: el.y, button: 'left', clickCount: 1,
|
|
@@ -537,140 +1007,52 @@ async function executeAction(page, act, elements) {
|
|
|
537
1007
|
}
|
|
538
1008
|
// 1. Detect editor type
|
|
539
1009
|
const editor = await detectEditor(page);
|
|
540
|
-
const isMonaco = editor?.type === 'monaco';
|
|
541
1010
|
const isCodeEditor = editor !== null;
|
|
542
1011
|
if (editor)
|
|
543
1012
|
console.log(`[browse] Detected editor: ${editor.type}`);
|
|
544
1013
|
// 2. Unescape LLM text: convert literal \n, \t, \\", etc. to real characters
|
|
545
|
-
const
|
|
1014
|
+
const rawText = act.text
|
|
546
1015
|
.replace(/\\n/g, '\n')
|
|
547
1016
|
.replace(/\\t/g, '\t')
|
|
548
1017
|
.replace(/\\"/g, '"')
|
|
549
1018
|
.replace(/\\\\/g, '\\');
|
|
1019
|
+
// 3. Normalize indentation for code editors (LLM often adds extra indent)
|
|
1020
|
+
const text = isCodeEditor ? normalizeCodeIndent(rawText) : rawText;
|
|
550
1021
|
console.log(`[browse] Typing ${text.length} characters...`);
|
|
551
|
-
if (
|
|
552
|
-
|
|
553
|
-
|
|
554
|
-
|
|
555
|
-
await
|
|
556
|
-
// 2. Clear: Ctrl+A → wait → Backspace (works when textarea has focus)
|
|
557
|
-
await cdpSend(page, 'Input.dispatchKeyEvent', {
|
|
558
|
-
type: 'keyDown', key: 'a', code: 'KeyA', modifiers: 2,
|
|
559
|
-
windowsVirtualKeyCode: 65, nativeVirtualKeyCode: 65,
|
|
560
|
-
});
|
|
561
|
-
await cdpSend(page, 'Input.dispatchKeyEvent', {
|
|
562
|
-
type: 'keyUp', key: 'a', code: 'KeyA', modifiers: 2,
|
|
563
|
-
windowsVirtualKeyCode: 65, nativeVirtualKeyCode: 65,
|
|
564
|
-
});
|
|
565
|
-
await sleep(200);
|
|
566
|
-
await cdpSend(page, 'Input.dispatchKeyEvent', {
|
|
567
|
-
type: 'keyDown', key: 'Backspace', code: 'Backspace',
|
|
568
|
-
windowsVirtualKeyCode: 8, nativeVirtualKeyCode: 8,
|
|
569
|
-
});
|
|
570
|
-
await cdpSend(page, 'Input.dispatchKeyEvent', {
|
|
571
|
-
type: 'keyUp', key: 'Backspace', code: 'Backspace',
|
|
572
|
-
windowsVirtualKeyCode: 8, nativeVirtualKeyCode: 8,
|
|
573
|
-
});
|
|
574
|
-
await sleep(200);
|
|
575
|
-
// 3. Re-focus textarea (Monaco may have stolen focus during Ctrl+A/Backspace)
|
|
576
|
-
await focusSelector(page, editor.selector);
|
|
577
|
-
await sleep(100);
|
|
578
|
-
// 4. Insert full text via Input.insertText — works for ALL chars including \n
|
|
579
|
-
await cdpSend(page, 'Input.insertText', { text });
|
|
1022
|
+
if (isCodeEditor) {
|
|
1023
|
+
await insertIntoCodeEditor(page, editor.selector, text, charDelayMs);
|
|
1024
|
+
}
|
|
1025
|
+
else {
|
|
1026
|
+
await insertIntoGenericInput(page, el, text, charDelayMs);
|
|
580
1027
|
}
|
|
581
|
-
|
|
582
|
-
|
|
583
|
-
|
|
584
|
-
|
|
585
|
-
|
|
586
|
-
|
|
587
|
-
|
|
588
|
-
|
|
589
|
-
|
|
590
|
-
|
|
591
|
-
|
|
592
|
-
|
|
593
|
-
|
|
594
|
-
|
|
595
|
-
|
|
596
|
-
|
|
597
|
-
|
|
598
|
-
|
|
599
|
-
|
|
600
|
-
|
|
601
|
-
|
|
602
|
-
|
|
603
|
-
|
|
604
|
-
|
|
1028
|
+
return true;
|
|
1029
|
+
}
|
|
1030
|
+
case 'write_code': {
|
|
1031
|
+
if (act.elementIndex === undefined || !act.code)
|
|
1032
|
+
break;
|
|
1033
|
+
const el = elements.find((e) => e.i === act.elementIndex);
|
|
1034
|
+
if (!el) {
|
|
1035
|
+
console.log(`[browse] Element index ${act.elementIndex} not found, will retry`);
|
|
1036
|
+
return false;
|
|
1037
|
+
}
|
|
1038
|
+
// 1. Detect editor type
|
|
1039
|
+
const editor = await detectEditor(page);
|
|
1040
|
+
const isCodeEditor = editor !== null;
|
|
1041
|
+
if (editor)
|
|
1042
|
+
console.log(`[browse] Detected editor: ${editor.type}`);
|
|
1043
|
+
// 2. Unescape LLM code: convert literal \n, \t, \\", etc. to real characters
|
|
1044
|
+
const text = act.code
|
|
1045
|
+
.replace(/\\n/g, '\n')
|
|
1046
|
+
.replace(/\\t/g, '\t')
|
|
1047
|
+
.replace(/\\"/g, '"')
|
|
1048
|
+
.replace(/\\\\/g, '\\');
|
|
1049
|
+
console.log(`[browse] Writing code (${text.length} chars)...`);
|
|
1050
|
+
// 3. write_code inserts EXACTLY as-is — no normalizeCodeIndent
|
|
1051
|
+
if (isCodeEditor) {
|
|
1052
|
+
await insertIntoCodeEditor(page, editor.selector, text, charDelayMs);
|
|
605
1053
|
}
|
|
606
1054
|
else {
|
|
607
|
-
|
|
608
|
-
await cdpSend(page, 'Input.dispatchMouseEvent', {
|
|
609
|
-
type: 'mousePressed', x: el.x, y: el.y, button: 'left', clickCount: 1,
|
|
610
|
-
});
|
|
611
|
-
await cdpSend(page, 'Input.dispatchMouseEvent', {
|
|
612
|
-
type: 'mouseReleased', x: el.x, y: el.y, button: 'left', clickCount: 1,
|
|
613
|
-
});
|
|
614
|
-
await sleep(200);
|
|
615
|
-
// Ctrl+A + Backspace to clear
|
|
616
|
-
await cdpSend(page, 'Input.dispatchKeyEvent', {
|
|
617
|
-
type: 'keyDown', key: 'a', code: 'KeyA', modifiers: 2,
|
|
618
|
-
windowsVirtualKeyCode: 65, nativeVirtualKeyCode: 65,
|
|
619
|
-
});
|
|
620
|
-
await cdpSend(page, 'Input.dispatchKeyEvent', {
|
|
621
|
-
type: 'keyUp', key: 'a', code: 'KeyA', modifiers: 2,
|
|
622
|
-
windowsVirtualKeyCode: 65, nativeVirtualKeyCode: 65,
|
|
623
|
-
});
|
|
624
|
-
await cdpSend(page, 'Input.dispatchKeyEvent', {
|
|
625
|
-
type: 'keyDown', key: 'Backspace', code: 'Backspace',
|
|
626
|
-
windowsVirtualKeyCode: 8, nativeVirtualKeyCode: 8,
|
|
627
|
-
});
|
|
628
|
-
await cdpSend(page, 'Input.dispatchKeyEvent', {
|
|
629
|
-
type: 'keyUp', key: 'Backspace', code: 'Backspace',
|
|
630
|
-
windowsVirtualKeyCode: 8, nativeVirtualKeyCode: 8,
|
|
631
|
-
});
|
|
632
|
-
await sleep(200);
|
|
633
|
-
// Char-by-char typing with natural timing
|
|
634
|
-
for (let i = 0; i < text.length; i++) {
|
|
635
|
-
const char = text[i];
|
|
636
|
-
if (char === '\n') {
|
|
637
|
-
await cdpSend(page, 'Input.dispatchKeyEvent', {
|
|
638
|
-
type: 'keyDown', key: 'Enter', code: 'Enter',
|
|
639
|
-
windowsVirtualKeyCode: 13, nativeVirtualKeyCode: 13,
|
|
640
|
-
});
|
|
641
|
-
await cdpSend(page, 'Input.dispatchKeyEvent', {
|
|
642
|
-
type: 'keyUp', key: 'Enter', code: 'Enter',
|
|
643
|
-
windowsVirtualKeyCode: 13, nativeVirtualKeyCode: 13,
|
|
644
|
-
});
|
|
645
|
-
}
|
|
646
|
-
else if (char === '\t') {
|
|
647
|
-
await cdpSend(page, 'Input.dispatchKeyEvent', {
|
|
648
|
-
type: 'keyDown', key: 'Tab', code: 'Tab',
|
|
649
|
-
windowsVirtualKeyCode: 9, nativeVirtualKeyCode: 9,
|
|
650
|
-
});
|
|
651
|
-
await cdpSend(page, 'Input.dispatchKeyEvent', {
|
|
652
|
-
type: 'keyUp', key: 'Tab', code: 'Tab',
|
|
653
|
-
windowsVirtualKeyCode: 9, nativeVirtualKeyCode: 9,
|
|
654
|
-
});
|
|
655
|
-
}
|
|
656
|
-
else {
|
|
657
|
-
const k = charToKey(char);
|
|
658
|
-
const mods = k.shift ? 2 : undefined;
|
|
659
|
-
const downParams = {
|
|
660
|
-
type: 'keyDown', key: k.key, code: k.code, text: char,
|
|
661
|
-
};
|
|
662
|
-
const upParams = {
|
|
663
|
-
type: 'keyUp', key: k.key, code: k.code,
|
|
664
|
-
};
|
|
665
|
-
if (mods !== undefined) {
|
|
666
|
-
downParams.modifiers = mods;
|
|
667
|
-
upParams.modifiers = mods;
|
|
668
|
-
}
|
|
669
|
-
await cdpSend(page, 'Input.dispatchKeyEvent', downParams);
|
|
670
|
-
await cdpSend(page, 'Input.dispatchKeyEvent', upParams);
|
|
671
|
-
}
|
|
672
|
-
await sleep(charDelay(char, i, text.length));
|
|
673
|
-
}
|
|
1055
|
+
await insertIntoGenericInput(page, el, text, charDelayMs);
|
|
674
1056
|
}
|
|
675
1057
|
return true;
|
|
676
1058
|
}
|
|
@@ -704,6 +1086,115 @@ async function executeAction(page, act, elements) {
|
|
|
704
1086
|
return false;
|
|
705
1087
|
}
|
|
706
1088
|
}
|
|
1089
|
+
/**
|
|
1090
|
+
* Hybrid action executor: OS-level keyboard/mouse for writing (invisible to detection).
|
|
1091
|
+
* Same logic as executeAction but uses clickOS/typeHumanLike/scrollOS/pressKeyOS.
|
|
1092
|
+
*/
|
|
1093
|
+
async function executeActionHybrid(page, act, elements, charDelayMs) {
|
|
1094
|
+
try {
|
|
1095
|
+
switch (act.action) {
|
|
1096
|
+
case 'click': {
|
|
1097
|
+
if (act.elementIndex === undefined)
|
|
1098
|
+
return false;
|
|
1099
|
+
let el = elements.find((e) => e.i === act.elementIndex);
|
|
1100
|
+
if (!el) {
|
|
1101
|
+
console.log(`[browse] Element index ${act.elementIndex} not found, will retry`);
|
|
1102
|
+
return false;
|
|
1103
|
+
}
|
|
1104
|
+
// Scroll off-screen element into view before clicking
|
|
1105
|
+
if (!el.inViewport) {
|
|
1106
|
+
console.log(`[browse] Element ${el.i} off-screen — scrolling to "${el.text?.slice(0, 30)}"...`);
|
|
1107
|
+
await scrollElementIntoView(page, el);
|
|
1108
|
+
// Re-extract elements to get updated coordinates
|
|
1109
|
+
const updatedElements = await extractElements(page);
|
|
1110
|
+
const updatedEl = updatedElements.find((e) => e.text === el.text && e.type === el.type && e.group === el.group);
|
|
1111
|
+
if (updatedEl) {
|
|
1112
|
+
el = updatedEl;
|
|
1113
|
+
console.log(`[browse] Scrolled — element now at index ${el.i}, position (${el.x},${el.y})`);
|
|
1114
|
+
}
|
|
1115
|
+
}
|
|
1116
|
+
console.log(`[browse] Clicking [${el.i}] ${el.tag} "${el.text.slice(0, 60)}" at (${el.x},${el.y})`);
|
|
1117
|
+
clickOS(el.x, el.y);
|
|
1118
|
+
return true;
|
|
1119
|
+
}
|
|
1120
|
+
case 'type': {
|
|
1121
|
+
if (act.elementIndex === undefined || !act.text)
|
|
1122
|
+
break;
|
|
1123
|
+
const el = elements.find((e) => e.i === act.elementIndex);
|
|
1124
|
+
if (!el) {
|
|
1125
|
+
console.log(`[browse] Element index ${act.elementIndex} not found, will retry`);
|
|
1126
|
+
return false;
|
|
1127
|
+
}
|
|
1128
|
+
const editor = await detectEditor(page);
|
|
1129
|
+
const isCodeEditor = editor !== null;
|
|
1130
|
+
if (editor)
|
|
1131
|
+
console.log(`[browse] Detected editor: ${editor.type}`);
|
|
1132
|
+
const rawText = act.text
|
|
1133
|
+
.replace(/\\n/g, '\n')
|
|
1134
|
+
.replace(/\\t/g, '\t')
|
|
1135
|
+
.replace(/\\"/g, '"')
|
|
1136
|
+
.replace(/\\\\/g, '\\');
|
|
1137
|
+
const text = isCodeEditor ? normalizeCodeIndent(rawText) : rawText;
|
|
1138
|
+
console.log(`[browse] Typing ${text.length} characters (OS-level)...`);
|
|
1139
|
+
await sleep(randomInt(200, 400));
|
|
1140
|
+
clickOS(el.x, el.y);
|
|
1141
|
+
await sleep(randomInt(100, 200));
|
|
1142
|
+
if (isCodeEditor) {
|
|
1143
|
+
pressKeyOS('ctrl+a');
|
|
1144
|
+
await sleep(randomInt(50, 100));
|
|
1145
|
+
}
|
|
1146
|
+
await typeHumanLike(text, charDelayMs);
|
|
1147
|
+
return true;
|
|
1148
|
+
}
|
|
1149
|
+
case 'write_code': {
|
|
1150
|
+
if (act.elementIndex === undefined || !act.code)
|
|
1151
|
+
break;
|
|
1152
|
+
const el = elements.find((e) => e.i === act.elementIndex);
|
|
1153
|
+
if (!el) {
|
|
1154
|
+
console.log(`[browse] Element index ${act.elementIndex} not found, will retry`);
|
|
1155
|
+
return false;
|
|
1156
|
+
}
|
|
1157
|
+
const editor = await detectEditor(page);
|
|
1158
|
+
const isCodeEditor = editor !== null;
|
|
1159
|
+
if (editor)
|
|
1160
|
+
console.log(`[browse] Detected editor: ${editor.type}`);
|
|
1161
|
+
const text = act.code
|
|
1162
|
+
.replace(/\\n/g, '\n')
|
|
1163
|
+
.replace(/\\t/g, '\t')
|
|
1164
|
+
.replace(/\\"/g, '"')
|
|
1165
|
+
.replace(/\\\\/g, '\\');
|
|
1166
|
+
console.log(`[browse] Writing code (${text.length} chars, OS-level)...`);
|
|
1167
|
+
await sleep(randomInt(200, 400));
|
|
1168
|
+
clickOS(el.x, el.y);
|
|
1169
|
+
await sleep(randomInt(100, 200));
|
|
1170
|
+
if (isCodeEditor) {
|
|
1171
|
+
pressKeyOS('ctrl+a');
|
|
1172
|
+
await sleep(randomInt(50, 100));
|
|
1173
|
+
}
|
|
1174
|
+
await typeHumanLike(text, charDelayMs);
|
|
1175
|
+
return true;
|
|
1176
|
+
}
|
|
1177
|
+
case 'scroll': {
|
|
1178
|
+
const delta = act.delta ?? 3;
|
|
1179
|
+
console.log(`[browse] Scrolling ${delta > 0 ? 'down' : 'up'} (${Math.abs(delta)} clicks, OS-level)...`);
|
|
1180
|
+
scrollOS(-delta);
|
|
1181
|
+
break;
|
|
1182
|
+
}
|
|
1183
|
+
case 'press': {
|
|
1184
|
+
if (!act.key)
|
|
1185
|
+
break;
|
|
1186
|
+
console.log(`[browse] Pressing ${act.key} (OS-level)...`);
|
|
1187
|
+
pressKeyOS(act.key.toLowerCase());
|
|
1188
|
+
break;
|
|
1189
|
+
}
|
|
1190
|
+
}
|
|
1191
|
+
return true;
|
|
1192
|
+
}
|
|
1193
|
+
catch (err) {
|
|
1194
|
+
console.error(`[browse] Action error: ${(0, config_1.msg)(err)}`);
|
|
1195
|
+
return false;
|
|
1196
|
+
}
|
|
1197
|
+
}
|
|
707
1198
|
// ── Key mapping for CDP keyboard events ──────────────────────────────────────
|
|
708
1199
|
/** Maps a character to its CDP key/code/shift properties. */
|
|
709
1200
|
function charToKey(c) {
|
|
@@ -759,24 +1250,596 @@ function charToKey(c) {
|
|
|
759
1250
|
// Fallback — use the char itself
|
|
760
1251
|
return { key: c, code: c, shift: false };
|
|
761
1252
|
}
|
|
762
|
-
// ──
|
|
763
|
-
/**
|
|
764
|
-
|
|
765
|
-
|
|
766
|
-
|
|
1253
|
+
// ── Code indentation normalization ─────────────────────────────────────────
|
|
1254
|
+
/**
|
|
1255
|
+
* Strips the common leading whitespace from all non-empty lines.
|
|
1256
|
+
* Fixes LLM output that has consistent extra indentation (e.g. 8 spaces
|
|
1257
|
+
* instead of 4). The relative indentation between lines is preserved.
|
|
1258
|
+
*/
|
|
1259
|
+
function normalizeCodeIndent(text) {
|
|
1260
|
+
const lines = text.split('\n');
|
|
1261
|
+
const nonEmpty = lines.filter((l) => l.trim().length > 0);
|
|
1262
|
+
if (nonEmpty.length <= 1)
|
|
1263
|
+
return text;
|
|
1264
|
+
// Find the minimum leading whitespace across all non-empty lines
|
|
1265
|
+
const minIndent = nonEmpty.reduce((min, line) => {
|
|
1266
|
+
const match = line.match(/^(\s*)/);
|
|
1267
|
+
return Math.min(min, match?.[1]?.length ?? 0);
|
|
1268
|
+
}, Infinity);
|
|
1269
|
+
if (minIndent === 0 || minIndent === Infinity)
|
|
1270
|
+
return text;
|
|
1271
|
+
return lines
|
|
1272
|
+
.map((line) => {
|
|
1273
|
+
if (line.trim().length === 0)
|
|
1274
|
+
return line;
|
|
1275
|
+
return line.slice(minIndent);
|
|
1276
|
+
})
|
|
1277
|
+
.join('\n');
|
|
1278
|
+
}
|
|
1279
|
+
// ── Editor insertion helpers ───────────────────────────────────────────────
|
|
1280
|
+
/** Clear editor content: Ctrl+A → Backspace. */
|
|
1281
|
+
async function clearEditor(page) {
|
|
1282
|
+
await cdpSend(page, 'Input.dispatchKeyEvent', {
|
|
1283
|
+
type: 'keyDown', key: 'a', code: 'KeyA', modifiers: 2,
|
|
1284
|
+
windowsVirtualKeyCode: 65, nativeVirtualKeyCode: 65,
|
|
1285
|
+
});
|
|
1286
|
+
await cdpSend(page, 'Input.dispatchKeyEvent', {
|
|
1287
|
+
type: 'keyUp', key: 'a', code: 'KeyA', modifiers: 2,
|
|
1288
|
+
windowsVirtualKeyCode: 65, nativeVirtualKeyCode: 65,
|
|
1289
|
+
});
|
|
1290
|
+
await sleep(100);
|
|
1291
|
+
await cdpSend(page, 'Input.dispatchKeyEvent', {
|
|
1292
|
+
type: 'keyDown', key: 'Backspace', code: 'Backspace',
|
|
1293
|
+
windowsVirtualKeyCode: 8, nativeVirtualKeyCode: 8,
|
|
1294
|
+
});
|
|
1295
|
+
await cdpSend(page, 'Input.dispatchKeyEvent', {
|
|
1296
|
+
type: 'keyUp', key: 'Backspace', code: 'Backspace',
|
|
1297
|
+
windowsVirtualKeyCode: 8, nativeVirtualKeyCode: 8,
|
|
1298
|
+
});
|
|
1299
|
+
}
|
|
1300
|
+
/** Read current editor content from Monaco/CM5/CM6/Ace. */
|
|
1301
|
+
async function readEditorContent(page) {
|
|
1302
|
+
try {
|
|
1303
|
+
const result = await cdpSend(page, 'Runtime.evaluate', {
|
|
1304
|
+
expression: `
|
|
1305
|
+
(() => {
|
|
1306
|
+
const monacoNs = typeof monaco !== 'undefined' ? monaco : window.monaco;
|
|
1307
|
+
const editors = monacoNs?.editor?.getEditors?.() || [];
|
|
1308
|
+
if (editors.length > 0) return editors[0].getValue();
|
|
1309
|
+
const cm6 = document.querySelector('.cm-content');
|
|
1310
|
+
if (cm6?.cmView?.view) return cm6.cmView.view.state.doc.toString();
|
|
1311
|
+
const cm5 = document.querySelector('.CodeMirror')?.CodeMirror;
|
|
1312
|
+
if (cm5) return cm5.getValue();
|
|
1313
|
+
const ace = document.querySelector('.ace_editor')?.env?.editor;
|
|
1314
|
+
if (ace) return ace.getValue();
|
|
1315
|
+
return '';
|
|
1316
|
+
})()
|
|
1317
|
+
`,
|
|
1318
|
+
returnByValue: true,
|
|
1319
|
+
});
|
|
1320
|
+
return result?.result?.value ?? '';
|
|
1321
|
+
}
|
|
1322
|
+
catch {
|
|
1323
|
+
return '';
|
|
1324
|
+
}
|
|
1325
|
+
}
|
|
1326
|
+
/** Scroll an off-screen element into view before clicking. */
|
|
1327
|
+
async function scrollElementIntoView(page, el) {
|
|
1328
|
+
const textSlice = el.text?.slice(0, 100) || '';
|
|
1329
|
+
await cdpSend(page, 'Runtime.evaluate', {
|
|
1330
|
+
expression: `
|
|
1331
|
+
(() => {
|
|
1332
|
+
const sels = [
|
|
1333
|
+
'input[type="radio"]', 'input[type="checkbox"]',
|
|
1334
|
+
'input[type="text"]', 'input[type="email"]', 'input[type="password"]',
|
|
1335
|
+
'input:not([type])', 'textarea', 'select', 'button', 'a',
|
|
1336
|
+
'[role="radio"]', '[role="checkbox"]', '[role="button"]',
|
|
1337
|
+
'[role="option"]', '[role="tab"]', '[role="menuitem"]', 'label',
|
|
1338
|
+
];
|
|
1339
|
+
const all = [...document.querySelectorAll(sels.join(','))];
|
|
1340
|
+
const target = all.find(el => {
|
|
1341
|
+
const t = (el.innerText || el.textContent || '').trim().replace(/\\s+/g, ' ').slice(0, 100);
|
|
1342
|
+
return t === ${JSON.stringify(textSlice)};
|
|
1343
|
+
});
|
|
1344
|
+
if (target) {
|
|
1345
|
+
target.scrollIntoView({ behavior: 'instant', block: 'center' });
|
|
1346
|
+
return true;
|
|
1347
|
+
}
|
|
1348
|
+
return false;
|
|
1349
|
+
})()
|
|
1350
|
+
`,
|
|
1351
|
+
returnByValue: true,
|
|
1352
|
+
});
|
|
1353
|
+
await sleep(300);
|
|
1354
|
+
}
|
|
1355
|
+
/** Check if there are genuinely unanswered questions — catches premature 'done' from LLM. */
|
|
1356
|
+
function hasUnansweredQuestions(elements) {
|
|
1357
|
+
const groups = new Map();
|
|
1358
|
+
for (const el of elements) {
|
|
1359
|
+
if (!groups.has(el.group))
|
|
1360
|
+
groups.set(el.group, []);
|
|
1361
|
+
groups.get(el.group).push(el);
|
|
1362
|
+
}
|
|
1363
|
+
for (const [, els] of groups) {
|
|
1364
|
+
const types = els.map((e) => e.type);
|
|
1365
|
+
const isRadio = types.some((t) => t === 'radio');
|
|
1366
|
+
const isCheckbox = types.some((t) => t === 'checkbox');
|
|
1367
|
+
const hasInput = els.some((e) => e.tag === 'input' || e.tag === 'textarea');
|
|
1368
|
+
const checkedCount = els.filter((e) => e.checked).length;
|
|
1369
|
+
if (isRadio && checkedCount === 0)
|
|
1370
|
+
return true;
|
|
1371
|
+
if (isCheckbox && checkedCount < els.length)
|
|
1372
|
+
return true;
|
|
1373
|
+
if (hasInput && !els.some((e) => e.text && e.tag === 'input'))
|
|
1374
|
+
return true;
|
|
1375
|
+
}
|
|
1376
|
+
return false;
|
|
1377
|
+
}
|
|
1378
|
+
/** Normalize code for fuzzy comparison — strip whitespace, normalize quotes. */
|
|
1379
|
+
function normalizeCode(code) {
|
|
1380
|
+
return code
|
|
1381
|
+
.replace(/\s+/g, '')
|
|
1382
|
+
.replace(/['`]/g, '"')
|
|
1383
|
+
.replace(/;$/gm, '')
|
|
1384
|
+
.trim();
|
|
1385
|
+
}
|
|
1386
|
+
/** Find how many leading characters two strings share. */
|
|
1387
|
+
function findCommonPrefix(a, b) {
|
|
1388
|
+
let i = 0;
|
|
1389
|
+
while (i < a.length && i < b.length && a[i] === b[i])
|
|
1390
|
+
i++;
|
|
1391
|
+
return i;
|
|
1392
|
+
}
|
|
1393
|
+
/** Ask the LLM to decide if an action is redundant — is the current code already correct? */
|
|
1394
|
+
async function isActionRedundant(baseUrl, apiKey, model, page, elements, action, timeoutMs) {
|
|
1395
|
+
// Click on already-selected radio → definitely redundant
|
|
1396
|
+
if (action.action === 'click' && action.elementIndex !== undefined) {
|
|
1397
|
+
const el = elements[action.elementIndex];
|
|
1398
|
+
if (el?.type === 'radio' && el.checked) {
|
|
1399
|
+
console.log(`[browse] Skipping click — radio already selected: "${el.text?.slice(0, 40)}"`);
|
|
1400
|
+
return { decision: 'skip', reason: 'Radio already selected' };
|
|
1401
|
+
}
|
|
1402
|
+
return { decision: 'full', reason: 'Action needed' };
|
|
1403
|
+
}
|
|
1404
|
+
// For write_code/type → read current content and ask LLM
|
|
1405
|
+
const targetCode = action.action === 'write_code' ? action.code : action.text;
|
|
1406
|
+
if (!targetCode || targetCode.length < 20) {
|
|
1407
|
+
return { decision: 'full', reason: 'Short input, always type' };
|
|
1408
|
+
}
|
|
1409
|
+
const currentContent = await readEditorContent(page);
|
|
1410
|
+
if (!currentContent) {
|
|
1411
|
+
return { decision: 'full', reason: 'Editor empty' };
|
|
1412
|
+
}
|
|
1413
|
+
// Quick exact check first
|
|
1414
|
+
if (currentContent === targetCode) {
|
|
1415
|
+
console.log(`[browse] Skipping ${action.action} — code is identical`);
|
|
1416
|
+
return { decision: 'skip', reason: 'Identical' };
|
|
1417
|
+
}
|
|
1418
|
+
// Quick normalized check (fast fallback, no API call needed)
|
|
1419
|
+
const normCurrent = normalizeCode(currentContent);
|
|
1420
|
+
const normTarget = normalizeCode(targetCode);
|
|
1421
|
+
if (normCurrent === normTarget) {
|
|
1422
|
+
console.log(`[browse] Skipping ${action.action} — code is semantically identical (formatting only)`);
|
|
1423
|
+
return { decision: 'skip', reason: 'Semantically identical' };
|
|
1424
|
+
}
|
|
1425
|
+
// Calculate match percentage for prefix check
|
|
1426
|
+
const commonLen = findCommonPrefix(normCurrent, normTarget);
|
|
1427
|
+
const matchPercent = Math.round((commonLen / Math.max(normCurrent.length, normTarget.length)) * 100);
|
|
1428
|
+
// If >90% match, it's likely just a small change → partial update
|
|
1429
|
+
if (matchPercent > 90) {
|
|
1430
|
+
console.log(`[browse] Code is ${matchPercent}% similar — partial update possible`);
|
|
1431
|
+
return { decision: 'partial', reason: `${matchPercent}% match`, matchPercent };
|
|
1432
|
+
}
|
|
1433
|
+
// Ask the LLM to judge
|
|
1434
|
+
console.log(`[browse] Code is ${matchPercent}% similar — asking LLM to judge...`);
|
|
1435
|
+
const prompt = `You are a code comparison expert. Two versions of code are provided.
|
|
1436
|
+
|
|
1437
|
+
CURRENT editor content (truncated to first 500 chars):
|
|
1438
|
+
\`\`\`
|
|
1439
|
+
${currentContent.slice(0, 500)}
|
|
1440
|
+
\`\`\`
|
|
1441
|
+
|
|
1442
|
+
TARGET code to write (truncated to first 500 chars):
|
|
1443
|
+
\`\`\`
|
|
1444
|
+
${targetCode.slice(0, 500)}
|
|
1445
|
+
\`\`\`
|
|
1446
|
+
|
|
1447
|
+
The user wants to write the TARGET code into the editor.
|
|
1448
|
+
|
|
1449
|
+
TASK: Compare these two code versions. Are they semantically the same (same logic, same algorithm)? Ignore whitespace, formatting, variable naming differences. Focus on LOGIC.
|
|
1450
|
+
|
|
1451
|
+
Return EXACTLY one JSON object:
|
|
1452
|
+
- {"decision":"skip","reason":"..."} — if the current code is already correct (same logic)
|
|
1453
|
+
- {"decision":"partial","reason":"..."} — if mostly correct, only small changes needed
|
|
1454
|
+
- {"decision":"full","reason":"..."} — if completely different code is needed
|
|
1455
|
+
|
|
1456
|
+
Return ONLY the JSON. No other text.`;
|
|
1457
|
+
try {
|
|
1458
|
+
const body = {
|
|
1459
|
+
model,
|
|
1460
|
+
messages: [
|
|
1461
|
+
{ role: 'system', content: prompt },
|
|
1462
|
+
{ role: 'user', content: 'Compare the two code versions and return your decision as JSON.' },
|
|
1463
|
+
],
|
|
1464
|
+
temperature: 0.1,
|
|
1465
|
+
max_tokens: 200,
|
|
1466
|
+
stream: false,
|
|
1467
|
+
};
|
|
1468
|
+
const controller = new AbortController();
|
|
1469
|
+
const timeout = setTimeout(() => controller.abort(), Math.min(timeoutMs, 30000));
|
|
1470
|
+
const res = await fetch(`${baseUrl.replace(/\/$/, '')}/chat/completions`, {
|
|
1471
|
+
method: 'POST',
|
|
1472
|
+
signal: controller.signal,
|
|
1473
|
+
headers: {
|
|
1474
|
+
'Content-Type': 'application/json',
|
|
1475
|
+
'Authorization': `Bearer ${apiKey}`,
|
|
1476
|
+
'api-key': apiKey,
|
|
1477
|
+
},
|
|
1478
|
+
body: JSON.stringify(body),
|
|
1479
|
+
});
|
|
1480
|
+
clearTimeout(timeout);
|
|
1481
|
+
if (!res.ok) {
|
|
1482
|
+
console.log(`[browse] Redundancy check API failed (${res.status}) — defaulting to full update`);
|
|
1483
|
+
return { decision: 'full', reason: 'API failed', matchPercent };
|
|
1484
|
+
}
|
|
1485
|
+
const json = (await res.json());
|
|
1486
|
+
let content = json.choices?.[0]?.message?.content ?? '';
|
|
1487
|
+
if (Array.isArray(content)) {
|
|
1488
|
+
content = content.filter((c) => c.type === 'text').map((c) => c.text).join('');
|
|
1489
|
+
}
|
|
1490
|
+
// Parse the JSON response
|
|
1491
|
+
const match = content.match(/\{[^}]+\}/);
|
|
1492
|
+
if (match) {
|
|
1493
|
+
const parsed = JSON.parse(match[0]);
|
|
1494
|
+
console.log(`[browse] LLM decision: ${parsed.decision} — ${parsed.reason}`);
|
|
1495
|
+
return { ...parsed, matchPercent };
|
|
1496
|
+
}
|
|
1497
|
+
}
|
|
1498
|
+
catch (err) {
|
|
1499
|
+
console.log(`[browse] Redundancy check failed: ${(0, config_1.msg)(err)} — defaulting to full update`);
|
|
1500
|
+
}
|
|
1501
|
+
return { decision: 'full', reason: 'Fallback', matchPercent };
|
|
1502
|
+
}
|
|
1503
|
+
/** Wait for the page to change by comparing screenshots periodically. Returns true if changed, false if stopped. */
|
|
1504
|
+
async function waitForScreenChange(page, previousScreenshot, intervalMs, quality, stopped) {
|
|
1505
|
+
console.log('[browse] Waiting for screen change... (Ctrl+C to stop)');
|
|
1506
|
+
while (!stopped()) {
|
|
1507
|
+
await sleep(intervalMs);
|
|
1508
|
+
if (stopped())
|
|
1509
|
+
break;
|
|
1510
|
+
const newScreenshot = await captureScreenshot(page, quality);
|
|
1511
|
+
if (newScreenshot && newScreenshot !== previousScreenshot) {
|
|
1512
|
+
console.log('[browse] Screen changed — resuming...');
|
|
1513
|
+
return true;
|
|
1514
|
+
}
|
|
1515
|
+
}
|
|
1516
|
+
return false;
|
|
1517
|
+
}
|
|
1518
|
+
/** Variable typing delay — random variation for natural feel. Never faster than 70% of base. */
|
|
1519
|
+
function variableDelay(baseMs, char, index, total) {
|
|
1520
|
+
let delay = baseMs + (Math.random() - 0.5) * baseMs * 0.6;
|
|
767
1521
|
if (char === ' ')
|
|
768
|
-
|
|
769
|
-
// Numbers — slightly slower
|
|
1522
|
+
delay += 20 + Math.random() * 60;
|
|
770
1523
|
if (/[0-9]/.test(char))
|
|
771
|
-
|
|
772
|
-
// Shift characters — shift key overhead
|
|
1524
|
+
delay += 10 + Math.random() * 30;
|
|
773
1525
|
if (/^[{}():"<>!@#$%^&*_+|~`?\[\]]$/.test(char))
|
|
774
|
-
|
|
775
|
-
// Occasional "think" pause every 15-25 chars
|
|
1526
|
+
delay += 50 + Math.random() * 120;
|
|
776
1527
|
if (index > 0 && index % (15 + Math.floor(Math.random() * 10)) === 0) {
|
|
777
|
-
|
|
1528
|
+
delay += 200 + Math.random() * 500;
|
|
1529
|
+
}
|
|
1530
|
+
return Math.max(delay, baseMs * 0.7);
|
|
1531
|
+
}
|
|
1532
|
+
/**
|
|
1533
|
+
* Detect and remove extra trailing closing brackets in the editor.
|
|
1534
|
+
* Handles: } } } → removes extras, keeps correct one.
|
|
1535
|
+
*/
|
|
1536
|
+
async function removeTrailingBrackets(page) {
|
|
1537
|
+
const content = await readEditorContent(page);
|
|
1538
|
+
if (!content)
|
|
1539
|
+
return;
|
|
1540
|
+
// Count trailing closing brackets
|
|
1541
|
+
const trailing = content.match(/[\}\)\]]+$/)?.[0] ?? '';
|
|
1542
|
+
if (!trailing || trailing.length === 0)
|
|
1543
|
+
return;
|
|
1544
|
+
// Count opening brackets in the full content
|
|
1545
|
+
let opens = 0;
|
|
1546
|
+
let closes = 0;
|
|
1547
|
+
for (const c of content) {
|
|
1548
|
+
if (c === '{' || c === '(' || c === '[')
|
|
1549
|
+
opens++;
|
|
1550
|
+
if (c === '}' || c === ')' || c === ']')
|
|
1551
|
+
closes++;
|
|
1552
|
+
}
|
|
1553
|
+
const extra = closes - opens;
|
|
1554
|
+
if (extra <= 0)
|
|
1555
|
+
return;
|
|
1556
|
+
// Move to end of editor
|
|
1557
|
+
await cdpSend(page, 'Input.dispatchKeyEvent', {
|
|
1558
|
+
type: 'keyDown', key: 'End', code: 'End', modifiers: 2,
|
|
1559
|
+
windowsVirtualKeyCode: 35, nativeVirtualKeyCode: 35,
|
|
1560
|
+
});
|
|
1561
|
+
await cdpSend(page, 'Input.dispatchKeyEvent', {
|
|
1562
|
+
type: 'keyUp', key: 'End', code: 'End', modifiers: 2,
|
|
1563
|
+
windowsVirtualKeyCode: 35, nativeVirtualKeyCode: 35,
|
|
1564
|
+
});
|
|
1565
|
+
await sleep(100);
|
|
1566
|
+
// Press Backspace extra times
|
|
1567
|
+
for (let i = 0; i < extra; i++) {
|
|
1568
|
+
await cdpSend(page, 'Input.dispatchKeyEvent', {
|
|
1569
|
+
type: 'keyDown', key: 'Backspace', code: 'Backspace',
|
|
1570
|
+
windowsVirtualKeyCode: 8, nativeVirtualKeyCode: 8,
|
|
1571
|
+
});
|
|
1572
|
+
await cdpSend(page, 'Input.dispatchKeyEvent', {
|
|
1573
|
+
type: 'keyUp', key: 'Backspace', code: 'Backspace',
|
|
1574
|
+
windowsVirtualKeyCode: 8, nativeVirtualKeyCode: 8,
|
|
1575
|
+
});
|
|
1576
|
+
await sleep(50);
|
|
1577
|
+
}
|
|
1578
|
+
console.log(`[browse] Removed ${extra} extra trailing bracket(s)`);
|
|
1579
|
+
}
|
|
1580
|
+
/** Insert text into ANY code editor — character-by-character typing, no pasting. */
|
|
1581
|
+
async function insertIntoCodeEditor(page, selector, text, charDelayMs) {
|
|
1582
|
+
// Focus the editor
|
|
1583
|
+
await focusSelector(page, selector);
|
|
1584
|
+
await sleep(100);
|
|
1585
|
+
// Dismiss any autocomplete popup
|
|
1586
|
+
await cdpSend(page, 'Input.dispatchKeyEvent', {
|
|
1587
|
+
type: 'keyDown', key: 'Escape', code: 'Escape',
|
|
1588
|
+
windowsVirtualKeyCode: 27, nativeVirtualKeyCode: 27,
|
|
1589
|
+
});
|
|
1590
|
+
await cdpSend(page, 'Input.dispatchKeyEvent', {
|
|
1591
|
+
type: 'keyUp', key: 'Escape', code: 'Escape',
|
|
1592
|
+
windowsVirtualKeyCode: 27, nativeVirtualKeyCode: 27,
|
|
1593
|
+
});
|
|
1594
|
+
await sleep(100);
|
|
1595
|
+
// Disable ALL auto-formatting to prevent cascading indentation
|
|
1596
|
+
await cdpSend(page, 'Runtime.evaluate', {
|
|
1597
|
+
expression: `
|
|
1598
|
+
(() => {
|
|
1599
|
+
const monacoNs = typeof monaco !== 'undefined' ? monaco : window.monaco;
|
|
1600
|
+
monacoNs?.editor?.getEditors?.().forEach(e => e.updateOptions({
|
|
1601
|
+
autoIndent: false,
|
|
1602
|
+
formatOnType: false,
|
|
1603
|
+
autoClosingBrackets: 'never',
|
|
1604
|
+
autoSurround: 'never',
|
|
1605
|
+
tabSize: 2,
|
|
1606
|
+
detectIndentation: false,
|
|
1607
|
+
}));
|
|
1608
|
+
const cm5 = document.querySelector('.CodeMirror')?.CodeMirror;
|
|
1609
|
+
if (cm5) cm5.setOption('electricChars', false);
|
|
1610
|
+
const ace = document.querySelector('.ace_editor')?.env?.editor;
|
|
1611
|
+
if (ace) {
|
|
1612
|
+
ace.setOption('autoScrollEditorIntoView', false);
|
|
1613
|
+
ace.setOption(' behavioursEnabled', false);
|
|
1614
|
+
}
|
|
1615
|
+
})()
|
|
1616
|
+
`,
|
|
1617
|
+
});
|
|
1618
|
+
// Smart retry: check if editor already has the correct code
|
|
1619
|
+
const currentContent = await readEditorContent(page);
|
|
1620
|
+
if (currentContent === text) {
|
|
1621
|
+
console.log('[browse] Code already correct — skipping re-type');
|
|
1622
|
+
return;
|
|
1623
|
+
}
|
|
1624
|
+
// Focus + clear existing content
|
|
1625
|
+
await focusSelector(page, selector);
|
|
1626
|
+
await sleep(100);
|
|
1627
|
+
await clearEditor(page);
|
|
1628
|
+
await sleep(100);
|
|
1629
|
+
// Re-focus after clear
|
|
1630
|
+
await focusSelector(page, selector);
|
|
1631
|
+
await sleep(100);
|
|
1632
|
+
// Type character by character — NO pasting
|
|
1633
|
+
const est = Math.round(text.length * charDelayMs / 1000);
|
|
1634
|
+
console.log(`[browse] Typing ${text.length} chars (~${est}s est.)...`);
|
|
1635
|
+
for (let i = 0; i < text.length; i++) {
|
|
1636
|
+
const char = text[i];
|
|
1637
|
+
if (char === '\n') {
|
|
1638
|
+
await cdpSend(page, 'Input.dispatchKeyEvent', {
|
|
1639
|
+
type: 'keyDown', key: 'Enter', code: 'Enter', text: '\r',
|
|
1640
|
+
windowsVirtualKeyCode: 13, nativeVirtualKeyCode: 13,
|
|
1641
|
+
});
|
|
1642
|
+
await cdpSend(page, 'Input.dispatchKeyEvent', {
|
|
1643
|
+
type: 'keyUp', key: 'Enter', code: 'Enter',
|
|
1644
|
+
windowsVirtualKeyCode: 13, nativeVirtualKeyCode: 13,
|
|
1645
|
+
});
|
|
1646
|
+
}
|
|
1647
|
+
else if (char === '\t') {
|
|
1648
|
+
await cdpSend(page, 'Input.dispatchKeyEvent', {
|
|
1649
|
+
type: 'keyDown', key: 'Tab', code: 'Tab', text: '\t',
|
|
1650
|
+
windowsVirtualKeyCode: 9, nativeVirtualKeyCode: 9,
|
|
1651
|
+
});
|
|
1652
|
+
await cdpSend(page, 'Input.dispatchKeyEvent', {
|
|
1653
|
+
type: 'keyUp', key: 'Tab', code: 'Tab',
|
|
1654
|
+
windowsVirtualKeyCode: 9, nativeVirtualKeyCode: 9,
|
|
1655
|
+
});
|
|
1656
|
+
}
|
|
1657
|
+
else {
|
|
1658
|
+
const k = charToKey(char);
|
|
1659
|
+
const mods = k.shift ? 2 : undefined;
|
|
1660
|
+
const downParams = {
|
|
1661
|
+
type: 'keyDown', key: k.key, code: k.code, text: char,
|
|
1662
|
+
};
|
|
1663
|
+
const upParams = {
|
|
1664
|
+
type: 'keyUp', key: k.key, code: k.code,
|
|
1665
|
+
};
|
|
1666
|
+
if (mods !== undefined) {
|
|
1667
|
+
downParams.modifiers = mods;
|
|
1668
|
+
upParams.modifiers = mods;
|
|
1669
|
+
}
|
|
1670
|
+
await cdpSend(page, 'Input.dispatchKeyEvent', downParams);
|
|
1671
|
+
await cdpSend(page, 'Input.dispatchKeyEvent', upParams);
|
|
1672
|
+
}
|
|
1673
|
+
await sleep(variableDelay(charDelayMs, char, i, text.length));
|
|
1674
|
+
}
|
|
1675
|
+
// Post-typing: detect and remove extra trailing brackets
|
|
1676
|
+
await removeTrailingBrackets(page);
|
|
1677
|
+
// Re-enable auto-formatting
|
|
1678
|
+
await cdpSend(page, 'Runtime.evaluate', {
|
|
1679
|
+
expression: `
|
|
1680
|
+
(() => {
|
|
1681
|
+
const monacoNs = typeof monaco !== 'undefined' ? monaco : window.monaco;
|
|
1682
|
+
monacoNs?.editor?.getEditors?.().forEach(e => e.updateOptions({
|
|
1683
|
+
autoIndent: true,
|
|
1684
|
+
formatOnType: true,
|
|
1685
|
+
autoClosingBrackets: 'languageDefined',
|
|
1686
|
+
autoSurround: 'languageDefined',
|
|
1687
|
+
detectIndentation: true,
|
|
1688
|
+
}));
|
|
1689
|
+
const cm5 = document.querySelector('.CodeMirror')?.CodeMirror;
|
|
1690
|
+
if (cm5) cm5.setOption('electricChars', true);
|
|
1691
|
+
})()
|
|
1692
|
+
`,
|
|
1693
|
+
});
|
|
1694
|
+
console.log(`[browse] Typing complete (${text.length} chars)`);
|
|
1695
|
+
}
|
|
1696
|
+
/** Insert text into a generic input — click focus + char-by-char typing. */
|
|
1697
|
+
async function insertIntoGenericInput(page, el, text, charDelayMs) {
|
|
1698
|
+
await cdpSend(page, 'Input.dispatchMouseEvent', {
|
|
1699
|
+
type: 'mousePressed', x: el.x, y: el.y, button: 'left', clickCount: 1,
|
|
1700
|
+
});
|
|
1701
|
+
await cdpSend(page, 'Input.dispatchMouseEvent', {
|
|
1702
|
+
type: 'mouseReleased', x: el.x, y: el.y, button: 'left', clickCount: 1,
|
|
1703
|
+
});
|
|
1704
|
+
await sleep(200);
|
|
1705
|
+
await clearEditor(page);
|
|
1706
|
+
await sleep(200);
|
|
1707
|
+
for (let i = 0; i < text.length; i++) {
|
|
1708
|
+
const char = text[i];
|
|
1709
|
+
if (char === '\n') {
|
|
1710
|
+
await cdpSend(page, 'Input.dispatchKeyEvent', {
|
|
1711
|
+
type: 'keyDown', key: 'Enter', code: 'Enter', text: '\r',
|
|
1712
|
+
windowsVirtualKeyCode: 13, nativeVirtualKeyCode: 13,
|
|
1713
|
+
});
|
|
1714
|
+
await cdpSend(page, 'Input.dispatchKeyEvent', {
|
|
1715
|
+
type: 'keyUp', key: 'Enter', code: 'Enter',
|
|
1716
|
+
windowsVirtualKeyCode: 13, nativeVirtualKeyCode: 13,
|
|
1717
|
+
});
|
|
1718
|
+
}
|
|
1719
|
+
else if (char === '\t') {
|
|
1720
|
+
await cdpSend(page, 'Input.dispatchKeyEvent', {
|
|
1721
|
+
type: 'keyDown', key: 'Tab', code: 'Tab', text: '\t',
|
|
1722
|
+
windowsVirtualKeyCode: 9, nativeVirtualKeyCode: 9,
|
|
1723
|
+
});
|
|
1724
|
+
await cdpSend(page, 'Input.dispatchKeyEvent', {
|
|
1725
|
+
type: 'keyUp', key: 'Tab', code: 'Tab',
|
|
1726
|
+
windowsVirtualKeyCode: 9, nativeVirtualKeyCode: 9,
|
|
1727
|
+
});
|
|
1728
|
+
}
|
|
1729
|
+
else {
|
|
1730
|
+
const k = charToKey(char);
|
|
1731
|
+
const mods = k.shift ? 2 : undefined;
|
|
1732
|
+
const downParams = {
|
|
1733
|
+
type: 'keyDown', key: k.key, code: k.code, text: char,
|
|
1734
|
+
};
|
|
1735
|
+
const upParams = {
|
|
1736
|
+
type: 'keyUp', key: k.key, code: k.code,
|
|
1737
|
+
};
|
|
1738
|
+
if (mods !== undefined) {
|
|
1739
|
+
downParams.modifiers = mods;
|
|
1740
|
+
upParams.modifiers = mods;
|
|
1741
|
+
}
|
|
1742
|
+
await cdpSend(page, 'Input.dispatchKeyEvent', downParams);
|
|
1743
|
+
await cdpSend(page, 'Input.dispatchKeyEvent', upParams);
|
|
1744
|
+
}
|
|
1745
|
+
await sleep(variableDelay(charDelayMs, char, i, text.length));
|
|
1746
|
+
}
|
|
1747
|
+
}
|
|
1748
|
+
/**
|
|
1749
|
+
* Extract page context for the LLM: title, URL, headings, errors, editor content.
|
|
1750
|
+
* This gives the LLM full visibility into what's on screen.
|
|
1751
|
+
*/
|
|
1752
|
+
async function extractPageContext(page) {
|
|
1753
|
+
const js = `
|
|
1754
|
+
(() => {
|
|
1755
|
+
const ctx = [];
|
|
1756
|
+
// Title + URL
|
|
1757
|
+
ctx.push('Title: ' + (document.title || 'unknown'));
|
|
1758
|
+
ctx.push('URL: ' + window.location.href);
|
|
1759
|
+
// Headings
|
|
1760
|
+
const headings = [...document.querySelectorAll('h1,h2,h3,h4,h5,h6')]
|
|
1761
|
+
.map(h => h.textContent?.trim())
|
|
1762
|
+
.filter(Boolean)
|
|
1763
|
+
.slice(0, 10);
|
|
1764
|
+
if (headings.length) ctx.push('Headings: ' + JSON.stringify(headings));
|
|
1765
|
+
// Errors / alerts / toasts / notifications
|
|
1766
|
+
const errSels = [
|
|
1767
|
+
'[role="alert"]', '[role="status"]',
|
|
1768
|
+
'[class*="error"]', '[class*="Error"]',
|
|
1769
|
+
'[class*="toast"]', '[class*="Toast"]',
|
|
1770
|
+
'[class*="alert"]', '[class*="Alert"]',
|
|
1771
|
+
'[class*="banner"]', '[class*="Banner"]',
|
|
1772
|
+
'[class*="notification"]', '[class*="Notification"]',
|
|
1773
|
+
'[class*="danger"]', '[class*="warning"]',
|
|
1774
|
+
'[class*="inline-error"]', '[class*="field-error"]',
|
|
1775
|
+
'[class*="form-error"]', '[class*="validation-error"]',
|
|
1776
|
+
'[data-sonner-toaster] [data-sonner-toast]',
|
|
1777
|
+
'.alert-danger', '.alert-warning', '.alert-error',
|
|
1778
|
+
'.text-red', '.text-danger', '.text-error',
|
|
1779
|
+
'[class*="err-msg"]', '[class*="error-msg"]', '[class*="error-message"]',
|
|
1780
|
+
];
|
|
1781
|
+
const errors = [];
|
|
1782
|
+
document.querySelectorAll(errSels.join(',')).forEach(el => {
|
|
1783
|
+
const text = el.innerText?.trim().replace(/\\s+/g, ' ').slice(0, 200);
|
|
1784
|
+
if (text && text.length > 2) errors.push(text);
|
|
1785
|
+
});
|
|
1786
|
+
if (errors.length) {
|
|
1787
|
+
ctx.push('Errors on screen:');
|
|
1788
|
+
errors.slice(0, 5).forEach(e => ctx.push(' - ' + e));
|
|
1789
|
+
}
|
|
1790
|
+
// Code editor content
|
|
1791
|
+
let editorContent = null;
|
|
1792
|
+
// Monaco
|
|
1793
|
+
const monacoNs = typeof monaco !== 'undefined' ? monaco : window.monaco;
|
|
1794
|
+
const editors = monacoNs?.editor?.getEditors?.() || [];
|
|
1795
|
+
if (editors.length > 0) editorContent = editors[0].getValue();
|
|
1796
|
+
// CodeMirror 6
|
|
1797
|
+
if (!editorContent) {
|
|
1798
|
+
const cm6 = document.querySelector('.cm-content');
|
|
1799
|
+
if (cm6?.cmView?.view) editorContent = cm6.cmView.view.state.doc.toString();
|
|
1800
|
+
}
|
|
1801
|
+
// CodeMirror 5
|
|
1802
|
+
if (!editorContent) {
|
|
1803
|
+
const cm5 = document.querySelector('.CodeMirror')?.CodeMirror;
|
|
1804
|
+
if (cm5) editorContent = cm5.getValue();
|
|
1805
|
+
}
|
|
1806
|
+
// Ace
|
|
1807
|
+
if (!editorContent) {
|
|
1808
|
+
const ace = document.querySelector('.ace_editor')?.env?.editor;
|
|
1809
|
+
if (ace) editorContent = ace.getValue();
|
|
1810
|
+
}
|
|
1811
|
+
if (editorContent && editorContent.trim().length > 0) {
|
|
1812
|
+
ctx.push('Editor content (' + editorContent.length + ' chars):');
|
|
1813
|
+
ctx.push(editorContent.slice(0, 1500));
|
|
1814
|
+
if (editorContent.length > 1500) ctx.push('... (truncated)');
|
|
1815
|
+
}
|
|
1816
|
+
// Problem description (LeetCode, HackerRank, etc.)
|
|
1817
|
+
const descSels = [
|
|
1818
|
+
'[class*="problem-description"]', '[class*="question-text"]',
|
|
1819
|
+
'[data-track-load="description_content"]', '.markdown-body',
|
|
1820
|
+
'[class*="content"]', '[class*="prose"]', '[class*="description"]',
|
|
1821
|
+
];
|
|
1822
|
+
for (const sel of descSels) {
|
|
1823
|
+
const el = document.querySelector(sel);
|
|
1824
|
+
if (el && el.innerText && el.innerText.trim().length > 20) {
|
|
1825
|
+
ctx.push('Problem description:');
|
|
1826
|
+
ctx.push(el.innerText.trim().slice(0, 3000));
|
|
1827
|
+
break;
|
|
1828
|
+
}
|
|
1829
|
+
}
|
|
1830
|
+
return ctx.join('\\n');
|
|
1831
|
+
})()
|
|
1832
|
+
`;
|
|
1833
|
+
try {
|
|
1834
|
+
const result = await cdpSend(page, 'Runtime.evaluate', {
|
|
1835
|
+
expression: js,
|
|
1836
|
+
returnByValue: true,
|
|
1837
|
+
});
|
|
1838
|
+
return result?.result?.value ?? '';
|
|
1839
|
+
}
|
|
1840
|
+
catch {
|
|
1841
|
+
return '';
|
|
778
1842
|
}
|
|
779
|
-
return base;
|
|
780
1843
|
}
|
|
781
1844
|
/**
|
|
782
1845
|
* Detect which code/rich-text editor is active on the page.
|
|
@@ -890,6 +1953,8 @@ function validateActions(actions) {
|
|
|
890
1953
|
return typeof a.elementIndex === 'number';
|
|
891
1954
|
if (a.action === 'type')
|
|
892
1955
|
return typeof a.elementIndex === 'number' && typeof a.text === 'string';
|
|
1956
|
+
if (a.action === 'write_code')
|
|
1957
|
+
return typeof a.elementIndex === 'number' && typeof a.code === 'string';
|
|
893
1958
|
if (a.action === 'scroll')
|
|
894
1959
|
return typeof a.delta === 'number';
|
|
895
1960
|
if (a.action === 'press')
|
|
@@ -992,7 +2057,7 @@ function extractScatteredActions(str) {
|
|
|
992
2057
|
}
|
|
993
2058
|
return actions;
|
|
994
2059
|
}
|
|
995
|
-
async function analyzeScreenshot(baseUrl, apiKey, model, screenshotBase64, elements, timeoutMs, attempt, maxAttempts) {
|
|
2060
|
+
async function analyzeScreenshot(baseUrl, apiKey, model, screenshotBase64, elements, pageContext, timeoutMs, attempt, maxAttempts) {
|
|
996
2061
|
const startTime = Date.now();
|
|
997
2062
|
// Format element list grouped by question
|
|
998
2063
|
const groups = new Map();
|
|
@@ -1038,11 +2103,16 @@ async function analyzeScreenshot(baseUrl, apiKey, model, screenshotBase64, eleme
|
|
|
1038
2103
|
}
|
|
1039
2104
|
for (const el of els) {
|
|
1040
2105
|
const checked = el.checked ? ' [CHECKED]' : '';
|
|
1041
|
-
|
|
2106
|
+
const visibility = el.inViewport ? '' : ' [OFF-SCREEN]';
|
|
2107
|
+
lines.push(` ${el.i}: [${el.type || el.tag}] "${el.text}"${checked}${visibility}`);
|
|
1042
2108
|
}
|
|
1043
2109
|
lines.push('');
|
|
1044
2110
|
}
|
|
1045
|
-
const userText =
|
|
2111
|
+
const userText = [
|
|
2112
|
+
pageContext ? `<page_context>\n${pageContext}\n</page_context>\n` : '',
|
|
2113
|
+
`Interactive elements on screen:\n${lines.join('\n')}`,
|
|
2114
|
+
`Analyze the screenshot. For each UNANSWERED question, return the element INDEX to click. For checkbox questions, return ONE action per correct answer (multiple clicks). If you see ERRORS in <page_context>, fix them by retyping the correct answer.`,
|
|
2115
|
+
].filter(Boolean).join('\n\n');
|
|
1046
2116
|
const body = {
|
|
1047
2117
|
model,
|
|
1048
2118
|
messages: [
|
|
@@ -1056,7 +2126,7 @@ async function analyzeScreenshot(baseUrl, apiKey, model, screenshotBase64, eleme
|
|
|
1056
2126
|
},
|
|
1057
2127
|
],
|
|
1058
2128
|
temperature: 0.1,
|
|
1059
|
-
max_tokens:
|
|
2129
|
+
max_tokens: 16384,
|
|
1060
2130
|
stream: false,
|
|
1061
2131
|
};
|
|
1062
2132
|
const controller = new AbortController();
|
|
@@ -1067,7 +2137,8 @@ async function analyzeScreenshot(baseUrl, apiKey, model, screenshotBase64, eleme
|
|
|
1067
2137
|
console.log(`[browse] Waiting for LLM... (${elapsed}s elapsed, attempt ${attempt}/${maxAttempts})`);
|
|
1068
2138
|
}, 30000);
|
|
1069
2139
|
const screenshotKB = Math.round((screenshotBase64.length * 3 / 4) / 1024);
|
|
1070
|
-
|
|
2140
|
+
const contextKB = Math.round(pageContext.length / 1024);
|
|
2141
|
+
console.log(`[browse] LLM request: ${model} | ${screenshotKB}KB image | ${contextKB}KB context | ${elements.length} elements | timeout ${Math.round(timeoutMs / 1000)}s | attempt ${attempt}/${maxAttempts}`);
|
|
1071
2142
|
try {
|
|
1072
2143
|
const res = await fetch(`${baseUrl.replace(/\/$/, '')}/chat/completions`, {
|
|
1073
2144
|
method: 'POST',
|
|
@@ -1123,7 +2194,508 @@ async function analyzeScreenshot(baseUrl, apiKey, model, screenshotBase64, eleme
|
|
|
1123
2194
|
clearInterval(heartbeat);
|
|
1124
2195
|
}
|
|
1125
2196
|
}
|
|
2197
|
+
/** Like analyzeScreenshot but accepts a custom system prompt (for optimization pass). */
|
|
2198
|
+
async function analyzeScreenshotWithPrompt(baseUrl, apiKey, model, screenshotBase64, elements, pageContext, timeoutMs, attempt, maxAttempts, customSystemPrompt) {
|
|
2199
|
+
const startTime = Date.now();
|
|
2200
|
+
const userText = [
|
|
2201
|
+
pageContext ? `<page_context>\n${pageContext}\n</page_context>\n` : '',
|
|
2202
|
+
`Analyze the screenshot and respond according to the system instructions.`,
|
|
2203
|
+
].filter(Boolean).join('\n\n');
|
|
2204
|
+
const body = {
|
|
2205
|
+
model,
|
|
2206
|
+
messages: [
|
|
2207
|
+
{ role: 'system', content: customSystemPrompt },
|
|
2208
|
+
{
|
|
2209
|
+
role: 'user',
|
|
2210
|
+
content: [
|
|
2211
|
+
{ type: 'text', text: userText },
|
|
2212
|
+
{ type: 'image_url', image_url: { url: `data:image/jpeg;base64,${screenshotBase64}` } },
|
|
2213
|
+
],
|
|
2214
|
+
},
|
|
2215
|
+
],
|
|
2216
|
+
temperature: 0.1,
|
|
2217
|
+
max_tokens: 16384,
|
|
2218
|
+
stream: false,
|
|
2219
|
+
};
|
|
2220
|
+
const controller = new AbortController();
|
|
2221
|
+
const timeout = setTimeout(() => controller.abort(), timeoutMs);
|
|
2222
|
+
try {
|
|
2223
|
+
const res = await fetch(`${baseUrl.replace(/\/$/, '')}/chat/completions`, {
|
|
2224
|
+
method: 'POST',
|
|
2225
|
+
signal: controller.signal,
|
|
2226
|
+
headers: {
|
|
2227
|
+
'Content-Type': 'application/json',
|
|
2228
|
+
'Authorization': `Bearer ${apiKey}`,
|
|
2229
|
+
'api-key': apiKey,
|
|
2230
|
+
},
|
|
2231
|
+
body: JSON.stringify(body),
|
|
2232
|
+
});
|
|
2233
|
+
if (!res.ok) {
|
|
2234
|
+
return { actions: [], attempt, durationMs: Date.now() - startTime, error: 'http_error' };
|
|
2235
|
+
}
|
|
2236
|
+
const json = (await res.json());
|
|
2237
|
+
let content = json.choices?.[0]?.message?.content;
|
|
2238
|
+
if (Array.isArray(content)) {
|
|
2239
|
+
content = content.filter((c) => c.type === 'text').map((c) => c.text).join('');
|
|
2240
|
+
}
|
|
2241
|
+
if (typeof content !== 'string' || !content) {
|
|
2242
|
+
const reasoning = json.choices?.[0]?.message?.reasoning_content;
|
|
2243
|
+
if (typeof reasoning === 'string' && reasoning)
|
|
2244
|
+
content = reasoning;
|
|
2245
|
+
}
|
|
2246
|
+
if (typeof content !== 'string' || !content) {
|
|
2247
|
+
return { actions: [], attempt, durationMs: Date.now() - startTime, error: 'empty' };
|
|
2248
|
+
}
|
|
2249
|
+
const parsed = extractJsonActions(content);
|
|
2250
|
+
if (parsed.length === 0) {
|
|
2251
|
+
return { actions: [], attempt, durationMs: Date.now() - startTime, error: 'parse_error' };
|
|
2252
|
+
}
|
|
2253
|
+
console.log(`[browse] Optimization pass: ${parsed.length} action(s) returned`);
|
|
2254
|
+
return { actions: parsed, attempt, durationMs: Date.now() - startTime };
|
|
2255
|
+
}
|
|
2256
|
+
catch (err) {
|
|
2257
|
+
if (err instanceof Error && err.name === 'AbortError') {
|
|
2258
|
+
return { actions: [], attempt, durationMs: Date.now() - startTime, error: 'timeout' };
|
|
2259
|
+
}
|
|
2260
|
+
return { actions: [], attempt, durationMs: Date.now() - startTime, error: 'http_error' };
|
|
2261
|
+
}
|
|
2262
|
+
finally {
|
|
2263
|
+
clearTimeout(timeout);
|
|
2264
|
+
}
|
|
2265
|
+
}
|
|
1126
2266
|
function sleep(ms) {
|
|
1127
2267
|
return new Promise((r) => setTimeout(r, ms));
|
|
1128
2268
|
}
|
|
2269
|
+
// ── Stealth mode: OS-level automation (no CDP) ──────────────────────────────
|
|
2270
|
+
function randomInt(min, max) {
|
|
2271
|
+
return Math.floor(Math.random() * (max - min + 1)) + min;
|
|
2272
|
+
}
|
|
2273
|
+
/**
|
|
2274
|
+
* OS-level screen capture using PowerShell.
|
|
2275
|
+
* Returns path to screenshot PNG.
|
|
2276
|
+
*/
|
|
2277
|
+
function captureScreenStealth() {
|
|
2278
|
+
const outPath = path.join(os.tmpdir(), `stealth_${Date.now()}.png`);
|
|
2279
|
+
const ps = [
|
|
2280
|
+
'Add-Type -AssemblyName System.Windows.Forms',
|
|
2281
|
+
'Add-Type -AssemblyName System.Drawing',
|
|
2282
|
+
'$screen = [System.Windows.Forms.Screen]::PrimaryScreen',
|
|
2283
|
+
'$bitmap = New-Object System.Drawing.Bitmap($screen.Bounds.Width, $screen.Bounds.Height)',
|
|
2284
|
+
'$graphics = [System.Drawing.Graphics]::FromImage($bitmap)',
|
|
2285
|
+
'$graphics.CopyFromScreen($screen.Bounds.Location, [System.Drawing.Point]::Empty, $screen.Bounds.Size)',
|
|
2286
|
+
`$bitmap.Save('${outPath.replace(/\\/g, '\\\\')}', [System.Drawing.Imaging.ImageFormat]::Png)`,
|
|
2287
|
+
'$graphics.Dispose()',
|
|
2288
|
+
'$bitmap.Dispose()',
|
|
2289
|
+
].join('; ');
|
|
2290
|
+
(0, child_process_1.execSync)(`powershell -ExecutionPolicy Bypass -Command "${ps}"`, { stdio: 'pipe', timeout: 10000 });
|
|
2291
|
+
return outPath;
|
|
2292
|
+
}
|
|
2293
|
+
/**
|
|
2294
|
+
* Compare two screenshots to detect screen change.
|
|
2295
|
+
* Returns true if screens are different (pixel diff > threshold).
|
|
2296
|
+
*/
|
|
2297
|
+
function screensChanged(prevPath, currPath) {
|
|
2298
|
+
try {
|
|
2299
|
+
const prevBuf = fs.readFileSync(prevPath);
|
|
2300
|
+
const currBuf = fs.readFileSync(currPath);
|
|
2301
|
+
if (prevBuf.length === currBuf.length) {
|
|
2302
|
+
let diff = 0;
|
|
2303
|
+
for (let i = 0; i < prevBuf.length; i += 100) {
|
|
2304
|
+
if (prevBuf[i] !== currBuf[i])
|
|
2305
|
+
diff++;
|
|
2306
|
+
}
|
|
2307
|
+
return diff > 50;
|
|
2308
|
+
}
|
|
2309
|
+
return true;
|
|
2310
|
+
}
|
|
2311
|
+
catch {
|
|
2312
|
+
return true;
|
|
2313
|
+
}
|
|
2314
|
+
}
|
|
2315
|
+
/**
|
|
2316
|
+
* OS-level keyboard character input using PowerShell SendKeys.
|
|
2317
|
+
*/
|
|
2318
|
+
function typeCharOS(char) {
|
|
2319
|
+
// Map special characters to SendKeys syntax
|
|
2320
|
+
const sendKeysMap = {
|
|
2321
|
+
'\n': '{ENTER}',
|
|
2322
|
+
'\r': '{ENTER}',
|
|
2323
|
+
'\t': '{TAB}',
|
|
2324
|
+
'\b': '{BACKSPACE}',
|
|
2325
|
+
'+': '{+}',
|
|
2326
|
+
'^': '{^}',
|
|
2327
|
+
'%': '{%}',
|
|
2328
|
+
'~': '{~}',
|
|
2329
|
+
'(': '{(}',
|
|
2330
|
+
')': '{)}',
|
|
2331
|
+
'{': '{{}',
|
|
2332
|
+
'}': '{}}',
|
|
2333
|
+
'[': '{[}',
|
|
2334
|
+
']': '{]}',
|
|
2335
|
+
};
|
|
2336
|
+
const escaped = sendKeysMap[char] ?? char;
|
|
2337
|
+
const ps = `Add-Type -AssemblyName System.Windows.Forms; [System.Windows.Forms.SendKeys]::SendWait('${escaped.replace(/'/g, "''")}')`;
|
|
2338
|
+
(0, child_process_1.execSync)(`powershell -ExecutionPolicy Bypass -Command "${ps}"`, { stdio: 'pipe', timeout: 5000 });
|
|
2339
|
+
}
|
|
2340
|
+
/**
|
|
2341
|
+
* OS-level mouse click at specific screen coordinates.
|
|
2342
|
+
* Writes C# to temp file, loads via Add-Type -Path, calls user32.dll.
|
|
2343
|
+
*/
|
|
2344
|
+
function clickOS(x, y) {
|
|
2345
|
+
const csCode = [
|
|
2346
|
+
'using System;',
|
|
2347
|
+
'using System.Runtime.InteropServices;',
|
|
2348
|
+
'public class MouseOps {',
|
|
2349
|
+
' [DllImport("user32.dll")] public static extern void mouse_event(uint dwFlags, int dx, int dy, uint dwData, IntPtr dwExtraInfo);',
|
|
2350
|
+
' [DllImport("user32.dll")] public static extern bool SetCursorPos(int X, int Y);',
|
|
2351
|
+
'}',
|
|
2352
|
+
].join('\n');
|
|
2353
|
+
const csPath = path.join(os.tmpdir(), 'ts_mouse_ops.cs');
|
|
2354
|
+
fs.writeFileSync(csPath, csCode, 'utf8');
|
|
2355
|
+
const ps = [
|
|
2356
|
+
`Add-Type -Path '${csPath.replace(/\\/g, '\\\\')}'`,
|
|
2357
|
+
`[MouseOps]::SetCursorPos(${Math.round(x)}, ${Math.round(y)})`,
|
|
2358
|
+
'Start-Sleep -Milliseconds 50',
|
|
2359
|
+
'[MouseOps]::mouse_event(0x0002, 0, 0, 0, [IntPtr]::Zero)',
|
|
2360
|
+
'Start-Sleep -Milliseconds 30',
|
|
2361
|
+
'[MouseOps]::mouse_event(0x0004, 0, 0, 0, [IntPtr]::Zero)',
|
|
2362
|
+
].join('; ');
|
|
2363
|
+
(0, child_process_1.execSync)(`powershell -ExecutionPolicy Bypass -Command "${ps}"`, { stdio: 'pipe', timeout: 5000 });
|
|
2364
|
+
}
|
|
2365
|
+
/**
|
|
2366
|
+
* OS-level scroll using mouse wheel.
|
|
2367
|
+
* positive = up, negative = down.
|
|
2368
|
+
*/
|
|
2369
|
+
function scrollOS(clicks) {
|
|
2370
|
+
const delta = clicks * 120; // WHEEL_DELTA = 120
|
|
2371
|
+
const csCode = [
|
|
2372
|
+
'using System;',
|
|
2373
|
+
'using System.Runtime.InteropServices;',
|
|
2374
|
+
'public class ScrollOps {',
|
|
2375
|
+
' [DllImport("user32.dll")] public static extern void mouse_event(uint dwFlags, int dx, int dy, uint dwData, IntPtr dwExtraInfo);',
|
|
2376
|
+
'}',
|
|
2377
|
+
].join('\n');
|
|
2378
|
+
const csPath = path.join(os.tmpdir(), 'ts_scroll_ops.cs');
|
|
2379
|
+
fs.writeFileSync(csPath, csCode, 'utf8');
|
|
2380
|
+
const ps = [
|
|
2381
|
+
`Add-Type -Path '${csPath.replace(/\\/g, '\\\\')}'`,
|
|
2382
|
+
`[ScrollOps]::mouse_event(0x0800, 0, 0, ${delta}, [IntPtr]::Zero)`,
|
|
2383
|
+
].join('; ');
|
|
2384
|
+
(0, child_process_1.execSync)(`powershell -ExecutionPolicy Bypass -Command "${ps}"`, { stdio: 'pipe', timeout: 5000 });
|
|
2385
|
+
}
|
|
2386
|
+
/**
|
|
2387
|
+
* OS-level keyboard shortcut (e.g., Ctrl+A, Tab, Enter).
|
|
2388
|
+
*/
|
|
2389
|
+
function pressKeyOS(key) {
|
|
2390
|
+
const keyMap = {
|
|
2391
|
+
'enter': '{ENTER}',
|
|
2392
|
+
'tab': '{TAB}',
|
|
2393
|
+
'escape': '{ESC}',
|
|
2394
|
+
'backspace': '{BACKSPACE}',
|
|
2395
|
+
'delete': '{DELETE}',
|
|
2396
|
+
'space': ' ',
|
|
2397
|
+
'up': '{UP}',
|
|
2398
|
+
'down': '{DOWN}',
|
|
2399
|
+
'left': '{LEFT}',
|
|
2400
|
+
'right': '{RIGHT}',
|
|
2401
|
+
'home': '{HOME}',
|
|
2402
|
+
'end': '{END}',
|
|
2403
|
+
'pageup': '{PGUP}',
|
|
2404
|
+
'pagedown': '{PGDN}',
|
|
2405
|
+
'ctrl+a': '^a',
|
|
2406
|
+
'ctrl+c': '^c',
|
|
2407
|
+
'ctrl+v': '^v',
|
|
2408
|
+
'ctrl+z': '^z',
|
|
2409
|
+
};
|
|
2410
|
+
const escaped = keyMap[key.toLowerCase()] ?? key;
|
|
2411
|
+
const ps = `Add-Type -AssemblyName System.Windows.Forms; [System.Windows.Forms.SendKeys]::SendWait('${escaped.replace(/'/g, "''")}')`;
|
|
2412
|
+
(0, child_process_1.execSync)(`powershell -ExecutionPolicy Bypass -Command "${ps}"`, { stdio: 'pipe', timeout: 5000 });
|
|
2413
|
+
}
|
|
2414
|
+
/**
|
|
2415
|
+
* Keyboard proximity map for generating realistic typos.
|
|
2416
|
+
*/
|
|
2417
|
+
const ADJACENT_KEYS = {
|
|
2418
|
+
'a': ['s', 'q', 'w', 'z'], 'b': ['v', 'g', 'h', 'n'],
|
|
2419
|
+
'c': ['x', 'd', 'f', 'v'], 'd': ['s', 'f', 'e', 'c', 'r'],
|
|
2420
|
+
'e': ['w', 'r', 'd', 's'], 'f': ['d', 'g', 'r', 'v', 't'],
|
|
2421
|
+
'g': ['f', 'h', 't', 'b', 'y'], 'h': ['g', 'j', 'y', 'n', 'u'],
|
|
2422
|
+
'i': ['u', 'o', 'k', 'j'], 'j': ['h', 'k', 'n', 'u', 'm'],
|
|
2423
|
+
'k': ['j', 'l', 'm', 'i', 'o'], 'l': ['k', 'o', 'p'],
|
|
2424
|
+
'm': ['n', 'j', 'k'], 'n': ['b', 'm', 'h', 'j'],
|
|
2425
|
+
'o': ['i', 'p', 'l', 'k'], 'p': ['o', 'l'],
|
|
2426
|
+
'q': ['w', 'a'], 'r': ['e', 't', 'f', 'd'],
|
|
2427
|
+
's': ['a', 'd', 'w', 'x', 'z'], 't': ['r', 'y', 'g', 'f'],
|
|
2428
|
+
'u': ['y', 'i', 'j', 'h'], 'v': ['c', 'b', 'f', 'g'],
|
|
2429
|
+
'w': ['q', 'e', 's', 'a'], 'x': ['z', 'c', 's', 'd'],
|
|
2430
|
+
'y': ['t', 'u', 'g', 'h'], 'z': ['x', 'a', 's'],
|
|
2431
|
+
'0': ['9', '1'], '1': ['0', '2', 'q'], '2': ['1', '3', 'w'],
|
|
2432
|
+
'3': ['2', '4', 'e'], '4': ['3', '5', 'r'], '5': ['4', '6', 't'],
|
|
2433
|
+
'6': ['5', '7', 'y'], '7': ['6', '8', 'u'], '8': ['7', '9', 'i'],
|
|
2434
|
+
'9': ['8', '0', 'o'],
|
|
2435
|
+
};
|
|
2436
|
+
/**
|
|
2437
|
+
* Generate a realistic typo by picking an adjacent key.
|
|
2438
|
+
*/
|
|
2439
|
+
function getAdjacentKey(char) {
|
|
2440
|
+
const lower = char.toLowerCase();
|
|
2441
|
+
const keys = ADJACENT_KEYS[lower];
|
|
2442
|
+
if (!keys)
|
|
2443
|
+
return char;
|
|
2444
|
+
const wrong = keys[Math.floor(Math.random() * keys.length)];
|
|
2445
|
+
return char === char.toUpperCase() ? wrong.toUpperCase() : wrong;
|
|
2446
|
+
}
|
|
2447
|
+
/**
|
|
2448
|
+
* Type text with human-like patterns: variable speed, occasional typos,
|
|
2449
|
+
* backspaces, natural pauses. Uses OS-level keyboard (no CDP).
|
|
2450
|
+
*/
|
|
2451
|
+
async function typeHumanLike(text, baseDelayMs = 120) {
|
|
2452
|
+
for (let i = 0; i < text.length; i++) {
|
|
2453
|
+
const char = text[i];
|
|
2454
|
+
// 1. Occasional typo (2% chance) — then backspace to correct
|
|
2455
|
+
if (char !== '\n' && char !== '\t' && Math.random() < 0.02) {
|
|
2456
|
+
const wrong = getAdjacentKey(char);
|
|
2457
|
+
typeCharOS(wrong);
|
|
2458
|
+
await sleep(randomInt(40, 120));
|
|
2459
|
+
typeCharOS('\b'); // backspace
|
|
2460
|
+
await sleep(randomInt(25, 70));
|
|
2461
|
+
}
|
|
2462
|
+
// 2. Type the correct character
|
|
2463
|
+
typeCharOS(char);
|
|
2464
|
+
// 3. Variable delay based on character type (human rhythm)
|
|
2465
|
+
let delay = baseDelayMs + randomInt(-Math.floor(baseDelayMs * 0.3), Math.floor(baseDelayMs * 0.4));
|
|
2466
|
+
if (char === ' ')
|
|
2467
|
+
delay += randomInt(30, 120); // pause at word boundary
|
|
2468
|
+
else if (char === '\n')
|
|
2469
|
+
delay += randomInt(100, 350); // pause at line end
|
|
2470
|
+
else if (char === '.')
|
|
2471
|
+
delay += randomInt(80, 200); // pause at sentence end
|
|
2472
|
+
else if (char === ',')
|
|
2473
|
+
delay += randomInt(40, 100); // pause at comma
|
|
2474
|
+
else if (/[A-Z]/.test(char))
|
|
2475
|
+
delay += randomInt(15, 50); // hesitate before uppercase
|
|
2476
|
+
else if (/[0-9]/.test(char))
|
|
2477
|
+
delay += randomInt(10, 35); // hesitate before number
|
|
2478
|
+
else if (/[!@#$%^&*()_+\-=\[\]{};':"\\|,.<>\/?]/.test(char))
|
|
2479
|
+
delay += randomInt(20, 60); // hesitate before symbol
|
|
2480
|
+
// 4. Occasional longer pause (thinking) — 3% chance
|
|
2481
|
+
if (Math.random() < 0.03) {
|
|
2482
|
+
await sleep(randomInt(200, 600));
|
|
2483
|
+
}
|
|
2484
|
+
await sleep(Math.max(delay, 30));
|
|
2485
|
+
}
|
|
2486
|
+
}
|
|
2487
|
+
/**
|
|
2488
|
+
* Wait for the screen to change by comparing screenshots.
|
|
2489
|
+
* Returns the new screenshot path, or null if timed out.
|
|
2490
|
+
*/
|
|
2491
|
+
function waitForScreenChangeStealth(prevPath, intervalMs = 2000, maxWaitMs = 60000, shouldStop = () => false) {
|
|
2492
|
+
const deadline = Date.now() + maxWaitMs;
|
|
2493
|
+
while (Date.now() < deadline && !shouldStop()) {
|
|
2494
|
+
sleep(intervalMs); // sync-ish wait
|
|
2495
|
+
const currPath = captureScreenStealth();
|
|
2496
|
+
if (screensChanged(prevPath, currPath)) {
|
|
2497
|
+
return currPath;
|
|
2498
|
+
}
|
|
2499
|
+
}
|
|
2500
|
+
return null;
|
|
2501
|
+
}
|
|
2502
|
+
/**
|
|
2503
|
+
* Stealth LLM call: send screenshot to vision LLM, get structured action.
|
|
2504
|
+
* Returns JSON: { action, coordinates?, text?, scroll?, reasoning }
|
|
2505
|
+
*/
|
|
2506
|
+
async function callStealthLLM(baseUrl, apiKey, model, screenshotPath, timeoutMs, promptOverride) {
|
|
2507
|
+
const imageData = fs.readFileSync(screenshotPath).toString('base64');
|
|
2508
|
+
const systemPrompt = promptOverride ?? [
|
|
2509
|
+
'You are a test-taking assistant analyzing a screenshot of a test/assessment.',
|
|
2510
|
+
'',
|
|
2511
|
+
'Analyze the screenshot and return a JSON object with the correct action.',
|
|
2512
|
+
'',
|
|
2513
|
+
'For MULTIPLE-CHOICE questions (radio buttons, checkboxes, A/B/C/D options):',
|
|
2514
|
+
' Return: {"action":"click","x":<center_x_of_correct_option>,"y":<center_y_of_correct_option>,"reasoning":"why this option"}',
|
|
2515
|
+
' Look at the screenshot carefully. The x,y coordinates should be the CENTER of the radio button or checkbox for the correct answer.',
|
|
2516
|
+
'',
|
|
2517
|
+
'For CODING questions (code editor visible, need to write code):',
|
|
2518
|
+
' Return: {"action":"type","text":"<the complete code>","reasoning":"why this code"}',
|
|
2519
|
+
'',
|
|
2520
|
+
'For FILL-IN-THE-BLANK (text input field visible):',
|
|
2521
|
+
' Return: {"action":"type","text":"<the answer text>","reasoning":"why this answer"}',
|
|
2522
|
+
'',
|
|
2523
|
+
'For questions that need SCROLLING (answer options are cut off at bottom):',
|
|
2524
|
+
' Return: {"action":"scroll","scroll":-3,"reasoning":"need to see more options"}',
|
|
2525
|
+
' scroll: negative = scroll down, positive = scroll up. Use -2 to -5 for typical scroll.',
|
|
2526
|
+
'',
|
|
2527
|
+
'For DONE (all questions answered, no more questions visible):',
|
|
2528
|
+
' Return: {"action":"done","reasoning":"all questions answered"}',
|
|
2529
|
+
'',
|
|
2530
|
+
'Rules:',
|
|
2531
|
+
'- Return ONLY valid JSON, nothing else. No markdown, no explanation outside the JSON.',
|
|
2532
|
+
'- For click actions, the x,y must be PIXEL COORDINATES on the screen where the correct option is.',
|
|
2533
|
+
'- Read the question carefully. Think step by step before answering.',
|
|
2534
|
+
'- If you need to scroll to see all options, return scroll action first.',
|
|
2535
|
+
].join('\n');
|
|
2536
|
+
const controller = new AbortController();
|
|
2537
|
+
const timeout = setTimeout(() => controller.abort(), timeoutMs);
|
|
2538
|
+
try {
|
|
2539
|
+
const res = await fetch(`${baseUrl}/chat/completions`, {
|
|
2540
|
+
method: 'POST',
|
|
2541
|
+
headers: {
|
|
2542
|
+
'Content-Type': 'application/json',
|
|
2543
|
+
'Authorization': `Bearer ${apiKey}`,
|
|
2544
|
+
},
|
|
2545
|
+
body: JSON.stringify({
|
|
2546
|
+
model,
|
|
2547
|
+
messages: [
|
|
2548
|
+
{ role: 'system', content: systemPrompt },
|
|
2549
|
+
{
|
|
2550
|
+
role: 'user',
|
|
2551
|
+
content: [
|
|
2552
|
+
{ type: 'text', text: 'Analyze this screenshot. Return the JSON action to take.' },
|
|
2553
|
+
{ type: 'image_url', image_url: { url: `data:image/png;base64,${imageData}` } },
|
|
2554
|
+
],
|
|
2555
|
+
},
|
|
2556
|
+
],
|
|
2557
|
+
max_tokens: 4096,
|
|
2558
|
+
temperature: 0.1,
|
|
2559
|
+
}),
|
|
2560
|
+
signal: controller.signal,
|
|
2561
|
+
});
|
|
2562
|
+
if (!res.ok)
|
|
2563
|
+
throw new Error(`HTTP ${res.status}`);
|
|
2564
|
+
const data = await res.json();
|
|
2565
|
+
const content = (data.choices?.[0]?.message?.content ?? '').trim();
|
|
2566
|
+
// Parse JSON response
|
|
2567
|
+
try {
|
|
2568
|
+
// Extract JSON from possible markdown code block
|
|
2569
|
+
const jsonMatch = content.match(/\{[\s\S]*\}/);
|
|
2570
|
+
if (jsonMatch) {
|
|
2571
|
+
const parsed = JSON.parse(jsonMatch[0]);
|
|
2572
|
+
return {
|
|
2573
|
+
action: parsed.action ?? 'type',
|
|
2574
|
+
x: parsed.x,
|
|
2575
|
+
y: parsed.y,
|
|
2576
|
+
text: parsed.text,
|
|
2577
|
+
scroll: parsed.scroll,
|
|
2578
|
+
reasoning: parsed.reasoning,
|
|
2579
|
+
};
|
|
2580
|
+
}
|
|
2581
|
+
}
|
|
2582
|
+
catch {
|
|
2583
|
+
// JSON parse failed — treat as plain text answer (type action)
|
|
2584
|
+
}
|
|
2585
|
+
// Fallback: treat entire response as text to type
|
|
2586
|
+
return { action: 'type', text: content, reasoning: 'fallback: plain text response' };
|
|
2587
|
+
}
|
|
2588
|
+
finally {
|
|
2589
|
+
clearTimeout(timeout);
|
|
2590
|
+
}
|
|
2591
|
+
}
|
|
2592
|
+
/**
|
|
2593
|
+
* Stealth agent loop: OS-level screen capture → vision LLM → OS-level keyboard/mouse.
|
|
2594
|
+
* No CDP connection, no clipboard, no DevTools detection.
|
|
2595
|
+
*/
|
|
2596
|
+
async function runStealthLoop(baseUrl, apiKey, model, maxActions, intervalMs, timeoutMs, charDelayMs, continuous) {
|
|
2597
|
+
let actionCount = 0;
|
|
2598
|
+
let llmCallCount = 0;
|
|
2599
|
+
let stopped = false;
|
|
2600
|
+
process.on('SIGINT', () => {
|
|
2601
|
+
console.log('\n[stealth] Stopped by user');
|
|
2602
|
+
stopped = true;
|
|
2603
|
+
});
|
|
2604
|
+
process.on('SIGTERM', () => { stopped = true; });
|
|
2605
|
+
const limit = continuous ? Number.MAX_SAFE_INTEGER : maxActions;
|
|
2606
|
+
console.log(`[stealth] Agent started. ${continuous ? 'Running until Ctrl+C.' : `Max ${limit} actions.`}`);
|
|
2607
|
+
console.log('[stealth] OS-level automation — no CDP, no clipboard, no DevTools');
|
|
2608
|
+
console.log('[stealth] Make sure the target window (browser/editor) is focused\n');
|
|
2609
|
+
// Take initial screenshot
|
|
2610
|
+
let prevScreenshot = captureScreenStealth();
|
|
2611
|
+
console.log('[stealth] Initial screenshot captured — waiting for screen change...\n');
|
|
2612
|
+
while (!stopped && actionCount < limit) {
|
|
2613
|
+
try {
|
|
2614
|
+
// Wait for screen to change (new question appears)
|
|
2615
|
+
const currScreenshot = captureScreenStealth();
|
|
2616
|
+
if (!screensChanged(prevScreenshot, currScreenshot)) {
|
|
2617
|
+
// No change — wait and retry
|
|
2618
|
+
await sleep(intervalMs);
|
|
2619
|
+
continue;
|
|
2620
|
+
}
|
|
2621
|
+
console.log(`[stealth] Screen changed — analyzing...`);
|
|
2622
|
+
prevScreenshot = currScreenshot;
|
|
2623
|
+
// Send to vision LLM
|
|
2624
|
+
const actionLabel = continuous ? `${actionCount + 1}/∞` : `${actionCount + 1}/${limit}`;
|
|
2625
|
+
console.log(`[stealth] [${actionLabel}] Calling vision LLM...`);
|
|
2626
|
+
const result = await callStealthLLM(baseUrl, apiKey, model, currScreenshot, timeoutMs);
|
|
2627
|
+
llmCallCount++;
|
|
2628
|
+
console.log(`[stealth] LLM returned: action="${result.action}" reasoning="${result.reasoning?.slice(0, 60) ?? ''}"`);
|
|
2629
|
+
// Execute the action
|
|
2630
|
+
switch (result.action) {
|
|
2631
|
+
case 'click': {
|
|
2632
|
+
if (result.x !== undefined && result.y !== undefined) {
|
|
2633
|
+
console.log(`[stealth] Clicking at (${Math.round(result.x)}, ${Math.round(result.y)})...`);
|
|
2634
|
+
await sleep(randomInt(200, 500)); // natural pause before click
|
|
2635
|
+
clickOS(result.x, result.y);
|
|
2636
|
+
actionCount++;
|
|
2637
|
+
}
|
|
2638
|
+
else {
|
|
2639
|
+
console.log('[stealth] Click action missing coordinates — skipping');
|
|
2640
|
+
}
|
|
2641
|
+
break;
|
|
2642
|
+
}
|
|
2643
|
+
case 'type': {
|
|
2644
|
+
if (result.text && result.text.length > 0) {
|
|
2645
|
+
console.log(`[stealth] Typing ${result.text.length} chars...`);
|
|
2646
|
+
await sleep(randomInt(300, 700)); // natural pause before typing
|
|
2647
|
+
await typeHumanLike(result.text, charDelayMs);
|
|
2648
|
+
actionCount++;
|
|
2649
|
+
}
|
|
2650
|
+
else {
|
|
2651
|
+
console.log('[stealth] Type action missing text — skipping');
|
|
2652
|
+
}
|
|
2653
|
+
break;
|
|
2654
|
+
}
|
|
2655
|
+
case 'scroll': {
|
|
2656
|
+
const clicks = result.scroll ?? -3;
|
|
2657
|
+
console.log(`[stealth] Scrolling ${clicks > 0 ? 'up' : 'down'} (${Math.abs(clicks)} clicks)...`);
|
|
2658
|
+
await sleep(randomInt(200, 400));
|
|
2659
|
+
scrollOS(clicks);
|
|
2660
|
+
actionCount++;
|
|
2661
|
+
// Wait for scroll to settle
|
|
2662
|
+
await sleep(randomInt(500, 1000));
|
|
2663
|
+
break;
|
|
2664
|
+
}
|
|
2665
|
+
case 'done': {
|
|
2666
|
+
console.log(`[stealth] LLM says done: ${result.reasoning ?? 'All questions answered'}`);
|
|
2667
|
+
if (!continuous) {
|
|
2668
|
+
stopped = true;
|
|
2669
|
+
}
|
|
2670
|
+
else {
|
|
2671
|
+
// In continuous mode, wait for next screen change
|
|
2672
|
+
console.log('[stealth] Waiting for next question...');
|
|
2673
|
+
await sleep(randomInt(3000, 6000));
|
|
2674
|
+
}
|
|
2675
|
+
break;
|
|
2676
|
+
}
|
|
2677
|
+
default: {
|
|
2678
|
+
console.log(`[stealth] Unknown action: ${result.action} — treating as type`);
|
|
2679
|
+
if (result.text) {
|
|
2680
|
+
await typeHumanLike(result.text, charDelayMs);
|
|
2681
|
+
actionCount++;
|
|
2682
|
+
}
|
|
2683
|
+
}
|
|
2684
|
+
}
|
|
2685
|
+
// Log progress
|
|
2686
|
+
if (llmCallCount > 50 && llmCallCount % 25 === 0) {
|
|
2687
|
+
console.log(`[stealth] Warning: ${llmCallCount} LLM calls made — may hit rate limits`);
|
|
2688
|
+
}
|
|
2689
|
+
// Natural pause before next question
|
|
2690
|
+
const pause = randomInt(1500, 3500);
|
|
2691
|
+
console.log(`[stealth] Waiting ${pause}ms for next question...\n`);
|
|
2692
|
+
await sleep(pause);
|
|
2693
|
+
}
|
|
2694
|
+
catch (err) {
|
|
2695
|
+
console.error(`[stealth] Error: ${(0, config_1.msg)(err)}`);
|
|
2696
|
+
await sleep(intervalMs);
|
|
2697
|
+
}
|
|
2698
|
+
}
|
|
2699
|
+
console.log(`[stealth] Agent stopped. ${actionCount} actions completed, ${llmCallCount} LLM calls.`);
|
|
2700
|
+
}
|
|
1129
2701
|
//# sourceMappingURL=browse.js.map
|