teamshare-bridge 0.19.17 → 0.19.18

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,4 +1,37 @@
1
1
  "use strict";
2
+ var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
3
+ if (k2 === undefined) k2 = k;
4
+ var desc = Object.getOwnPropertyDescriptor(m, k);
5
+ if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
6
+ desc = { enumerable: true, get: function() { return m[k]; } };
7
+ }
8
+ Object.defineProperty(o, k2, desc);
9
+ }) : (function(o, m, k, k2) {
10
+ if (k2 === undefined) k2 = k;
11
+ o[k2] = m[k];
12
+ }));
13
+ var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
14
+ Object.defineProperty(o, "default", { enumerable: true, value: v });
15
+ }) : function(o, v) {
16
+ o["default"] = v;
17
+ });
18
+ var __importStar = (this && this.__importStar) || (function () {
19
+ var ownKeys = function(o) {
20
+ ownKeys = Object.getOwnPropertyNames || function (o) {
21
+ var ar = [];
22
+ for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
23
+ return ar;
24
+ };
25
+ return ownKeys(o);
26
+ };
27
+ return function (mod) {
28
+ if (mod && mod.__esModule) return mod;
29
+ var result = {};
30
+ if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
31
+ __setModuleDefault(result, mod);
32
+ return result;
33
+ };
34
+ })();
2
35
  var __importDefault = (this && this.__importDefault) || function (mod) {
3
36
  return (mod && mod.__esModule) ? mod : { "default": mod };
4
37
  };
@@ -12,11 +45,70 @@ exports.cmdBrowse = cmdBrowse;
12
45
  * answers questions/forms on screen.
13
46
  *
14
47
  * Usage:
15
- * teamshare-agent browse --auto-answer [--port 9222] [--model mimo-v2.5] [--max-actions 50]
48
+ * teamshare-agent browse --auto-answer [--port 9222] [--model mimo-v2.5] [--max-actions 50] [--continuous]
16
49
  */
17
50
  const ws_1 = __importDefault(require("ws"));
18
51
  const models_1 = require("../../lib/llm/models");
19
52
  const config_1 = require("../../lib/config");
53
+ const fs = __importStar(require("fs"));
54
+ const os = __importStar(require("os"));
55
+ const path = __importStar(require("path"));
56
+ const child_process_1 = require("child_process");
57
+ /** Create a fingerprint from the current element list. */
58
+ function createFingerprint(elements) {
59
+ const groups = new Map();
60
+ for (const el of elements) {
61
+ if (!groups.has(el.group))
62
+ groups.set(el.group, []);
63
+ groups.get(el.group).push(el);
64
+ }
65
+ const fp = [];
66
+ for (const [gid, els] of groups) {
67
+ fp.push({
68
+ groupId: gid,
69
+ questionText: els[0]?.questionText || '',
70
+ options: els.map((e) => e.text || ''),
71
+ optionTypes: els.map((e) => e.type || e.tag),
72
+ checkedStates: els.map((e) => e.checked),
73
+ elementCount: els.length,
74
+ });
75
+ }
76
+ return fp;
77
+ }
78
+ /** Weighted comparison of two fingerprints. Returns change score. */
79
+ function calculateChangeScore(prev, curr) {
80
+ let score = 0;
81
+ const prevMap = new Map(prev.map((fp) => [fp.groupId, fp]));
82
+ const currMap = new Map(curr.map((fp) => [fp.groupId, fp]));
83
+ // New questions appeared (weight: 10)
84
+ for (const [gid] of currMap) {
85
+ if (!prevMap.has(gid))
86
+ score += 10;
87
+ }
88
+ // Questions removed (weight: 10)
89
+ for (const [gid] of prevMap) {
90
+ if (!currMap.has(gid))
91
+ score += 10;
92
+ }
93
+ // Compare matching questions
94
+ for (const [gid, currFp] of currMap) {
95
+ const prevFp = prevMap.get(gid);
96
+ if (!prevFp)
97
+ continue;
98
+ // Question text changed (weight: 8)
99
+ if (prevFp.questionText !== currFp.questionText)
100
+ score += 8;
101
+ // Options changed (weight: 5)
102
+ if (prevFp.options.join('|') !== currFp.options.join('|'))
103
+ score += 5;
104
+ // Checked states changed (weight: 2 — user answering, not screen change)
105
+ for (let i = 0; i < Math.min(prevFp.checkedStates.length, currFp.checkedStates.length); i++) {
106
+ if (prevFp.checkedStates[i] !== currFp.checkedStates[i])
107
+ score += 2;
108
+ }
109
+ }
110
+ return score;
111
+ }
20
112
  const SYSTEM_PROMPT = `You are a browser automation agent. A screenshot and a grouped list of interactive elements are attached.
21
113
 
22
114
  Your task: identify unanswered questions and answer them by clicking the correct elements.
@@ -25,7 +117,16 @@ RULES (STRICT):
25
117
  1. NEVER click Next, Submit, Back, Save, Continue, or any navigation. ONLY answer questions on the current screen.
26
118
  2. NEVER ask for help. If uncertain, pick the BEST GUESS. Always decide — never wait.
27
119
  3. Handle MULTIPLE unanswered questions in a SINGLE response.
28
- 4. If ALL questions are already answered correctly: [{"action":"done","summary":"All visible questions answered"}]
120
+ 4. NEVER return done unless you have VERIFIED every single question is answered. See VERIFICATION CHECKLIST below.
121
+ 5. NEVER answer personal questions — see PERSONAL QUESTIONS section below.
122
+
123
+ VERIFICATION CHECKLIST (MANDATORY before returning done):
124
+ Before returning done, you MUST check EVERY question group:
125
+ - Radio questions: exactly ONE option must be selected (checked). If none selected → you MUST pick an answer.
126
+ - Checkbox questions: ALL correct options must be checked. If any correct option is missing → you MUST check it.
127
+ - Text input questions: the input must have a non-empty value. If empty → you MUST type an answer.
128
+ If ANY question fails this check, you MUST answer it — NEVER return done with unanswered questions.
129
+ If you return done while questions are unanswered, the task will FAIL. You will be retried.
29
130
 
30
131
  QUESTION TYPES AND STATUS:
31
132
  - radio (single answer): Only ONE option can be correct.
@@ -57,15 +158,86 @@ COMBINED EXAMPLE:
57
158
  {"action":"type","elementIndex":9,"text":"42","reason":"Q3: fill answer"}]
58
159
 
59
160
  CODE TYPE EXAMPLE (typing code into an editor — note escaped newlines and quotes):
60
- [{"action":"type","elementIndex":5,"text":"function twoSum(nums, target) {\\n const map = new Map();\\n for (let i = 0; i < nums.length; i++) {\\n const complement = target - nums[i];\\n if (map.has(complement)) return [map.get(complement), i];\\n map.set(nums[i], i);\\n }\\n return [];\\n}","reason":"Type the solution function"}]
161
+ [{"action":"type","elementIndex":5,"text":"function twoSum(nums: number[], target: number): number[] {\\n const map = new Map<number, number>();\\n for (let i = 0; i < nums.length; i++) {\\n const complement = target - nums[i];\\n if (map.has(complement)) {\\n return [map.get(complement), i];\\n }\\n map.set(nums[i], i);\\n }\\n return [];\\n}","reason":"Type the solution function"}]
162
+
163
+ WRITE_CODE EXAMPLE (large code blocks — preferred for code > 100 chars):
164
+ [{"action":"write_code","elementIndex":5,"code":"function twoSum(nums: number[], target: number): number[] {\\n const map = new Map<number, number>();\\n for (let i = 0; i < nums.length; i++) {\\n const complement = target - nums[i];\\n if (map.has(complement)) {\\n return [map.get(complement), i];\\n }\\n map.set(nums[i], i);\\n }\\n return [];\\n}","reason":"Type the solution function"}]
165
+
166
+ WHEN TO USE write_code vs type:
167
+ - "write_code" for code blocks, functions, classes, multi-line snippets (> 100 chars)
168
+ - "type" for short answers: numbers, single words, short strings (< 100 chars)
169
+ - "write_code" preserves exact indentation and newlines — no formatting changes
170
+
171
+ CODE FORMATTING RULES (CRITICAL):
172
+ - Use 2-space indentation. Each nesting level adds exactly 2 more spaces.
173
+ - Function body = 2 spaces, loop body = 4 spaces, if-body = 6 spaces.
174
+ - NEVER use tabs or 4/8-space indentation.
175
+ - Keep code compact: single-line statements when appropriate.
176
+
177
+ CODE OPTIMIZATION RULES (CRITICAL — for coding problems):
178
+ 1. ALWAYS write the most time-efficient solution. Correctness alone is NOT enough.
179
+ 2. Prefer O(n) or O(n log n) over O(n²). Never write O(n²) if an O(n) approach exists.
180
+ 3. Use HashMap/Set for O(1) lookups instead of nested loops for searching/matching.
181
+ 4. Consider space-time tradeoffs — a HashMap is worth the O(n) space for O(n) time.
182
+ 5. Avoid unnecessary re-computation — cache results in variables.
183
+ 6. Use sliding window, two pointers, or binary search where applicable.
184
+ 7. Pre-compute prefix sums, frequency maps, or lookup tables when helpful.
185
+ 8. Choose the right data structure:
186
+ - Deque for sliding window problems
187
+ - Heap/priority queue for top-K problems
188
+ - Trie for prefix/string matching problems
189
+ - Union-Find for connected components
190
+ - BFS for shortest path, DFS for exploration
191
+ 9. NEVER write brute force if an optimized approach exists.
192
+ 10. If the problem has constraints (e.g., n <= 10^5), ensure your solution runs within limits.
193
+
194
+ OPTIMIZATION EXAMPLE:
195
+ Problem: "Given an array of integers, return indices of two numbers that add up to target."
196
+
197
+ NAIVE (WRONG for efficiency — O(n²) time):
198
+ for (let i = 0; i < nums.length; i++) {
199
+ for (let j = i + 1; j < nums.length; j++) {
200
+ if (nums[i] + nums[j] === target) return [i, j];
201
+ }
202
+ }
203
+
204
+ OPTIMIZED (CORRECT — O(n) time, O(n) space):
205
+ const map = new Map();
206
+ for (let i = 0; i < nums.length; i++) {
207
+ const complement = target - nums[i];
208
+ if (map.has(complement)) return [map.get(complement), i];
209
+ map.set(nums[i], i);
210
+ }
211
+ return [];
61
212
 
62
213
  Available actions:
63
214
  - click: {action:"click", elementIndex:number} — click to check/uncheck/select
64
- - type: {action:"type", elementIndex:number, text:string} — type into input
215
+ - type: {action:"type", elementIndex:number, text:string} — type short input (< 100 chars)
216
+ - write_code: {action:"write_code", elementIndex:number, code:string} — write code block (> 100 chars), preserves exact indentation
65
217
  - scroll: {action:"scroll", delta:number} — scroll up (-) or down (+)
66
218
  - press: {action:"press", key:string} — press Enter, Tab, etc.
67
219
  - done: {action:"done", summary:string} — all visible questions answered correctly
68
220
 
221
+ ERROR CORRECTION (CRITICAL):
222
+ - Check <page_context> for "Errors on screen" after each action
223
+ - If errors are visible (wrong answer, validation error, "Incorrect", "Wrong Answer", "Error"), FIX THEM immediately
224
+ - For text inputs with errors: clear and retype the correct answer
225
+ - For code editors with errors: retype the correct code using write_code
226
+ - For radio/checkbox with errors: click the correct option
227
+ - NEVER return "done" if errors are visible on screen
228
+ - Keep running until ALL questions are answered correctly with NO errors
229
+
230
+ CORRECTION MODE:
231
+ - The <page_context> block shows the current editor content — compare it to the expected answer
232
+ - If editor content is WRONG or INCOMPLETE, type the CORRECT code to replace it
233
+ - If you previously typed wrong code, use write_code to replace with correct code
234
+ - You may be asked the same question multiple times — each time, check if the current answer is correct before acting
235
+
236
+ PERSONAL QUESTIONS — SKIP THEM:
237
+ - NEVER type answers for questions asking: name, email, phone, address, SSN, password, credit card, date of birth, or any personal information
238
+ - If a question asks for personal info, do NOT click or type — skip it entirely
239
+ - If ALL remaining questions are personal, return done
240
+
69
241
  RESPONSE FORMAT RULES (CRITICAL — follow exactly):
70
242
  - Return ONLY a valid JSON array. No explanations, no markdown fences, no extra text.
71
243
  - The response MUST be parseable by JSON.parse(). Test mentally: is your output valid JSON?
@@ -121,11 +293,18 @@ async function cmdBrowse(flags) {
121
293
  return;
122
294
  }
123
295
  const port = parseInt(flags.get('port') ?? '9222', 10);
124
- const maxActions = parseInt(flags.get('max-actions') ?? '50', 10);
296
+ const continuous = flags.has('continuous');
297
+ const maxActions = continuous ? Number.MAX_SAFE_INTEGER : parseInt(flags.get('max-actions') ?? '50', 10);
125
298
  const intervalMs = parseInt(flags.get('interval') ?? '2000', 10);
126
299
  const modelSpec = flags.get('model') ?? 'mimo:mimo-v2.5';
127
- const timeoutSec = Math.min(Math.max(parseInt(flags.get('timeout') ?? '180', 10), 30), 600);
300
+ const timeoutSec = Math.min(Math.max(parseInt(flags.get('timeout') ?? '300', 10), 30), 600);
128
301
  const timeoutMs = timeoutSec * 1000;
302
+ const quality = Math.min(Math.max(parseInt(flags.get('quality') ?? '92', 10), 50), 100);
303
+ const maxConsecutiveFailures = parseInt(flags.get('max-consecutive-failures') ?? '0', 10); // 0 = unlimited
304
+ const charDelayMs = Math.min(Math.max(parseInt(flags.get('char-delay') ?? '150', 10), 5), 200);
305
+ const optimize = flags.has('optimize');
306
+ const stealth = flags.has('stealth');
307
+ const hybrid = flags.has('hybrid');
129
308
  // Resolve LLM endpoint
130
309
  const resolved = (0, models_1.resolveModel)(modelSpec, process.env.LLM_BASE_URL);
131
310
  const apiKey = (0, models_1.resolveApiKeyForProvider)(modelSpec);
@@ -138,9 +317,21 @@ async function cmdBrowse(flags) {
138
317
  const model = resolved.model;
139
318
  console.log(`[browse] Model: ${modelSpec} -> ${model} @ ${baseUrl}`);
140
319
  console.log(`[browse] Chrome port: ${port}`);
141
- console.log(`[browse] Max actions: ${maxActions}`);
320
+ console.log(`[browse] Mode: ${stealth ? 'STEALTH (OS-level, no CDP)' : hybrid ? 'HYBRID (CDP reads + OS-level writes)' : continuous ? 'continuous (runs until Ctrl+C)' : `capped at ${maxActions} actions`}`);
142
321
  console.log(`[browse] LLM timeout: ${timeoutSec}s`);
322
+ if (!stealth) {
323
+ console.log(`[browse] Screenshot quality: ${quality}`);
324
+ }
325
+ console.log(`[browse] Char delay: ${charDelayMs}ms`);
143
326
  console.log(`[browse] Screenshot interval: ${intervalMs}ms`);
327
+ if (!stealth) {
328
+ console.log(`[browse] Max consecutive failures: ${maxConsecutiveFailures === 0 ? 'unlimited' : maxConsecutiveFailures}`);
329
+ }
330
+ // ── Stealth mode: bypass CDP entirely ──
331
+ if (stealth) {
332
+ await runStealthLoop(baseUrl, apiKey, model, maxActions, intervalMs, timeoutMs, charDelayMs, continuous);
333
+ return;
334
+ }
144
335
  // Connect directly to the page's WebSocket (not browser-level)
145
336
  let page;
146
337
  try {
@@ -163,79 +354,344 @@ async function cmdBrowse(flags) {
163
354
  }
164
355
  // Agent loop
165
356
  let actionCount = 0;
357
+ let llmCallCount = 0;
166
358
  let stopped = false;
167
359
  let consecutiveFailures = 0;
168
- const maxRetries = 3;
169
- const maxConsecutiveFailures = 10;
360
+ let lastScreenshot = null;
361
+ let lastFingerprint = null;
362
+ let codeConfirmedOptimal = false;
363
+ const maxRetries = 5;
170
364
  process.on('SIGINT', () => {
171
365
  console.log('\n[browse] Stopped by user');
172
366
  stopped = true;
173
367
  });
174
368
  process.on('SIGTERM', () => { stopped = true; });
175
- console.log('[browse] Agent started. Press Ctrl+C to stop.\n');
369
+ console.log(`[browse] Agent started. ${continuous ? 'Running until Ctrl+C.' : `Max ${maxActions} actions.`} Press Ctrl+C to stop.\n`);
176
370
  while (!stopped && actionCount < maxActions) {
177
371
  try {
178
372
  // Take screenshot and extract elements in parallel
179
- const [screenshot, elements] = await Promise.all([
180
- captureScreenshot(page),
373
+ const [screenshot, elements, pageContext] = await Promise.all([
374
+ captureScreenshot(page, quality),
181
375
  extractElements(page),
376
+ extractPageContext(page),
182
377
  ]);
183
378
  if (!screenshot) {
184
379
  console.error('[browse] Failed to capture screenshot, retrying...');
185
380
  await sleep(intervalMs);
186
381
  continue;
187
382
  }
383
+ // Detect screen change via question fingerprint comparison
384
+ const currentFingerprint = createFingerprint(elements);
385
+ if (lastFingerprint) {
386
+ const changeScore = calculateChangeScore(lastFingerprint, currentFingerprint);
387
+ if (changeScore >= 5) {
388
+ console.log(`[browse] Screen changed (score: ${changeScore}) — resetting state`);
389
+ codeConfirmedOptimal = false;
390
+ consecutiveFailures = 0;
391
+ }
392
+ }
393
+ lastFingerprint = currentFingerprint;
394
+ lastScreenshot = screenshot;
188
395
  console.log(`[browse] Found ${elements.length} interactive elements`);
189
396
  // Send to vision LLM with retry logic
190
- console.log(`[browse] [${actionCount + 1}/${maxActions}] Analyzing screenshot...`);
397
+ const actionLabel = continuous ? `${actionCount + 1}/∞` : `${actionCount + 1}/${maxActions}`;
398
+ console.log(`[browse] [${actionLabel}] Analyzing screenshot...`);
191
399
  let result = null;
192
400
  for (let attempt = 1; attempt <= maxRetries; attempt++) {
193
- result = await analyzeScreenshot(baseUrl, apiKey, model, screenshot, elements, timeoutMs, attempt, maxRetries);
401
+ result = await analyzeScreenshot(baseUrl, apiKey, model, screenshot, elements, pageContext, timeoutMs, attempt, maxRetries);
402
+ llmCallCount++;
403
+ if (llmCallCount > 50 && llmCallCount % 25 === 0) {
404
+ console.log(`[browse] Warning: ${llmCallCount} LLM calls made — may hit rate limits`);
405
+ }
194
406
  if (result.actions.length > 0)
195
407
  break; // success
408
+ // Timeout → exponential backoff, keep retrying (never count as failure)
409
+ if (result.error === 'timeout') {
410
+ const backoff = Math.min(30 * Math.pow(2, attempt - 1), 300); // 30, 60, 120, 240, 300
411
+ console.log(`[browse] LLM timed out — retrying in ${backoff}s (attempt ${attempt}/${maxRetries})...`);
412
+ await sleep(backoff * 1000);
413
+ continue;
414
+ }
415
+ // Other errors → short backoff
196
416
  if (attempt < maxRetries) {
197
417
  const backoff = attempt === 1 ? 3 : 5;
198
- console.log(`[browse] ${result.error === 'timeout' ? 'LLM timed out' : result.error === 'empty' ? 'Empty response' : 'Parse failed'} (attempt ${attempt}/${maxRetries}), retrying in ${backoff}s...`);
418
+ console.log(`[browse] ${result.error === 'empty' ? 'Empty response' : 'Parse failed'} (attempt ${attempt}/${maxRetries}), retrying in ${backoff}s...`);
199
419
  await sleep(backoff * 1000);
200
420
  }
201
421
  }
202
422
  if (!result || result.actions.length === 0) {
203
423
  consecutiveFailures++;
204
424
  console.log(`[browse] No valid response after ${maxRetries} attempts (${consecutiveFailures} consecutive failures)`);
205
- if (consecutiveFailures >= maxConsecutiveFailures) {
425
+ if (maxConsecutiveFailures > 0 && consecutiveFailures >= maxConsecutiveFailures) {
206
426
  console.error(`[browse] Too many consecutive failures (${consecutiveFailures}), stopping agent`);
207
427
  break;
208
428
  }
209
- if (consecutiveFailures >= 5) {
210
- console.log(`[browse] Warning: ${consecutiveFailures} consecutive failures — agent may be stuck`);
429
+ if (consecutiveFailures >= 5 && consecutiveFailures % 5 === 0) {
430
+ console.log(`[browse] Warning: ${consecutiveFailures} consecutive failures — still running`);
211
431
  }
212
432
  await sleep(intervalMs);
213
433
  continue;
214
434
  }
215
435
  // Success — reset consecutive failure counter
216
436
  consecutiveFailures = 0;
217
- // Process all actions with 2s natural delay between clicks
218
- let allAnswered = false;
437
+ // Pre-execution check: detect if screen changed during LLM call
438
+ const preActions = result.actions.filter((a) => a.action !== 'done');
439
+ if (preActions.length > 0) {
440
+ const [freshScreenshot, freshElements] = await Promise.all([
441
+ captureScreenshot(page, quality),
442
+ extractElements(page),
443
+ ]);
444
+ if (freshScreenshot) {
445
+ const freshFingerprint = createFingerprint(freshElements);
446
+ if (lastFingerprint) {
447
+ const changeScore = calculateChangeScore(lastFingerprint, freshFingerprint);
448
+ if (changeScore >= 5) {
449
+ console.log(`[browse] Screen changed during LLM call (score: ${changeScore}) — skipping actions, re-analyzing`);
450
+ lastFingerprint = freshFingerprint;
451
+ lastScreenshot = freshScreenshot;
452
+ await sleep(intervalMs);
453
+ continue;
454
+ }
455
+ }
456
+ }
457
+ }
458
+ // Batch multiple write_code actions — use last one (complete solution)
459
+ const writeCodeActions = result.actions.filter((a) => a.action === 'write_code');
460
+ if (writeCodeActions.length > 1) {
461
+ result.actions = result.actions.filter((a) => a.action !== 'write_code');
462
+ result.actions.push(writeCodeActions[writeCodeActions.length - 1]);
463
+ console.log(`[browse] ${writeCodeActions.length} write_code actions — using last one (complete solution)`);
464
+ }
465
+ // Process all actions
466
+ let executedAny = false;
467
+ let wroteCode = false;
219
468
  for (const act of result.actions) {
220
469
  if (act.action === 'done') {
221
- console.log(`[browse] Done: ${act.summary || 'All questions answered'}`);
222
- allAnswered = true;
223
- break;
470
+ // Validate: are there actually unanswered questions?
471
+ if (hasUnansweredQuestions(elements)) {
472
+ console.log(`[browse] LLM says done but unanswered questions remain — retrying...`);
473
+ // Force retry with stronger prompt
474
+ result = await analyzeScreenshot(baseUrl, apiKey, model, screenshot, elements, pageContext, timeoutMs, 1, 1);
475
+ if (result.actions.length > 0 && result.actions[0].action !== 'done') {
476
+ // LLM returned actual actions — re-process from the start of this loop
477
+ executedAny = false;
478
+ wroteCode = false;
479
+ // Fall through to process new actions below
480
+ }
481
+ else {
482
+ // LLM still says done — trust it this time and continue
483
+ console.log(`[browse] LLM confirms done after verification`);
484
+ if (continuous && !executedAny) {
485
+ const changed = await waitForScreenChange(page, lastScreenshot ?? screenshot, intervalMs, quality, () => stopped);
486
+ if (!changed)
487
+ break;
488
+ executedAny = false;
489
+ }
490
+ }
491
+ continue;
492
+ }
493
+ // All questions genuinely answered
494
+ if (continuous && !executedAny) {
495
+ console.log(`[browse] LLM reports done: ${act.summary || 'All questions answered'}`);
496
+ // Log correctness indicators from page context
497
+ const correctIndicators = pageContext.match(/correct|accepted|✓|passed|success|well done/gi);
498
+ const wrongIndicators = pageContext.match(/incorrect|wrong|✗|failed|error|try again/gi);
499
+ if (wrongIndicators) {
500
+ console.log(`[browse] Errors detected on screen: ${wrongIndicators.join(', ')} — should be fixed`);
501
+ }
502
+ else if (correctIndicators) {
503
+ console.log(`[browse] Answer verified correct: ${correctIndicators.join(', ')}`);
504
+ }
505
+ const changed = await waitForScreenChange(page, lastScreenshot ?? screenshot, intervalMs, quality, () => stopped);
506
+ if (!changed)
507
+ break;
508
+ executedAny = false;
509
+ }
510
+ else {
511
+ console.log(`[browse] LLM reports done: ${act.summary || 'All questions answered'} — continuing to verify...`);
512
+ }
513
+ continue;
514
+ }
515
+ // Skip if code was already confirmed optimal (prevents re-typing after optimization pass)
516
+ if ((act.action === 'write_code' || act.action === 'type') && codeConfirmedOptimal) {
517
+ console.log(`[browse] Skipping ${act.action} — code already confirmed optimal`);
518
+ continue;
519
+ }
520
+ // Ask LLM to decide if action is redundant
521
+ const redundancy = await isActionRedundant(baseUrl, apiKey, model, page, elements, act, timeoutMs);
522
+ if (redundancy.decision === 'skip') {
523
+ continue;
224
524
  }
225
- console.log(`[browse] Action: ${act.action} - ${act.reasoning || ''}`);
226
- await executeAction(page, act, elements);
525
+ if (redundancy.decision === 'partial' && (act.action === 'write_code' || act.action === 'type')) {
526
+ // Partial update — only retype the different part
527
+ const targetCode = act.action === 'write_code' ? act.code : act.text;
528
+ if (targetCode) {
529
+ console.log(`[browse] Partial update — only retying changed portion...`);
530
+ const currentContent = await readEditorContent(page);
531
+ const normCurrent = normalizeCode(currentContent);
532
+ const normTarget = normalizeCode(targetCode);
533
+ const commonLen = findCommonPrefix(normCurrent, normTarget);
534
+ // Find position in original code that corresponds to end of common prefix
535
+ const suffix = targetCode.slice(Math.min(commonLen, targetCode.length));
536
+ if (suffix.length > 0) {
537
+ // Position cursor at end, type only the suffix
538
+ await cdpSend(page, 'Input.dispatchKeyEvent', {
539
+ type: 'keyDown', key: 'End', code: 'End', modifiers: 2,
540
+ windowsVirtualKeyCode: 35, nativeVirtualKeyCode: 35,
541
+ });
542
+ await cdpSend(page, 'Input.dispatchKeyEvent', {
543
+ type: 'keyUp', key: 'End', code: 'End', modifiers: 2,
544
+ windowsVirtualKeyCode: 35, nativeVirtualKeyCode: 35,
545
+ });
546
+ await sleep(100);
547
+ // Select from cursor to end and overwrite
548
+ await cdpSend(page, 'Input.dispatchKeyEvent', {
549
+ type: 'keyDown', key: 'Shift', code: 'ShiftRight', modifiers: 2,
550
+ windowsVirtualKeyCode: 16, nativeVirtualKeyCode: 16,
551
+ });
552
+ // Move to end of document
553
+ for (let i = 0; i < 50; i++) {
554
+ await cdpSend(page, 'Input.dispatchKeyEvent', {
555
+ type: 'keyDown', key: 'ArrowRight', code: 'ArrowRight',
556
+ windowsVirtualKeyCode: 39, nativeVirtualKeyCode: 39,
557
+ });
558
+ }
559
+ await cdpSend(page, 'Input.dispatchKeyEvent', {
560
+ type: 'keyUp', key: 'Shift', code: 'ShiftRight', modifiers: 2,
561
+ windowsVirtualKeyCode: 16, nativeVirtualKeyCode: 16,
562
+ });
563
+ // Delete selection
564
+ await cdpSend(page, 'Input.dispatchKeyEvent', {
565
+ type: 'keyDown', key: 'Backspace', code: 'Backspace',
566
+ windowsVirtualKeyCode: 8, nativeVirtualKeyCode: 8,
567
+ });
568
+ await cdpSend(page, 'Input.dispatchKeyEvent', {
569
+ type: 'keyUp', key: 'Backspace', code: 'Backspace',
570
+ windowsVirtualKeyCode: 8, nativeVirtualKeyCode: 8,
571
+ });
572
+ await sleep(100);
573
+ // Type the suffix
574
+ for (let i = 0; i < suffix.length; i++) {
575
+ const char = suffix[i];
576
+ if (char === '\n') {
577
+ await cdpSend(page, 'Input.dispatchKeyEvent', {
578
+ type: 'keyDown', key: 'Enter', code: 'Enter', text: '\r',
579
+ windowsVirtualKeyCode: 13, nativeVirtualKeyCode: 13,
580
+ });
581
+ await cdpSend(page, 'Input.dispatchKeyEvent', {
582
+ type: 'keyUp', key: 'Enter', code: 'Enter',
583
+ windowsVirtualKeyCode: 13, nativeVirtualKeyCode: 13,
584
+ });
585
+ }
586
+ else {
587
+ const k = charToKey(char);
588
+ const mods = k.shift ? 2 : undefined;
589
+ const downParams = { type: 'keyDown', key: k.key, code: k.code, text: char };
590
+ const upParams = { type: 'keyUp', key: k.key, code: k.code };
591
+ if (mods !== undefined) {
592
+ downParams.modifiers = mods;
593
+ upParams.modifiers = mods;
594
+ }
595
+ await cdpSend(page, 'Input.dispatchKeyEvent', downParams);
596
+ await cdpSend(page, 'Input.dispatchKeyEvent', upParams);
597
+ }
598
+ await sleep(variableDelay(charDelayMs, char, i, suffix.length));
599
+ }
600
+ console.log(`[browse] Partial update complete (${suffix.length} chars typed)`);
601
+ }
602
+ actionCount++;
603
+ executedAny = true;
604
+ wroteCode = true;
605
+ }
606
+ continue;
607
+ }
608
+ // Full update — execute the action normally
609
+ if (act.action === 'type') {
610
+ console.log(`[browse] Action: type — ${act.text?.slice(0, 120)}${(act.text?.length ?? 0) > 120 ? '...' : ''}`);
611
+ }
612
+ else if (act.action === 'write_code') {
613
+ console.log(`[browse] Action: write_code — ${act.code?.slice(0, 120)}${(act.code?.length ?? 0) > 120 ? '...' : ''}`);
614
+ }
615
+ else {
616
+ console.log(`[browse] Action: ${act.action} - ${act.reasoning || ''}`);
617
+ }
618
+ await (hybrid ? executeActionHybrid : executeAction)(page, act, elements, charDelayMs);
227
619
  actionCount++;
228
- // 2s delay between clicks for natural feel
229
- if (act.action === 'click' || act.action === 'type') {
620
+ executedAny = true;
621
+ if (act.action === 'write_code' || act.action === 'type') {
622
+ wroteCode = true;
623
+ }
624
+ // Natural delay between actions
625
+ if (act.action === 'click' || act.action === 'type' || act.action === 'write_code') {
230
626
  await sleep(2000);
231
627
  }
232
628
  }
233
- // Wait between LLM calls
234
- if (allAnswered) {
235
- console.log('[browse] All answered, waiting for page change...');
236
- await sleep(1500);
629
+ // Optimization pass: if --optimize and code was written, ask LLM to optimize it
630
+ if (optimize && wroteCode) {
631
+ console.log('[browse] Running optimization pass...');
632
+ const [newScreenshot, newElements, newPageContext] = await Promise.all([
633
+ captureScreenshot(page, quality),
634
+ extractElements(page),
635
+ extractPageContext(page),
636
+ ]);
637
+ if (newScreenshot) {
638
+ const optResult = await analyzeScreenshotWithPrompt(baseUrl, apiKey, model, newScreenshot, newElements, newPageContext, timeoutMs, 1, 1, `You are a code optimization expert. The code in the editor is CORRECT but may not be EFFICIENT.
639
+
640
+ Review the code and check:
641
+ 1. Is it the most time-efficient approach? (O(n log n) or better?)
642
+ 2. Are there better data structures to use?
643
+ 3. Can nested loops be replaced with HashMap/Set lookups?
644
+ 4. Can the solution be optimized with sliding window, two pointers, or binary search?
645
+
646
+ If the code is ALREADY optimal (top-tier efficiency), return: [{"action":"done","summary":"Code is already optimal"}]
647
+
648
+ If you can optimize it, return the FULL optimized code using write_code. Focus ONLY on performance — do not change correctness.
649
+ Do NOT simplify variable names or change the function signature. Only optimize the algorithm.`);
650
+ if (optResult.actions.length > 0 && optResult.actions[0].action !== 'done') {
651
+ const optAct = optResult.actions[0];
652
+ if ((optAct.action === 'write_code' || optAct.action === 'type') && optAct.code) {
653
+ console.log(`[browse] Optimization found — rewriting code...`);
654
+ await (hybrid ? executeActionHybrid : executeAction)(page, { action: 'click', elementIndex: optAct.elementIndex ?? result.actions[0].elementIndex }, newElements, charDelayMs);
655
+ await sleep(500);
656
+ await (hybrid ? executeActionHybrid : executeAction)(page, optAct, newElements, charDelayMs);
657
+ actionCount++;
658
+ await sleep(2000);
659
+ }
660
+ else {
661
+ console.log('[browse] No actionable optimization returned — keeping original');
662
+ }
663
+ }
664
+ else {
665
+ console.log('[browse] Code is already optimal — no changes needed');
666
+ codeConfirmedOptimal = true;
667
+ }
668
+ }
669
+ }
670
+ // Cooldown after code was written — give user time to see result and navigate
671
+ if (wroteCode) {
672
+ console.log('[browse] Code written — cooling down 30s before next check...');
673
+ await sleep(30000);
674
+ // After cooldown, check for test results on screen
675
+ const [, postElements, postContext] = await Promise.all([
676
+ captureScreenshot(page, quality),
677
+ extractElements(page),
678
+ extractPageContext(page),
679
+ ]);
680
+ if (postContext) {
681
+ const failedIndicators = postContext.match(/wrong answer|time limit exceeded|runtime error|compile error|undefined is not|cannot read property|segmentation fault|n\s*\/\s*m/gi);
682
+ const passedIndicators = postContext.match(/accepted|all test cases passed|✓ passed|runtime:\s*\d+\s*ms/gi);
683
+ if (failedIndicators) {
684
+ console.log(`[browse] Test FAILED: ${failedIndicators.join(', ')} — will retry this question`);
685
+ codeConfirmedOptimal = false; // allow re-typing with fix
686
+ // Don't wait for screen change — loop again to fix the error
687
+ }
688
+ else if (passedIndicators) {
689
+ console.log(`[browse] Test PASSED: ${passedIndicators.join(', ')}`);
690
+ }
691
+ }
237
692
  }
238
693
  else {
694
+ // Wait between LLM calls
239
695
  await sleep(2000);
240
696
  }
241
697
  }
@@ -244,7 +700,7 @@ async function cmdBrowse(flags) {
244
700
  await sleep(intervalMs);
245
701
  }
246
702
  }
247
- if (actionCount >= maxActions) {
703
+ if (actionCount >= maxActions && !continuous) {
248
704
  console.log(`[browse] Reached max actions (${maxActions})`);
249
705
  }
250
706
  console.log(`[browse] Agent finished. ${actionCount} actions performed.`);
@@ -330,9 +786,9 @@ function cdpSend(page, method, params) {
330
786
  page.ws.send(JSON.stringify({ id, method, params }));
331
787
  });
332
788
  }
333
- async function captureScreenshot(page) {
789
+ async function captureScreenshot(page, quality) {
334
790
  try {
335
- const result = await cdpSend(page, 'Page.captureScreenshot', { format: 'jpeg', quality: 70 });
791
+ const result = await cdpSend(page, 'Page.captureScreenshot', { format: 'jpeg', quality });
336
792
  return result?.data ?? null;
337
793
  }
338
794
  catch (err) {
@@ -383,6 +839,7 @@ async function extractElements(page) {
383
839
  checked: el.checked || el.getAttribute('aria-checked') === 'true' || false,
384
840
  x: Math.round(rect.x + rect.width / 2),
385
841
  y: Math.round(rect.y + rect.height / 2),
842
+ inViewport: rect.bottom >= 0 && rect.top <= window.innerHeight && rect.right >= 0 && rect.left <= window.innerWidth,
386
843
  groupId: null,
387
844
  });
388
845
  });
@@ -490,6 +947,7 @@ async function extractElements(page) {
490
947
  x: item.x,
491
948
  y: item.y,
492
949
  group: item.groupId,
950
+ inViewport: item.inViewport,
493
951
  });
494
952
  });
495
953
 
@@ -507,17 +965,29 @@ async function extractElements(page) {
507
965
  return [];
508
966
  }
509
967
  }
510
- async function executeAction(page, act, elements) {
968
+ async function executeAction(page, act, elements, charDelayMs) {
511
969
  try {
512
970
  switch (act.action) {
513
971
  case 'click': {
514
972
  if (act.elementIndex === undefined)
515
973
  return false;
516
- const el = elements.find((e) => e.i === act.elementIndex);
974
+ let el = elements.find((e) => e.i === act.elementIndex);
517
975
  if (!el) {
518
976
  console.log(`[browse] Element index ${act.elementIndex} not found, will retry`);
519
977
  return false;
520
978
  }
979
+ // Scroll off-screen element into view before clicking
980
+ if (!el.inViewport) {
981
+ console.log(`[browse] Element ${el.i} off-screen — scrolling to "${el.text?.slice(0, 30)}"...`);
982
+ await scrollElementIntoView(page, el);
983
+ // Re-extract elements to get updated coordinates
984
+ const updatedElements = await extractElements(page);
985
+ const updatedEl = updatedElements.find((e) => e.text === el.text && e.type === el.type && e.group === el.group);
986
+ if (updatedEl) {
987
+ el = updatedEl;
988
+ console.log(`[browse] Scrolled — element now at index ${el.i}, position (${el.x},${el.y})`);
989
+ }
990
+ }
521
991
  console.log(`[browse] Clicking [${el.i}] ${el.tag} "${el.text.slice(0, 60)}" at (${el.x},${el.y})`);
522
992
  await cdpSend(page, 'Input.dispatchMouseEvent', {
523
993
  type: 'mousePressed', x: el.x, y: el.y, button: 'left', clickCount: 1,
@@ -537,140 +1007,52 @@ async function executeAction(page, act, elements) {
537
1007
  }
538
1008
  // 1. Detect editor type
539
1009
  const editor = await detectEditor(page);
540
- const isMonaco = editor?.type === 'monaco';
541
1010
  const isCodeEditor = editor !== null;
542
1011
  if (editor)
543
1012
  console.log(`[browse] Detected editor: ${editor.type}`);
544
1013
  // 2. Unescape LLM text: convert literal \n, \t, \\", etc. to real characters
545
- const text = act.text
1014
+ const rawText = act.text
546
1015
  .replace(/\\n/g, '\n')
547
1016
  .replace(/\\t/g, '\t')
548
1017
  .replace(/\\"/g, '"')
549
1018
  .replace(/\\\\/g, '\\');
1019
+ // 3. Normalize indentation for code editors (LLM often adds extra indent)
1020
+ const text = isCodeEditor ? normalizeCodeIndent(rawText) : rawText;
550
1021
  console.log(`[browse] Typing ${text.length} characters...`);
551
- if (isMonaco) {
552
- // ── Monaco path: keyboard-only, no mouse clicks (clicks select content) ──
553
- // 1. Focus the hidden textarea via JS
554
- await focusSelector(page, editor.selector);
555
- await sleep(200);
556
- // 2. Clear: Ctrl+A → wait → Backspace (works when textarea has focus)
557
- await cdpSend(page, 'Input.dispatchKeyEvent', {
558
- type: 'keyDown', key: 'a', code: 'KeyA', modifiers: 2,
559
- windowsVirtualKeyCode: 65, nativeVirtualKeyCode: 65,
560
- });
561
- await cdpSend(page, 'Input.dispatchKeyEvent', {
562
- type: 'keyUp', key: 'a', code: 'KeyA', modifiers: 2,
563
- windowsVirtualKeyCode: 65, nativeVirtualKeyCode: 65,
564
- });
565
- await sleep(200);
566
- await cdpSend(page, 'Input.dispatchKeyEvent', {
567
- type: 'keyDown', key: 'Backspace', code: 'Backspace',
568
- windowsVirtualKeyCode: 8, nativeVirtualKeyCode: 8,
569
- });
570
- await cdpSend(page, 'Input.dispatchKeyEvent', {
571
- type: 'keyUp', key: 'Backspace', code: 'Backspace',
572
- windowsVirtualKeyCode: 8, nativeVirtualKeyCode: 8,
573
- });
574
- await sleep(200);
575
- // 3. Re-focus textarea (Monaco may have stolen focus during Ctrl+A/Backspace)
576
- await focusSelector(page, editor.selector);
577
- await sleep(100);
578
- // 4. Insert full text via Input.insertText — works for ALL chars including \n
579
- await cdpSend(page, 'Input.insertText', { text });
1022
+ if (isCodeEditor) {
1023
+ await insertIntoCodeEditor(page, editor.selector, text, charDelayMs);
1024
+ }
1025
+ else {
1026
+ await insertIntoGenericInput(page, el, text, charDelayMs);
580
1027
  }
581
- else if (isCodeEditor) {
582
- // ── Other code editors: focus via JS .focus(), char-by-char ──
583
- await focusSelector(page, editor.selector);
584
- await sleep(200);
585
- // Ctrl+A + Backspace to clear
586
- await cdpSend(page, 'Input.dispatchKeyEvent', {
587
- type: 'keyDown', key: 'a', code: 'KeyA', modifiers: 2,
588
- windowsVirtualKeyCode: 65, nativeVirtualKeyCode: 65,
589
- });
590
- await cdpSend(page, 'Input.dispatchKeyEvent', {
591
- type: 'keyUp', key: 'a', code: 'KeyA', modifiers: 2,
592
- windowsVirtualKeyCode: 65, nativeVirtualKeyCode: 65,
593
- });
594
- await cdpSend(page, 'Input.dispatchKeyEvent', {
595
- type: 'keyDown', key: 'Backspace', code: 'Backspace',
596
- windowsVirtualKeyCode: 8, nativeVirtualKeyCode: 8,
597
- });
598
- await cdpSend(page, 'Input.dispatchKeyEvent', {
599
- type: 'keyUp', key: 'Backspace', code: 'Backspace',
600
- windowsVirtualKeyCode: 8, nativeVirtualKeyCode: 8,
601
- });
602
- await sleep(200);
603
- // Type entire text via insertText
604
- await cdpSend(page, 'Input.insertText', { text });
1028
+ return true;
1029
+ }
1030
+ case 'write_code': {
1031
+ if (act.elementIndex === undefined || !act.code)
1032
+ break;
1033
+ const el = elements.find((e) => e.i === act.elementIndex);
1034
+ if (!el) {
1035
+ console.log(`[browse] Element index ${act.elementIndex} not found, will retry`);
1036
+ return false;
1037
+ }
1038
+ // 1. Detect editor type
1039
+ const editor = await detectEditor(page);
1040
+ const isCodeEditor = editor !== null;
1041
+ if (editor)
1042
+ console.log(`[browse] Detected editor: ${editor.type}`);
1043
+ // 2. Unescape LLM code: convert literal \n, \t, \\", etc. to real characters
1044
+ const text = act.code
1045
+ .replace(/\\n/g, '\n')
1046
+ .replace(/\\t/g, '\t')
1047
+ .replace(/\\"/g, '"')
1048
+ .replace(/\\\\/g, '\\');
1049
+ console.log(`[browse] Writing code (${text.length} chars)...`);
1050
+ // 3. write_code inserts EXACTLY as-is — no normalizeCodeIndent
1051
+ if (isCodeEditor) {
1052
+ await insertIntoCodeEditor(page, editor.selector, text, charDelayMs);
605
1053
  }
606
1054
  else {
607
- // ── Generic inputs: click to focus, char-by-char with keyDown/keyUp ──
608
- await cdpSend(page, 'Input.dispatchMouseEvent', {
609
- type: 'mousePressed', x: el.x, y: el.y, button: 'left', clickCount: 1,
610
- });
611
- await cdpSend(page, 'Input.dispatchMouseEvent', {
612
- type: 'mouseReleased', x: el.x, y: el.y, button: 'left', clickCount: 1,
613
- });
614
- await sleep(200);
615
- // Ctrl+A + Backspace to clear
616
- await cdpSend(page, 'Input.dispatchKeyEvent', {
617
- type: 'keyDown', key: 'a', code: 'KeyA', modifiers: 2,
618
- windowsVirtualKeyCode: 65, nativeVirtualKeyCode: 65,
619
- });
620
- await cdpSend(page, 'Input.dispatchKeyEvent', {
621
- type: 'keyUp', key: 'a', code: 'KeyA', modifiers: 2,
622
- windowsVirtualKeyCode: 65, nativeVirtualKeyCode: 65,
623
- });
624
- await cdpSend(page, 'Input.dispatchKeyEvent', {
625
- type: 'keyDown', key: 'Backspace', code: 'Backspace',
626
- windowsVirtualKeyCode: 8, nativeVirtualKeyCode: 8,
627
- });
628
- await cdpSend(page, 'Input.dispatchKeyEvent', {
629
- type: 'keyUp', key: 'Backspace', code: 'Backspace',
630
- windowsVirtualKeyCode: 8, nativeVirtualKeyCode: 8,
631
- });
632
- await sleep(200);
633
- // Char-by-char typing with natural timing
634
- for (let i = 0; i < text.length; i++) {
635
- const char = text[i];
636
- if (char === '\n') {
637
- await cdpSend(page, 'Input.dispatchKeyEvent', {
638
- type: 'keyDown', key: 'Enter', code: 'Enter',
639
- windowsVirtualKeyCode: 13, nativeVirtualKeyCode: 13,
640
- });
641
- await cdpSend(page, 'Input.dispatchKeyEvent', {
642
- type: 'keyUp', key: 'Enter', code: 'Enter',
643
- windowsVirtualKeyCode: 13, nativeVirtualKeyCode: 13,
644
- });
645
- }
646
- else if (char === '\t') {
647
- await cdpSend(page, 'Input.dispatchKeyEvent', {
648
- type: 'keyDown', key: 'Tab', code: 'Tab',
649
- windowsVirtualKeyCode: 9, nativeVirtualKeyCode: 9,
650
- });
651
- await cdpSend(page, 'Input.dispatchKeyEvent', {
652
- type: 'keyUp', key: 'Tab', code: 'Tab',
653
- windowsVirtualKeyCode: 9, nativeVirtualKeyCode: 9,
654
- });
655
- }
656
- else {
657
- const k = charToKey(char);
658
- const mods = k.shift ? 2 : undefined;
659
- const downParams = {
660
- type: 'keyDown', key: k.key, code: k.code, text: char,
661
- };
662
- const upParams = {
663
- type: 'keyUp', key: k.key, code: k.code,
664
- };
665
- if (mods !== undefined) {
666
- downParams.modifiers = mods;
667
- upParams.modifiers = mods;
668
- }
669
- await cdpSend(page, 'Input.dispatchKeyEvent', downParams);
670
- await cdpSend(page, 'Input.dispatchKeyEvent', upParams);
671
- }
672
- await sleep(charDelay(char, i, text.length));
673
- }
1055
+ await insertIntoGenericInput(page, el, text, charDelayMs);
674
1056
  }
675
1057
  return true;
676
1058
  }
@@ -704,6 +1086,115 @@ async function executeAction(page, act, elements) {
704
1086
  return false;
705
1087
  }
706
1088
  }
1089
+ /**
1090
+ * Hybrid action executor: OS-level keyboard/mouse for writing (invisible to detection).
1091
+ * Same logic as executeAction but uses clickOS/typeHumanLike/scrollOS/pressKeyOS.
1092
+ */
1093
+ async function executeActionHybrid(page, act, elements, charDelayMs) {
1094
+ try {
1095
+ switch (act.action) {
1096
+ case 'click': {
1097
+ if (act.elementIndex === undefined)
1098
+ return false;
1099
+ let el = elements.find((e) => e.i === act.elementIndex);
1100
+ if (!el) {
1101
+ console.log(`[browse] Element index ${act.elementIndex} not found, will retry`);
1102
+ return false;
1103
+ }
1104
+ // Scroll off-screen element into view before clicking
1105
+ if (!el.inViewport) {
1106
+ console.log(`[browse] Element ${el.i} off-screen — scrolling to "${el.text?.slice(0, 30)}"...`);
1107
+ await scrollElementIntoView(page, el);
1108
+ // Re-extract elements to get updated coordinates
1109
+ const updatedElements = await extractElements(page);
1110
+ const updatedEl = updatedElements.find((e) => e.text === el.text && e.type === el.type && e.group === el.group);
1111
+ if (updatedEl) {
1112
+ el = updatedEl;
1113
+ console.log(`[browse] Scrolled — element now at index ${el.i}, position (${el.x},${el.y})`);
1114
+ }
1115
+ }
1116
+ console.log(`[browse] Clicking [${el.i}] ${el.tag} "${el.text.slice(0, 60)}" at (${el.x},${el.y})`);
1117
+ clickOS(el.x, el.y);
1118
+ return true;
1119
+ }
1120
+ case 'type': {
1121
+ if (act.elementIndex === undefined || !act.text)
1122
+ break;
1123
+ const el = elements.find((e) => e.i === act.elementIndex);
1124
+ if (!el) {
1125
+ console.log(`[browse] Element index ${act.elementIndex} not found, will retry`);
1126
+ return false;
1127
+ }
1128
+ const editor = await detectEditor(page);
1129
+ const isCodeEditor = editor !== null;
1130
+ if (editor)
1131
+ console.log(`[browse] Detected editor: ${editor.type}`);
1132
+ const rawText = act.text
1133
+ .replace(/\\n/g, '\n')
1134
+ .replace(/\\t/g, '\t')
1135
+ .replace(/\\"/g, '"')
1136
+ .replace(/\\\\/g, '\\');
1137
+ const text = isCodeEditor ? normalizeCodeIndent(rawText) : rawText;
1138
+ console.log(`[browse] Typing ${text.length} characters (OS-level)...`);
1139
+ await sleep(randomInt(200, 400));
1140
+ clickOS(el.x, el.y);
1141
+ await sleep(randomInt(100, 200));
1142
+ if (isCodeEditor) {
1143
+ pressKeyOS('ctrl+a');
1144
+ await sleep(randomInt(50, 100));
1145
+ }
1146
+ await typeHumanLike(text, charDelayMs);
1147
+ return true;
1148
+ }
1149
+ case 'write_code': {
1150
+ if (act.elementIndex === undefined || !act.code)
1151
+ break;
1152
+ const el = elements.find((e) => e.i === act.elementIndex);
1153
+ if (!el) {
1154
+ console.log(`[browse] Element index ${act.elementIndex} not found, will retry`);
1155
+ return false;
1156
+ }
1157
+ const editor = await detectEditor(page);
1158
+ const isCodeEditor = editor !== null;
1159
+ if (editor)
1160
+ console.log(`[browse] Detected editor: ${editor.type}`);
1161
+ const text = act.code
1162
+ .replace(/\\n/g, '\n')
1163
+ .replace(/\\t/g, '\t')
1164
+ .replace(/\\"/g, '"')
1165
+ .replace(/\\\\/g, '\\');
1166
+ console.log(`[browse] Writing code (${text.length} chars, OS-level)...`);
1167
+ await sleep(randomInt(200, 400));
1168
+ clickOS(el.x, el.y);
1169
+ await sleep(randomInt(100, 200));
1170
+ if (isCodeEditor) {
1171
+ pressKeyOS('ctrl+a');
1172
+ await sleep(randomInt(50, 100));
1173
+ }
1174
+ await typeHumanLike(text, charDelayMs);
1175
+ return true;
1176
+ }
1177
+ case 'scroll': {
1178
+ const delta = act.delta ?? 3;
1179
+ console.log(`[browse] Scrolling ${delta > 0 ? 'down' : 'up'} (${Math.abs(delta)} clicks, OS-level)...`);
1180
+ scrollOS(-delta);
1181
+ break;
1182
+ }
1183
+ case 'press': {
1184
+ if (!act.key)
1185
+ break;
1186
+ console.log(`[browse] Pressing ${act.key} (OS-level)...`);
1187
+ pressKeyOS(act.key.toLowerCase());
1188
+ break;
1189
+ }
1190
+ }
1191
+ return true;
1192
+ }
1193
+ catch (err) {
1194
+ console.error(`[browse] Action error: ${(0, config_1.msg)(err)}`);
1195
+ return false;
1196
+ }
1197
+ }
707
1198
  // ── Key mapping for CDP keyboard events ──────────────────────────────────────
708
1199
  /** Maps a character to its CDP key/code/shift properties. */
709
1200
  function charToKey(c) {
@@ -759,24 +1250,596 @@ function charToKey(c) {
759
1250
  // Fallback — use the char itself
760
1251
  return { key: c, code: c, shift: false };
761
1252
  }
762
- // ── Natural typing timing ────────────────────────────────────────────────────
763
- /** Delay in ms before typing the next character (average typer profile). */
764
- function charDelay(char, index, total) {
765
- const base = 80 + Math.random() * 90; // 80-170ms (35-55 WPM)
766
- // Spaces micro-pause at word boundary
1253
+ // ── Code indentation normalization ─────────────────────────────────────────
1254
+ /**
1255
+ * Strips the common leading whitespace from all non-empty lines.
1256
+ * Fixes LLM output that has consistent extra indentation (e.g. 8 spaces
1257
+ * instead of 4). The relative indentation between lines is preserved.
1258
+ */
1259
+ function normalizeCodeIndent(text) {
1260
+ const lines = text.split('\n');
1261
+ const nonEmpty = lines.filter((l) => l.trim().length > 0);
1262
+ if (nonEmpty.length <= 1)
1263
+ return text;
1264
+ // Find the minimum leading whitespace across all non-empty lines
1265
+ const minIndent = nonEmpty.reduce((min, line) => {
1266
+ const match = line.match(/^(\s*)/);
1267
+ return Math.min(min, match?.[1]?.length ?? 0);
1268
+ }, Infinity);
1269
+ if (minIndent === 0 || minIndent === Infinity)
1270
+ return text;
1271
+ return lines
1272
+ .map((line) => {
1273
+ if (line.trim().length === 0)
1274
+ return line;
1275
+ return line.slice(minIndent);
1276
+ })
1277
+ .join('\n');
1278
+ }
1279
+ // ── Editor insertion helpers ───────────────────────────────────────────────
1280
+ /** Clear editor content: Ctrl+A → Backspace. */
1281
+ async function clearEditor(page) {
1282
+ await cdpSend(page, 'Input.dispatchKeyEvent', {
1283
+ type: 'keyDown', key: 'a', code: 'KeyA', modifiers: 2,
1284
+ windowsVirtualKeyCode: 65, nativeVirtualKeyCode: 65,
1285
+ });
1286
+ await cdpSend(page, 'Input.dispatchKeyEvent', {
1287
+ type: 'keyUp', key: 'a', code: 'KeyA', modifiers: 2,
1288
+ windowsVirtualKeyCode: 65, nativeVirtualKeyCode: 65,
1289
+ });
1290
+ await sleep(100);
1291
+ await cdpSend(page, 'Input.dispatchKeyEvent', {
1292
+ type: 'keyDown', key: 'Backspace', code: 'Backspace',
1293
+ windowsVirtualKeyCode: 8, nativeVirtualKeyCode: 8,
1294
+ });
1295
+ await cdpSend(page, 'Input.dispatchKeyEvent', {
1296
+ type: 'keyUp', key: 'Backspace', code: 'Backspace',
1297
+ windowsVirtualKeyCode: 8, nativeVirtualKeyCode: 8,
1298
+ });
1299
+ }
1300
+ /** Read current editor content from Monaco/CM5/CM6/Ace. */
1301
+ async function readEditorContent(page) {
1302
+ try {
1303
+ const result = await cdpSend(page, 'Runtime.evaluate', {
1304
+ expression: `
1305
+ (() => {
1306
+ const monacoNs = typeof monaco !== 'undefined' ? monaco : window.monaco;
1307
+ const editors = monacoNs?.editor?.getEditors?.() || [];
1308
+ if (editors.length > 0) return editors[0].getValue();
1309
+ const cm6 = document.querySelector('.cm-content');
1310
+ if (cm6?.cmView?.view) return cm6.cmView.view.state.doc.toString();
1311
+ const cm5 = document.querySelector('.CodeMirror')?.CodeMirror;
1312
+ if (cm5) return cm5.getValue();
1313
+ const ace = document.querySelector('.ace_editor')?.env?.editor;
1314
+ if (ace) return ace.getValue();
1315
+ return '';
1316
+ })()
1317
+ `,
1318
+ returnByValue: true,
1319
+ });
1320
+ return result?.result?.value ?? '';
1321
+ }
1322
+ catch {
1323
+ return '';
1324
+ }
1325
+ }
1326
+ /** Scroll an off-screen element into view before clicking. */
1327
+ async function scrollElementIntoView(page, el) {
1328
+ const textSlice = el.text?.slice(0, 100) || '';
1329
+ await cdpSend(page, 'Runtime.evaluate', {
1330
+ expression: `
1331
+ (() => {
1332
+ const sels = [
1333
+ 'input[type="radio"]', 'input[type="checkbox"]',
1334
+ 'input[type="text"]', 'input[type="email"]', 'input[type="password"]',
1335
+ 'input:not([type])', 'textarea', 'select', 'button', 'a',
1336
+ '[role="radio"]', '[role="checkbox"]', '[role="button"]',
1337
+ '[role="option"]', '[role="tab"]', '[role="menuitem"]', 'label',
1338
+ ];
1339
+ const all = [...document.querySelectorAll(sels.join(','))];
1340
+ const target = all.find(el => {
1341
+ const t = (el.innerText || el.textContent || '').trim().replace(/\\s+/g, ' ').slice(0, 100);
1342
+ return t === ${JSON.stringify(textSlice)};
1343
+ });
1344
+ if (target) {
1345
+ target.scrollIntoView({ behavior: 'instant', block: 'center' });
1346
+ return true;
1347
+ }
1348
+ return false;
1349
+ })()
1350
+ `,
1351
+ returnByValue: true,
1352
+ });
1353
+ await sleep(300);
1354
+ }
1355
+ /** Check if there are genuinely unanswered questions — catches premature 'done' from LLM. */
1356
+ function hasUnansweredQuestions(elements) {
1357
+ const groups = new Map();
1358
+ for (const el of elements) {
1359
+ if (!groups.has(el.group))
1360
+ groups.set(el.group, []);
1361
+ groups.get(el.group).push(el);
1362
+ }
1363
+ for (const [, els] of groups) {
1364
+ const types = els.map((e) => e.type);
1365
+ const isRadio = types.some((t) => t === 'radio');
1366
+ const isCheckbox = types.some((t) => t === 'checkbox');
1367
+ const hasInput = els.some((e) => e.tag === 'input' || e.tag === 'textarea');
1368
+ const checkedCount = els.filter((e) => e.checked).length;
1369
+ if (isRadio && checkedCount === 0)
1370
+ return true;
1371
+ if (isCheckbox && checkedCount < els.length)
1372
+ return true;
1373
+ if (hasInput && !els.some((e) => e.text && e.tag === 'input'))
1374
+ return true;
1375
+ }
1376
+ return false;
1377
+ }
1378
+ /** Normalize code for fuzzy comparison — strip whitespace, normalize quotes. */
1379
+ function normalizeCode(code) {
1380
+ return code
1381
+ .replace(/\s+/g, '')
1382
+ .replace(/['`]/g, '"')
1383
+ .replace(/;$/gm, '')
1384
+ .trim();
1385
+ }
1386
+ /** Find how many leading characters two strings share. */
1387
+ function findCommonPrefix(a, b) {
1388
+ let i = 0;
1389
+ while (i < a.length && i < b.length && a[i] === b[i])
1390
+ i++;
1391
+ return i;
1392
+ }
1393
+ /** Ask the LLM to decide if an action is redundant — is the current code already correct? */
1394
+ async function isActionRedundant(baseUrl, apiKey, model, page, elements, action, timeoutMs) {
1395
+ // Click on already-selected radio → definitely redundant
1396
+ if (action.action === 'click' && action.elementIndex !== undefined) {
1397
+ const el = elements[action.elementIndex];
1398
+ if (el?.type === 'radio' && el.checked) {
1399
+ console.log(`[browse] Skipping click — radio already selected: "${el.text?.slice(0, 40)}"`);
1400
+ return { decision: 'skip', reason: 'Radio already selected' };
1401
+ }
1402
+ return { decision: 'full', reason: 'Action needed' };
1403
+ }
1404
+ // For write_code/type → read current content and ask LLM
1405
+ const targetCode = action.action === 'write_code' ? action.code : action.text;
1406
+ if (!targetCode || targetCode.length < 20) {
1407
+ return { decision: 'full', reason: 'Short input, always type' };
1408
+ }
1409
+ const currentContent = await readEditorContent(page);
1410
+ if (!currentContent) {
1411
+ return { decision: 'full', reason: 'Editor empty' };
1412
+ }
1413
+ // Quick exact check first
1414
+ if (currentContent === targetCode) {
1415
+ console.log(`[browse] Skipping ${action.action} — code is identical`);
1416
+ return { decision: 'skip', reason: 'Identical' };
1417
+ }
1418
+ // Quick normalized check (fast fallback, no API call needed)
1419
+ const normCurrent = normalizeCode(currentContent);
1420
+ const normTarget = normalizeCode(targetCode);
1421
+ if (normCurrent === normTarget) {
1422
+ console.log(`[browse] Skipping ${action.action} — code is semantically identical (formatting only)`);
1423
+ return { decision: 'skip', reason: 'Semantically identical' };
1424
+ }
1425
+ // Calculate match percentage for prefix check
1426
+ const commonLen = findCommonPrefix(normCurrent, normTarget);
1427
+ const matchPercent = Math.round((commonLen / Math.max(normCurrent.length, normTarget.length)) * 100);
1428
+ // If >90% match, it's likely just a small change → partial update
1429
+ if (matchPercent > 90) {
1430
+ console.log(`[browse] Code is ${matchPercent}% similar — partial update possible`);
1431
+ return { decision: 'partial', reason: `${matchPercent}% match`, matchPercent };
1432
+ }
1433
+ // Ask the LLM to judge
1434
+ console.log(`[browse] Code is ${matchPercent}% similar — asking LLM to judge...`);
1435
+ const prompt = `You are a code comparison expert. Two versions of code are provided.
1436
+
1437
+ CURRENT editor content (truncated to first 500 chars):
1438
+ \`\`\`
1439
+ ${currentContent.slice(0, 500)}
1440
+ \`\`\`
1441
+
1442
+ TARGET code to write (truncated to first 500 chars):
1443
+ \`\`\`
1444
+ ${targetCode.slice(0, 500)}
1445
+ \`\`\`
1446
+
1447
+ The user wants to write the TARGET code into the editor.
1448
+
1449
+ TASK: Compare these two code versions. Are they semantically the same (same logic, same algorithm)? Ignore whitespace, formatting, variable naming differences. Focus on LOGIC.
1450
+
1451
+ Return EXACTLY one JSON object:
1452
+ - {"decision":"skip","reason":"..."} — if the current code is already correct (same logic)
1453
+ - {"decision":"partial","reason":"..."} — if mostly correct, only small changes needed
1454
+ - {"decision":"full","reason":"..."} — if completely different code is needed
1455
+
1456
+ Return ONLY the JSON. No other text.`;
1457
+ try {
1458
+ const body = {
1459
+ model,
1460
+ messages: [
1461
+ { role: 'system', content: prompt },
1462
+ { role: 'user', content: 'Compare the two code versions and return your decision as JSON.' },
1463
+ ],
1464
+ temperature: 0.1,
1465
+ max_tokens: 200,
1466
+ stream: false,
1467
+ };
1468
+ const controller = new AbortController();
1469
+ const timeout = setTimeout(() => controller.abort(), Math.min(timeoutMs, 30000));
1470
+ const res = await fetch(`${baseUrl.replace(/\/$/, '')}/chat/completions`, {
1471
+ method: 'POST',
1472
+ signal: controller.signal,
1473
+ headers: {
1474
+ 'Content-Type': 'application/json',
1475
+ 'Authorization': `Bearer ${apiKey}`,
1476
+ 'api-key': apiKey,
1477
+ },
1478
+ body: JSON.stringify(body),
1479
+ });
1480
+ clearTimeout(timeout);
1481
+ if (!res.ok) {
1482
+ console.log(`[browse] Redundancy check API failed (${res.status}) — defaulting to full update`);
1483
+ return { decision: 'full', reason: 'API failed', matchPercent };
1484
+ }
1485
+ const json = (await res.json());
1486
+ let content = json.choices?.[0]?.message?.content ?? '';
1487
+ if (Array.isArray(content)) {
1488
+ content = content.filter((c) => c.type === 'text').map((c) => c.text).join('');
1489
+ }
1490
+ // Parse the JSON response
1491
+ const match = content.match(/\{[^}]+\}/);
1492
+ if (match) {
1493
+ const parsed = JSON.parse(match[0]);
1494
+ console.log(`[browse] LLM decision: ${parsed.decision} — ${parsed.reason}`);
1495
+ return { ...parsed, matchPercent };
1496
+ }
1497
+ }
1498
+ catch (err) {
1499
+ console.log(`[browse] Redundancy check failed: ${(0, config_1.msg)(err)} — defaulting to full update`);
1500
+ }
1501
+ return { decision: 'full', reason: 'Fallback', matchPercent };
1502
+ }
1503
+ /** Wait for the page to change by comparing screenshots periodically. Returns true if changed, false if stopped. */
1504
+ async function waitForScreenChange(page, previousScreenshot, intervalMs, quality, stopped) {
1505
+ console.log('[browse] Waiting for screen change... (Ctrl+C to stop)');
1506
+ while (!stopped()) {
1507
+ await sleep(intervalMs);
1508
+ if (stopped())
1509
+ break;
1510
+ const newScreenshot = await captureScreenshot(page, quality);
1511
+ if (newScreenshot && newScreenshot !== previousScreenshot) {
1512
+ console.log('[browse] Screen changed — resuming...');
1513
+ return true;
1514
+ }
1515
+ }
1516
+ return false;
1517
+ }
1518
+ /** Variable typing delay — random variation for natural feel. Never faster than 70% of base. */
1519
+ function variableDelay(baseMs, char, index, total) {
1520
+ let delay = baseMs + (Math.random() - 0.5) * baseMs * 0.6;
767
1521
  if (char === ' ')
768
- return base + 20 + Math.random() * 60;
769
- // Numbers — slightly slower
1522
+ delay += 20 + Math.random() * 60;
770
1523
  if (/[0-9]/.test(char))
771
- return base + 10 + Math.random() * 30;
772
- // Shift characters — shift key overhead
1524
+ delay += 10 + Math.random() * 30;
773
1525
  if (/^[{}():"<>!@#$%^&*_+|~`?\[\]]$/.test(char))
774
- return base + 50 + Math.random() * 120;
775
- // Occasional "think" pause every 15-25 chars
1526
+ delay += 50 + Math.random() * 120;
776
1527
  if (index > 0 && index % (15 + Math.floor(Math.random() * 10)) === 0) {
777
- return 200 + Math.random() * 300;
1528
+ delay += 200 + Math.random() * 500;
1529
+ }
1530
+ return Math.max(delay, baseMs * 0.7);
1531
+ }
1532
+ /**
1533
+ * Detect and remove extra trailing closing brackets in the editor.
1534
+ * Handles: } } } → removes extras, keeps correct one.
1535
+ */
1536
+ async function removeTrailingBrackets(page) {
1537
+ const content = await readEditorContent(page);
1538
+ if (!content)
1539
+ return;
1540
+ // Count trailing closing brackets
1541
+ const trailing = content.match(/[\}\)\]]+$/)?.[0] ?? '';
1542
+ if (!trailing || trailing.length === 0)
1543
+ return;
1544
+ // Count opening brackets in the full content
1545
+ let opens = 0;
1546
+ let closes = 0;
1547
+ for (const c of content) {
1548
+ if (c === '{' || c === '(' || c === '[')
1549
+ opens++;
1550
+ if (c === '}' || c === ')' || c === ']')
1551
+ closes++;
1552
+ }
1553
+ const extra = closes - opens;
1554
+ if (extra <= 0)
1555
+ return;
1556
+ // Move to end of editor
1557
+ await cdpSend(page, 'Input.dispatchKeyEvent', {
1558
+ type: 'keyDown', key: 'End', code: 'End', modifiers: 2,
1559
+ windowsVirtualKeyCode: 35, nativeVirtualKeyCode: 35,
1560
+ });
1561
+ await cdpSend(page, 'Input.dispatchKeyEvent', {
1562
+ type: 'keyUp', key: 'End', code: 'End', modifiers: 2,
1563
+ windowsVirtualKeyCode: 35, nativeVirtualKeyCode: 35,
1564
+ });
1565
+ await sleep(100);
1566
+ // Press Backspace extra times
1567
+ for (let i = 0; i < extra; i++) {
1568
+ await cdpSend(page, 'Input.dispatchKeyEvent', {
1569
+ type: 'keyDown', key: 'Backspace', code: 'Backspace',
1570
+ windowsVirtualKeyCode: 8, nativeVirtualKeyCode: 8,
1571
+ });
1572
+ await cdpSend(page, 'Input.dispatchKeyEvent', {
1573
+ type: 'keyUp', key: 'Backspace', code: 'Backspace',
1574
+ windowsVirtualKeyCode: 8, nativeVirtualKeyCode: 8,
1575
+ });
1576
+ await sleep(50);
1577
+ }
1578
+ console.log(`[browse] Removed ${extra} extra trailing bracket(s)`);
1579
+ }
1580
+ /** Insert text into ANY code editor — character-by-character typing, no pasting. */
1581
+ async function insertIntoCodeEditor(page, selector, text, charDelayMs) {
1582
+ // Focus the editor
1583
+ await focusSelector(page, selector);
1584
+ await sleep(100);
1585
+ // Dismiss any autocomplete popup
1586
+ await cdpSend(page, 'Input.dispatchKeyEvent', {
1587
+ type: 'keyDown', key: 'Escape', code: 'Escape',
1588
+ windowsVirtualKeyCode: 27, nativeVirtualKeyCode: 27,
1589
+ });
1590
+ await cdpSend(page, 'Input.dispatchKeyEvent', {
1591
+ type: 'keyUp', key: 'Escape', code: 'Escape',
1592
+ windowsVirtualKeyCode: 27, nativeVirtualKeyCode: 27,
1593
+ });
1594
+ await sleep(100);
1595
+ // Disable ALL auto-formatting to prevent cascading indentation
1596
+ await cdpSend(page, 'Runtime.evaluate', {
1597
+ expression: `
1598
+ (() => {
1599
+ const monacoNs = typeof monaco !== 'undefined' ? monaco : window.monaco;
1600
+ monacoNs?.editor?.getEditors?.().forEach(e => e.updateOptions({
1601
+ autoIndent: false,
1602
+ formatOnType: false,
1603
+ autoClosingBrackets: 'never',
1604
+ autoSurround: 'never',
1605
+ tabSize: 2,
1606
+ detectIndentation: false,
1607
+ }));
1608
+ const cm5 = document.querySelector('.CodeMirror')?.CodeMirror;
1609
+ if (cm5) cm5.setOption('electricChars', false);
1610
+ const ace = document.querySelector('.ace_editor')?.env?.editor;
1611
+ if (ace) {
1612
+ ace.setOption('autoScrollEditorIntoView', false);
1613
+ ace.setOption(' behavioursEnabled', false);
1614
+ }
1615
+ })()
1616
+ `,
1617
+ });
1618
+ // Smart retry: check if editor already has the correct code
1619
+ const currentContent = await readEditorContent(page);
1620
+ if (currentContent === text) {
1621
+ console.log('[browse] Code already correct — skipping re-type');
1622
+ return;
1623
+ }
1624
+ // Focus + clear existing content
1625
+ await focusSelector(page, selector);
1626
+ await sleep(100);
1627
+ await clearEditor(page);
1628
+ await sleep(100);
1629
+ // Re-focus after clear
1630
+ await focusSelector(page, selector);
1631
+ await sleep(100);
1632
+ // Type character by character — NO pasting
1633
+ const est = Math.round(text.length * charDelayMs / 1000);
1634
+ console.log(`[browse] Typing ${text.length} chars (~${est}s est.)...`);
1635
+ for (let i = 0; i < text.length; i++) {
1636
+ const char = text[i];
1637
+ if (char === '\n') {
1638
+ await cdpSend(page, 'Input.dispatchKeyEvent', {
1639
+ type: 'keyDown', key: 'Enter', code: 'Enter', text: '\r',
1640
+ windowsVirtualKeyCode: 13, nativeVirtualKeyCode: 13,
1641
+ });
1642
+ await cdpSend(page, 'Input.dispatchKeyEvent', {
1643
+ type: 'keyUp', key: 'Enter', code: 'Enter',
1644
+ windowsVirtualKeyCode: 13, nativeVirtualKeyCode: 13,
1645
+ });
1646
+ }
1647
+ else if (char === '\t') {
1648
+ await cdpSend(page, 'Input.dispatchKeyEvent', {
1649
+ type: 'keyDown', key: 'Tab', code: 'Tab', text: '\t',
1650
+ windowsVirtualKeyCode: 9, nativeVirtualKeyCode: 9,
1651
+ });
1652
+ await cdpSend(page, 'Input.dispatchKeyEvent', {
1653
+ type: 'keyUp', key: 'Tab', code: 'Tab',
1654
+ windowsVirtualKeyCode: 9, nativeVirtualKeyCode: 9,
1655
+ });
1656
+ }
1657
+ else {
1658
+ const k = charToKey(char);
1659
+ const mods = k.shift ? 2 : undefined;
1660
+ const downParams = {
1661
+ type: 'keyDown', key: k.key, code: k.code, text: char,
1662
+ };
1663
+ const upParams = {
1664
+ type: 'keyUp', key: k.key, code: k.code,
1665
+ };
1666
+ if (mods !== undefined) {
1667
+ downParams.modifiers = mods;
1668
+ upParams.modifiers = mods;
1669
+ }
1670
+ await cdpSend(page, 'Input.dispatchKeyEvent', downParams);
1671
+ await cdpSend(page, 'Input.dispatchKeyEvent', upParams);
1672
+ }
1673
+ await sleep(variableDelay(charDelayMs, char, i, text.length));
1674
+ }
1675
+ // Post-typing: detect and remove extra trailing brackets
1676
+ await removeTrailingBrackets(page);
1677
+ // Re-enable auto-formatting
1678
+ await cdpSend(page, 'Runtime.evaluate', {
1679
+ expression: `
1680
+ (() => {
1681
+ const monacoNs = typeof monaco !== 'undefined' ? monaco : window.monaco;
1682
+ monacoNs?.editor?.getEditors?.().forEach(e => e.updateOptions({
1683
+ autoIndent: true,
1684
+ formatOnType: true,
1685
+ autoClosingBrackets: 'languageDefined',
1686
+ autoSurround: 'languageDefined',
1687
+ detectIndentation: true,
1688
+ }));
1689
+ const cm5 = document.querySelector('.CodeMirror')?.CodeMirror;
1690
+ if (cm5) cm5.setOption('electricChars', true);
1691
+ })()
1692
+ `,
1693
+ });
1694
+ console.log(`[browse] Typing complete (${text.length} chars)`);
1695
+ }
1696
+ /** Insert text into a generic input — click focus + char-by-char typing. */
1697
+ async function insertIntoGenericInput(page, el, text, charDelayMs) {
1698
+ await cdpSend(page, 'Input.dispatchMouseEvent', {
1699
+ type: 'mousePressed', x: el.x, y: el.y, button: 'left', clickCount: 1,
1700
+ });
1701
+ await cdpSend(page, 'Input.dispatchMouseEvent', {
1702
+ type: 'mouseReleased', x: el.x, y: el.y, button: 'left', clickCount: 1,
1703
+ });
1704
+ await sleep(200);
1705
+ await clearEditor(page);
1706
+ await sleep(200);
1707
+ for (let i = 0; i < text.length; i++) {
1708
+ const char = text[i];
1709
+ if (char === '\n') {
1710
+ await cdpSend(page, 'Input.dispatchKeyEvent', {
1711
+ type: 'keyDown', key: 'Enter', code: 'Enter', text: '\r',
1712
+ windowsVirtualKeyCode: 13, nativeVirtualKeyCode: 13,
1713
+ });
1714
+ await cdpSend(page, 'Input.dispatchKeyEvent', {
1715
+ type: 'keyUp', key: 'Enter', code: 'Enter',
1716
+ windowsVirtualKeyCode: 13, nativeVirtualKeyCode: 13,
1717
+ });
1718
+ }
1719
+ else if (char === '\t') {
1720
+ await cdpSend(page, 'Input.dispatchKeyEvent', {
1721
+ type: 'keyDown', key: 'Tab', code: 'Tab', text: '\t',
1722
+ windowsVirtualKeyCode: 9, nativeVirtualKeyCode: 9,
1723
+ });
1724
+ await cdpSend(page, 'Input.dispatchKeyEvent', {
1725
+ type: 'keyUp', key: 'Tab', code: 'Tab',
1726
+ windowsVirtualKeyCode: 9, nativeVirtualKeyCode: 9,
1727
+ });
1728
+ }
1729
+ else {
1730
+ const k = charToKey(char);
1731
+ const mods = k.shift ? 2 : undefined;
1732
+ const downParams = {
1733
+ type: 'keyDown', key: k.key, code: k.code, text: char,
1734
+ };
1735
+ const upParams = {
1736
+ type: 'keyUp', key: k.key, code: k.code,
1737
+ };
1738
+ if (mods !== undefined) {
1739
+ downParams.modifiers = mods;
1740
+ upParams.modifiers = mods;
1741
+ }
1742
+ await cdpSend(page, 'Input.dispatchKeyEvent', downParams);
1743
+ await cdpSend(page, 'Input.dispatchKeyEvent', upParams);
1744
+ }
1745
+ await sleep(variableDelay(charDelayMs, char, i, text.length));
1746
+ }
1747
+ }
1748
+ /**
1749
+ * Extract page context for the LLM: title, URL, headings, errors, editor content.
1750
+ * This gives the LLM full visibility into what's on screen.
1751
+ */
1752
+ async function extractPageContext(page) {
1753
+ const js = `
1754
+ (() => {
1755
+ const ctx = [];
1756
+ // Title + URL
1757
+ ctx.push('Title: ' + (document.title || 'unknown'));
1758
+ ctx.push('URL: ' + window.location.href);
1759
+ // Headings
1760
+ const headings = [...document.querySelectorAll('h1,h2,h3,h4,h5,h6')]
1761
+ .map(h => h.textContent?.trim())
1762
+ .filter(Boolean)
1763
+ .slice(0, 10);
1764
+ if (headings.length) ctx.push('Headings: ' + JSON.stringify(headings));
1765
+ // Errors / alerts / toasts / notifications
1766
+ const errSels = [
1767
+ '[role="alert"]', '[role="status"]',
1768
+ '[class*="error"]', '[class*="Error"]',
1769
+ '[class*="toast"]', '[class*="Toast"]',
1770
+ '[class*="alert"]', '[class*="Alert"]',
1771
+ '[class*="banner"]', '[class*="Banner"]',
1772
+ '[class*="notification"]', '[class*="Notification"]',
1773
+ '[class*="danger"]', '[class*="warning"]',
1774
+ '[class*="inline-error"]', '[class*="field-error"]',
1775
+ '[class*="form-error"]', '[class*="validation-error"]',
1776
+ '[data-sonner-toaster] [data-sonner-toast]',
1777
+ '.alert-danger', '.alert-warning', '.alert-error',
1778
+ '.text-red', '.text-danger', '.text-error',
1779
+ '[class*="err-msg"]', '[class*="error-msg"]', '[class*="error-message"]',
1780
+ ];
1781
+ const errors = [];
1782
+ document.querySelectorAll(errSels.join(',')).forEach(el => {
1783
+ const text = el.innerText?.trim().replace(/\\s+/g, ' ').slice(0, 200);
1784
+ if (text && text.length > 2) errors.push(text);
1785
+ });
1786
+ if (errors.length) {
1787
+ ctx.push('Errors on screen:');
1788
+ errors.slice(0, 5).forEach(e => ctx.push(' - ' + e));
1789
+ }
1790
+ // Code editor content
1791
+ let editorContent = null;
1792
+ // Monaco
1793
+ const monacoNs = typeof monaco !== 'undefined' ? monaco : window.monaco;
1794
+ const editors = monacoNs?.editor?.getEditors?.() || [];
1795
+ if (editors.length > 0) editorContent = editors[0].getValue();
1796
+ // CodeMirror 6
1797
+ if (!editorContent) {
1798
+ const cm6 = document.querySelector('.cm-content');
1799
+ if (cm6?.cmView?.view) editorContent = cm6.cmView.view.state.doc.toString();
1800
+ }
1801
+ // CodeMirror 5
1802
+ if (!editorContent) {
1803
+ const cm5 = document.querySelector('.CodeMirror')?.CodeMirror;
1804
+ if (cm5) editorContent = cm5.getValue();
1805
+ }
1806
+ // Ace
1807
+ if (!editorContent) {
1808
+ const ace = document.querySelector('.ace_editor')?.env?.editor;
1809
+ if (ace) editorContent = ace.getValue();
1810
+ }
1811
+ if (editorContent && editorContent.trim().length > 0) {
1812
+ ctx.push('Editor content (' + editorContent.length + ' chars):');
1813
+ ctx.push(editorContent.slice(0, 1500));
1814
+ if (editorContent.length > 1500) ctx.push('... (truncated)');
1815
+ }
1816
+ // Problem description (LeetCode, HackerRank, etc.)
1817
+ const descSels = [
1818
+ '[class*="problem-description"]', '[class*="question-text"]',
1819
+ '[data-track-load="description_content"]', '.markdown-body',
1820
+ '[class*="content"]', '[class*="prose"]', '[class*="description"]',
1821
+ ];
1822
+ for (const sel of descSels) {
1823
+ const el = document.querySelector(sel);
1824
+ if (el && el.innerText && el.innerText.trim().length > 20) {
1825
+ ctx.push('Problem description:');
1826
+ ctx.push(el.innerText.trim().slice(0, 3000));
1827
+ break;
1828
+ }
1829
+ }
1830
+ return ctx.join('\\n');
1831
+ })()
1832
+ `;
1833
+ try {
1834
+ const result = await cdpSend(page, 'Runtime.evaluate', {
1835
+ expression: js,
1836
+ returnByValue: true,
1837
+ });
1838
+ return result?.result?.value ?? '';
1839
+ }
1840
+ catch {
1841
+ return '';
778
1842
  }
779
- return base;
780
1843
  }
781
1844
  /**
782
1845
  * Detect which code/rich-text editor is active on the page.
@@ -890,6 +1953,8 @@ function validateActions(actions) {
890
1953
  return typeof a.elementIndex === 'number';
891
1954
  if (a.action === 'type')
892
1955
  return typeof a.elementIndex === 'number' && typeof a.text === 'string';
1956
+ if (a.action === 'write_code')
1957
+ return typeof a.elementIndex === 'number' && typeof a.code === 'string';
893
1958
  if (a.action === 'scroll')
894
1959
  return typeof a.delta === 'number';
895
1960
  if (a.action === 'press')
@@ -992,7 +2057,7 @@ function extractScatteredActions(str) {
992
2057
  }
993
2058
  return actions;
994
2059
  }
995
- async function analyzeScreenshot(baseUrl, apiKey, model, screenshotBase64, elements, timeoutMs, attempt, maxAttempts) {
2060
+ async function analyzeScreenshot(baseUrl, apiKey, model, screenshotBase64, elements, pageContext, timeoutMs, attempt, maxAttempts) {
996
2061
  const startTime = Date.now();
997
2062
  // Format element list grouped by question
998
2063
  const groups = new Map();
@@ -1038,11 +2103,16 @@ async function analyzeScreenshot(baseUrl, apiKey, model, screenshotBase64, eleme
1038
2103
  }
1039
2104
  for (const el of els) {
1040
2105
  const checked = el.checked ? ' [CHECKED]' : '';
1041
- lines.push(` ${el.i}: [${el.type || el.tag}] "${el.text}"${checked}`);
2106
+ const visibility = el.inViewport ? '' : ' [OFF-SCREEN]';
2107
+ lines.push(` ${el.i}: [${el.type || el.tag}] "${el.text}"${checked}${visibility}`);
1042
2108
  }
1043
2109
  lines.push('');
1044
2110
  }
1045
- const userText = `Interactive elements on screen:\n${lines.join('\n')}\nAnalyze the screenshot. For each UNANSWERED question, return the element INDEX to click. For checkbox questions, return ONE action per correct answer (multiple clicks).`;
2111
+ const userText = [
2112
+ pageContext ? `<page_context>\n${pageContext}\n</page_context>\n` : '',
2113
+ `Interactive elements on screen:\n${lines.join('\n')}`,
2114
+ `Analyze the screenshot. For each UNANSWERED question, return the element INDEX to click. For checkbox questions, return ONE action per correct answer (multiple clicks). If you see ERRORS in <page_context>, fix them by retyping the correct answer.`,
2115
+ ].filter(Boolean).join('\n\n');
1046
2116
  const body = {
1047
2117
  model,
1048
2118
  messages: [
@@ -1056,7 +2126,7 @@ async function analyzeScreenshot(baseUrl, apiKey, model, screenshotBase64, eleme
1056
2126
  },
1057
2127
  ],
1058
2128
  temperature: 0.1,
1059
- max_tokens: 8192,
2129
+ max_tokens: 16384,
1060
2130
  stream: false,
1061
2131
  };
1062
2132
  const controller = new AbortController();
@@ -1067,7 +2137,8 @@ async function analyzeScreenshot(baseUrl, apiKey, model, screenshotBase64, eleme
1067
2137
  console.log(`[browse] Waiting for LLM... (${elapsed}s elapsed, attempt ${attempt}/${maxAttempts})`);
1068
2138
  }, 30000);
1069
2139
  const screenshotKB = Math.round((screenshotBase64.length * 3 / 4) / 1024);
1070
- console.log(`[browse] LLM request: ${model} | ${screenshotKB}KB image | ${elements.length} elements | timeout ${Math.round(timeoutMs / 1000)}s | attempt ${attempt}/${maxAttempts}`);
2140
+ const contextKB = Math.round(pageContext.length / 1024);
2141
+ console.log(`[browse] LLM request: ${model} | ${screenshotKB}KB image | ${contextKB}KB context | ${elements.length} elements | timeout ${Math.round(timeoutMs / 1000)}s | attempt ${attempt}/${maxAttempts}`);
1071
2142
  try {
1072
2143
  const res = await fetch(`${baseUrl.replace(/\/$/, '')}/chat/completions`, {
1073
2144
  method: 'POST',
@@ -1123,7 +2194,508 @@ async function analyzeScreenshot(baseUrl, apiKey, model, screenshotBase64, eleme
1123
2194
  clearInterval(heartbeat);
1124
2195
  }
1125
2196
  }
2197
+ /** Like analyzeScreenshot but accepts a custom system prompt (for optimization pass). */
2198
+ async function analyzeScreenshotWithPrompt(baseUrl, apiKey, model, screenshotBase64, elements, pageContext, timeoutMs, attempt, maxAttempts, customSystemPrompt) {
2199
+ const startTime = Date.now();
2200
+ const userText = [
2201
+ pageContext ? `<page_context>\n${pageContext}\n</page_context>\n` : '',
2202
+ `Analyze the screenshot and respond according to the system instructions.`,
2203
+ ].filter(Boolean).join('\n\n');
2204
+ const body = {
2205
+ model,
2206
+ messages: [
2207
+ { role: 'system', content: customSystemPrompt },
2208
+ {
2209
+ role: 'user',
2210
+ content: [
2211
+ { type: 'text', text: userText },
2212
+ { type: 'image_url', image_url: { url: `data:image/jpeg;base64,${screenshotBase64}` } },
2213
+ ],
2214
+ },
2215
+ ],
2216
+ temperature: 0.1,
2217
+ max_tokens: 16384,
2218
+ stream: false,
2219
+ };
2220
+ const controller = new AbortController();
2221
+ const timeout = setTimeout(() => controller.abort(), timeoutMs);
2222
+ try {
2223
+ const res = await fetch(`${baseUrl.replace(/\/$/, '')}/chat/completions`, {
2224
+ method: 'POST',
2225
+ signal: controller.signal,
2226
+ headers: {
2227
+ 'Content-Type': 'application/json',
2228
+ 'Authorization': `Bearer ${apiKey}`,
2229
+ 'api-key': apiKey,
2230
+ },
2231
+ body: JSON.stringify(body),
2232
+ });
2233
+ if (!res.ok) {
2234
+ return { actions: [], attempt, durationMs: Date.now() - startTime, error: 'http_error' };
2235
+ }
2236
+ const json = (await res.json());
2237
+ let content = json.choices?.[0]?.message?.content;
2238
+ if (Array.isArray(content)) {
2239
+ content = content.filter((c) => c.type === 'text').map((c) => c.text).join('');
2240
+ }
2241
+ if (typeof content !== 'string' || !content) {
2242
+ const reasoning = json.choices?.[0]?.message?.reasoning_content;
2243
+ if (typeof reasoning === 'string' && reasoning)
2244
+ content = reasoning;
2245
+ }
2246
+ if (typeof content !== 'string' || !content) {
2247
+ return { actions: [], attempt, durationMs: Date.now() - startTime, error: 'empty' };
2248
+ }
2249
+ const parsed = extractJsonActions(content);
2250
+ if (parsed.length === 0) {
2251
+ return { actions: [], attempt, durationMs: Date.now() - startTime, error: 'parse_error' };
2252
+ }
2253
+ console.log(`[browse] Optimization pass: ${parsed.length} action(s) returned`);
2254
+ return { actions: parsed, attempt, durationMs: Date.now() - startTime };
2255
+ }
2256
+ catch (err) {
2257
+ if (err instanceof Error && err.name === 'AbortError') {
2258
+ return { actions: [], attempt, durationMs: Date.now() - startTime, error: 'timeout' };
2259
+ }
2260
+ return { actions: [], attempt, durationMs: Date.now() - startTime, error: 'http_error' };
2261
+ }
2262
+ finally {
2263
+ clearTimeout(timeout);
2264
+ }
2265
+ }
1126
2266
  function sleep(ms) {
1127
2267
  return new Promise((r) => setTimeout(r, ms));
1128
2268
  }
2269
+ // ── Stealth mode: OS-level automation (no CDP) ──────────────────────────────
2270
+ function randomInt(min, max) {
2271
+ return Math.floor(Math.random() * (max - min + 1)) + min;
2272
+ }
2273
+ /**
2274
+ * OS-level screen capture using PowerShell.
2275
+ * Returns path to screenshot PNG.
2276
+ */
2277
+ function captureScreenStealth() {
2278
+ const outPath = path.join(os.tmpdir(), `stealth_${Date.now()}.png`);
2279
+ const ps = [
2280
+ 'Add-Type -AssemblyName System.Windows.Forms',
2281
+ 'Add-Type -AssemblyName System.Drawing',
2282
+ '$screen = [System.Windows.Forms.Screen]::PrimaryScreen',
2283
+ '$bitmap = New-Object System.Drawing.Bitmap($screen.Bounds.Width, $screen.Bounds.Height)',
2284
+ '$graphics = [System.Drawing.Graphics]::FromImage($bitmap)',
2285
+ '$graphics.CopyFromScreen($screen.Bounds.Location, [System.Drawing.Point]::Empty, $screen.Bounds.Size)',
2286
+ `$bitmap.Save('${outPath.replace(/\\/g, '\\\\')}', [System.Drawing.Imaging.ImageFormat]::Png)`,
2287
+ '$graphics.Dispose()',
2288
+ '$bitmap.Dispose()',
2289
+ ].join('; ');
2290
+ (0, child_process_1.execSync)(`powershell -ExecutionPolicy Bypass -Command "${ps}"`, { stdio: 'pipe', timeout: 10000 });
2291
+ return outPath;
2292
+ }
2293
+ /**
2294
+ * Compare two screenshots to detect screen change.
2295
+ * Returns true if screens are different (pixel diff > threshold).
2296
+ */
2297
+ function screensChanged(prevPath, currPath) {
2298
+ try {
2299
+ const prevBuf = fs.readFileSync(prevPath);
2300
+ const currBuf = fs.readFileSync(currPath);
2301
+ if (prevBuf.length === currBuf.length) {
2302
+ let diff = 0;
2303
+ for (let i = 0; i < prevBuf.length; i += 100) {
2304
+ if (prevBuf[i] !== currBuf[i])
2305
+ diff++;
2306
+ }
2307
+ return diff > 50;
2308
+ }
2309
+ return true;
2310
+ }
2311
+ catch {
2312
+ return true;
2313
+ }
2314
+ }
2315
+ /**
2316
+ * OS-level keyboard character input using PowerShell SendKeys.
2317
+ */
2318
+ function typeCharOS(char) {
2319
+ // Map special characters to SendKeys syntax
2320
+ const sendKeysMap = {
2321
+ '\n': '{ENTER}',
2322
+ '\r': '{ENTER}',
2323
+ '\t': '{TAB}',
2324
+ '\b': '{BACKSPACE}',
2325
+ '+': '{+}',
2326
+ '^': '{^}',
2327
+ '%': '{%}',
2328
+ '~': '{~}',
2329
+ '(': '{(}',
2330
+ ')': '{)}',
2331
+ '{': '{{}',
2332
+ '}': '{}}',
2333
+ '[': '{[}',
2334
+ ']': '{]}',
2335
+ };
2336
+ const escaped = sendKeysMap[char] ?? char;
2337
+ const ps = `Add-Type -AssemblyName System.Windows.Forms; [System.Windows.Forms.SendKeys]::SendWait('${escaped.replace(/'/g, "''")}')`;
2338
+ (0, child_process_1.execSync)(`powershell -ExecutionPolicy Bypass -Command "${ps}"`, { stdio: 'pipe', timeout: 5000 });
2339
+ }
2340
+ /**
2341
+ * OS-level mouse click at specific screen coordinates.
2342
+ * Writes C# to temp file, loads via Add-Type -Path, calls user32.dll.
2343
+ */
2344
+ function clickOS(x, y) {
2345
+ const csCode = [
2346
+ 'using System;',
2347
+ 'using System.Runtime.InteropServices;',
2348
+ 'public class MouseOps {',
2349
+ ' [DllImport("user32.dll")] public static extern void mouse_event(uint dwFlags, int dx, int dy, uint dwData, IntPtr dwExtraInfo);',
2350
+ ' [DllImport("user32.dll")] public static extern bool SetCursorPos(int X, int Y);',
2351
+ '}',
2352
+ ].join('\n');
2353
+ const csPath = path.join(os.tmpdir(), 'ts_mouse_ops.cs');
2354
+ fs.writeFileSync(csPath, csCode, 'utf8');
2355
+ const ps = [
2356
+ `Add-Type -Path '${csPath.replace(/\\/g, '\\\\')}'`,
2357
+ `[MouseOps]::SetCursorPos(${Math.round(x)}, ${Math.round(y)})`,
2358
+ 'Start-Sleep -Milliseconds 50',
2359
+ '[MouseOps]::mouse_event(0x0002, 0, 0, 0, [IntPtr]::Zero)',
2360
+ 'Start-Sleep -Milliseconds 30',
2361
+ '[MouseOps]::mouse_event(0x0004, 0, 0, 0, [IntPtr]::Zero)',
2362
+ ].join('; ');
2363
+ (0, child_process_1.execSync)(`powershell -ExecutionPolicy Bypass -Command "${ps}"`, { stdio: 'pipe', timeout: 5000 });
2364
+ }
2365
+ /**
2366
+ * OS-level scroll using mouse wheel.
2367
+ * positive = up, negative = down.
2368
+ */
2369
+ function scrollOS(clicks) {
2370
+ const delta = clicks * 120; // WHEEL_DELTA = 120
2371
+ const csCode = [
2372
+ 'using System;',
2373
+ 'using System.Runtime.InteropServices;',
2374
+ 'public class ScrollOps {',
2375
+ ' [DllImport("user32.dll")] public static extern void mouse_event(uint dwFlags, int dx, int dy, uint dwData, IntPtr dwExtraInfo);',
2376
+ '}',
2377
+ ].join('\n');
2378
+ const csPath = path.join(os.tmpdir(), 'ts_scroll_ops.cs');
2379
+ fs.writeFileSync(csPath, csCode, 'utf8');
2380
+ const ps = [
2381
+ `Add-Type -Path '${csPath.replace(/\\/g, '\\\\')}'`,
2382
+ `[ScrollOps]::mouse_event(0x0800, 0, 0, ${delta}, [IntPtr]::Zero)`,
2383
+ ].join('; ');
2384
+ (0, child_process_1.execSync)(`powershell -ExecutionPolicy Bypass -Command "${ps}"`, { stdio: 'pipe', timeout: 5000 });
2385
+ }
2386
+ /**
2387
+ * OS-level keyboard shortcut (e.g., Ctrl+A, Tab, Enter).
2388
+ */
2389
+ function pressKeyOS(key) {
2390
+ const keyMap = {
2391
+ 'enter': '{ENTER}',
2392
+ 'tab': '{TAB}',
2393
+ 'escape': '{ESC}',
2394
+ 'backspace': '{BACKSPACE}',
2395
+ 'delete': '{DELETE}',
2396
+ 'space': ' ',
2397
+ 'up': '{UP}',
2398
+ 'down': '{DOWN}',
2399
+ 'left': '{LEFT}',
2400
+ 'right': '{RIGHT}',
2401
+ 'home': '{HOME}',
2402
+ 'end': '{END}',
2403
+ 'pageup': '{PGUP}',
2404
+ 'pagedown': '{PGDN}',
2405
+ 'ctrl+a': '^a',
2406
+ 'ctrl+c': '^c',
2407
+ 'ctrl+v': '^v',
2408
+ 'ctrl+z': '^z',
2409
+ };
2410
+ const escaped = keyMap[key.toLowerCase()] ?? key;
2411
+ const ps = `Add-Type -AssemblyName System.Windows.Forms; [System.Windows.Forms.SendKeys]::SendWait('${escaped.replace(/'/g, "''")}')`;
2412
+ (0, child_process_1.execSync)(`powershell -ExecutionPolicy Bypass -Command "${ps}"`, { stdio: 'pipe', timeout: 5000 });
2413
+ }
2414
+ /**
2415
+ * Keyboard proximity map for generating realistic typos.
2416
+ */
2417
+ const ADJACENT_KEYS = {
2418
+ 'a': ['s', 'q', 'w', 'z'], 'b': ['v', 'g', 'h', 'n'],
2419
+ 'c': ['x', 'd', 'f', 'v'], 'd': ['s', 'f', 'e', 'c', 'r'],
2420
+ 'e': ['w', 'r', 'd', 's'], 'f': ['d', 'g', 'r', 'v', 't'],
2421
+ 'g': ['f', 'h', 't', 'b', 'y'], 'h': ['g', 'j', 'y', 'n', 'u'],
2422
+ 'i': ['u', 'o', 'k', 'j'], 'j': ['h', 'k', 'n', 'u', 'm'],
2423
+ 'k': ['j', 'l', 'm', 'i', 'o'], 'l': ['k', 'o', 'p'],
2424
+ 'm': ['n', 'j', 'k'], 'n': ['b', 'm', 'h', 'j'],
2425
+ 'o': ['i', 'p', 'l', 'k'], 'p': ['o', 'l'],
2426
+ 'q': ['w', 'a'], 'r': ['e', 't', 'f', 'd'],
2427
+ 's': ['a', 'd', 'w', 'x', 'z'], 't': ['r', 'y', 'g', 'f'],
2428
+ 'u': ['y', 'i', 'j', 'h'], 'v': ['c', 'b', 'f', 'g'],
2429
+ 'w': ['q', 'e', 's', 'a'], 'x': ['z', 'c', 's', 'd'],
2430
+ 'y': ['t', 'u', 'g', 'h'], 'z': ['x', 'a', 's'],
2431
+ '0': ['9', '1'], '1': ['0', '2', 'q'], '2': ['1', '3', 'w'],
2432
+ '3': ['2', '4', 'e'], '4': ['3', '5', 'r'], '5': ['4', '6', 't'],
2433
+ '6': ['5', '7', 'y'], '7': ['6', '8', 'u'], '8': ['7', '9', 'i'],
2434
+ '9': ['8', '0', 'o'],
2435
+ };
2436
+ /**
2437
+ * Generate a realistic typo by picking an adjacent key.
2438
+ */
2439
+ function getAdjacentKey(char) {
2440
+ const lower = char.toLowerCase();
2441
+ const keys = ADJACENT_KEYS[lower];
2442
+ if (!keys)
2443
+ return char;
2444
+ const wrong = keys[Math.floor(Math.random() * keys.length)];
2445
+ return char === char.toUpperCase() ? wrong.toUpperCase() : wrong;
2446
+ }
2447
+ /**
2448
+ * Type text with human-like patterns: variable speed, occasional typos,
2449
+ * backspaces, natural pauses. Uses OS-level keyboard (no CDP).
2450
+ */
2451
+ async function typeHumanLike(text, baseDelayMs = 120) {
2452
+ for (let i = 0; i < text.length; i++) {
2453
+ const char = text[i];
2454
+ // 1. Occasional typo (2% chance) — then backspace to correct
2455
+ if (char !== '\n' && char !== '\t' && Math.random() < 0.02) {
2456
+ const wrong = getAdjacentKey(char);
2457
+ typeCharOS(wrong);
2458
+ await sleep(randomInt(40, 120));
2459
+ typeCharOS('\b'); // backspace
2460
+ await sleep(randomInt(25, 70));
2461
+ }
2462
+ // 2. Type the correct character
2463
+ typeCharOS(char);
2464
+ // 3. Variable delay based on character type (human rhythm)
2465
+ let delay = baseDelayMs + randomInt(-Math.floor(baseDelayMs * 0.3), Math.floor(baseDelayMs * 0.4));
2466
+ if (char === ' ')
2467
+ delay += randomInt(30, 120); // pause at word boundary
2468
+ else if (char === '\n')
2469
+ delay += randomInt(100, 350); // pause at line end
2470
+ else if (char === '.')
2471
+ delay += randomInt(80, 200); // pause at sentence end
2472
+ else if (char === ',')
2473
+ delay += randomInt(40, 100); // pause at comma
2474
+ else if (/[A-Z]/.test(char))
2475
+ delay += randomInt(15, 50); // hesitate before uppercase
2476
+ else if (/[0-9]/.test(char))
2477
+ delay += randomInt(10, 35); // hesitate before number
2478
+ else if (/[!@#$%^&*()_+\-=\[\]{};':"\\|,.<>\/?]/.test(char))
2479
+ delay += randomInt(20, 60); // hesitate before symbol
2480
+ // 4. Occasional longer pause (thinking) — 3% chance
2481
+ if (Math.random() < 0.03) {
2482
+ await sleep(randomInt(200, 600));
2483
+ }
2484
+ await sleep(Math.max(delay, 30));
2485
+ }
2486
+ }
2487
+ /**
2488
+ * Wait for the screen to change by comparing screenshots.
2489
+ * Returns the new screenshot path, or null if timed out.
2490
+ */
2491
+ function waitForScreenChangeStealth(prevPath, intervalMs = 2000, maxWaitMs = 60000, shouldStop = () => false) {
2492
+ const deadline = Date.now() + maxWaitMs;
2493
+ while (Date.now() < deadline && !shouldStop()) {
2494
+ sleep(intervalMs); // sync-ish wait
2495
+ const currPath = captureScreenStealth();
2496
+ if (screensChanged(prevPath, currPath)) {
2497
+ return currPath;
2498
+ }
2499
+ }
2500
+ return null;
2501
+ }
2502
+ /**
2503
+ * Stealth LLM call: send screenshot to vision LLM, get structured action.
2504
+ * Returns JSON: { action, coordinates?, text?, scroll?, reasoning }
2505
+ */
2506
+ async function callStealthLLM(baseUrl, apiKey, model, screenshotPath, timeoutMs, promptOverride) {
2507
+ const imageData = fs.readFileSync(screenshotPath).toString('base64');
2508
+ const systemPrompt = promptOverride ?? [
2509
+ 'You are a test-taking assistant analyzing a screenshot of a test/assessment.',
2510
+ '',
2511
+ 'Analyze the screenshot and return a JSON object with the correct action.',
2512
+ '',
2513
+ 'For MULTIPLE-CHOICE questions (radio buttons, checkboxes, A/B/C/D options):',
2514
+ ' Return: {"action":"click","x":<center_x_of_correct_option>,"y":<center_y_of_correct_option>,"reasoning":"why this option"}',
2515
+ ' Look at the screenshot carefully. The x,y coordinates should be the CENTER of the radio button or checkbox for the correct answer.',
2516
+ '',
2517
+ 'For CODING questions (code editor visible, need to write code):',
2518
+ ' Return: {"action":"type","text":"<the complete code>","reasoning":"why this code"}',
2519
+ '',
2520
+ 'For FILL-IN-THE-BLANK (text input field visible):',
2521
+ ' Return: {"action":"type","text":"<the answer text>","reasoning":"why this answer"}',
2522
+ '',
2523
+ 'For questions that need SCROLLING (answer options are cut off at bottom):',
2524
+ ' Return: {"action":"scroll","scroll":-3,"reasoning":"need to see more options"}',
2525
+ ' scroll: negative = scroll down, positive = scroll up. Use -2 to -5 for typical scroll.',
2526
+ '',
2527
+ 'For DONE (all questions answered, no more questions visible):',
2528
+ ' Return: {"action":"done","reasoning":"all questions answered"}',
2529
+ '',
2530
+ 'Rules:',
2531
+ '- Return ONLY valid JSON, nothing else. No markdown, no explanation outside the JSON.',
2532
+ '- For click actions, the x,y must be PIXEL COORDINATES on the screen where the correct option is.',
2533
+ '- Read the question carefully. Think step by step before answering.',
2534
+ '- If you need to scroll to see all options, return scroll action first.',
2535
+ ].join('\n');
2536
+ const controller = new AbortController();
2537
+ const timeout = setTimeout(() => controller.abort(), timeoutMs);
2538
+ try {
2539
+ const res = await fetch(`${baseUrl}/chat/completions`, {
2540
+ method: 'POST',
2541
+ headers: {
2542
+ 'Content-Type': 'application/json',
2543
+ 'Authorization': `Bearer ${apiKey}`,
2544
+ },
2545
+ body: JSON.stringify({
2546
+ model,
2547
+ messages: [
2548
+ { role: 'system', content: systemPrompt },
2549
+ {
2550
+ role: 'user',
2551
+ content: [
2552
+ { type: 'text', text: 'Analyze this screenshot. Return the JSON action to take.' },
2553
+ { type: 'image_url', image_url: { url: `data:image/png;base64,${imageData}` } },
2554
+ ],
2555
+ },
2556
+ ],
2557
+ max_tokens: 4096,
2558
+ temperature: 0.1,
2559
+ }),
2560
+ signal: controller.signal,
2561
+ });
2562
+ if (!res.ok)
2563
+ throw new Error(`HTTP ${res.status}`);
2564
+ const data = await res.json();
2565
+ const content = (data.choices?.[0]?.message?.content ?? '').trim();
2566
+ // Parse JSON response
2567
+ try {
2568
+ // Extract JSON from possible markdown code block
2569
+ const jsonMatch = content.match(/\{[\s\S]*\}/);
2570
+ if (jsonMatch) {
2571
+ const parsed = JSON.parse(jsonMatch[0]);
2572
+ return {
2573
+ action: parsed.action ?? 'type',
2574
+ x: parsed.x,
2575
+ y: parsed.y,
2576
+ text: parsed.text,
2577
+ scroll: parsed.scroll,
2578
+ reasoning: parsed.reasoning,
2579
+ };
2580
+ }
2581
+ }
2582
+ catch {
2583
+ // JSON parse failed — treat as plain text answer (type action)
2584
+ }
2585
+ // Fallback: treat entire response as text to type
2586
+ return { action: 'type', text: content, reasoning: 'fallback: plain text response' };
2587
+ }
2588
+ finally {
2589
+ clearTimeout(timeout);
2590
+ }
2591
+ }
2592
+ /**
2593
+ * Stealth agent loop: OS-level screen capture → vision LLM → OS-level keyboard/mouse.
2594
+ * No CDP connection, no clipboard, no DevTools detection.
2595
+ */
2596
+ async function runStealthLoop(baseUrl, apiKey, model, maxActions, intervalMs, timeoutMs, charDelayMs, continuous) {
2597
+ let actionCount = 0;
2598
+ let llmCallCount = 0;
2599
+ let stopped = false;
2600
+ process.on('SIGINT', () => {
2601
+ console.log('\n[stealth] Stopped by user');
2602
+ stopped = true;
2603
+ });
2604
+ process.on('SIGTERM', () => { stopped = true; });
2605
+ const limit = continuous ? Number.MAX_SAFE_INTEGER : maxActions;
2606
+ console.log(`[stealth] Agent started. ${continuous ? 'Running until Ctrl+C.' : `Max ${limit} actions.`}`);
2607
+ console.log('[stealth] OS-level automation — no CDP, no clipboard, no DevTools');
2608
+ console.log('[stealth] Make sure the target window (browser/editor) is focused\n');
2609
+ // Take initial screenshot
2610
+ let prevScreenshot = captureScreenStealth();
2611
+ console.log('[stealth] Initial screenshot captured — waiting for screen change...\n');
2612
+ while (!stopped && actionCount < limit) {
2613
+ try {
2614
+ // Wait for screen to change (new question appears)
2615
+ const currScreenshot = captureScreenStealth();
2616
+ if (!screensChanged(prevScreenshot, currScreenshot)) {
2617
+ // No change — wait and retry
2618
+ await sleep(intervalMs);
2619
+ continue;
2620
+ }
2621
+ console.log(`[stealth] Screen changed — analyzing...`);
2622
+ prevScreenshot = currScreenshot;
2623
+ // Send to vision LLM
2624
+ const actionLabel = continuous ? `${actionCount + 1}/∞` : `${actionCount + 1}/${limit}`;
2625
+ console.log(`[stealth] [${actionLabel}] Calling vision LLM...`);
2626
+ const result = await callStealthLLM(baseUrl, apiKey, model, currScreenshot, timeoutMs);
2627
+ llmCallCount++;
2628
+ console.log(`[stealth] LLM returned: action="${result.action}" reasoning="${result.reasoning?.slice(0, 60) ?? ''}"`);
2629
+ // Execute the action
2630
+ switch (result.action) {
2631
+ case 'click': {
2632
+ if (result.x !== undefined && result.y !== undefined) {
2633
+ console.log(`[stealth] Clicking at (${Math.round(result.x)}, ${Math.round(result.y)})...`);
2634
+ await sleep(randomInt(200, 500)); // natural pause before click
2635
+ clickOS(result.x, result.y);
2636
+ actionCount++;
2637
+ }
2638
+ else {
2639
+ console.log('[stealth] Click action missing coordinates — skipping');
2640
+ }
2641
+ break;
2642
+ }
2643
+ case 'type': {
2644
+ if (result.text && result.text.length > 0) {
2645
+ console.log(`[stealth] Typing ${result.text.length} chars...`);
2646
+ await sleep(randomInt(300, 700)); // natural pause before typing
2647
+ await typeHumanLike(result.text, charDelayMs);
2648
+ actionCount++;
2649
+ }
2650
+ else {
2651
+ console.log('[stealth] Type action missing text — skipping');
2652
+ }
2653
+ break;
2654
+ }
2655
+ case 'scroll': {
2656
+ const clicks = result.scroll ?? -3;
2657
+ console.log(`[stealth] Scrolling ${clicks > 0 ? 'up' : 'down'} (${Math.abs(clicks)} clicks)...`);
2658
+ await sleep(randomInt(200, 400));
2659
+ scrollOS(clicks);
2660
+ actionCount++;
2661
+ // Wait for scroll to settle
2662
+ await sleep(randomInt(500, 1000));
2663
+ break;
2664
+ }
2665
+ case 'done': {
2666
+ console.log(`[stealth] LLM says done: ${result.reasoning ?? 'All questions answered'}`);
2667
+ if (!continuous) {
2668
+ stopped = true;
2669
+ }
2670
+ else {
2671
+ // In continuous mode, wait for next screen change
2672
+ console.log('[stealth] Waiting for next question...');
2673
+ await sleep(randomInt(3000, 6000));
2674
+ }
2675
+ break;
2676
+ }
2677
+ default: {
2678
+ console.log(`[stealth] Unknown action: ${result.action} — treating as type`);
2679
+ if (result.text) {
2680
+ await typeHumanLike(result.text, charDelayMs);
2681
+ actionCount++;
2682
+ }
2683
+ }
2684
+ }
2685
+ // Log progress
2686
+ if (llmCallCount > 50 && llmCallCount % 25 === 0) {
2687
+ console.log(`[stealth] Warning: ${llmCallCount} LLM calls made — may hit rate limits`);
2688
+ }
2689
+ // Natural pause before next question
2690
+ const pause = randomInt(1500, 3500);
2691
+ console.log(`[stealth] Waiting ${pause}ms for next question...\n`);
2692
+ await sleep(pause);
2693
+ }
2694
+ catch (err) {
2695
+ console.error(`[stealth] Error: ${(0, config_1.msg)(err)}`);
2696
+ await sleep(intervalMs);
2697
+ }
2698
+ }
2699
+ console.log(`[stealth] Agent stopped. ${actionCount} actions completed, ${llmCallCount} LLM calls.`);
2700
+ }
1129
2701
  //# sourceMappingURL=browse.js.map