real-browser-mcp-server 1.5.0 → 2.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (113) hide show
  1. package/README.md +46 -41
  2. package/dist/lib/cjs/index.d.ts +14 -1
  3. package/dist/lib/cjs/index.d.ts.map +1 -1
  4. package/dist/lib/cjs/index.js +5 -3
  5. package/dist/lib/cjs/index.js.map +1 -1
  6. package/dist/lib/cjs/module/pageController.d.ts +7 -1
  7. package/dist/lib/cjs/module/pageController.d.ts.map +1 -1
  8. package/dist/lib/cjs/module/pageController.js +4 -23
  9. package/dist/lib/cjs/module/pageController.js.map +1 -1
  10. package/dist/lib/cjs/module/turnstile.d.ts +4 -1
  11. package/dist/lib/cjs/module/turnstile.d.ts.map +1 -1
  12. package/dist/lib/cjs/module/turnstile.js +14 -13
  13. package/dist/lib/cjs/module/turnstile.js.map +1 -1
  14. package/dist/lib/esm/index.d.mjs +11 -0
  15. package/dist/lib/esm/index.mjs +4 -0
  16. package/dist/lib/esm/module/pageController.d.mjs +3 -0
  17. package/dist/lib/esm/module/turnstile.d.mjs +3 -0
  18. package/dist/scripts/generate-esm.d.ts +2 -0
  19. package/dist/scripts/generate-esm.d.ts.map +1 -0
  20. package/dist/scripts/generate-esm.js +100 -0
  21. package/dist/scripts/generate-esm.js.map +1 -0
  22. package/dist/src/index.d.ts.map +1 -1
  23. package/dist/src/index.js +0 -1
  24. package/dist/src/index.js.map +1 -1
  25. package/dist/src/mcp/handlers/browser.d.ts +8 -7
  26. package/dist/src/mcp/handlers/browser.d.ts.map +1 -1
  27. package/dist/src/mcp/handlers/browser.js +19 -2
  28. package/dist/src/mcp/handlers/browser.js.map +1 -1
  29. package/dist/src/mcp/handlers/dom.d.ts +19 -18
  30. package/dist/src/mcp/handlers/dom.d.ts.map +1 -1
  31. package/dist/src/mcp/handlers/dom.js +7 -28
  32. package/dist/src/mcp/handlers/dom.js.map +1 -1
  33. package/dist/src/mcp/handlers/extract.js.map +1 -1
  34. package/dist/src/mcp/handlers/handler-utils.d.ts +14 -0
  35. package/dist/src/mcp/handlers/handler-utils.d.ts.map +1 -0
  36. package/dist/src/mcp/handlers/handler-utils.js +42 -0
  37. package/dist/src/mcp/handlers/handler-utils.js.map +1 -0
  38. package/dist/src/mcp/handlers/helpers.d.ts +0 -7
  39. package/dist/src/mcp/handlers/helpers.d.ts.map +1 -1
  40. package/dist/src/mcp/handlers/helpers.js +0 -15
  41. package/dist/src/mcp/handlers/helpers.js.map +1 -1
  42. package/dist/src/mcp/handlers/index.d.ts.map +1 -1
  43. package/dist/src/mcp/handlers/index.js +11 -0
  44. package/dist/src/mcp/handlers/index.js.map +1 -1
  45. package/dist/src/mcp/handlers/media-handlers.d.ts +2 -1
  46. package/dist/src/mcp/handlers/media-handlers.d.ts.map +1 -1
  47. package/dist/src/mcp/handlers/media-handlers.js +24 -8
  48. package/dist/src/mcp/handlers/media-handlers.js.map +1 -1
  49. package/dist/src/mcp/handlers/network-extractors.d.ts +2 -0
  50. package/dist/src/mcp/handlers/network-extractors.d.ts.map +1 -0
  51. package/dist/src/mcp/handlers/network-extractors.js +651 -0
  52. package/dist/src/mcp/handlers/network-extractors.js.map +1 -0
  53. package/dist/src/mcp/handlers/network-recorder.d.ts +119 -0
  54. package/dist/src/mcp/handlers/network-recorder.d.ts.map +1 -0
  55. package/dist/src/mcp/handlers/network-recorder.js +337 -0
  56. package/dist/src/mcp/handlers/network-recorder.js.map +1 -0
  57. package/dist/src/mcp/handlers/network.d.ts +30 -118
  58. package/dist/src/mcp/handlers/network.d.ts.map +1 -1
  59. package/dist/src/mcp/handlers/network.js +28 -1187
  60. package/dist/src/mcp/handlers/network.js.map +1 -1
  61. package/dist/src/mcp/handlers/state.d.ts +1 -0
  62. package/dist/src/mcp/handlers/state.d.ts.map +1 -1
  63. package/dist/src/mcp/handlers/state.js +17 -0
  64. package/dist/src/mcp/handlers/state.js.map +1 -1
  65. package/dist/src/mcp/handlers/utility-handlers.d.ts +8 -37
  66. package/dist/src/mcp/handlers/utility-handlers.d.ts.map +1 -1
  67. package/dist/src/mcp/handlers/utility-handlers.js +69 -31
  68. package/dist/src/mcp/handlers/utility-handlers.js.map +1 -1
  69. package/dist/src/mcp/handlers/vision-captcha.d.ts +221 -0
  70. package/dist/src/mcp/handlers/vision-captcha.d.ts.map +1 -0
  71. package/dist/src/mcp/handlers/vision-captcha.js +238 -0
  72. package/dist/src/mcp/handlers/vision-captcha.js.map +1 -0
  73. package/dist/src/mcp/handlers/vision-see-page.d.ts +32 -0
  74. package/dist/src/mcp/handlers/vision-see-page.d.ts.map +1 -0
  75. package/dist/src/mcp/handlers/vision-see-page.js +260 -0
  76. package/dist/src/mcp/handlers/vision-see-page.js.map +1 -0
  77. package/dist/src/mcp/handlers/vision.d.ts +50 -27
  78. package/dist/src/mcp/handlers/vision.d.ts.map +1 -1
  79. package/dist/src/mcp/handlers/vision.js +4 -606
  80. package/dist/src/mcp/handlers/vision.js.map +1 -1
  81. package/dist/src/mcp/index.d.ts.map +1 -1
  82. package/dist/src/mcp/index.js +11 -3
  83. package/dist/src/mcp/index.js.map +1 -1
  84. package/dist/src/mcp/server.d.ts.map +1 -1
  85. package/dist/src/mcp/server.js +6 -1
  86. package/dist/src/mcp/server.js.map +1 -1
  87. package/dist/src/shared/cache-manager.d.ts +0 -2
  88. package/dist/src/shared/cache-manager.d.ts.map +1 -1
  89. package/dist/src/shared/cache-manager.js +1 -3
  90. package/dist/src/shared/cache-manager.js.map +1 -1
  91. package/dist/src/shared/lib-core.d.ts +2 -1
  92. package/dist/src/shared/lib-core.d.ts.map +1 -1
  93. package/dist/src/shared/lib-core.js +5 -38
  94. package/dist/src/shared/lib-core.js.map +1 -1
  95. package/dist/src/shared/tools.d.ts.map +1 -1
  96. package/dist/src/shared/tools.js +6 -18
  97. package/dist/src/shared/tools.js.map +1 -1
  98. package/dist/src/types.d.ts +25 -6
  99. package/dist/src/types.d.ts.map +1 -1
  100. package/dist/test/cjs/test.js +29 -33
  101. package/dist/test/cjs/test.js.map +1 -1
  102. package/dist/test/mcp/smoke-test.d.ts +1 -1
  103. package/dist/test/mcp/smoke-test.d.ts.map +1 -1
  104. package/dist/test/mcp/smoke-test.js +16 -4
  105. package/dist/test/mcp/smoke-test.js.map +1 -1
  106. package/dist/test/unit/handler-test.d.ts +3 -0
  107. package/dist/test/unit/handler-test.d.ts.map +1 -0
  108. package/dist/test/unit/handler-test.js +133 -0
  109. package/dist/test/unit/handler-test.js.map +1 -0
  110. package/lib/esm/module/pageController.mjs +4 -22
  111. package/lib/esm/module/turnstile.mjs +16 -13
  112. package/package.json +5 -4
  113. package/typings.d.ts +5 -40
@@ -1,616 +1,14 @@
1
1
  "use strict";
2
- var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
3
- if (k2 === undefined) k2 = k;
4
- var desc = Object.getOwnPropertyDescriptor(m, k);
5
- if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
6
- desc = { enumerable: true, get: function() { return m[k]; } };
7
- }
8
- Object.defineProperty(o, k2, desc);
9
- }) : (function(o, m, k, k2) {
10
- if (k2 === undefined) k2 = k;
11
- o[k2] = m[k];
12
- }));
13
- var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
14
- Object.defineProperty(o, "default", { enumerable: true, value: v });
15
- }) : function(o, v) {
16
- o["default"] = v;
17
- });
18
- var __importStar = (this && this.__importStar) || (function () {
19
- var ownKeys = function(o) {
20
- ownKeys = Object.getOwnPropertyNames || function (o) {
21
- var ar = [];
22
- for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
23
- return ar;
24
- };
25
- return ownKeys(o);
26
- };
27
- return function (mod) {
28
- if (mod && mod.__esModule) return mod;
29
- var result = {};
30
- if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
31
- __setModuleDefault(result, mod);
32
- return result;
33
- };
34
- })();
35
2
  Object.defineProperty(exports, "__esModule", { value: true });
36
3
  exports.visionHandlers = void 0;
37
- const path = __importStar(require("path"));
38
- const fs = __importStar(require("fs"));
39
- const state_1 = require("./state");
40
- const helpers_1 = require("./helpers");
41
- // Auto-generated vision handlers
4
+ const vision_captcha_1 = require("./vision-captcha");
5
+ const vision_see_page_1 = require("./vision-see-page");
42
6
  exports.visionHandlers = {
43
7
  async solve_captcha(params = {}) {
44
- const { page } = (0, state_1.requireBrowser)();
45
- const { type = 'auto', timeout = 30000,
46
- // OCR-specific options
47
- captchaSelector, inputSelector, refreshSelector, lang = 'eng', expectedLength, allowedChars, maxRetries = 3,
48
- // iFrame Support
49
- iframe = null, iframeSelector = null,
50
- // Accuracy options
51
- analyzeFirst = true, // Analyze page before solving
52
- autoRetry = true, // Auto-retry until success
53
- // MERGED: Form automation options (from form_automator)
54
- formData, // Form field data to fill
55
- formSelector, // Form selector (auto-detect if not provided)
56
- submit = false, // Auto-submit after filling
57
- humanLike = true, // Human-like typing delays
58
- aiMatch = true, // AI matches fields even if names differ
59
- } = params;
60
- // Resolve target frame
61
- let targetFrame = page;
62
- if (iframe !== null && iframe !== undefined || iframeSelector) {
63
- const resolved = await helpers_1.helpersHandlers._resolveIframeContext(page, iframe, iframeSelector);
64
- if (resolved.success) {
65
- targetFrame = resolved.targetFrame;
66
- (0, state_1.notifyProgress)('solve_captcha', 'progress', `🎯 Targeting iframe ${iframe ?? iframeSelector}...`);
67
- }
68
- else {
69
- return { success: false, error: resolved.error };
70
- }
71
- }
72
- // ═══════════════════════════════════════════════════════════════
73
- // STEP 0: FORM AUTOMATION (if formData is provided)
74
- // ═══════════════════════════════════════════════════════════════
75
- let formResult = null;
76
- if (formData && Object.keys(formData).length > 0) {
77
- (0, state_1.notifyProgress)('solve_captcha', 'started', `📋 Smart Form + Captcha Mode: Filling ${Object.keys(formData).length} fields...`);
78
- formResult = await helpers_1.helpersHandlers._fillFormFields(page, formData, formSelector, humanLike, aiMatch);
79
- }
80
- else {
81
- (0, state_1.notifyProgress)('solve_captcha', 'started', `🎯 100% Accuracy Mode: Solving ${type} captcha...`);
82
- }
83
- // ═══════════════════════════════════════════════════════════════
84
- // STEP 0.5: AUTO-DETECT JS CAPTCHA TYPE (Cloudflare/reCAPTCHA/hCaptcha)
85
- // ═══════════════════════════════════════════════════════════════
86
- let detectedJsType = null;
87
- if (type === 'auto') {
88
- detectedJsType = await page.evaluate(() => {
89
- // Cloudflare WAF challenge page
90
- if (document.title.includes('Just a moment') ||
91
- document.querySelector('#challenge-stage') !== null ||
92
- document.querySelector('.cf-turnstile') !== null ||
93
- document.querySelector('input[name="cf-turnstile-response"]') !== null) {
94
- return 'turnstile';
95
- }
96
- // reCAPTCHA v2/v3
97
- if (document.querySelector('.g-recaptcha') !== null ||
98
- document.querySelector('iframe[src*="recaptcha"]') !== null ||
99
- document.querySelector('#g-recaptcha-response') !== null) {
100
- return 'recaptcha';
101
- }
102
- // hCaptcha
103
- if (document.querySelector('.h-captcha') !== null ||
104
- document.querySelector('iframe[src*="hcaptcha"]') !== null) {
105
- return 'hcaptcha';
106
- }
107
- return null;
108
- });
109
- if (detectedJsType) {
110
- (0, state_1.notifyProgress)('solve_captcha', 'progress', `🔍 Auto-detected JS captcha type: ${detectedJsType}`);
111
- }
112
- }
113
- // ═══════════════════════════════════════════════════════════════
114
- // STEP 1: ANALYZE PAGE (if enabled)
115
- // ═══════════════════════════════════════════════════════════════
116
- let pageAnalysis = null;
117
- let detectedCaptchaSelector = captchaSelector;
118
- let detectedInputSelector = inputSelector;
119
- if (analyzeFirst) {
120
- (0, state_1.notifyProgress)('solve_captcha', 'progress', '🔍 Analyzing page structure...');
121
- pageAnalysis = await targetFrame.evaluate(() => {
122
- const result = {
123
- captchas: [],
124
- captchaInputs: [],
125
- forms: [],
126
- };
127
- // Find all captcha images
128
- const captchaSelectors = [
129
- 'img[src*="captcha"]', 'img[alt*="captcha"]', 'img[id*="captcha"]',
130
- '.captcha-image', '#captcha_image', '#captchaImg', '.captcha',
131
- 'img[src*="Captcha"]', 'canvas[id*="captcha"]'
132
- ];
133
- captchaSelectors.forEach(sel => {
134
- document.querySelectorAll(sel).forEach((el) => {
135
- if (el.offsetParent !== null) { // Visible
136
- // AI is smart: we don't need hardcoded filters. We just grab candidate images.
137
- // To avoid grabbing top-header logos, we can just check if the image is inside a form.
138
- const isInsideForm = el.closest('form') !== null;
139
- result.captchas.push({
140
- selector: el.id ? `#${el.id}` : sel,
141
- src: el.src || null,
142
- width: el.width,
143
- height: el.height,
144
- isInsideForm: isInsideForm
145
- });
146
- }
147
- });
148
- });
149
- // Find captcha input fields
150
- const inputSelectors = [
151
- 'input[name*="captcha"]', 'input[id*="captcha"]', '#fcaptcha_code',
152
- 'input[placeholder*="captcha"]', 'input[placeholder*="Enter"]',
153
- '#captchaInput', '.captcha-input'
154
- ];
155
- inputSelectors.forEach(sel => {
156
- document.querySelectorAll(sel).forEach((el) => {
157
- if (el.offsetParent !== null && el.type !== 'hidden') {
158
- result.captchaInputs.push({
159
- selector: el.id ? `#${el.id}` : (el.name ? `[name="${el.name}"]` : sel),
160
- placeholder: el.placeholder,
161
- maxLength: el.maxLength > 0 ? el.maxLength : null,
162
- });
163
- }
164
- });
165
- });
166
- // Find forms
167
- document.querySelectorAll('form').forEach(form => {
168
- const hasCaptcha = form.querySelector('img[src*="captcha"], input[name*="captcha"]');
169
- if (hasCaptcha) {
170
- result.forms.push({
171
- id: form.id || null,
172
- action: form.action,
173
- submitBtn: form.querySelector('button[type="submit"], input[type="submit"]')?.id || null,
174
- });
175
- }
176
- });
177
- return result;
178
- });
179
- // Auto-detect selectors if not provided
180
- if (!captchaSelector && pageAnalysis.captchas.length > 0) {
181
- detectedCaptchaSelector = pageAnalysis.captchas[0].selector;
182
- (0, state_1.notifyProgress)('solve_captcha', 'progress', `📍 Auto-detected captcha: ${detectedCaptchaSelector}`);
183
- }
184
- if (!inputSelector && pageAnalysis.captchaInputs.length > 0) {
185
- detectedInputSelector = pageAnalysis.captchaInputs[0].selector;
186
- // Auto-detect expected length from maxLength
187
- if (!expectedLength && pageAnalysis.captchaInputs[0].maxLength) {
188
- params.expectedLength = pageAnalysis.captchaInputs[0].maxLength;
189
- }
190
- (0, state_1.notifyProgress)('solve_captcha', 'progress', `📍 Auto-detected input: ${detectedInputSelector}`);
191
- }
192
- }
193
- // Handle text/image captcha with OCR/Vision
194
- // ponytail: only enter OCR path if explicitly text/image or auto-detected an image captcha (not a JS captcha)
195
- if (type === 'text' || type === 'image' || (type === 'auto' && detectedCaptchaSelector && !detectedJsType)) {
196
- if (!detectedCaptchaSelector) {
197
- return { success: false, error: 'captchaSelector not provided and could not auto-detect' };
198
- }
199
- const effectiveMaxRetries = autoRetry ? maxRetries : 1;
200
- for (let attempt = 1; attempt <= effectiveMaxRetries; attempt++) {
201
- try {
202
- (0, state_1.notifyProgress)('solve_captcha', 'progress', `📸 Capturing CAPTCHA image... (attempt ${attempt}/${effectiveMaxRetries})`);
203
- // If retrying, refresh captcha first
204
- if (attempt > 1 && refreshSelector) {
205
- try {
206
- (0, state_1.notifyProgress)('solve_captcha', 'progress', '🔄 Refreshing CAPTCHA...');
207
- await targetFrame.click(refreshSelector);
208
- await new Promise(r => setTimeout(r, 1500));
209
- }
210
- catch (refreshErr) {
211
- (0, state_1.notifyProgress)('solve_captcha', 'progress', `⚠️ Could not refresh captcha: ${refreshErr.message}`);
212
- }
213
- }
214
- let targetHandle = null;
215
- // 1. SMART AI CONTEXT: Try to capture the form container so AI sees the full context
216
- if (detectedInputSelector) {
217
- try {
218
- const containerHandle = await targetFrame.evaluateHandle((sel) => {
219
- const input = document.querySelector(sel);
220
- if (!input)
221
- return null;
222
- return input.closest('form') || input.closest('div.captcha-wrapper') || input.parentElement?.parentElement || input.parentElement;
223
- }, detectedInputSelector);
224
- if (containerHandle && await containerHandle.evaluate((n) => n !== null)) {
225
- targetHandle = containerHandle.asElement();
226
- }
227
- }
228
- catch (e) { }
229
- }
230
- // 2. Fallback to just the captcha image if no container found
231
- if (!targetHandle && detectedCaptchaSelector) {
232
- try {
233
- targetHandle = await targetFrame.waitForSelector(detectedCaptchaSelector, { timeout: 5000 });
234
- }
235
- catch (e) { }
236
- }
237
- if (!targetHandle) {
238
- return { success: false, error: 'Could not find elements to capture' };
239
- }
240
- // Scroll into view
241
- await targetHandle.evaluate((el) => el.scrollIntoView({ behavior: 'smooth', block: 'center' }));
242
- await new Promise(r => setTimeout(r, 500));
243
- // CAPTCHA इमेज का स्क्रीनशॉट — सीधे AI IDE एजेंट को भेजना है
244
- let captchaImageBase64 = null;
245
- try {
246
- const captchaEl = await targetFrame.$(detectedCaptchaSelector);
247
- if (captchaEl) {
248
- captchaImageBase64 = (await captchaEl.screenshot()).toString('base64');
249
- }
250
- }
251
- catch (e) { }
252
- // पूरे context का स्क्रीनशॉट (form + captcha एक साथ दिखे)
253
- const screenshotBase64 = (await targetHandle.screenshot()).toString('base64');
254
- // ═══════════════════════════════════════════════════════════
255
- // AI IDE एजेंट को CAPTCHA इमेज भेजो — वो पढ़ कर जवाब देगा
256
- // ponytail: कोई external API नहीं, AI IDE एजेंट ही काफ़ी है
257
- // ═══════════════════════════════════════════════════════════
258
- const langHint = lang !== 'eng' ? `\nध्यान दें: टेक्स्ट ${lang === 'hin' ? 'हिन्दी' : lang} भाषा में हो सकता है।` : '';
259
- (0, state_1.notifyProgress)('solve_captcha', 'progress', '📤 CAPTCHA इमेज AI IDE एजेंट को भेज रहे हैं...');
260
- const instructions = `[कार्रवाई आवश्यक: CAPTCHA हल करें]\n\nCAPTCHA इमेज सफलतापूर्वक कैप्चर की गई (संलग्न देखें)।${langHint}\n\n1. अपनी Vision क्षमता से इमेज में दिखे टेक्स्ट/अक्षर पढ़ें।\n2. टेक्स्ट मिलने पर \`type\` टूल से selector \`${detectedInputSelector || '<input_selector>'}\` में भरें।\n3. सबमिट अनुरोध: ${submit ? 'हाँ — फॉर्म सबमिट भी करें' : 'नहीं'}।\n\nनोट: इस CAPTCHA के लिए \`solve_captcha\` दोबारा न बुलाएं, इमेज पहले ही मिल चुकी है।`;
261
- return {
262
- success: true,
263
- mcpContent: [
264
- { type: 'text', text: instructions },
265
- { type: 'image', data: captchaImageBase64 || screenshotBase64, mimeType: 'image/png' }
266
- ]
267
- };
268
- }
269
- catch (err) {
270
- (0, state_1.notifyProgress)('solve_captcha', 'error', `Vision capture error (attempt ${attempt}): ${err.message}`);
271
- if (attempt >= effectiveMaxRetries) {
272
- return { success: false, error: err.message, type: 'vision_capture', formResult };
273
- }
274
- }
275
- }
276
- return { success: false, error: 'All captcha solving attempts exhausted', formResult };
277
- }
278
- // ═══════════════════════════════════════════════════════════════
279
- // JS-BASED CAPTCHA HANDLING (Turnstile / reCAPTCHA / hCaptcha)
280
- // ═══════════════════════════════════════════════════════════════
281
- const effectiveType = type !== 'auto' ? type : (detectedJsType || 'turnstile');
282
- // reCAPTCHA / hCaptcha — not yet implemented, return honest error
283
- // ponytail: no fake support — tell the user the truth
284
- if (effectiveType === 'recaptcha' || effectiveType === 'hcaptcha') {
285
- return {
286
- success: false,
287
- error: `${effectiveType} solving is not yet implemented. Use a third-party service (e.g. 2Captcha) or the browser extension approach.`,
288
- detectedType: effectiveType,
289
- formResult
290
- };
291
- }
292
- // Turnstile / Cloudflare WAF handling — active click + token polling
293
- const start = Date.now();
294
- let attempts = 0;
295
- while (Date.now() - start < timeout) {
296
- attempts++;
297
- // ACTIVE: Try clicking the Cloudflare checkbox (like turnstile.ts does)
298
- // ponytail: click attempt every 3rd iteration to avoid spam, passive poll otherwise
299
- if (attempts % 3 === 1) {
300
- try {
301
- await page.evaluate(() => {
302
- // Method 1: Find turnstile widget container and click checkbox area
303
- const coordinates = [];
304
- document.querySelectorAll('div').forEach(item => {
305
- try {
306
- const rect = item.getBoundingClientRect();
307
- const css = window.getComputedStyle(item);
308
- // Turnstile checkbox container: ~300px wide, no children, minimal styling
309
- if (rect.width > 290 && rect.width <= 310 && !item.querySelector('*')) {
310
- if (css.margin === '0px' && css.padding === '0px') {
311
- coordinates.push({ x: rect.x, y: rect.y, h: rect.height });
312
- }
313
- }
314
- }
315
- catch (_) { }
316
- });
317
- return coordinates;
318
- }).then(async (coords) => {
319
- for (const item of coords) {
320
- await page.mouse.click(item.x + 30, item.y + item.h / 2);
321
- }
322
- });
323
- }
324
- catch (_) { }
325
- }
326
- // PASSIVE: Check if token is ready
327
- const turnstileToken = await page.evaluate(() => {
328
- const input = document.querySelector('input[name="cf-turnstile-response"]');
329
- return input && input.value ? input.value : null;
330
- });
331
- if (turnstileToken) {
332
- (0, state_1.notifyProgress)('solve_captcha', 'completed', `✅ Turnstile solved after ${attempts} checks`, { type: 'turnstile', attempts });
333
- // Handle form submission if requested
334
- if (submit) {
335
- (0, state_1.notifyProgress)('solve_captcha', 'progress', '🚀 Submitting form...');
336
- const submitResult = await helpers_1.helpersHandlers._submitForm(page);
337
- return {
338
- success: true,
339
- type: 'turnstile',
340
- solved: true,
341
- formResult,
342
- submitted: submitResult.success,
343
- submitMessage: submitResult.message
344
- };
345
- }
346
- return { success: true, type: 'turnstile', solved: true, formResult };
347
- }
348
- // Check if Cloudflare challenge page has resolved (redirect happened)
349
- const stillOnChallenge = await page.evaluate(() => {
350
- return document.title.includes('Just a moment') ||
351
- document.querySelector('#challenge-stage') !== null;
352
- });
353
- if (!stillOnChallenge && attempts > 3) {
354
- // Challenge page gone — WAF solved via redirect
355
- (0, state_1.notifyProgress)('solve_captcha', 'completed', `✅ Cloudflare WAF challenge passed after ${attempts} checks (page redirected)`);
356
- return {
357
- success: true,
358
- type: 'cloudflare_waf',
359
- solved: true,
360
- method: 'challenge_redirect',
361
- attempts,
362
- formResult
363
- };
364
- }
365
- if (attempts % 10 === 0) {
366
- (0, state_1.notifyProgress)('solve_captcha', 'progress', `Still solving... (${attempts} checks)`, { attempts });
367
- }
368
- await new Promise(r => setTimeout(r, 1000));
369
- }
370
- (0, state_1.notifyProgress)('solve_captcha', 'error', 'Captcha solving timeout');
371
- return { success: false, error: 'Captcha solving timeout', type: effectiveType, formResult };
8
+ return (0, vision_captcha_1.solveCaptcha)(params);
372
9
  },
373
10
  async see_page(params = {}) {
374
- const { page } = (0, state_1.requireBrowser)();
375
- const { fullPage = false, format = 'jpeg', quality = 70, includeElements = true, annotate = false, includeDomText = false, maxElements = 60, path: savePath, autoHover = false, watchMutations = false } = params;
376
- (0, state_1.notifyProgress)('see_page', 'started', `👁️ Looking at the page (${fullPage ? 'full page' : 'viewport'})${annotate ? ' with Super Annotations' : ''}...`);
377
- let mutationsSinceLastCheck = [];
378
- if (watchMutations) {
379
- mutationsSinceLastCheck = await page.evaluate(() => {
380
- if (!window.__mutations) {
381
- window.__mutations = [];
382
- const observer = new MutationObserver((mutations) => {
383
- for (const m of mutations) {
384
- if (m.addedNodes.length) {
385
- const text = Array.from(m.addedNodes).map((n) => n.innerText || '').join(' ').trim();
386
- if (text.length > 5)
387
- window.__mutations.push({ time: Date.now(), text: text.slice(0, 100) });
388
- }
389
- }
390
- });
391
- observer.observe(document.body, { childList: true, subtree: true });
392
- }
393
- const recent = [...window.__mutations];
394
- window.__mutations = []; // clear after reading
395
- return recent;
396
- });
397
- }
398
- if (autoHover) {
399
- (0, state_1.notifyProgress)('see_page', 'progress', 'Hovering over menus to reveal dropdowns...');
400
- await page.evaluate(() => {
401
- const hoverables = document.querySelectorAll('nav, li, [role="menuitem"], .dropdown, [aria-haspopup="true"]');
402
- hoverables.forEach(el => {
403
- try {
404
- el.dispatchEvent(new MouseEvent('mouseover', { bubbles: true }));
405
- el.dispatchEvent(new MouseEvent('mouseenter', { bubbles: true }));
406
- }
407
- catch (e) { }
408
- });
409
- });
410
- await new Promise(r => setTimeout(r, 400)); // Wait for CSS transitions
411
- }
412
- let elements = [];
413
- let pageInfo = {};
414
- // 1. Build a "visual map" of visible interactive elements (what a human can act on)
415
- // If annotate=true, this also injects visual bounding boxes into the DOM
416
- if (includeElements || annotate) {
417
- const data = await page.evaluate(({ maxEls, isFullPage, doAnnotate }) => {
418
- const out = [];
419
- const seen = new Set();
420
- const sel = 'a[href], button, input, select, textarea, [role="button"], [role="link"], [onclick], [tabindex]';
421
- const nodes = document.querySelectorAll(sel);
422
- const cssPath = (el) => {
423
- if (el.id)
424
- return `#${CSS.escape(el.id)}`;
425
- if (el.name)
426
- return `${el.tagName.toLowerCase()}[name="${el.name}"]`;
427
- const parts = [];
428
- let node = el;
429
- while (node && node.nodeType === 1 && parts.length < 4) {
430
- let part = node.tagName.toLowerCase();
431
- if (node.classList.length) {
432
- const cls = Array.from(node.classList).slice(0, 2).map((c) => '.' + CSS.escape(c)).join('');
433
- part += cls;
434
- }
435
- const parent = node.parentElement;
436
- if (parent) {
437
- const sibs = Array.from(parent.children).filter((c) => c.tagName === node.tagName);
438
- if (sibs.length > 1)
439
- part += `:nth-of-type(${sibs.indexOf(node) + 1})`;
440
- }
441
- parts.unshift(part);
442
- node = node.parentElement;
443
- }
444
- return parts.join(' > ');
445
- };
446
- let elementIdCounter = 1;
447
- const boxesToInject = [];
448
- for (const el of nodes) {
449
- if (out.length >= maxEls)
450
- break;
451
- const rect = el.getBoundingClientRect();
452
- // Only elements actually visible on screen
453
- if (rect.width < 2 || rect.height < 2)
454
- continue;
455
- const style = window.getComputedStyle(el);
456
- if (style.display === 'none' || style.visibility === 'hidden' || style.opacity === '0')
457
- continue;
458
- if (!isFullPage) {
459
- if (rect.bottom < 0 || rect.right < 0 || rect.top > window.innerHeight || rect.left > window.innerWidth) {
460
- // outside current viewport — skip (we report what is seen)
461
- continue;
462
- }
463
- }
464
- const tag = el.tagName.toLowerCase();
465
- let label = (el.innerText || el.value || el.getAttribute('aria-label') || el.getAttribute('placeholder') || el.getAttribute('title') || el.getAttribute('alt') || '').trim().replace(/\s+/g, ' ').slice(0, 80);
466
- let kind = tag;
467
- if (tag === 'a')
468
- kind = 'link';
469
- else if (tag === 'button' || el.getAttribute('role') === 'button')
470
- kind = 'button';
471
- else if (tag === 'input')
472
- kind = `input:${el.type || 'text'}`;
473
- else if (tag === 'select')
474
- kind = 'select';
475
- else if (tag === 'textarea')
476
- kind = 'textarea';
477
- const selector = cssPath(el);
478
- if (seen.has(selector + '|' + label))
479
- continue;
480
- seen.add(selector + '|' + label);
481
- const annotationId = elementIdCounter++;
482
- if (doAnnotate) {
483
- boxesToInject.push({ id: annotationId, rect, tag: kind });
484
- }
485
- out.push({
486
- id: annotationId,
487
- kind,
488
- text: label,
489
- selector,
490
- href: tag === 'a' ? el.href : undefined,
491
- box: { x: Math.round(rect.x), y: Math.round(rect.y), w: Math.round(rect.width), h: Math.round(rect.height) }
492
- });
493
- }
494
- // Inject the boxes into the DOM
495
- if (doAnnotate && boxesToInject.length > 0) {
496
- let container = document.getElementById('real-browser-annotations');
497
- if (container)
498
- container.remove();
499
- container = document.createElement('div');
500
- container.id = 'real-browser-annotations';
501
- // Ensure container sits exactly on top of the document without affecting layout
502
- container.style.position = 'absolute';
503
- container.style.top = '0';
504
- container.style.left = '0';
505
- container.style.width = '100%';
506
- container.style.height = '100%';
507
- container.style.pointerEvents = 'none';
508
- container.style.zIndex = '2147483647'; // Max z-index
509
- for (const box of boxesToInject) {
510
- // Adjust box coordinates by scroll position because container is absolute (document relative)
511
- const absoluteY = box.rect.y + window.scrollY;
512
- const absoluteX = box.rect.x + window.scrollX;
513
- const div = document.createElement('div');
514
- div.style.position = 'absolute';
515
- div.style.border = '2px solid red';
516
- div.style.backgroundColor = 'rgba(255, 0, 0, 0.1)';
517
- div.style.left = absoluteX + 'px';
518
- div.style.top = absoluteY + 'px';
519
- div.style.width = box.rect.width + 'px';
520
- div.style.height = box.rect.height + 'px';
521
- div.style.boxSizing = 'border-box';
522
- const label = document.createElement('div');
523
- label.innerText = String(box.id);
524
- label.style.position = 'absolute';
525
- label.style.top = '-2px';
526
- label.style.left = '-2px';
527
- label.style.backgroundColor = 'red';
528
- label.style.color = 'white';
529
- label.style.fontSize = '12px';
530
- label.style.fontWeight = 'bold';
531
- label.style.padding = '1px 4px';
532
- label.style.fontFamily = 'monospace';
533
- label.style.borderBottomRightRadius = '4px';
534
- div.appendChild(label);
535
- container.appendChild(div);
536
- }
537
- document.body.appendChild(container);
538
- }
539
- return {
540
- elements: out,
541
- info: {
542
- title: document.title,
543
- url: location.href,
544
- viewport: { width: window.innerWidth, height: window.innerHeight },
545
- scrollY: Math.round(window.scrollY),
546
- scrollHeight: document.body ? document.body.scrollHeight : 0
547
- }
548
- };
549
- }, { maxEls: maxElements, isFullPage: fullPage, doAnnotate: annotate }).catch((e) => { console.error(e); return { elements: [], info: {} }; });
550
- elements = data.elements || [];
551
- pageInfo = data.info || {};
552
- // Save elements to state for quick annotation ID mapping
553
- state_1.state.activeAnnotations = {};
554
- for (const el of elements) {
555
- state_1.state.activeAnnotations[el.id] = { selector: el.selector, text: el.text, type: el.kind };
556
- }
557
- }
558
- // 2. Capture what the page looks like (with annotations if injected)
559
- const shotOpts = { type: format, fullPage };
560
- if (format === 'jpeg' && typeof quality === 'number')
561
- shotOpts.quality = quality;
562
- let buffer;
563
- try {
564
- buffer = await page.screenshot(shotOpts);
565
- }
566
- catch (e) {
567
- return { success: false, error: `Vision capture failed: ${e.message}` };
568
- }
569
- // 2.1 Cleanup annotations if we injected them
570
- if (annotate) {
571
- await page.evaluate(() => {
572
- const container = document.getElementById('real-browser-annotations');
573
- if (container)
574
- container.remove();
575
- }).catch(() => { });
576
- }
577
- let domText = undefined;
578
- if (includeDomText) {
579
- domText = await page.evaluate(() => {
580
- return document.body ? document.body.innerText : '';
581
- }).catch(() => '');
582
- }
583
- // 3. Optionally save the image too
584
- let savedTo = null;
585
- if (savePath) {
586
- const dir = path.dirname(savePath);
587
- if (dir && !fs.existsSync(dir))
588
- fs.mkdirSync(dir, { recursive: true });
589
- fs.writeFileSync(savePath, buffer);
590
- savedTo = savePath;
591
- }
592
- (0, state_1.notifyProgress)('see_page', 'completed', `👁️ Saw the page: ${elements.length} interactive elements visible${savedTo ? ' (saved ' + savedTo + ')' : ''}`);
593
- const base64 = Buffer.from(buffer).toString('base64');
594
- const summary = {
595
- success: true,
596
- url: pageInfo.url || page.url(),
597
- title: pageInfo.title,
598
- viewport: pageInfo.viewport,
599
- scroll: { y: pageInfo.scrollY, pageHeight: pageInfo.scrollHeight },
600
- visibleInteractiveElements: elements.length,
601
- domText,
602
- mutationsSinceLastCheck: watchMutations ? mutationsSinceLastCheck : undefined,
603
- elements,
604
- savedTo
605
- };
606
- // Return BOTH the actual image (so the AI literally "sees" it) and the visual map text
607
- return {
608
- mcpContent: [
609
- { type: 'image', data: base64, mimeType: format === 'jpeg' ? 'image/jpeg' : 'image/png' },
610
- { type: 'text', text: JSON.stringify(summary, null, 2) }
611
- ],
612
- ...summary
613
- };
11
+ return (0, vision_see_page_1.seePage)(params);
614
12
  }
615
13
  };
616
14
  //# sourceMappingURL=vision.js.map