real-browser-mcp-server 1.2.0 → 1.2.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (120) hide show
  1. package/README.md +96 -9
  2. package/dist/lib/cjs/index.d.ts +2 -0
  3. package/dist/lib/cjs/index.d.ts.map +1 -0
  4. package/dist/lib/cjs/index.js +385 -0
  5. package/dist/lib/cjs/index.js.map +1 -0
  6. package/dist/lib/cjs/module/pageController.d.ts +2 -0
  7. package/dist/lib/cjs/module/pageController.d.ts.map +1 -0
  8. package/{lib → dist/lib}/cjs/module/pageController.js +28 -29
  9. package/dist/lib/cjs/module/pageController.js.map +1 -0
  10. package/dist/lib/cjs/module/turnstile.d.ts +2 -0
  11. package/dist/lib/cjs/module/turnstile.d.ts.map +1 -0
  12. package/{lib → dist/lib}/cjs/module/turnstile.js +24 -12
  13. package/dist/lib/cjs/module/turnstile.js.map +1 -0
  14. package/dist/src/index.d.ts +11 -0
  15. package/dist/src/index.d.ts.map +1 -0
  16. package/dist/src/index.js +118 -0
  17. package/dist/src/index.js.map +1 -0
  18. package/dist/src/mcp/handlers/browser.d.ts +30 -0
  19. package/dist/src/mcp/handlers/browser.d.ts.map +1 -0
  20. package/dist/src/mcp/handlers/browser.js +231 -0
  21. package/dist/src/mcp/handlers/browser.js.map +1 -0
  22. package/dist/src/mcp/handlers/dom.d.ts +134 -0
  23. package/dist/src/mcp/handlers/dom.d.ts.map +1 -0
  24. package/dist/src/mcp/handlers/dom.js +551 -0
  25. package/dist/src/mcp/handlers/dom.js.map +1 -0
  26. package/dist/src/mcp/handlers/extract.d.ts +59 -0
  27. package/dist/src/mcp/handlers/extract.d.ts.map +1 -0
  28. package/dist/src/mcp/handlers/extract.js +455 -0
  29. package/dist/src/mcp/handlers/extract.js.map +1 -0
  30. package/dist/src/mcp/handlers/form-handlers.d.ts +9 -0
  31. package/dist/src/mcp/handlers/form-handlers.d.ts.map +1 -0
  32. package/dist/src/mcp/handlers/form-handlers.js +56 -0
  33. package/dist/src/mcp/handlers/form-handlers.js.map +1 -0
  34. package/dist/src/mcp/handlers/helpers.d.ts +47 -0
  35. package/dist/src/mcp/handlers/helpers.d.ts.map +1 -0
  36. package/dist/src/mcp/handlers/helpers.js +515 -0
  37. package/dist/src/mcp/handlers/helpers.js.map +1 -0
  38. package/dist/src/mcp/handlers/index.d.ts +6 -0
  39. package/dist/src/mcp/handlers/index.d.ts.map +1 -0
  40. package/dist/src/mcp/handlers/index.js +61 -0
  41. package/dist/src/mcp/handlers/index.js.map +1 -0
  42. package/dist/src/mcp/handlers/media-handlers.d.ts +10 -0
  43. package/dist/src/mcp/handlers/media-handlers.d.ts.map +1 -0
  44. package/dist/src/mcp/handlers/media-handlers.js +535 -0
  45. package/dist/src/mcp/handlers/media-handlers.js.map +1 -0
  46. package/dist/src/mcp/handlers/network.d.ts +147 -0
  47. package/dist/src/mcp/handlers/network.d.ts.map +1 -0
  48. package/dist/src/mcp/handlers/network.js +1135 -0
  49. package/dist/src/mcp/handlers/network.js.map +1 -0
  50. package/dist/src/mcp/handlers/state.d.ts +34 -0
  51. package/dist/src/mcp/handlers/state.d.ts.map +1 -0
  52. package/dist/src/mcp/handlers/state.js +225 -0
  53. package/dist/src/mcp/handlers/state.js.map +1 -0
  54. package/dist/src/mcp/handlers/utility-handlers.d.ts +167 -0
  55. package/dist/src/mcp/handlers/utility-handlers.d.ts.map +1 -0
  56. package/dist/src/mcp/handlers/utility-handlers.js +280 -0
  57. package/dist/src/mcp/handlers/utility-handlers.js.map +1 -0
  58. package/dist/src/mcp/handlers/vision.d.ts +127 -0
  59. package/dist/src/mcp/handlers/vision.d.ts.map +1 -0
  60. package/dist/src/mcp/handlers/vision.js +483 -0
  61. package/dist/src/mcp/handlers/vision.js.map +1 -0
  62. package/dist/src/mcp/index.d.ts +3 -0
  63. package/dist/src/mcp/index.d.ts.map +1 -0
  64. package/dist/src/mcp/index.js +166 -0
  65. package/dist/src/mcp/index.js.map +1 -0
  66. package/dist/src/mcp/server.d.ts +2 -0
  67. package/dist/src/mcp/server.d.ts.map +1 -0
  68. package/dist/src/mcp/server.js +117 -0
  69. package/dist/src/mcp/server.js.map +1 -0
  70. package/dist/src/mcp/tools.d.ts +8 -0
  71. package/dist/src/mcp/tools.d.ts.map +1 -0
  72. package/{src → dist/src}/mcp/tools.js +12 -11
  73. package/dist/src/mcp/tools.js.map +1 -0
  74. package/dist/src/shared/cache-manager.d.ts +80 -0
  75. package/dist/src/shared/cache-manager.d.ts.map +1 -0
  76. package/dist/src/shared/cache-manager.js +221 -0
  77. package/dist/src/shared/cache-manager.js.map +1 -0
  78. package/dist/src/shared/tools.d.ts +2 -0
  79. package/dist/src/shared/tools.d.ts.map +1 -0
  80. package/dist/src/shared/tools.js +599 -0
  81. package/dist/src/shared/tools.js.map +1 -0
  82. package/dist/src/types.d.ts +365 -0
  83. package/dist/src/types.d.ts.map +1 -0
  84. package/dist/src/types.js +9 -0
  85. package/dist/src/types.js.map +1 -0
  86. package/dist/test/cjs/test.d.ts +11 -0
  87. package/dist/test/cjs/test.d.ts.map +1 -0
  88. package/dist/test/cjs/test.js +289 -0
  89. package/dist/test/cjs/test.js.map +1 -0
  90. package/dist/test/mcp/smoke-test.d.ts +29 -0
  91. package/dist/test/mcp/smoke-test.d.ts.map +1 -0
  92. package/dist/test/mcp/smoke-test.js +132 -0
  93. package/dist/test/mcp/smoke-test.js.map +1 -0
  94. package/lib/esm/index.mjs +232 -79
  95. package/lib/esm/module/pageController.mjs +21 -18
  96. package/lib/esm/module/turnstile.mjs +7 -0
  97. package/package.json +25 -15
  98. package/typings.d.ts +12 -6
  99. package/.github/ISSUE_TEMPLATE/general_issue.yaml +0 -58
  100. package/.github/SETUP.md +0 -111
  101. package/.github/workflows/publish.yml +0 -135
  102. package/Dockerfile +0 -79
  103. package/lib/cjs/adblocker.bin +0 -0
  104. package/lib/cjs/index.js +0 -249
  105. package/src/ai/action-parser.js +0 -274
  106. package/src/ai/core.js +0 -378
  107. package/src/ai/element-finder.js +0 -466
  108. package/src/ai/index.js +0 -82
  109. package/src/ai/page-analyzer.js +0 -304
  110. package/src/ai/selector-healer.js +0 -236
  111. package/src/index.js +0 -121
  112. package/src/mcp/handlers.js +0 -5071
  113. package/src/mcp/index.js +0 -190
  114. package/src/mcp/server.js +0 -144
  115. package/src/shared/tools.js +0 -618
  116. package/test/cjs/test.js +0 -259
  117. package/test/esm/package.json +0 -13
  118. package/test/esm/test.js +0 -226
  119. package/test/esm/test_option2.js +0 -46
  120. package/test/esm/test_playwright_ghost.js +0 -30
@@ -0,0 +1,1135 @@
1
+ "use strict";
2
+ Object.defineProperty(exports, "__esModule", { value: true });
3
+ exports.networkHandlers = void 0;
4
+ const state_1 = require("./state");
5
+ // Auto-generated network handlers
6
+ exports.networkHandlers = {
7
+ async redirect_tracer(params) {
8
+ const { page } = (0, state_1.requireBrowser)();
9
+ const { url, maxRedirects = 20, includeHeaders = false, followJS = true, timeout = 30000 } = params;
10
+ (0, state_1.notifyProgress)('redirect_tracer', 'started', `Tracing redirects for: ${url}`);
11
+ const redirects = [];
12
+ const jsNavigations = [];
13
+ let currentUrl = url;
14
+ // HTTP redirect handler
15
+ const responseHandler = (response) => {
16
+ if ([301, 302, 303, 307, 308].includes(response.status())) {
17
+ redirects.push({
18
+ url: response.url(),
19
+ status: response.status(),
20
+ type: 'http',
21
+ headers: includeHeaders ? response.headers() : undefined
22
+ });
23
+ (0, state_1.notifyProgress)('redirect_tracer', 'progress', `HTTP Redirect ${redirects.length}: ${response.status()}`, { status: response.status() });
24
+ }
25
+ };
26
+ // JS/Navigation handler for tracking window.location changes
27
+ const frameNavigatedHandler = (frame) => {
28
+ if (frame === page.mainFrame()) {
29
+ const newUrl = frame.url();
30
+ if (newUrl !== currentUrl && newUrl !== 'about:blank') {
31
+ jsNavigations.push({
32
+ url: newUrl,
33
+ type: 'js_navigation',
34
+ fromUrl: currentUrl,
35
+ timestamp: Date.now()
36
+ });
37
+ (0, state_1.notifyProgress)('redirect_tracer', 'progress', `JS Navigation: ${newUrl}`, { type: 'js' });
38
+ currentUrl = newUrl;
39
+ }
40
+ }
41
+ };
42
+ page.on('response', responseHandler);
43
+ page.on('framenavigated', frameNavigatedHandler);
44
+ try {
45
+ await page.goto(url, { waitUntil: 'networkidle', timeout });
46
+ // If followJS is enabled, wait a bit and check for meta refreshes and JS redirects
47
+ if (followJS) {
48
+ // Check for meta refresh tags
49
+ const metaRefresh = await page.evaluate(() => {
50
+ const meta = document.querySelector('meta[http-equiv="refresh"]');
51
+ if (meta) {
52
+ const content = meta.getAttribute('content');
53
+ const match = content?.match(/url=(.+)/i);
54
+ return match ? match[1].trim().replace(/['"]/g, '') : null;
55
+ }
56
+ return null;
57
+ }).catch(() => null);
58
+ if (metaRefresh) {
59
+ jsNavigations.push({
60
+ url: metaRefresh,
61
+ type: 'meta_refresh',
62
+ fromUrl: page.url()
63
+ });
64
+ }
65
+ // Extract any onclick/href javascript: URLs
66
+ const jsLinks = await page.evaluate(() => {
67
+ const links = [];
68
+ document.querySelectorAll('a[href^="javascript:"], [onclick]').forEach(el => {
69
+ const onclick = el.getAttribute('onclick');
70
+ const href = el.getAttribute('href');
71
+ if (onclick) {
72
+ const match = onclick.match(/location\.href\s*=\s*['"]([^'"]+)['"]/);
73
+ if (match)
74
+ links.push({ url: match[1], type: 'onclick' });
75
+ }
76
+ if (href && href.includes('location')) {
77
+ links.push({ url: href, type: 'javascript_href' });
78
+ }
79
+ });
80
+ return links;
81
+ }).catch(() => []);
82
+ jsNavigations.push(...jsLinks);
83
+ }
84
+ }
85
+ catch (e) {
86
+ (0, state_1.notifyProgress)('redirect_tracer', 'progress', `Navigation error: ${e.message}`);
87
+ }
88
+ page.off('response', responseHandler);
89
+ page.off('framenavigated', frameNavigatedHandler);
90
+ const allRedirects = [
91
+ ...redirects,
92
+ ...jsNavigations.filter(nav => nav.url && nav.url.startsWith('http'))
93
+ ];
94
+ (0, state_1.notifyProgress)('redirect_tracer', 'completed', `Found ${redirects.length} HTTP + ${jsNavigations.length} JS redirects`, { httpRedirects: redirects.length, jsNavigations: jsNavigations.length, finalUrl: page.url() });
95
+ return {
96
+ success: true,
97
+ originalUrl: url,
98
+ finalUrl: page.url(),
99
+ redirectCount: allRedirects.length,
100
+ httpRedirects: redirects,
101
+ jsNavigations: jsNavigations,
102
+ allRedirects: allRedirects
103
+ };
104
+ },
105
+ async network_recorder(params = {}) {
106
+ const { page } = (0, state_1.requireBrowser)();
107
+ const { action = 'get', filter = {}, captureResponses = false } = params;
108
+ switch (action) {
109
+ case 'start':
110
+ state_1.state.networkRecords = [];
111
+ state_1.state.isRecordingNetwork = true;
112
+ // ====== FEATURE 2: Pre-page-load Runtime API Interception ======
113
+ // Inject BEFORE any JS runs — catches calls from obfuscated/webpack code
114
+ try {
115
+ await page.addInitScript(() => {
116
+ window.__interceptedApis = [];
117
+ window.__wsMessages = [];
118
+ // --- Monkey-patch fetch ---
119
+ const origFetch = window.fetch;
120
+ window.fetch = function (...args) {
121
+ try {
122
+ const url = typeof args[0] === 'string' ? args[0] : (args[0]?.url || String(args[0]));
123
+ const opts = args[1] || {};
124
+ const entry = {
125
+ type: 'fetch', url, method: opts.method || 'GET',
126
+ headers: opts.headers ? JSON.parse(JSON.stringify(opts.headers)) : null,
127
+ body: typeof opts.body === 'string' ? opts.body.substring(0, 2000) : null,
128
+ timestamp: Date.now()
129
+ };
130
+ window.__interceptedApis.push(entry);
131
+ }
132
+ catch (e) { }
133
+ return origFetch.apply(this, args);
134
+ };
135
+ // --- Monkey-patch XMLHttpRequest ---
136
+ const origOpen = XMLHttpRequest.prototype.open;
137
+ const origSend = XMLHttpRequest.prototype.send;
138
+ const origSetHeader = XMLHttpRequest.prototype.setRequestHeader;
139
+ XMLHttpRequest.prototype.open = function (method, url, ...rest) {
140
+ this.__iUrl = url;
141
+ this.__iMethod = method;
142
+ this.__iHeaders = {};
143
+ return origOpen.apply(this, [method, url, ...rest]);
144
+ };
145
+ XMLHttpRequest.prototype.setRequestHeader = function (name, value) {
146
+ if (this.__iHeaders)
147
+ this.__iHeaders[name] = value;
148
+ return origSetHeader.apply(this, [name, value]);
149
+ };
150
+ XMLHttpRequest.prototype.send = function (body) {
151
+ try {
152
+ window.__interceptedApis.push({
153
+ type: 'xhr', url: this.__iUrl, method: this.__iMethod,
154
+ headers: this.__iHeaders || null,
155
+ body: typeof body === 'string' ? body.substring(0, 2000) : null,
156
+ timestamp: Date.now()
157
+ });
158
+ }
159
+ catch (e) { }
160
+ return origSend.apply(this, [body]);
161
+ };
162
+ // --- Monkey-patch navigator.sendBeacon ---
163
+ if (navigator.sendBeacon) {
164
+ const origBeacon = navigator.sendBeacon.bind(navigator);
165
+ navigator.sendBeacon = function (url, data) {
166
+ try {
167
+ window.__interceptedApis.push({
168
+ type: 'beacon', url, method: 'POST',
169
+ body: typeof data === 'string' ? data.substring(0, 2000) : null,
170
+ timestamp: Date.now()
171
+ });
172
+ }
173
+ catch (e) { }
174
+ return origBeacon(url, data);
175
+ };
176
+ }
177
+ // ====== FEATURE 3: WebSocket Recording ======
178
+ const OrigWS = window.WebSocket;
179
+ window.WebSocket = function (url, protocols) {
180
+ const ws = protocols ? new OrigWS(url, protocols) : new OrigWS(url);
181
+ const wsId = window.__wsMessages.length;
182
+ const wsEntry = { id: wsId, url, openedAt: Date.now(), messages: [], status: 'connecting' };
183
+ window.__wsMessages.push(wsEntry);
184
+ ws.addEventListener('open', () => { wsEntry.status = 'open'; });
185
+ ws.addEventListener('close', (e) => { wsEntry.status = 'closed'; wsEntry.closedAt = Date.now(); wsEntry.closeCode = e.code; });
186
+ ws.addEventListener('error', () => { wsEntry.status = 'error'; });
187
+ ws.addEventListener('message', (e) => {
188
+ try {
189
+ let data = e.data;
190
+ let dataType = 'text';
191
+ if (data instanceof Blob) {
192
+ dataType = 'blob';
193
+ data = `[Blob ${data.size} bytes]`;
194
+ }
195
+ else if (data instanceof ArrayBuffer) {
196
+ dataType = 'binary';
197
+ data = `[ArrayBuffer ${data.byteLength} bytes]`;
198
+ }
199
+ else if (typeof data === 'string' && data.length > 5000) {
200
+ data = data.substring(0, 5000) + '...';
201
+ }
202
+ wsEntry.messages.push({ direction: 'received', data, dataType, timestamp: Date.now() });
203
+ }
204
+ catch (e) { }
205
+ });
206
+ // Intercept send
207
+ const origWsSend = ws.send.bind(ws);
208
+ ws.send = function (data) {
209
+ try {
210
+ let sendData = data;
211
+ let dataType = 'text';
212
+ if (data instanceof Blob) {
213
+ dataType = 'blob';
214
+ sendData = `[Blob ${data.size} bytes]`;
215
+ }
216
+ else if (data instanceof ArrayBuffer) {
217
+ dataType = 'binary';
218
+ sendData = `[ArrayBuffer ${data.byteLength} bytes]`;
219
+ }
220
+ else if (typeof data === 'string' && data.length > 5000) {
221
+ sendData = data.substring(0, 5000) + '...';
222
+ }
223
+ wsEntry.messages.push({ direction: 'sent', data: sendData, dataType, timestamp: Date.now() });
224
+ }
225
+ catch (e) { }
226
+ return origWsSend(data);
227
+ };
228
+ return ws;
229
+ };
230
+ window.WebSocket.prototype = OrigWS.prototype;
231
+ window.WebSocket.CONNECTING = OrigWS.CONNECTING;
232
+ window.WebSocket.OPEN = OrigWS.OPEN;
233
+ window.WebSocket.CLOSING = OrigWS.CLOSING;
234
+ window.WebSocket.CLOSED = OrigWS.CLOSED;
235
+ });
236
+ }
237
+ catch (e) { /* addInitScript may fail on already-loaded pages, that's OK */ }
238
+ // Request handler
239
+ page.on('request', req => {
240
+ if (state_1.state.isRecordingNetwork) {
241
+ state_1.state.networkRecords.push({
242
+ type: 'request',
243
+ url: req.url(),
244
+ method: req.method(),
245
+ resourceType: req.resourceType(),
246
+ headers: req.headers(),
247
+ timestamp: Date.now()
248
+ });
249
+ }
250
+ });
251
+ // Response handler for capturing video/media URLs
252
+ page.on('response', async (res) => {
253
+ if (state_1.state.isRecordingNetwork) {
254
+ const url = res.url();
255
+ const contentType = res.headers()['content-type'] || '';
256
+ const isMedia = contentType.includes('video') ||
257
+ contentType.includes('audio') ||
258
+ contentType.includes('mpegurl') ||
259
+ url.includes('.m3u8') ||
260
+ url.includes('.mpd') ||
261
+ url.includes('.mp4') ||
262
+ url.includes('.ts');
263
+ const isApiCall = contentType.includes('json') ||
264
+ contentType.includes('x-www-form-urlencoded') ||
265
+ url.match(/\.(php|api|json|do|action)($|\?)/) ||
266
+ (res.request().resourceType() === 'xhr') ||
267
+ (res.request().resourceType() === 'fetch');
268
+ const record = {
269
+ type: 'response',
270
+ url: url,
271
+ method: res.request().method(),
272
+ status: res.status(),
273
+ contentType: contentType,
274
+ isMedia: isMedia,
275
+ isApiCall: isApiCall,
276
+ resourceType: res.request().resourceType(),
277
+ timestamp: Date.now()
278
+ };
279
+ // For media URLs, try to get more details
280
+ if (isMedia) {
281
+ record.mediaType = url.includes('.m3u8') ? 'hls' :
282
+ url.includes('.mpd') ? 'dash' :
283
+ url.includes('.mp4') ? 'mp4' : 'other';
284
+ }
285
+ // Capture request/response body for API calls
286
+ if (isApiCall) {
287
+ try {
288
+ const postData = res.request().postData();
289
+ if (postData)
290
+ record.requestBody = postData.substring(0, 2000);
291
+ const responseBody = await res.text().catch(() => null);
292
+ if (responseBody) {
293
+ record.responseBody = responseBody.substring(0, 5000);
294
+ try {
295
+ record.responseJson = JSON.parse(responseBody);
296
+ }
297
+ catch (e) { }
298
+ }
299
+ }
300
+ catch (e) { }
301
+ }
302
+ state_1.state.networkRecords.push(record);
303
+ }
304
+ });
305
+ // Frame navigation handler for JS redirects
306
+ page.on('framenavigated', frame => {
307
+ if (state_1.state.isRecordingNetwork && frame === page.mainFrame()) {
308
+ state_1.state.networkRecords.push({
309
+ type: 'navigation',
310
+ url: frame.url(),
311
+ timestamp: Date.now()
312
+ });
313
+ }
314
+ });
315
+ (0, state_1.notifyProgress)('network_recorder', 'started', 'Power recording started (requests + responses + API interception + WebSocket + navigations)');
316
+ break;
317
+ case 'stop':
318
+ state_1.state.isRecordingNetwork = false;
319
+ (0, state_1.notifyProgress)('network_recorder', 'completed', `Recording stopped: ${state_1.state.networkRecords.length} events captured`);
320
+ break;
321
+ case 'clear':
322
+ state_1.state.networkRecords = [];
323
+ // Also clear intercepted data
324
+ try {
325
+ await page.evaluate(() => { window.__interceptedApis = []; window.__wsMessages = []; });
326
+ }
327
+ catch (e) { }
328
+ (0, state_1.notifyProgress)('network_recorder', 'completed', 'Network records cleared');
329
+ break;
330
+ case 'get_media':
331
+ // Special action to get only media URLs
332
+ const mediaRecords = state_1.state.networkRecords.filter(r => r.isMedia);
333
+ return {
334
+ success: true,
335
+ count: mediaRecords.length,
336
+ mediaUrls: mediaRecords.map(r => ({ url: r.url, type: r.mediaType }))
337
+ };
338
+ case 'get_navigations':
339
+ // Get only navigation events (for tracking JS redirects)
340
+ const navRecords = state_1.state.networkRecords.filter(r => r.type === 'navigation');
341
+ return {
342
+ success: true,
343
+ count: navRecords.length,
344
+ navigations: navRecords
345
+ };
346
+ case 'get_api_calls': {
347
+ const apiRecords = state_1.state.networkRecords.filter(r => r.isApiCall);
348
+ return {
349
+ success: true,
350
+ count: apiRecords.length,
351
+ apiCalls: apiRecords.map(r => ({
352
+ url: r.url, method: r.method || 'GET', status: r.status,
353
+ contentType: r.contentType, resourceType: r.resourceType,
354
+ requestBody: r.requestBody || null, responseBody: r.responseBody || null,
355
+ responseJson: r.responseJson || null, timestamp: r.timestamp
356
+ }))
357
+ };
358
+ }
359
+ // ====== FEATURE 2: Get Intercepted APIs (from monkey-patched fetch/XHR/beacon) ======
360
+ case 'get_intercepted_apis': {
361
+ try {
362
+ const intercepted = await page.evaluate(() => window.__interceptedApis || []);
363
+ return {
364
+ success: true,
365
+ count: intercepted.length,
366
+ interceptedApis: intercepted,
367
+ note: 'These are runtime-intercepted API calls captured via monkey-patched fetch/XHR/sendBeacon (pre-page-load injection)'
368
+ };
369
+ }
370
+ catch (e) {
371
+ return { success: false, error: 'Failed to retrieve intercepted APIs: ' + e.message, interceptedApis: [] };
372
+ }
373
+ }
374
+ // ====== FEATURE 3: Get WebSocket Messages ======
375
+ case 'get_websockets': {
376
+ try {
377
+ const wsData = await page.evaluate(() => window.__wsMessages || []);
378
+ const totalMessages = wsData.reduce((sum, ws) => sum + ws.messages.length, 0);
379
+ return {
380
+ success: true,
381
+ count: wsData.length,
382
+ totalMessages: totalMessages,
383
+ websockets: wsData,
384
+ note: 'WebSocket connections and messages captured via constructor monkey-patch'
385
+ };
386
+ }
387
+ catch (e) {
388
+ return { success: false, error: 'Failed to retrieve WebSocket data: ' + e.message, websockets: [] };
389
+ }
390
+ }
391
+ }
392
+ let records = state_1.state.networkRecords;
393
+ if (filter.resourceType) {
394
+ records = records.filter(r => r.resourceType === filter.resourceType);
395
+ }
396
+ if (filter.urlPattern) {
397
+ const regex = new RegExp(filter.urlPattern);
398
+ records = records.filter(r => regex.test(r.url));
399
+ }
400
+ if (filter.type) {
401
+ records = records.filter(r => r.type === filter.type);
402
+ }
403
+ if (filter.mediaOnly) {
404
+ records = records.filter(r => r.isMedia);
405
+ }
406
+ return { success: true, recording: state_1.state.isRecordingNetwork, count: records.length, records: records.slice(-200) };
407
+ },
408
+ async cookie_manager(params = {}) {
409
+ const { page } = (0, state_1.requireBrowser)();
410
+ const { action = 'get', name, value, domain, expires } = params;
411
+ (0, state_1.notifyProgress)('cookie_manager', 'started', `Cookie action: ${action}`);
412
+ // Playwright/Patchright: cookies are managed via the BrowserContext, not the page
413
+ const context = page.context();
414
+ switch (action) {
415
+ case 'get': {
416
+ const cookies = await context.cookies();
417
+ (0, state_1.notifyProgress)('cookie_manager', 'completed', `Retrieved ${cookies.length} cookies`);
418
+ return { success: true, cookies: name ? cookies.filter(c => c.name === name) : cookies };
419
+ }
420
+ case 'set': {
421
+ await context.addCookies([{
422
+ name,
423
+ value,
424
+ domain: domain || new URL(page.url()).hostname,
425
+ path: '/',
426
+ ...(expires ? { expires } : {})
427
+ }]);
428
+ (0, state_1.notifyProgress)('cookie_manager', 'completed', `Cookie set: ${name}`);
429
+ return { success: true, message: `Cookie ${name} set` };
430
+ }
431
+ case 'delete': {
432
+ // Playwright has no per-cookie delete: clear all, then re-add the ones we keep
433
+ const toDelete = await context.cookies();
434
+ const remaining = name ? toDelete.filter(c => c.name !== name) : [];
435
+ const removedCount = toDelete.length - remaining.length;
436
+ await context.clearCookies();
437
+ if (remaining.length) {
438
+ await context.addCookies(remaining.map(c => ({
439
+ name: c.name, value: c.value, domain: c.domain, path: c.path,
440
+ ...(c.expires && c.expires > 0 ? { expires: c.expires } : {}),
441
+ httpOnly: c.httpOnly, secure: c.secure, sameSite: c.sameSite
442
+ })));
443
+ }
444
+ (0, state_1.notifyProgress)('cookie_manager', 'completed', `Deleted ${removedCount} cookie(s)`);
445
+ return { success: true, message: `Deleted ${removedCount} cookie(s)` };
446
+ }
447
+ case 'clear': {
448
+ const allCookies = await context.cookies();
449
+ await context.clearCookies();
450
+ (0, state_1.notifyProgress)('cookie_manager', 'completed', `Cleared ${allCookies.length} cookies`);
451
+ return { success: true, message: `Cleared ${allCookies.length} cookies` };
452
+ }
453
+ }
454
+ return { success: false, error: 'Invalid action' };
455
+ },
456
+ async extract_data(params = {}) {
457
+ const { page } = (0, state_1.requireBrowser)();
458
+ const { type = 'auto', pattern, selector, jsonPath, source = 'all', autoDecode = true, flags = 'gi', types = ['all'], includeTitle = true, includeCanonical = true, maxMatches = 100, maxJsonObjects = 50, waitForSelector = false, selectorTimeout = 10000 } = params;
459
+ (0, state_1.notifyProgress)('extract_data', 'started', `Extracting data (type: ${type})...`);
460
+ const results = {
461
+ success: true,
462
+ type,
463
+ url: page.url(),
464
+ extracted: {}
465
+ };
466
+ // Helper: Extract regex matches
467
+ const extractRegex = async (regexPattern, regexFlags, contentSource) => {
468
+ let content;
469
+ if (contentSource === 'html') {
470
+ content = await page.content();
471
+ }
472
+ else if (contentSource === 'scripts') {
473
+ content = await page.$$eval('script', scripts => scripts.map(s => s.textContent).join('\n'));
474
+ }
475
+ else if (contentSource === 'text') {
476
+ content = await page.evaluate(() => document.body.innerText);
477
+ }
478
+ else {
479
+ // 'all' - search in both HTML and scripts
480
+ const html = await page.content();
481
+ const scripts = await page.$$eval('script', scripts => scripts.map(s => s.textContent).join('\n'));
482
+ content = html + '\n' + scripts;
483
+ }
484
+ const regex = new RegExp(regexPattern, regexFlags);
485
+ const matches = content.match(regex) || [];
486
+ return {
487
+ pattern: regexPattern,
488
+ flags: regexFlags,
489
+ matchCount: matches.length,
490
+ matches: matches.slice(0, maxMatches)
491
+ };
492
+ };
493
+ // Helper: Extract JSON data
494
+ const extractJson = async (jsonSource, sel, path) => {
495
+ const jsonData = [];
496
+ if (jsonSource === 'ld+json') {
497
+ const ldJson = await page.$$eval('script[type="application/ld+json"]', scripts => scripts.map(s => {
498
+ try {
499
+ return JSON.parse(s.textContent);
500
+ }
501
+ catch {
502
+ return null;
503
+ }
504
+ }).filter(Boolean));
505
+ jsonData.push(...ldJson);
506
+ }
507
+ else if (jsonSource === 'scripts') {
508
+ const content = await page.$$eval('script', scripts => scripts.map(s => s.textContent).join('\n'));
509
+ // Look for JSON objects in scripts
510
+ const jsonRegex = /\{[^{}]*(?:\{[^{}]*\}[^{}]*)*\}|\[[^\[\]]*(?:\[[^\[\]]*\][^\[\]]*)*\]/g;
511
+ const matches = content.match(jsonRegex) || [];
512
+ for (const match of matches.slice(0, maxJsonObjects)) {
513
+ try {
514
+ const parsed = JSON.parse(match);
515
+ jsonData.push(parsed);
516
+ }
517
+ catch { }
518
+ }
519
+ }
520
+ else if (jsonSource === 'api') {
521
+ // Try to find API responses in page data
522
+ const apiData = await page.evaluate(() => {
523
+ const data = [];
524
+ // Look for common API data storage patterns
525
+ if (window.__DATA__)
526
+ data.push(window.__DATA__);
527
+ if (window.__INITIAL_STATE__)
528
+ data.push(window.__INITIAL_STATE__);
529
+ if (window.__APP_DATA__)
530
+ data.push(window.__APP_DATA__);
531
+ if (window.data)
532
+ data.push(window.data);
533
+ if (window.config)
534
+ data.push(window.config);
535
+ return data;
536
+ });
537
+ jsonData.push(...apiData);
538
+ }
539
+ else if (sel) {
540
+ try {
541
+ const text = await page.$eval(sel, el => el.textContent);
542
+ const parsed = JSON.parse(text);
543
+ jsonData.push(parsed);
544
+ }
545
+ catch { }
546
+ }
547
+ else {
548
+ // 'page' - try all sources
549
+ const ldJson = await page.$$eval('script[type="application/ld+json"]', scripts => scripts.map(s => {
550
+ try {
551
+ return JSON.parse(s.textContent);
552
+ }
553
+ catch {
554
+ return null;
555
+ }
556
+ }).filter(Boolean));
557
+ jsonData.push(...ldJson);
558
+ }
559
+ // Apply JSONPath if specified
560
+ if (path && jsonData.length > 0) {
561
+ // Simple JSONPath implementation
562
+ const getPath = (obj, pathStr) => {
563
+ const parts = pathStr.replace(/^\$\./, '').split('.');
564
+ let current = obj;
565
+ for (const part of parts) {
566
+ if (current === null || current === undefined)
567
+ return undefined;
568
+ if (part.includes('[') && part.includes(']')) {
569
+ const arrName = part.substring(0, part.indexOf('['));
570
+ const idx = parseInt(part.match(/\[(\d+)\]/)?.[1] || '0');
571
+ current = current[arrName]?.[idx];
572
+ }
573
+ else {
574
+ current = current[part];
575
+ }
576
+ }
577
+ return current;
578
+ };
579
+ return jsonData.map(obj => ({
580
+ original: obj,
581
+ extracted: getPath(obj, path)
582
+ }));
583
+ }
584
+ return jsonData;
585
+ };
586
+ // Helper: Extract meta tags
587
+ const extractMeta = async (metaTypes) => {
588
+ const meta = await page.evaluate(([includeTitle, includeCanonical]) => {
589
+ const result = { meta: {}, og: {}, twitter: {} };
590
+ document.querySelectorAll('meta').forEach(tag => {
591
+ const name = tag.getAttribute('name') || tag.getAttribute('property');
592
+ const content = tag.getAttribute('content');
593
+ if (name && content) {
594
+ if (name.startsWith('og:')) {
595
+ result.og[name.replace('og:', '')] = content;
596
+ }
597
+ else if (name.startsWith('twitter:')) {
598
+ result.twitter[name.replace('twitter:', '')] = content;
599
+ }
600
+ else {
601
+ result.meta[name] = content;
602
+ }
603
+ }
604
+ });
605
+ if (includeTitle) {
606
+ result.title = document.title;
607
+ }
608
+ if (includeCanonical) {
609
+ result.canonical = document.querySelector('link[rel="canonical"]')?.href;
610
+ }
611
+ return result;
612
+ }, [includeTitle, includeCanonical]);
613
+ // Filter by requested types
614
+ const filtered = {};
615
+ if (metaTypes.includes('all')) {
616
+ return meta;
617
+ }
618
+ if (metaTypes.includes('meta'))
619
+ filtered.meta = meta.meta;
620
+ if (metaTypes.includes('og'))
621
+ filtered.og = meta.og;
622
+ if (metaTypes.includes('twitter'))
623
+ filtered.twitter = meta.twitter;
624
+ if (includeTitle)
625
+ filtered.title = meta.title;
626
+ if (includeCanonical)
627
+ filtered.canonical = meta.canonical;
628
+ return filtered;
629
+ };
630
+ // Helper: Extract structured data from selector
631
+ const extractStructured = async (sel, wait = false, timeout = 10000) => {
632
+ if (wait) {
633
+ await page.waitForSelector(sel, { timeout });
634
+ }
635
+ const element = await page.$(sel);
636
+ if (!element) {
637
+ return { error: `Element not found: ${sel}` };
638
+ }
639
+ const data = await element.evaluate(el => ({
640
+ tagName: el.tagName,
641
+ text: el.innerText,
642
+ html: el.innerHTML,
643
+ attributes: Object.fromEntries([...el.attributes].map(a => [a.name, a.value])),
644
+ childCount: el.children.length,
645
+ boundingBox: el.getBoundingClientRect ? {
646
+ x: el.getBoundingClientRect().x,
647
+ y: el.getBoundingClientRect().y,
648
+ width: el.getBoundingClientRect().width,
649
+ height: el.getBoundingClientRect().height
650
+ } : null
651
+ }));
652
+ return data;
653
+ };
654
+ // Helper: Auto-detect and extract all
655
+ const extractAuto = async () => {
656
+ const autoResults = {
657
+ meta: null,
658
+ json: null,
659
+ structured: null,
660
+ patterns: []
661
+ };
662
+ // Extract meta tags
663
+ try {
664
+ autoResults.meta = await extractMeta(['all']);
665
+ }
666
+ catch (e) { }
667
+ // Extract JSON-LD
668
+ try {
669
+ autoResults.json = await extractJson('ld+json');
670
+ }
671
+ catch (e) { }
672
+ // Look for common data patterns
673
+ const commonPatterns = [
674
+ { name: 'emails', pattern: '[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}' },
675
+ { name: 'phones', pattern: '(\+?1?[-.\s]?)?\(?[0-9]{3}\)?[-.\s]?[0-9]{3}[-.\s]?[0-9]{4}' },
676
+ { name: 'urls', pattern: 'https?://[^\s<>"{}|\\^`\[\]]+' },
677
+ { name: 'ipv4', pattern: '\b(?:(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.){3}(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\b' }
678
+ ];
679
+ const pageText = await page.evaluate(() => document.body.innerText);
680
+ for (const { name, pattern } of commonPatterns) {
681
+ const regex = new RegExp(pattern, 'gi');
682
+ const matches = [...new Set(pageText.match(regex) || [])];
683
+ if (matches.length > 0) {
684
+ autoResults.patterns.push({ type: name, count: matches.length, samples: matches.slice(0, 10) });
685
+ }
686
+ }
687
+ return autoResults;
688
+ };
689
+ // Main switch based on type
690
+ switch (type) {
691
+ case 'regex': {
692
+ if (!pattern) {
693
+ return { success: false, error: 'Pattern is required for regex extraction' };
694
+ }
695
+ results.extracted = await extractRegex(pattern, flags, source);
696
+ (0, state_1.notifyProgress)('extract_data', 'completed', `Regex: ${results.extracted.matchCount} matches`);
697
+ break;
698
+ }
699
+ case 'json': {
700
+ results.extracted = await extractJson(source, selector, jsonPath);
701
+ results.count = Array.isArray(results.extracted) ? results.extracted.length : 0;
702
+ (0, state_1.notifyProgress)('extract_data', 'completed', `JSON: ${results.count} objects`);
703
+ break;
704
+ }
705
+ case 'meta': {
706
+ results.extracted = await extractMeta(types);
707
+ const tagCount = Object.values(results.extracted).reduce((sum, val) => {
708
+ if (typeof val === 'object' && val !== null) {
709
+ return sum + Object.keys(val).length;
710
+ }
711
+ return sum + (val ? 1 : 0);
712
+ }, 0);
713
+ (0, state_1.notifyProgress)('extract_data', 'completed', `Meta: ${tagCount} tags`);
714
+ break;
715
+ }
716
+ case 'structured': {
717
+ if (!selector) {
718
+ return { success: false, error: 'Selector is required for structured extraction' };
719
+ }
720
+ results.extracted = await extractStructured(selector, waitForSelector, selectorTimeout);
721
+ if (results.extracted.error) {
722
+ results.success = false;
723
+ results.error = results.extracted.error;
724
+ delete results.extracted;
725
+ }
726
+ (0, state_1.notifyProgress)('extract_data', 'completed', results.success ? 'Structured data extracted' : 'Extraction failed');
727
+ break;
728
+ }
729
+ case 'auto': {
730
+ results.extracted = await extractAuto();
731
+ const summary = [];
732
+ if (results.extracted.meta)
733
+ summary.push('meta');
734
+ if (results.extracted.json?.length)
735
+ summary.push('json');
736
+ if (results.extracted.patterns?.length)
737
+ summary.push('patterns');
738
+ (0, state_1.notifyProgress)('extract_data', 'completed', `Auto: ${summary.join(', ')}`);
739
+ break;
740
+ }
741
+ case 'deobfuscate': {
742
+ (0, state_1.notifyProgress)('extract_data', 'in_progress', 'Deobfuscating JavaScript (enhanced)...');
743
+ const scriptContents = await page.evaluate(() => {
744
+ return Array.from(document.querySelectorAll('script')).map(s => s.textContent).join('\n');
745
+ }).catch(() => '');
746
+ const externalScripts = await page.evaluate(() => {
747
+ return Array.from(document.querySelectorAll('script[src]')).map(s => s.src);
748
+ }).catch(() => []);
749
+ let allJs = scriptContents;
750
+ for (const src of externalScripts.slice(0, 10)) {
751
+ try {
752
+ const resp = await fetch(src);
753
+ allJs += '\n' + await resp.text();
754
+ }
755
+ catch (e) { }
756
+ }
757
+ const deobfuscated = {
758
+ stringArrays: [], decodedStrings: [], functionMappings: [],
759
+ apiEndpoints: [], urls: [], fetchCalls: [],
760
+ webpackModules: [], evalUnpacked: [], resolvedConcats: [], unicodeDecoded: []
761
+ };
762
+ // 1. Original _0x style string arrays
763
+ const arrayPattern = /(?:const|var|let)\s+(_0x[a-f0-9]+)\s*=\s*\[([^\]]{20,})\]/g;
764
+ let match;
765
+ while ((match = arrayPattern.exec(allJs)) !== null) {
766
+ const varName = match[1];
767
+ try {
768
+ const items = match[2].match(/'([^']*)'|"([^"]*)"/g) || [];
769
+ const decoded = items.map(s => s.replace(/^['"]|['"]$/g, ''));
770
+ deobfuscated.stringArrays.push({ variable: varName, count: decoded.length, strings: decoded });
771
+ deobfuscated.decodedStrings.push(...decoded);
772
+ }
773
+ catch (e) { }
774
+ }
775
+ // 2. Hex-encoded strings
776
+ const hexStrings = [...new Set((allJs.match(/(?:'(?:\\x[0-9a-f]{2})+[^']*'|"(?:\\x[0-9a-f]{2})+[^"]*")/gi) || []))];
777
+ for (const hs of hexStrings.slice(0, 50)) {
778
+ try {
779
+ const decoded = hs.slice(1, -1).replace(/\\x([0-9a-f]{2})/gi, (_, h) => String.fromCharCode(parseInt(h, 16)));
780
+ if (decoded.length > 2)
781
+ deobfuscated.decodedStrings.push(decoded);
782
+ }
783
+ catch (e) { }
784
+ }
785
+ // 3. NEW: Unicode escape sequences (\u0066\u0065\u0074\u0063\u0068 → fetch)
786
+ const unicodePattern = /(?:'(?:\\u[0-9a-f]{4})+[^']*'|"(?:\\u[0-9a-f]{4})+[^"]*")/gi;
787
+ const unicodeMatches = allJs.match(unicodePattern) || [];
788
+ for (const um of unicodeMatches.slice(0, 50)) {
789
+ try {
790
+ const decoded = um.slice(1, -1).replace(/\\u([0-9a-f]{4})/gi, (_, h) => String.fromCharCode(parseInt(h, 16)));
791
+ if (decoded.length > 1) {
792
+ deobfuscated.unicodeDecoded.push(decoded);
793
+ deobfuscated.decodedStrings.push(decoded);
794
+ }
795
+ }
796
+ catch (e) { }
797
+ }
798
+ // 4. NEW: Eval unpacker — eval(function(p,a,c,k,e,d){...})
799
+ const evalPattern = /eval\s*\(\s*function\s*\(\s*p\s*,\s*a\s*,\s*c\s*,\s*k\s*,\s*e\s*,?\s*[dr]?\s*\)\s*\{[^}]*\}\s*\(\s*'([^']*)'(?:\s*,\s*(\d+)){2}\s*,\s*'([^']*)'/g;
800
+ let evalMatch;
801
+ while ((evalMatch = evalPattern.exec(allJs)) !== null) {
802
+ try {
803
+ const p = evalMatch[1], a = parseInt(evalMatch[2]) || 62;
804
+ const keywords = evalMatch[3].split('|');
805
+ const unpacked = p.replace(/\b\w+\b/g, w => {
806
+ const n = parseInt(w, a);
807
+ return (n < keywords.length && keywords[n]) ? keywords[n] : w;
808
+ });
809
+ deobfuscated.evalUnpacked.push(unpacked.substring(0, 3000));
810
+ // Extract strings from unpacked code
811
+ const unpackedStrings = unpacked.match(/['"]([^'"]{3,})['"]/g) || [];
812
+ for (const s of unpackedStrings.slice(0, 100)) {
813
+ deobfuscated.decodedStrings.push(s.replace(/^['"]|['"]$/g, ''));
814
+ }
815
+ }
816
+ catch (e) { }
817
+ }
818
+ // Also handle simpler eval patterns
819
+ const simpleEval = /eval\s*\(\s*['"]([^'"]{10,})['"]\s*\)/g;
820
+ let seMatch;
821
+ while ((seMatch = simpleEval.exec(allJs)) !== null) {
822
+ deobfuscated.evalUnpacked.push(seMatch[1].substring(0, 2000));
823
+ }
824
+ // 5. NEW: Webpack module detection
825
+ const webpackPatterns = [
826
+ /(?:__webpack_require__|__webpack_modules__)\s*\[\s*['"]?(\w+)['"]?\s*\]/g,
827
+ /(?:const|var|let)\s+\w+\s*=\s*\{[\s\S]{0,50}__webpack_require__/g,
828
+ /\(\s*function\s*\(\s*modules\s*\)\s*\{[\s\S]{0,200}__webpack_require__/g
829
+ ];
830
+ const webpackExports = allJs.match(/(?:module\.exports|exports\.\w+)\s*=\s*['"]([^'"]+)['"]/g) || [];
831
+ for (const exp of webpackExports.slice(0, 30)) {
832
+ const val = exp.match(/=\s*['"]([^'"]+)['"]/);
833
+ if (val) {
834
+ deobfuscated.webpackModules.push(val[1]);
835
+ deobfuscated.decodedStrings.push(val[1]);
836
+ }
837
+ }
838
+ // Detect webpack chunk loading and module IDs
839
+ const chunkIds = allJs.match(/webpackChunk\w*\.push\s*\(\s*\[\s*\[([^\]]+)\]/g) || [];
840
+ for (const ci of chunkIds.slice(0, 10)) {
841
+ deobfuscated.webpackModules.push(`chunk: ${ci.substring(0, 100)}`);
842
+ }
843
+ // 6. NEW: Terser/UglifyJS single-letter variable mappings
844
+ const terserPattern = /(?:var|let|const)\s+([a-z])\s*=\s*['"]([^'"]{2,})['"]/gi;
845
+ let terserMatch;
846
+ const terserMappings = {};
847
+ while ((terserMatch = terserPattern.exec(allJs)) !== null) {
848
+ const varName = terserMatch[1], value = terserMatch[2];
849
+ if (value.length > 2 && value.length < 200) {
850
+ terserMappings[varName] = value;
851
+ deobfuscated.functionMappings.push({ variable: varName, value: value });
852
+ deobfuscated.decodedStrings.push(value);
853
+ }
854
+ }
855
+ // 7. NEW: String concatenation resolution ("htt"+"ps://" → "https://")
856
+ const concatPattern = /(?:['"][^'"]*['"]\s*\+\s*){2,}['"][^'"]*['"]/g;
857
+ const concatMatches = allJs.match(concatPattern) || [];
858
+ for (const cm of concatMatches.slice(0, 50)) {
859
+ try {
860
+ const parts = cm.match(/['"]([^'"]*)['"]|(['"])/g) || [];
861
+ const resolved = parts.map(p => p.replace(/^['"]|['"]$/g, '')).join('');
862
+ if (resolved.length > 3) {
863
+ deobfuscated.resolvedConcats.push(resolved);
864
+ deobfuscated.decodedStrings.push(resolved);
865
+ }
866
+ }
867
+ catch (e) { }
868
+ }
869
+ // 8. NEW: Array rotation detection — function with push/shift on array
870
+ const rotationPattern = /function\s+\w*\s*\(\s*(_0x[a-f0-9]+)\s*,\s*\w+\s*\)\s*\{[\s\S]{0,500}push\s*\(\s*\1\s*\.\s*shift\s*\(\s*\)\s*\)/g;
871
+ const rotations = allJs.match(rotationPattern) || [];
872
+ if (rotations.length > 0) {
873
+ deobfuscated.functionMappings.push({ type: 'array_rotation', count: rotations.length, note: 'Array rotation functions detected — strings may be shifted' });
874
+ }
875
+ // Extract URLs and API endpoints from all decoded strings
876
+ deobfuscated.urls = [...new Set(deobfuscated.decodedStrings.filter((s) => s.match(/^(https?:\/\/|\/)/) || s.match(/\.(php|json|api|asp|jsp)$/i)))].slice(0, 50);
877
+ deobfuscated.apiEndpoints = [...new Set(deobfuscated.decodedStrings.filter((s) => s.match(/^\/[a-z]/i) && s.length > 3 && s.length < 100))].slice(0, 30);
878
+ // Find fetch patterns
879
+ const fetchPatterns = allJs.match(/fetch\s*\(\s*['"]([^'"]+)['"]/g) || [];
880
+ deobfuscated.fetchCalls = fetchPatterns.map(f => f.replace(/fetch\s*\(\s*['"]/, '').replace(/['"]$/, '')).slice(0, 20);
881
+ deobfuscated.decodedStrings = [...new Set(deobfuscated.decodedStrings)].slice(0, 500);
882
+ results.extracted = deobfuscated;
883
+ const summary = `${deobfuscated.stringArrays.length} arrays, ${deobfuscated.decodedStrings.length} strings, ${deobfuscated.evalUnpacked.length} eval unpacked, ${deobfuscated.webpackModules.length} webpack modules, ${deobfuscated.resolvedConcats.length} concats resolved, ${deobfuscated.unicodeDecoded.length} unicode decoded`;
884
+ (0, state_1.notifyProgress)('extract_data', 'completed', `Deobfuscated(enhanced): ${summary}`);
885
+ break;
886
+ }
887
+ case 'apiDiscovery': {
888
+ (0, state_1.notifyProgress)('extract_data', 'in_progress', 'Discovering hidden API endpoints...');
889
+ const apiResults = {
890
+ fetchEndpoints: [], xhrEndpoints: [], formActions: [],
891
+ scriptSources: [], inlineApiPatterns: [], postBodies: [], dynamicApis: []
892
+ };
893
+ // 1. Intercept runtime fetch/XHR
894
+ try {
895
+ const runtimeApis = await page.evaluate(() => {
896
+ return new Promise((resolve) => {
897
+ const found = [];
898
+ if (window.__capturedApis) {
899
+ resolve(window.__capturedApis);
900
+ return;
901
+ }
902
+ const origFetch = window.fetch;
903
+ window.fetch = function (...args) {
904
+ try {
905
+ const url = typeof args[0] === 'string' ? args[0] : args[0]?.url;
906
+ const opts = args[1] || {};
907
+ found.push({
908
+ type: 'fetch', url, method: opts.method || 'GET',
909
+ body: typeof opts.body === 'string' ? opts.body.substring(0, 500) : null
910
+ });
911
+ }
912
+ catch (e) { }
913
+ return origFetch.apply(this, args);
914
+ };
915
+ const origOpen = XMLHttpRequest.prototype.open;
916
+ const origSend = XMLHttpRequest.prototype.send;
917
+ XMLHttpRequest.prototype.open = function (method, url, ...rest) { this.__apiUrl = url; this.__apiMethod = method; return origOpen.apply(this, [method, url, ...rest]); };
918
+ XMLHttpRequest.prototype.send = function (body) {
919
+ found.push({
920
+ type: 'xhr', url: this.__apiUrl, method: this.__apiMethod,
921
+ body: typeof body === 'string' ? body.substring(0, 500) : null
922
+ });
923
+ return origSend.apply(this, [body]);
924
+ };
925
+ window.__capturedApis = found;
926
+ setTimeout(() => resolve(found), 3000);
927
+ });
928
+ });
929
+ apiResults.dynamicApis = runtimeApis;
930
+ }
931
+ catch (e) {
932
+ apiResults.dynamicApis = [];
933
+ }
934
+ // 2. Static analysis
935
+ const allScriptContent = await page.evaluate(() => {
936
+ return Array.from(document.querySelectorAll('script')).map(s => s.textContent).join('\n');
937
+ }).catch(() => '');
938
+ const fetchRegex = /fetch\s*\(\s*(?:['"`]([^'"`]+)['"`]|([a-zA-Z_$][a-zA-Z0-9_$]*))/g;
939
+ let fMatch;
940
+ while ((fMatch = fetchRegex.exec(allScriptContent)) !== null) {
941
+ apiResults.fetchEndpoints.push((fMatch[1] || fMatch[2]));
942
+ }
943
+ apiResults.fetchEndpoints = [...new Set(apiResults.fetchEndpoints)].slice(0, 30);
944
+ const xhrRegex = /\.open\s*\(\s*['"](?:GET|POST|PUT|DELETE)['"]\s*,\s*['"`]([^'"`]+)['"`]/gi;
945
+ let xMatch;
946
+ while ((xMatch = xhrRegex.exec(allScriptContent)) !== null) {
947
+ apiResults.xhrEndpoints.push(xMatch[1]);
948
+ }
949
+ apiResults.xhrEndpoints = [...new Set(apiResults.xhrEndpoints)].slice(0, 30);
950
+ apiResults.formActions = await page.evaluate(() => {
951
+ return Array.from(document.querySelectorAll('form[action]')).map((f) => ({ action: f.action, method: f.method || 'GET', id: f.id || null }));
952
+ }).catch(() => []);
953
+ const postBodyPatterns = allScriptContent.match(/(?:URLSearchParams|FormData|JSON\.stringify)\s*\(\s*\{[^}]{5,200}\}/g) || [];
954
+ apiResults.postBodies = postBodyPatterns.slice(0, 10);
955
+ const apiUrlPattern = /['"`]((?:https?:\/\/[^'"`]+|\/)(?:[a-zA-Z0-9_\-\/]+\.(?:php|json|api|asp|aspx|do|action))[^'"`]*)['"`]/g;
956
+ let apiMatch;
957
+ while ((apiMatch = apiUrlPattern.exec(allScriptContent)) !== null) {
958
+ apiResults.inlineApiPatterns.push(apiMatch[1]);
959
+ }
960
+ apiResults.inlineApiPatterns = [...new Set(apiResults.inlineApiPatterns)].slice(0, 30);
961
+ apiResults.scriptSources = await page.evaluate(() => {
962
+ return Array.from(document.querySelectorAll('script[src]')).map(s => s.src);
963
+ }).catch(() => []);
964
+ results.extracted = apiResults;
965
+ const totalFound = apiResults.fetchEndpoints.length + apiResults.xhrEndpoints.length +
966
+ apiResults.inlineApiPatterns.length + apiResults.dynamicApis.length;
967
+ (0, state_1.notifyProgress)('extract_data', 'completed', `API Discovery: ${totalFound} endpoints found`);
968
+ break;
969
+ }
970
+ // ====== FEATURE 4: Response Auto-Decryption ======
971
+ case 'decrypt': {
972
+ (0, state_1.notifyProgress)('extract_data', 'in_progress', 'Auto-decrypting data...');
973
+ const { encryptedData, autoFindKey = true } = params;
974
+ const decryptResults = {
975
+ original: null, decoded: [], detectedEncoding: [], extractedKeys: [], aesDecrypted: null
976
+ };
977
+ // Get data to decrypt — from param or from page
978
+ let dataToDecrypt = encryptedData;
979
+ if (!dataToDecrypt) {
980
+ // Try to get from clipboard or last API response
981
+ const lastApiResponse = state_1.state.networkRecords.filter(r => r.responseBody).pop();
982
+ if (lastApiResponse)
983
+ dataToDecrypt = lastApiResponse.responseBody;
984
+ }
985
+ if (!dataToDecrypt) {
986
+ return { success: false, error: 'No data to decrypt. Provide encryptedData parameter or start network_recorder first.' };
987
+ }
988
+ decryptResults.original = dataToDecrypt.substring(0, 500);
989
+ // 1. Base64 chain decode (recursive, up to 5 levels)
990
+ let b64Data = dataToDecrypt.trim();
991
+ for (let level = 0; level < 5; level++) {
992
+ if (!/^[A-Za-z0-9+/=]+$/.test(b64Data) || b64Data.length < 4)
993
+ break;
994
+ try {
995
+ const decoded = Buffer.from(b64Data, 'base64').toString('utf-8');
996
+ if (decoded && decoded.length > 0 && !/[\x00-\x08\x0e-\x1f]/.test(decoded.substring(0, 100))) {
997
+ decryptResults.decoded.push({ level: level + 1, type: 'base64', value: decoded.substring(0, 5000) });
998
+ decryptResults.detectedEncoding.push('base64');
999
+ // Check if result is JSON
1000
+ try {
1001
+ const json = JSON.parse(decoded);
1002
+ decryptResults.decoded.push({ level: level + 1, type: 'base64_json', value: json });
1003
+ }
1004
+ catch (e) { }
1005
+ b64Data = decoded; // Continue chain
1006
+ }
1007
+ else
1008
+ break;
1009
+ }
1010
+ catch (e) {
1011
+ break;
1012
+ }
1013
+ }
1014
+ // 2. Hex decode
1015
+ const hexClean = dataToDecrypt.replace(/\s+/g, '');
1016
+ if (/^[0-9a-f]+$/i.test(hexClean) && hexClean.length >= 6 && hexClean.length % 2 === 0) {
1017
+ try {
1018
+ const hexDecoded = Buffer.from(hexClean, 'hex').toString('utf-8');
1019
+ if (hexDecoded && !/[\x00-\x08\x0e-\x1f]/.test(hexDecoded.substring(0, 50))) {
1020
+ decryptResults.decoded.push({ type: 'hex', value: hexDecoded.substring(0, 5000) });
1021
+ decryptResults.detectedEncoding.push('hex');
1022
+ }
1023
+ }
1024
+ catch (e) { }
1025
+ }
1026
+ // 3. URL decode (multi-level)
1027
+ if (dataToDecrypt.includes('%')) {
1028
+ try {
1029
+ let urlDecoded = decodeURIComponent(dataToDecrypt);
1030
+ decryptResults.decoded.push({ type: 'url', value: urlDecoded.substring(0, 5000) });
1031
+ decryptResults.detectedEncoding.push('url');
1032
+ // Double URL decode
1033
+ if (urlDecoded.includes('%')) {
1034
+ urlDecoded = decodeURIComponent(urlDecoded);
1035
+ decryptResults.decoded.push({ type: 'url_double', value: urlDecoded.substring(0, 5000) });
1036
+ }
1037
+ }
1038
+ catch (e) { }
1039
+ }
1040
+ // 4. ROT13
1041
+ try {
1042
+ const rot13 = dataToDecrypt.replace(/[a-zA-Z]/g, (c) => {
1043
+ const base = c <= 'Z' ? 65 : 97;
1044
+ return String.fromCharCode(((c.charCodeAt(0) - base + 13) % 26) + base);
1045
+ });
1046
+ if (rot13 !== dataToDecrypt && (rot13.includes('http') || rot13.includes('www') || rot13.includes('.com'))) {
1047
+ decryptResults.decoded.push({ type: 'rot13', value: rot13.substring(0, 5000) });
1048
+ decryptResults.detectedEncoding.push('rot13');
1049
+ }
1050
+ }
1051
+ catch (e) { }
1052
+ // 5. Auto-extract encryption keys from page scripts
1053
+ if (autoFindKey) {
1054
+ try {
1055
+ const keys = await page.evaluate(() => {
1056
+ const scripts = Array.from(document.querySelectorAll('script')).map(s => s.textContent).join('\n');
1057
+ const found = [];
1058
+ // CryptoJS patterns
1059
+ const cryptoPatterns = [
1060
+ /CryptoJS\.AES\.decrypt\s*\(\s*\w+\s*,\s*['"]([^'"]+)['"]/g,
1061
+ /CryptoJS\.AES\.encrypt\s*\(\s*\w+\s*,\s*['"]([^'"]+)['"]/g,
1062
+ /CryptoJS\.enc\.Utf8\.parse\s*\(\s*['"]([^'"]+)['"]/g,
1063
+ /(?:secret|key|pass|password|iv|salt)\s*[:=]\s*['"]([^'"]{8,})['"]/gi,
1064
+ /aes(?:Key|_key|Secret)\s*[:=]\s*['"]([^'"]{8,})['"]/gi
1065
+ ];
1066
+ for (const pat of cryptoPatterns) {
1067
+ let m;
1068
+ while ((m = pat.exec(scripts)) !== null) {
1069
+ found.push({ pattern: pat.source.substring(0, 50), key: m[1] });
1070
+ }
1071
+ }
1072
+ return found;
1073
+ });
1074
+ decryptResults.extractedKeys = keys.slice(0, 20);
1075
+ }
1076
+ catch (e) { }
1077
+ }
1078
+ // 6. AES decryption — try with extracted keys or user-provided key
1079
+ const aesKey = params.aesKey || (decryptResults.extractedKeys[0]?.key);
1080
+ if (aesKey && dataToDecrypt.length > 10) {
1081
+ try {
1082
+ const crypto = require('crypto');
1083
+ // Try AES-256-CBC
1084
+ for (const keyEncoding of ['utf8', 'hex', 'base64']) {
1085
+ try {
1086
+ let keyBuf;
1087
+ if (keyEncoding === 'utf8')
1088
+ keyBuf = Buffer.alloc(32); // pad to 32 bytes
1089
+ else
1090
+ keyBuf = Buffer.from(aesKey, keyEncoding);
1091
+ if (keyEncoding === 'utf8') {
1092
+ const kb = Buffer.from(aesKey, 'utf8');
1093
+ kb.copy(keyBuf);
1094
+ }
1095
+ // Try to decode the data from base64 first
1096
+ const dataBuf = Buffer.from(dataToDecrypt, 'base64');
1097
+ if (dataBuf.length > 16) {
1098
+ // IV might be first 16 bytes
1099
+ const iv = params.aesIV ? Buffer.from(params.aesIV, keyEncoding) : dataBuf.slice(0, 16);
1100
+ const encrypted = params.aesIV ? dataBuf : dataBuf.slice(16);
1101
+ const decipher = crypto.createDecipheriv('aes-256-cbc', keyBuf, iv);
1102
+ decipher.setAutoPadding(true);
1103
+ let decrypted = decipher.update(encrypted, undefined, 'utf8');
1104
+ decrypted += decipher.final('utf8');
1105
+ if (decrypted && decrypted.length > 0) {
1106
+ decryptResults.aesDecrypted = decrypted.substring(0, 5000);
1107
+ decryptResults.detectedEncoding.push('aes-256-cbc');
1108
+ // Try JSON parse
1109
+ try {
1110
+ decryptResults.aesDecrypted = JSON.parse(decrypted);
1111
+ }
1112
+ catch (e) { }
1113
+ break;
1114
+ }
1115
+ }
1116
+ }
1117
+ catch (e) {
1118
+ continue;
1119
+ }
1120
+ }
1121
+ }
1122
+ catch (e) { }
1123
+ }
1124
+ results.extracted = decryptResults;
1125
+ const decodedCount = decryptResults.decoded.length + (decryptResults.aesDecrypted ? 1 : 0);
1126
+ (0, state_1.notifyProgress)('extract_data', 'completed', `Decrypted: ${decodedCount} decodings, ${decryptResults.extractedKeys.length} keys found, encodings: ${decryptResults.detectedEncoding.join(', ') || 'none'}`);
1127
+ break;
1128
+ }
1129
+ default:
1130
+ return { success: false, error: `Unknown type: ${type}. Supported: regex, json, meta, structured, auto, deobfuscate, apiDiscovery, decrypt` };
1131
+ }
1132
+ return results;
1133
+ }
1134
+ };
1135
+ //# sourceMappingURL=network.js.map