real-browser-mcp-server 1.2.0 → 1.2.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +82 -9
- package/dist/lib/cjs/index.d.ts +2 -0
- package/dist/lib/cjs/index.d.ts.map +1 -0
- package/dist/lib/cjs/index.js +385 -0
- package/dist/lib/cjs/index.js.map +1 -0
- package/dist/lib/cjs/module/pageController.d.ts +2 -0
- package/dist/lib/cjs/module/pageController.d.ts.map +1 -0
- package/{lib → dist/lib}/cjs/module/pageController.js +28 -29
- package/dist/lib/cjs/module/pageController.js.map +1 -0
- package/dist/lib/cjs/module/turnstile.d.ts +2 -0
- package/dist/lib/cjs/module/turnstile.d.ts.map +1 -0
- package/{lib → dist/lib}/cjs/module/turnstile.js +24 -12
- package/dist/lib/cjs/module/turnstile.js.map +1 -0
- package/dist/src/index.d.ts +11 -0
- package/dist/src/index.d.ts.map +1 -0
- package/dist/src/index.js +118 -0
- package/dist/src/index.js.map +1 -0
- package/dist/src/mcp/handlers/browser.d.ts +30 -0
- package/dist/src/mcp/handlers/browser.d.ts.map +1 -0
- package/dist/src/mcp/handlers/browser.js +231 -0
- package/dist/src/mcp/handlers/browser.js.map +1 -0
- package/dist/src/mcp/handlers/dom.d.ts +134 -0
- package/dist/src/mcp/handlers/dom.d.ts.map +1 -0
- package/dist/src/mcp/handlers/dom.js +551 -0
- package/dist/src/mcp/handlers/dom.js.map +1 -0
- package/dist/src/mcp/handlers/extract.d.ts +59 -0
- package/dist/src/mcp/handlers/extract.d.ts.map +1 -0
- package/dist/src/mcp/handlers/extract.js +455 -0
- package/dist/src/mcp/handlers/extract.js.map +1 -0
- package/dist/src/mcp/handlers/form-handlers.d.ts +9 -0
- package/dist/src/mcp/handlers/form-handlers.d.ts.map +1 -0
- package/dist/src/mcp/handlers/form-handlers.js +56 -0
- package/dist/src/mcp/handlers/form-handlers.js.map +1 -0
- package/dist/src/mcp/handlers/helpers.d.ts +47 -0
- package/dist/src/mcp/handlers/helpers.d.ts.map +1 -0
- package/dist/src/mcp/handlers/helpers.js +515 -0
- package/dist/src/mcp/handlers/helpers.js.map +1 -0
- package/dist/src/mcp/handlers/index.d.ts +6 -0
- package/dist/src/mcp/handlers/index.d.ts.map +1 -0
- package/dist/src/mcp/handlers/index.js +61 -0
- package/dist/src/mcp/handlers/index.js.map +1 -0
- package/dist/src/mcp/handlers/media-handlers.d.ts +10 -0
- package/dist/src/mcp/handlers/media-handlers.d.ts.map +1 -0
- package/dist/src/mcp/handlers/media-handlers.js +535 -0
- package/dist/src/mcp/handlers/media-handlers.js.map +1 -0
- package/dist/src/mcp/handlers/network.d.ts +147 -0
- package/dist/src/mcp/handlers/network.d.ts.map +1 -0
- package/dist/src/mcp/handlers/network.js +1135 -0
- package/dist/src/mcp/handlers/network.js.map +1 -0
- package/dist/src/mcp/handlers/state.d.ts +34 -0
- package/dist/src/mcp/handlers/state.d.ts.map +1 -0
- package/dist/src/mcp/handlers/state.js +225 -0
- package/dist/src/mcp/handlers/state.js.map +1 -0
- package/dist/src/mcp/handlers/utility-handlers.d.ts +167 -0
- package/dist/src/mcp/handlers/utility-handlers.d.ts.map +1 -0
- package/dist/src/mcp/handlers/utility-handlers.js +280 -0
- package/dist/src/mcp/handlers/utility-handlers.js.map +1 -0
- package/dist/src/mcp/handlers/vision.d.ts +127 -0
- package/dist/src/mcp/handlers/vision.d.ts.map +1 -0
- package/dist/src/mcp/handlers/vision.js +483 -0
- package/dist/src/mcp/handlers/vision.js.map +1 -0
- package/dist/src/mcp/index.d.ts +3 -0
- package/dist/src/mcp/index.d.ts.map +1 -0
- package/dist/src/mcp/index.js +166 -0
- package/dist/src/mcp/index.js.map +1 -0
- package/dist/src/mcp/server.d.ts +2 -0
- package/dist/src/mcp/server.d.ts.map +1 -0
- package/dist/src/mcp/server.js +117 -0
- package/dist/src/mcp/server.js.map +1 -0
- package/dist/src/mcp/tools.d.ts +8 -0
- package/dist/src/mcp/tools.d.ts.map +1 -0
- package/{src → dist/src}/mcp/tools.js +12 -11
- package/dist/src/mcp/tools.js.map +1 -0
- package/dist/src/shared/cache-manager.d.ts +80 -0
- package/dist/src/shared/cache-manager.d.ts.map +1 -0
- package/dist/src/shared/cache-manager.js +221 -0
- package/dist/src/shared/cache-manager.js.map +1 -0
- package/dist/src/shared/tools.d.ts +2 -0
- package/dist/src/shared/tools.d.ts.map +1 -0
- package/dist/src/shared/tools.js +599 -0
- package/dist/src/shared/tools.js.map +1 -0
- package/dist/src/types.d.ts +365 -0
- package/dist/src/types.d.ts.map +1 -0
- package/dist/src/types.js +9 -0
- package/dist/src/types.js.map +1 -0
- package/dist/test/cjs/test.d.ts +11 -0
- package/dist/test/cjs/test.d.ts.map +1 -0
- package/dist/test/cjs/test.js +289 -0
- package/dist/test/cjs/test.js.map +1 -0
- package/dist/test/mcp/smoke-test.d.ts +29 -0
- package/dist/test/mcp/smoke-test.d.ts.map +1 -0
- package/dist/test/mcp/smoke-test.js +132 -0
- package/dist/test/mcp/smoke-test.js.map +1 -0
- package/lib/esm/index.mjs +232 -79
- package/lib/esm/module/pageController.mjs +21 -18
- package/lib/esm/module/turnstile.mjs +7 -0
- package/package.json +25 -15
- package/typings.d.ts +12 -6
- package/.github/ISSUE_TEMPLATE/general_issue.yaml +0 -58
- package/.github/SETUP.md +0 -111
- package/.github/workflows/publish.yml +0 -135
- package/Dockerfile +0 -79
- package/lib/cjs/adblocker.bin +0 -0
- package/lib/cjs/index.js +0 -249
- package/src/ai/action-parser.js +0 -274
- package/src/ai/core.js +0 -378
- package/src/ai/element-finder.js +0 -466
- package/src/ai/index.js +0 -82
- package/src/ai/page-analyzer.js +0 -304
- package/src/ai/selector-healer.js +0 -236
- package/src/index.js +0 -121
- package/src/mcp/handlers.js +0 -5071
- package/src/mcp/index.js +0 -190
- package/src/mcp/server.js +0 -144
- package/src/shared/tools.js +0 -618
- package/test/cjs/test.js +0 -259
- package/test/esm/package.json +0 -13
- package/test/esm/test.js +0 -226
- package/test/esm/test_option2.js +0 -46
- package/test/esm/test_playwright_ghost.js +0 -30
|
@@ -0,0 +1,1135 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.networkHandlers = void 0;
|
|
4
|
+
const state_1 = require("./state");
|
|
5
|
+
// Auto-generated network handlers
|
|
6
|
+
exports.networkHandlers = {
|
|
7
|
+
async redirect_tracer(params) {
|
|
8
|
+
const { page } = (0, state_1.requireBrowser)();
|
|
9
|
+
const { url, maxRedirects = 20, includeHeaders = false, followJS = true, timeout = 30000 } = params;
|
|
10
|
+
(0, state_1.notifyProgress)('redirect_tracer', 'started', `Tracing redirects for: ${url}`);
|
|
11
|
+
const redirects = [];
|
|
12
|
+
const jsNavigations = [];
|
|
13
|
+
let currentUrl = url;
|
|
14
|
+
// HTTP redirect handler
|
|
15
|
+
const responseHandler = (response) => {
|
|
16
|
+
if ([301, 302, 303, 307, 308].includes(response.status())) {
|
|
17
|
+
redirects.push({
|
|
18
|
+
url: response.url(),
|
|
19
|
+
status: response.status(),
|
|
20
|
+
type: 'http',
|
|
21
|
+
headers: includeHeaders ? response.headers() : undefined
|
|
22
|
+
});
|
|
23
|
+
(0, state_1.notifyProgress)('redirect_tracer', 'progress', `HTTP Redirect ${redirects.length}: ${response.status()}`, { status: response.status() });
|
|
24
|
+
}
|
|
25
|
+
};
|
|
26
|
+
// JS/Navigation handler for tracking window.location changes
|
|
27
|
+
const frameNavigatedHandler = (frame) => {
|
|
28
|
+
if (frame === page.mainFrame()) {
|
|
29
|
+
const newUrl = frame.url();
|
|
30
|
+
if (newUrl !== currentUrl && newUrl !== 'about:blank') {
|
|
31
|
+
jsNavigations.push({
|
|
32
|
+
url: newUrl,
|
|
33
|
+
type: 'js_navigation',
|
|
34
|
+
fromUrl: currentUrl,
|
|
35
|
+
timestamp: Date.now()
|
|
36
|
+
});
|
|
37
|
+
(0, state_1.notifyProgress)('redirect_tracer', 'progress', `JS Navigation: ${newUrl}`, { type: 'js' });
|
|
38
|
+
currentUrl = newUrl;
|
|
39
|
+
}
|
|
40
|
+
}
|
|
41
|
+
};
|
|
42
|
+
page.on('response', responseHandler);
|
|
43
|
+
page.on('framenavigated', frameNavigatedHandler);
|
|
44
|
+
try {
|
|
45
|
+
await page.goto(url, { waitUntil: 'networkidle', timeout });
|
|
46
|
+
// If followJS is enabled, wait a bit and check for meta refreshes and JS redirects
|
|
47
|
+
if (followJS) {
|
|
48
|
+
// Check for meta refresh tags
|
|
49
|
+
const metaRefresh = await page.evaluate(() => {
|
|
50
|
+
const meta = document.querySelector('meta[http-equiv="refresh"]');
|
|
51
|
+
if (meta) {
|
|
52
|
+
const content = meta.getAttribute('content');
|
|
53
|
+
const match = content?.match(/url=(.+)/i);
|
|
54
|
+
return match ? match[1].trim().replace(/['"]/g, '') : null;
|
|
55
|
+
}
|
|
56
|
+
return null;
|
|
57
|
+
}).catch(() => null);
|
|
58
|
+
if (metaRefresh) {
|
|
59
|
+
jsNavigations.push({
|
|
60
|
+
url: metaRefresh,
|
|
61
|
+
type: 'meta_refresh',
|
|
62
|
+
fromUrl: page.url()
|
|
63
|
+
});
|
|
64
|
+
}
|
|
65
|
+
// Extract any onclick/href javascript: URLs
|
|
66
|
+
const jsLinks = await page.evaluate(() => {
|
|
67
|
+
const links = [];
|
|
68
|
+
document.querySelectorAll('a[href^="javascript:"], [onclick]').forEach(el => {
|
|
69
|
+
const onclick = el.getAttribute('onclick');
|
|
70
|
+
const href = el.getAttribute('href');
|
|
71
|
+
if (onclick) {
|
|
72
|
+
const match = onclick.match(/location\.href\s*=\s*['"]([^'"]+)['"]/);
|
|
73
|
+
if (match)
|
|
74
|
+
links.push({ url: match[1], type: 'onclick' });
|
|
75
|
+
}
|
|
76
|
+
if (href && href.includes('location')) {
|
|
77
|
+
links.push({ url: href, type: 'javascript_href' });
|
|
78
|
+
}
|
|
79
|
+
});
|
|
80
|
+
return links;
|
|
81
|
+
}).catch(() => []);
|
|
82
|
+
jsNavigations.push(...jsLinks);
|
|
83
|
+
}
|
|
84
|
+
}
|
|
85
|
+
catch (e) {
|
|
86
|
+
(0, state_1.notifyProgress)('redirect_tracer', 'progress', `Navigation error: ${e.message}`);
|
|
87
|
+
}
|
|
88
|
+
page.off('response', responseHandler);
|
|
89
|
+
page.off('framenavigated', frameNavigatedHandler);
|
|
90
|
+
const allRedirects = [
|
|
91
|
+
...redirects,
|
|
92
|
+
...jsNavigations.filter(nav => nav.url && nav.url.startsWith('http'))
|
|
93
|
+
];
|
|
94
|
+
(0, state_1.notifyProgress)('redirect_tracer', 'completed', `Found ${redirects.length} HTTP + ${jsNavigations.length} JS redirects`, { httpRedirects: redirects.length, jsNavigations: jsNavigations.length, finalUrl: page.url() });
|
|
95
|
+
return {
|
|
96
|
+
success: true,
|
|
97
|
+
originalUrl: url,
|
|
98
|
+
finalUrl: page.url(),
|
|
99
|
+
redirectCount: allRedirects.length,
|
|
100
|
+
httpRedirects: redirects,
|
|
101
|
+
jsNavigations: jsNavigations,
|
|
102
|
+
allRedirects: allRedirects
|
|
103
|
+
};
|
|
104
|
+
},
|
|
105
|
+
async network_recorder(params = {}) {
|
|
106
|
+
const { page } = (0, state_1.requireBrowser)();
|
|
107
|
+
const { action = 'get', filter = {}, captureResponses = false } = params;
|
|
108
|
+
switch (action) {
|
|
109
|
+
case 'start':
|
|
110
|
+
state_1.state.networkRecords = [];
|
|
111
|
+
state_1.state.isRecordingNetwork = true;
|
|
112
|
+
// ====== FEATURE 2: Pre-page-load Runtime API Interception ======
|
|
113
|
+
// Inject BEFORE any JS runs — catches calls from obfuscated/webpack code
|
|
114
|
+
try {
|
|
115
|
+
await page.addInitScript(() => {
|
|
116
|
+
window.__interceptedApis = [];
|
|
117
|
+
window.__wsMessages = [];
|
|
118
|
+
// --- Monkey-patch fetch ---
|
|
119
|
+
const origFetch = window.fetch;
|
|
120
|
+
window.fetch = function (...args) {
|
|
121
|
+
try {
|
|
122
|
+
const url = typeof args[0] === 'string' ? args[0] : (args[0]?.url || String(args[0]));
|
|
123
|
+
const opts = args[1] || {};
|
|
124
|
+
const entry = {
|
|
125
|
+
type: 'fetch', url, method: opts.method || 'GET',
|
|
126
|
+
headers: opts.headers ? JSON.parse(JSON.stringify(opts.headers)) : null,
|
|
127
|
+
body: typeof opts.body === 'string' ? opts.body.substring(0, 2000) : null,
|
|
128
|
+
timestamp: Date.now()
|
|
129
|
+
};
|
|
130
|
+
window.__interceptedApis.push(entry);
|
|
131
|
+
}
|
|
132
|
+
catch (e) { }
|
|
133
|
+
return origFetch.apply(this, args);
|
|
134
|
+
};
|
|
135
|
+
// --- Monkey-patch XMLHttpRequest ---
|
|
136
|
+
const origOpen = XMLHttpRequest.prototype.open;
|
|
137
|
+
const origSend = XMLHttpRequest.prototype.send;
|
|
138
|
+
const origSetHeader = XMLHttpRequest.prototype.setRequestHeader;
|
|
139
|
+
XMLHttpRequest.prototype.open = function (method, url, ...rest) {
|
|
140
|
+
this.__iUrl = url;
|
|
141
|
+
this.__iMethod = method;
|
|
142
|
+
this.__iHeaders = {};
|
|
143
|
+
return origOpen.apply(this, [method, url, ...rest]);
|
|
144
|
+
};
|
|
145
|
+
XMLHttpRequest.prototype.setRequestHeader = function (name, value) {
|
|
146
|
+
if (this.__iHeaders)
|
|
147
|
+
this.__iHeaders[name] = value;
|
|
148
|
+
return origSetHeader.apply(this, [name, value]);
|
|
149
|
+
};
|
|
150
|
+
XMLHttpRequest.prototype.send = function (body) {
|
|
151
|
+
try {
|
|
152
|
+
window.__interceptedApis.push({
|
|
153
|
+
type: 'xhr', url: this.__iUrl, method: this.__iMethod,
|
|
154
|
+
headers: this.__iHeaders || null,
|
|
155
|
+
body: typeof body === 'string' ? body.substring(0, 2000) : null,
|
|
156
|
+
timestamp: Date.now()
|
|
157
|
+
});
|
|
158
|
+
}
|
|
159
|
+
catch (e) { }
|
|
160
|
+
return origSend.apply(this, [body]);
|
|
161
|
+
};
|
|
162
|
+
// --- Monkey-patch navigator.sendBeacon ---
|
|
163
|
+
if (navigator.sendBeacon) {
|
|
164
|
+
const origBeacon = navigator.sendBeacon.bind(navigator);
|
|
165
|
+
navigator.sendBeacon = function (url, data) {
|
|
166
|
+
try {
|
|
167
|
+
window.__interceptedApis.push({
|
|
168
|
+
type: 'beacon', url, method: 'POST',
|
|
169
|
+
body: typeof data === 'string' ? data.substring(0, 2000) : null,
|
|
170
|
+
timestamp: Date.now()
|
|
171
|
+
});
|
|
172
|
+
}
|
|
173
|
+
catch (e) { }
|
|
174
|
+
return origBeacon(url, data);
|
|
175
|
+
};
|
|
176
|
+
}
|
|
177
|
+
// ====== FEATURE 3: WebSocket Recording ======
|
|
178
|
+
const OrigWS = window.WebSocket;
|
|
179
|
+
window.WebSocket = function (url, protocols) {
|
|
180
|
+
const ws = protocols ? new OrigWS(url, protocols) : new OrigWS(url);
|
|
181
|
+
const wsId = window.__wsMessages.length;
|
|
182
|
+
const wsEntry = { id: wsId, url, openedAt: Date.now(), messages: [], status: 'connecting' };
|
|
183
|
+
window.__wsMessages.push(wsEntry);
|
|
184
|
+
ws.addEventListener('open', () => { wsEntry.status = 'open'; });
|
|
185
|
+
ws.addEventListener('close', (e) => { wsEntry.status = 'closed'; wsEntry.closedAt = Date.now(); wsEntry.closeCode = e.code; });
|
|
186
|
+
ws.addEventListener('error', () => { wsEntry.status = 'error'; });
|
|
187
|
+
ws.addEventListener('message', (e) => {
|
|
188
|
+
try {
|
|
189
|
+
let data = e.data;
|
|
190
|
+
let dataType = 'text';
|
|
191
|
+
if (data instanceof Blob) {
|
|
192
|
+
dataType = 'blob';
|
|
193
|
+
data = `[Blob ${data.size} bytes]`;
|
|
194
|
+
}
|
|
195
|
+
else if (data instanceof ArrayBuffer) {
|
|
196
|
+
dataType = 'binary';
|
|
197
|
+
data = `[ArrayBuffer ${data.byteLength} bytes]`;
|
|
198
|
+
}
|
|
199
|
+
else if (typeof data === 'string' && data.length > 5000) {
|
|
200
|
+
data = data.substring(0, 5000) + '...';
|
|
201
|
+
}
|
|
202
|
+
wsEntry.messages.push({ direction: 'received', data, dataType, timestamp: Date.now() });
|
|
203
|
+
}
|
|
204
|
+
catch (e) { }
|
|
205
|
+
});
|
|
206
|
+
// Intercept send
|
|
207
|
+
const origWsSend = ws.send.bind(ws);
|
|
208
|
+
ws.send = function (data) {
|
|
209
|
+
try {
|
|
210
|
+
let sendData = data;
|
|
211
|
+
let dataType = 'text';
|
|
212
|
+
if (data instanceof Blob) {
|
|
213
|
+
dataType = 'blob';
|
|
214
|
+
sendData = `[Blob ${data.size} bytes]`;
|
|
215
|
+
}
|
|
216
|
+
else if (data instanceof ArrayBuffer) {
|
|
217
|
+
dataType = 'binary';
|
|
218
|
+
sendData = `[ArrayBuffer ${data.byteLength} bytes]`;
|
|
219
|
+
}
|
|
220
|
+
else if (typeof data === 'string' && data.length > 5000) {
|
|
221
|
+
sendData = data.substring(0, 5000) + '...';
|
|
222
|
+
}
|
|
223
|
+
wsEntry.messages.push({ direction: 'sent', data: sendData, dataType, timestamp: Date.now() });
|
|
224
|
+
}
|
|
225
|
+
catch (e) { }
|
|
226
|
+
return origWsSend(data);
|
|
227
|
+
};
|
|
228
|
+
return ws;
|
|
229
|
+
};
|
|
230
|
+
window.WebSocket.prototype = OrigWS.prototype;
|
|
231
|
+
window.WebSocket.CONNECTING = OrigWS.CONNECTING;
|
|
232
|
+
window.WebSocket.OPEN = OrigWS.OPEN;
|
|
233
|
+
window.WebSocket.CLOSING = OrigWS.CLOSING;
|
|
234
|
+
window.WebSocket.CLOSED = OrigWS.CLOSED;
|
|
235
|
+
});
|
|
236
|
+
}
|
|
237
|
+
catch (e) { /* addInitScript may fail on already-loaded pages, that's OK */ }
|
|
238
|
+
// Request handler
|
|
239
|
+
page.on('request', req => {
|
|
240
|
+
if (state_1.state.isRecordingNetwork) {
|
|
241
|
+
state_1.state.networkRecords.push({
|
|
242
|
+
type: 'request',
|
|
243
|
+
url: req.url(),
|
|
244
|
+
method: req.method(),
|
|
245
|
+
resourceType: req.resourceType(),
|
|
246
|
+
headers: req.headers(),
|
|
247
|
+
timestamp: Date.now()
|
|
248
|
+
});
|
|
249
|
+
}
|
|
250
|
+
});
|
|
251
|
+
// Response handler for capturing video/media URLs
|
|
252
|
+
page.on('response', async (res) => {
|
|
253
|
+
if (state_1.state.isRecordingNetwork) {
|
|
254
|
+
const url = res.url();
|
|
255
|
+
const contentType = res.headers()['content-type'] || '';
|
|
256
|
+
const isMedia = contentType.includes('video') ||
|
|
257
|
+
contentType.includes('audio') ||
|
|
258
|
+
contentType.includes('mpegurl') ||
|
|
259
|
+
url.includes('.m3u8') ||
|
|
260
|
+
url.includes('.mpd') ||
|
|
261
|
+
url.includes('.mp4') ||
|
|
262
|
+
url.includes('.ts');
|
|
263
|
+
const isApiCall = contentType.includes('json') ||
|
|
264
|
+
contentType.includes('x-www-form-urlencoded') ||
|
|
265
|
+
url.match(/\.(php|api|json|do|action)($|\?)/) ||
|
|
266
|
+
(res.request().resourceType() === 'xhr') ||
|
|
267
|
+
(res.request().resourceType() === 'fetch');
|
|
268
|
+
const record = {
|
|
269
|
+
type: 'response',
|
|
270
|
+
url: url,
|
|
271
|
+
method: res.request().method(),
|
|
272
|
+
status: res.status(),
|
|
273
|
+
contentType: contentType,
|
|
274
|
+
isMedia: isMedia,
|
|
275
|
+
isApiCall: isApiCall,
|
|
276
|
+
resourceType: res.request().resourceType(),
|
|
277
|
+
timestamp: Date.now()
|
|
278
|
+
};
|
|
279
|
+
// For media URLs, try to get more details
|
|
280
|
+
if (isMedia) {
|
|
281
|
+
record.mediaType = url.includes('.m3u8') ? 'hls' :
|
|
282
|
+
url.includes('.mpd') ? 'dash' :
|
|
283
|
+
url.includes('.mp4') ? 'mp4' : 'other';
|
|
284
|
+
}
|
|
285
|
+
// Capture request/response body for API calls
|
|
286
|
+
if (isApiCall) {
|
|
287
|
+
try {
|
|
288
|
+
const postData = res.request().postData();
|
|
289
|
+
if (postData)
|
|
290
|
+
record.requestBody = postData.substring(0, 2000);
|
|
291
|
+
const responseBody = await res.text().catch(() => null);
|
|
292
|
+
if (responseBody) {
|
|
293
|
+
record.responseBody = responseBody.substring(0, 5000);
|
|
294
|
+
try {
|
|
295
|
+
record.responseJson = JSON.parse(responseBody);
|
|
296
|
+
}
|
|
297
|
+
catch (e) { }
|
|
298
|
+
}
|
|
299
|
+
}
|
|
300
|
+
catch (e) { }
|
|
301
|
+
}
|
|
302
|
+
state_1.state.networkRecords.push(record);
|
|
303
|
+
}
|
|
304
|
+
});
|
|
305
|
+
// Frame navigation handler for JS redirects
|
|
306
|
+
page.on('framenavigated', frame => {
|
|
307
|
+
if (state_1.state.isRecordingNetwork && frame === page.mainFrame()) {
|
|
308
|
+
state_1.state.networkRecords.push({
|
|
309
|
+
type: 'navigation',
|
|
310
|
+
url: frame.url(),
|
|
311
|
+
timestamp: Date.now()
|
|
312
|
+
});
|
|
313
|
+
}
|
|
314
|
+
});
|
|
315
|
+
(0, state_1.notifyProgress)('network_recorder', 'started', 'Power recording started (requests + responses + API interception + WebSocket + navigations)');
|
|
316
|
+
break;
|
|
317
|
+
case 'stop':
|
|
318
|
+
state_1.state.isRecordingNetwork = false;
|
|
319
|
+
(0, state_1.notifyProgress)('network_recorder', 'completed', `Recording stopped: ${state_1.state.networkRecords.length} events captured`);
|
|
320
|
+
break;
|
|
321
|
+
case 'clear':
|
|
322
|
+
state_1.state.networkRecords = [];
|
|
323
|
+
// Also clear intercepted data
|
|
324
|
+
try {
|
|
325
|
+
await page.evaluate(() => { window.__interceptedApis = []; window.__wsMessages = []; });
|
|
326
|
+
}
|
|
327
|
+
catch (e) { }
|
|
328
|
+
(0, state_1.notifyProgress)('network_recorder', 'completed', 'Network records cleared');
|
|
329
|
+
break;
|
|
330
|
+
case 'get_media':
|
|
331
|
+
// Special action to get only media URLs
|
|
332
|
+
const mediaRecords = state_1.state.networkRecords.filter(r => r.isMedia);
|
|
333
|
+
return {
|
|
334
|
+
success: true,
|
|
335
|
+
count: mediaRecords.length,
|
|
336
|
+
mediaUrls: mediaRecords.map(r => ({ url: r.url, type: r.mediaType }))
|
|
337
|
+
};
|
|
338
|
+
case 'get_navigations':
|
|
339
|
+
// Get only navigation events (for tracking JS redirects)
|
|
340
|
+
const navRecords = state_1.state.networkRecords.filter(r => r.type === 'navigation');
|
|
341
|
+
return {
|
|
342
|
+
success: true,
|
|
343
|
+
count: navRecords.length,
|
|
344
|
+
navigations: navRecords
|
|
345
|
+
};
|
|
346
|
+
case 'get_api_calls': {
|
|
347
|
+
const apiRecords = state_1.state.networkRecords.filter(r => r.isApiCall);
|
|
348
|
+
return {
|
|
349
|
+
success: true,
|
|
350
|
+
count: apiRecords.length,
|
|
351
|
+
apiCalls: apiRecords.map(r => ({
|
|
352
|
+
url: r.url, method: r.method || 'GET', status: r.status,
|
|
353
|
+
contentType: r.contentType, resourceType: r.resourceType,
|
|
354
|
+
requestBody: r.requestBody || null, responseBody: r.responseBody || null,
|
|
355
|
+
responseJson: r.responseJson || null, timestamp: r.timestamp
|
|
356
|
+
}))
|
|
357
|
+
};
|
|
358
|
+
}
|
|
359
|
+
// ====== FEATURE 2: Get Intercepted APIs (from monkey-patched fetch/XHR/beacon) ======
|
|
360
|
+
case 'get_intercepted_apis': {
|
|
361
|
+
try {
|
|
362
|
+
const intercepted = await page.evaluate(() => window.__interceptedApis || []);
|
|
363
|
+
return {
|
|
364
|
+
success: true,
|
|
365
|
+
count: intercepted.length,
|
|
366
|
+
interceptedApis: intercepted,
|
|
367
|
+
note: 'These are runtime-intercepted API calls captured via monkey-patched fetch/XHR/sendBeacon (pre-page-load injection)'
|
|
368
|
+
};
|
|
369
|
+
}
|
|
370
|
+
catch (e) {
|
|
371
|
+
return { success: false, error: 'Failed to retrieve intercepted APIs: ' + e.message, interceptedApis: [] };
|
|
372
|
+
}
|
|
373
|
+
}
|
|
374
|
+
// ====== FEATURE 3: Get WebSocket Messages ======
|
|
375
|
+
case 'get_websockets': {
|
|
376
|
+
try {
|
|
377
|
+
const wsData = await page.evaluate(() => window.__wsMessages || []);
|
|
378
|
+
const totalMessages = wsData.reduce((sum, ws) => sum + ws.messages.length, 0);
|
|
379
|
+
return {
|
|
380
|
+
success: true,
|
|
381
|
+
count: wsData.length,
|
|
382
|
+
totalMessages: totalMessages,
|
|
383
|
+
websockets: wsData,
|
|
384
|
+
note: 'WebSocket connections and messages captured via constructor monkey-patch'
|
|
385
|
+
};
|
|
386
|
+
}
|
|
387
|
+
catch (e) {
|
|
388
|
+
return { success: false, error: 'Failed to retrieve WebSocket data: ' + e.message, websockets: [] };
|
|
389
|
+
}
|
|
390
|
+
}
|
|
391
|
+
}
|
|
392
|
+
let records = state_1.state.networkRecords;
|
|
393
|
+
if (filter.resourceType) {
|
|
394
|
+
records = records.filter(r => r.resourceType === filter.resourceType);
|
|
395
|
+
}
|
|
396
|
+
if (filter.urlPattern) {
|
|
397
|
+
const regex = new RegExp(filter.urlPattern);
|
|
398
|
+
records = records.filter(r => regex.test(r.url));
|
|
399
|
+
}
|
|
400
|
+
if (filter.type) {
|
|
401
|
+
records = records.filter(r => r.type === filter.type);
|
|
402
|
+
}
|
|
403
|
+
if (filter.mediaOnly) {
|
|
404
|
+
records = records.filter(r => r.isMedia);
|
|
405
|
+
}
|
|
406
|
+
return { success: true, recording: state_1.state.isRecordingNetwork, count: records.length, records: records.slice(-200) };
|
|
407
|
+
},
|
|
408
|
+
async cookie_manager(params = {}) {
|
|
409
|
+
const { page } = (0, state_1.requireBrowser)();
|
|
410
|
+
const { action = 'get', name, value, domain, expires } = params;
|
|
411
|
+
(0, state_1.notifyProgress)('cookie_manager', 'started', `Cookie action: ${action}`);
|
|
412
|
+
// Playwright/Patchright: cookies are managed via the BrowserContext, not the page
|
|
413
|
+
const context = page.context();
|
|
414
|
+
switch (action) {
|
|
415
|
+
case 'get': {
|
|
416
|
+
const cookies = await context.cookies();
|
|
417
|
+
(0, state_1.notifyProgress)('cookie_manager', 'completed', `Retrieved ${cookies.length} cookies`);
|
|
418
|
+
return { success: true, cookies: name ? cookies.filter(c => c.name === name) : cookies };
|
|
419
|
+
}
|
|
420
|
+
case 'set': {
|
|
421
|
+
await context.addCookies([{
|
|
422
|
+
name,
|
|
423
|
+
value,
|
|
424
|
+
domain: domain || new URL(page.url()).hostname,
|
|
425
|
+
path: '/',
|
|
426
|
+
...(expires ? { expires } : {})
|
|
427
|
+
}]);
|
|
428
|
+
(0, state_1.notifyProgress)('cookie_manager', 'completed', `Cookie set: ${name}`);
|
|
429
|
+
return { success: true, message: `Cookie ${name} set` };
|
|
430
|
+
}
|
|
431
|
+
case 'delete': {
|
|
432
|
+
// Playwright has no per-cookie delete: clear all, then re-add the ones we keep
|
|
433
|
+
const toDelete = await context.cookies();
|
|
434
|
+
const remaining = name ? toDelete.filter(c => c.name !== name) : [];
|
|
435
|
+
const removedCount = toDelete.length - remaining.length;
|
|
436
|
+
await context.clearCookies();
|
|
437
|
+
if (remaining.length) {
|
|
438
|
+
await context.addCookies(remaining.map(c => ({
|
|
439
|
+
name: c.name, value: c.value, domain: c.domain, path: c.path,
|
|
440
|
+
...(c.expires && c.expires > 0 ? { expires: c.expires } : {}),
|
|
441
|
+
httpOnly: c.httpOnly, secure: c.secure, sameSite: c.sameSite
|
|
442
|
+
})));
|
|
443
|
+
}
|
|
444
|
+
(0, state_1.notifyProgress)('cookie_manager', 'completed', `Deleted ${removedCount} cookie(s)`);
|
|
445
|
+
return { success: true, message: `Deleted ${removedCount} cookie(s)` };
|
|
446
|
+
}
|
|
447
|
+
case 'clear': {
|
|
448
|
+
const allCookies = await context.cookies();
|
|
449
|
+
await context.clearCookies();
|
|
450
|
+
(0, state_1.notifyProgress)('cookie_manager', 'completed', `Cleared ${allCookies.length} cookies`);
|
|
451
|
+
return { success: true, message: `Cleared ${allCookies.length} cookies` };
|
|
452
|
+
}
|
|
453
|
+
}
|
|
454
|
+
return { success: false, error: 'Invalid action' };
|
|
455
|
+
},
|
|
456
|
+
async extract_data(params = {}) {
|
|
457
|
+
const { page } = (0, state_1.requireBrowser)();
|
|
458
|
+
const { type = 'auto', pattern, selector, jsonPath, source = 'all', autoDecode = true, flags = 'gi', types = ['all'], includeTitle = true, includeCanonical = true, maxMatches = 100, maxJsonObjects = 50, waitForSelector = false, selectorTimeout = 10000 } = params;
|
|
459
|
+
(0, state_1.notifyProgress)('extract_data', 'started', `Extracting data (type: ${type})...`);
|
|
460
|
+
const results = {
|
|
461
|
+
success: true,
|
|
462
|
+
type,
|
|
463
|
+
url: page.url(),
|
|
464
|
+
extracted: {}
|
|
465
|
+
};
|
|
466
|
+
// Helper: Extract regex matches
|
|
467
|
+
const extractRegex = async (regexPattern, regexFlags, contentSource) => {
|
|
468
|
+
let content;
|
|
469
|
+
if (contentSource === 'html') {
|
|
470
|
+
content = await page.content();
|
|
471
|
+
}
|
|
472
|
+
else if (contentSource === 'scripts') {
|
|
473
|
+
content = await page.$$eval('script', scripts => scripts.map(s => s.textContent).join('\n'));
|
|
474
|
+
}
|
|
475
|
+
else if (contentSource === 'text') {
|
|
476
|
+
content = await page.evaluate(() => document.body.innerText);
|
|
477
|
+
}
|
|
478
|
+
else {
|
|
479
|
+
// 'all' - search in both HTML and scripts
|
|
480
|
+
const html = await page.content();
|
|
481
|
+
const scripts = await page.$$eval('script', scripts => scripts.map(s => s.textContent).join('\n'));
|
|
482
|
+
content = html + '\n' + scripts;
|
|
483
|
+
}
|
|
484
|
+
const regex = new RegExp(regexPattern, regexFlags);
|
|
485
|
+
const matches = content.match(regex) || [];
|
|
486
|
+
return {
|
|
487
|
+
pattern: regexPattern,
|
|
488
|
+
flags: regexFlags,
|
|
489
|
+
matchCount: matches.length,
|
|
490
|
+
matches: matches.slice(0, maxMatches)
|
|
491
|
+
};
|
|
492
|
+
};
|
|
493
|
+
// Helper: Extract JSON data
|
|
494
|
+
const extractJson = async (jsonSource, sel, path) => {
|
|
495
|
+
const jsonData = [];
|
|
496
|
+
if (jsonSource === 'ld+json') {
|
|
497
|
+
const ldJson = await page.$$eval('script[type="application/ld+json"]', scripts => scripts.map(s => {
|
|
498
|
+
try {
|
|
499
|
+
return JSON.parse(s.textContent);
|
|
500
|
+
}
|
|
501
|
+
catch {
|
|
502
|
+
return null;
|
|
503
|
+
}
|
|
504
|
+
}).filter(Boolean));
|
|
505
|
+
jsonData.push(...ldJson);
|
|
506
|
+
}
|
|
507
|
+
else if (jsonSource === 'scripts') {
|
|
508
|
+
const content = await page.$$eval('script', scripts => scripts.map(s => s.textContent).join('\n'));
|
|
509
|
+
// Look for JSON objects in scripts
|
|
510
|
+
const jsonRegex = /\{[^{}]*(?:\{[^{}]*\}[^{}]*)*\}|\[[^\[\]]*(?:\[[^\[\]]*\][^\[\]]*)*\]/g;
|
|
511
|
+
const matches = content.match(jsonRegex) || [];
|
|
512
|
+
for (const match of matches.slice(0, maxJsonObjects)) {
|
|
513
|
+
try {
|
|
514
|
+
const parsed = JSON.parse(match);
|
|
515
|
+
jsonData.push(parsed);
|
|
516
|
+
}
|
|
517
|
+
catch { }
|
|
518
|
+
}
|
|
519
|
+
}
|
|
520
|
+
else if (jsonSource === 'api') {
|
|
521
|
+
// Try to find API responses in page data
|
|
522
|
+
const apiData = await page.evaluate(() => {
|
|
523
|
+
const data = [];
|
|
524
|
+
// Look for common API data storage patterns
|
|
525
|
+
if (window.__DATA__)
|
|
526
|
+
data.push(window.__DATA__);
|
|
527
|
+
if (window.__INITIAL_STATE__)
|
|
528
|
+
data.push(window.__INITIAL_STATE__);
|
|
529
|
+
if (window.__APP_DATA__)
|
|
530
|
+
data.push(window.__APP_DATA__);
|
|
531
|
+
if (window.data)
|
|
532
|
+
data.push(window.data);
|
|
533
|
+
if (window.config)
|
|
534
|
+
data.push(window.config);
|
|
535
|
+
return data;
|
|
536
|
+
});
|
|
537
|
+
jsonData.push(...apiData);
|
|
538
|
+
}
|
|
539
|
+
else if (sel) {
|
|
540
|
+
try {
|
|
541
|
+
const text = await page.$eval(sel, el => el.textContent);
|
|
542
|
+
const parsed = JSON.parse(text);
|
|
543
|
+
jsonData.push(parsed);
|
|
544
|
+
}
|
|
545
|
+
catch { }
|
|
546
|
+
}
|
|
547
|
+
else {
|
|
548
|
+
// 'page' - try all sources
|
|
549
|
+
const ldJson = await page.$$eval('script[type="application/ld+json"]', scripts => scripts.map(s => {
|
|
550
|
+
try {
|
|
551
|
+
return JSON.parse(s.textContent);
|
|
552
|
+
}
|
|
553
|
+
catch {
|
|
554
|
+
return null;
|
|
555
|
+
}
|
|
556
|
+
}).filter(Boolean));
|
|
557
|
+
jsonData.push(...ldJson);
|
|
558
|
+
}
|
|
559
|
+
// Apply JSONPath if specified
|
|
560
|
+
if (path && jsonData.length > 0) {
|
|
561
|
+
// Simple JSONPath implementation
|
|
562
|
+
const getPath = (obj, pathStr) => {
|
|
563
|
+
const parts = pathStr.replace(/^\$\./, '').split('.');
|
|
564
|
+
let current = obj;
|
|
565
|
+
for (const part of parts) {
|
|
566
|
+
if (current === null || current === undefined)
|
|
567
|
+
return undefined;
|
|
568
|
+
if (part.includes('[') && part.includes(']')) {
|
|
569
|
+
const arrName = part.substring(0, part.indexOf('['));
|
|
570
|
+
const idx = parseInt(part.match(/\[(\d+)\]/)?.[1] || '0');
|
|
571
|
+
current = current[arrName]?.[idx];
|
|
572
|
+
}
|
|
573
|
+
else {
|
|
574
|
+
current = current[part];
|
|
575
|
+
}
|
|
576
|
+
}
|
|
577
|
+
return current;
|
|
578
|
+
};
|
|
579
|
+
return jsonData.map(obj => ({
|
|
580
|
+
original: obj,
|
|
581
|
+
extracted: getPath(obj, path)
|
|
582
|
+
}));
|
|
583
|
+
}
|
|
584
|
+
return jsonData;
|
|
585
|
+
};
|
|
586
|
+
// Helper: Extract meta tags
|
|
587
|
+
const extractMeta = async (metaTypes) => {
|
|
588
|
+
const meta = await page.evaluate(([includeTitle, includeCanonical]) => {
|
|
589
|
+
const result = { meta: {}, og: {}, twitter: {} };
|
|
590
|
+
document.querySelectorAll('meta').forEach(tag => {
|
|
591
|
+
const name = tag.getAttribute('name') || tag.getAttribute('property');
|
|
592
|
+
const content = tag.getAttribute('content');
|
|
593
|
+
if (name && content) {
|
|
594
|
+
if (name.startsWith('og:')) {
|
|
595
|
+
result.og[name.replace('og:', '')] = content;
|
|
596
|
+
}
|
|
597
|
+
else if (name.startsWith('twitter:')) {
|
|
598
|
+
result.twitter[name.replace('twitter:', '')] = content;
|
|
599
|
+
}
|
|
600
|
+
else {
|
|
601
|
+
result.meta[name] = content;
|
|
602
|
+
}
|
|
603
|
+
}
|
|
604
|
+
});
|
|
605
|
+
if (includeTitle) {
|
|
606
|
+
result.title = document.title;
|
|
607
|
+
}
|
|
608
|
+
if (includeCanonical) {
|
|
609
|
+
result.canonical = document.querySelector('link[rel="canonical"]')?.href;
|
|
610
|
+
}
|
|
611
|
+
return result;
|
|
612
|
+
}, [includeTitle, includeCanonical]);
|
|
613
|
+
// Filter by requested types
|
|
614
|
+
const filtered = {};
|
|
615
|
+
if (metaTypes.includes('all')) {
|
|
616
|
+
return meta;
|
|
617
|
+
}
|
|
618
|
+
if (metaTypes.includes('meta'))
|
|
619
|
+
filtered.meta = meta.meta;
|
|
620
|
+
if (metaTypes.includes('og'))
|
|
621
|
+
filtered.og = meta.og;
|
|
622
|
+
if (metaTypes.includes('twitter'))
|
|
623
|
+
filtered.twitter = meta.twitter;
|
|
624
|
+
if (includeTitle)
|
|
625
|
+
filtered.title = meta.title;
|
|
626
|
+
if (includeCanonical)
|
|
627
|
+
filtered.canonical = meta.canonical;
|
|
628
|
+
return filtered;
|
|
629
|
+
};
|
|
630
|
+
// Helper: Extract structured data from selector
|
|
631
|
+
const extractStructured = async (sel, wait = false, timeout = 10000) => {
|
|
632
|
+
if (wait) {
|
|
633
|
+
await page.waitForSelector(sel, { timeout });
|
|
634
|
+
}
|
|
635
|
+
const element = await page.$(sel);
|
|
636
|
+
if (!element) {
|
|
637
|
+
return { error: `Element not found: ${sel}` };
|
|
638
|
+
}
|
|
639
|
+
const data = await element.evaluate(el => ({
|
|
640
|
+
tagName: el.tagName,
|
|
641
|
+
text: el.innerText,
|
|
642
|
+
html: el.innerHTML,
|
|
643
|
+
attributes: Object.fromEntries([...el.attributes].map(a => [a.name, a.value])),
|
|
644
|
+
childCount: el.children.length,
|
|
645
|
+
boundingBox: el.getBoundingClientRect ? {
|
|
646
|
+
x: el.getBoundingClientRect().x,
|
|
647
|
+
y: el.getBoundingClientRect().y,
|
|
648
|
+
width: el.getBoundingClientRect().width,
|
|
649
|
+
height: el.getBoundingClientRect().height
|
|
650
|
+
} : null
|
|
651
|
+
}));
|
|
652
|
+
return data;
|
|
653
|
+
};
|
|
654
|
+
// Helper: Auto-detect and extract all
|
|
655
|
+
const extractAuto = async () => {
|
|
656
|
+
const autoResults = {
|
|
657
|
+
meta: null,
|
|
658
|
+
json: null,
|
|
659
|
+
structured: null,
|
|
660
|
+
patterns: []
|
|
661
|
+
};
|
|
662
|
+
// Extract meta tags
|
|
663
|
+
try {
|
|
664
|
+
autoResults.meta = await extractMeta(['all']);
|
|
665
|
+
}
|
|
666
|
+
catch (e) { }
|
|
667
|
+
// Extract JSON-LD
|
|
668
|
+
try {
|
|
669
|
+
autoResults.json = await extractJson('ld+json');
|
|
670
|
+
}
|
|
671
|
+
catch (e) { }
|
|
672
|
+
// Look for common data patterns
|
|
673
|
+
const commonPatterns = [
|
|
674
|
+
{ name: 'emails', pattern: '[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}' },
|
|
675
|
+
{ name: 'phones', pattern: '(\+?1?[-.\s]?)?\(?[0-9]{3}\)?[-.\s]?[0-9]{3}[-.\s]?[0-9]{4}' },
|
|
676
|
+
{ name: 'urls', pattern: 'https?://[^\s<>"{}|\\^`\[\]]+' },
|
|
677
|
+
{ name: 'ipv4', pattern: '\b(?:(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.){3}(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\b' }
|
|
678
|
+
];
|
|
679
|
+
const pageText = await page.evaluate(() => document.body.innerText);
|
|
680
|
+
for (const { name, pattern } of commonPatterns) {
|
|
681
|
+
const regex = new RegExp(pattern, 'gi');
|
|
682
|
+
const matches = [...new Set(pageText.match(regex) || [])];
|
|
683
|
+
if (matches.length > 0) {
|
|
684
|
+
autoResults.patterns.push({ type: name, count: matches.length, samples: matches.slice(0, 10) });
|
|
685
|
+
}
|
|
686
|
+
}
|
|
687
|
+
return autoResults;
|
|
688
|
+
};
|
|
689
|
+
// Main switch based on type
|
|
690
|
+
switch (type) {
|
|
691
|
+
case 'regex': {
|
|
692
|
+
if (!pattern) {
|
|
693
|
+
return { success: false, error: 'Pattern is required for regex extraction' };
|
|
694
|
+
}
|
|
695
|
+
results.extracted = await extractRegex(pattern, flags, source);
|
|
696
|
+
(0, state_1.notifyProgress)('extract_data', 'completed', `Regex: ${results.extracted.matchCount} matches`);
|
|
697
|
+
break;
|
|
698
|
+
}
|
|
699
|
+
case 'json': {
|
|
700
|
+
results.extracted = await extractJson(source, selector, jsonPath);
|
|
701
|
+
results.count = Array.isArray(results.extracted) ? results.extracted.length : 0;
|
|
702
|
+
(0, state_1.notifyProgress)('extract_data', 'completed', `JSON: ${results.count} objects`);
|
|
703
|
+
break;
|
|
704
|
+
}
|
|
705
|
+
case 'meta': {
|
|
706
|
+
results.extracted = await extractMeta(types);
|
|
707
|
+
const tagCount = Object.values(results.extracted).reduce((sum, val) => {
|
|
708
|
+
if (typeof val === 'object' && val !== null) {
|
|
709
|
+
return sum + Object.keys(val).length;
|
|
710
|
+
}
|
|
711
|
+
return sum + (val ? 1 : 0);
|
|
712
|
+
}, 0);
|
|
713
|
+
(0, state_1.notifyProgress)('extract_data', 'completed', `Meta: ${tagCount} tags`);
|
|
714
|
+
break;
|
|
715
|
+
}
|
|
716
|
+
case 'structured': {
|
|
717
|
+
if (!selector) {
|
|
718
|
+
return { success: false, error: 'Selector is required for structured extraction' };
|
|
719
|
+
}
|
|
720
|
+
results.extracted = await extractStructured(selector, waitForSelector, selectorTimeout);
|
|
721
|
+
if (results.extracted.error) {
|
|
722
|
+
results.success = false;
|
|
723
|
+
results.error = results.extracted.error;
|
|
724
|
+
delete results.extracted;
|
|
725
|
+
}
|
|
726
|
+
(0, state_1.notifyProgress)('extract_data', 'completed', results.success ? 'Structured data extracted' : 'Extraction failed');
|
|
727
|
+
break;
|
|
728
|
+
}
|
|
729
|
+
case 'auto': {
|
|
730
|
+
results.extracted = await extractAuto();
|
|
731
|
+
const summary = [];
|
|
732
|
+
if (results.extracted.meta)
|
|
733
|
+
summary.push('meta');
|
|
734
|
+
if (results.extracted.json?.length)
|
|
735
|
+
summary.push('json');
|
|
736
|
+
if (results.extracted.patterns?.length)
|
|
737
|
+
summary.push('patterns');
|
|
738
|
+
(0, state_1.notifyProgress)('extract_data', 'completed', `Auto: ${summary.join(', ')}`);
|
|
739
|
+
break;
|
|
740
|
+
}
|
|
741
|
+
case 'deobfuscate': {
|
|
742
|
+
(0, state_1.notifyProgress)('extract_data', 'in_progress', 'Deobfuscating JavaScript (enhanced)...');
|
|
743
|
+
const scriptContents = await page.evaluate(() => {
|
|
744
|
+
return Array.from(document.querySelectorAll('script')).map(s => s.textContent).join('\n');
|
|
745
|
+
}).catch(() => '');
|
|
746
|
+
const externalScripts = await page.evaluate(() => {
|
|
747
|
+
return Array.from(document.querySelectorAll('script[src]')).map(s => s.src);
|
|
748
|
+
}).catch(() => []);
|
|
749
|
+
let allJs = scriptContents;
|
|
750
|
+
for (const src of externalScripts.slice(0, 10)) {
|
|
751
|
+
try {
|
|
752
|
+
const resp = await fetch(src);
|
|
753
|
+
allJs += '\n' + await resp.text();
|
|
754
|
+
}
|
|
755
|
+
catch (e) { }
|
|
756
|
+
}
|
|
757
|
+
const deobfuscated = {
|
|
758
|
+
stringArrays: [], decodedStrings: [], functionMappings: [],
|
|
759
|
+
apiEndpoints: [], urls: [], fetchCalls: [],
|
|
760
|
+
webpackModules: [], evalUnpacked: [], resolvedConcats: [], unicodeDecoded: []
|
|
761
|
+
};
|
|
762
|
+
// 1. Original _0x style string arrays
|
|
763
|
+
const arrayPattern = /(?:const|var|let)\s+(_0x[a-f0-9]+)\s*=\s*\[([^\]]{20,})\]/g;
|
|
764
|
+
let match;
|
|
765
|
+
while ((match = arrayPattern.exec(allJs)) !== null) {
|
|
766
|
+
const varName = match[1];
|
|
767
|
+
try {
|
|
768
|
+
const items = match[2].match(/'([^']*)'|"([^"]*)"/g) || [];
|
|
769
|
+
const decoded = items.map(s => s.replace(/^['"]|['"]$/g, ''));
|
|
770
|
+
deobfuscated.stringArrays.push({ variable: varName, count: decoded.length, strings: decoded });
|
|
771
|
+
deobfuscated.decodedStrings.push(...decoded);
|
|
772
|
+
}
|
|
773
|
+
catch (e) { }
|
|
774
|
+
}
|
|
775
|
+
// 2. Hex-encoded strings
|
|
776
|
+
const hexStrings = [...new Set((allJs.match(/(?:'(?:\\x[0-9a-f]{2})+[^']*'|"(?:\\x[0-9a-f]{2})+[^"]*")/gi) || []))];
|
|
777
|
+
for (const hs of hexStrings.slice(0, 50)) {
|
|
778
|
+
try {
|
|
779
|
+
const decoded = hs.slice(1, -1).replace(/\\x([0-9a-f]{2})/gi, (_, h) => String.fromCharCode(parseInt(h, 16)));
|
|
780
|
+
if (decoded.length > 2)
|
|
781
|
+
deobfuscated.decodedStrings.push(decoded);
|
|
782
|
+
}
|
|
783
|
+
catch (e) { }
|
|
784
|
+
}
|
|
785
|
+
// 3. NEW: Unicode escape sequences (\u0066\u0065\u0074\u0063\u0068 → fetch)
|
|
786
|
+
const unicodePattern = /(?:'(?:\\u[0-9a-f]{4})+[^']*'|"(?:\\u[0-9a-f]{4})+[^"]*")/gi;
|
|
787
|
+
const unicodeMatches = allJs.match(unicodePattern) || [];
|
|
788
|
+
for (const um of unicodeMatches.slice(0, 50)) {
|
|
789
|
+
try {
|
|
790
|
+
const decoded = um.slice(1, -1).replace(/\\u([0-9a-f]{4})/gi, (_, h) => String.fromCharCode(parseInt(h, 16)));
|
|
791
|
+
if (decoded.length > 1) {
|
|
792
|
+
deobfuscated.unicodeDecoded.push(decoded);
|
|
793
|
+
deobfuscated.decodedStrings.push(decoded);
|
|
794
|
+
}
|
|
795
|
+
}
|
|
796
|
+
catch (e) { }
|
|
797
|
+
}
|
|
798
|
+
// 4. NEW: Eval unpacker — eval(function(p,a,c,k,e,d){...})
|
|
799
|
+
const evalPattern = /eval\s*\(\s*function\s*\(\s*p\s*,\s*a\s*,\s*c\s*,\s*k\s*,\s*e\s*,?\s*[dr]?\s*\)\s*\{[^}]*\}\s*\(\s*'([^']*)'(?:\s*,\s*(\d+)){2}\s*,\s*'([^']*)'/g;
|
|
800
|
+
let evalMatch;
|
|
801
|
+
while ((evalMatch = evalPattern.exec(allJs)) !== null) {
|
|
802
|
+
try {
|
|
803
|
+
const p = evalMatch[1], a = parseInt(evalMatch[2]) || 62;
|
|
804
|
+
const keywords = evalMatch[3].split('|');
|
|
805
|
+
const unpacked = p.replace(/\b\w+\b/g, w => {
|
|
806
|
+
const n = parseInt(w, a);
|
|
807
|
+
return (n < keywords.length && keywords[n]) ? keywords[n] : w;
|
|
808
|
+
});
|
|
809
|
+
deobfuscated.evalUnpacked.push(unpacked.substring(0, 3000));
|
|
810
|
+
// Extract strings from unpacked code
|
|
811
|
+
const unpackedStrings = unpacked.match(/['"]([^'"]{3,})['"]/g) || [];
|
|
812
|
+
for (const s of unpackedStrings.slice(0, 100)) {
|
|
813
|
+
deobfuscated.decodedStrings.push(s.replace(/^['"]|['"]$/g, ''));
|
|
814
|
+
}
|
|
815
|
+
}
|
|
816
|
+
catch (e) { }
|
|
817
|
+
}
|
|
818
|
+
// Also handle simpler eval patterns
|
|
819
|
+
const simpleEval = /eval\s*\(\s*['"]([^'"]{10,})['"]\s*\)/g;
|
|
820
|
+
let seMatch;
|
|
821
|
+
while ((seMatch = simpleEval.exec(allJs)) !== null) {
|
|
822
|
+
deobfuscated.evalUnpacked.push(seMatch[1].substring(0, 2000));
|
|
823
|
+
}
|
|
824
|
+
// 5. NEW: Webpack module detection
|
|
825
|
+
const webpackPatterns = [
|
|
826
|
+
/(?:__webpack_require__|__webpack_modules__)\s*\[\s*['"]?(\w+)['"]?\s*\]/g,
|
|
827
|
+
/(?:const|var|let)\s+\w+\s*=\s*\{[\s\S]{0,50}__webpack_require__/g,
|
|
828
|
+
/\(\s*function\s*\(\s*modules\s*\)\s*\{[\s\S]{0,200}__webpack_require__/g
|
|
829
|
+
];
|
|
830
|
+
const webpackExports = allJs.match(/(?:module\.exports|exports\.\w+)\s*=\s*['"]([^'"]+)['"]/g) || [];
|
|
831
|
+
for (const exp of webpackExports.slice(0, 30)) {
|
|
832
|
+
const val = exp.match(/=\s*['"]([^'"]+)['"]/);
|
|
833
|
+
if (val) {
|
|
834
|
+
deobfuscated.webpackModules.push(val[1]);
|
|
835
|
+
deobfuscated.decodedStrings.push(val[1]);
|
|
836
|
+
}
|
|
837
|
+
}
|
|
838
|
+
// Detect webpack chunk loading and module IDs
|
|
839
|
+
const chunkIds = allJs.match(/webpackChunk\w*\.push\s*\(\s*\[\s*\[([^\]]+)\]/g) || [];
|
|
840
|
+
for (const ci of chunkIds.slice(0, 10)) {
|
|
841
|
+
deobfuscated.webpackModules.push(`chunk: ${ci.substring(0, 100)}`);
|
|
842
|
+
}
|
|
843
|
+
// 6. NEW: Terser/UglifyJS single-letter variable mappings
|
|
844
|
+
const terserPattern = /(?:var|let|const)\s+([a-z])\s*=\s*['"]([^'"]{2,})['"]/gi;
|
|
845
|
+
let terserMatch;
|
|
846
|
+
const terserMappings = {};
|
|
847
|
+
while ((terserMatch = terserPattern.exec(allJs)) !== null) {
|
|
848
|
+
const varName = terserMatch[1], value = terserMatch[2];
|
|
849
|
+
if (value.length > 2 && value.length < 200) {
|
|
850
|
+
terserMappings[varName] = value;
|
|
851
|
+
deobfuscated.functionMappings.push({ variable: varName, value: value });
|
|
852
|
+
deobfuscated.decodedStrings.push(value);
|
|
853
|
+
}
|
|
854
|
+
}
|
|
855
|
+
// 7. NEW: String concatenation resolution ("htt"+"ps://" → "https://")
|
|
856
|
+
const concatPattern = /(?:['"][^'"]*['"]\s*\+\s*){2,}['"][^'"]*['"]/g;
|
|
857
|
+
const concatMatches = allJs.match(concatPattern) || [];
|
|
858
|
+
for (const cm of concatMatches.slice(0, 50)) {
|
|
859
|
+
try {
|
|
860
|
+
const parts = cm.match(/['"]([^'"]*)['"]|(['"])/g) || [];
|
|
861
|
+
const resolved = parts.map(p => p.replace(/^['"]|['"]$/g, '')).join('');
|
|
862
|
+
if (resolved.length > 3) {
|
|
863
|
+
deobfuscated.resolvedConcats.push(resolved);
|
|
864
|
+
deobfuscated.decodedStrings.push(resolved);
|
|
865
|
+
}
|
|
866
|
+
}
|
|
867
|
+
catch (e) { }
|
|
868
|
+
}
|
|
869
|
+
// 8. NEW: Array rotation detection — function with push/shift on array
|
|
870
|
+
const rotationPattern = /function\s+\w*\s*\(\s*(_0x[a-f0-9]+)\s*,\s*\w+\s*\)\s*\{[\s\S]{0,500}push\s*\(\s*\1\s*\.\s*shift\s*\(\s*\)\s*\)/g;
|
|
871
|
+
const rotations = allJs.match(rotationPattern) || [];
|
|
872
|
+
if (rotations.length > 0) {
|
|
873
|
+
deobfuscated.functionMappings.push({ type: 'array_rotation', count: rotations.length, note: 'Array rotation functions detected — strings may be shifted' });
|
|
874
|
+
}
|
|
875
|
+
// Extract URLs and API endpoints from all decoded strings
|
|
876
|
+
deobfuscated.urls = [...new Set(deobfuscated.decodedStrings.filter((s) => s.match(/^(https?:\/\/|\/)/) || s.match(/\.(php|json|api|asp|jsp)$/i)))].slice(0, 50);
|
|
877
|
+
deobfuscated.apiEndpoints = [...new Set(deobfuscated.decodedStrings.filter((s) => s.match(/^\/[a-z]/i) && s.length > 3 && s.length < 100))].slice(0, 30);
|
|
878
|
+
// Find fetch patterns
|
|
879
|
+
const fetchPatterns = allJs.match(/fetch\s*\(\s*['"]([^'"]+)['"]/g) || [];
|
|
880
|
+
deobfuscated.fetchCalls = fetchPatterns.map(f => f.replace(/fetch\s*\(\s*['"]/, '').replace(/['"]$/, '')).slice(0, 20);
|
|
881
|
+
deobfuscated.decodedStrings = [...new Set(deobfuscated.decodedStrings)].slice(0, 500);
|
|
882
|
+
results.extracted = deobfuscated;
|
|
883
|
+
const summary = `${deobfuscated.stringArrays.length} arrays, ${deobfuscated.decodedStrings.length} strings, ${deobfuscated.evalUnpacked.length} eval unpacked, ${deobfuscated.webpackModules.length} webpack modules, ${deobfuscated.resolvedConcats.length} concats resolved, ${deobfuscated.unicodeDecoded.length} unicode decoded`;
|
|
884
|
+
(0, state_1.notifyProgress)('extract_data', 'completed', `Deobfuscated(enhanced): ${summary}`);
|
|
885
|
+
break;
|
|
886
|
+
}
|
|
887
|
+
case 'apiDiscovery': {
|
|
888
|
+
(0, state_1.notifyProgress)('extract_data', 'in_progress', 'Discovering hidden API endpoints...');
|
|
889
|
+
const apiResults = {
|
|
890
|
+
fetchEndpoints: [], xhrEndpoints: [], formActions: [],
|
|
891
|
+
scriptSources: [], inlineApiPatterns: [], postBodies: [], dynamicApis: []
|
|
892
|
+
};
|
|
893
|
+
// 1. Intercept runtime fetch/XHR
|
|
894
|
+
try {
|
|
895
|
+
const runtimeApis = await page.evaluate(() => {
|
|
896
|
+
return new Promise((resolve) => {
|
|
897
|
+
const found = [];
|
|
898
|
+
if (window.__capturedApis) {
|
|
899
|
+
resolve(window.__capturedApis);
|
|
900
|
+
return;
|
|
901
|
+
}
|
|
902
|
+
const origFetch = window.fetch;
|
|
903
|
+
window.fetch = function (...args) {
|
|
904
|
+
try {
|
|
905
|
+
const url = typeof args[0] === 'string' ? args[0] : args[0]?.url;
|
|
906
|
+
const opts = args[1] || {};
|
|
907
|
+
found.push({
|
|
908
|
+
type: 'fetch', url, method: opts.method || 'GET',
|
|
909
|
+
body: typeof opts.body === 'string' ? opts.body.substring(0, 500) : null
|
|
910
|
+
});
|
|
911
|
+
}
|
|
912
|
+
catch (e) { }
|
|
913
|
+
return origFetch.apply(this, args);
|
|
914
|
+
};
|
|
915
|
+
const origOpen = XMLHttpRequest.prototype.open;
|
|
916
|
+
const origSend = XMLHttpRequest.prototype.send;
|
|
917
|
+
XMLHttpRequest.prototype.open = function (method, url, ...rest) { this.__apiUrl = url; this.__apiMethod = method; return origOpen.apply(this, [method, url, ...rest]); };
|
|
918
|
+
XMLHttpRequest.prototype.send = function (body) {
|
|
919
|
+
found.push({
|
|
920
|
+
type: 'xhr', url: this.__apiUrl, method: this.__apiMethod,
|
|
921
|
+
body: typeof body === 'string' ? body.substring(0, 500) : null
|
|
922
|
+
});
|
|
923
|
+
return origSend.apply(this, [body]);
|
|
924
|
+
};
|
|
925
|
+
window.__capturedApis = found;
|
|
926
|
+
setTimeout(() => resolve(found), 3000);
|
|
927
|
+
});
|
|
928
|
+
});
|
|
929
|
+
apiResults.dynamicApis = runtimeApis;
|
|
930
|
+
}
|
|
931
|
+
catch (e) {
|
|
932
|
+
apiResults.dynamicApis = [];
|
|
933
|
+
}
|
|
934
|
+
// 2. Static analysis
|
|
935
|
+
const allScriptContent = await page.evaluate(() => {
|
|
936
|
+
return Array.from(document.querySelectorAll('script')).map(s => s.textContent).join('\n');
|
|
937
|
+
}).catch(() => '');
|
|
938
|
+
const fetchRegex = /fetch\s*\(\s*(?:['"`]([^'"`]+)['"`]|([a-zA-Z_$][a-zA-Z0-9_$]*))/g;
|
|
939
|
+
let fMatch;
|
|
940
|
+
while ((fMatch = fetchRegex.exec(allScriptContent)) !== null) {
|
|
941
|
+
apiResults.fetchEndpoints.push((fMatch[1] || fMatch[2]));
|
|
942
|
+
}
|
|
943
|
+
apiResults.fetchEndpoints = [...new Set(apiResults.fetchEndpoints)].slice(0, 30);
|
|
944
|
+
const xhrRegex = /\.open\s*\(\s*['"](?:GET|POST|PUT|DELETE)['"]\s*,\s*['"`]([^'"`]+)['"`]/gi;
|
|
945
|
+
let xMatch;
|
|
946
|
+
while ((xMatch = xhrRegex.exec(allScriptContent)) !== null) {
|
|
947
|
+
apiResults.xhrEndpoints.push(xMatch[1]);
|
|
948
|
+
}
|
|
949
|
+
apiResults.xhrEndpoints = [...new Set(apiResults.xhrEndpoints)].slice(0, 30);
|
|
950
|
+
apiResults.formActions = await page.evaluate(() => {
|
|
951
|
+
return Array.from(document.querySelectorAll('form[action]')).map((f) => ({ action: f.action, method: f.method || 'GET', id: f.id || null }));
|
|
952
|
+
}).catch(() => []);
|
|
953
|
+
const postBodyPatterns = allScriptContent.match(/(?:URLSearchParams|FormData|JSON\.stringify)\s*\(\s*\{[^}]{5,200}\}/g) || [];
|
|
954
|
+
apiResults.postBodies = postBodyPatterns.slice(0, 10);
|
|
955
|
+
const apiUrlPattern = /['"`]((?:https?:\/\/[^'"`]+|\/)(?:[a-zA-Z0-9_\-\/]+\.(?:php|json|api|asp|aspx|do|action))[^'"`]*)['"`]/g;
|
|
956
|
+
let apiMatch;
|
|
957
|
+
while ((apiMatch = apiUrlPattern.exec(allScriptContent)) !== null) {
|
|
958
|
+
apiResults.inlineApiPatterns.push(apiMatch[1]);
|
|
959
|
+
}
|
|
960
|
+
apiResults.inlineApiPatterns = [...new Set(apiResults.inlineApiPatterns)].slice(0, 30);
|
|
961
|
+
apiResults.scriptSources = await page.evaluate(() => {
|
|
962
|
+
return Array.from(document.querySelectorAll('script[src]')).map(s => s.src);
|
|
963
|
+
}).catch(() => []);
|
|
964
|
+
results.extracted = apiResults;
|
|
965
|
+
const totalFound = apiResults.fetchEndpoints.length + apiResults.xhrEndpoints.length +
|
|
966
|
+
apiResults.inlineApiPatterns.length + apiResults.dynamicApis.length;
|
|
967
|
+
(0, state_1.notifyProgress)('extract_data', 'completed', `API Discovery: ${totalFound} endpoints found`);
|
|
968
|
+
break;
|
|
969
|
+
}
|
|
970
|
+
// ====== FEATURE 4: Response Auto-Decryption ======
|
|
971
|
+
case 'decrypt': {
|
|
972
|
+
(0, state_1.notifyProgress)('extract_data', 'in_progress', 'Auto-decrypting data...');
|
|
973
|
+
const { encryptedData, autoFindKey = true } = params;
|
|
974
|
+
const decryptResults = {
|
|
975
|
+
original: null, decoded: [], detectedEncoding: [], extractedKeys: [], aesDecrypted: null
|
|
976
|
+
};
|
|
977
|
+
// Get data to decrypt — from param or from page
|
|
978
|
+
let dataToDecrypt = encryptedData;
|
|
979
|
+
if (!dataToDecrypt) {
|
|
980
|
+
// Try to get from clipboard or last API response
|
|
981
|
+
const lastApiResponse = state_1.state.networkRecords.filter(r => r.responseBody).pop();
|
|
982
|
+
if (lastApiResponse)
|
|
983
|
+
dataToDecrypt = lastApiResponse.responseBody;
|
|
984
|
+
}
|
|
985
|
+
if (!dataToDecrypt) {
|
|
986
|
+
return { success: false, error: 'No data to decrypt. Provide encryptedData parameter or start network_recorder first.' };
|
|
987
|
+
}
|
|
988
|
+
decryptResults.original = dataToDecrypt.substring(0, 500);
|
|
989
|
+
// 1. Base64 chain decode (recursive, up to 5 levels)
|
|
990
|
+
let b64Data = dataToDecrypt.trim();
|
|
991
|
+
for (let level = 0; level < 5; level++) {
|
|
992
|
+
if (!/^[A-Za-z0-9+/=]+$/.test(b64Data) || b64Data.length < 4)
|
|
993
|
+
break;
|
|
994
|
+
try {
|
|
995
|
+
const decoded = Buffer.from(b64Data, 'base64').toString('utf-8');
|
|
996
|
+
if (decoded && decoded.length > 0 && !/[\x00-\x08\x0e-\x1f]/.test(decoded.substring(0, 100))) {
|
|
997
|
+
decryptResults.decoded.push({ level: level + 1, type: 'base64', value: decoded.substring(0, 5000) });
|
|
998
|
+
decryptResults.detectedEncoding.push('base64');
|
|
999
|
+
// Check if result is JSON
|
|
1000
|
+
try {
|
|
1001
|
+
const json = JSON.parse(decoded);
|
|
1002
|
+
decryptResults.decoded.push({ level: level + 1, type: 'base64_json', value: json });
|
|
1003
|
+
}
|
|
1004
|
+
catch (e) { }
|
|
1005
|
+
b64Data = decoded; // Continue chain
|
|
1006
|
+
}
|
|
1007
|
+
else
|
|
1008
|
+
break;
|
|
1009
|
+
}
|
|
1010
|
+
catch (e) {
|
|
1011
|
+
break;
|
|
1012
|
+
}
|
|
1013
|
+
}
|
|
1014
|
+
// 2. Hex decode
|
|
1015
|
+
const hexClean = dataToDecrypt.replace(/\s+/g, '');
|
|
1016
|
+
if (/^[0-9a-f]+$/i.test(hexClean) && hexClean.length >= 6 && hexClean.length % 2 === 0) {
|
|
1017
|
+
try {
|
|
1018
|
+
const hexDecoded = Buffer.from(hexClean, 'hex').toString('utf-8');
|
|
1019
|
+
if (hexDecoded && !/[\x00-\x08\x0e-\x1f]/.test(hexDecoded.substring(0, 50))) {
|
|
1020
|
+
decryptResults.decoded.push({ type: 'hex', value: hexDecoded.substring(0, 5000) });
|
|
1021
|
+
decryptResults.detectedEncoding.push('hex');
|
|
1022
|
+
}
|
|
1023
|
+
}
|
|
1024
|
+
catch (e) { }
|
|
1025
|
+
}
|
|
1026
|
+
// 3. URL decode (multi-level)
|
|
1027
|
+
if (dataToDecrypt.includes('%')) {
|
|
1028
|
+
try {
|
|
1029
|
+
let urlDecoded = decodeURIComponent(dataToDecrypt);
|
|
1030
|
+
decryptResults.decoded.push({ type: 'url', value: urlDecoded.substring(0, 5000) });
|
|
1031
|
+
decryptResults.detectedEncoding.push('url');
|
|
1032
|
+
// Double URL decode
|
|
1033
|
+
if (urlDecoded.includes('%')) {
|
|
1034
|
+
urlDecoded = decodeURIComponent(urlDecoded);
|
|
1035
|
+
decryptResults.decoded.push({ type: 'url_double', value: urlDecoded.substring(0, 5000) });
|
|
1036
|
+
}
|
|
1037
|
+
}
|
|
1038
|
+
catch (e) { }
|
|
1039
|
+
}
|
|
1040
|
+
// 4. ROT13
|
|
1041
|
+
try {
|
|
1042
|
+
const rot13 = dataToDecrypt.replace(/[a-zA-Z]/g, (c) => {
|
|
1043
|
+
const base = c <= 'Z' ? 65 : 97;
|
|
1044
|
+
return String.fromCharCode(((c.charCodeAt(0) - base + 13) % 26) + base);
|
|
1045
|
+
});
|
|
1046
|
+
if (rot13 !== dataToDecrypt && (rot13.includes('http') || rot13.includes('www') || rot13.includes('.com'))) {
|
|
1047
|
+
decryptResults.decoded.push({ type: 'rot13', value: rot13.substring(0, 5000) });
|
|
1048
|
+
decryptResults.detectedEncoding.push('rot13');
|
|
1049
|
+
}
|
|
1050
|
+
}
|
|
1051
|
+
catch (e) { }
|
|
1052
|
+
// 5. Auto-extract encryption keys from page scripts
|
|
1053
|
+
if (autoFindKey) {
|
|
1054
|
+
try {
|
|
1055
|
+
const keys = await page.evaluate(() => {
|
|
1056
|
+
const scripts = Array.from(document.querySelectorAll('script')).map(s => s.textContent).join('\n');
|
|
1057
|
+
const found = [];
|
|
1058
|
+
// CryptoJS patterns
|
|
1059
|
+
const cryptoPatterns = [
|
|
1060
|
+
/CryptoJS\.AES\.decrypt\s*\(\s*\w+\s*,\s*['"]([^'"]+)['"]/g,
|
|
1061
|
+
/CryptoJS\.AES\.encrypt\s*\(\s*\w+\s*,\s*['"]([^'"]+)['"]/g,
|
|
1062
|
+
/CryptoJS\.enc\.Utf8\.parse\s*\(\s*['"]([^'"]+)['"]/g,
|
|
1063
|
+
/(?:secret|key|pass|password|iv|salt)\s*[:=]\s*['"]([^'"]{8,})['"]/gi,
|
|
1064
|
+
/aes(?:Key|_key|Secret)\s*[:=]\s*['"]([^'"]{8,})['"]/gi
|
|
1065
|
+
];
|
|
1066
|
+
for (const pat of cryptoPatterns) {
|
|
1067
|
+
let m;
|
|
1068
|
+
while ((m = pat.exec(scripts)) !== null) {
|
|
1069
|
+
found.push({ pattern: pat.source.substring(0, 50), key: m[1] });
|
|
1070
|
+
}
|
|
1071
|
+
}
|
|
1072
|
+
return found;
|
|
1073
|
+
});
|
|
1074
|
+
decryptResults.extractedKeys = keys.slice(0, 20);
|
|
1075
|
+
}
|
|
1076
|
+
catch (e) { }
|
|
1077
|
+
}
|
|
1078
|
+
// 6. AES decryption — try with extracted keys or user-provided key
|
|
1079
|
+
const aesKey = params.aesKey || (decryptResults.extractedKeys[0]?.key);
|
|
1080
|
+
if (aesKey && dataToDecrypt.length > 10) {
|
|
1081
|
+
try {
|
|
1082
|
+
const crypto = require('crypto');
|
|
1083
|
+
// Try AES-256-CBC
|
|
1084
|
+
for (const keyEncoding of ['utf8', 'hex', 'base64']) {
|
|
1085
|
+
try {
|
|
1086
|
+
let keyBuf;
|
|
1087
|
+
if (keyEncoding === 'utf8')
|
|
1088
|
+
keyBuf = Buffer.alloc(32); // pad to 32 bytes
|
|
1089
|
+
else
|
|
1090
|
+
keyBuf = Buffer.from(aesKey, keyEncoding);
|
|
1091
|
+
if (keyEncoding === 'utf8') {
|
|
1092
|
+
const kb = Buffer.from(aesKey, 'utf8');
|
|
1093
|
+
kb.copy(keyBuf);
|
|
1094
|
+
}
|
|
1095
|
+
// Try to decode the data from base64 first
|
|
1096
|
+
const dataBuf = Buffer.from(dataToDecrypt, 'base64');
|
|
1097
|
+
if (dataBuf.length > 16) {
|
|
1098
|
+
// IV might be first 16 bytes
|
|
1099
|
+
const iv = params.aesIV ? Buffer.from(params.aesIV, keyEncoding) : dataBuf.slice(0, 16);
|
|
1100
|
+
const encrypted = params.aesIV ? dataBuf : dataBuf.slice(16);
|
|
1101
|
+
const decipher = crypto.createDecipheriv('aes-256-cbc', keyBuf, iv);
|
|
1102
|
+
decipher.setAutoPadding(true);
|
|
1103
|
+
let decrypted = decipher.update(encrypted, undefined, 'utf8');
|
|
1104
|
+
decrypted += decipher.final('utf8');
|
|
1105
|
+
if (decrypted && decrypted.length > 0) {
|
|
1106
|
+
decryptResults.aesDecrypted = decrypted.substring(0, 5000);
|
|
1107
|
+
decryptResults.detectedEncoding.push('aes-256-cbc');
|
|
1108
|
+
// Try JSON parse
|
|
1109
|
+
try {
|
|
1110
|
+
decryptResults.aesDecrypted = JSON.parse(decrypted);
|
|
1111
|
+
}
|
|
1112
|
+
catch (e) { }
|
|
1113
|
+
break;
|
|
1114
|
+
}
|
|
1115
|
+
}
|
|
1116
|
+
}
|
|
1117
|
+
catch (e) {
|
|
1118
|
+
continue;
|
|
1119
|
+
}
|
|
1120
|
+
}
|
|
1121
|
+
}
|
|
1122
|
+
catch (e) { }
|
|
1123
|
+
}
|
|
1124
|
+
results.extracted = decryptResults;
|
|
1125
|
+
const decodedCount = decryptResults.decoded.length + (decryptResults.aesDecrypted ? 1 : 0);
|
|
1126
|
+
(0, state_1.notifyProgress)('extract_data', 'completed', `Decrypted: ${decodedCount} decodings, ${decryptResults.extractedKeys.length} keys found, encodings: ${decryptResults.detectedEncoding.join(', ') || 'none'}`);
|
|
1127
|
+
break;
|
|
1128
|
+
}
|
|
1129
|
+
default:
|
|
1130
|
+
return { success: false, error: `Unknown type: ${type}. Supported: regex, json, meta, structured, auto, deobfuscate, apiDiscovery, decrypt` };
|
|
1131
|
+
}
|
|
1132
|
+
return results;
|
|
1133
|
+
}
|
|
1134
|
+
};
|
|
1135
|
+
//# sourceMappingURL=network.js.map
|