unbrowse 3.1.0-experiments.995f8bb → 3.1.0-experiments.9cbcb13
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cli.js +79 -33
- package/dist/index.js +2 -6
- package/dist/mcp.js +73 -22
- package/dist/server.js +25994 -0
- package/package.json +1 -2
- package/vendor/kuri/manifest.json +1 -1
- package/runtime-src/agent-outcome.ts +0 -166
- package/runtime-src/analytics-session.ts +0 -55
- package/runtime-src/api/browse-index.ts +0 -343
- package/runtime-src/api/browse-session.ts +0 -572
- package/runtime-src/api/browse-submit-prereqs.ts +0 -48
- package/runtime-src/api/browse-submit.ts +0 -1184
- package/runtime-src/api/routes.ts +0 -1833
- package/runtime-src/auth/browser-cookies.ts +0 -423
- package/runtime-src/auth/index.ts +0 -535
- package/runtime-src/auth/runtime.ts +0 -116
- package/runtime-src/browser/index.ts +0 -659
- package/runtime-src/browser/types.ts +0 -41
- package/runtime-src/build-info.generated.ts +0 -6
- package/runtime-src/capture/index.ts +0 -1929
- package/runtime-src/capture/prefetch.ts +0 -95
- package/runtime-src/capture/rsc.ts +0 -45
- package/runtime-src/cli/shortcuts.ts +0 -273
- package/runtime-src/cli.ts +0 -1734
- package/runtime-src/client/graph-client.ts +0 -100
- package/runtime-src/client/index.ts +0 -1425
- package/runtime-src/debug-trace.ts +0 -18
- package/runtime-src/domain.ts +0 -38
- package/runtime-src/execution/index.ts +0 -3407
- package/runtime-src/execution/retry.ts +0 -46
- package/runtime-src/execution/robots.ts +0 -167
- package/runtime-src/execution/search-forms.ts +0 -188
- package/runtime-src/execution/token-resolver.ts +0 -135
- package/runtime-src/extraction/index.ts +0 -1507
- package/runtime-src/foundry/publish-bundle.ts +0 -392
- package/runtime-src/graph/agent-augment.ts +0 -315
- package/runtime-src/graph/index.ts +0 -1532
- package/runtime-src/graph/local-fixtures.ts +0 -393
- package/runtime-src/graph/local-harness.ts +0 -646
- package/runtime-src/graph/planner.ts +0 -411
- package/runtime-src/graph/session.ts +0 -294
- package/runtime-src/graph/trace-store.ts +0 -136
- package/runtime-src/impact-log.ts +0 -227
- package/runtime-src/index.ts +0 -24
- package/runtime-src/indexer/index.ts +0 -465
- package/runtime-src/intent-match.ts +0 -1515
- package/runtime-src/kuri/client.ts +0 -1839
- package/runtime-src/logger.ts +0 -30
- package/runtime-src/marketplace/index.ts +0 -111
- package/runtime-src/mcp.ts +0 -1911
- package/runtime-src/orchestrator/browser-agent.ts +0 -374
- package/runtime-src/orchestrator/dag-advisor.ts +0 -59
- package/runtime-src/orchestrator/dag-feedback.ts +0 -257
- package/runtime-src/orchestrator/first-pass-action.ts +0 -403
- package/runtime-src/orchestrator/index.ts +0 -4480
- package/runtime-src/orchestrator/passive-publish.ts +0 -187
- package/runtime-src/orchestrator/timing-economics.ts +0 -80
- package/runtime-src/payments/cascade.ts +0 -137
- package/runtime-src/payments/index.ts +0 -270
- package/runtime-src/payments/lobster-pay.ts +0 -182
- package/runtime-src/payments/wallet.ts +0 -98
- package/runtime-src/publish/review-context.ts +0 -93
- package/runtime-src/publish/sanitize.ts +0 -197
- package/runtime-src/publish/schema-review.ts +0 -192
- package/runtime-src/publish-admission.ts +0 -388
- package/runtime-src/ratelimit/index.ts +0 -23
- package/runtime-src/reverse-engineer/bundle-scanner.ts +0 -127
- package/runtime-src/reverse-engineer/description-prompt.ts +0 -213
- package/runtime-src/reverse-engineer/index.ts +0 -1551
- package/runtime-src/reverse-engineer/token-sources.ts +0 -379
- package/runtime-src/router.ts +0 -17
- package/runtime-src/routing-telemetry.ts +0 -395
- package/runtime-src/runtime/browser-access.ts +0 -11
- package/runtime-src/runtime/browser-auth.ts +0 -12
- package/runtime-src/runtime/browser-host.ts +0 -48
- package/runtime-src/runtime/lifecycle.ts +0 -17
- package/runtime-src/runtime/local-server.ts +0 -311
- package/runtime-src/runtime/paths.ts +0 -99
- package/runtime-src/runtime/setup.ts +0 -251
- package/runtime-src/runtime/supervisor.ts +0 -69
- package/runtime-src/runtime/update-hints.ts +0 -351
- package/runtime-src/server.ts +0 -100
- package/runtime-src/session-logs.ts +0 -142
- package/runtime-src/settings.ts +0 -221
- package/runtime-src/single-binary.ts +0 -143
- package/runtime-src/site-policy.ts +0 -54
- package/runtime-src/stale-cleanup-runner.ts +0 -144
- package/runtime-src/stale-cleanup.ts +0 -133
- package/runtime-src/telemetry-attribution.ts +0 -120
- package/runtime-src/telemetry.ts +0 -253
- package/runtime-src/template-params.ts +0 -141
- package/runtime-src/transform/drift.ts +0 -60
- package/runtime-src/transform/index.ts +0 -277
- package/runtime-src/types/index.ts +0 -1
- package/runtime-src/types/skill.ts +0 -931
- package/runtime-src/vault/index.ts +0 -196
- package/runtime-src/verification/auth-gate.ts +0 -8
- package/runtime-src/verification/candidates.ts +0 -27
- package/runtime-src/verification/index.ts +0 -120
- package/runtime-src/verification/matrix.ts +0 -30
- package/runtime-src/version.ts +0 -148
- package/runtime-src/workflow/artifact.ts +0 -161
- package/runtime-src/workflow/compile.ts +0 -808
- package/runtime-src/workflow/publish.ts +0 -225
- package/runtime-src/workflow/runtime.ts +0 -213
|
@@ -1,1929 +0,0 @@
|
|
|
1
|
-
import * as kuri from "../kuri/client.js";
|
|
2
|
-
import { nanoid } from "nanoid";
|
|
3
|
-
import { getRegistrableDomain } from "../domain.js";
|
|
4
|
-
import { log } from "../logger.js";
|
|
5
|
-
import type { BrowserAccessConfig } from "../runtime/browser-access.js";
|
|
6
|
-
import { DEFAULT_BROWSER_ACCESS } from "../runtime/browser-access.js";
|
|
7
|
-
|
|
8
|
-
/**
|
|
9
|
-
* Check whether the current BrowserAccessConfig allows browser-based capture.
|
|
10
|
-
* Returns true when the default or fallback path is "unbrowse" or "direct"
|
|
11
|
-
* (i.e. not proxy-only), meaning we can launch a local browser via Kuri.
|
|
12
|
-
*/
|
|
13
|
-
export function isBrowserAccessAvailable(
|
|
14
|
-
config: BrowserAccessConfig = DEFAULT_BROWSER_ACCESS,
|
|
15
|
-
): boolean {
|
|
16
|
-
return config.default_path !== "proxy" || config.fallback_path !== "proxy";
|
|
17
|
-
}
|
|
18
|
-
|
|
19
|
-
// BUG-GC-012: Use a real Chrome UA — HeadlessChrome is actively blocked by Google and others.
|
|
20
|
-
const CHROME_UA = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36";
|
|
21
|
-
|
|
22
|
-
// Tab semaphore: max 3 concurrent capture tabs
|
|
23
|
-
const MAX_CONCURRENT_TABS = 3;
|
|
24
|
-
let activeTabs = 0;
|
|
25
|
-
const waitQueue: Array<() => void> = [];
|
|
26
|
-
|
|
27
|
-
// Active tab registry — tracked for graceful shutdown
|
|
28
|
-
const activeTabRegistry = new Set<string>();
|
|
29
|
-
|
|
30
|
-
// Tracks tabs where scriptInject has been registered (persistent across navigations)
|
|
31
|
-
const interceptorInjectedTabs = new Set<string>();
|
|
32
|
-
|
|
33
|
-
// Tracks tabs where CDP-level document-start injection has been registered
|
|
34
|
-
const cdpDocStartTabs = new Set<string>();
|
|
35
|
-
|
|
36
|
-
/**
|
|
37
|
-
* Register a script via Chrome's Page.addScriptToEvaluateOnNewDocument CDP method directly.
|
|
38
|
-
* This runs BEFORE any page JS on every navigation — critical for catching early fetch() calls.
|
|
39
|
-
* Falls back silently if CDP is unavailable.
|
|
40
|
-
*/
|
|
41
|
-
export async function registerDocumentStartScript(tabId: string, source: string): Promise<boolean> {
|
|
42
|
-
if (cdpDocStartTabs.has(tabId)) return true;
|
|
43
|
-
const cdpPort = kuri.getCdpPort();
|
|
44
|
-
if (!cdpPort) return false;
|
|
45
|
-
|
|
46
|
-
try {
|
|
47
|
-
// Get the WebSocket URL for this tab
|
|
48
|
-
const resp = await fetch(`http://127.0.0.1:${cdpPort}/json`);
|
|
49
|
-
const targets = await resp.json() as Array<{ id: string; webSocketDebuggerUrl?: string }>;
|
|
50
|
-
const target = targets.find((t) => t.id === tabId);
|
|
51
|
-
if (!target?.webSocketDebuggerUrl) return false;
|
|
52
|
-
|
|
53
|
-
// Connect via WebSocket and send CDP command
|
|
54
|
-
const ws = new WebSocket(target.webSocketDebuggerUrl);
|
|
55
|
-
const result = await new Promise<boolean>((resolve) => {
|
|
56
|
-
const timeout = setTimeout(() => { ws.close(); resolve(false); }, 3000);
|
|
57
|
-
ws.onopen = () => {
|
|
58
|
-
ws.send(JSON.stringify({
|
|
59
|
-
id: 1,
|
|
60
|
-
method: "Page.addScriptToEvaluateOnNewDocument",
|
|
61
|
-
params: { source },
|
|
62
|
-
}));
|
|
63
|
-
};
|
|
64
|
-
ws.onmessage = (event) => {
|
|
65
|
-
try {
|
|
66
|
-
const msg = JSON.parse(String(event.data));
|
|
67
|
-
if (msg.id === 1) {
|
|
68
|
-
clearTimeout(timeout);
|
|
69
|
-
ws.close();
|
|
70
|
-
resolve(!msg.error);
|
|
71
|
-
}
|
|
72
|
-
} catch { /* ignore parse errors */ }
|
|
73
|
-
};
|
|
74
|
-
ws.onerror = () => { clearTimeout(timeout); ws.close(); resolve(false); };
|
|
75
|
-
});
|
|
76
|
-
|
|
77
|
-
if (result) {
|
|
78
|
-
cdpDocStartTabs.add(tabId);
|
|
79
|
-
log("capture", `document-start interceptor registered via direct CDP for tab ${tabId}`);
|
|
80
|
-
}
|
|
81
|
-
return result;
|
|
82
|
-
} catch {
|
|
83
|
-
return false;
|
|
84
|
-
}
|
|
85
|
-
}
|
|
86
|
-
|
|
87
|
-
// Tracks captured request headers from CDP Network events
|
|
88
|
-
const cdpCapturedHeaders = new Map<string, Map<string, Record<string, string>>>();
|
|
89
|
-
|
|
90
|
-
/**
|
|
91
|
-
* Enable CDP Network.requestWillBeSent listener to capture full request headers
|
|
92
|
-
* including auth headers that HAR/interceptor miss. Stores headers indexed by URL.
|
|
93
|
-
*/
|
|
94
|
-
export async function enableNetworkHeaderCapture(tabId: string): Promise<void> {
|
|
95
|
-
const cdpPort = kuri.getCdpPort();
|
|
96
|
-
if (!cdpPort) return;
|
|
97
|
-
|
|
98
|
-
try {
|
|
99
|
-
const resp = await fetch(`http://127.0.0.1:${cdpPort}/json`);
|
|
100
|
-
const targets = await resp.json() as Array<{ id: string; webSocketDebuggerUrl?: string }>;
|
|
101
|
-
const target = targets.find((t) => t.id === tabId);
|
|
102
|
-
if (!target?.webSocketDebuggerUrl) return;
|
|
103
|
-
|
|
104
|
-
const headers = new Map<string, Record<string, string>>();
|
|
105
|
-
cdpCapturedHeaders.set(tabId, headers);
|
|
106
|
-
|
|
107
|
-
const ws = new WebSocket(target.webSocketDebuggerUrl);
|
|
108
|
-
ws.onopen = () => {
|
|
109
|
-
// Enable network domain (may already be enabled, that's OK)
|
|
110
|
-
ws.send(JSON.stringify({ id: 1, method: "Network.enable", params: {} }));
|
|
111
|
-
};
|
|
112
|
-
ws.onmessage = (event) => {
|
|
113
|
-
try {
|
|
114
|
-
const msg = JSON.parse(String(event.data));
|
|
115
|
-
if (msg.method === "Network.requestWillBeSent") {
|
|
116
|
-
const req = msg.params?.request;
|
|
117
|
-
if (req?.url && req?.headers) {
|
|
118
|
-
// Only capture headers for API-like URLs
|
|
119
|
-
if (/\/(api|graphql|v\d+)\b/i.test(req.url) || /ads-api|voyager/i.test(req.url)) {
|
|
120
|
-
headers.set(req.url, req.headers);
|
|
121
|
-
}
|
|
122
|
-
}
|
|
123
|
-
}
|
|
124
|
-
} catch { /* ignore */ }
|
|
125
|
-
};
|
|
126
|
-
// Keep ws alive — it will be cleaned up when tab closes
|
|
127
|
-
ws.onerror = () => { cdpCapturedHeaders.delete(tabId); };
|
|
128
|
-
|
|
129
|
-
log("capture", `CDP network header capture enabled for tab ${tabId}`);
|
|
130
|
-
} catch { /* best-effort */ }
|
|
131
|
-
}
|
|
132
|
-
|
|
133
|
-
/**
|
|
134
|
-
* Get captured request headers for a tab. Returns a map of URL → headers.
|
|
135
|
-
* These come from CDP Network.requestWillBeSent which includes auth headers
|
|
136
|
-
* that HAR and the JS interceptor miss.
|
|
137
|
-
*/
|
|
138
|
-
export function getCapturedNetworkHeaders(tabId: string): Map<string, Record<string, string>> {
|
|
139
|
-
return cdpCapturedHeaders.get(tabId) ?? new Map();
|
|
140
|
-
}
|
|
141
|
-
// Hard timeout per capture: 90s prevents stuck tabs from holding slots forever
|
|
142
|
-
const CAPTURE_TIMEOUT_MS = 90_000;
|
|
143
|
-
const CAPTURE_NAV_TIMEOUT_MS = 20_000;
|
|
144
|
-
|
|
145
|
-
/**
|
|
146
|
-
* Races `fn()` against an AbortSignal so that each CDP phase exits immediately
|
|
147
|
-
* when the overall capture timeout fires — instead of waiting for kuri's own
|
|
148
|
-
* 30s per-request timeout to expire (which caused 120s+ hangs, bug #113).
|
|
149
|
-
*/
|
|
150
|
-
export function withBrowserPhaseTimeout<T>(
|
|
151
|
-
signal: AbortSignal,
|
|
152
|
-
label: string,
|
|
153
|
-
fn: () => Promise<T>,
|
|
154
|
-
): Promise<T> {
|
|
155
|
-
if (signal.aborted) return Promise.reject(new Error(`capture phase '${label}' timed out`));
|
|
156
|
-
return new Promise<T>((resolve, reject) => {
|
|
157
|
-
const onAbort = () => reject(new Error(`capture phase '${label}' timed out`));
|
|
158
|
-
signal.addEventListener("abort", onAbort, { once: true });
|
|
159
|
-
fn().then(
|
|
160
|
-
(val) => { signal.removeEventListener("abort", onAbort); resolve(val); },
|
|
161
|
-
(err) => { signal.removeEventListener("abort", onAbort); reject(err); },
|
|
162
|
-
);
|
|
163
|
-
});
|
|
164
|
-
}
|
|
165
|
-
|
|
166
|
-
// Client hint headers to avoid headless detection
|
|
167
|
-
const CLIENT_HINT_HEADERS: Record<string, string> = {
|
|
168
|
-
"sec-ch-ua": '"Not_A Brand";v="8", "Chromium";v="131", "Google Chrome";v="131"',
|
|
169
|
-
"sec-ch-ua-mobile": "?0",
|
|
170
|
-
"sec-ch-ua-platform": '"macOS"',
|
|
171
|
-
};
|
|
172
|
-
|
|
173
|
-
async function resetTab(tabId: string): Promise<void> {
|
|
174
|
-
try {
|
|
175
|
-
await kuri.navigate(tabId, "about:blank");
|
|
176
|
-
} catch { /* best-effort */ }
|
|
177
|
-
}
|
|
178
|
-
|
|
179
|
-
async function openFreshCaptureTab(): Promise<string> {
|
|
180
|
-
const tabId = await kuri.newTab("about:blank");
|
|
181
|
-
if (!tabId) throw new Error("Failed to create a fresh browser tab");
|
|
182
|
-
return tabId;
|
|
183
|
-
}
|
|
184
|
-
|
|
185
|
-
type CaptureNavigationPage = {
|
|
186
|
-
goto(url: string, options: { waitUntil: "domcontentloaded"; timeout: number }): Promise<unknown>;
|
|
187
|
-
};
|
|
188
|
-
|
|
189
|
-
export async function navigatePageForCapture(page: CaptureNavigationPage, url: string): Promise<void> {
|
|
190
|
-
await page.goto(url, { waitUntil: "domcontentloaded", timeout: CAPTURE_NAV_TIMEOUT_MS });
|
|
191
|
-
}
|
|
192
|
-
|
|
193
|
-
async function acquireTabSlot(): Promise<void> {
|
|
194
|
-
if (activeTabs < MAX_CONCURRENT_TABS) {
|
|
195
|
-
activeTabs++;
|
|
196
|
-
return;
|
|
197
|
-
}
|
|
198
|
-
return new Promise<void>((resolve) => {
|
|
199
|
-
waitQueue.push(() => { activeTabs++; resolve(); });
|
|
200
|
-
});
|
|
201
|
-
}
|
|
202
|
-
|
|
203
|
-
function releaseTabSlot(tabId?: string): void {
|
|
204
|
-
if (tabId) { activeTabRegistry.delete(tabId); interceptorInjectedTabs.delete(tabId); }
|
|
205
|
-
activeTabs--;
|
|
206
|
-
const next = waitQueue.shift();
|
|
207
|
-
if (next) next();
|
|
208
|
-
}
|
|
209
|
-
|
|
210
|
-
/** Close all active tabs — called on server shutdown. */
|
|
211
|
-
export async function shutdownAllBrowsers(): Promise<void> {
|
|
212
|
-
await Promise.allSettled([...activeTabRegistry].map((t) => resetTab(t)));
|
|
213
|
-
activeTabRegistry.clear();
|
|
214
|
-
await kuri.stop();
|
|
215
|
-
}
|
|
216
|
-
|
|
217
|
-
export interface CapturedWsMessage {
|
|
218
|
-
url: string;
|
|
219
|
-
direction: "sent" | "received";
|
|
220
|
-
data: string;
|
|
221
|
-
timestamp: string;
|
|
222
|
-
}
|
|
223
|
-
|
|
224
|
-
export interface CaptureResult {
|
|
225
|
-
requests: RawRequest[];
|
|
226
|
-
har_lineage_id: string;
|
|
227
|
-
domain: string;
|
|
228
|
-
cookies?: Array<{ name: string; value: string; domain: string; path?: string; httpOnly?: boolean; secure?: boolean }>;
|
|
229
|
-
final_url: string;
|
|
230
|
-
ws_messages?: CapturedWsMessage[];
|
|
231
|
-
html?: string;
|
|
232
|
-
js_bundles?: Map<string, string>;
|
|
233
|
-
graph_session?: {
|
|
234
|
-
observed_operations: string[];
|
|
235
|
-
known_bindings: Record<string, unknown>;
|
|
236
|
-
suggested_next: string[];
|
|
237
|
-
};
|
|
238
|
-
}
|
|
239
|
-
|
|
240
|
-
export interface RawRequest {
|
|
241
|
-
url: string;
|
|
242
|
-
method: string;
|
|
243
|
-
request_headers: Record<string, string>;
|
|
244
|
-
request_body?: string;
|
|
245
|
-
response_status: number;
|
|
246
|
-
response_headers: Record<string, string>;
|
|
247
|
-
response_body?: string;
|
|
248
|
-
timestamp: string;
|
|
249
|
-
/** Query hook bridge: which action step triggered this request (#114) */
|
|
250
|
-
triggered_by_step?: number;
|
|
251
|
-
triggered_by_action?: string;
|
|
252
|
-
triggered_by_ref?: string;
|
|
253
|
-
}
|
|
254
|
-
|
|
255
|
-
export interface QueryHookEvent {
|
|
256
|
-
step_index: number;
|
|
257
|
-
action_type: string;
|
|
258
|
-
selector: string;
|
|
259
|
-
requests_triggered: Array<{ url: string; method: string }>;
|
|
260
|
-
timestamp: number;
|
|
261
|
-
}
|
|
262
|
-
|
|
263
|
-
interface ExtensionEntry {
|
|
264
|
-
url: string;
|
|
265
|
-
method: string;
|
|
266
|
-
type: string; // "xmlhttprequest", "fetch", "main_frame", etc.
|
|
267
|
-
statusCode?: number;
|
|
268
|
-
requestHeaders?: Array<{ name: string; value: string }>;
|
|
269
|
-
responseHeaders?: Array<{ name: string; value: string }>;
|
|
270
|
-
tabId?: number;
|
|
271
|
-
timestamp: number;
|
|
272
|
-
}
|
|
273
|
-
|
|
274
|
-
interface PerformanceResourceEntry {
|
|
275
|
-
name: string;
|
|
276
|
-
initiatorType?: string;
|
|
277
|
-
transferSize?: number;
|
|
278
|
-
}
|
|
279
|
-
|
|
280
|
-
export type CapturedCookie = {
|
|
281
|
-
name: string;
|
|
282
|
-
value: string;
|
|
283
|
-
domain: string;
|
|
284
|
-
path?: string;
|
|
285
|
-
httpOnly?: boolean;
|
|
286
|
-
secure?: boolean;
|
|
287
|
-
};
|
|
288
|
-
|
|
289
|
-
export function filterFirstPartySessionCookies(
|
|
290
|
-
cookies: CapturedCookie[],
|
|
291
|
-
...urls: Array<string | undefined>
|
|
292
|
-
): CapturedCookie[] {
|
|
293
|
-
const hosts = new Set<string>();
|
|
294
|
-
const domains = new Set<string>();
|
|
295
|
-
for (const rawUrl of urls) {
|
|
296
|
-
if (!rawUrl) continue;
|
|
297
|
-
try {
|
|
298
|
-
const host = new URL(rawUrl).hostname.toLowerCase();
|
|
299
|
-
hosts.add(host);
|
|
300
|
-
domains.add(getRegistrableDomain(host));
|
|
301
|
-
} catch {
|
|
302
|
-
// ignore bad urls
|
|
303
|
-
}
|
|
304
|
-
}
|
|
305
|
-
if (hosts.size === 0 && domains.size === 0) return cookies;
|
|
306
|
-
return cookies.filter((cookie) => {
|
|
307
|
-
const cookieDomain = cookie.domain.replace(/^\./, "").toLowerCase();
|
|
308
|
-
if (!cookieDomain) return false;
|
|
309
|
-
if (hosts.has(cookieDomain)) return true;
|
|
310
|
-
try {
|
|
311
|
-
return domains.has(getRegistrableDomain(cookieDomain));
|
|
312
|
-
} catch {
|
|
313
|
-
return false;
|
|
314
|
-
}
|
|
315
|
-
});
|
|
316
|
-
}
|
|
317
|
-
|
|
318
|
-
export function isBlockedAppShell(html?: string): boolean {
|
|
319
|
-
if (!html) return false;
|
|
320
|
-
return (
|
|
321
|
-
/JavaScript is not available\./i.test(html) ||
|
|
322
|
-
/switch to a supported browser/i.test(html) ||
|
|
323
|
-
/Something went wrong, but don.?t fret/i.test(html) ||
|
|
324
|
-
/class=["']errorContainer["']/i.test(html) ||
|
|
325
|
-
/#placeholder,\s*#react-root\s*\{\s*display:\s*none/i.test(html) ||
|
|
326
|
-
/Attention Required!\s*\|\s*Cloudflare/i.test(html) ||
|
|
327
|
-
/cf-error-details|cf\.errors\.css/i.test(html)
|
|
328
|
-
);
|
|
329
|
-
}
|
|
330
|
-
|
|
331
|
-
function shouldRetryEphemeralProfileError(error: unknown): boolean {
|
|
332
|
-
const message = error instanceof Error ? error.message : String(error ?? "");
|
|
333
|
-
return /persistentcontext|target page, context or browser has been closed|browser has been closed|page has been closed|fetch failed|econnrefused|connection refused/i.test(message);
|
|
334
|
-
}
|
|
335
|
-
|
|
336
|
-
/**
|
|
337
|
-
* Extract a route hint keyword from a URL path for intent-aware API waiting.
|
|
338
|
-
* e.g., "/i/bookmarks" → "bookmark", "/dashboard/analytics" → "analytic"
|
|
339
|
-
* Returns null if the path is too generic (root, single char, common SPA prefixes).
|
|
340
|
-
*/
|
|
341
|
-
function extractRouteHint(url: string): string | null {
|
|
342
|
-
try {
|
|
343
|
-
const pathname = new URL(url).pathname;
|
|
344
|
-
// Walk path segments from the end — the last meaningful segment is most specific
|
|
345
|
-
const segments = pathname.split("/").filter(Boolean);
|
|
346
|
-
// Skip generic SPA prefixes
|
|
347
|
-
const GENERIC = /^(i|app|dashboard|page|view|en|es|fr|de|#|_|index|home|main|me|@me|users?|channels?|guilds?|servers?|messages?|threads?|conversations?)$/i;
|
|
348
|
-
for (let i = segments.length - 1; i >= 0; i--) {
|
|
349
|
-
const seg = segments[i];
|
|
350
|
-
if (seg.length <= 2 || GENERIC.test(seg) || /^\d+$/.test(seg) || /^\{.+\}$/.test(seg) || /^[0-9a-f-]{8,}$/i.test(seg)) continue;
|
|
351
|
-
// Return lowercased stem (strip trailing 's' for simple plural handling)
|
|
352
|
-
return seg.toLowerCase().replace(/s$/, "");
|
|
353
|
-
}
|
|
354
|
-
} catch { /* bad URL */ }
|
|
355
|
-
return null;
|
|
356
|
-
}
|
|
357
|
-
|
|
358
|
-
function deriveIntentHints(captureUrl?: string, intent?: string): string[] {
|
|
359
|
-
const derivedHints = new Set<string>();
|
|
360
|
-
if (captureUrl) {
|
|
361
|
-
const routeHint = extractRouteHint(captureUrl);
|
|
362
|
-
if (routeHint) derivedHints.add(routeHint);
|
|
363
|
-
}
|
|
364
|
-
const lowerIntent = intent?.toLowerCase() ?? "";
|
|
365
|
-
if (/\b(person|people|profile|profiles|user|users|member|members)\b/.test(lowerIntent)) {
|
|
366
|
-
derivedHints.add("profile");
|
|
367
|
-
derivedHints.add("users");
|
|
368
|
-
derivedHints.add("userby");
|
|
369
|
-
}
|
|
370
|
-
if (/\b(company|companies|organization|organisations|business)\b/.test(lowerIntent)) {
|
|
371
|
-
derivedHints.add("company");
|
|
372
|
-
derivedHints.add("organization");
|
|
373
|
-
derivedHints.add("about");
|
|
374
|
-
}
|
|
375
|
-
if (/\b(post|posts|tweet|tweets|status|statuses)\b/.test(lowerIntent)) {
|
|
376
|
-
derivedHints.add("tweet");
|
|
377
|
-
derivedHints.add("timeline");
|
|
378
|
-
derivedHints.add("status");
|
|
379
|
-
}
|
|
380
|
-
if (/\b(guild|guilds|server|servers)\b/.test(lowerIntent)) {
|
|
381
|
-
derivedHints.add("guild");
|
|
382
|
-
derivedHints.add("guilds");
|
|
383
|
-
}
|
|
384
|
-
if (/\b(channel|channels)\b/.test(lowerIntent)) {
|
|
385
|
-
derivedHints.add("channel");
|
|
386
|
-
derivedHints.add("channels");
|
|
387
|
-
}
|
|
388
|
-
if (/\b(message|messages|dm|dms|chat|thread|threads|conversation|conversations)\b/.test(lowerIntent)) {
|
|
389
|
-
derivedHints.add("message");
|
|
390
|
-
derivedHints.add("messages");
|
|
391
|
-
derivedHints.add("thread");
|
|
392
|
-
derivedHints.add("threads");
|
|
393
|
-
}
|
|
394
|
-
if (/\b(topic|topics|trend|trending|hashtag|hashtags)\b/.test(lowerIntent)) {
|
|
395
|
-
derivedHints.add("trend");
|
|
396
|
-
derivedHints.add("explore");
|
|
397
|
-
derivedHints.add("topic");
|
|
398
|
-
}
|
|
399
|
-
return [...derivedHints];
|
|
400
|
-
}
|
|
401
|
-
|
|
402
|
-
function hasCapturedHint(responseUrls: Iterable<string>, hint: string): boolean {
|
|
403
|
-
const lowerHint = hint.toLowerCase();
|
|
404
|
-
for (const url of responseUrls) {
|
|
405
|
-
if (url.toLowerCase().includes(lowerHint)) return true;
|
|
406
|
-
}
|
|
407
|
-
return false;
|
|
408
|
-
}
|
|
409
|
-
|
|
410
|
-
async function maybeProbeIntentApis(
|
|
411
|
-
tabId: string,
|
|
412
|
-
captureUrl: string | undefined,
|
|
413
|
-
intent: string | undefined,
|
|
414
|
-
responseBodies: Map<string, string> | undefined,
|
|
415
|
-
): Promise<void> {
|
|
416
|
-
if (!captureUrl || !responseBodies) return;
|
|
417
|
-
const lowerIntent = intent?.toLowerCase() ?? "";
|
|
418
|
-
let hostname = "";
|
|
419
|
-
try {
|
|
420
|
-
hostname = new URL(captureUrl).hostname.toLowerCase();
|
|
421
|
-
} catch {
|
|
422
|
-
return;
|
|
423
|
-
}
|
|
424
|
-
|
|
425
|
-
if (/discord\.com$/.test(hostname) && /\b(guild|guilds|server|servers)\b/.test(lowerIntent)) {
|
|
426
|
-
if (hasCapturedHint(responseBodies.keys(), "/guild")) return;
|
|
427
|
-
const probes = [
|
|
428
|
-
{ label: "User affinities guilds", path: "/api/v9/users/@me/affinities/guilds" },
|
|
429
|
-
{ label: "User guilds", path: "/api/v9/users/@me/guilds?with_counts=true" },
|
|
430
|
-
];
|
|
431
|
-
try {
|
|
432
|
-
for (const probe of probes) {
|
|
433
|
-
if (hasCapturedHint(responseBodies.keys(), "/guild")) break;
|
|
434
|
-
log("spa", `probing ${probe.label}: GET https://discord.com${probe.path}`);
|
|
435
|
-
await kuri.evaluate(tabId, `(async function() {
|
|
436
|
-
try {
|
|
437
|
-
var response = await fetch(${JSON.stringify(probe.path)}, { credentials: "include" });
|
|
438
|
-
await response.text();
|
|
439
|
-
} catch(e) { /* best-effort probe */ }
|
|
440
|
-
})()`);
|
|
441
|
-
await new Promise((r) => setTimeout(r, 1200));
|
|
442
|
-
}
|
|
443
|
-
} catch {
|
|
444
|
-
// non-fatal
|
|
445
|
-
}
|
|
446
|
-
}
|
|
447
|
-
}
|
|
448
|
-
|
|
449
|
-
const CAPTURE_RESPONSE_NOISE = /user_flow|datasavermode|ces\/p2|intercom|badge_count|settings\.json|paymentfailure|saved_searches|launcher_settings|conversations|\/ping\b|verifiedorg|xchatdmsettings|scheduledpromotions|storytopic|sidebaruserrecommendations|subscriptions|live_pipeline|fleetline|authorizetoken|logintwittertoken/i;
|
|
450
|
-
|
|
451
|
-
export function hasUsefulCapturedResponses(
|
|
452
|
-
responseUrls: Iterable<string>,
|
|
453
|
-
captureUrl?: string,
|
|
454
|
-
intent?: string,
|
|
455
|
-
): boolean {
|
|
456
|
-
const usefulUrls = [...responseUrls].filter((url) => !CAPTURE_RESPONSE_NOISE.test(url));
|
|
457
|
-
if (usefulUrls.length === 0) return false;
|
|
458
|
-
const hints = deriveIntentHints(captureUrl, intent);
|
|
459
|
-
if (hints.length === 0) return usefulUrls.length > 0;
|
|
460
|
-
return usefulUrls.some((url) => {
|
|
461
|
-
const lower = url.toLowerCase();
|
|
462
|
-
return hints.some((hint) => lower.includes(hint));
|
|
463
|
-
});
|
|
464
|
-
}
|
|
465
|
-
|
|
466
|
-
/**
|
|
467
|
-
* Inject a fetch/XHR interceptor into the page to capture request/response data.
|
|
468
|
-
* Returns captured entries via __unbrowse_intercepted global.
|
|
469
|
-
*/
|
|
470
|
-
export const INTERCEPTOR_SCRIPT = `(function() {
|
|
471
|
-
if (window.__unbrowse_interceptor_installed) return;
|
|
472
|
-
window.__unbrowse_interceptor_installed = true;
|
|
473
|
-
window.__unbrowse_intercepted = [];
|
|
474
|
-
var MAX_BODY = 2 * 1024 * 1024;
|
|
475
|
-
var MAX_JS_BODY = 2 * 1024 * 1024;
|
|
476
|
-
var MAX_ENTRIES = 500;
|
|
477
|
-
|
|
478
|
-
// Intercept fetch
|
|
479
|
-
var origFetch = window.fetch;
|
|
480
|
-
window.fetch = function() {
|
|
481
|
-
var args = arguments;
|
|
482
|
-
var url = typeof args[0] === 'string' ? args[0] : (args[0] && args[0].url ? args[0].url : '');
|
|
483
|
-
var opts = args[1] || {};
|
|
484
|
-
var method = (opts.method || 'GET').toUpperCase();
|
|
485
|
-
var reqBody = opts.body ? String(opts.body).substring(0, MAX_BODY) : undefined;
|
|
486
|
-
var reqHeaders = {};
|
|
487
|
-
// Extract headers from Request object (first arg)
|
|
488
|
-
if (args[0] && typeof args[0] === 'object' && args[0].headers && typeof args[0].headers.forEach === 'function') {
|
|
489
|
-
args[0].headers.forEach(function(v, k) { reqHeaders[k] = v; });
|
|
490
|
-
}
|
|
491
|
-
// Override/merge with explicit opts.headers (second arg)
|
|
492
|
-
if (opts.headers) {
|
|
493
|
-
if (typeof opts.headers.forEach === 'function') {
|
|
494
|
-
opts.headers.forEach(function(v, k) { reqHeaders[k] = v; });
|
|
495
|
-
} else {
|
|
496
|
-
Object.keys(opts.headers).forEach(function(k) { reqHeaders[k] = opts.headers[k]; });
|
|
497
|
-
}
|
|
498
|
-
}
|
|
499
|
-
return origFetch.apply(this, args).then(function(response) {
|
|
500
|
-
if (window.__unbrowse_intercepted.length >= MAX_ENTRIES) return response;
|
|
501
|
-
var ct = response.headers.get('content-type') || '';
|
|
502
|
-
var isJs = ct.indexOf('javascript') !== -1 || /\\.js(\\?|$)/.test(url);
|
|
503
|
-
var isData = ct.indexOf('json') !== -1 || ct.indexOf('application/x-protobuf') !== -1 ||
|
|
504
|
-
ct.indexOf('text/plain') !== -1 ||
|
|
505
|
-
url.indexOf('batchexecute') !== -1 || url.indexOf('/api/') !== -1 ||
|
|
506
|
-
url.indexOf('graphql') !== -1 || url.indexOf('voyager') !== -1;
|
|
507
|
-
if (!isJs && !isData) return response;
|
|
508
|
-
if (/\\.(css|woff2?|png|jpg|svg|ico)(\\?|$)/.test(url)) return response;
|
|
509
|
-
var clone = response.clone();
|
|
510
|
-
clone.text().then(function(body) {
|
|
511
|
-
var limit = isJs ? MAX_JS_BODY : MAX_BODY;
|
|
512
|
-
if (body.length > limit) return;
|
|
513
|
-
var respHeaders = {};
|
|
514
|
-
response.headers.forEach(function(v, k) { respHeaders[k] = v; });
|
|
515
|
-
window.__unbrowse_intercepted.push({
|
|
516
|
-
url: url,
|
|
517
|
-
method: method,
|
|
518
|
-
request_headers: reqHeaders,
|
|
519
|
-
request_body: reqBody,
|
|
520
|
-
response_status: response.status,
|
|
521
|
-
response_headers: respHeaders,
|
|
522
|
-
response_body: body,
|
|
523
|
-
content_type: ct,
|
|
524
|
-
is_js: isJs,
|
|
525
|
-
timestamp: new Date().toISOString()
|
|
526
|
-
});
|
|
527
|
-
}).catch(function() {});
|
|
528
|
-
return response;
|
|
529
|
-
}).catch(function(err) { throw err; });
|
|
530
|
-
};
|
|
531
|
-
|
|
532
|
-
// Intercept XMLHttpRequest
|
|
533
|
-
var origOpen = XMLHttpRequest.prototype.open;
|
|
534
|
-
var origSend = XMLHttpRequest.prototype.send;
|
|
535
|
-
XMLHttpRequest.prototype.open = function(method, url) {
|
|
536
|
-
this.__unbrowse_method = method;
|
|
537
|
-
this.__unbrowse_url = url;
|
|
538
|
-
this.__unbrowse_reqHeaders = {};
|
|
539
|
-
var origSetHeader = this.setRequestHeader.bind(this);
|
|
540
|
-
this.setRequestHeader = function(k, v) {
|
|
541
|
-
this.__unbrowse_reqHeaders[k] = v;
|
|
542
|
-
origSetHeader(k, v);
|
|
543
|
-
}.bind(this);
|
|
544
|
-
return origOpen.apply(this, arguments);
|
|
545
|
-
};
|
|
546
|
-
XMLHttpRequest.prototype.send = function(body) {
|
|
547
|
-
var xhr = this;
|
|
548
|
-
xhr.addEventListener('load', function() {
|
|
549
|
-
if (window.__unbrowse_intercepted.length >= MAX_ENTRIES) return;
|
|
550
|
-
var ct = xhr.getResponseHeader('content-type') || '';
|
|
551
|
-
var url = xhr.__unbrowse_url || '';
|
|
552
|
-
var isJs = ct.indexOf('javascript') !== -1 || /\\.js(\\?|$)/.test(url);
|
|
553
|
-
var isData = ct.indexOf('json') !== -1 || ct.indexOf('application/x-protobuf') !== -1 ||
|
|
554
|
-
ct.indexOf('text/plain') !== -1 ||
|
|
555
|
-
url.indexOf('batchexecute') !== -1 || url.indexOf('/api/') !== -1 ||
|
|
556
|
-
url.indexOf('graphql') !== -1 || url.indexOf('voyager') !== -1;
|
|
557
|
-
if (!isJs && !isData) return;
|
|
558
|
-
if (/\\.(css|woff2?|png|jpg|svg|ico)(\\?|$)/.test(url)) return;
|
|
559
|
-
var respBody = xhr.responseText || '';
|
|
560
|
-
var limit = isJs ? MAX_JS_BODY : MAX_BODY;
|
|
561
|
-
if (respBody.length > limit) return;
|
|
562
|
-
window.__unbrowse_intercepted.push({
|
|
563
|
-
url: url,
|
|
564
|
-
method: (xhr.__unbrowse_method || 'GET').toUpperCase(),
|
|
565
|
-
request_headers: xhr.__unbrowse_reqHeaders || {},
|
|
566
|
-
request_body: body ? String(body).substring(0, MAX_BODY) : undefined,
|
|
567
|
-
response_status: xhr.status,
|
|
568
|
-
response_headers: {},
|
|
569
|
-
response_body: respBody,
|
|
570
|
-
content_type: ct,
|
|
571
|
-
is_js: isJs,
|
|
572
|
-
timestamp: new Date().toISOString()
|
|
573
|
-
});
|
|
574
|
-
});
|
|
575
|
-
return origSend.apply(this, arguments);
|
|
576
|
-
};
|
|
577
|
-
})()`;
|
|
578
|
-
|
|
579
|
-
/**
|
|
580
|
-
* Collect intercepted requests from the page.
|
|
581
|
-
*/
|
|
582
|
-
export async function collectInterceptedRequests(tabId: string): Promise<Array<{
|
|
583
|
-
url: string;
|
|
584
|
-
method: string;
|
|
585
|
-
request_headers: Record<string, string>;
|
|
586
|
-
request_body?: string;
|
|
587
|
-
response_status: number;
|
|
588
|
-
response_headers: Record<string, string>;
|
|
589
|
-
response_body?: string;
|
|
590
|
-
content_type?: string;
|
|
591
|
-
is_js?: boolean;
|
|
592
|
-
timestamp: string;
|
|
593
|
-
}>> {
|
|
594
|
-
try {
|
|
595
|
-
const result = await kuri.evaluate(tabId, "JSON.stringify(window.__unbrowse_intercepted || [])");
|
|
596
|
-
if (typeof result === "string" && result.startsWith("[")) {
|
|
597
|
-
return JSON.parse(result);
|
|
598
|
-
}
|
|
599
|
-
} catch { /* non-fatal */ }
|
|
600
|
-
return [];
|
|
601
|
-
}
|
|
602
|
-
|
|
603
|
-
/**
|
|
604
|
-
* Inject the interceptor script in chunks to work around kuri's ~1KB evaluate limit.
|
|
605
|
-
* Falls back to scriptInject for persistent injection on new navigations.
|
|
606
|
-
*/
|
|
607
|
-
export async function injectInterceptor(tabId: string): Promise<void> {
|
|
608
|
-
// Split into setup (globals + fetch) and XHR parts
|
|
609
|
-
const SETUP = `(function(){if(window.__unbrowse_interceptor_installed)return;window.__unbrowse_interceptor_installed=true;window.__unbrowse_intercepted=[];window.__UB_MAX=2*1024*1024;window.__UB_MAX_JS=2*1024*1024;window.__UB_MAX_N=500;})()`;
|
|
610
|
-
|
|
611
|
-
const FETCH_PATCH = `(function(){if(!window.__unbrowse_interceptor_installed)return;var M=window.__UB_MAX,MJ=window.__UB_MAX_JS,MN=window.__UB_MAX_N;var oF=window.fetch;window.fetch=function(){var a=arguments,u=typeof a[0]==='string'?a[0]:(a[0]&&a[0].url?a[0].url:''),o=a[1]||{},m=(o.method||'GET').toUpperCase(),rb=o.body?String(o.body).substring(0,M):void 0,rh={};if(o.headers){if(typeof o.headers.forEach==='function')o.headers.forEach(function(v,k){rh[k]=v});else Object.keys(o.headers).forEach(function(k){rh[k]=o.headers[k]})}return oF.apply(this,a).then(function(r){if(window.__unbrowse_intercepted.length>=MN)return r;var ct=r.headers.get('content-type')||'';var isJ=ct.indexOf('javascript')!==-1||/\\.js(\\?|$)/.test(u);var isD=ct.indexOf('json')!==-1||ct.indexOf('x-protobuf')!==-1||ct.indexOf('text/plain')!==-1||u.indexOf('/api/')!==-1||u.indexOf('graphql')!==-1||u.indexOf('voyager')!==-1;if(!isJ&&!isD)return r;if(/\\.(css|woff2?|png|jpg|svg|ico)(\\?|$)/.test(u))return r;var c=r.clone();c.text().then(function(b){var lim=isJ?MJ:M;if(b.length>lim)return;var rr={};r.headers.forEach(function(v,k){rr[k]=v});window.__unbrowse_intercepted.push({url:u,method:m,request_headers:rh,request_body:rb,response_status:r.status,response_headers:rr,response_body:b,content_type:ct,is_js:isJ,timestamp:new Date().toISOString()})}).catch(function(){});return r}).catch(function(e){throw e})}})()`;
|
|
612
|
-
|
|
613
|
-
const XHR_PATCH = `(function(){if(!window.__unbrowse_interceptor_installed)return;var M=window.__UB_MAX,MJ=window.__UB_MAX_JS,MN=window.__UB_MAX_N;var oO=XMLHttpRequest.prototype.open,oS=XMLHttpRequest.prototype.send;XMLHttpRequest.prototype.open=function(m,u){this.__ub_m=m;this.__ub_u=u;this.__ub_h={};var oSH=this.setRequestHeader.bind(this);this.setRequestHeader=function(k,v){this.__ub_h[k]=v;oSH(k,v)}.bind(this);return oO.apply(this,arguments)};XMLHttpRequest.prototype.send=function(b){var x=this;x.addEventListener('load',function(){if(window.__unbrowse_intercepted.length>=MN)return;var ct=x.getResponseHeader('content-type')||'',u=x.__ub_u||'';var isJ=ct.indexOf('javascript')!==-1||/\\.js(\\?|$)/.test(u);var isD=ct.indexOf('json')!==-1||ct.indexOf('x-protobuf')!==-1||ct.indexOf('text/plain')!==-1||u.indexOf('/api/')!==-1||u.indexOf('graphql')!==-1||u.indexOf('voyager')!==-1;if(!isJ&&!isD)return;if(/\\.(css|woff2?|png|jpg|svg|ico)(\\?|$)/.test(u))return;var rb=x.responseText||'';var lim=isJ?MJ:M;if(rb.length>lim)return;window.__unbrowse_intercepted.push({url:u,method:(x.__ub_m||'GET').toUpperCase(),request_headers:x.__ub_h||{},request_body:b?String(b).substring(0,M):void 0,response_status:x.status,response_headers:{},response_body:rb,content_type:ct,is_js:isJ,timestamp:new Date().toISOString()})});return oS.apply(this,arguments)}})()`;
|
|
614
|
-
|
|
615
|
-
// Inject in 3 small chunks
|
|
616
|
-
for (const chunk of [SETUP, FETCH_PATCH, XHR_PATCH]) {
|
|
617
|
-
await kuri.evaluate(tabId, chunk).catch(() => {});
|
|
618
|
-
}
|
|
619
|
-
}
|
|
620
|
-
|
|
621
|
-
|
|
622
|
-
/**
|
|
623
|
-
* Merge three passive capture data sources into a unified RawRequest list.
|
|
624
|
-
* Priority: JS interceptor (has bodies) > HAR entries > extension observer > responseBodies-only.
|
|
625
|
-
* Deduplicates by URL, keeps the highest-priority version.
|
|
626
|
-
*/
|
|
627
|
-
export function mergePassiveCaptureData(
|
|
628
|
-
intercepted: Array<{ url: string; method: string; response_body?: string; response_status: number; request_headers: Record<string, string>; response_headers: Record<string, string>; request_body?: string; content_type?: string; is_js?: boolean; timestamp: string }>,
|
|
629
|
-
harEntries: kuri.KuriHarEntry[],
|
|
630
|
-
extensionEntries: ExtensionEntry[],
|
|
631
|
-
responseBodies: Map<string, string>,
|
|
632
|
-
performanceUrls: string[] = [],
|
|
633
|
-
): RawRequest[] {
|
|
634
|
-
const seen = new Map<string, RawRequest>();
|
|
635
|
-
|
|
636
|
-
// Priority 1: JS-intercepted entries (have response bodies)
|
|
637
|
-
for (const entry of intercepted) {
|
|
638
|
-
if (entry.is_js) continue; // skip JS bundles
|
|
639
|
-
seen.set(entry.url, {
|
|
640
|
-
url: entry.url,
|
|
641
|
-
method: entry.method,
|
|
642
|
-
request_headers: entry.request_headers,
|
|
643
|
-
request_body: entry.request_body,
|
|
644
|
-
response_status: entry.response_status,
|
|
645
|
-
response_headers: entry.response_headers,
|
|
646
|
-
response_body: entry.response_body,
|
|
647
|
-
timestamp: entry.timestamp,
|
|
648
|
-
});
|
|
649
|
-
}
|
|
650
|
-
|
|
651
|
-
// Priority 2: HAR entries (supplement with responseBodies map)
|
|
652
|
-
// Skip OPTIONS (CORS preflight) — they pollute method attribution
|
|
653
|
-
for (const entry of harEntries) {
|
|
654
|
-
const url = entry.request?.url;
|
|
655
|
-
if (!url || seen.has(url)) continue;
|
|
656
|
-
if (entry.request.method === "OPTIONS") continue;
|
|
657
|
-
const reqHeaders: Record<string, string> = {};
|
|
658
|
-
for (const h of entry.request.headers ?? []) reqHeaders[h.name] = h.value;
|
|
659
|
-
const respHeaders: Record<string, string> = {};
|
|
660
|
-
for (const h of entry.response.headers ?? []) respHeaders[h.name] = h.value;
|
|
661
|
-
seen.set(url, {
|
|
662
|
-
url,
|
|
663
|
-
method: entry.request.method,
|
|
664
|
-
request_headers: reqHeaders,
|
|
665
|
-
request_body: entry.request.postData?.text,
|
|
666
|
-
response_status: entry.response.status,
|
|
667
|
-
response_headers: respHeaders,
|
|
668
|
-
response_body: responseBodies.get(url) ?? entry.response.content?.text,
|
|
669
|
-
timestamp: entry.startedDateTime,
|
|
670
|
-
});
|
|
671
|
-
}
|
|
672
|
-
|
|
673
|
-
// Priority 3: Extension entries (chrome.webRequest — has full auth headers HAR strips)
|
|
674
|
-
for (const entry of extensionEntries) {
|
|
675
|
-
const reqHeaders: Record<string, string> = {};
|
|
676
|
-
for (const h of entry.requestHeaders ?? []) reqHeaders[h.name] = h.value;
|
|
677
|
-
const respHeaders: Record<string, string> = {};
|
|
678
|
-
for (const h of entry.responseHeaders ?? []) respHeaders[h.name] = h.value;
|
|
679
|
-
|
|
680
|
-
const existing = seen.get(entry.url);
|
|
681
|
-
if (existing) {
|
|
682
|
-
// Merge auth headers from extension into existing entry (HAR strips them)
|
|
683
|
-
for (const [k, v] of Object.entries(reqHeaders)) {
|
|
684
|
-
if (!existing.request_headers[k]) existing.request_headers[k] = v;
|
|
685
|
-
}
|
|
686
|
-
continue;
|
|
687
|
-
}
|
|
688
|
-
seen.set(entry.url, {
|
|
689
|
-
url: entry.url,
|
|
690
|
-
method: entry.method,
|
|
691
|
-
request_headers: reqHeaders,
|
|
692
|
-
response_status: entry.statusCode ?? 0,
|
|
693
|
-
response_headers: respHeaders,
|
|
694
|
-
response_body: responseBodies.get(entry.url),
|
|
695
|
-
timestamp: new Date(entry.timestamp).toISOString(),
|
|
696
|
-
});
|
|
697
|
-
}
|
|
698
|
-
|
|
699
|
-
// Priority 4: responseBodies-only entries (from Performance API replay / HAR replay)
|
|
700
|
-
// These URLs have bodies but weren't in HAR, interceptor, or extension data
|
|
701
|
-
for (const perfUrl of performanceUrls) {
|
|
702
|
-
if (seen.has(perfUrl) || responseBodies.has(perfUrl)) continue;
|
|
703
|
-
seen.set(perfUrl, {
|
|
704
|
-
url: perfUrl,
|
|
705
|
-
method: "GET",
|
|
706
|
-
request_headers: {},
|
|
707
|
-
response_status: 200,
|
|
708
|
-
response_headers: {},
|
|
709
|
-
timestamp: new Date().toISOString(),
|
|
710
|
-
});
|
|
711
|
-
}
|
|
712
|
-
|
|
713
|
-
// Priority 5: responseBodies-only entries (from Performance API replay / HAR replay)
|
|
714
|
-
// These URLs have bodies but weren't in HAR, interceptor, extension, or synthetic perf data
|
|
715
|
-
for (const [bodyUrl, body] of responseBodies) {
|
|
716
|
-
if (seen.has(bodyUrl)) continue;
|
|
717
|
-
seen.set(bodyUrl, {
|
|
718
|
-
url: bodyUrl,
|
|
719
|
-
method: "GET",
|
|
720
|
-
request_headers: {},
|
|
721
|
-
response_status: 200,
|
|
722
|
-
response_headers: {},
|
|
723
|
-
response_body: body,
|
|
724
|
-
timestamp: new Date().toISOString(),
|
|
725
|
-
});
|
|
726
|
-
}
|
|
727
|
-
|
|
728
|
-
return Array.from(seen.values());
|
|
729
|
-
}
|
|
730
|
-
|
|
731
|
-
function appendInterceptedResponseBodies(
|
|
732
|
-
entries: Array<{ url: string; response_body?: string; is_js?: boolean }>,
|
|
733
|
-
responseBodies: Map<string, string>,
|
|
734
|
-
): void {
|
|
735
|
-
for (const entry of entries) {
|
|
736
|
-
if (!entry.response_body || entry.is_js) continue;
|
|
737
|
-
responseBodies.set(entry.url, entry.response_body);
|
|
738
|
-
}
|
|
739
|
-
}
|
|
740
|
-
|
|
741
|
-
function normalizeCapturedUrl(url: string, baseUrl?: string): string {
|
|
742
|
-
if (!url) return url;
|
|
743
|
-
try {
|
|
744
|
-
return new URL(url).toString();
|
|
745
|
-
} catch {
|
|
746
|
-
if (!baseUrl) return url;
|
|
747
|
-
try {
|
|
748
|
-
return new URL(url, baseUrl).toString();
|
|
749
|
-
} catch {
|
|
750
|
-
return url;
|
|
751
|
-
}
|
|
752
|
-
}
|
|
753
|
-
}
|
|
754
|
-
|
|
755
|
-
const REPLAY_SKIP = /\.(js|css|woff2?|png|jpe?g|gif|svg|ico|webp|avif|map|ttf)(\?|$)/i;
|
|
756
|
-
const HAR_REPLAY_CT = /application\/json|text\/plain|\+json/i;
|
|
757
|
-
|
|
758
|
-
function isSameOriginUrl(url: string, captureUrl?: string): boolean {
|
|
759
|
-
if (!captureUrl) return false;
|
|
760
|
-
try {
|
|
761
|
-
return new URL(url).origin === new URL(captureUrl).origin;
|
|
762
|
-
} catch {
|
|
763
|
-
return false;
|
|
764
|
-
}
|
|
765
|
-
}
|
|
766
|
-
|
|
767
|
-
export function isReplayableApiUrl(url: string): boolean {
|
|
768
|
-
const normalized = normalizeCapturedUrl(url).toLowerCase();
|
|
769
|
-
if (!normalized || REPLAY_SKIP.test(normalized)) return false;
|
|
770
|
-
if (/\/manifest\.json([?#]|$)/.test(normalized)) return false;
|
|
771
|
-
if (/cloudflareinsights|cdn-cgi\/rum|sentry\.io|piwik|analytics|disqus|doubleclick|googletagmanager|google-analytics/.test(normalized)) return false;
|
|
772
|
-
if (/\/api\//.test(normalized)) return true;
|
|
773
|
-
if (/graphql|voyager|_search/.test(normalized)) return true;
|
|
774
|
-
if (/\/v\d+(?:[./]|\/)/.test(normalized)) return true;
|
|
775
|
-
try {
|
|
776
|
-
const parsed = new URL(normalized);
|
|
777
|
-
if (/^api\./.test(parsed.hostname) && parsed.pathname.endsWith(".json")) return true;
|
|
778
|
-
} catch {
|
|
779
|
-
// ignore bad URL
|
|
780
|
-
}
|
|
781
|
-
if (/(\?|&)format=json(?:&|$)/.test(normalized) || /(\?|&)output=json(?:&|$)/.test(normalized)) return true;
|
|
782
|
-
return false;
|
|
783
|
-
}
|
|
784
|
-
|
|
785
|
-
export function selectPerformanceReplayCandidates(
|
|
786
|
-
entries: PerformanceResourceEntry[],
|
|
787
|
-
params: {
|
|
788
|
-
captureUrl?: string;
|
|
789
|
-
intent?: string;
|
|
790
|
-
knownUrls?: Iterable<string>;
|
|
791
|
-
limit?: number;
|
|
792
|
-
},
|
|
793
|
-
): string[] {
|
|
794
|
-
const knownUrls = new Set(params.knownUrls ?? []);
|
|
795
|
-
const wantedHints = new Set(deriveIntentHints(params.captureUrl, params.intent));
|
|
796
|
-
const scored = new Map<string, number>();
|
|
797
|
-
|
|
798
|
-
for (const entry of entries) {
|
|
799
|
-
const perfUrl = normalizeCapturedUrl(entry.name, params.captureUrl);
|
|
800
|
-
if (!perfUrl || knownUrls.has(perfUrl) || REPLAY_SKIP.test(perfUrl)) continue;
|
|
801
|
-
|
|
802
|
-
const lower = perfUrl.toLowerCase();
|
|
803
|
-
const initiator = (entry.initiatorType ?? "").toLowerCase();
|
|
804
|
-
const apiLike = isReplayableApiUrl(perfUrl);
|
|
805
|
-
const hintMatch = [...wantedHints].some((hint) => lower.includes(hint));
|
|
806
|
-
const initiatorEligible =
|
|
807
|
-
initiator === "fetch" ||
|
|
808
|
-
initiator === "xmlhttprequest" ||
|
|
809
|
-
(initiator === "link" && apiLike);
|
|
810
|
-
if (!initiatorEligible && !apiLike) continue;
|
|
811
|
-
|
|
812
|
-
let score = 0;
|
|
813
|
-
if (hintMatch) score += 10;
|
|
814
|
-
if (initiator === "fetch" || initiator === "xmlhttprequest") score += 5;
|
|
815
|
-
if (initiator === "link") score += 2;
|
|
816
|
-
if (apiLike) score += 6;
|
|
817
|
-
if (isSameOriginUrl(perfUrl, params.captureUrl)) score += 2;
|
|
818
|
-
if (/\.json([?#]|$)/.test(lower)) score += 2;
|
|
819
|
-
if (/\/v\d+(?:[./]|\/)/.test(lower)) score += 2;
|
|
820
|
-
if (score <= 0) continue;
|
|
821
|
-
|
|
822
|
-
scored.set(perfUrl, Math.max(scored.get(perfUrl) ?? 0, score));
|
|
823
|
-
}
|
|
824
|
-
|
|
825
|
-
return [...scored.entries()]
|
|
826
|
-
.sort((a, b) => b[1] - a[1] || a[0].localeCompare(b[0]))
|
|
827
|
-
.slice(0, params.limit ?? 30)
|
|
828
|
-
.map(([url]) => url);
|
|
829
|
-
}
|
|
830
|
-
|
|
831
|
-
async function collectPerformanceResourceEntries(tabId: string): Promise<PerformanceResourceEntry[]> {
|
|
832
|
-
const perfRaw = await kuri.evaluate(tabId, `JSON.stringify(
|
|
833
|
-
performance.getEntriesByType('resource')
|
|
834
|
-
.map(function(e) { return { name: e.name, initiatorType: e.initiatorType, transferSize: e.transferSize || 0 }; })
|
|
835
|
-
)`);
|
|
836
|
-
if (typeof perfRaw !== "string" || !perfRaw.startsWith("[")) return [];
|
|
837
|
-
const perfEntries: PerformanceResourceEntry[] = JSON.parse(perfRaw);
|
|
838
|
-
return perfEntries.filter((entry) => typeof entry?.name === "string" && entry.name.length > 0);
|
|
839
|
-
}
|
|
840
|
-
|
|
841
|
-
async function replayPerformanceApiResponses(
|
|
842
|
-
tabId: string,
|
|
843
|
-
perfEntries: PerformanceResourceEntry[],
|
|
844
|
-
responseBodies: Map<string, string>,
|
|
845
|
-
captureUrl?: string,
|
|
846
|
-
intent?: string,
|
|
847
|
-
): Promise<string[]> {
|
|
848
|
-
const perfUrls = selectPerformanceReplayCandidates(perfEntries, {
|
|
849
|
-
captureUrl,
|
|
850
|
-
intent,
|
|
851
|
-
knownUrls: responseBodies.keys(),
|
|
852
|
-
limit: 30,
|
|
853
|
-
});
|
|
854
|
-
if (perfUrls.length === 0) return [];
|
|
855
|
-
|
|
856
|
-
log("capture", `Performance API selected ${perfUrls.length} replayable resource URLs`);
|
|
857
|
-
let replayCount = 0;
|
|
858
|
-
for (const perfUrl of perfUrls) {
|
|
859
|
-
if (responseBodies.has(perfUrl)) continue;
|
|
860
|
-
try {
|
|
861
|
-
const body = await kuri.evaluate(
|
|
862
|
-
tabId,
|
|
863
|
-
`(function(){var x=new XMLHttpRequest();x.open('GET','${perfUrl.replace(/'/g, "\\'")}',false);x.send();return x.status>=200&&x.status<400?x.responseText:''})()`,
|
|
864
|
-
);
|
|
865
|
-
if (typeof body === "string" && body.length > 0 && body.length < 512 * 1024) {
|
|
866
|
-
responseBodies.set(perfUrl, body);
|
|
867
|
-
replayCount++;
|
|
868
|
-
log("capture", `replay-fetched ${perfUrl.substring(0, 80)} (${body.length}B)`);
|
|
869
|
-
}
|
|
870
|
-
} catch {
|
|
871
|
-
// non-fatal
|
|
872
|
-
}
|
|
873
|
-
}
|
|
874
|
-
|
|
875
|
-
if (replayCount > 0) {
|
|
876
|
-
log("capture", `replay-fetched ${replayCount} API response bodies via Performance API`);
|
|
877
|
-
}
|
|
878
|
-
return perfUrls;
|
|
879
|
-
}
|
|
880
|
-
|
|
881
|
-
async function replayHarApiResponses(
|
|
882
|
-
tabId: string,
|
|
883
|
-
harEntries: kuri.KuriHarEntry[],
|
|
884
|
-
responseBodies: Map<string, string>,
|
|
885
|
-
): Promise<void> {
|
|
886
|
-
let harReplayCount = 0;
|
|
887
|
-
|
|
888
|
-
for (const entry of harEntries) {
|
|
889
|
-
const harUrl = entry.request?.url;
|
|
890
|
-
if (!harUrl || responseBodies.has(harUrl)) continue;
|
|
891
|
-
if (REPLAY_SKIP.test(harUrl)) continue;
|
|
892
|
-
const method = entry.request?.method?.toUpperCase();
|
|
893
|
-
if (method === "OPTIONS" || method === "HEAD") continue;
|
|
894
|
-
const status = entry.response?.status ?? 0;
|
|
895
|
-
if (status < 200 || status >= 400) continue;
|
|
896
|
-
const ct = (entry.response?.headers ?? []).find((header: { name: string }) => header.name.toLowerCase() === "content-type")?.value ?? "";
|
|
897
|
-
if (!HAR_REPLAY_CT.test(ct) && !harUrl.includes("/api/") && !harUrl.includes("_search") && !harUrl.includes("graphql") && !harUrl.includes("voyager")) continue;
|
|
898
|
-
if (harReplayCount >= 20) break;
|
|
899
|
-
try {
|
|
900
|
-
const postData = method !== "GET" ? entry.request?.postData?.text : undefined;
|
|
901
|
-
const replayScript = method === "GET" || !postData
|
|
902
|
-
? `(function(){var x=new XMLHttpRequest();x.open('GET','${harUrl.replace(/'/g, "\\'")}',false);x.send();return x.status>=200&&x.status<400?x.responseText:''})()`
|
|
903
|
-
: `(function(){var x=new XMLHttpRequest();x.open('${method}','${harUrl.replace(/'/g, "\\'")}',false);x.setRequestHeader('Content-Type','application/json');x.send(${JSON.stringify(postData)});return x.status>=200&&x.status<400?x.responseText:''})()`;
|
|
904
|
-
const body = await kuri.evaluate(tabId, replayScript);
|
|
905
|
-
if (typeof body === "string" && body.length > 0 && body.length < 512 * 1024) {
|
|
906
|
-
responseBodies.set(harUrl, body);
|
|
907
|
-
harReplayCount++;
|
|
908
|
-
log("capture", `har-replay-fetched ${harUrl.substring(0, 80)} (${body.length}B)`);
|
|
909
|
-
}
|
|
910
|
-
} catch {
|
|
911
|
-
// non-fatal
|
|
912
|
-
}
|
|
913
|
-
}
|
|
914
|
-
|
|
915
|
-
if (harReplayCount > 0) {
|
|
916
|
-
log("capture", `har-replay-fetched ${harReplayCount} API response bodies from HAR entries`);
|
|
917
|
-
}
|
|
918
|
-
}
|
|
919
|
-
|
|
920
|
-
export async function enrichPassiveCaptureRequests(params: {
|
|
921
|
-
tabId: string;
|
|
922
|
-
captureUrl: string;
|
|
923
|
-
harEntries: kuri.KuriHarEntry[];
|
|
924
|
-
intent?: string;
|
|
925
|
-
}): Promise<RawRequest[]> {
|
|
926
|
-
const { tabId, captureUrl, harEntries, intent } = params;
|
|
927
|
-
const responseBodies = new Map<string, string>();
|
|
928
|
-
const perfEntries = await collectPerformanceResourceEntries(tabId).catch(() => []);
|
|
929
|
-
|
|
930
|
-
let intercepted = await collectInterceptedRequests(tabId).catch(() => []);
|
|
931
|
-
appendInterceptedResponseBodies(intercepted, responseBodies);
|
|
932
|
-
|
|
933
|
-
if (intercepted.length === 0 && harEntries.length === 0) {
|
|
934
|
-
await waitForContentReady(tabId, captureUrl, intent, responseBodies).catch(() => {});
|
|
935
|
-
intercepted = await collectInterceptedRequests(tabId).catch(() => intercepted);
|
|
936
|
-
appendInterceptedResponseBodies(intercepted, responseBodies);
|
|
937
|
-
}
|
|
938
|
-
|
|
939
|
-
const performanceUrls = await replayPerformanceApiResponses(tabId, perfEntries, responseBodies, captureUrl, intent).catch(() => []);
|
|
940
|
-
await replayHarApiResponses(tabId, harEntries, responseBodies).catch(() => {});
|
|
941
|
-
|
|
942
|
-
const extensionEntries = await collectExtensionRequests(tabId).catch(() => []);
|
|
943
|
-
const requests = mergePassiveCaptureData(intercepted, harEntries, extensionEntries, responseBodies, performanceUrls).map((request) => ({
|
|
944
|
-
...request,
|
|
945
|
-
url: normalizeCapturedUrl(request.url, captureUrl),
|
|
946
|
-
}));
|
|
947
|
-
|
|
948
|
-
// HAR strips auth headers. Infer their presence from cookies so
|
|
949
|
-
// enrichEndpointsWithTokenSources can create DAG bindings.
|
|
950
|
-
if (extensionEntries.length === 0) {
|
|
951
|
-
const tabCookies = await kuri.getCookies(tabId).catch(() => []) as Array<{ name: string; value: string }>;
|
|
952
|
-
const hasAuthCookie = tabCookies.some((c) => /^(ct0|csrf_token|_csrf|csrftoken|XSRF-TOKEN|auth_token)$/i.test(c.name));
|
|
953
|
-
if (hasAuthCookie) {
|
|
954
|
-
for (const req of requests) {
|
|
955
|
-
if (/\/(api|graphql|v\d+)\b/i.test(req.url) || /ads-api|voyager/i.test(req.url)) {
|
|
956
|
-
if (!req.request_headers["authorization"]) req.request_headers["authorization"] = "[REDACTED]";
|
|
957
|
-
if (!req.request_headers["x-csrf-token"]) req.request_headers["x-csrf-token"] = "[REDACTED]";
|
|
958
|
-
}
|
|
959
|
-
}
|
|
960
|
-
}
|
|
961
|
-
}
|
|
962
|
-
|
|
963
|
-
log("capture", `browse-checkpoint tracked ${harEntries.length} HAR, ${intercepted.length} intercepted, ${extensionEntries.length} extension, ${responseBodies.size} bodies → ${requests.length} merged`);
|
|
964
|
-
return requests;
|
|
965
|
-
}
|
|
966
|
-
/**
|
|
967
|
-
* Collect network requests observed by kuri's builtin extension (chrome.webRequest).
|
|
968
|
-
* Gracefully returns [] if the extension relay is not yet wired.
|
|
969
|
-
*/
|
|
970
|
-
async function collectExtensionRequests(tabId: string): Promise<ExtensionEntry[]> {
|
|
971
|
-
try {
|
|
972
|
-
const raw = await kuri.evaluate(tabId, `
|
|
973
|
-
(function() {
|
|
974
|
-
if (!window.__kuri || !window.__kuri._networkLog) return '[]';
|
|
975
|
-
var log = window.__kuri._networkLog;
|
|
976
|
-
window.__kuri._networkLog = [];
|
|
977
|
-
return JSON.stringify(log);
|
|
978
|
-
})()
|
|
979
|
-
`);
|
|
980
|
-
if (typeof raw !== "string" || !raw.startsWith("[")) return [];
|
|
981
|
-
const entries: ExtensionEntry[] = JSON.parse(raw);
|
|
982
|
-
log("capture", `extension observer: ${entries.length} entries collected`);
|
|
983
|
-
return entries;
|
|
984
|
-
} catch {
|
|
985
|
-
log("capture", "extension observer: not available (expected if relay not wired)");
|
|
986
|
-
return [];
|
|
987
|
-
}
|
|
988
|
-
}
|
|
989
|
-
|
|
990
|
-
/**
|
|
991
|
-
* Poll document.readyState until "complete" or timeout.
|
|
992
|
-
* Replaces page.waitForLoadState("networkidle").
|
|
993
|
-
*/
|
|
994
|
-
async function waitForReadyState(tabId: string, timeoutMs = 8000): Promise<void> {
|
|
995
|
-
const start = Date.now();
|
|
996
|
-
while (Date.now() - start < timeoutMs) {
|
|
997
|
-
try {
|
|
998
|
-
const state = await kuri.evaluate(tabId, "document.readyState");
|
|
999
|
-
if (state === "complete") return;
|
|
1000
|
-
} catch { /* tab may not be ready */ }
|
|
1001
|
-
await new Promise((r) => setTimeout(r, 500));
|
|
1002
|
-
}
|
|
1003
|
-
}
|
|
1004
|
-
|
|
1005
|
-
/**
|
|
1006
|
-
* Adaptive content-ready wait. Replaces flat 5s timeout.
|
|
1007
|
-
* Phase 1: 2s initial settle
|
|
1008
|
-
* Phase 2: If Cloudflare challenge detected, poll up to 15s for clearance
|
|
1009
|
-
* Phase 3: Wait for document.readyState complete (replaces networkidle)
|
|
1010
|
-
* Phase 4: Intent-aware API wait — poll intercepted requests for matching URLs
|
|
1011
|
-
* Phase 5: SPA scroll stimulus for search/explore pages
|
|
1012
|
-
*/
|
|
1013
|
-
async function waitForContentReady(
|
|
1014
|
-
tabId: string,
|
|
1015
|
-
captureUrl?: string,
|
|
1016
|
-
intent?: string,
|
|
1017
|
-
responseBodies?: Map<string, string>,
|
|
1018
|
-
): Promise<void> {
|
|
1019
|
-
// Phase 1: Initial settle — let the page start rendering
|
|
1020
|
-
await new Promise((r) => setTimeout(r, 1000));
|
|
1021
|
-
|
|
1022
|
-
// Early exit: if interceptor already captured API responses, page is loaded enough
|
|
1023
|
-
if (responseBodies && responseBodies.size > 0) {
|
|
1024
|
-
log("capture", `early exit: ${responseBodies.size} API responses already captured during navigation`);
|
|
1025
|
-
// Brief extra settle to catch any trailing responses
|
|
1026
|
-
await new Promise((r) => setTimeout(r, 500));
|
|
1027
|
-
return;
|
|
1028
|
-
}
|
|
1029
|
-
|
|
1030
|
-
// Phase 2: Cloudflare challenge detection and wait
|
|
1031
|
-
try {
|
|
1032
|
-
const hasCf = await kuri.hasCloudflareChallenge(tabId);
|
|
1033
|
-
if (hasCf) {
|
|
1034
|
-
log("capture", "Cloudflare challenge detected, waiting for clearance...");
|
|
1035
|
-
const cleared = await kuri.waitForCloudflare(tabId, 15000);
|
|
1036
|
-
if (cleared) {
|
|
1037
|
-
log("capture", "Cloudflare cleared");
|
|
1038
|
-
}
|
|
1039
|
-
}
|
|
1040
|
-
} catch {
|
|
1041
|
-
// Tab not available — skip CF detection
|
|
1042
|
-
}
|
|
1043
|
-
|
|
1044
|
-
// Phase 3: Wait for document ready state (replaces networkidle)
|
|
1045
|
-
await waitForReadyState(tabId, 5000);
|
|
1046
|
-
|
|
1047
|
-
// Early exit: check again after readyState — SPAs often fire API calls during hydration
|
|
1048
|
-
if (responseBodies) {
|
|
1049
|
-
const intercepted = await collectInterceptedRequests(tabId);
|
|
1050
|
-
for (const entry of intercepted) {
|
|
1051
|
-
if (entry.response_body && !entry.is_js) {
|
|
1052
|
-
responseBodies.set(entry.url, entry.response_body);
|
|
1053
|
-
}
|
|
1054
|
-
}
|
|
1055
|
-
if (responseBodies.size > 0) {
|
|
1056
|
-
log("capture", `early exit after readyState: ${responseBodies.size} API responses captured`);
|
|
1057
|
-
return;
|
|
1058
|
-
}
|
|
1059
|
-
}
|
|
1060
|
-
|
|
1061
|
-
// Phase 4: Intent-aware API wait — poll intercepted requests for matching API URLs
|
|
1062
|
-
if (captureUrl && responseBodies) {
|
|
1063
|
-
const derivedHints = new Set(deriveIntentHints(captureUrl, intent));
|
|
1064
|
-
const wantedHints = [...derivedHints].filter((hint) =>
|
|
1065
|
-
![...responseBodies.keys()].some((u) => u.toLowerCase().includes(hint))
|
|
1066
|
-
);
|
|
1067
|
-
if (wantedHints.length > 0) {
|
|
1068
|
-
log("capture", `intent-aware wait: looking for API matching one of [${wantedHints.join(", ")}] (from ${captureUrl})`);
|
|
1069
|
-
const intentStart = Date.now();
|
|
1070
|
-
const INTENT_MAX_WAIT = 8000;
|
|
1071
|
-
const INTENT_POLL_INTERVAL = 1000;
|
|
1072
|
-
while (Date.now() - intentStart < INTENT_MAX_WAIT) {
|
|
1073
|
-
await new Promise((r) => setTimeout(r, INTENT_POLL_INTERVAL));
|
|
1074
|
-
// Check newly intercepted requests
|
|
1075
|
-
const intercepted = await collectInterceptedRequests(tabId);
|
|
1076
|
-
for (const entry of intercepted) {
|
|
1077
|
-
const respUrl = entry.url.toLowerCase();
|
|
1078
|
-
const matchedHint = wantedHints.find((hint) => respUrl.includes(hint));
|
|
1079
|
-
if (matchedHint) {
|
|
1080
|
-
log("capture", `intent-aware wait: matched ${matchedHint} via ${entry.url.substring(0, 120)}`);
|
|
1081
|
-
// Add to responseBodies so downstream sees it
|
|
1082
|
-
if (entry.response_body && !entry.is_js) {
|
|
1083
|
-
responseBodies.set(entry.url, entry.response_body);
|
|
1084
|
-
}
|
|
1085
|
-
await new Promise((r) => setTimeout(r, 500));
|
|
1086
|
-
return;
|
|
1087
|
-
}
|
|
1088
|
-
}
|
|
1089
|
-
}
|
|
1090
|
-
} else if (derivedHints.size > 0) {
|
|
1091
|
-
log("capture", `intent-aware wait: already captured API matching one of [${[...derivedHints].join(", ")}], skipping`);
|
|
1092
|
-
}
|
|
1093
|
-
}
|
|
1094
|
-
|
|
1095
|
-
// Phase 5: generic SPA stimulus via scroll
|
|
1096
|
-
const lowerIntent = intent?.toLowerCase() ?? "";
|
|
1097
|
-
if (
|
|
1098
|
-
captureUrl &&
|
|
1099
|
-
responseBodies &&
|
|
1100
|
-
(
|
|
1101
|
-
/search|explore|trending|tabs|discover/i.test(captureUrl) ||
|
|
1102
|
-
/\b(person|people|profile|profiles|user|users|member|members|company|companies|organization|organisations|business|post|posts|tweet|tweets|status|statuses)\b/.test(lowerIntent)
|
|
1103
|
-
)
|
|
1104
|
-
) {
|
|
1105
|
-
try {
|
|
1106
|
-
const before = responseBodies.size;
|
|
1107
|
-
await kuri.evaluate(tabId, "window.scrollTo(0, Math.max(window.innerHeight, Math.min(document.body.scrollHeight, window.innerHeight * 2)))");
|
|
1108
|
-
await new Promise((r) => setTimeout(r, 1200));
|
|
1109
|
-
await kuri.evaluate(tabId, "window.scrollTo(0, 0)");
|
|
1110
|
-
if (responseBodies.size === before) {
|
|
1111
|
-
await new Promise((r) => setTimeout(r, 1500));
|
|
1112
|
-
}
|
|
1113
|
-
} catch {
|
|
1114
|
-
// non-fatal
|
|
1115
|
-
}
|
|
1116
|
-
}
|
|
1117
|
-
|
|
1118
|
-
await maybeProbeIntentApis(tabId, captureUrl, intent, responseBodies);
|
|
1119
|
-
}
|
|
1120
|
-
|
|
1121
|
-
/**
|
|
1122
|
-
* Inject cookies into tab via kuri.setCookies.
|
|
1123
|
-
*/
|
|
1124
|
-
async function injectCookies(
|
|
1125
|
-
tabId: string,
|
|
1126
|
-
cookies: Array<{
|
|
1127
|
-
name: string;
|
|
1128
|
-
value: string;
|
|
1129
|
-
domain: string;
|
|
1130
|
-
path?: string;
|
|
1131
|
-
secure?: boolean;
|
|
1132
|
-
httpOnly?: boolean;
|
|
1133
|
-
sameSite?: string;
|
|
1134
|
-
expires?: number;
|
|
1135
|
-
}>
|
|
1136
|
-
): Promise<void> {
|
|
1137
|
-
const sanitized = cookies.map((c) => ({
|
|
1138
|
-
name: c.name,
|
|
1139
|
-
value: c.value,
|
|
1140
|
-
domain: c.domain.startsWith(".") ? c.domain : `.${c.domain}`,
|
|
1141
|
-
path: c.path ?? "/",
|
|
1142
|
-
...(c.secure != null ? { secure: c.secure } : {}),
|
|
1143
|
-
...(c.httpOnly != null ? { httpOnly: c.httpOnly } : {}),
|
|
1144
|
-
...(c.sameSite != null ? { sameSite: c.sameSite } : {}),
|
|
1145
|
-
...(c.expires != null && c.expires > 0 ? { expires: c.expires } : {}),
|
|
1146
|
-
}));
|
|
1147
|
-
|
|
1148
|
-
log("capture", `injecting ${sanitized.length} cookies for domains: ${[...new Set(sanitized.map((c) => c.domain))].join(", ")}`);
|
|
1149
|
-
try {
|
|
1150
|
-
await kuri.setCookies(tabId, sanitized);
|
|
1151
|
-
} catch (batchErr) {
|
|
1152
|
-
log("capture", `batch cookie injection failed: ${batchErr instanceof Error ? batchErr.message : batchErr} — falling back to per-cookie`);
|
|
1153
|
-
let injected = 0;
|
|
1154
|
-
for (const cookie of sanitized) {
|
|
1155
|
-
try {
|
|
1156
|
-
await kuri.setCookie(tabId, cookie);
|
|
1157
|
-
injected++;
|
|
1158
|
-
} catch (err) {
|
|
1159
|
-
log("capture", `failed to inject cookie "${cookie.name}" for ${cookie.domain}: ${err instanceof Error ? err.message : err}`);
|
|
1160
|
-
}
|
|
1161
|
-
}
|
|
1162
|
-
log("capture", `per-cookie fallback: ${injected}/${sanitized.length} injected`);
|
|
1163
|
-
}
|
|
1164
|
-
}
|
|
1165
|
-
|
|
1166
|
-
/**
|
|
1167
|
-
* Extract cookies from page via document.cookie (CDP getCookies crashes Kuri).
|
|
1168
|
-
* Parses simple name=value pairs — httpOnly cookies are NOT visible via JS.
|
|
1169
|
-
*/
|
|
1170
|
-
async function extractCookiesFromPage(tabId: string, pageUrl: string): Promise<CapturedCookie[]> {
|
|
1171
|
-
try {
|
|
1172
|
-
const raw = await kuri.evaluate(tabId, "document.cookie");
|
|
1173
|
-
if (typeof raw !== "string" || !raw) return [];
|
|
1174
|
-
let hostname = "";
|
|
1175
|
-
try { hostname = new URL(pageUrl).hostname; } catch { return []; }
|
|
1176
|
-
const domain = hostname.startsWith(".") ? hostname : `.${hostname}`;
|
|
1177
|
-
return raw.split(";").map((pair) => {
|
|
1178
|
-
const eqIdx = pair.indexOf("=");
|
|
1179
|
-
if (eqIdx < 0) return null;
|
|
1180
|
-
return {
|
|
1181
|
-
name: pair.substring(0, eqIdx).trim(),
|
|
1182
|
-
value: pair.substring(eqIdx + 1).trim(),
|
|
1183
|
-
domain,
|
|
1184
|
-
};
|
|
1185
|
-
}).filter((c): c is CapturedCookie => c !== null && c.name.length > 0);
|
|
1186
|
-
} catch {
|
|
1187
|
-
return [];
|
|
1188
|
-
}
|
|
1189
|
-
}
|
|
1190
|
-
|
|
1191
|
-
export async function captureSession(
|
|
1192
|
-
url: string,
|
|
1193
|
-
authHeaders?: Record<string, string>,
|
|
1194
|
-
cookies?: Array<{ name: string; value: string; domain: string; path?: string; secure?: boolean; httpOnly?: boolean; sameSite?: string; expires?: number }>,
|
|
1195
|
-
intent?: string,
|
|
1196
|
-
options?: { forceEphemeral?: boolean },
|
|
1197
|
-
): Promise<CaptureResult> {
|
|
1198
|
-
await acquireTabSlot();
|
|
1199
|
-
|
|
1200
|
-
// Guard: check browser access config before launching
|
|
1201
|
-
if (!isBrowserAccessAvailable()) {
|
|
1202
|
-
releaseTabSlot("no-tab");
|
|
1203
|
-
throw new Error("Browser access is not available (proxy-only config)");
|
|
1204
|
-
}
|
|
1205
|
-
|
|
1206
|
-
// Ensure Kuri is running and tabs are discovered
|
|
1207
|
-
await kuri.start();
|
|
1208
|
-
await kuri.discoverTabs(); // Sync Chrome tabs into Kuri's registry
|
|
1209
|
-
|
|
1210
|
-
let tabId: string;
|
|
1211
|
-
try {
|
|
1212
|
-
tabId = await openFreshCaptureTab();
|
|
1213
|
-
} catch (error) {
|
|
1214
|
-
releaseTabSlot("no-tab");
|
|
1215
|
-
throw error;
|
|
1216
|
-
}
|
|
1217
|
-
activeTabRegistry.add(tabId);
|
|
1218
|
-
|
|
1219
|
-
const domain = new URL(url).hostname;
|
|
1220
|
-
let captureTimedOut = false;
|
|
1221
|
-
let retryFreshTab = false;
|
|
1222
|
-
let captureError: unknown;
|
|
1223
|
-
const abortController = new AbortController();
|
|
1224
|
-
const { signal } = abortController;
|
|
1225
|
-
// phase() races each CDP call against the overall capture timeout (bug #113 fix)
|
|
1226
|
-
const phase = <T>(label: string, fn: () => Promise<T>) =>
|
|
1227
|
-
withBrowserPhaseTimeout(signal, label, fn);
|
|
1228
|
-
const timeoutHandle = setTimeout(() => {
|
|
1229
|
-
captureTimedOut = true;
|
|
1230
|
-
abortController.abort();
|
|
1231
|
-
resetTab(tabId).catch(() => {});
|
|
1232
|
-
}, CAPTURE_TIMEOUT_MS);
|
|
1233
|
-
|
|
1234
|
-
try {
|
|
1235
|
-
// Set headers: client hints + auth headers
|
|
1236
|
-
const allHeaders = { ...CLIENT_HINT_HEADERS, ...(authHeaders ?? {}) };
|
|
1237
|
-
await phase("setHeaders", () => kuri.setHeaders(tabId, allHeaders));
|
|
1238
|
-
|
|
1239
|
-
// Navigate to origin first so cookies are set in the correct domain context
|
|
1240
|
-
// (CDP setCookie on about:blank doesn't bind to the target domain properly)
|
|
1241
|
-
if (cookies && cookies.length > 0) {
|
|
1242
|
-
const origin = new URL(url).origin;
|
|
1243
|
-
await phase("navigate:origin", () => kuri.navigate(tabId, origin));
|
|
1244
|
-
await phase("waitForLoad:origin", () => kuri.waitForLoad(tabId, 10_000).catch(() => {}));
|
|
1245
|
-
|
|
1246
|
-
// Check if browser already has a valid session (not redirected to login).
|
|
1247
|
-
// If so, skip vault cookie injection — browser cookies are fresher and
|
|
1248
|
-
// injecting stale vault cookies (e.g. JSESSIONID) causes HTTP 400.
|
|
1249
|
-
const postOriginUrl = await phase("checkOriginUrl", () => kuri.getCurrentUrl(tabId));
|
|
1250
|
-
const LOGIN_PATHS_RE = /\/(login|signin|sign-in|sso|auth|uas\/login|checkpoint|oauth)/i;
|
|
1251
|
-
const originHost = new URL(origin).hostname;
|
|
1252
|
-
let browserAlreadyAuthed = false;
|
|
1253
|
-
try {
|
|
1254
|
-
const postHost = new URL(postOriginUrl).hostname;
|
|
1255
|
-
browserAlreadyAuthed = postHost === originHost && !LOGIN_PATHS_RE.test(new URL(postOriginUrl).pathname);
|
|
1256
|
-
} catch { /* bad URL — inject cookies as fallback */ }
|
|
1257
|
-
|
|
1258
|
-
if (browserAlreadyAuthed) {
|
|
1259
|
-
log("capture", `browser already authenticated at ${postOriginUrl} — skipping vault cookie injection`);
|
|
1260
|
-
} else {
|
|
1261
|
-
await phase("injectCookies", () => injectCookies(tabId, cookies!));
|
|
1262
|
-
}
|
|
1263
|
-
}
|
|
1264
|
-
|
|
1265
|
-
// Inject interceptor persistently — survives navigations via Page.addScriptToEvaluateOnNewDocument
|
|
1266
|
-
if (!interceptorInjectedTabs.has(tabId)) {
|
|
1267
|
-
try {
|
|
1268
|
-
await phase("scriptInject", () => kuri.scriptInject(tabId, INTERCEPTOR_SCRIPT));
|
|
1269
|
-
interceptorInjectedTabs.add(tabId);
|
|
1270
|
-
log("capture", "interceptor installed via scriptInject (persistent)");
|
|
1271
|
-
} catch {
|
|
1272
|
-
// Fallback for older kuri versions without scriptInject
|
|
1273
|
-
log("capture", "scriptInject unavailable — falling back to evaluate injection");
|
|
1274
|
-
}
|
|
1275
|
-
}
|
|
1276
|
-
|
|
1277
|
-
// Start HAR recording
|
|
1278
|
-
await phase("harStart", () => kuri.harStart(tabId));
|
|
1279
|
-
|
|
1280
|
-
// Determine page domain for JS bundle filtering
|
|
1281
|
-
let pageDomain: string | undefined;
|
|
1282
|
-
try { pageDomain = getRegistrableDomain(new URL(url).hostname); } catch { /* bad url */ }
|
|
1283
|
-
|
|
1284
|
-
// Navigate to target URL
|
|
1285
|
-
await phase("navigate", () => kuri.navigate(tabId, url));
|
|
1286
|
-
|
|
1287
|
-
// If scriptInject wasn't available, fall back to single evaluate injection after navigation
|
|
1288
|
-
if (!interceptorInjectedTabs.has(tabId)) {
|
|
1289
|
-
await phase("evaluate:interceptor", () => kuri.evaluate(tabId, INTERCEPTOR_SCRIPT).catch(() => {}));
|
|
1290
|
-
log("capture", "interceptor re-injected via evaluate fallback");
|
|
1291
|
-
}
|
|
1292
|
-
|
|
1293
|
-
// Build response bodies map from intercepted requests
|
|
1294
|
-
const responseBodies = new Map<string, string>();
|
|
1295
|
-
const jsBundleBodies = new Map<string, string>();
|
|
1296
|
-
const MAX_JS_BUNDLES = 20;
|
|
1297
|
-
|
|
1298
|
-
// Helper: drain intercepted data into responseBodies/jsBundleBodies
|
|
1299
|
-
const drainIntercepted = async () => {
|
|
1300
|
-
try {
|
|
1301
|
-
const entries = await collectInterceptedRequests(tabId);
|
|
1302
|
-
for (const entry of entries) {
|
|
1303
|
-
if (entry.is_js && jsBundleBodies.size < MAX_JS_BUNDLES && pageDomain) {
|
|
1304
|
-
try {
|
|
1305
|
-
const jsDomain = getRegistrableDomain(new URL(entry.url).hostname);
|
|
1306
|
-
if (jsDomain === pageDomain && entry.response_body) {
|
|
1307
|
-
jsBundleBodies.set(entry.url, entry.response_body);
|
|
1308
|
-
}
|
|
1309
|
-
} catch { /* bad url */ }
|
|
1310
|
-
} else if (entry.response_body && !entry.is_js) {
|
|
1311
|
-
responseBodies.set(entry.url, entry.response_body);
|
|
1312
|
-
}
|
|
1313
|
-
}
|
|
1314
|
-
return entries;
|
|
1315
|
-
} catch { return []; }
|
|
1316
|
-
};
|
|
1317
|
-
|
|
1318
|
-
// Adaptive wait: handle Cloudflare challenges + SPA content loading + intent-aware API wait
|
|
1319
|
-
await phase("waitForContentReady", () => waitForContentReady(tabId, url, intent, responseBodies));
|
|
1320
|
-
|
|
1321
|
-
// Incremental collection: drain intercepted data after content ready
|
|
1322
|
-
await drainIntercepted();
|
|
1323
|
-
|
|
1324
|
-
// Collect all intercepted requests (final sweep)
|
|
1325
|
-
const intercepted = await phase("collectIntercepted", () => collectInterceptedRequests(tabId));
|
|
1326
|
-
const extensionEntries = await phase("collectExtension", () => collectExtensionRequests(tabId));
|
|
1327
|
-
|
|
1328
|
-
// Separate JS bundles from data responses (from final sweep)
|
|
1329
|
-
for (const entry of intercepted) {
|
|
1330
|
-
if (entry.is_js && jsBundleBodies.size < MAX_JS_BUNDLES && pageDomain) {
|
|
1331
|
-
try {
|
|
1332
|
-
const jsDomain = getRegistrableDomain(new URL(entry.url).hostname);
|
|
1333
|
-
if (jsDomain === pageDomain && entry.response_body) {
|
|
1334
|
-
jsBundleBodies.set(entry.url, entry.response_body);
|
|
1335
|
-
}
|
|
1336
|
-
} catch { /* bad url */ }
|
|
1337
|
-
} else if (entry.response_body && !entry.is_js) {
|
|
1338
|
-
responseBodies.set(entry.url, entry.response_body);
|
|
1339
|
-
}
|
|
1340
|
-
}
|
|
1341
|
-
|
|
1342
|
-
// --- Performance API discovery + sync XHR replay ---
|
|
1343
|
-
// The JS interceptor can miss early requests (SPA API calls fire before injection).
|
|
1344
|
-
// HAR captures URLs but not always bodies.
|
|
1345
|
-
// Use Performance API to discover replayable API URLs, including API-ish preloads.
|
|
1346
|
-
let performanceUrls: string[] = [];
|
|
1347
|
-
try {
|
|
1348
|
-
const perfEntries = await phase("evaluate:perf", () => collectPerformanceResourceEntries(tabId));
|
|
1349
|
-
performanceUrls = await phase("replay-fetch", () =>
|
|
1350
|
-
replayPerformanceApiResponses(tabId, perfEntries, responseBodies, url, intent),
|
|
1351
|
-
);
|
|
1352
|
-
} catch { /* non-fatal */ }
|
|
1353
|
-
|
|
1354
|
-
// Stop HAR recording and merge with intercepted data
|
|
1355
|
-
let harEntries: kuri.KuriHarEntry[] = [];
|
|
1356
|
-
try {
|
|
1357
|
-
const harResult = await phase("harStop", () => kuri.harStop(tabId));
|
|
1358
|
-
harEntries = harResult.entries;
|
|
1359
|
-
} catch { /* HAR may not be available */ }
|
|
1360
|
-
|
|
1361
|
-
// --- HAR-based replay ---
|
|
1362
|
-
// CDP HAR sees ALL network requests (even ones the JS interceptor missed).
|
|
1363
|
-
// For any API-like HAR entry without a body in responseBodies, replay-fetch it.
|
|
1364
|
-
const HAR_REPLAY_CT = /application\/json|text\/plain|\+json/i;
|
|
1365
|
-
let harReplayCount = 0;
|
|
1366
|
-
for (const entry of harEntries) {
|
|
1367
|
-
const harUrl = entry.request?.url;
|
|
1368
|
-
if (!harUrl || responseBodies.has(harUrl)) continue;
|
|
1369
|
-
if (REPLAY_SKIP.test(harUrl)) continue;
|
|
1370
|
-
const method = entry.request?.method?.toUpperCase();
|
|
1371
|
-
if (method === "OPTIONS" || method === "HEAD") continue;
|
|
1372
|
-
const status = entry.response?.status ?? 0;
|
|
1373
|
-
if (status < 200 || status >= 400) continue;
|
|
1374
|
-
const ct = (entry.response?.headers ?? []).find((h: { name: string }) => h.name.toLowerCase() === "content-type")?.value ?? "";
|
|
1375
|
-
if (!HAR_REPLAY_CT.test(ct) && !harUrl.includes("/api/") && !harUrl.includes("_search") && !harUrl.includes("graphql")) continue;
|
|
1376
|
-
if (harReplayCount >= 20) break;
|
|
1377
|
-
try {
|
|
1378
|
-
const postData = method !== "GET" ? entry.request?.postData?.text : undefined;
|
|
1379
|
-
const replayScript = method === "GET" || !postData
|
|
1380
|
-
? `(function(){var x=new XMLHttpRequest();x.open('GET','${harUrl.replace(/'/g, "\\'")}',false);x.send();return x.status>=200&&x.status<400?x.responseText:''})()`
|
|
1381
|
-
: `(function(){var x=new XMLHttpRequest();x.open('${method}','${harUrl.replace(/'/g, "\\'")}',false);x.setRequestHeader('Content-Type','application/json');x.send(${JSON.stringify(postData)});return x.status>=200&&x.status<400?x.responseText:''})()`;
|
|
1382
|
-
const body = await phase("har-replay", () => kuri.evaluate(tabId, replayScript));
|
|
1383
|
-
if (typeof body === "string" && body.length > 0 && body.length < 512 * 1024) {
|
|
1384
|
-
responseBodies.set(harUrl, body);
|
|
1385
|
-
harReplayCount++;
|
|
1386
|
-
log("capture", `har-replay-fetched ${harUrl.substring(0, 80)} (${body.length}B)`);
|
|
1387
|
-
}
|
|
1388
|
-
} catch { /* non-fatal */ }
|
|
1389
|
-
}
|
|
1390
|
-
if (harReplayCount > 0) {
|
|
1391
|
-
log("capture", `har-replay-fetched ${harReplayCount} API response bodies from HAR entries`);
|
|
1392
|
-
}
|
|
1393
|
-
|
|
1394
|
-
const har_lineage_id = nanoid();
|
|
1395
|
-
|
|
1396
|
-
// Debug: log captured counts
|
|
1397
|
-
log("capture", `tracked ${harEntries.length} HAR, ${intercepted.length} intercepted, ${extensionEntries.length} extension, ${responseBodies.size} bodies`);
|
|
1398
|
-
for (const [bodyUrl] of responseBodies) {
|
|
1399
|
-
log("capture", `response body captured: ${bodyUrl.substring(0, 150)}`);
|
|
1400
|
-
}
|
|
1401
|
-
|
|
1402
|
-
let final_url = url;
|
|
1403
|
-
let html: string | undefined;
|
|
1404
|
-
try {
|
|
1405
|
-
const rawUrl = await phase("getCurrentUrl", () => kuri.getCurrentUrl(tabId));
|
|
1406
|
-
final_url = typeof rawUrl === "string" ? rawUrl : String(rawUrl ?? url);
|
|
1407
|
-
// Validate it's actually a URL, fall back to original if not
|
|
1408
|
-
try { new URL(final_url); } catch { final_url = url; }
|
|
1409
|
-
html = await phase("getPageHtml", () => kuri.getPageHtml(tabId));
|
|
1410
|
-
} catch {}
|
|
1411
|
-
|
|
1412
|
-
// Merge all passive capture sources into unified request list
|
|
1413
|
-
const requests: RawRequest[] = mergePassiveCaptureData(intercepted, harEntries, extensionEntries, responseBodies, performanceUrls);
|
|
1414
|
-
log("capture", `tracked ${harEntries.length} HAR, ${intercepted.length} intercepted, ${extensionEntries.length} extension, ${responseBodies.size} bodies → ${requests.length} merged`);
|
|
1415
|
-
|
|
1416
|
-
// Extract session cookies via document.cookie
|
|
1417
|
-
const rawCookies = await phase("extractCookies", () => extractCookiesFromPage(tabId, url));
|
|
1418
|
-
const sessionCookies = filterFirstPartySessionCookies(rawCookies, url, final_url);
|
|
1419
|
-
|
|
1420
|
-
if (captureTimedOut) throw new Error(`captureSession timed out after ${CAPTURE_TIMEOUT_MS}ms for ${url}`);
|
|
1421
|
-
log("capture", `captured ${jsBundleBodies.size} JS bundles for route scanning`);
|
|
1422
|
-
|
|
1423
|
-
const responseBodyCount = responseBodies.size;
|
|
1424
|
-
const capturedRequestUrls = requests.map((request) => request.url);
|
|
1425
|
-
const hasUsefulResponseBodies = hasUsefulCapturedResponses(responseBodies.keys(), url, intent);
|
|
1426
|
-
const hasUsefulCapturedRequests = hasUsefulCapturedResponses(capturedRequestUrls, url, intent);
|
|
1427
|
-
const hasRichCapturedTraffic = requests.length >= 25;
|
|
1428
|
-
if (
|
|
1429
|
-
isBlockedAppShell(html) &&
|
|
1430
|
-
responseBodyCount < 10 &&
|
|
1431
|
-
!hasUsefulResponseBodies &&
|
|
1432
|
-
!hasUsefulCapturedRequests &&
|
|
1433
|
-
!hasRichCapturedTraffic
|
|
1434
|
-
) {
|
|
1435
|
-
// On ephemeral retry, surface a structured auth/block instead of throwing a
|
|
1436
|
-
// generic "failed without returning" error after the second pass.
|
|
1437
|
-
if (options?.forceEphemeral) {
|
|
1438
|
-
const cloudflareBlocked = !!html && /Cloudflare|cf\.errors\.css|cf-error-details/i.test(html);
|
|
1439
|
-
throw Object.assign(new Error(cloudflareBlocked ? "cloudflare_waf_block" : "blocked_app_shell"), {
|
|
1440
|
-
code: "auth_required",
|
|
1441
|
-
login_url: final_url || url,
|
|
1442
|
-
...(cloudflareBlocked ? { reason: "cloudflare_waf" } : { reason: "blocked_app_shell" }),
|
|
1443
|
-
});
|
|
1444
|
-
}
|
|
1445
|
-
retryFreshTab = true;
|
|
1446
|
-
log("capture", `rendered blocked app shell for ${url}; retrying with fresh tab`);
|
|
1447
|
-
} else {
|
|
1448
|
-
return {
|
|
1449
|
-
requests,
|
|
1450
|
-
har_lineage_id,
|
|
1451
|
-
domain,
|
|
1452
|
-
cookies: sessionCookies.length > 0 ? sessionCookies : undefined,
|
|
1453
|
-
final_url,
|
|
1454
|
-
// WebSocket capture skipped (Kuri limitation)
|
|
1455
|
-
ws_messages: undefined,
|
|
1456
|
-
html,
|
|
1457
|
-
js_bundles: jsBundleBodies.size > 0 ? jsBundleBodies : undefined,
|
|
1458
|
-
};
|
|
1459
|
-
}
|
|
1460
|
-
} catch (error) {
|
|
1461
|
-
captureError = error;
|
|
1462
|
-
if (shouldRetryEphemeralProfileError(error)) {
|
|
1463
|
-
retryFreshTab = true;
|
|
1464
|
-
log("capture", `tab failed for ${url}; retrying with fresh tab (${error instanceof Error ? error.message : String(error)})`);
|
|
1465
|
-
} else {
|
|
1466
|
-
throw error;
|
|
1467
|
-
}
|
|
1468
|
-
} finally {
|
|
1469
|
-
clearTimeout(timeoutHandle);
|
|
1470
|
-
abortController.abort(); // no-op if already aborted; prevents stale phase rejections
|
|
1471
|
-
await resetTab(tabId);
|
|
1472
|
-
releaseTabSlot(tabId);
|
|
1473
|
-
}
|
|
1474
|
-
if (retryFreshTab && !options?.forceEphemeral) {
|
|
1475
|
-
return captureSession(url, authHeaders, cookies, intent, { forceEphemeral: true });
|
|
1476
|
-
}
|
|
1477
|
-
if (captureError) throw captureError;
|
|
1478
|
-
throw new Error(`captureSession failed without returning a result for ${url}`);
|
|
1479
|
-
}
|
|
1480
|
-
|
|
1481
|
-
export async function executeInBrowser(
|
|
1482
|
-
url: string,
|
|
1483
|
-
method: string,
|
|
1484
|
-
requestHeaders: Record<string, string>,
|
|
1485
|
-
body?: unknown,
|
|
1486
|
-
authHeaders?: Record<string, string>,
|
|
1487
|
-
cookies?: Array<{ name: string; value: string; domain: string; path?: string; secure?: boolean; httpOnly?: boolean; sameSite?: string; expires?: number }>
|
|
1488
|
-
): Promise<{ status: number; data: unknown; trace_id: string }> {
|
|
1489
|
-
if (!isBrowserAccessAvailable()) {
|
|
1490
|
-
throw new Error("Browser access is not available (proxy-only config)");
|
|
1491
|
-
}
|
|
1492
|
-
await kuri.start();
|
|
1493
|
-
await kuri.discoverTabs();
|
|
1494
|
-
|
|
1495
|
-
let tabId: string;
|
|
1496
|
-
try {
|
|
1497
|
-
tabId = await openFreshCaptureTab();
|
|
1498
|
-
} catch (error) {
|
|
1499
|
-
releaseTabSlot("no-tab");
|
|
1500
|
-
throw error;
|
|
1501
|
-
}
|
|
1502
|
-
activeTabRegistry.add(tabId);
|
|
1503
|
-
|
|
1504
|
-
try {
|
|
1505
|
-
const allHeaders = { ...CLIENT_HINT_HEADERS, ...authHeaders, ...requestHeaders };
|
|
1506
|
-
await kuri.setHeaders(tabId, allHeaders);
|
|
1507
|
-
|
|
1508
|
-
if (cookies && cookies.length > 0) {
|
|
1509
|
-
await injectCookies(tabId, cookies);
|
|
1510
|
-
}
|
|
1511
|
-
|
|
1512
|
-
// Navigate to origin so in-page fetch inherits cookies/CORS
|
|
1513
|
-
const origin = new URL(url).origin;
|
|
1514
|
-
await kuri.navigate(tabId, origin);
|
|
1515
|
-
await waitForReadyState(tabId, 5000);
|
|
1516
|
-
|
|
1517
|
-
const result = await kuri.executeInPageFetch(tabId, url, method, requestHeaders, body);
|
|
1518
|
-
return { ...result, trace_id: nanoid() };
|
|
1519
|
-
} finally {
|
|
1520
|
-
await resetTab(tabId);
|
|
1521
|
-
releaseTabSlot(tabId);
|
|
1522
|
-
}
|
|
1523
|
-
}
|
|
1524
|
-
|
|
1525
|
-
/**
|
|
1526
|
-
* Trigger-and-intercept execution: navigate to the page that originally
|
|
1527
|
-
* triggered an API call, let the site's own JS make the request (passing
|
|
1528
|
-
* CSRF checks, TLS fingerprinting, etc.), and intercept the response.
|
|
1529
|
-
*
|
|
1530
|
-
* Uses the injected fetch/XHR interceptor to capture the target API response.
|
|
1531
|
-
*/
|
|
1532
|
-
export async function triggerAndIntercept(
|
|
1533
|
-
triggerUrl: string,
|
|
1534
|
-
targetUrlPattern: string,
|
|
1535
|
-
cookies: Array<{ name: string; value: string; domain: string; path?: string; secure?: boolean; httpOnly?: boolean; sameSite?: string; expires?: number }>,
|
|
1536
|
-
authHeaders?: Record<string, string>,
|
|
1537
|
-
): Promise<{ status: number; data: unknown; trace_id: string }> {
|
|
1538
|
-
await acquireTabSlot();
|
|
1539
|
-
await kuri.start();
|
|
1540
|
-
await kuri.discoverTabs();
|
|
1541
|
-
|
|
1542
|
-
let tabId: string;
|
|
1543
|
-
try {
|
|
1544
|
-
tabId = await openFreshCaptureTab();
|
|
1545
|
-
} catch (error) {
|
|
1546
|
-
releaseTabSlot("no-tab");
|
|
1547
|
-
throw error;
|
|
1548
|
-
}
|
|
1549
|
-
activeTabRegistry.add(tabId);
|
|
1550
|
-
|
|
1551
|
-
try {
|
|
1552
|
-
// Set headers
|
|
1553
|
-
const headers = { ...CLIENT_HINT_HEADERS, ...authHeaders };
|
|
1554
|
-
await kuri.setHeaders(tabId, headers);
|
|
1555
|
-
await injectCookies(tabId, cookies);
|
|
1556
|
-
|
|
1557
|
-
// Build a URL matcher
|
|
1558
|
-
const targetBase = targetUrlPattern.replace(/\{[^}]+\}/g, "").split("?")[0];
|
|
1559
|
-
let targetQueryId: string | null = null;
|
|
1560
|
-
try {
|
|
1561
|
-
const tu = new URL(targetUrlPattern.replace(/\{[^}]+\}/g, "x"));
|
|
1562
|
-
const rawQueryId = tu.searchParams.get("queryId");
|
|
1563
|
-
targetQueryId = rawQueryId ? rawQueryId.split(".")[0] : null;
|
|
1564
|
-
} catch { /* skip */ }
|
|
1565
|
-
|
|
1566
|
-
log("capture", `trigger-and-intercept: navigating to ${triggerUrl}, waiting for ${targetBase}${targetQueryId ? `?queryId=${targetQueryId.slice(0, 40)}` : ""}`);
|
|
1567
|
-
|
|
1568
|
-
// Navigate to trigger origin first, inject interceptor
|
|
1569
|
-
try {
|
|
1570
|
-
const origin = new URL(triggerUrl).origin;
|
|
1571
|
-
await kuri.navigate(tabId, origin);
|
|
1572
|
-
await new Promise((r) => setTimeout(r, 500));
|
|
1573
|
-
await kuri.evaluate(tabId, INTERCEPTOR_SCRIPT);
|
|
1574
|
-
} catch { /* best-effort */ }
|
|
1575
|
-
|
|
1576
|
-
// Navigate to the trigger page — the site's JS will make the API call
|
|
1577
|
-
await kuri.navigate(tabId, triggerUrl);
|
|
1578
|
-
// Re-inject interceptor after navigation
|
|
1579
|
-
try {
|
|
1580
|
-
await new Promise((r) => setTimeout(r, 300));
|
|
1581
|
-
await kuri.evaluate(tabId, INTERCEPTOR_SCRIPT);
|
|
1582
|
-
} catch { /* page may not be ready */ }
|
|
1583
|
-
|
|
1584
|
-
// Poll for the target response
|
|
1585
|
-
const POLL_INTERVAL = 1000;
|
|
1586
|
-
const MAX_WAIT = 15000;
|
|
1587
|
-
const startTime = Date.now();
|
|
1588
|
-
|
|
1589
|
-
while (Date.now() - startTime < MAX_WAIT) {
|
|
1590
|
-
await new Promise((r) => setTimeout(r, POLL_INTERVAL));
|
|
1591
|
-
const intercepted = await collectInterceptedRequests(tabId);
|
|
1592
|
-
for (const entry of intercepted) {
|
|
1593
|
-
const respUrl = entry.url;
|
|
1594
|
-
const baseMatch = respUrl.includes(targetBase);
|
|
1595
|
-
const queryIdMatch = !targetQueryId || respUrl.includes(targetQueryId);
|
|
1596
|
-
if (baseMatch && queryIdMatch) {
|
|
1597
|
-
let data: unknown = entry.response_body;
|
|
1598
|
-
try { data = JSON.parse(entry.response_body ?? ""); } catch { /* keep as string */ }
|
|
1599
|
-
log("capture", `trigger-and-intercept: got status ${entry.response_status} for ${targetBase}`);
|
|
1600
|
-
return { status: entry.response_status, data, trace_id: nanoid() };
|
|
1601
|
-
}
|
|
1602
|
-
}
|
|
1603
|
-
}
|
|
1604
|
-
|
|
1605
|
-
log("capture", `trigger-and-intercept: timeout waiting for ${targetBase}`);
|
|
1606
|
-
return {
|
|
1607
|
-
status: 0,
|
|
1608
|
-
data: { error: "trigger_timeout", message: "Target API call not intercepted within 15s" },
|
|
1609
|
-
trace_id: nanoid(),
|
|
1610
|
-
};
|
|
1611
|
-
} finally {
|
|
1612
|
-
await resetTab(tabId);
|
|
1613
|
-
releaseTabSlot(tabId);
|
|
1614
|
-
}
|
|
1615
|
-
}
|
|
1616
|
-
|
|
1617
|
-
// ---------------------------------------------------------------------------
|
|
1618
|
-
// First-pass browser action execution (Lane 1: browser action floor)
|
|
1619
|
-
//
|
|
1620
|
-
// When no reusable second-pass route exists, the browser completes the task
|
|
1621
|
-
// directly using action primitives. Passive reverse-engineering (HAR capture,
|
|
1622
|
-
// interceptor, bundle scanning) runs in the background simultaneously.
|
|
1623
|
-
// ---------------------------------------------------------------------------
|
|
1624
|
-
|
|
1625
|
-
/** A single step in a browser action sequence. */
|
|
1626
|
-
export interface BrowserActionStep {
|
|
1627
|
-
/** Action to perform. "snapshot" takes an a11y snapshot to discover refs. */
|
|
1628
|
-
action: kuri.KuriActionType | "snapshot" | "navigate" | "wait" | "type" | "evaluate";
|
|
1629
|
-
/** Element ref from the a11y snapshot (e.g. "e5"). Required for click/fill/select/etc. */
|
|
1630
|
-
ref?: string;
|
|
1631
|
-
/** Value for fill/select/type/press/evaluate, URL for navigate, selector for wait. */
|
|
1632
|
-
value?: string;
|
|
1633
|
-
/** Timeout in ms for wait actions. */
|
|
1634
|
-
timeoutMs?: number;
|
|
1635
|
-
}
|
|
1636
|
-
|
|
1637
|
-
/** Internal record of when each action step started, used for request provenance tagging. */
|
|
1638
|
-
interface StepTimingEntry {
|
|
1639
|
-
stepIndex: number;
|
|
1640
|
-
action: string;
|
|
1641
|
-
ref?: string;
|
|
1642
|
-
startedAt: string; // ISO timestamp
|
|
1643
|
-
}
|
|
1644
|
-
|
|
1645
|
-
/**
|
|
1646
|
-
* Tag captured requests with the action step that triggered them.
|
|
1647
|
-
* Uses timestamp comparison: a request is attributed to the latest step
|
|
1648
|
-
* whose startedAt is <= the request's timestamp. Requests that arrived
|
|
1649
|
-
* before any step (e.g. during initial navigation) are left untagged.
|
|
1650
|
-
*
|
|
1651
|
-
* Exported for testability.
|
|
1652
|
-
*/
|
|
1653
|
-
export function tagRequestProvenance(
|
|
1654
|
-
requests: RawRequest[],
|
|
1655
|
-
stepTimings: StepTimingEntry[],
|
|
1656
|
-
): void {
|
|
1657
|
-
if (stepTimings.length === 0) return;
|
|
1658
|
-
|
|
1659
|
-
// Sort step timings by startedAt ascending (should already be, but be safe)
|
|
1660
|
-
const sorted = [...stepTimings].sort(
|
|
1661
|
-
(a, b) => new Date(a.startedAt).getTime() - new Date(b.startedAt).getTime(),
|
|
1662
|
-
);
|
|
1663
|
-
|
|
1664
|
-
for (const req of requests) {
|
|
1665
|
-
if (!req.timestamp) continue;
|
|
1666
|
-
const reqTime = new Date(req.timestamp).getTime();
|
|
1667
|
-
|
|
1668
|
-
// Find the latest step that started at or before this request's timestamp
|
|
1669
|
-
let matchedStep: StepTimingEntry | undefined;
|
|
1670
|
-
for (const step of sorted) {
|
|
1671
|
-
if (new Date(step.startedAt).getTime() <= reqTime) {
|
|
1672
|
-
matchedStep = step;
|
|
1673
|
-
} else {
|
|
1674
|
-
break;
|
|
1675
|
-
}
|
|
1676
|
-
}
|
|
1677
|
-
|
|
1678
|
-
if (matchedStep) {
|
|
1679
|
-
req.triggered_by_step = matchedStep.stepIndex;
|
|
1680
|
-
req.triggered_by_action = matchedStep.action;
|
|
1681
|
-
req.triggered_by_ref = matchedStep.ref;
|
|
1682
|
-
}
|
|
1683
|
-
}
|
|
1684
|
-
}
|
|
1685
|
-
|
|
1686
|
-
/** Result of a first-pass browser action execution. */
|
|
1687
|
-
export interface BrowserActionResult {
|
|
1688
|
-
/** Whether the action sequence completed without errors. */
|
|
1689
|
-
ok: boolean;
|
|
1690
|
-
/** Per-step results. */
|
|
1691
|
-
steps: Array<{ action: string; ok: boolean; result?: unknown; error?: string }>;
|
|
1692
|
-
/** Final page URL after all actions. */
|
|
1693
|
-
final_url: string;
|
|
1694
|
-
/** Final page HTML (for downstream extraction). */
|
|
1695
|
-
html?: string;
|
|
1696
|
-
/** Last a11y snapshot (compact text-tree). */
|
|
1697
|
-
snapshot?: string;
|
|
1698
|
-
/** Passive capture result gathered during action execution. */
|
|
1699
|
-
capture?: CaptureResult;
|
|
1700
|
-
trace_id: string;
|
|
1701
|
-
}
|
|
1702
|
-
|
|
1703
|
-
/**
|
|
1704
|
-
* Execute a sequence of browser actions on a page while running passive
|
|
1705
|
-
* capture in the background. This is the "first pass" — the browser does
|
|
1706
|
-
* the task now, and the captured network traffic feeds reverse-engineering
|
|
1707
|
-
* for a cheaper second pass next time.
|
|
1708
|
-
*/
|
|
1709
|
-
export async function executeActionSequence(
|
|
1710
|
-
url: string,
|
|
1711
|
-
steps: BrowserActionStep[],
|
|
1712
|
-
options?: {
|
|
1713
|
-
authHeaders?: Record<string, string>;
|
|
1714
|
-
cookies?: Array<{ name: string; value: string; domain: string; path?: string; secure?: boolean; httpOnly?: boolean; sameSite?: string; expires?: number }>;
|
|
1715
|
-
intent?: string;
|
|
1716
|
-
/** Skip passive capture (useful for quick action-only flows). */
|
|
1717
|
-
skipCapture?: boolean;
|
|
1718
|
-
},
|
|
1719
|
-
): Promise<BrowserActionResult> {
|
|
1720
|
-
await acquireTabSlot();
|
|
1721
|
-
await kuri.start();
|
|
1722
|
-
await kuri.discoverTabs();
|
|
1723
|
-
|
|
1724
|
-
let tabId: string;
|
|
1725
|
-
try {
|
|
1726
|
-
tabId = await openFreshCaptureTab();
|
|
1727
|
-
} catch (error) {
|
|
1728
|
-
releaseTabSlot("no-tab");
|
|
1729
|
-
throw error;
|
|
1730
|
-
}
|
|
1731
|
-
activeTabRegistry.add(tabId);
|
|
1732
|
-
|
|
1733
|
-
const traceId = nanoid();
|
|
1734
|
-
const stepResults: BrowserActionResult["steps"] = [];
|
|
1735
|
-
const stepTimings: StepTimingEntry[] = [];
|
|
1736
|
-
try {
|
|
1737
|
-
// Setup: headers + cookies
|
|
1738
|
-
const allHeaders = { ...CLIENT_HINT_HEADERS, ...(options?.authHeaders ?? {}) };
|
|
1739
|
-
await kuri.setHeaders(tabId, allHeaders);
|
|
1740
|
-
if (options?.cookies?.length) {
|
|
1741
|
-
await injectCookies(tabId, options.cookies);
|
|
1742
|
-
}
|
|
1743
|
-
|
|
1744
|
-
// Start background passive capture (HAR + interceptor)
|
|
1745
|
-
if (!options?.skipCapture) {
|
|
1746
|
-
await kuri.harStart(tabId);
|
|
1747
|
-
await kuri.evaluate(tabId, INTERCEPTOR_SCRIPT).catch(() => {});
|
|
1748
|
-
}
|
|
1749
|
-
|
|
1750
|
-
// Navigate to the starting URL
|
|
1751
|
-
await kuri.navigate(tabId, url);
|
|
1752
|
-
await kuri.waitForLoad(tabId, CAPTURE_NAV_TIMEOUT_MS);
|
|
1753
|
-
|
|
1754
|
-
// Re-inject interceptor after navigation
|
|
1755
|
-
if (!options?.skipCapture) {
|
|
1756
|
-
await kuri.evaluate(tabId, INTERCEPTOR_SCRIPT).catch(() => {});
|
|
1757
|
-
}
|
|
1758
|
-
|
|
1759
|
-
// Execute each action step
|
|
1760
|
-
for (let stepIdx = 0; stepIdx < steps.length; stepIdx++) {
|
|
1761
|
-
const step = steps[stepIdx];
|
|
1762
|
-
// Record step start time for request provenance tagging (#214)
|
|
1763
|
-
stepTimings.push({
|
|
1764
|
-
stepIndex: stepIdx,
|
|
1765
|
-
action: step.action,
|
|
1766
|
-
ref: step.ref,
|
|
1767
|
-
startedAt: new Date().toISOString(),
|
|
1768
|
-
});
|
|
1769
|
-
try {
|
|
1770
|
-
let result: unknown;
|
|
1771
|
-
switch (step.action) {
|
|
1772
|
-
case "snapshot":
|
|
1773
|
-
result = await kuri.snapshot(tabId, step.value);
|
|
1774
|
-
break;
|
|
1775
|
-
|
|
1776
|
-
case "navigate":
|
|
1777
|
-
if (!step.value) throw new Error("navigate requires a value (URL)");
|
|
1778
|
-
await kuri.navigate(tabId, step.value);
|
|
1779
|
-
await kuri.waitForLoad(tabId, step.timeoutMs ?? CAPTURE_NAV_TIMEOUT_MS);
|
|
1780
|
-
// Re-inject interceptor after navigation
|
|
1781
|
-
if (!options?.skipCapture) {
|
|
1782
|
-
await kuri.evaluate(tabId, INTERCEPTOR_SCRIPT).catch(() => {});
|
|
1783
|
-
}
|
|
1784
|
-
result = await kuri.getCurrentUrl(tabId);
|
|
1785
|
-
break;
|
|
1786
|
-
|
|
1787
|
-
case "wait":
|
|
1788
|
-
result = await kuri.waitForSelector(tabId, step.value, step.timeoutMs ?? 5000);
|
|
1789
|
-
break;
|
|
1790
|
-
|
|
1791
|
-
case "type":
|
|
1792
|
-
if (!step.value) throw new Error("type requires a value (text)");
|
|
1793
|
-
result = await kuri.keyboardType(tabId, step.value);
|
|
1794
|
-
break;
|
|
1795
|
-
|
|
1796
|
-
case "evaluate":
|
|
1797
|
-
if (!step.value) throw new Error("evaluate requires a value (expression)");
|
|
1798
|
-
result = await kuri.evaluate(tabId, step.value);
|
|
1799
|
-
break;
|
|
1800
|
-
|
|
1801
|
-
case "click":
|
|
1802
|
-
case "dblclick":
|
|
1803
|
-
case "hover":
|
|
1804
|
-
case "focus":
|
|
1805
|
-
case "blur":
|
|
1806
|
-
case "check":
|
|
1807
|
-
case "uncheck":
|
|
1808
|
-
if (!step.ref) throw new Error(`${step.action} requires a ref`);
|
|
1809
|
-
result = await kuri.action(tabId, step.action, step.ref);
|
|
1810
|
-
break;
|
|
1811
|
-
|
|
1812
|
-
case "fill":
|
|
1813
|
-
case "select":
|
|
1814
|
-
if (!step.ref) throw new Error(`${step.action} requires a ref`);
|
|
1815
|
-
if (!step.value) throw new Error(`${step.action} requires a value`);
|
|
1816
|
-
result = await kuri.action(tabId, step.action, step.ref, step.value);
|
|
1817
|
-
break;
|
|
1818
|
-
|
|
1819
|
-
case "scroll":
|
|
1820
|
-
result = await kuri.scroll(tabId);
|
|
1821
|
-
break;
|
|
1822
|
-
|
|
1823
|
-
case "press":
|
|
1824
|
-
if (!step.value) throw new Error("press requires a value (key)");
|
|
1825
|
-
result = await kuri.press(tabId, step.value);
|
|
1826
|
-
break;
|
|
1827
|
-
}
|
|
1828
|
-
|
|
1829
|
-
stepResults.push({ action: step.action, ok: true, result });
|
|
1830
|
-
} catch (err) {
|
|
1831
|
-
stepResults.push({
|
|
1832
|
-
action: step.action,
|
|
1833
|
-
ok: false,
|
|
1834
|
-
error: err instanceof Error ? err.message : String(err),
|
|
1835
|
-
});
|
|
1836
|
-
}
|
|
1837
|
-
}
|
|
1838
|
-
|
|
1839
|
-
// Collect final state
|
|
1840
|
-
const rawFinalUrl = await kuri.getCurrentUrl(tabId).catch(() => url);
|
|
1841
|
-
const finalUrl = typeof rawFinalUrl === "string" && rawFinalUrl.startsWith("http") ? rawFinalUrl : url;
|
|
1842
|
-
const rawHtml = await kuri.getPageHtml(tabId).catch(() => undefined);
|
|
1843
|
-
const html = typeof rawHtml === "string" && rawHtml.trimStart().startsWith("<") ? rawHtml : undefined;
|
|
1844
|
-
const lastSnapshot = await kuri.snapshot(tabId).catch(() => undefined);
|
|
1845
|
-
|
|
1846
|
-
// Collect passive capture in the background
|
|
1847
|
-
let capture: CaptureResult | undefined;
|
|
1848
|
-
if (!options?.skipCapture) {
|
|
1849
|
-
try {
|
|
1850
|
-
const domain = new URL(url).hostname;
|
|
1851
|
-
const harResult = await kuri.harStop(tabId);
|
|
1852
|
-
const intercepted = await collectInterceptedRequests(tabId);
|
|
1853
|
-
const responseBodies = new Map<string, string>();
|
|
1854
|
-
for (const entry of intercepted) {
|
|
1855
|
-
if (entry.response_body && !entry.is_js) {
|
|
1856
|
-
responseBodies.set(entry.url, entry.response_body);
|
|
1857
|
-
}
|
|
1858
|
-
}
|
|
1859
|
-
|
|
1860
|
-
const requests: RawRequest[] = harResult.entries.map((entry) => {
|
|
1861
|
-
const reqHeaders: Record<string, string> = {};
|
|
1862
|
-
for (const h of entry.request.headers ?? []) reqHeaders[h.name] = h.value;
|
|
1863
|
-
const respHeaders: Record<string, string> = {};
|
|
1864
|
-
for (const h of entry.response.headers ?? []) respHeaders[h.name] = h.value;
|
|
1865
|
-
return {
|
|
1866
|
-
url: entry.request.url,
|
|
1867
|
-
method: entry.request.method,
|
|
1868
|
-
request_headers: reqHeaders,
|
|
1869
|
-
request_body: entry.request.postData?.text,
|
|
1870
|
-
response_status: entry.response.status,
|
|
1871
|
-
response_headers: respHeaders,
|
|
1872
|
-
response_body: responseBodies.get(entry.request.url) ?? entry.response.content?.text,
|
|
1873
|
-
timestamp: entry.startedDateTime,
|
|
1874
|
-
};
|
|
1875
|
-
});
|
|
1876
|
-
|
|
1877
|
-
// Add intercepted requests not in HAR
|
|
1878
|
-
const harUrls = new Set(harResult.entries.map((e) => e.request.url));
|
|
1879
|
-
for (const entry of intercepted) {
|
|
1880
|
-
if (entry.is_js || harUrls.has(entry.url)) continue;
|
|
1881
|
-
requests.push({
|
|
1882
|
-
url: entry.url,
|
|
1883
|
-
method: entry.method,
|
|
1884
|
-
request_headers: entry.request_headers,
|
|
1885
|
-
request_body: entry.request_body,
|
|
1886
|
-
response_status: entry.response_status,
|
|
1887
|
-
response_headers: entry.response_headers,
|
|
1888
|
-
response_body: entry.response_body,
|
|
1889
|
-
timestamp: entry.timestamp,
|
|
1890
|
-
});
|
|
1891
|
-
}
|
|
1892
|
-
|
|
1893
|
-
// Tag each request with the action step that triggered it (#214)
|
|
1894
|
-
tagRequestProvenance(requests, stepTimings);
|
|
1895
|
-
|
|
1896
|
-
const sessionCookies = filterFirstPartySessionCookies(
|
|
1897
|
-
await extractCookiesFromPage(tabId, url),
|
|
1898
|
-
url,
|
|
1899
|
-
finalUrl,
|
|
1900
|
-
);
|
|
1901
|
-
|
|
1902
|
-
capture = {
|
|
1903
|
-
requests,
|
|
1904
|
-
har_lineage_id: nanoid(),
|
|
1905
|
-
domain,
|
|
1906
|
-
cookies: sessionCookies.length > 0 ? sessionCookies : undefined,
|
|
1907
|
-
final_url: finalUrl,
|
|
1908
|
-
ws_messages: undefined,
|
|
1909
|
-
html,
|
|
1910
|
-
};
|
|
1911
|
-
} catch (err) {
|
|
1912
|
-
log("capture", `background capture collection failed: ${err instanceof Error ? err.message : String(err)}`);
|
|
1913
|
-
}
|
|
1914
|
-
}
|
|
1915
|
-
|
|
1916
|
-
return {
|
|
1917
|
-
ok: stepResults.every((s) => s.ok),
|
|
1918
|
-
steps: stepResults,
|
|
1919
|
-
final_url: finalUrl,
|
|
1920
|
-
html,
|
|
1921
|
-
snapshot: lastSnapshot,
|
|
1922
|
-
capture,
|
|
1923
|
-
trace_id: traceId,
|
|
1924
|
-
};
|
|
1925
|
-
} finally {
|
|
1926
|
-
await resetTab(tabId);
|
|
1927
|
-
releaseTabSlot(tabId);
|
|
1928
|
-
}
|
|
1929
|
-
}
|