webcrawl 1.2.4 → 1.2.7

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (116) hide show
  1. package/dist/cli.js +3 -2
  2. package/dist/core/engine/awswaf.d.ts +37 -0
  3. package/dist/core/engine/awswaf.js +189 -0
  4. package/dist/core/engine/cache.d.ts +70 -0
  5. package/dist/core/engine/cache.js +297 -0
  6. package/dist/core/engine/crawl-policy.d.ts +36 -0
  7. package/dist/core/engine/crawl-policy.js +134 -0
  8. package/dist/core/engine/crawler.d.ts +151 -0
  9. package/dist/core/engine/crawler.js +885 -0
  10. package/dist/core/engine/fetcher.d.ts +66 -0
  11. package/dist/core/engine/fetcher.js +185 -0
  12. package/dist/core/engine/index.d.ts +4 -0
  13. package/dist/core/engine/index.js +4 -0
  14. package/dist/core/engine/markdown.d.ts +33 -0
  15. package/dist/core/engine/markdown.js +334 -0
  16. package/dist/core/engine/path-scope.d.ts +21 -0
  17. package/dist/core/engine/path-scope.js +88 -0
  18. package/dist/core/engine/safe-url.d.ts +23 -0
  19. package/dist/core/engine/safe-url.js +92 -0
  20. package/dist/core/engine/stealth.d.ts +79 -0
  21. package/dist/core/engine/stealth.js +155 -0
  22. package/dist/core/engine.d.ts +1 -0
  23. package/dist/core/engine.js +1 -0
  24. package/dist/core/extract/business.d.ts +28 -0
  25. package/dist/core/extract/business.js +344 -0
  26. package/dist/core/extract/currency.d.ts +15 -0
  27. package/dist/core/extract/currency.js +259 -0
  28. package/dist/core/extract/cve-cache.d.ts +62 -0
  29. package/dist/core/extract/cve-cache.js +277 -0
  30. package/dist/core/extract/cve.d.ts +57 -0
  31. package/dist/core/extract/cve.js +397 -0
  32. package/dist/core/extract/dedupe-key.d.ts +12 -0
  33. package/dist/core/extract/dedupe-key.js +36 -0
  34. package/dist/core/extract/flights.d.ts +37 -0
  35. package/dist/core/extract/flights.js +373 -0
  36. package/dist/core/extract/hotels.d.ts +35 -0
  37. package/dist/core/extract/hotels.js +473 -0
  38. package/dist/core/extract/index.d.ts +8 -0
  39. package/dist/core/extract/index.js +8 -0
  40. package/dist/core/extract/real-estate.d.ts +53 -0
  41. package/dist/core/extract/real-estate.js +835 -0
  42. package/dist/core/extract/scaledown.d.ts +14 -0
  43. package/dist/core/extract/scaledown.js +484 -0
  44. package/dist/core/extract/shopping.d.ts +44 -0
  45. package/dist/core/extract/shopping.js +393 -0
  46. package/dist/core/image-url.d.ts +5 -0
  47. package/dist/core/image-url.js +13 -0
  48. package/dist/core/interact/driver.d.ts +49 -0
  49. package/dist/core/interact/driver.js +264 -0
  50. package/dist/core/interact/remote.d.ts +41 -0
  51. package/dist/core/interact/remote.js +133 -0
  52. package/dist/core/interact/spans.d.ts +46 -0
  53. package/dist/core/interact/spans.js +32 -0
  54. package/dist/core/interact/validate.d.ts +34 -0
  55. package/dist/core/interact/validate.js +135 -0
  56. package/dist/core/interact/vm.d.ts +42 -0
  57. package/dist/core/interact/vm.js +296 -0
  58. package/dist/core/local-engine.d.ts +35 -0
  59. package/dist/core/local-engine.js +367 -0
  60. package/dist/core/parse/detect.d.ts +29 -0
  61. package/dist/core/parse/detect.js +152 -0
  62. package/dist/core/parse/fetch.d.ts +20 -0
  63. package/dist/core/parse/fetch.js +39 -0
  64. package/dist/core/parse/image.d.ts +13 -0
  65. package/dist/core/parse/image.js +100 -0
  66. package/dist/core/parse/index.d.ts +53 -0
  67. package/dist/core/parse/index.js +153 -0
  68. package/dist/core/parse/limits.d.ts +12 -0
  69. package/dist/core/parse/limits.js +29 -0
  70. package/dist/core/parse/media.d.ts +42 -0
  71. package/dist/core/parse/media.js +238 -0
  72. package/dist/core/parse/merge.d.ts +19 -0
  73. package/dist/core/parse/merge.js +48 -0
  74. package/dist/core/parse/ocr.d.ts +25 -0
  75. package/dist/core/parse/ocr.js +121 -0
  76. package/dist/core/parse/office.d.ts +28 -0
  77. package/dist/core/parse/office.js +83 -0
  78. package/dist/core/parse/pdf.d.ts +27 -0
  79. package/dist/core/parse/pdf.js +156 -0
  80. package/dist/core/parse/types.d.ts +38 -0
  81. package/dist/core/parse/types.js +1 -0
  82. package/dist/core/parse/youtube.d.ts +30 -0
  83. package/dist/core/parse/youtube.js +264 -0
  84. package/dist/core/ranker/decision.d.ts +28 -0
  85. package/dist/core/ranker/decision.js +151 -0
  86. package/dist/core/ranker/index.d.ts +5 -0
  87. package/dist/core/ranker/index.js +361 -0
  88. package/dist/core/ranker/lexical.d.ts +82 -0
  89. package/dist/core/ranker/lexical.js +642 -0
  90. package/dist/core/ranker/types.d.ts +90 -0
  91. package/dist/core/ranker/types.js +1 -0
  92. package/dist/core/ranker.d.ts +1 -0
  93. package/dist/core/ranker.js +1 -0
  94. package/dist/core/sandbox.d.ts +29 -0
  95. package/dist/core/sandbox.js +182 -0
  96. package/dist/core/search/catalog.d.ts +174 -0
  97. package/dist/core/search/catalog.js +488 -0
  98. package/dist/core/search/index.d.ts +3 -0
  99. package/dist/core/search/index.js +3 -0
  100. package/dist/core/search/pipeline.d.ts +7 -0
  101. package/dist/core/search/pipeline.js +704 -0
  102. package/dist/core/search/providers.d.ts +62 -0
  103. package/dist/core/search/providers.js +1175 -0
  104. package/dist/core/search.d.ts +1 -0
  105. package/dist/core/search.js +1 -0
  106. package/dist/core/targets/developer.d.ts +44 -0
  107. package/dist/core/targets/developer.js +338 -0
  108. package/dist/core/targets/index.d.ts +3 -0
  109. package/dist/core/targets/index.js +3 -0
  110. package/dist/core/targets/registry.d.ts +32 -0
  111. package/dist/core/targets/registry.js +411 -0
  112. package/dist/core/targets/types.d.ts +72 -0
  113. package/dist/core/targets/types.js +8 -0
  114. package/dist/core/targets.d.ts +1 -0
  115. package/dist/core/targets.js +1 -0
  116. package/package.json +82 -4
@@ -0,0 +1,264 @@
1
+ /**
2
+ * Browser driver for interact runs. Node-only: called from workflow steps.
3
+ *
4
+ * Two backends share one step runner (STEP_SCRIPT):
5
+ * - sandbox: Chromium inside a Vercel Sandbox, reached by sandboxId so any
6
+ * step invocation (on any function instance) can reattach.
7
+ * - local: Chromium on this machine for dev (INTERACT_LOCAL_PLAYWRIGHT=true).
8
+ * Only works while steps run in the same long-lived process
9
+ * (Local World / `next dev`).
10
+ */
11
+ import { execFile, spawn } from "node:child_process";
12
+ import { mkdtempSync, writeFileSync } from "node:fs";
13
+ import { tmpdir } from "node:os";
14
+ import path from "node:path";
15
+ import { Sandbox } from "@vercel/sandbox";
16
+ import { CDP_PORT, SETUP_SCRIPT, STEP_SCRIPT, STEP_SCRIPT_PATH, VM_ROOT, WAIT_FOR_CDP_SCRIPT, chromeArgs, parseStepOutput, } from "./vm.js";
17
+ import { startRemoteSession, stopRemoteSession } from "./remote.js";
18
+ const snapshotId = () => process.env.WEBCRAWL_INTERACT_SNAPSHOT_ID || process.env.INTERACT_SNAPSHOT_ID || "";
19
+ const localMode = () => process.env.INTERACT_LOCAL_PLAYWRIGHT === "true";
20
+ /**
21
+ * True when some browser backend can run. Checked before starting a run so we never fake success.
22
+ * On Vercel (VERCEL=1) the OIDC token is not in env at request time: it arrives in the
23
+ * `x-vercel-oidc-token` request header, which @vercel/sandbox reads via @vercel/oidc.
24
+ */
25
+ export function interactBrowserAvailable() {
26
+ if (localMode())
27
+ return true;
28
+ return Boolean(process.env.VERCEL === "1" ||
29
+ process.env.VERCEL_OIDC_TOKEN ||
30
+ (process.env.VERCEL_TOKEN && process.env.VERCEL_TEAM_ID && process.env.VERCEL_PROJECT_ID));
31
+ }
32
+ export function interactBackend() {
33
+ if (localMode())
34
+ return "local";
35
+ return interactBrowserAvailable() ? "sandbox" : "none";
36
+ }
37
+ /* ----------------------------- sandbox backend ---------------------------- */
38
+ async function sh(sandbox, script, sudo = true) {
39
+ const r = await sandbox.runCommand({ cmd: "bash", args: ["-lc", script], sudo });
40
+ return { exitCode: r.exitCode, stdout: await r.stdout(), stderr: await r.stderr() };
41
+ }
42
+ const SANDBOX_RESOURCES = { vcpus: 2 };
43
+ /**
44
+ * Boots from the prebuilt snapshot (Chromium + Playwright already installed,
45
+ * see scripts/build-interact-snapshot.ts). If the snapshot is unset, expired
46
+ * or deleted, falls back to a cold VM and runs the full setup.
47
+ */
48
+ async function createBrowserSandbox(timeoutMs) {
49
+ const snap = snapshotId();
50
+ if (snap) {
51
+ try {
52
+ const sandbox = await Sandbox.create({
53
+ source: { type: "snapshot", snapshotId: snap },
54
+ timeout: timeoutMs,
55
+ resources: SANDBOX_RESOURCES,
56
+ });
57
+ return { sandbox, fromSnapshot: true };
58
+ }
59
+ catch (err) {
60
+ console.warn("[interact] snapshot boot failed, using cold setup", {
61
+ snapshotId: snap,
62
+ error: err instanceof Error ? err.message : String(err),
63
+ });
64
+ }
65
+ }
66
+ const sandbox = await Sandbox.create({ runtime: "node22", timeout: timeoutMs, resources: SANDBOX_RESOURCES });
67
+ return { sandbox, fromSnapshot: false };
68
+ }
69
+ async function startSandboxBrowser(opts) {
70
+ const { sandbox, fromSnapshot } = await createBrowserSandbox(opts.timeoutMs);
71
+ try {
72
+ if (!fromSnapshot) {
73
+ const setup = await sh(sandbox, SETUP_SCRIPT);
74
+ if (setup.exitCode !== 0)
75
+ throw new Error(`Browser setup failed: ${(setup.stderr || setup.stdout).slice(-300)}`);
76
+ }
77
+ // Always write the current step runner so a snapshot built from older code
78
+ // never runs a stale copy. It is a few KB, so this costs one round trip.
79
+ await sandbox.writeFiles([{ path: STEP_SCRIPT_PATH, content: Buffer.from(STEP_SCRIPT) }]);
80
+ const exe = (await sh(sandbox, `cat ${VM_ROOT}/chrome-path`)).stdout.trim().split("\n").pop();
81
+ if (!exe)
82
+ throw new Error("Browser binary not found in sandbox");
83
+ await sandbox.runCommand({ cmd: exe, args: chromeArgs({ userAgent: opts.userAgent }), detached: true, sudo: true });
84
+ const ready = await sh(sandbox, WAIT_FOR_CDP_SCRIPT, false);
85
+ if (ready.exitCode !== 0)
86
+ throw new Error("Browser did not start");
87
+ return { kind: "sandbox", sandboxId: sandbox.sandboxId };
88
+ }
89
+ catch (err) {
90
+ await sandbox.stop().catch(() => { });
91
+ throw err;
92
+ }
93
+ }
94
+ /* ------------------------------ local backend ----------------------------- */
95
+ async function resolveLocalPlaywright() {
96
+ const candidates = [
97
+ process.env.WC_PLAYWRIGHT,
98
+ path.join(process.cwd(), "node_modules/playwright/index.mjs"),
99
+ path.join(process.cwd(), "../node_modules/playwright/index.mjs"),
100
+ ].filter(Boolean);
101
+ for (const p of candidates) {
102
+ try {
103
+ const mod = (await import(/* webpackIgnore: true */ p));
104
+ const chrome = process.env.INTERACT_CHROME_PATH || mod.chromium.executablePath();
105
+ return { playwright: p, chrome };
106
+ }
107
+ catch {
108
+ /* try next */
109
+ }
110
+ }
111
+ throw new Error("Local browser unavailable: install playwright or set WC_PLAYWRIGHT");
112
+ }
113
+ async function startLocalBrowser(opts) {
114
+ const { playwright, chrome } = await resolveLocalPlaywright();
115
+ const port = 9300 + Math.floor(Math.random() * 600);
116
+ const dir = mkdtempSync(path.join(tmpdir(), "wc-interact-"));
117
+ const args = chromeArgs({ userAgent: opts.userAgent })
118
+ .map((a) => (a.startsWith("--remote-debugging-port=") ? `--remote-debugging-port=${port}` : a))
119
+ .map((a) => (a.startsWith("--user-data-dir=") ? `--user-data-dir=${dir}/profile` : a));
120
+ args.unshift("--disable-crash-reporter", "--disable-breakpad");
121
+ const child = spawn(chrome, args, { detached: true, stdio: "ignore" });
122
+ child.unref();
123
+ const cdp = `http://127.0.0.1:${port}`;
124
+ // A cold first launch (new profile) can take over 10s on macOS.
125
+ for (let i = 0; i < 75; i++) {
126
+ try {
127
+ const r = await fetch(`${cdp}/json/version`);
128
+ if (r.ok)
129
+ break;
130
+ }
131
+ catch {
132
+ /* not up yet */
133
+ }
134
+ if (i === 74) {
135
+ try {
136
+ process.kill(child.pid);
137
+ }
138
+ catch { /* gone */ }
139
+ throw new Error("Local browser did not start");
140
+ }
141
+ await new Promise((r) => setTimeout(r, 200));
142
+ }
143
+ const scriptPath = path.join(dir, "step.mjs");
144
+ writeFileSync(scriptPath, STEP_SCRIPT);
145
+ return { kind: "local", cdp, pid: child.pid, scriptPath, playwright };
146
+ }
147
+ /* ------------------------------ remote backend ---------------------------- */
148
+ /**
149
+ * A place to run STEP_SCRIPT without starting Chromium: the provider hosts the
150
+ * browser. Local Playwright when installed (dev), else a sandbox VM.
151
+ */
152
+ async function startStepRunner(timeoutMs) {
153
+ if (localMode()) {
154
+ const { playwright } = await resolveLocalPlaywright();
155
+ const dir = mkdtempSync(path.join(tmpdir(), "wc-remote-"));
156
+ const scriptPath = path.join(dir, "step.mjs");
157
+ writeFileSync(scriptPath, STEP_SCRIPT);
158
+ return { kind: "local", scriptPath, playwright };
159
+ }
160
+ const { sandbox, fromSnapshot } = await createBrowserSandbox(timeoutMs);
161
+ try {
162
+ if (!fromSnapshot) {
163
+ const setup = await sh(sandbox, SETUP_SCRIPT);
164
+ if (setup.exitCode !== 0)
165
+ throw new Error(`Step runner setup failed: ${(setup.stderr || setup.stdout).slice(-300)}`);
166
+ }
167
+ await sandbox.writeFiles([{ path: STEP_SCRIPT_PATH, content: Buffer.from(STEP_SCRIPT) }]);
168
+ return { kind: "sandbox", sandboxId: sandbox.sandboxId };
169
+ }
170
+ catch (err) {
171
+ await sandbox.stop().catch(() => { });
172
+ throw err;
173
+ }
174
+ }
175
+ async function startRemoteBrowser(provider, timeoutMs) {
176
+ // Both start in parallel; the provider bills from session start, so stop it if the runner fails.
177
+ const [session, runner] = await Promise.allSettled([startRemoteSession({ provider, timeoutMs }), startStepRunner(timeoutMs)]);
178
+ if (session.status === "fulfilled" && runner.status === "fulfilled") {
179
+ const s = session.value;
180
+ return {
181
+ kind: "remote",
182
+ provider,
183
+ sessionId: s.sessionId,
184
+ cdpUrl: s.cdpUrl,
185
+ ...(s.cdpHeaders ? { cdpHeaders: s.cdpHeaders } : {}),
186
+ startedAt: s.startedAt,
187
+ runner: runner.value,
188
+ };
189
+ }
190
+ if (session.status === "fulfilled")
191
+ await stopRemoteSession(session.value);
192
+ if (runner.status === "fulfilled")
193
+ await stopRunner(runner.value);
194
+ const reason = session.status === "rejected" ? session.reason : runner.reason;
195
+ throw reason instanceof Error ? reason : new Error(String(reason));
196
+ }
197
+ async function stopRunner(runner) {
198
+ if (runner.kind === "local")
199
+ return;
200
+ try {
201
+ const sandbox = await Sandbox.get({ sandboxId: runner.sandboxId });
202
+ await sandbox.stop();
203
+ }
204
+ catch {
205
+ /* already stopped or expired */
206
+ }
207
+ }
208
+ /* --------------------------------- public --------------------------------- */
209
+ export async function startBrowser(opts = {}) {
210
+ if (opts.remote)
211
+ return startRemoteBrowser(opts.remote, opts.timeoutMs ?? 5 * 60_000);
212
+ if (localMode())
213
+ return startLocalBrowser(opts);
214
+ if (!interactBrowserAvailable())
215
+ throw new Error("browser_unavailable");
216
+ return startSandboxBrowser({ timeoutMs: opts.timeoutMs ?? 5 * 60_000, userAgent: opts.userAgent });
217
+ }
218
+ function runLocalStep(scriptPath, playwright, cdp, action, extraEnv = {}) {
219
+ return new Promise((resolve) => {
220
+ execFile(process.execPath, [scriptPath, JSON.stringify(action)], {
221
+ env: { ...process.env, WC_PLAYWRIGHT: playwright, WC_CDP: cdp, ...extraEnv },
222
+ // A stealth fetchMany reads many documents in one step.
223
+ timeout: action.type === "fetchMany" ? 300_000 : 45_000,
224
+ maxBuffer: 16 * 1024 * 1024,
225
+ }, (_err, stdout, stderr) => resolve(parseStepOutput(String(stdout ?? ""), String(stderr ?? ""))));
226
+ });
227
+ }
228
+ async function runSandboxStep(sandboxId, action, env) {
229
+ const sandbox = await Sandbox.get({ sandboxId });
230
+ const r = await sandbox.runCommand({ cmd: "node", args: [STEP_SCRIPT_PATH, JSON.stringify(action)], sudo: true, ...(env ? { env } : {}) });
231
+ return parseStepOutput(await r.stdout(), await r.stderr());
232
+ }
233
+ export async function runBrowserStep(handle, action) {
234
+ if (handle.kind === "local")
235
+ return runLocalStep(handle.scriptPath, handle.playwright, handle.cdp, action);
236
+ if (handle.kind === "remote") {
237
+ // The provider's browser carries its own fingerprint; overriding it would undo that.
238
+ const remoteAction = { ...action, keepFingerprint: true, lightweight: true };
239
+ const headerEnv = handle.cdpHeaders ? { WC_CDP_HEADERS: JSON.stringify(handle.cdpHeaders) } : {};
240
+ return handle.runner.kind === "local"
241
+ ? runLocalStep(handle.runner.scriptPath, handle.runner.playwright, handle.cdpUrl, remoteAction, headerEnv)
242
+ : runSandboxStep(handle.runner.sandboxId, remoteAction, { WC_CDP: handle.cdpUrl, ...headerEnv });
243
+ }
244
+ return runSandboxStep(handle.sandboxId, action);
245
+ }
246
+ /** Stops the browser. For a remote provider, returns what the session cost (null when unknown). */
247
+ export async function stopBrowser(handle) {
248
+ if (handle.kind === "local") {
249
+ try {
250
+ process.kill(handle.pid);
251
+ }
252
+ catch {
253
+ /* already gone */
254
+ }
255
+ return null;
256
+ }
257
+ if (handle.kind === "remote") {
258
+ const [cost] = await Promise.all([stopRemoteSession(handle), stopRunner(handle.runner)]);
259
+ return cost;
260
+ }
261
+ await stopRunner(handle);
262
+ return null;
263
+ }
264
+ export { CDP_PORT };
@@ -0,0 +1,41 @@
1
+ /**
2
+ * Remote CDP browser providers for stealth fetches.
3
+ *
4
+ * A remote provider hosts the browser and hands back a CDP URL. Our step runner
5
+ * (sandbox or local Node) connects to it over CDP, so STEP_SCRIPT runs unchanged.
6
+ *
7
+ * WEBCRAWL_STEALTH_PROVIDER picks one:
8
+ * - browser-use: BROWSER_USE_API_KEY. Custom Chromium + residential proxy ($5/GB).
9
+ * - cloudflare: CLOUDFLARE_ACCOUNT_ID + CLOUDFLARE_BROWSER_TOKEN (Browser Rendering - Edit).
10
+ * Cloudflare Browser Run, $0.09 per browser hour, Cloudflare egress IPs.
11
+ * Only the stealth path (stealthFetch) uses it; /v1/interact keeps the sandbox.
12
+ */
13
+ export type RemoteProvider = "browser-use" | "cloudflare";
14
+ export interface RemoteSession {
15
+ provider: RemoteProvider;
16
+ sessionId: string;
17
+ cdpUrl: string;
18
+ /** Headers the CDP websocket needs (Cloudflare wants the bearer token on connect). */
19
+ cdpHeaders?: Record<string, string>;
20
+ startedAt: number;
21
+ }
22
+ export interface RemoteSessionCost {
23
+ /** Provider charge for the session in USD (browser time + proxy traffic). */
24
+ costUsd: number;
25
+ proxyMb: number;
26
+ }
27
+ /** Cloudflare Browser Run: $0.09 per browser hour past the included hours. */
28
+ export declare const CLOUDFLARE_BROWSER_HOUR_USD = 0.09;
29
+ /** The configured remote provider, or null when stealth runs on our own browser. */
30
+ export declare function remoteStealthProvider(): RemoteProvider | null;
31
+ /** Starts a remote browser. `timeoutMs` caps how long the provider keeps it (and bills it) if we never stop it. */
32
+ export declare function startRemoteSession(opts?: {
33
+ provider?: RemoteProvider;
34
+ timeoutMs: number;
35
+ fetchImpl?: typeof fetch;
36
+ }): Promise<RemoteSession>;
37
+ /** Stops the session (unused time is refunded) and returns what it cost. Never throws. */
38
+ export declare function stopRemoteSession(session: Pick<RemoteSession, "sessionId"> & Partial<Pick<RemoteSession, "provider" | "startedAt">>, opts?: {
39
+ fetchImpl?: typeof fetch;
40
+ now?: number;
41
+ }): Promise<RemoteSessionCost | null>;
@@ -0,0 +1,133 @@
1
+ /**
2
+ * Remote CDP browser providers for stealth fetches.
3
+ *
4
+ * A remote provider hosts the browser and hands back a CDP URL. Our step runner
5
+ * (sandbox or local Node) connects to it over CDP, so STEP_SCRIPT runs unchanged.
6
+ *
7
+ * WEBCRAWL_STEALTH_PROVIDER picks one:
8
+ * - browser-use: BROWSER_USE_API_KEY. Custom Chromium + residential proxy ($5/GB).
9
+ * - cloudflare: CLOUDFLARE_ACCOUNT_ID + CLOUDFLARE_BROWSER_TOKEN (Browser Rendering - Edit).
10
+ * Cloudflare Browser Run, $0.09 per browser hour, Cloudflare egress IPs.
11
+ * Only the stealth path (stealthFetch) uses it; /v1/interact keeps the sandbox.
12
+ */
13
+ /** Cloudflare Browser Run: $0.09 per browser hour past the included hours. */
14
+ export const CLOUDFLARE_BROWSER_HOUR_USD = 0.09;
15
+ const BROWSER_USE_API = () => (process.env.BROWSER_USE_API_URL || "https://api.browser-use.com/api/v4").replace(/\/+$/, "");
16
+ function cloudflareCreds() {
17
+ const accountId = (process.env.CLOUDFLARE_ACCOUNT_ID || "").trim();
18
+ const token = (process.env.CLOUDFLARE_BROWSER_TOKEN || process.env.CLOUDFLARE_API_TOKEN || "").trim();
19
+ return accountId && token ? { accountId, token } : null;
20
+ }
21
+ const CF_API = (accountId) => `${(process.env.CLOUDFLARE_API_URL || "https://api.cloudflare.com/client/v4").replace(/\/+$/, "")}/accounts/${encodeURIComponent(accountId)}/browser-run`;
22
+ /** The configured remote provider, or null when stealth runs on our own browser. */
23
+ export function remoteStealthProvider() {
24
+ const name = (process.env.WEBCRAWL_STEALTH_PROVIDER || "").trim().toLowerCase();
25
+ if (name === "browser-use" && process.env.BROWSER_USE_API_KEY)
26
+ return "browser-use";
27
+ if (name === "cloudflare" && cloudflareCreds())
28
+ return "cloudflare";
29
+ return null;
30
+ }
31
+ function browserUseHeaders() {
32
+ return { "X-Browser-Use-API-Key": process.env.BROWSER_USE_API_KEY || "", "Content-Type": "application/json" };
33
+ }
34
+ function num(v) {
35
+ const n = typeof v === "number" ? v : Number.parseFloat(String(v ?? "0"));
36
+ return Number.isFinite(n) ? n : 0;
37
+ }
38
+ function errorDetail(body) {
39
+ const b = (body ?? {});
40
+ if (typeof b.detail === "string")
41
+ return b.detail;
42
+ if (Array.isArray(b.errors) && b.errors[0]?.message)
43
+ return String(b.errors[0].message);
44
+ return b.detail ? JSON.stringify(b.detail).slice(0, 200) : "";
45
+ }
46
+ /** Starts a remote browser. `timeoutMs` caps how long the provider keeps it (and bills it) if we never stop it. */
47
+ export async function startRemoteSession(opts = { timeoutMs: 5 * 60_000 }) {
48
+ const provider = opts.provider ?? "browser-use";
49
+ return provider === "cloudflare" ? startCloudflareSession(opts) : startBrowserUseSession(opts);
50
+ }
51
+ async function startBrowserUseSession(opts) {
52
+ const f = opts.fetchImpl ?? fetch;
53
+ const country = (process.env.BROWSER_USE_PROXY_COUNTRY || "us").trim().toLowerCase();
54
+ const res = await f(`${BROWSER_USE_API()}/browsers`, {
55
+ method: "POST",
56
+ headers: browserUseHeaders(),
57
+ body: JSON.stringify({
58
+ timeout: Math.max(1, Math.min(240, Math.ceil(opts.timeoutMs / 60_000))),
59
+ // "none" turns the residential proxy off (cheaper, weaker against IP reputation checks).
60
+ proxyCountryCode: country === "none" ? null : country,
61
+ solveCaptchas: process.env.BROWSER_USE_SOLVE_CAPTCHAS !== "false",
62
+ metadata: { app: "webcrawl" },
63
+ }),
64
+ signal: AbortSignal.timeout(30_000),
65
+ });
66
+ const body = (await res.json().catch(() => ({})));
67
+ if (!res.ok || !body.id || !body.cdpUrl) {
68
+ const detail = errorDetail(body);
69
+ throw new Error(`Browser Use session failed (HTTP ${res.status})${detail ? `: ${detail}` : ""}`);
70
+ }
71
+ return { provider: "browser-use", sessionId: body.id, cdpUrl: body.cdpUrl, startedAt: Date.now() };
72
+ }
73
+ async function startCloudflareSession(opts) {
74
+ const creds = cloudflareCreds();
75
+ if (!creds)
76
+ throw new Error("Cloudflare Browser Run is not configured");
77
+ const f = opts.fetchImpl ?? fetch;
78
+ // keep_alive is how long the session idles before Cloudflare closes it: 10s..20min.
79
+ const keepAlive = Math.max(10_000, Math.min(1_200_000, opts.timeoutMs));
80
+ const res = await f(`${CF_API(creds.accountId)}/devtools/browser?keep_alive=${keepAlive}`, {
81
+ method: "POST",
82
+ headers: { Authorization: `Bearer ${creds.token}` },
83
+ signal: AbortSignal.timeout(30_000),
84
+ });
85
+ const raw = (await res.json().catch(() => ({})));
86
+ const body = raw.result ?? raw;
87
+ if (!res.ok || !body.sessionId) {
88
+ const detail = errorDetail(raw);
89
+ throw new Error(`Cloudflare Browser Run session failed (HTTP ${res.status})${detail ? `: ${detail}` : ""}`);
90
+ }
91
+ return {
92
+ provider: "cloudflare",
93
+ sessionId: body.sessionId,
94
+ cdpUrl: body.webSocketDebuggerUrl ||
95
+ `${CF_API(creds.accountId).replace(/^http/, "ws")}/devtools/browser/${encodeURIComponent(body.sessionId)}`,
96
+ cdpHeaders: { Authorization: `Bearer ${creds.token}` },
97
+ startedAt: Date.now(),
98
+ };
99
+ }
100
+ /** Stops the session (unused time is refunded) and returns what it cost. Never throws. */
101
+ export async function stopRemoteSession(session, opts = {}) {
102
+ const f = opts.fetchImpl ?? fetch;
103
+ if (session.provider === "cloudflare") {
104
+ const creds = cloudflareCreds();
105
+ if (creds) {
106
+ await f(`${CF_API(creds.accountId)}/devtools/browser/${encodeURIComponent(session.sessionId)}`, {
107
+ method: "DELETE",
108
+ headers: { Authorization: `Bearer ${creds.token}` },
109
+ signal: AbortSignal.timeout(20_000),
110
+ }).catch(() => null);
111
+ }
112
+ // Cloudflare does not report per-session cost; bill browser time (rounded up to a minute).
113
+ if (!session.startedAt)
114
+ return null;
115
+ const minutes = Math.max(1, Math.ceil(((opts.now ?? Date.now()) - session.startedAt) / 60_000));
116
+ return { costUsd: (minutes / 60) * CLOUDFLARE_BROWSER_HOUR_USD, proxyMb: 0 };
117
+ }
118
+ try {
119
+ const res = await f(`${BROWSER_USE_API()}/browsers/${encodeURIComponent(session.sessionId)}`, {
120
+ method: "PATCH",
121
+ headers: browserUseHeaders(),
122
+ body: JSON.stringify({ action: "stop" }),
123
+ signal: AbortSignal.timeout(20_000),
124
+ });
125
+ if (!res.ok)
126
+ return null;
127
+ const body = (await res.json().catch(() => ({})));
128
+ return { costUsd: num(body.proxyCost) + num(body.browserCost), proxyMb: num(body.proxyUsedMb) };
129
+ }
130
+ catch {
131
+ return null;
132
+ }
133
+ }
@@ -0,0 +1,46 @@
1
+ /**
2
+ * Span events streamed from interact / crawl workflow runs and rendered by
3
+ * the trace view. Kept free of Node APIs so the workflow body and the
4
+ * browser bundle can both import it.
5
+ */
6
+ export type SpanStatus = "running" | "ok" | "failed" | "skipped";
7
+ export type SpanKind = "setup" | "open" | "action" | "read" | "map" | "page" | "charge" | "fetch" | "ocr" | "transcribe";
8
+ export interface Span {
9
+ /** Stable id within a run, e.g. `action-3`. Later events with the same id replace earlier ones. */
10
+ id: string;
11
+ kind: SpanKind;
12
+ label: string;
13
+ detail?: string;
14
+ status: SpanStatus;
15
+ /** Milliseconds since the run started. */
16
+ startMs: number;
17
+ durationMs?: number;
18
+ error?: string;
19
+ }
20
+ export type RunEvent = {
21
+ type: "run";
22
+ runId: string;
23
+ kind: "interact" | "crawl" | "parse";
24
+ url: string;
25
+ startedAt: number;
26
+ } | {
27
+ type: "span";
28
+ span: Span;
29
+ } | {
30
+ type: "done";
31
+ status: "completed" | "failed";
32
+ error?: string;
33
+ };
34
+ /** Folds a list of events into the latest state of each span, in first-seen order. */
35
+ export declare function foldSpans(events: RunEvent[]): Span[];
36
+ export interface SpanSummary {
37
+ total: number;
38
+ ok: number;
39
+ failed: number;
40
+ skipped: number;
41
+ running: number;
42
+ durationMs: number;
43
+ /** 1-based index among spans of the given kind, of the first failure. */
44
+ firstFailedIndex?: number;
45
+ }
46
+ export declare function summarizeSpans(spans: Span[], kind?: SpanKind): SpanSummary;
@@ -0,0 +1,32 @@
1
+ /**
2
+ * Span events streamed from interact / crawl workflow runs and rendered by
3
+ * the trace view. Kept free of Node APIs so the workflow body and the
4
+ * browser bundle can both import it.
5
+ */
6
+ /** Folds a list of events into the latest state of each span, in first-seen order. */
7
+ export function foldSpans(events) {
8
+ const order = [];
9
+ const byId = new Map();
10
+ for (const e of events) {
11
+ if (e.type !== "span")
12
+ continue;
13
+ if (!byId.has(e.span.id))
14
+ order.push(e.span.id);
15
+ byId.set(e.span.id, e.span);
16
+ }
17
+ return order.map((id) => byId.get(id));
18
+ }
19
+ export function summarizeSpans(spans, kind) {
20
+ const list = kind ? spans.filter((s) => s.kind === kind) : spans;
21
+ let durationMs = 0;
22
+ let firstFailedIndex;
23
+ const s = { total: list.length, ok: 0, failed: 0, skipped: 0, running: 0 };
24
+ list.forEach((span, i) => {
25
+ s[span.status === "ok" ? "ok" : span.status] += 1;
26
+ if (span.status === "failed" && firstFailedIndex === undefined)
27
+ firstFailedIndex = i + 1;
28
+ });
29
+ for (const span of spans)
30
+ durationMs = Math.max(durationMs, span.startMs + (span.durationMs ?? 0));
31
+ return { ...s, durationMs, firstFailedIndex };
32
+ }
@@ -0,0 +1,34 @@
1
+ /**
2
+ * Input checks for /v1/interact. Pure functions so they can be unit-tested
3
+ * and reused by the API route before a workflow run is started.
4
+ */
5
+ export declare const INTERACT_ACTION_TYPES: readonly ["click", "fill", "type", "hover", "select", "press", "wait", "scroll", "evaluate", "extract"];
6
+ export type InteractActionType = (typeof INTERACT_ACTION_TYPES)[number];
7
+ export declare const MAX_INTERACT_ACTIONS = 20;
8
+ export interface InteractAction {
9
+ type: InteractActionType;
10
+ selector?: string;
11
+ text?: string;
12
+ value?: string;
13
+ key?: string;
14
+ ms?: number;
15
+ y?: number;
16
+ script?: string;
17
+ timeoutMs?: number;
18
+ }
19
+ /** Browser session length: 2 min by default (first minute in the base price), 10 min max. */
20
+ export declare const DEFAULT_SESSION_SECONDS = 120;
21
+ export declare const MAX_SESSION_SECONDS = 600;
22
+ export type ValidateResult = {
23
+ ok: true;
24
+ url: string;
25
+ actions: InteractAction[];
26
+ sessionSeconds: number;
27
+ } | {
28
+ ok: false;
29
+ error: string;
30
+ };
31
+ export declare function normalizeActionType(raw: unknown): InteractActionType | undefined;
32
+ /** Human label used in the trace, e.g. `Click #submit`. */
33
+ export declare function describeAction(a: InteractAction): string;
34
+ export declare function validateInteractRequest(body: Record<string, unknown>): ValidateResult;