surfsky 0.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +471 -0
- package/dist/index.d.ts +1722 -0
- package/dist/index.js +2754 -0
- package/package.json +64 -0
package/dist/index.js
ADDED
|
@@ -0,0 +1,2754 @@
|
|
|
1
|
+
import { z } from "zod";
|
|
2
|
+
//#region src/browser/actions.ts
|
|
3
|
+
function cdpParams(given) {
|
|
4
|
+
const params = {};
|
|
5
|
+
for (const [key, value] of Object.entries(given)) if (value !== void 0) params[key] = value;
|
|
6
|
+
return params;
|
|
7
|
+
}
|
|
8
|
+
var Keyboard = class {
|
|
9
|
+
#send;
|
|
10
|
+
constructor(send) {
|
|
11
|
+
this.#send = send;
|
|
12
|
+
}
|
|
13
|
+
type(text) {
|
|
14
|
+
return this.#send("Human.type", { text });
|
|
15
|
+
}
|
|
16
|
+
press(key, options = {}) {
|
|
17
|
+
return this.#send("Human.press", cdpParams({
|
|
18
|
+
key,
|
|
19
|
+
...options
|
|
20
|
+
}));
|
|
21
|
+
}
|
|
22
|
+
};
|
|
23
|
+
var Mouse = class {
|
|
24
|
+
#send;
|
|
25
|
+
constructor(send) {
|
|
26
|
+
this.#send = send;
|
|
27
|
+
}
|
|
28
|
+
move(x, y) {
|
|
29
|
+
return this.#send("Human.moveTo", {
|
|
30
|
+
x,
|
|
31
|
+
y
|
|
32
|
+
});
|
|
33
|
+
}
|
|
34
|
+
click(x, y, options = {}) {
|
|
35
|
+
return this.#send("Human.click", cdpParams({
|
|
36
|
+
x,
|
|
37
|
+
y,
|
|
38
|
+
...options
|
|
39
|
+
}));
|
|
40
|
+
}
|
|
41
|
+
down(x, y, options = {}) {
|
|
42
|
+
return this.#send("Human.mouseDown", cdpParams({
|
|
43
|
+
x,
|
|
44
|
+
y,
|
|
45
|
+
...options
|
|
46
|
+
}));
|
|
47
|
+
}
|
|
48
|
+
up(x, y, options = {}) {
|
|
49
|
+
return this.#send("Human.mouseUp", cdpParams({
|
|
50
|
+
x,
|
|
51
|
+
y,
|
|
52
|
+
...options
|
|
53
|
+
}));
|
|
54
|
+
}
|
|
55
|
+
wheel(options = {}) {
|
|
56
|
+
return this.#send("Human.wheel", cdpParams({ ...options }));
|
|
57
|
+
}
|
|
58
|
+
drag(options) {
|
|
59
|
+
return this.#send("Human.drag", cdpParams({ ...options }));
|
|
60
|
+
}
|
|
61
|
+
};
|
|
62
|
+
var Actions = class {
|
|
63
|
+
keyboard;
|
|
64
|
+
mouse;
|
|
65
|
+
constructor() {
|
|
66
|
+
const send = (method, params) => this.send(method, params);
|
|
67
|
+
this.keyboard = new Keyboard(send);
|
|
68
|
+
this.mouse = new Mouse(send);
|
|
69
|
+
}
|
|
70
|
+
click(selector, options = {}) {
|
|
71
|
+
return this.send("Human.click", cdpParams({
|
|
72
|
+
selector,
|
|
73
|
+
...options
|
|
74
|
+
}));
|
|
75
|
+
}
|
|
76
|
+
dblclick(selector, options = {}) {
|
|
77
|
+
return this.send("Human.dblclick", cdpParams({
|
|
78
|
+
selector,
|
|
79
|
+
...options
|
|
80
|
+
}));
|
|
81
|
+
}
|
|
82
|
+
hover(selector, options = {}) {
|
|
83
|
+
return this.send("Human.moveTo", cdpParams({
|
|
84
|
+
selector,
|
|
85
|
+
...options
|
|
86
|
+
}));
|
|
87
|
+
}
|
|
88
|
+
async type(selector, text) {
|
|
89
|
+
await this.click(selector);
|
|
90
|
+
return this.keyboard.type(text);
|
|
91
|
+
}
|
|
92
|
+
async fill(selector, text) {
|
|
93
|
+
await this.click(selector, { clickCount: 3 });
|
|
94
|
+
return this.keyboard.type(text);
|
|
95
|
+
}
|
|
96
|
+
scroll(options = {}) {
|
|
97
|
+
return this.send("Human.scroll", cdpParams({ ...options }));
|
|
98
|
+
}
|
|
99
|
+
scrollIntoView(selector, options = {}) {
|
|
100
|
+
return this.send("Human.scrollIntoView", cdpParams({
|
|
101
|
+
selector,
|
|
102
|
+
...options
|
|
103
|
+
}));
|
|
104
|
+
}
|
|
105
|
+
scrollTo(options = {}) {
|
|
106
|
+
return this.send("Human.scrollTo", cdpParams({ ...options }));
|
|
107
|
+
}
|
|
108
|
+
};
|
|
109
|
+
//#endregion
|
|
110
|
+
//#region src/errors.ts
|
|
111
|
+
var SurfskyError = class extends Error {
|
|
112
|
+
constructor(message, options) {
|
|
113
|
+
super(message, options);
|
|
114
|
+
this.name = new.target.name;
|
|
115
|
+
}
|
|
116
|
+
};
|
|
117
|
+
/** No API token, or another unusable client setting. */
|
|
118
|
+
var ConfigurationError = class extends SurfskyError {};
|
|
119
|
+
/** A request that failed validation before it was sent. `cause` is the ZodError. */
|
|
120
|
+
var ValidationError = class extends SurfskyError {
|
|
121
|
+
issues;
|
|
122
|
+
constructor(message, error) {
|
|
123
|
+
super(message, { cause: error });
|
|
124
|
+
this.issues = error.issues;
|
|
125
|
+
}
|
|
126
|
+
};
|
|
127
|
+
/** An error response from the API, or a request that never got one. */
|
|
128
|
+
var APIError = class extends SurfskyError {
|
|
129
|
+
statusCode;
|
|
130
|
+
body;
|
|
131
|
+
code;
|
|
132
|
+
requestId;
|
|
133
|
+
headers;
|
|
134
|
+
retryAfter;
|
|
135
|
+
constructor(message, options = {}) {
|
|
136
|
+
super(message, options.cause === void 0 ? void 0 : { cause: options.cause });
|
|
137
|
+
this.statusCode = options.statusCode;
|
|
138
|
+
this.body = options.body;
|
|
139
|
+
this.code = errorCode$1(options.body);
|
|
140
|
+
this.requestId = options.requestId;
|
|
141
|
+
this.headers = options.headers;
|
|
142
|
+
this.retryAfter = options.retryAfter;
|
|
143
|
+
}
|
|
144
|
+
toString() {
|
|
145
|
+
if (this.statusCode === void 0) return this.message;
|
|
146
|
+
return `[${this.statusCode}] ${this.message}`;
|
|
147
|
+
}
|
|
148
|
+
};
|
|
149
|
+
function errorCode$1(body) {
|
|
150
|
+
if (typeof body === "object" && body !== null && "code" in body) {
|
|
151
|
+
const code = body.code;
|
|
152
|
+
return typeof code === "string" ? code : void 0;
|
|
153
|
+
}
|
|
154
|
+
}
|
|
155
|
+
/** The request never reached the API. */
|
|
156
|
+
var APIConnectionError = class extends APIError {};
|
|
157
|
+
/** No response within the timeout. */
|
|
158
|
+
var APITimeoutError = class extends APIConnectionError {};
|
|
159
|
+
/** 400 or 422. */
|
|
160
|
+
var BadRequestError = class extends APIError {};
|
|
161
|
+
/** 401, or a 422 that rejects the token header. */
|
|
162
|
+
var AuthenticationError = class extends APIError {};
|
|
163
|
+
/** 402. */
|
|
164
|
+
var PaymentRequiredError = class extends APIError {};
|
|
165
|
+
/** 403. */
|
|
166
|
+
var ForbiddenError = class extends APIError {};
|
|
167
|
+
/** 404. */
|
|
168
|
+
var NotFoundError = class extends APIError {};
|
|
169
|
+
/** 409. */
|
|
170
|
+
var ConflictError = class extends APIError {};
|
|
171
|
+
/** 429. `retryAfter` is the server's hint in ms. */
|
|
172
|
+
var RateLimitError = class extends APIError {};
|
|
173
|
+
/** 429 with `shared_traffic_limit_reached`. */
|
|
174
|
+
var SharedTrafficLimitError = class extends APIError {};
|
|
175
|
+
/** 429 with `premium_traffic_limit_reached`. */
|
|
176
|
+
var PremiumTrafficLimitError = class extends APIError {};
|
|
177
|
+
/** 429 with `monthly_session_limit_reached`. */
|
|
178
|
+
var MonthlySessionLimitError = class extends APIError {};
|
|
179
|
+
/** 5xx. */
|
|
180
|
+
var ServerError = class extends APIError {};
|
|
181
|
+
/** An error reply to a CDP command, or a lost connection. */
|
|
182
|
+
var CDPError = class extends SurfskyError {
|
|
183
|
+
code;
|
|
184
|
+
constructor(message, options = {}) {
|
|
185
|
+
super(message, options.cause === void 0 ? void 0 : { cause: options.cause });
|
|
186
|
+
this.code = options.code;
|
|
187
|
+
}
|
|
188
|
+
};
|
|
189
|
+
/** A browser wait or command ran out of time. */
|
|
190
|
+
var BrowserTimeoutError = class extends SurfskyError {};
|
|
191
|
+
/** The page is gone. */
|
|
192
|
+
var PageClosedError = class extends SurfskyError {};
|
|
193
|
+
//#endregion
|
|
194
|
+
//#region src/util.ts
|
|
195
|
+
/** Resolve after `ms`, or right away once `signal` aborts. */
|
|
196
|
+
function sleep(ms, signal) {
|
|
197
|
+
return new Promise((resolve) => {
|
|
198
|
+
if (signal?.aborted) return resolve();
|
|
199
|
+
const timer = setTimeout(() => {
|
|
200
|
+
signal?.removeEventListener("abort", onAbort);
|
|
201
|
+
resolve();
|
|
202
|
+
}, ms);
|
|
203
|
+
function onAbort() {
|
|
204
|
+
clearTimeout(timer);
|
|
205
|
+
resolve();
|
|
206
|
+
}
|
|
207
|
+
signal?.addEventListener("abort", onAbort, { once: true });
|
|
208
|
+
});
|
|
209
|
+
}
|
|
210
|
+
/** Race a promise against a deadline; the timer is cleared either way. */
|
|
211
|
+
async function withTimeout(promise, ms, onTimeout) {
|
|
212
|
+
if (ms === null || ms === void 0) return promise;
|
|
213
|
+
let timer;
|
|
214
|
+
const deadline = new Promise((_, reject) => {
|
|
215
|
+
timer = setTimeout(() => reject(onTimeout()), ms);
|
|
216
|
+
});
|
|
217
|
+
try {
|
|
218
|
+
return await Promise.race([promise, deadline]);
|
|
219
|
+
} finally {
|
|
220
|
+
clearTimeout(timer);
|
|
221
|
+
}
|
|
222
|
+
}
|
|
223
|
+
/** A one-shot event: `wait()` resolves once `set()` was called. */
|
|
224
|
+
var Flag = class {
|
|
225
|
+
#resolvers = Promise.withResolvers();
|
|
226
|
+
#set = false;
|
|
227
|
+
get isSet() {
|
|
228
|
+
return this.#set;
|
|
229
|
+
}
|
|
230
|
+
set() {
|
|
231
|
+
if (this.#set) return;
|
|
232
|
+
this.#set = true;
|
|
233
|
+
this.#resolvers.resolve();
|
|
234
|
+
}
|
|
235
|
+
wait() {
|
|
236
|
+
return this.#resolvers.promise;
|
|
237
|
+
}
|
|
238
|
+
};
|
|
239
|
+
/** Drop `undefined` and `null` values, recursively: what pydantic's `exclude_none` did. */
|
|
240
|
+
function dropNulls(value) {
|
|
241
|
+
if (Array.isArray(value)) return value.map(dropNulls);
|
|
242
|
+
if (typeof value !== "object" || value === null) return value;
|
|
243
|
+
if (value instanceof Blob || value instanceof FormData) return value;
|
|
244
|
+
return Object.fromEntries(Object.entries(value).filter(([, item]) => item !== void 0 && item !== null).map(([key, item]) => [key, dropNulls(item)]));
|
|
245
|
+
}
|
|
246
|
+
function fromBase64(text) {
|
|
247
|
+
if (typeof Buffer !== "undefined") return new Uint8Array(Buffer.from(text, "base64"));
|
|
248
|
+
return Uint8Array.from(atob(text), (char) => char.charCodeAt(0));
|
|
249
|
+
}
|
|
250
|
+
//#endregion
|
|
251
|
+
//#region src/types.ts
|
|
252
|
+
z.enum([
|
|
253
|
+
"win",
|
|
254
|
+
"mac",
|
|
255
|
+
"android"
|
|
256
|
+
]);
|
|
257
|
+
z.enum(["x86", "arm"]);
|
|
258
|
+
z.enum(["phone", "tablet"]);
|
|
259
|
+
const waitUntil = z.enum([
|
|
260
|
+
"domcontentloaded",
|
|
261
|
+
"load",
|
|
262
|
+
"networkidle",
|
|
263
|
+
"commit"
|
|
264
|
+
]);
|
|
265
|
+
const proxyType = z.enum(["residential", "mobile"]);
|
|
266
|
+
const profileOrdering = z.enum([
|
|
267
|
+
"created",
|
|
268
|
+
"-created",
|
|
269
|
+
"active",
|
|
270
|
+
"-active",
|
|
271
|
+
"title",
|
|
272
|
+
"-title"
|
|
273
|
+
]);
|
|
274
|
+
const regionalPool = z.enum([
|
|
275
|
+
"western",
|
|
276
|
+
"europe",
|
|
277
|
+
"westeurope",
|
|
278
|
+
"northamerica",
|
|
279
|
+
"southamerica",
|
|
280
|
+
"asia",
|
|
281
|
+
"centralasia",
|
|
282
|
+
"southasia",
|
|
283
|
+
"eastasia",
|
|
284
|
+
"sea",
|
|
285
|
+
"oceania",
|
|
286
|
+
"mena"
|
|
287
|
+
]);
|
|
288
|
+
const loose = () => z.string();
|
|
289
|
+
function isRecord(value) {
|
|
290
|
+
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
291
|
+
}
|
|
292
|
+
const noise = z.looseObject({
|
|
293
|
+
webgl: z.boolean().nullish(),
|
|
294
|
+
canvas: z.boolean().nullish(),
|
|
295
|
+
audio: z.boolean().nullish(),
|
|
296
|
+
client_rects: z.boolean().nullish()
|
|
297
|
+
});
|
|
298
|
+
const mediaDevices = z.looseObject({
|
|
299
|
+
video_in: z.number().nullish(),
|
|
300
|
+
audio_out: z.number().nullish(),
|
|
301
|
+
audio_in: z.number().nullish()
|
|
302
|
+
});
|
|
303
|
+
const geolocation = z.looseObject({
|
|
304
|
+
latitude: z.number().nullish(),
|
|
305
|
+
longitude: z.number().nullish(),
|
|
306
|
+
accuracy: z.number().nullish()
|
|
307
|
+
});
|
|
308
|
+
const fingerprint = z.looseObject({
|
|
309
|
+
os: loose().nullish(),
|
|
310
|
+
os_arch: loose().nullish(),
|
|
311
|
+
os_version: z.string().nullish(),
|
|
312
|
+
device_model: z.string().nullish(),
|
|
313
|
+
device_type: loose().nullish(),
|
|
314
|
+
user_agent: z.string().nullish(),
|
|
315
|
+
cpu: z.number().nullish(),
|
|
316
|
+
ram: z.number().nullish(),
|
|
317
|
+
renderer: z.string().nullish(),
|
|
318
|
+
noise: noise.nullish(),
|
|
319
|
+
media_devices: mediaDevices.nullish(),
|
|
320
|
+
screen: z.string().nullish(),
|
|
321
|
+
languages: z.array(z.string()).nullish(),
|
|
322
|
+
timezone: z.string().nullish(),
|
|
323
|
+
geolocation: geolocation.nullish(),
|
|
324
|
+
dns: z.string().nullish()
|
|
325
|
+
});
|
|
326
|
+
const storageOptions = z.looseObject({
|
|
327
|
+
cookies: z.boolean().nullish(),
|
|
328
|
+
passwords: z.boolean().nullish(),
|
|
329
|
+
extensions: z.boolean().nullish(),
|
|
330
|
+
localstorage: z.boolean().nullish(),
|
|
331
|
+
history: z.boolean().nullish(),
|
|
332
|
+
bookmarks: z.boolean().nullish(),
|
|
333
|
+
serviceworkers: z.boolean().nullish()
|
|
334
|
+
});
|
|
335
|
+
const browserSettings = z.strictObject({
|
|
336
|
+
inactive_kill_timeout: z.number().int().optional(),
|
|
337
|
+
cache_enabled: z.boolean().optional(),
|
|
338
|
+
cache_key: z.string().optional()
|
|
339
|
+
});
|
|
340
|
+
const shared = z.strictObject({
|
|
341
|
+
tier: z.literal("shared"),
|
|
342
|
+
country: z.string().min(1).optional()
|
|
343
|
+
});
|
|
344
|
+
const premium = z.strictObject({
|
|
345
|
+
tier: z.literal("premium").optional(),
|
|
346
|
+
country: z.string().min(1).optional(),
|
|
347
|
+
region: z.string().min(1).optional(),
|
|
348
|
+
city: z.string().min(1).optional(),
|
|
349
|
+
type: proxyType.optional(),
|
|
350
|
+
pool: regionalPool.optional(),
|
|
351
|
+
asn: z.number().int().min(1).max(4294967295).optional(),
|
|
352
|
+
lat: z.number().min(-90).max(90).optional(),
|
|
353
|
+
lon: z.number().min(-180).max(180).optional(),
|
|
354
|
+
session_minutes: z.number().int().min(1).max(10080).optional(),
|
|
355
|
+
unique_ip: z.boolean().optional(),
|
|
356
|
+
keep_asn: z.boolean().optional(),
|
|
357
|
+
keep_ip: z.boolean().optional()
|
|
358
|
+
}).superRefine((p, ctx) => {
|
|
359
|
+
const gps = p.lat !== void 0 || p.lon !== void 0;
|
|
360
|
+
const set = (v) => v !== void 0;
|
|
361
|
+
const geo = set(p.country) || set(p.region) || set(p.city) || set(p.asn);
|
|
362
|
+
if ([
|
|
363
|
+
set(p.pool),
|
|
364
|
+
gps,
|
|
365
|
+
geo
|
|
366
|
+
].filter(Boolean).length > 1) ctx.addIssue("'pool', 'lat'/'lon' and 'country'/'region'/'city'/'asn' are 3 separate targeting modes - use one!");
|
|
367
|
+
if (gps && (p.lat === void 0 || p.lon === void 0)) ctx.addIssue("'lat' and 'lon' must be sent together");
|
|
368
|
+
if (set(p.region) && !set(p.country)) ctx.addIssue("region targeting requires country");
|
|
369
|
+
if (set(p.city) && !set(p.region)) ctx.addIssue("city targeting requires region");
|
|
370
|
+
if (set(p.asn) && !set(p.country)) ctx.addIssue("asn targeting requires country");
|
|
371
|
+
});
|
|
372
|
+
const proxySelector = z.discriminatedUnion("tier", [shared, premium], { error: (issue) => issue.code === "invalid_union" ? "tier must be \"shared\" or \"premium\"" : void 0 });
|
|
373
|
+
const proxyLike = z.custom().superRefine((value, ctx) => {
|
|
374
|
+
if (typeof value === "string") return;
|
|
375
|
+
const proto = isRecord(value) ? Object.getPrototypeOf(value) : null;
|
|
376
|
+
if (proto !== null && proto !== Object.prototype) {
|
|
377
|
+
ctx.addIssue("a proxy is a URL string or a plain selector object");
|
|
378
|
+
return;
|
|
379
|
+
}
|
|
380
|
+
for (const issue of proxySelector.safeParse(value).error?.issues ?? []) ctx.addIssue({ ...issue });
|
|
381
|
+
});
|
|
382
|
+
const domainRoute = z.strictObject({
|
|
383
|
+
proxy: z.string(),
|
|
384
|
+
domain: z.array(z.string()).optional(),
|
|
385
|
+
domain_suffix: z.array(z.string()).optional(),
|
|
386
|
+
domain_keyword: z.array(z.string()).optional(),
|
|
387
|
+
domain_regex: z.array(z.string()).optional()
|
|
388
|
+
});
|
|
389
|
+
const cookieInput = z.union([z.string(), z.array(z.record(z.string(), z.unknown()))], { error: "expected a cookie string or an array of cookie objects" });
|
|
390
|
+
const oneTimeStartRequest = z.strictObject({
|
|
391
|
+
fingerprint: fingerprint.optional(),
|
|
392
|
+
proxy: proxyLike.nullish(),
|
|
393
|
+
browser_settings: browserSettings.optional(),
|
|
394
|
+
enable_chromedriver: z.boolean().optional(),
|
|
395
|
+
extensions: z.array(z.string()).max(5).optional(),
|
|
396
|
+
proxy_blacklist: z.array(z.string()).optional(),
|
|
397
|
+
domain_routes: z.array(domainRoute).optional(),
|
|
398
|
+
cookies: cookieInput.optional()
|
|
399
|
+
});
|
|
400
|
+
const oneTimeOnly = (field) => z.undefined({ error: `${field} applies to one-time sessions only` }).nullish();
|
|
401
|
+
const profileStartRequest = oneTimeStartRequest.extend({
|
|
402
|
+
fingerprint: oneTimeOnly("fingerprint"),
|
|
403
|
+
cookies: oneTimeOnly("cookies")
|
|
404
|
+
});
|
|
405
|
+
const profileCreateRequest = z.strictObject({
|
|
406
|
+
title: z.string().min(1),
|
|
407
|
+
fingerprint,
|
|
408
|
+
description: z.string().optional(),
|
|
409
|
+
proxy: proxyLike.nullish(),
|
|
410
|
+
cookies: cookieInput.optional(),
|
|
411
|
+
storage_options: storageOptions.optional()
|
|
412
|
+
});
|
|
413
|
+
const profileUpdateRequest = z.strictObject({
|
|
414
|
+
title: z.string().optional(),
|
|
415
|
+
/** `null` clears it. */
|
|
416
|
+
description: z.string().nullish(),
|
|
417
|
+
proxy: proxyLike.nullish(),
|
|
418
|
+
fingerprint: fingerprint.transform(({ os, os_arch, os_version, device_model, device_type, ...rest }) => rest).optional(),
|
|
419
|
+
storage_options: storageOptions.nullish()
|
|
420
|
+
});
|
|
421
|
+
const listPageRequest = z.strictObject({
|
|
422
|
+
page: z.number().int().min(0).optional(),
|
|
423
|
+
page_len: z.number().int().min(1).optional(),
|
|
424
|
+
ordering: profileOrdering.optional()
|
|
425
|
+
});
|
|
426
|
+
const scrapeRequest = z.strictObject({
|
|
427
|
+
url: z.string(),
|
|
428
|
+
screenshot: z.boolean().optional(),
|
|
429
|
+
/** Seconds, 0-60. */
|
|
430
|
+
wait: z.number().min(0).max(60).optional(),
|
|
431
|
+
wait_until: waitUntil.optional(),
|
|
432
|
+
wait_for: z.string().optional(),
|
|
433
|
+
/** 0-3. */
|
|
434
|
+
human_actions: z.number().int().min(0).max(3).optional()
|
|
435
|
+
});
|
|
436
|
+
const parser = (schema, what) => (value) => {
|
|
437
|
+
const result = schema.safeParse(value);
|
|
438
|
+
if (result.success) return dropNulls(result.data);
|
|
439
|
+
throw new ValidationError(`${what}: ${z.prettifyError(result.error)}`, result.error);
|
|
440
|
+
};
|
|
441
|
+
const parseOneTimeStartRequest = parser(oneTimeStartRequest, "start_one_time");
|
|
442
|
+
const parseProfileStartRequest = parser(profileStartRequest, "start");
|
|
443
|
+
const parseProfileCreateRequest = parser(profileCreateRequest, "create");
|
|
444
|
+
const parseProfileUpdateRequest = parser(profileUpdateRequest, "update");
|
|
445
|
+
const parseScrapeRequest = parser(scrapeRequest, "scrape");
|
|
446
|
+
const parseListPageRequest = parser(listPageRequest, "listPage");
|
|
447
|
+
function parseProxyLike(value, what = "proxy") {
|
|
448
|
+
return parser(proxyLike, what)(value);
|
|
449
|
+
}
|
|
450
|
+
//#endregion
|
|
451
|
+
//#region src/transport.ts
|
|
452
|
+
const AUTH_HEADER = "X-Cloud-Api-Token";
|
|
453
|
+
const MAX_BACKOFF = 3e4;
|
|
454
|
+
const STATUS_ERRORS = {
|
|
455
|
+
400: BadRequestError,
|
|
456
|
+
401: AuthenticationError,
|
|
457
|
+
402: PaymentRequiredError,
|
|
458
|
+
403: ForbiddenError,
|
|
459
|
+
404: NotFoundError,
|
|
460
|
+
409: ConflictError,
|
|
461
|
+
422: BadRequestError,
|
|
462
|
+
429: RateLimitError
|
|
463
|
+
};
|
|
464
|
+
const QUOTA_ERRORS = {
|
|
465
|
+
shared_traffic_limit_reached: SharedTrafficLimitError,
|
|
466
|
+
premium_traffic_limit_reached: PremiumTrafficLimitError,
|
|
467
|
+
monthly_session_limit_reached: MonthlySessionLimitError
|
|
468
|
+
};
|
|
469
|
+
const PLAN_FULL = "parallel_browsers_limit_reached";
|
|
470
|
+
const NO_RETRY_CODES = /* @__PURE__ */ new Set([...Object.keys(QUOTA_ERRORS), PLAN_FULL]);
|
|
471
|
+
const IDEMPOTENT = /* @__PURE__ */ new Set([
|
|
472
|
+
"GET",
|
|
473
|
+
"HEAD",
|
|
474
|
+
"PUT",
|
|
475
|
+
"DELETE",
|
|
476
|
+
"OPTIONS"
|
|
477
|
+
]);
|
|
478
|
+
const RETRY_STATUSES = /* @__PURE__ */ new Set([
|
|
479
|
+
500,
|
|
480
|
+
502,
|
|
481
|
+
503,
|
|
482
|
+
504
|
|
483
|
+
]);
|
|
484
|
+
const NEVER_SENT = /* @__PURE__ */ new Set([
|
|
485
|
+
"ECONNREFUSED",
|
|
486
|
+
"ENOTFOUND",
|
|
487
|
+
"EAI_AGAIN",
|
|
488
|
+
"UND_ERR_CONNECT_TIMEOUT",
|
|
489
|
+
"ConnectionRefused",
|
|
490
|
+
"FailedToOpenSocket",
|
|
491
|
+
"DNSError"
|
|
492
|
+
]);
|
|
493
|
+
const noop = () => {};
|
|
494
|
+
/** Warnings and errors go to the console unless told otherwise; `null` silences all. */
|
|
495
|
+
function makeLogger(given) {
|
|
496
|
+
if (given === null) return {
|
|
497
|
+
debug: noop,
|
|
498
|
+
info: noop,
|
|
499
|
+
warn: noop,
|
|
500
|
+
error: noop
|
|
501
|
+
};
|
|
502
|
+
return {
|
|
503
|
+
debug: given?.debug ?? noop,
|
|
504
|
+
info: given?.info ?? noop,
|
|
505
|
+
warn: given?.warn ?? ((message) => console.warn(message)),
|
|
506
|
+
error: given?.error ?? ((message) => console.error(message))
|
|
507
|
+
};
|
|
508
|
+
}
|
|
509
|
+
const defaultSleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
|
|
510
|
+
function buildUrl(baseUrl, path, params) {
|
|
511
|
+
const url = new URL(baseUrl + path);
|
|
512
|
+
for (const [key, value] of Object.entries(params ?? {})) if (value !== void 0) url.searchParams.set(key, String(value));
|
|
513
|
+
return url.toString();
|
|
514
|
+
}
|
|
515
|
+
/** Send with retries. Throws only when no response ever came back. */
|
|
516
|
+
async function send(spec, options) {
|
|
517
|
+
const method = spec.method.toUpperCase();
|
|
518
|
+
const url = buildUrl(options.baseUrl, spec.path, spec.params);
|
|
519
|
+
const idempotent = IDEMPOTENT.has(method);
|
|
520
|
+
const headers = { ...options.headers };
|
|
521
|
+
let body = spec.body;
|
|
522
|
+
if (spec.json !== void 0) {
|
|
523
|
+
body = JSON.stringify(spec.json);
|
|
524
|
+
headers["Content-Type"] = "application/json";
|
|
525
|
+
}
|
|
526
|
+
const timeout = spec.timeout ?? options.timeout;
|
|
527
|
+
const sleep = options.sleep ?? defaultSleep;
|
|
528
|
+
const random = options.random ?? Math.random;
|
|
529
|
+
for (let attempt = 0;; attempt++) {
|
|
530
|
+
let response;
|
|
531
|
+
let error;
|
|
532
|
+
try {
|
|
533
|
+
response = await options.fetch(url, {
|
|
534
|
+
method,
|
|
535
|
+
headers,
|
|
536
|
+
body,
|
|
537
|
+
signal: timeout > 0 ? AbortSignal.timeout(timeout) : void 0
|
|
538
|
+
});
|
|
539
|
+
} catch (err) {
|
|
540
|
+
error = err;
|
|
541
|
+
}
|
|
542
|
+
const last = attempt >= options.retries;
|
|
543
|
+
let delay;
|
|
544
|
+
if (response !== void 0) {
|
|
545
|
+
if (last || !await retryResponse(response, idempotent)) return response;
|
|
546
|
+
const hinted = retryAfterMs(response);
|
|
547
|
+
delay = hinted === void 0 ? backoffDelay(attempt, options.backoff, random) : Math.min(hinted, MAX_BACKOFF);
|
|
548
|
+
await response.body?.cancel();
|
|
549
|
+
} else {
|
|
550
|
+
if (last || !retryError(error, idempotent)) {
|
|
551
|
+
if (!isTransportError(error)) throw error;
|
|
552
|
+
throw networkError(spec.path, error);
|
|
553
|
+
}
|
|
554
|
+
delay = backoffDelay(attempt, options.backoff, random);
|
|
555
|
+
}
|
|
556
|
+
const got = response !== void 0 ? response.status : describe(error);
|
|
557
|
+
options.logger.warn(`${method} ${spec.path} -> ${got}, retry in ${(delay / 1e3).toFixed(1)}s`);
|
|
558
|
+
await sleep(delay);
|
|
559
|
+
}
|
|
560
|
+
}
|
|
561
|
+
function backoffDelay(attempt, backoff, random) {
|
|
562
|
+
return Math.min(backoff * 2 ** attempt, MAX_BACKOFF) + random() * backoff;
|
|
563
|
+
}
|
|
564
|
+
async function retryResponse(response, idempotent) {
|
|
565
|
+
if (response.status === 429) return !neverRetried(await parseBody(response.clone()));
|
|
566
|
+
return idempotent && RETRY_STATUSES.has(response.status);
|
|
567
|
+
}
|
|
568
|
+
function errorCode(error) {
|
|
569
|
+
if (!isRecord(error)) return void 0;
|
|
570
|
+
const cause = error.cause;
|
|
571
|
+
if (isRecord(cause) && typeof cause.code === "string") return cause.code;
|
|
572
|
+
return typeof error.code === "string" ? error.code : void 0;
|
|
573
|
+
}
|
|
574
|
+
const TIMEOUT_CODES = /* @__PURE__ */ new Set(["UND_ERR_CONNECT_TIMEOUT", "ConnectionTimeout"]);
|
|
575
|
+
function isTimeout(error) {
|
|
576
|
+
if (error instanceof Error && (error.name === "TimeoutError" || error.name === "AbortError")) return true;
|
|
577
|
+
const code = errorCode(error);
|
|
578
|
+
return code !== void 0 && TIMEOUT_CODES.has(code);
|
|
579
|
+
}
|
|
580
|
+
/** A failure of the transport itself, as opposed to a bad argument thrown by fetch. */
|
|
581
|
+
function isTransportError(error) {
|
|
582
|
+
if (isTimeout(error) || errorCode(error) !== void 0) return true;
|
|
583
|
+
return error instanceof TypeError && error.cause !== void 0;
|
|
584
|
+
}
|
|
585
|
+
function retryError(error, idempotent) {
|
|
586
|
+
const code = errorCode(error);
|
|
587
|
+
if (code !== void 0 && NEVER_SENT.has(code)) return true;
|
|
588
|
+
return idempotent && isTransportError(error);
|
|
589
|
+
}
|
|
590
|
+
function describe(error) {
|
|
591
|
+
return error instanceof Error ? `${error.name}: ${error.message}` : String(error);
|
|
592
|
+
}
|
|
593
|
+
/** JSON when it parses, the text otherwise, `null` when empty. */
|
|
594
|
+
async function parseBody(response) {
|
|
595
|
+
const text = await response.text();
|
|
596
|
+
if (!text) return null;
|
|
597
|
+
try {
|
|
598
|
+
return JSON.parse(text);
|
|
599
|
+
} catch {
|
|
600
|
+
return text;
|
|
601
|
+
}
|
|
602
|
+
}
|
|
603
|
+
/** Read a reply: map an error status, unwrap `{success, msg, data}`, parse. */
|
|
604
|
+
async function result(spec, response) {
|
|
605
|
+
const status = response.status;
|
|
606
|
+
const body = await parseBody(response);
|
|
607
|
+
if (status >= 400 && !spec.acceptError?.(status, body)) throw apiError(spec.method, spec.path, response, body);
|
|
608
|
+
let data = body;
|
|
609
|
+
if (isRecord(body) && "data" in body) {
|
|
610
|
+
if (body.success === false && spec.acceptError === void 0) throw new APIError(String(body.msg || "request failed"), { body });
|
|
611
|
+
data = body.data;
|
|
612
|
+
}
|
|
613
|
+
try {
|
|
614
|
+
return spec.parse ? spec.parse(data) : data;
|
|
615
|
+
} catch (err) {
|
|
616
|
+
throw new APIError(`unexpected response from ${spec.method} ${spec.path}: ${describe(err)}`, {
|
|
617
|
+
statusCode: status,
|
|
618
|
+
body,
|
|
619
|
+
requestId: requestId(response),
|
|
620
|
+
headers: headersOf(response),
|
|
621
|
+
cause: err
|
|
622
|
+
});
|
|
623
|
+
}
|
|
624
|
+
}
|
|
625
|
+
function neverRetried(body) {
|
|
626
|
+
return isRecord(body) && typeof body.code === "string" && NO_RETRY_CODES.has(body.code);
|
|
627
|
+
}
|
|
628
|
+
/** The Retry-After header in ms: seconds or an HTTP date. */
|
|
629
|
+
function retryAfterMs(response) {
|
|
630
|
+
const value = response.headers.get("Retry-After")?.trim();
|
|
631
|
+
if (!value) return void 0;
|
|
632
|
+
if (/^\d+$/.test(value)) return Number(value) * 1e3;
|
|
633
|
+
const when = Date.parse(value);
|
|
634
|
+
if (Number.isNaN(when)) return void 0;
|
|
635
|
+
return Math.max(0, when - Date.now());
|
|
636
|
+
}
|
|
637
|
+
function requestId(response) {
|
|
638
|
+
return response.headers.get("cf-ray") ?? void 0;
|
|
639
|
+
}
|
|
640
|
+
function headersOf(response) {
|
|
641
|
+
return Object.fromEntries(response.headers);
|
|
642
|
+
}
|
|
643
|
+
function tokenRejected(body) {
|
|
644
|
+
if (!isRecord(body) || !isRecord(body.data) || !Array.isArray(body.data.errors)) return false;
|
|
645
|
+
return body.data.errors.some((error) => isRecord(error) && Array.isArray(error.loc) && error.loc.some((part) => String(part).toLowerCase() === "X-Cloud-Api-Token".toLowerCase()));
|
|
646
|
+
}
|
|
647
|
+
/** Escape one path segment. An empty id would silently hit the list endpoint. */
|
|
648
|
+
function ref(value) {
|
|
649
|
+
if (!value) throw new TypeError("an empty id would address a different endpoint");
|
|
650
|
+
if (value === "." || value === "..") throw new TypeError(`'${value}' is not a path segment`);
|
|
651
|
+
return encodeURIComponent(value);
|
|
652
|
+
}
|
|
653
|
+
function fieldErrors(details) {
|
|
654
|
+
if (Array.isArray(details)) return details.filter(isRecord).map((e) => `${(Array.isArray(e.loc) ? e.loc : []).map(String).join(".")}: ${e.msg}`).join("; ");
|
|
655
|
+
if (isRecord(details)) return Object.entries(details).map(([name, why]) => `${name}: ${why}`).join("; ");
|
|
656
|
+
return "";
|
|
657
|
+
}
|
|
658
|
+
function apiError(method, path, response, body) {
|
|
659
|
+
const status = response.status;
|
|
660
|
+
let message = `HTTP ${status}`;
|
|
661
|
+
if (isRecord(body)) {
|
|
662
|
+
message = String(body.msg || body.message || body.detail || body.error || message);
|
|
663
|
+
const fields = fieldErrors(isRecord(body.data) ? body.data.errors : body.errors);
|
|
664
|
+
if (fields) message += `: ${fields}`;
|
|
665
|
+
} else if (typeof body === "string" && body) message = body.slice(0, 500);
|
|
666
|
+
message = `${method.toUpperCase()} ${path}: ${message}`;
|
|
667
|
+
let error = STATUS_ERRORS[status] ?? (status >= 500 ? ServerError : APIError);
|
|
668
|
+
let retryAfter;
|
|
669
|
+
if (status === 422 && tokenRejected(body)) error = AuthenticationError;
|
|
670
|
+
if (status === 429) {
|
|
671
|
+
const code = isRecord(body) && typeof body.code === "string" ? body.code : "";
|
|
672
|
+
error = QUOTA_ERRORS[code] ?? RateLimitError;
|
|
673
|
+
if (error === RateLimitError) retryAfter = retryAfterMs(response);
|
|
674
|
+
}
|
|
675
|
+
return new error(message, {
|
|
676
|
+
statusCode: status,
|
|
677
|
+
body,
|
|
678
|
+
requestId: requestId(response),
|
|
679
|
+
headers: headersOf(response),
|
|
680
|
+
retryAfter
|
|
681
|
+
});
|
|
682
|
+
}
|
|
683
|
+
function networkError(path, error) {
|
|
684
|
+
if (isTimeout(error)) return new APITimeoutError(`request to ${path} timed out`, { cause: error });
|
|
685
|
+
return new APIConnectionError(`request to ${path} failed: ${describe(error)}`, { cause: error });
|
|
686
|
+
}
|
|
687
|
+
//#endregion
|
|
688
|
+
//#region src/browser/cdp.ts
|
|
689
|
+
/** A tiny Chrome DevTools Protocol client: JSON-RPC over a WebSocket. */
|
|
690
|
+
const nativeWebSocket = (url) => new WebSocket(url);
|
|
691
|
+
var CDPClient = class {
|
|
692
|
+
wsUrl;
|
|
693
|
+
onClose;
|
|
694
|
+
#create;
|
|
695
|
+
#logger;
|
|
696
|
+
#ws;
|
|
697
|
+
#closed = false;
|
|
698
|
+
#closeReported = false;
|
|
699
|
+
#nextId = 0;
|
|
700
|
+
#pending = /* @__PURE__ */ new Map();
|
|
701
|
+
#handlers = /* @__PURE__ */ new Map();
|
|
702
|
+
constructor(wsUrl, options = {}) {
|
|
703
|
+
this.wsUrl = wsUrl;
|
|
704
|
+
this.onClose = options.onClose;
|
|
705
|
+
this.#create = options.createWebSocket ?? nativeWebSocket;
|
|
706
|
+
this.#logger = makeLogger(options.logger);
|
|
707
|
+
}
|
|
708
|
+
get connected() {
|
|
709
|
+
return this.#ws !== void 0 && !this.#closed;
|
|
710
|
+
}
|
|
711
|
+
async start() {
|
|
712
|
+
if (this.#ws !== void 0) throw new Error("CDP client is already started");
|
|
713
|
+
const ws = this.#create(this.wsUrl);
|
|
714
|
+
this.#ws = ws;
|
|
715
|
+
await new Promise((resolve, reject) => {
|
|
716
|
+
let opened = false;
|
|
717
|
+
ws.addEventListener("open", () => {
|
|
718
|
+
opened = true;
|
|
719
|
+
resolve();
|
|
720
|
+
});
|
|
721
|
+
ws.addEventListener("error", (event) => {
|
|
722
|
+
if (opened) return;
|
|
723
|
+
reject(new CDPError(`CDP connection failed: ${describeEvent(event)}`));
|
|
724
|
+
});
|
|
725
|
+
ws.addEventListener("close", (event) => {
|
|
726
|
+
if (!opened) reject(new CDPError(`CDP connection failed: ${describeEvent(event)}`));
|
|
727
|
+
this.#onSocketClose();
|
|
728
|
+
});
|
|
729
|
+
ws.addEventListener("message", (event) => this.#onMessage(event.data));
|
|
730
|
+
}).catch((err) => {
|
|
731
|
+
this.#closed = true;
|
|
732
|
+
this.#ws = void 0;
|
|
733
|
+
try {
|
|
734
|
+
ws.close();
|
|
735
|
+
} catch {}
|
|
736
|
+
throw err;
|
|
737
|
+
});
|
|
738
|
+
}
|
|
739
|
+
async stop() {
|
|
740
|
+
this.#closed = true;
|
|
741
|
+
const ws = this.#ws;
|
|
742
|
+
this.#ws = void 0;
|
|
743
|
+
if (ws !== void 0) try {
|
|
744
|
+
ws.close(1e3);
|
|
745
|
+
} catch {}
|
|
746
|
+
this.#failPending();
|
|
747
|
+
this.#reportClose();
|
|
748
|
+
}
|
|
749
|
+
on(event, handler) {
|
|
750
|
+
this.#handlers.set(event, handler);
|
|
751
|
+
}
|
|
752
|
+
/** Put a command on the wire and hand back its reply to await later. */
|
|
753
|
+
post(method, params, sessionId) {
|
|
754
|
+
if (this.#closed) throw new CDPError("CDP connection closed");
|
|
755
|
+
const ws = this.#ws;
|
|
756
|
+
if (ws === void 0) throw new Error("CDP client is not connected; call start() first");
|
|
757
|
+
this.#nextId += 1;
|
|
758
|
+
const id = this.#nextId;
|
|
759
|
+
const message = {
|
|
760
|
+
id,
|
|
761
|
+
method,
|
|
762
|
+
params: params ?? {}
|
|
763
|
+
};
|
|
764
|
+
if (sessionId !== void 0) message.sessionId = sessionId;
|
|
765
|
+
const reply = Promise.withResolvers();
|
|
766
|
+
reply.promise.catch(() => {});
|
|
767
|
+
this.#pending.set(id, reply);
|
|
768
|
+
try {
|
|
769
|
+
ws.send(JSON.stringify(message));
|
|
770
|
+
} catch (err) {
|
|
771
|
+
this.#pending.delete(id);
|
|
772
|
+
throw new CDPError(`CDP connection closed: ${describeEvent(err)}`, { cause: err });
|
|
773
|
+
}
|
|
774
|
+
return {
|
|
775
|
+
id,
|
|
776
|
+
reply: reply.promise
|
|
777
|
+
};
|
|
778
|
+
}
|
|
779
|
+
/** Drop a posted command nobody is going to await. */
|
|
780
|
+
forget(id) {
|
|
781
|
+
this.#pending.delete(id);
|
|
782
|
+
}
|
|
783
|
+
async send(method, params, sessionId) {
|
|
784
|
+
const { id, reply } = this.post(method, params, sessionId);
|
|
785
|
+
try {
|
|
786
|
+
return await reply;
|
|
787
|
+
} finally {
|
|
788
|
+
this.#pending.delete(id);
|
|
789
|
+
}
|
|
790
|
+
}
|
|
791
|
+
#onMessage(raw) {
|
|
792
|
+
if (typeof raw !== "string") {
|
|
793
|
+
this.#logger.warn("dropping a binary CDP frame");
|
|
794
|
+
return;
|
|
795
|
+
}
|
|
796
|
+
let message;
|
|
797
|
+
try {
|
|
798
|
+
message = JSON.parse(raw);
|
|
799
|
+
} catch {
|
|
800
|
+
this.#logger.warn("dropping unparsable CDP frame");
|
|
801
|
+
return;
|
|
802
|
+
}
|
|
803
|
+
if (isRecord(message)) this.#dispatch(message);
|
|
804
|
+
}
|
|
805
|
+
#dispatch(message) {
|
|
806
|
+
const id = message.id;
|
|
807
|
+
if (typeof id === "number") {
|
|
808
|
+
const reply = this.#pending.get(id);
|
|
809
|
+
this.#pending.delete(id);
|
|
810
|
+
if (reply === void 0) return;
|
|
811
|
+
const error = message.error;
|
|
812
|
+
if (isRecord(error)) {
|
|
813
|
+
let text = typeof error.message === "string" ? error.message : "CDP error";
|
|
814
|
+
if (error.data) text = `${text}: ${String(error.data)}`;
|
|
815
|
+
const code = typeof error.code === "number" ? error.code : void 0;
|
|
816
|
+
reply.reject(new CDPError(text, { code }));
|
|
817
|
+
} else reply.resolve(message.result ?? {});
|
|
818
|
+
return;
|
|
819
|
+
}
|
|
820
|
+
const method = typeof message.method === "string" ? message.method : "";
|
|
821
|
+
const handler = this.#handlers.get(method);
|
|
822
|
+
if (handler === void 0) return;
|
|
823
|
+
const sessionId = typeof message.sessionId === "string" ? message.sessionId : void 0;
|
|
824
|
+
try {
|
|
825
|
+
handler(message.params ?? {}, sessionId);
|
|
826
|
+
} catch (err) {
|
|
827
|
+
this.#logger.error(`CDP event handler for ${method} failed: ${describeEvent(err)}`);
|
|
828
|
+
}
|
|
829
|
+
}
|
|
830
|
+
#onSocketClose() {
|
|
831
|
+
if (!this.#closed) this.#logger.warn("CDP connection closed by the server");
|
|
832
|
+
this.#closed = true;
|
|
833
|
+
this.#ws = void 0;
|
|
834
|
+
this.#failPending();
|
|
835
|
+
this.#reportClose();
|
|
836
|
+
}
|
|
837
|
+
#reportClose() {
|
|
838
|
+
if (this.#closeReported) return;
|
|
839
|
+
this.#closeReported = true;
|
|
840
|
+
this.onClose?.();
|
|
841
|
+
}
|
|
842
|
+
#failPending() {
|
|
843
|
+
const pending = [...this.#pending.values()];
|
|
844
|
+
this.#pending.clear();
|
|
845
|
+
for (const reply of pending) reply.reject(new CDPError("CDP connection closed"));
|
|
846
|
+
}
|
|
847
|
+
};
|
|
848
|
+
function describeEvent(event) {
|
|
849
|
+
if (event instanceof Error) return event.message;
|
|
850
|
+
if (isRecord(event)) {
|
|
851
|
+
if (event.error instanceof Error) return event.error.message;
|
|
852
|
+
if (typeof event.message === "string") return event.message;
|
|
853
|
+
if (typeof event.reason === "string" && event.reason) return event.reason;
|
|
854
|
+
if (typeof event.code === "number") return `code ${event.code}`;
|
|
855
|
+
}
|
|
856
|
+
return String(event);
|
|
857
|
+
}
|
|
858
|
+
const LIFECYCLE_EVENT = {
|
|
859
|
+
commit: "commit",
|
|
860
|
+
domcontentloaded: "DOMContentLoaded",
|
|
861
|
+
load: "load",
|
|
862
|
+
networkidle: "networkIdle"
|
|
863
|
+
};
|
|
864
|
+
const WORLD_NAME = "utility";
|
|
865
|
+
/** A dialog is answered after this long, as a person would. */
|
|
866
|
+
const DIALOG_DELAY = [600, 1400];
|
|
867
|
+
const FUNCTION = /^\s*(async\s+)?(function\b|\([^()]*\)\s*=>|[\w$]+\s*=>)/;
|
|
868
|
+
const SELECT = `(selector, value, label) => {
|
|
869
|
+
const el = document.querySelector(selector);
|
|
870
|
+
if (!el) return null;
|
|
871
|
+
const option = Array.from(el.options).find(
|
|
872
|
+
o => label === null ? o.value === value : o.label === label || o.text === label
|
|
873
|
+
);
|
|
874
|
+
if (!option) return false;
|
|
875
|
+
el.value = option.value;
|
|
876
|
+
el.dispatchEvent(new Event("input", {bubbles: true}));
|
|
877
|
+
el.dispatchEvent(new Event("change", {bubbles: true}));
|
|
878
|
+
return option.value;
|
|
879
|
+
}`;
|
|
880
|
+
function cookieParam(cookie) {
|
|
881
|
+
const out = dropNulls(cookie);
|
|
882
|
+
const expires = out.expirationDate;
|
|
883
|
+
return expires === void 0 ? out : {
|
|
884
|
+
expires,
|
|
885
|
+
...out
|
|
886
|
+
};
|
|
887
|
+
}
|
|
888
|
+
function fromCdpCookie(raw) {
|
|
889
|
+
const { expires, ...rest } = raw;
|
|
890
|
+
return expires === void 0 ? rest : {
|
|
891
|
+
expirationDate: expires,
|
|
892
|
+
...rest
|
|
893
|
+
};
|
|
894
|
+
}
|
|
895
|
+
function commandFailed(method, params, err) {
|
|
896
|
+
const selector = params?.selector;
|
|
897
|
+
const where = typeof selector === "string" ? `${method} '${selector}'` : method;
|
|
898
|
+
if (err.message.includes("Could not find node with given id")) return new CDPError(`${where}: the page navigated while the command ran.`, {
|
|
899
|
+
code: err.code,
|
|
900
|
+
cause: err
|
|
901
|
+
});
|
|
902
|
+
return new CDPError(`${where}: ${err.message}`, {
|
|
903
|
+
code: err.code,
|
|
904
|
+
cause: err
|
|
905
|
+
});
|
|
906
|
+
}
|
|
907
|
+
/** A wait's time budget: poll loops `check()` it, single awaits `race()` against it. `null` is no limit. */
|
|
908
|
+
var Deadline = class {
|
|
909
|
+
timeout;
|
|
910
|
+
message;
|
|
911
|
+
until;
|
|
912
|
+
constructor(timeout, message) {
|
|
913
|
+
this.timeout = timeout ?? null;
|
|
914
|
+
this.message = message;
|
|
915
|
+
this.until = this.timeout === null ? Number.POSITIVE_INFINITY : Date.now() + this.timeout;
|
|
916
|
+
}
|
|
917
|
+
error() {
|
|
918
|
+
return new BrowserTimeoutError(`${this.message} within ${this.timeout}ms`);
|
|
919
|
+
}
|
|
920
|
+
check() {
|
|
921
|
+
if (Date.now() >= this.until) throw this.error();
|
|
922
|
+
}
|
|
923
|
+
race(promise) {
|
|
924
|
+
if (this.timeout === null) return promise;
|
|
925
|
+
return withTimeout(promise, Math.max(0, this.until - Date.now()), () => this.error());
|
|
926
|
+
}
|
|
927
|
+
};
|
|
928
|
+
var CapturedResponse = class {
|
|
929
|
+
url;
|
|
930
|
+
status;
|
|
931
|
+
headers;
|
|
932
|
+
body;
|
|
933
|
+
constructor(url, status, headers, body) {
|
|
934
|
+
this.url = url;
|
|
935
|
+
this.status = status;
|
|
936
|
+
this.headers = headers;
|
|
937
|
+
this.body = body;
|
|
938
|
+
}
|
|
939
|
+
get text() {
|
|
940
|
+
return new TextDecoder().decode(this.body);
|
|
941
|
+
}
|
|
942
|
+
json() {
|
|
943
|
+
return JSON.parse(this.text);
|
|
944
|
+
}
|
|
945
|
+
};
|
|
946
|
+
/** The lifecycle wait of one `goto`. */
|
|
947
|
+
var NavWaiter = class {
|
|
948
|
+
milestone;
|
|
949
|
+
done = new Flag();
|
|
950
|
+
error;
|
|
951
|
+
loaderId;
|
|
952
|
+
created = [];
|
|
953
|
+
reached = /* @__PURE__ */ new Set();
|
|
954
|
+
constructor(milestone) {
|
|
955
|
+
this.milestone = milestone;
|
|
956
|
+
}
|
|
957
|
+
follow(loaderId) {
|
|
958
|
+
let id = loaderId;
|
|
959
|
+
if (this.created.includes(id)) id = this.created[this.created.length - 1];
|
|
960
|
+
this.loaderId = id;
|
|
961
|
+
return this.reached.has(id);
|
|
962
|
+
}
|
|
963
|
+
followNewest() {
|
|
964
|
+
const newest = this.created[this.created.length - 1];
|
|
965
|
+
if (newest !== void 0) return this.follow(newest);
|
|
966
|
+
this.loaderId = "";
|
|
967
|
+
return false;
|
|
968
|
+
}
|
|
969
|
+
observe(name, loaderId) {
|
|
970
|
+
if (this.done.isSet || typeof loaderId !== "string" || !loaderId) return;
|
|
971
|
+
if (name === "init") {
|
|
972
|
+
if (this.loaderId === void 0) this.created.push(loaderId);
|
|
973
|
+
else if (loaderId !== this.loaderId) this.loaderId = loaderId;
|
|
974
|
+
} else if (name === this.milestone) {
|
|
975
|
+
if (this.loaderId === void 0) this.reached.add(loaderId);
|
|
976
|
+
else if (loaderId === this.loaderId) this.done.set();
|
|
977
|
+
}
|
|
978
|
+
}
|
|
979
|
+
fail(reason) {
|
|
980
|
+
this.error = reason;
|
|
981
|
+
this.done.set();
|
|
982
|
+
}
|
|
983
|
+
};
|
|
984
|
+
/** One page target. `Browser` is the session's first page plus the connection. */
|
|
985
|
+
var Page = class extends Actions {
|
|
986
|
+
/** @internal */ _browser;
|
|
987
|
+
/** @internal */ _targetId;
|
|
988
|
+
/** @internal */ _sessionId;
|
|
989
|
+
/** @internal */ _frameId;
|
|
990
|
+
/** @internal */ _closed;
|
|
991
|
+
/** Answers this page's dialogs; the browser's handler when unset. */
|
|
992
|
+
onDialog = null;
|
|
993
|
+
/** @internal the main frame's newest document */
|
|
994
|
+
_loaderId;
|
|
995
|
+
/** @internal the lifecycle events it reached */
|
|
996
|
+
_milestones;
|
|
997
|
+
/** @internal the isolated world's context */
|
|
998
|
+
_worldId;
|
|
999
|
+
/** @internal commands wait for the setup */
|
|
1000
|
+
_ready;
|
|
1001
|
+
/** @internal */ _setupError;
|
|
1002
|
+
/** @internal */ _waiter;
|
|
1003
|
+
/** @internal */ _status;
|
|
1004
|
+
/** @internal */ _storageReady;
|
|
1005
|
+
/** @internal */ _captures;
|
|
1006
|
+
/** @internal */ _responses;
|
|
1007
|
+
/** @internal */ _inFlight;
|
|
1008
|
+
constructor(browser, targetId, sessionId) {
|
|
1009
|
+
super();
|
|
1010
|
+
this._browser = browser ?? this;
|
|
1011
|
+
this._reset(targetId, sessionId);
|
|
1012
|
+
}
|
|
1013
|
+
/** @internal fresh page state; the browser reuses it on connect */
|
|
1014
|
+
_reset(targetId, sessionId) {
|
|
1015
|
+
this._targetId = targetId;
|
|
1016
|
+
this._sessionId = sessionId;
|
|
1017
|
+
this._frameId = "";
|
|
1018
|
+
this._closed = false;
|
|
1019
|
+
this._loaderId = void 0;
|
|
1020
|
+
this._milestones = /* @__PURE__ */ new Set();
|
|
1021
|
+
this._worldId = void 0;
|
|
1022
|
+
this._ready = new Flag();
|
|
1023
|
+
this._ready.set();
|
|
1024
|
+
this._setupError = void 0;
|
|
1025
|
+
this._waiter = void 0;
|
|
1026
|
+
this._status = void 0;
|
|
1027
|
+
this._storageReady = false;
|
|
1028
|
+
this._captures = [];
|
|
1029
|
+
this._responses = [];
|
|
1030
|
+
this._inFlight = /* @__PURE__ */ new Map();
|
|
1031
|
+
}
|
|
1032
|
+
get cdp() {
|
|
1033
|
+
return this._browser.cdp;
|
|
1034
|
+
}
|
|
1035
|
+
get targetId() {
|
|
1036
|
+
return this._targetId;
|
|
1037
|
+
}
|
|
1038
|
+
get closed() {
|
|
1039
|
+
return this._closed;
|
|
1040
|
+
}
|
|
1041
|
+
get status() {
|
|
1042
|
+
return this._status;
|
|
1043
|
+
}
|
|
1044
|
+
async send(method, params) {
|
|
1045
|
+
this._requireOpen();
|
|
1046
|
+
await this._waitReady();
|
|
1047
|
+
return this._send(method, params);
|
|
1048
|
+
}
|
|
1049
|
+
/** @internal */
|
|
1050
|
+
_requireOpen() {
|
|
1051
|
+
if (this._closed) throw new PageClosedError("page is closed");
|
|
1052
|
+
}
|
|
1053
|
+
/** @internal */
|
|
1054
|
+
async _send(method, params) {
|
|
1055
|
+
const timeout = this._browser.commandTimeout;
|
|
1056
|
+
try {
|
|
1057
|
+
return await withTimeout(this.cdp.send(method, params, this._sessionId), timeout, () => new BrowserTimeoutError(`${method} did not answer within ${timeout}ms`));
|
|
1058
|
+
} catch (err) {
|
|
1059
|
+
if (err instanceof CDPError) throw commandFailed(method, params, err);
|
|
1060
|
+
throw err;
|
|
1061
|
+
}
|
|
1062
|
+
}
|
|
1063
|
+
/** @internal */
|
|
1064
|
+
async _waitReady() {
|
|
1065
|
+
await this._ready.wait();
|
|
1066
|
+
if (this._setupError !== void 0) throw this._setupError;
|
|
1067
|
+
}
|
|
1068
|
+
/** @internal */
|
|
1069
|
+
async _setup(waiting) {
|
|
1070
|
+
const commands = [
|
|
1071
|
+
["Page.enable", void 0],
|
|
1072
|
+
["Page.setLifecycleEventsEnabled", { enabled: true }],
|
|
1073
|
+
["Fetch.enable", { patterns: this._browser._fetchPatterns }]
|
|
1074
|
+
];
|
|
1075
|
+
if (waiting) commands.push(["Runtime.runIfWaitingForDebugger", void 0]);
|
|
1076
|
+
const timeout = this._browser.commandTimeout;
|
|
1077
|
+
let posted = [];
|
|
1078
|
+
try {
|
|
1079
|
+
posted = commands.map(([method, params]) => this.cdp.post(method, params, this._sessionId));
|
|
1080
|
+
const replies = posted.map(async ({ reply }, i) => {
|
|
1081
|
+
const [method, params] = commands[i];
|
|
1082
|
+
try {
|
|
1083
|
+
await reply;
|
|
1084
|
+
} catch (err) {
|
|
1085
|
+
if (err instanceof CDPError) throw commandFailed(method, params, err);
|
|
1086
|
+
throw err;
|
|
1087
|
+
}
|
|
1088
|
+
});
|
|
1089
|
+
await withTimeout(Promise.all(replies), timeout, () => new BrowserTimeoutError(`the page was not ready within ${timeout}ms`));
|
|
1090
|
+
} catch (err) {
|
|
1091
|
+
this._setupError = err;
|
|
1092
|
+
} finally {
|
|
1093
|
+
for (const { id } of posted) this.cdp.forget(id);
|
|
1094
|
+
this._ready.set();
|
|
1095
|
+
}
|
|
1096
|
+
}
|
|
1097
|
+
async close() {
|
|
1098
|
+
if (this._closed) return;
|
|
1099
|
+
const timeout = this._browser.commandTimeout;
|
|
1100
|
+
await withTimeout(this.cdp.send("Target.closeTarget", { targetId: this._targetId }), timeout, () => new BrowserTimeoutError(`the page did not close within ${timeout}ms`));
|
|
1101
|
+
this._browser._drop(this);
|
|
1102
|
+
}
|
|
1103
|
+
async bringToFront() {
|
|
1104
|
+
await this.send("Page.bringToFront");
|
|
1105
|
+
}
|
|
1106
|
+
async waitForLoadState(state = "load", options = {}) {
|
|
1107
|
+
const milestone = LIFECYCLE_EVENT[state];
|
|
1108
|
+
const deadline = new Deadline(options.timeout ?? 3e4, `the page did not reach '${state}'`);
|
|
1109
|
+
while (!this._milestones.has(milestone)) {
|
|
1110
|
+
this._requireOpen();
|
|
1111
|
+
deadline.check();
|
|
1112
|
+
await sleep(100);
|
|
1113
|
+
}
|
|
1114
|
+
}
|
|
1115
|
+
async goto(url, options = {}) {
|
|
1116
|
+
const waitUntil = options.waitUntil ?? "load";
|
|
1117
|
+
if (this._waiter !== void 0) throw new Error("navigation already in progress");
|
|
1118
|
+
const waiter = new NavWaiter(LIFECYCLE_EVENT[waitUntil]);
|
|
1119
|
+
this._waiter = waiter;
|
|
1120
|
+
this._status = void 0;
|
|
1121
|
+
const deadline = new Deadline(options.timeout ?? 3e4, `navigation to ${url} did not reach '${waitUntil}'`);
|
|
1122
|
+
try {
|
|
1123
|
+
const result = await deadline.race(this.send("Page.navigate", { url }));
|
|
1124
|
+
if (result.errorText) throw new CDPError(`navigation to ${url} failed: ${result.errorText}`);
|
|
1125
|
+
const loaderId = result.loaderId;
|
|
1126
|
+
if (waitUntil === "commit" || typeof loaderId !== "string") return;
|
|
1127
|
+
if (!waiter.follow(loaderId)) await deadline.race(waiter.done.wait());
|
|
1128
|
+
if (waiter.error !== void 0) throw new CDPError(waiter.error);
|
|
1129
|
+
} finally {
|
|
1130
|
+
if (this._waiter === waiter) this._waiter = void 0;
|
|
1131
|
+
}
|
|
1132
|
+
}
|
|
1133
|
+
async reload(options = {}) {
|
|
1134
|
+
const waitUntil = options.waitUntil ?? "load";
|
|
1135
|
+
if (this._waiter !== void 0) throw new Error("navigation already in progress");
|
|
1136
|
+
const waiter = new NavWaiter(LIFECYCLE_EVENT[waitUntil]);
|
|
1137
|
+
this._waiter = waiter;
|
|
1138
|
+
this._status = void 0;
|
|
1139
|
+
const deadline = new Deadline(options.timeout ?? 3e4, `reload did not reach '${waitUntil}'`);
|
|
1140
|
+
try {
|
|
1141
|
+
await deadline.race(this.send("Page.reload"));
|
|
1142
|
+
if (!waiter.followNewest()) await deadline.race(waiter.done.wait());
|
|
1143
|
+
if (waiter.error !== void 0) throw new CDPError(waiter.error);
|
|
1144
|
+
} finally {
|
|
1145
|
+
if (this._waiter === waiter) this._waiter = void 0;
|
|
1146
|
+
}
|
|
1147
|
+
}
|
|
1148
|
+
goBack(options = {}) {
|
|
1149
|
+
return this.#historyStep(-1, options.timeout ?? 3e4);
|
|
1150
|
+
}
|
|
1151
|
+
goForward(options = {}) {
|
|
1152
|
+
return this.#historyStep(1, options.timeout ?? 3e4);
|
|
1153
|
+
}
|
|
1154
|
+
async #historyStep(delta, timeout) {
|
|
1155
|
+
const history = await this.send("Page.getNavigationHistory");
|
|
1156
|
+
const entries = history.entries;
|
|
1157
|
+
const index = history.currentIndex + delta;
|
|
1158
|
+
const entry = entries[index];
|
|
1159
|
+
if (index < 0 || entry === void 0) return null;
|
|
1160
|
+
await this.send("Page.navigateToHistoryEntry", { entryId: entry.id });
|
|
1161
|
+
const deadline = new Deadline(timeout, "history did not move");
|
|
1162
|
+
while (true) {
|
|
1163
|
+
if ((await deadline.race(this.send("Page.getNavigationHistory"))).currentIndex === index) break;
|
|
1164
|
+
deadline.check();
|
|
1165
|
+
await sleep(100);
|
|
1166
|
+
}
|
|
1167
|
+
return this.url();
|
|
1168
|
+
}
|
|
1169
|
+
get responses() {
|
|
1170
|
+
return [...this._responses];
|
|
1171
|
+
}
|
|
1172
|
+
/** Record responses whose URL contains a fragment. Call before navigating. */
|
|
1173
|
+
async captureResponses(...fragments) {
|
|
1174
|
+
if (fragments.length === 0) throw new TypeError("captureResponses needs at least one URL fragment");
|
|
1175
|
+
if (this._captures.length === 0) await this.send("Network.enable");
|
|
1176
|
+
this._captures.push(...fragments);
|
|
1177
|
+
}
|
|
1178
|
+
async stopCapturing() {
|
|
1179
|
+
if (this._captures.length === 0) return;
|
|
1180
|
+
this._captures = [];
|
|
1181
|
+
this._responses = [];
|
|
1182
|
+
this._inFlight.clear();
|
|
1183
|
+
await this.send("Network.disable");
|
|
1184
|
+
}
|
|
1185
|
+
async waitForResponse(fragment, options = {}) {
|
|
1186
|
+
if (this._captures.length === 0) throw new Error("call captureResponses() before the navigation");
|
|
1187
|
+
const deadline = new Deadline(options.timeout ?? 3e4, `no response matching '${fragment}'`);
|
|
1188
|
+
while (true) {
|
|
1189
|
+
const found = this._responses.find((response) => response.url.includes(fragment));
|
|
1190
|
+
if (found !== void 0) return found;
|
|
1191
|
+
this._requireOpen();
|
|
1192
|
+
deadline.check();
|
|
1193
|
+
await sleep(100);
|
|
1194
|
+
}
|
|
1195
|
+
}
|
|
1196
|
+
/** @internal */
|
|
1197
|
+
_onResponse(event) {
|
|
1198
|
+
const response = isRecord(event.response) ? event.response : {};
|
|
1199
|
+
const url = typeof response.url === "string" ? response.url : "";
|
|
1200
|
+
if (this._captures.some((fragment) => url.includes(fragment))) this._inFlight.set(String(event.requestId), response);
|
|
1201
|
+
}
|
|
1202
|
+
/** @internal */
|
|
1203
|
+
_onBodyReady(event) {
|
|
1204
|
+
const requestId = String(event.requestId ?? "");
|
|
1205
|
+
const response = this._inFlight.get(requestId);
|
|
1206
|
+
if (response === void 0) return;
|
|
1207
|
+
this._inFlight.delete(requestId);
|
|
1208
|
+
this._spawn(this.#collect(requestId, response, this._responses));
|
|
1209
|
+
}
|
|
1210
|
+
async #collect(requestId, response, into) {
|
|
1211
|
+
let body = /* @__PURE__ */ new Uint8Array(0);
|
|
1212
|
+
try {
|
|
1213
|
+
const result = await this.send("Network.getResponseBody", { requestId });
|
|
1214
|
+
const raw = typeof result.body === "string" ? result.body : "";
|
|
1215
|
+
body = result.base64Encoded ? fromBase64(raw) : new TextEncoder().encode(raw);
|
|
1216
|
+
} catch (err) {
|
|
1217
|
+
this._browser.logger.debug(`no body for ${String(response.url)}: ${String(err)}`);
|
|
1218
|
+
}
|
|
1219
|
+
into.push(new CapturedResponse(typeof response.url === "string" ? response.url : "", typeof response.status === "number" ? response.status : 0, isRecord(response.headers) ? response.headers : {}, body));
|
|
1220
|
+
}
|
|
1221
|
+
localStorage() {
|
|
1222
|
+
return this.#storage(true);
|
|
1223
|
+
}
|
|
1224
|
+
setLocalStorage(values) {
|
|
1225
|
+
return this.#setStorage(values, true);
|
|
1226
|
+
}
|
|
1227
|
+
sessionStorage() {
|
|
1228
|
+
return this.#storage(false);
|
|
1229
|
+
}
|
|
1230
|
+
setSessionStorage(values) {
|
|
1231
|
+
return this.#setStorage(values, false);
|
|
1232
|
+
}
|
|
1233
|
+
async clearCookies() {
|
|
1234
|
+
await this.send("Network.clearBrowserCookies");
|
|
1235
|
+
}
|
|
1236
|
+
async #storage(local) {
|
|
1237
|
+
const items = await this.send("DOMStorage.getDOMStorageItems", { storageId: await this.#storageId(local) });
|
|
1238
|
+
return Object.fromEntries(items.entries ?? []);
|
|
1239
|
+
}
|
|
1240
|
+
async #setStorage(values, local) {
|
|
1241
|
+
const storageId = await this.#storageId(local);
|
|
1242
|
+
for (const [key, value] of Object.entries(values)) await this.send("DOMStorage.setDOMStorageItem", {
|
|
1243
|
+
storageId,
|
|
1244
|
+
key,
|
|
1245
|
+
value
|
|
1246
|
+
});
|
|
1247
|
+
}
|
|
1248
|
+
async #storageId(local) {
|
|
1249
|
+
if (!this._storageReady) {
|
|
1250
|
+
await this.send("DOMStorage.enable");
|
|
1251
|
+
this._storageReady = true;
|
|
1252
|
+
}
|
|
1253
|
+
const frame = (await this.send("Page.getFrameTree")).frameTree.frame;
|
|
1254
|
+
const storageId = { isLocalStorage: local };
|
|
1255
|
+
if (frame.storageKey) storageId.storageKey = frame.storageKey;
|
|
1256
|
+
else storageId.securityOrigin = frame.securityOrigin;
|
|
1257
|
+
return storageId;
|
|
1258
|
+
}
|
|
1259
|
+
/**
|
|
1260
|
+
* Run script in the page. A function (or a string that looks like one) is called
|
|
1261
|
+
* with `args` as JSON; anything else is an expression. It runs in an isolated world:
|
|
1262
|
+
* the page's DOM, but not its globals, so the page's own script cannot see the call
|
|
1263
|
+
* or hook it. Page variables (`window.__data`) need `isolated: false`, the main
|
|
1264
|
+
* world, where a site that hooks the natives can notice.
|
|
1265
|
+
*/
|
|
1266
|
+
async evaluate(expression, options = {}) {
|
|
1267
|
+
const isolated = options.isolated ?? true;
|
|
1268
|
+
const awaitPromise = options.awaitPromise ?? true;
|
|
1269
|
+
let expr = typeof expression === "function" ? expression.toString() : expression;
|
|
1270
|
+
if (options.args !== void 0 && options.args.length > 0) expr = `(${expr})(...${JSON.stringify(options.args)})`;
|
|
1271
|
+
else if (FUNCTION.test(expr)) expr = `(${expr})()`;
|
|
1272
|
+
let remote = await this.#evaluate(expr, isolated, awaitPromise);
|
|
1273
|
+
if (remote.type === "function" && expr.includes("=>")) remote = await this.#evaluate(`(${expr})()`, isolated, awaitPromise);
|
|
1274
|
+
return remote.value;
|
|
1275
|
+
}
|
|
1276
|
+
async #evaluate(expression, isolated, awaitPromise) {
|
|
1277
|
+
const params = {
|
|
1278
|
+
expression,
|
|
1279
|
+
returnByValue: true,
|
|
1280
|
+
awaitPromise
|
|
1281
|
+
};
|
|
1282
|
+
if (isolated) params.contextId = await this.#world();
|
|
1283
|
+
let result;
|
|
1284
|
+
try {
|
|
1285
|
+
result = await this.send("Runtime.evaluate", params);
|
|
1286
|
+
} catch (err) {
|
|
1287
|
+
if (!isolated || !(err instanceof CDPError) || !err.message.includes("Cannot find context")) throw err;
|
|
1288
|
+
this._worldId = void 0;
|
|
1289
|
+
params.contextId = await this.#world();
|
|
1290
|
+
result = await this.send("Runtime.evaluate", params);
|
|
1291
|
+
}
|
|
1292
|
+
const details = result.exceptionDetails;
|
|
1293
|
+
if (details) throw new CDPError(`evaluate failed: ${(details.exception ?? {}).description ?? details.text}`);
|
|
1294
|
+
return result.result;
|
|
1295
|
+
}
|
|
1296
|
+
async #world() {
|
|
1297
|
+
if (this._worldId === void 0) {
|
|
1298
|
+
const created = await this.send("Page.createIsolatedWorld", {
|
|
1299
|
+
frameId: this._frameId,
|
|
1300
|
+
worldName: WORLD_NAME
|
|
1301
|
+
});
|
|
1302
|
+
this._worldId = created.executionContextId;
|
|
1303
|
+
}
|
|
1304
|
+
return this._worldId;
|
|
1305
|
+
}
|
|
1306
|
+
/** Poll until the expression is truthy. Returns the value. */
|
|
1307
|
+
async waitForFunction(expression, options = {}) {
|
|
1308
|
+
const { timeout = 3e4, ...rest } = options;
|
|
1309
|
+
const deadline = new Deadline(timeout, `${typeof expression === "function" ? "the function" : `'${expression}'`} was not truthy`);
|
|
1310
|
+
while (true) {
|
|
1311
|
+
const value = await deadline.race(this.evaluate(expression, rest));
|
|
1312
|
+
if (value) return value;
|
|
1313
|
+
deadline.check();
|
|
1314
|
+
await sleep(100);
|
|
1315
|
+
}
|
|
1316
|
+
}
|
|
1317
|
+
innerText(selector) {
|
|
1318
|
+
return this.evaluate("s => document.querySelector(s)?.innerText ?? null", { args: [selector] });
|
|
1319
|
+
}
|
|
1320
|
+
allInnerTexts(selector) {
|
|
1321
|
+
return this.evaluate("s => Array.from(document.querySelectorAll(s), e => e.innerText)", { args: [selector] });
|
|
1322
|
+
}
|
|
1323
|
+
async getAttribute(selector, name) {
|
|
1324
|
+
const nodeId = await this.#nodeId(selector);
|
|
1325
|
+
if (nodeId === void 0) return null;
|
|
1326
|
+
const pairs = (await this.send("DOM.getAttributes", { nodeId })).attributes ?? [];
|
|
1327
|
+
for (let i = 0; i + 1 < pairs.length; i += 2) if (pairs[i] === name) return pairs[i + 1];
|
|
1328
|
+
return null;
|
|
1329
|
+
}
|
|
1330
|
+
async count(selector) {
|
|
1331
|
+
const document = await this.send("DOM.getDocument", { depth: 0 });
|
|
1332
|
+
return ((await this.send("DOM.querySelectorAll", {
|
|
1333
|
+
nodeId: document.root.nodeId,
|
|
1334
|
+
selector
|
|
1335
|
+
})).nodeIds ?? []).length;
|
|
1336
|
+
}
|
|
1337
|
+
async selectOption(selector, option) {
|
|
1338
|
+
const { value, label } = typeof option === "string" ? { value: option } : option;
|
|
1339
|
+
if (value === void 0 === (label === void 0)) throw new TypeError("selectOption takes either a value or a label");
|
|
1340
|
+
const picked = await this.evaluate(SELECT, { args: [
|
|
1341
|
+
selector,
|
|
1342
|
+
value ?? null,
|
|
1343
|
+
label ?? null
|
|
1344
|
+
] });
|
|
1345
|
+
if (picked === null) throw new Error(`nothing matches '${selector}'`);
|
|
1346
|
+
if (picked === false) throw new Error(`'${selector}' has no option '${label ?? value}'`);
|
|
1347
|
+
return picked;
|
|
1348
|
+
}
|
|
1349
|
+
async waitForSelector(selector, options = {}) {
|
|
1350
|
+
const visible = options.visible ?? true;
|
|
1351
|
+
const state = visible ? "visible" : "in the document";
|
|
1352
|
+
const deadline = new Deadline(options.timeout ?? 3e4, `'${selector}' was not ${state}`);
|
|
1353
|
+
while (!await deadline.race(this.#matches(selector, visible))) {
|
|
1354
|
+
deadline.check();
|
|
1355
|
+
await sleep(100);
|
|
1356
|
+
}
|
|
1357
|
+
}
|
|
1358
|
+
async #matches(selector, visible) {
|
|
1359
|
+
let box;
|
|
1360
|
+
try {
|
|
1361
|
+
const nodeId = await this.#nodeId(selector);
|
|
1362
|
+
if (nodeId === void 0) return false;
|
|
1363
|
+
if (!visible) return true;
|
|
1364
|
+
box = await this.send("DOM.getBoxModel", { nodeId });
|
|
1365
|
+
} catch (err) {
|
|
1366
|
+
if (err instanceof CDPError) return false;
|
|
1367
|
+
throw err;
|
|
1368
|
+
}
|
|
1369
|
+
const model = box.model ?? {};
|
|
1370
|
+
return Boolean(model.width) && Boolean(model.height);
|
|
1371
|
+
}
|
|
1372
|
+
async #nodeId(selector) {
|
|
1373
|
+
const document = await this.send("DOM.getDocument", { depth: 0 });
|
|
1374
|
+
return (await this.send("DOM.querySelector", {
|
|
1375
|
+
nodeId: document.root.nodeId,
|
|
1376
|
+
selector
|
|
1377
|
+
})).nodeId || void 0;
|
|
1378
|
+
}
|
|
1379
|
+
/** Wait until the URL contains the fragment. Returns the URL. */
|
|
1380
|
+
async waitForUrl(fragment, options = {}) {
|
|
1381
|
+
const deadline = new Deadline(options.timeout ?? 3e4, `the page did not reach '${fragment}'`);
|
|
1382
|
+
while (true) {
|
|
1383
|
+
const url = await deadline.race(this.url());
|
|
1384
|
+
if (url.includes(fragment)) return url;
|
|
1385
|
+
deadline.check();
|
|
1386
|
+
await sleep(100);
|
|
1387
|
+
}
|
|
1388
|
+
}
|
|
1389
|
+
/** The first match has a bounding box. */
|
|
1390
|
+
isVisible(selector) {
|
|
1391
|
+
return this.#matches(selector, true);
|
|
1392
|
+
}
|
|
1393
|
+
async outerHtml(selector) {
|
|
1394
|
+
const nodeId = await this.#nodeId(selector);
|
|
1395
|
+
if (nodeId === void 0) return null;
|
|
1396
|
+
return (await this.send("DOM.getOuterHTML", { nodeId })).outerHTML;
|
|
1397
|
+
}
|
|
1398
|
+
async setCookies(cookies) {
|
|
1399
|
+
await this.send("Network.setCookies", { cookies: cookies.map(cookieParam) });
|
|
1400
|
+
}
|
|
1401
|
+
async content() {
|
|
1402
|
+
const document = await this.send("DOM.getDocument", { depth: 0 });
|
|
1403
|
+
return (await this.send("DOM.getOuterHTML", { nodeId: document.root.nodeId })).outerHTML;
|
|
1404
|
+
}
|
|
1405
|
+
async title() {
|
|
1406
|
+
return (await this.send("Target.getTargetInfo", { targetId: this._targetId })).targetInfo.title;
|
|
1407
|
+
}
|
|
1408
|
+
async url() {
|
|
1409
|
+
return (await this.send("Target.getTargetInfo", { targetId: this._targetId })).targetInfo.url;
|
|
1410
|
+
}
|
|
1411
|
+
async cookies() {
|
|
1412
|
+
return ((await this.send("Network.getCookies")).cookies ?? []).map(fromCdpCookie);
|
|
1413
|
+
}
|
|
1414
|
+
async screenshot(options = {}) {
|
|
1415
|
+
const format = options.format ?? "png";
|
|
1416
|
+
const fullPage = options.fullPage ?? false;
|
|
1417
|
+
const params = {
|
|
1418
|
+
format,
|
|
1419
|
+
captureBeyondViewport: fullPage
|
|
1420
|
+
};
|
|
1421
|
+
if (options.quality !== void 0 && format !== "png") params.quality = options.quality;
|
|
1422
|
+
if (options.selector !== void 0) params.clip = await this.#clip(options.selector);
|
|
1423
|
+
else if (fullPage) {
|
|
1424
|
+
const metrics = await this.send("Page.getLayoutMetrics");
|
|
1425
|
+
params.clip = {
|
|
1426
|
+
x: 0,
|
|
1427
|
+
y: 0,
|
|
1428
|
+
scale: 1,
|
|
1429
|
+
...metrics.cssContentSize ?? metrics.contentSize
|
|
1430
|
+
};
|
|
1431
|
+
}
|
|
1432
|
+
return fromBase64((await this.send("Page.captureScreenshot", params)).data);
|
|
1433
|
+
}
|
|
1434
|
+
async #clip(selector) {
|
|
1435
|
+
const nodeId = await this.#nodeId(selector);
|
|
1436
|
+
if (nodeId === void 0) throw new Error(`nothing matches '${selector}'`);
|
|
1437
|
+
await this.send("DOM.scrollIntoViewIfNeeded", { nodeId });
|
|
1438
|
+
const box = await this.send("DOM.getBoxModel", { nodeId });
|
|
1439
|
+
const metrics = await this.send("Page.getLayoutMetrics");
|
|
1440
|
+
const viewport = metrics.cssVisualViewport ?? metrics.visualViewport ?? {};
|
|
1441
|
+
const quad = box.model.content;
|
|
1442
|
+
return {
|
|
1443
|
+
x: quad[0] + (viewport.pageX ?? 0),
|
|
1444
|
+
y: quad[1] + (viewport.pageY ?? 0),
|
|
1445
|
+
width: quad[2] - quad[0],
|
|
1446
|
+
height: quad[5] - quad[1],
|
|
1447
|
+
scale: 1
|
|
1448
|
+
};
|
|
1449
|
+
}
|
|
1450
|
+
/** @internal */
|
|
1451
|
+
_onLifecycle(event) {
|
|
1452
|
+
if (event.frameId !== this._frameId) return;
|
|
1453
|
+
const name = event.name ?? "";
|
|
1454
|
+
const loaderId = event.loaderId;
|
|
1455
|
+
if (name === "init") {
|
|
1456
|
+
this._loaderId = loaderId;
|
|
1457
|
+
this._milestones = /* @__PURE__ */ new Set();
|
|
1458
|
+
this._worldId = void 0;
|
|
1459
|
+
} else if (loaderId === this._loaderId) this._milestones.add(name);
|
|
1460
|
+
else if (this._loaderId === void 0) {
|
|
1461
|
+
this._loaderId = loaderId;
|
|
1462
|
+
this._milestones = /* @__PURE__ */ new Set([name]);
|
|
1463
|
+
}
|
|
1464
|
+
this._waiter?.observe(name, loaderId);
|
|
1465
|
+
}
|
|
1466
|
+
/** @internal */
|
|
1467
|
+
_onDialog(event) {
|
|
1468
|
+
const kind = event.type ?? "";
|
|
1469
|
+
const message = event.message ?? "";
|
|
1470
|
+
const handler = this.onDialog ?? this._browser.onDialog;
|
|
1471
|
+
let verdict;
|
|
1472
|
+
try {
|
|
1473
|
+
verdict = handler ? handler(kind, message) : void 0;
|
|
1474
|
+
} catch (err) {
|
|
1475
|
+
this._browser.logger.error(`onDialog failed, falling back to the default: ${String(err)}`);
|
|
1476
|
+
verdict = void 0;
|
|
1477
|
+
}
|
|
1478
|
+
if (verdict === void 0 || verdict === null) verdict = kind === "beforeunload";
|
|
1479
|
+
const params = { accept: verdict !== false };
|
|
1480
|
+
if (typeof verdict === "string") params.promptText = verdict;
|
|
1481
|
+
this._browser.logger.info(`${kind} dialog '${message}' ${params.accept ? "accepted" : "dismissed"}`);
|
|
1482
|
+
this._spawn(this.#answerDialog(params));
|
|
1483
|
+
}
|
|
1484
|
+
async #answerDialog(params) {
|
|
1485
|
+
const [min, max] = DIALOG_DELAY;
|
|
1486
|
+
await sleep(min + Math.random() * (max - min), this._browser._signal);
|
|
1487
|
+
if (this._browser._signal.aborted) return;
|
|
1488
|
+
await this._send("Page.handleJavaScriptDialog", params);
|
|
1489
|
+
}
|
|
1490
|
+
/** @internal */
|
|
1491
|
+
_onRequestPaused(event) {
|
|
1492
|
+
const requestId = event.requestId;
|
|
1493
|
+
if (requestId === void 0 || requestId === null) {
|
|
1494
|
+
this._browser.logger.warn("Fetch.requestPaused without a requestId");
|
|
1495
|
+
return;
|
|
1496
|
+
}
|
|
1497
|
+
const status = event.responseStatusCode;
|
|
1498
|
+
if (status !== void 0 && status !== null) {
|
|
1499
|
+
if (event.frameId === this._frameId) this._status = status;
|
|
1500
|
+
this._spawn(this._send("Fetch.continueResponse", { requestId }));
|
|
1501
|
+
} else if (event.responseErrorReason) this._spawn(this._send("Fetch.continueRequest", { requestId }));
|
|
1502
|
+
else this._spawn(this._send("Fetch.failRequest", {
|
|
1503
|
+
requestId,
|
|
1504
|
+
errorReason: "BlockedByClient"
|
|
1505
|
+
}));
|
|
1506
|
+
}
|
|
1507
|
+
/** @internal */
|
|
1508
|
+
_spawn(task) {
|
|
1509
|
+
this._browser._spawn(task);
|
|
1510
|
+
}
|
|
1511
|
+
};
|
|
1512
|
+
//#endregion
|
|
1513
|
+
//#region src/browser/browser.ts
|
|
1514
|
+
const STATUS_PATTERN = {
|
|
1515
|
+
urlPattern: "*",
|
|
1516
|
+
resourceType: "Document",
|
|
1517
|
+
requestStage: "Response"
|
|
1518
|
+
};
|
|
1519
|
+
const AUTO_ATTACH = {
|
|
1520
|
+
autoAttach: true,
|
|
1521
|
+
waitForDebuggerOnStart: true,
|
|
1522
|
+
flatten: true,
|
|
1523
|
+
filter: [{ type: "page" }]
|
|
1524
|
+
};
|
|
1525
|
+
const NON_WEB_SCHEMES = [
|
|
1526
|
+
"chrome-extension://",
|
|
1527
|
+
"devtools://",
|
|
1528
|
+
"chrome://"
|
|
1529
|
+
];
|
|
1530
|
+
const START_PAGES = ["chrome://newtab", "chrome://new-tab-page"];
|
|
1531
|
+
function isStartPage(url) {
|
|
1532
|
+
return START_PAGES.some((page) => url.startsWith(page));
|
|
1533
|
+
}
|
|
1534
|
+
const RESOURCE_TYPES = Object.fromEntries([
|
|
1535
|
+
"Stylesheet",
|
|
1536
|
+
"Image",
|
|
1537
|
+
"Media",
|
|
1538
|
+
"Font",
|
|
1539
|
+
"Script",
|
|
1540
|
+
"TextTrack",
|
|
1541
|
+
"XHR",
|
|
1542
|
+
"Fetch",
|
|
1543
|
+
"Prefetch",
|
|
1544
|
+
"EventSource",
|
|
1545
|
+
"WebSocket",
|
|
1546
|
+
"Manifest",
|
|
1547
|
+
"SignedExchange",
|
|
1548
|
+
"Ping",
|
|
1549
|
+
"CSPViolationReport",
|
|
1550
|
+
"Preflight",
|
|
1551
|
+
"FedCM",
|
|
1552
|
+
"Other"
|
|
1553
|
+
].map((name) => [name.toLowerCase(), name]));
|
|
1554
|
+
function normalizeBlocked(resources) {
|
|
1555
|
+
if (typeof resources === "string") throw new TypeError("blockResources takes a list of resource types, not 1 string");
|
|
1556
|
+
const blocked = new Set([...resources ?? []].map((name) => name.toLowerCase()));
|
|
1557
|
+
if (blocked.has("document")) throw new TypeError("blocking 'document' blocks the page itself");
|
|
1558
|
+
const unknown = [...blocked].filter((name) => !(name in RESOURCE_TYPES)).sort();
|
|
1559
|
+
if (unknown.length > 0) {
|
|
1560
|
+
const valid = Object.keys(RESOURCE_TYPES).sort();
|
|
1561
|
+
throw new TypeError(`unknown resource types ${JSON.stringify(unknown)}; valid: ${JSON.stringify(valid)}`);
|
|
1562
|
+
}
|
|
1563
|
+
return blocked;
|
|
1564
|
+
}
|
|
1565
|
+
function normalizeUrls(patterns) {
|
|
1566
|
+
if (typeof patterns === "string") throw new TypeError("blockUrls takes a list of patterns, not 1 string");
|
|
1567
|
+
return [...patterns ?? []];
|
|
1568
|
+
}
|
|
1569
|
+
/** A connected cloud browser, and the page its session started with. */
|
|
1570
|
+
var Browser = class extends Page {
|
|
1571
|
+
session;
|
|
1572
|
+
data = {};
|
|
1573
|
+
blockedResources;
|
|
1574
|
+
blockedUrls;
|
|
1575
|
+
connectTimeout;
|
|
1576
|
+
commandTimeout;
|
|
1577
|
+
logger;
|
|
1578
|
+
/** @internal */ _pages = /* @__PURE__ */ new Map();
|
|
1579
|
+
/** @internal aborts on close: background sleeps end early */
|
|
1580
|
+
_signal;
|
|
1581
|
+
#createWebSocket;
|
|
1582
|
+
#onClose;
|
|
1583
|
+
#client;
|
|
1584
|
+
#abort = new AbortController();
|
|
1585
|
+
#pending = /* @__PURE__ */ new Set();
|
|
1586
|
+
#closing;
|
|
1587
|
+
#retired = false;
|
|
1588
|
+
#useCount = 0;
|
|
1589
|
+
constructor(session, options = {}) {
|
|
1590
|
+
super(void 0, "", "");
|
|
1591
|
+
this.session = session;
|
|
1592
|
+
this.blockedResources = normalizeBlocked(options.blockResources);
|
|
1593
|
+
this.blockedUrls = normalizeUrls(options.blockUrls);
|
|
1594
|
+
this.connectTimeout = options.connectTimeout ?? 3e4;
|
|
1595
|
+
this.commandTimeout = options.commandTimeout === void 0 ? 6e4 : options.commandTimeout;
|
|
1596
|
+
this.logger = makeLogger(options.logger);
|
|
1597
|
+
this.#createWebSocket = options.createWebSocket;
|
|
1598
|
+
this.#onClose = options.onClose;
|
|
1599
|
+
this._signal = this.#abort.signal;
|
|
1600
|
+
}
|
|
1601
|
+
get internalUuid() {
|
|
1602
|
+
return this.session.internal_uuid;
|
|
1603
|
+
}
|
|
1604
|
+
get cdp() {
|
|
1605
|
+
if (this.#client === void 0) throw new Error("browser is not connected; call connect() first");
|
|
1606
|
+
return this.#client;
|
|
1607
|
+
}
|
|
1608
|
+
get connected() {
|
|
1609
|
+
return this.#client?.connected ?? false;
|
|
1610
|
+
}
|
|
1611
|
+
get shouldRecycle() {
|
|
1612
|
+
return this.#retired || !this.connected;
|
|
1613
|
+
}
|
|
1614
|
+
/**
|
|
1615
|
+
* Leases so far, the current one included. Rotate the browser from your own loop:
|
|
1616
|
+
*
|
|
1617
|
+
* if (browser.useCount >= 20) browser.retire();
|
|
1618
|
+
*/
|
|
1619
|
+
get useCount() {
|
|
1620
|
+
return this.#useCount;
|
|
1621
|
+
}
|
|
1622
|
+
/** @internal */
|
|
1623
|
+
_lease() {
|
|
1624
|
+
this.#useCount += 1;
|
|
1625
|
+
}
|
|
1626
|
+
get pages() {
|
|
1627
|
+
return [...this._pages.values()];
|
|
1628
|
+
}
|
|
1629
|
+
/** @internal what a page's Fetch.enable intercepts */
|
|
1630
|
+
get _fetchPatterns() {
|
|
1631
|
+
const patterns = [...this.blockedResources].sort().map((name) => ({
|
|
1632
|
+
urlPattern: "*",
|
|
1633
|
+
resourceType: RESOURCE_TYPES[name]
|
|
1634
|
+
}));
|
|
1635
|
+
for (const pattern of this.blockedUrls) patterns.push({ urlPattern: pattern });
|
|
1636
|
+
return [...patterns, STATUS_PATTERN];
|
|
1637
|
+
}
|
|
1638
|
+
/** Replace this browser with a fresh one after the lease. */
|
|
1639
|
+
retire() {
|
|
1640
|
+
this.#retired = true;
|
|
1641
|
+
}
|
|
1642
|
+
async newPage() {
|
|
1643
|
+
const deadline = new Deadline(this.commandTimeout, "the new page did not open");
|
|
1644
|
+
const created = await deadline.race(this.cdp.send("Target.createTarget", {
|
|
1645
|
+
url: "about:blank",
|
|
1646
|
+
newWindow: true
|
|
1647
|
+
}));
|
|
1648
|
+
return this._pageReady(created.targetId, deadline);
|
|
1649
|
+
}
|
|
1650
|
+
/**
|
|
1651
|
+
* Run `action`, the click that opens a window, and return the page it opened:
|
|
1652
|
+
*
|
|
1653
|
+
* const popup = await browser.waitForPage(browser.click("a[target=_blank]"));
|
|
1654
|
+
*
|
|
1655
|
+
* A site can open the window after an async step, so `pages` right after the
|
|
1656
|
+
* click may not have it yet.
|
|
1657
|
+
*/
|
|
1658
|
+
async waitForPage(action, options = {}) {
|
|
1659
|
+
const before = new Set(this._pages.keys());
|
|
1660
|
+
const deadline = new Deadline(options.timeout ?? 3e4, "no page opened");
|
|
1661
|
+
await deadline.race(typeof action === "function" ? action() : action);
|
|
1662
|
+
while (true) {
|
|
1663
|
+
const opened = [...this._pages.entries()].filter(([sessionId]) => !before.has(sessionId)).map(([, page]) => page);
|
|
1664
|
+
const last = opened[opened.length - 1];
|
|
1665
|
+
if (last !== void 0) return last;
|
|
1666
|
+
this._requireOpen();
|
|
1667
|
+
deadline.check();
|
|
1668
|
+
await sleep(100);
|
|
1669
|
+
}
|
|
1670
|
+
}
|
|
1671
|
+
async connect() {
|
|
1672
|
+
if (this.#client !== void 0) throw new Error("browser is already connected");
|
|
1673
|
+
const handler = this.onDialog;
|
|
1674
|
+
this._reset("", "");
|
|
1675
|
+
this.onDialog = handler;
|
|
1676
|
+
this._pages = /* @__PURE__ */ new Map();
|
|
1677
|
+
this.#abort = new AbortController();
|
|
1678
|
+
this._signal = this.#abort.signal;
|
|
1679
|
+
this.#closing = void 0;
|
|
1680
|
+
const deadline = new Deadline(this.connectTimeout, "could not connect");
|
|
1681
|
+
try {
|
|
1682
|
+
const client = new CDPClient(this.session.ws_url, {
|
|
1683
|
+
createWebSocket: this.#createWebSocket,
|
|
1684
|
+
onClose: () => this._onDisconnect(),
|
|
1685
|
+
logger: this.logger
|
|
1686
|
+
});
|
|
1687
|
+
this.#client = client;
|
|
1688
|
+
await deadline.race(client.start());
|
|
1689
|
+
client.on("Target.attachedToTarget", (event) => this._onAttached(event));
|
|
1690
|
+
client.on("Target.detachedFromTarget", (event) => this._onDetached(event));
|
|
1691
|
+
client.on("Inspector.targetCrashed", this.#toPage((page) => this._drop(page)));
|
|
1692
|
+
client.on("Page.lifecycleEvent", this.#toPage((page, event) => page._onLifecycle(event)));
|
|
1693
|
+
client.on("Fetch.requestPaused", this.#toPage((page, event) => page._onRequestPaused(event)));
|
|
1694
|
+
client.on("Network.responseReceived", this.#toPage((page, event) => page._onResponse(event)));
|
|
1695
|
+
client.on("Network.loadingFinished", this.#toPage((page, event) => page._onBodyReady(event)));
|
|
1696
|
+
client.on("Page.javascriptDialogOpening", this.#toPage((page, event) => page._onDialog(event)));
|
|
1697
|
+
await deadline.race(client.send("Target.setAutoAttach", AUTO_ATTACH));
|
|
1698
|
+
await deadline.race(client.send("Target.getTargets"));
|
|
1699
|
+
if (!this._sessionId) {
|
|
1700
|
+
const created = await deadline.race(client.send("Target.createTarget", { url: "about:blank" }));
|
|
1701
|
+
await this._pageReady(created.targetId, deadline);
|
|
1702
|
+
}
|
|
1703
|
+
await deadline.race(this._waitReady());
|
|
1704
|
+
} catch (err) {
|
|
1705
|
+
await this.close();
|
|
1706
|
+
throw err;
|
|
1707
|
+
}
|
|
1708
|
+
}
|
|
1709
|
+
close() {
|
|
1710
|
+
this.#closing ??= this.#close();
|
|
1711
|
+
return this.#closing;
|
|
1712
|
+
}
|
|
1713
|
+
async #close() {
|
|
1714
|
+
this.#abort.abort();
|
|
1715
|
+
const pending = [...this.#pending];
|
|
1716
|
+
this.#pending.clear();
|
|
1717
|
+
const client = this.#client;
|
|
1718
|
+
this.#client = void 0;
|
|
1719
|
+
await withTimeout(Promise.allSettled(pending), 5e3, () => new BrowserTimeoutError("close timed out")).catch((err) => this.logger.debug(`close: ${String(err)}`));
|
|
1720
|
+
if (client !== void 0) await client.stop();
|
|
1721
|
+
this._closed = true;
|
|
1722
|
+
await this.#onClose?.();
|
|
1723
|
+
}
|
|
1724
|
+
[Symbol.asyncDispose]() {
|
|
1725
|
+
return this.close();
|
|
1726
|
+
}
|
|
1727
|
+
/** @internal a lease ends: other pages closed, captures dropped, dialogs reset; `data` stays */
|
|
1728
|
+
async _endLease() {
|
|
1729
|
+
for (const page of this.pages) if (page !== this) await page.close();
|
|
1730
|
+
await this.stopCapturing();
|
|
1731
|
+
this.onDialog = null;
|
|
1732
|
+
const timeout = this.commandTimeout;
|
|
1733
|
+
await withTimeout(this.cdp.send("Browser.getVersion"), timeout, () => new BrowserTimeoutError(`the browser did not answer within ${timeout}ms`));
|
|
1734
|
+
}
|
|
1735
|
+
/** @internal */
|
|
1736
|
+
async _pageReady(targetId, deadline) {
|
|
1737
|
+
let page = this.#pageFor(targetId);
|
|
1738
|
+
while (page === void 0) {
|
|
1739
|
+
this._requireOpen();
|
|
1740
|
+
deadline.check();
|
|
1741
|
+
await sleep(100);
|
|
1742
|
+
page = this.#pageFor(targetId);
|
|
1743
|
+
}
|
|
1744
|
+
await deadline.race(page._waitReady());
|
|
1745
|
+
return page;
|
|
1746
|
+
}
|
|
1747
|
+
#pageFor(targetId) {
|
|
1748
|
+
for (const page of this._pages.values()) if (page._targetId === targetId) return page;
|
|
1749
|
+
}
|
|
1750
|
+
/** @internal */
|
|
1751
|
+
_onAttached(event) {
|
|
1752
|
+
const info = event.targetInfo ?? {};
|
|
1753
|
+
const sessionId = event.sessionId;
|
|
1754
|
+
const waiting = Boolean(event.waitingForDebugger);
|
|
1755
|
+
const url = info.url ?? "";
|
|
1756
|
+
const adoptable = isStartPage(url) && !this._sessionId;
|
|
1757
|
+
if (info.type !== "page" || NON_WEB_SCHEMES.some((scheme) => url.startsWith(scheme)) && !adoptable) {
|
|
1758
|
+
this._spawn(this.#letGo(sessionId, waiting));
|
|
1759
|
+
return;
|
|
1760
|
+
}
|
|
1761
|
+
let page;
|
|
1762
|
+
if (this._sessionId) page = new Page(this, info.targetId, sessionId);
|
|
1763
|
+
else {
|
|
1764
|
+
page = this;
|
|
1765
|
+
this._targetId = info.targetId;
|
|
1766
|
+
this._sessionId = sessionId;
|
|
1767
|
+
}
|
|
1768
|
+
page._frameId = page._targetId;
|
|
1769
|
+
page._ready = new Flag();
|
|
1770
|
+
this._pages.set(sessionId, page);
|
|
1771
|
+
this._spawn(page._setup(waiting));
|
|
1772
|
+
}
|
|
1773
|
+
async #letGo(sessionId, waiting) {
|
|
1774
|
+
const timeout = this.commandTimeout;
|
|
1775
|
+
const detach = async () => {
|
|
1776
|
+
if (waiting) await this.cdp.send("Runtime.runIfWaitingForDebugger", void 0, sessionId);
|
|
1777
|
+
await this.cdp.send("Target.detachFromTarget", { sessionId });
|
|
1778
|
+
};
|
|
1779
|
+
await withTimeout(detach(), timeout, () => new BrowserTimeoutError(`the target did not let go within ${timeout}ms`));
|
|
1780
|
+
}
|
|
1781
|
+
/** @internal */
|
|
1782
|
+
_onDetached(event) {
|
|
1783
|
+
const page = this._pages.get(String(event.sessionId ?? ""));
|
|
1784
|
+
if (page !== void 0) this._drop(page);
|
|
1785
|
+
}
|
|
1786
|
+
/** @internal */
|
|
1787
|
+
_drop(page) {
|
|
1788
|
+
this._pages.delete(page._sessionId);
|
|
1789
|
+
page._closed = true;
|
|
1790
|
+
page._waiter?.fail("page closed");
|
|
1791
|
+
if (page === this) {
|
|
1792
|
+
this.logger.warn(`page target gone for ${this.internalUuid}`);
|
|
1793
|
+
this.retire();
|
|
1794
|
+
}
|
|
1795
|
+
}
|
|
1796
|
+
#toPage(handler) {
|
|
1797
|
+
return (event, sessionId) => {
|
|
1798
|
+
const page = this._pages.get(sessionId ?? "");
|
|
1799
|
+
if (page !== void 0) handler(page, event);
|
|
1800
|
+
};
|
|
1801
|
+
}
|
|
1802
|
+
/** @internal */
|
|
1803
|
+
_onDisconnect() {
|
|
1804
|
+
for (const page of this._pages.values()) {
|
|
1805
|
+
page._closed = true;
|
|
1806
|
+
page._waiter?.fail("CDP connection closed");
|
|
1807
|
+
}
|
|
1808
|
+
}
|
|
1809
|
+
/** @internal track a background task; its failure is logged at debug level */
|
|
1810
|
+
_spawn(task) {
|
|
1811
|
+
const tracked = task.then(() => void 0, (err) => this.logger.debug(`background task failed: ${String(err)}`));
|
|
1812
|
+
this.#pending.add(tracked);
|
|
1813
|
+
tracked.then(() => this.#pending.delete(tracked));
|
|
1814
|
+
}
|
|
1815
|
+
};
|
|
1816
|
+
//#endregion
|
|
1817
|
+
//#region src/proxy.ts
|
|
1818
|
+
function proxyList(proxies, owner) {
|
|
1819
|
+
if (typeof proxies === "string") throw new TypeError(`${owner} takes an iterable of proxies, not a single URL`);
|
|
1820
|
+
const list = [...proxies];
|
|
1821
|
+
if (list.length === 0) throw new RangeError(`${owner} needs at least one proxy`);
|
|
1822
|
+
for (const proxy of list) parseProxyLike(proxy, owner);
|
|
1823
|
+
return list;
|
|
1824
|
+
}
|
|
1825
|
+
/** Round-robin over a list. */
|
|
1826
|
+
var ProxyCycle = class {
|
|
1827
|
+
proxies;
|
|
1828
|
+
#index = 0;
|
|
1829
|
+
constructor(proxies) {
|
|
1830
|
+
this.proxies = proxyList(proxies, "ProxyCycle");
|
|
1831
|
+
}
|
|
1832
|
+
pick() {
|
|
1833
|
+
const proxy = this.proxies[this.#index % this.proxies.length];
|
|
1834
|
+
this.#index += 1;
|
|
1835
|
+
return proxy;
|
|
1836
|
+
}
|
|
1837
|
+
};
|
|
1838
|
+
var ProxyRandom = class {
|
|
1839
|
+
proxies;
|
|
1840
|
+
constructor(proxies) {
|
|
1841
|
+
this.proxies = proxyList(proxies, "ProxyRandom");
|
|
1842
|
+
}
|
|
1843
|
+
pick() {
|
|
1844
|
+
return this.proxies[Math.floor(Math.random() * this.proxies.length)];
|
|
1845
|
+
}
|
|
1846
|
+
};
|
|
1847
|
+
const PLACEHOLDER = /\{\{|\}\}|\{([^{}]*)\}/g;
|
|
1848
|
+
function format(template, values) {
|
|
1849
|
+
return template.replace(PLACEHOLDER, (match, name) => {
|
|
1850
|
+
if (match === "{{") return "{";
|
|
1851
|
+
if (match === "}}") return "}";
|
|
1852
|
+
if (name === void 0 || !Object.hasOwn(values, name)) throw new TypeError(`unknown placeholder in proxy template: '${name ?? ""}'`);
|
|
1853
|
+
return String(values[name]);
|
|
1854
|
+
});
|
|
1855
|
+
}
|
|
1856
|
+
function hex(bytes) {
|
|
1857
|
+
const buffer = new Uint8Array(bytes);
|
|
1858
|
+
crypto.getRandomValues(buffer);
|
|
1859
|
+
return Array.from(buffer, (b) => b.toString(16).padStart(2, "0")).join("");
|
|
1860
|
+
}
|
|
1861
|
+
/**
|
|
1862
|
+
* A proxy URL from a template: `{session}` is a fresh hex id, `{n}` a counter.
|
|
1863
|
+
* Covers the sticky-session syntax providers put in the username:
|
|
1864
|
+
*
|
|
1865
|
+
* new ProxyTemplate("http://user-cc-us-sessid-{session}:pw@gate.example.com:7000")
|
|
1866
|
+
*
|
|
1867
|
+
* Literal braces are `{{` and `}}`.
|
|
1868
|
+
*/
|
|
1869
|
+
var ProxyTemplate = class {
|
|
1870
|
+
template;
|
|
1871
|
+
#count = 0;
|
|
1872
|
+
constructor(template) {
|
|
1873
|
+
format(template, {
|
|
1874
|
+
session: "0".repeat(12),
|
|
1875
|
+
n: 0
|
|
1876
|
+
});
|
|
1877
|
+
this.template = template;
|
|
1878
|
+
}
|
|
1879
|
+
pick() {
|
|
1880
|
+
const n = this.#count;
|
|
1881
|
+
this.#count += 1;
|
|
1882
|
+
return format(this.template, {
|
|
1883
|
+
session: hex(6),
|
|
1884
|
+
n
|
|
1885
|
+
});
|
|
1886
|
+
}
|
|
1887
|
+
};
|
|
1888
|
+
function isProxySource(value) {
|
|
1889
|
+
return isRecord(value) && typeof value.pick === "function";
|
|
1890
|
+
}
|
|
1891
|
+
async function resolveProxy(proxy) {
|
|
1892
|
+
return typeof proxy === "function" ? proxy() : isProxySource(proxy) ? proxy.pick() : proxy;
|
|
1893
|
+
}
|
|
1894
|
+
//#endregion
|
|
1895
|
+
//#region src/resources/profiles.ts
|
|
1896
|
+
const SCRAPE_TIMEOUT = 15e4;
|
|
1897
|
+
function sessionUuid(session) {
|
|
1898
|
+
return ref(typeof session === "string" ? session : session.internal_uuid);
|
|
1899
|
+
}
|
|
1900
|
+
function startOneTimeSpec(options, proxy) {
|
|
1901
|
+
return {
|
|
1902
|
+
method: "POST",
|
|
1903
|
+
path: "/profiles/one_time",
|
|
1904
|
+
json: parseOneTimeStartRequest({
|
|
1905
|
+
...options,
|
|
1906
|
+
proxy
|
|
1907
|
+
})
|
|
1908
|
+
};
|
|
1909
|
+
}
|
|
1910
|
+
function startSpec(uuid, options, proxy) {
|
|
1911
|
+
const request = parseProfileStartRequest({
|
|
1912
|
+
...options,
|
|
1913
|
+
proxy
|
|
1914
|
+
});
|
|
1915
|
+
return {
|
|
1916
|
+
method: "POST",
|
|
1917
|
+
path: `/profiles/${ref(uuid)}/start`,
|
|
1918
|
+
json: request
|
|
1919
|
+
};
|
|
1920
|
+
}
|
|
1921
|
+
function stopSpec(session) {
|
|
1922
|
+
return {
|
|
1923
|
+
method: "POST",
|
|
1924
|
+
path: `/profiles/${sessionUuid(session)}/stop`,
|
|
1925
|
+
parse: (data) => data ? data : null
|
|
1926
|
+
};
|
|
1927
|
+
}
|
|
1928
|
+
function stopAllSpec() {
|
|
1929
|
+
return {
|
|
1930
|
+
method: "POST",
|
|
1931
|
+
path: "/profiles/stop"
|
|
1932
|
+
};
|
|
1933
|
+
}
|
|
1934
|
+
function listActiveSpec() {
|
|
1935
|
+
return {
|
|
1936
|
+
method: "GET",
|
|
1937
|
+
path: "/profiles/active",
|
|
1938
|
+
parse: (data) => data ?? []
|
|
1939
|
+
};
|
|
1940
|
+
}
|
|
1941
|
+
function createSpec(options, proxy) {
|
|
1942
|
+
return {
|
|
1943
|
+
method: "POST",
|
|
1944
|
+
path: "/profiles",
|
|
1945
|
+
json: parseProfileCreateRequest({
|
|
1946
|
+
...options,
|
|
1947
|
+
proxy
|
|
1948
|
+
})
|
|
1949
|
+
};
|
|
1950
|
+
}
|
|
1951
|
+
function getSpec$1(uuid) {
|
|
1952
|
+
return {
|
|
1953
|
+
method: "GET",
|
|
1954
|
+
path: `/profiles/${ref(uuid)}`
|
|
1955
|
+
};
|
|
1956
|
+
}
|
|
1957
|
+
function updateSpec$1(uuid, fields) {
|
|
1958
|
+
const body = parseProfileUpdateRequest(fields);
|
|
1959
|
+
for (const [key, value] of Object.entries(fields)) if (value === null) body[key] = null;
|
|
1960
|
+
return {
|
|
1961
|
+
method: "PATCH",
|
|
1962
|
+
path: `/profiles/${ref(uuid)}`,
|
|
1963
|
+
json: body
|
|
1964
|
+
};
|
|
1965
|
+
}
|
|
1966
|
+
function deleteSpec$1(uuid) {
|
|
1967
|
+
return {
|
|
1968
|
+
method: "DELETE",
|
|
1969
|
+
path: `/profiles/${ref(uuid)}`
|
|
1970
|
+
};
|
|
1971
|
+
}
|
|
1972
|
+
function deleteManySpec(uuids) {
|
|
1973
|
+
return {
|
|
1974
|
+
method: "DELETE",
|
|
1975
|
+
path: "/profiles",
|
|
1976
|
+
json: { uuids },
|
|
1977
|
+
acceptError: (status, body) => status === 400 && isRecord(body) && isRecord(body.data) && "deleted_uuids" in body.data
|
|
1978
|
+
};
|
|
1979
|
+
}
|
|
1980
|
+
function listPageSpec(options) {
|
|
1981
|
+
const { page, page_len, ordering } = parseListPageRequest(options);
|
|
1982
|
+
return {
|
|
1983
|
+
method: "GET",
|
|
1984
|
+
path: "/profiles",
|
|
1985
|
+
params: {
|
|
1986
|
+
page,
|
|
1987
|
+
page_len,
|
|
1988
|
+
ordering
|
|
1989
|
+
},
|
|
1990
|
+
parse: (data) => data ?? []
|
|
1991
|
+
};
|
|
1992
|
+
}
|
|
1993
|
+
function exportCookiesSpec(uuid, exportFormat) {
|
|
1994
|
+
return {
|
|
1995
|
+
method: "GET",
|
|
1996
|
+
path: `/profiles/${ref(uuid)}/cookies`,
|
|
1997
|
+
params: { export_format: exportFormat },
|
|
1998
|
+
parse: (data) => {
|
|
1999
|
+
const cookies = isRecord(data) ? data.cookies : void 0;
|
|
2000
|
+
if (typeof cookies === "string") return cookies;
|
|
2001
|
+
return cookies ?? [];
|
|
2002
|
+
}
|
|
2003
|
+
};
|
|
2004
|
+
}
|
|
2005
|
+
function importCookiesSpec(uuid, cookies) {
|
|
2006
|
+
const text = typeof cookies === "string" ? cookies : JSON.stringify(dropNulls(cookies));
|
|
2007
|
+
return {
|
|
2008
|
+
method: "POST",
|
|
2009
|
+
path: `/profiles/${ref(uuid)}/cookies`,
|
|
2010
|
+
json: { cookies: text },
|
|
2011
|
+
parse: () => void 0
|
|
2012
|
+
};
|
|
2013
|
+
}
|
|
2014
|
+
function scrapeSpec(session, request) {
|
|
2015
|
+
return {
|
|
2016
|
+
method: "POST",
|
|
2017
|
+
path: `/profiles/${sessionUuid(session)}/scrape`,
|
|
2018
|
+
json: parseScrapeRequest(request),
|
|
2019
|
+
timeout: SCRAPE_TIMEOUT
|
|
2020
|
+
};
|
|
2021
|
+
}
|
|
2022
|
+
const STOP_TIMEOUT = 8e3;
|
|
2023
|
+
const STOP_DEADLINE = 1e4;
|
|
2024
|
+
/** One fast attempt to stop a session; a failure is logged, never thrown. */
|
|
2025
|
+
async function stopSession(client, internalUuid) {
|
|
2026
|
+
const fast = client.withOptions({
|
|
2027
|
+
timeout: STOP_TIMEOUT,
|
|
2028
|
+
maxRetries: 0
|
|
2029
|
+
});
|
|
2030
|
+
const timedOut = Symbol("timed out");
|
|
2031
|
+
try {
|
|
2032
|
+
await withTimeout(fast.profiles.stop(internalUuid), STOP_DEADLINE, () => Object.assign(/* @__PURE__ */ new Error("stop timed out"), { [timedOut]: true }));
|
|
2033
|
+
} catch (err) {
|
|
2034
|
+
if (err instanceof Error && timedOut in err) client.logger.warn(`stopping session ${internalUuid} timed out; it may keep billing`);
|
|
2035
|
+
else client.logger.warn(`failed to stop session ${internalUuid}: ${String(err)}`);
|
|
2036
|
+
}
|
|
2037
|
+
}
|
|
2038
|
+
var Profiles = class {
|
|
2039
|
+
client;
|
|
2040
|
+
constructor(client) {
|
|
2041
|
+
this.client = client;
|
|
2042
|
+
}
|
|
2043
|
+
/** Start a session on a fresh browser that is gone once stopped. */
|
|
2044
|
+
async startOneTime(options = {}) {
|
|
2045
|
+
const proxy = await resolveProxy(options.proxy);
|
|
2046
|
+
return this.client.call(startOneTimeSpec(options, proxy));
|
|
2047
|
+
}
|
|
2048
|
+
/** Start a session on a saved profile. */
|
|
2049
|
+
async start(uuid, options = {}) {
|
|
2050
|
+
const proxy = await resolveProxy(options.proxy);
|
|
2051
|
+
return this.client.call(startSpec(uuid, options, proxy));
|
|
2052
|
+
}
|
|
2053
|
+
async stop(session) {
|
|
2054
|
+
return this.client.call(stopSpec(session));
|
|
2055
|
+
}
|
|
2056
|
+
async stopAll() {
|
|
2057
|
+
return this.client.call(stopAllSpec());
|
|
2058
|
+
}
|
|
2059
|
+
async listActive() {
|
|
2060
|
+
return this.client.call(listActiveSpec());
|
|
2061
|
+
}
|
|
2062
|
+
async create(options) {
|
|
2063
|
+
const proxy = await resolveProxy(options.proxy);
|
|
2064
|
+
return this.client.call(createSpec(options, proxy));
|
|
2065
|
+
}
|
|
2066
|
+
async get(uuid) {
|
|
2067
|
+
return this.client.call(getSpec$1(uuid));
|
|
2068
|
+
}
|
|
2069
|
+
async update(uuid, fields) {
|
|
2070
|
+
const { proxy, ...rest } = fields;
|
|
2071
|
+
return this.client.call(updateSpec$1(uuid, {
|
|
2072
|
+
...rest,
|
|
2073
|
+
proxy: await resolveProxy(proxy)
|
|
2074
|
+
}));
|
|
2075
|
+
}
|
|
2076
|
+
async delete(uuid) {
|
|
2077
|
+
return this.client.call(deleteSpec$1(uuid));
|
|
2078
|
+
}
|
|
2079
|
+
async deleteMany(uuids) {
|
|
2080
|
+
return this.client.call(deleteManySpec(uuids));
|
|
2081
|
+
}
|
|
2082
|
+
async listPage(options = {}) {
|
|
2083
|
+
return this.client.call(listPageSpec(options));
|
|
2084
|
+
}
|
|
2085
|
+
/** Every profile, page by page. */
|
|
2086
|
+
async *iterAll(options = {}) {
|
|
2087
|
+
const pageLen = Math.max(1, Math.min(options.page_len ?? 100, 100));
|
|
2088
|
+
const ordering = options.ordering ?? "created";
|
|
2089
|
+
for (let page = 0;; page++) {
|
|
2090
|
+
const batch = await this.listPage({
|
|
2091
|
+
page,
|
|
2092
|
+
page_len: pageLen,
|
|
2093
|
+
ordering
|
|
2094
|
+
});
|
|
2095
|
+
yield* batch;
|
|
2096
|
+
if (batch.length < pageLen) return;
|
|
2097
|
+
}
|
|
2098
|
+
}
|
|
2099
|
+
async exportCookies(uuid, options = {}) {
|
|
2100
|
+
return this.client.call(exportCookiesSpec(uuid, options.export_format ?? "json"));
|
|
2101
|
+
}
|
|
2102
|
+
async importCookies(uuid, cookies) {
|
|
2103
|
+
return this.client.call(importCookiesSpec(uuid, cookies));
|
|
2104
|
+
}
|
|
2105
|
+
async scrape(session, url, options = {}) {
|
|
2106
|
+
return this.client.call(scrapeSpec(session, {
|
|
2107
|
+
url,
|
|
2108
|
+
...options
|
|
2109
|
+
}));
|
|
2110
|
+
}
|
|
2111
|
+
};
|
|
2112
|
+
//#endregion
|
|
2113
|
+
//#region src/browser/pool.ts
|
|
2114
|
+
/** Throw from a `map` handler to end the run after this item. */
|
|
2115
|
+
var StopRun = class extends Error {
|
|
2116
|
+
constructor(message = "run stopped") {
|
|
2117
|
+
super(message);
|
|
2118
|
+
this.name = "StopRun";
|
|
2119
|
+
}
|
|
2120
|
+
};
|
|
2121
|
+
function planIsFull(err) {
|
|
2122
|
+
return err instanceof RateLimitError && err.code === "parallel_browsers_limit_reached";
|
|
2123
|
+
}
|
|
2124
|
+
var Semaphore = class {
|
|
2125
|
+
#free;
|
|
2126
|
+
#waiters = [];
|
|
2127
|
+
constructor(slots) {
|
|
2128
|
+
this.#free = slots;
|
|
2129
|
+
}
|
|
2130
|
+
acquire() {
|
|
2131
|
+
if (this.#free > 0) {
|
|
2132
|
+
this.#free -= 1;
|
|
2133
|
+
return Promise.resolve();
|
|
2134
|
+
}
|
|
2135
|
+
return new Promise((resolve) => this.#waiters.push(resolve));
|
|
2136
|
+
}
|
|
2137
|
+
release() {
|
|
2138
|
+
const next = this.#waiters.shift();
|
|
2139
|
+
if (next !== void 0) next();
|
|
2140
|
+
else this.#free += 1;
|
|
2141
|
+
}
|
|
2142
|
+
};
|
|
2143
|
+
var BrowserPool = class {
|
|
2144
|
+
sessionOptions;
|
|
2145
|
+
blockedResources;
|
|
2146
|
+
blockedUrls;
|
|
2147
|
+
#client;
|
|
2148
|
+
#concurrency;
|
|
2149
|
+
#createWebSocket;
|
|
2150
|
+
#idle = [];
|
|
2151
|
+
#owned = 0;
|
|
2152
|
+
#planFull = false;
|
|
2153
|
+
#planFullError;
|
|
2154
|
+
#waiters = [];
|
|
2155
|
+
#capacity = 0;
|
|
2156
|
+
#slots;
|
|
2157
|
+
constructor(client, options = {}) {
|
|
2158
|
+
const { concurrency = "auto", blockResources, blockUrls, createWebSocket, ...sessionOptions } = options;
|
|
2159
|
+
if (concurrency !== "auto" && !Number.isInteger(concurrency)) throw new TypeError("concurrency must be a whole number or \"auto\"");
|
|
2160
|
+
this.#client = client;
|
|
2161
|
+
this.#concurrency = concurrency;
|
|
2162
|
+
this.blockedResources = normalizeBlocked(blockResources);
|
|
2163
|
+
this.blockedUrls = normalizeUrls(blockUrls);
|
|
2164
|
+
this.#createWebSocket = createWebSocket;
|
|
2165
|
+
const { proxy } = sessionOptions;
|
|
2166
|
+
parseOneTimeStartRequest({
|
|
2167
|
+
...sessionOptions,
|
|
2168
|
+
proxy: typeof proxy === "function" || isProxySource(proxy) ? void 0 : proxy
|
|
2169
|
+
});
|
|
2170
|
+
this.sessionOptions = sessionOptions;
|
|
2171
|
+
}
|
|
2172
|
+
/** Max live browsers. */
|
|
2173
|
+
get capacity() {
|
|
2174
|
+
this.#requireOpen();
|
|
2175
|
+
return this.#capacity;
|
|
2176
|
+
}
|
|
2177
|
+
#requireOpen() {
|
|
2178
|
+
if (this.#slots === void 0) throw new Error("the pool is not open: use `await client.browsers()` or `open()`");
|
|
2179
|
+
return this.#slots;
|
|
2180
|
+
}
|
|
2181
|
+
async open() {
|
|
2182
|
+
const total = this.#concurrency === "auto" ? await this.#client.account.maxBrowsers() : this.#concurrency;
|
|
2183
|
+
this.#capacity = Math.max(1, total);
|
|
2184
|
+
this.#slots = new Semaphore(this.#capacity);
|
|
2185
|
+
this.#planFull = false;
|
|
2186
|
+
this.#planFullError = void 0;
|
|
2187
|
+
this.#owned = 0;
|
|
2188
|
+
return this;
|
|
2189
|
+
}
|
|
2190
|
+
/** Stop every idle browser. A browser still on lease is stopped when its lease ends. */
|
|
2191
|
+
async close() {
|
|
2192
|
+
this.#slots = void 0;
|
|
2193
|
+
const idle = this.#idle;
|
|
2194
|
+
this.#idle = [];
|
|
2195
|
+
this.#owned -= idle.length;
|
|
2196
|
+
await Promise.all(idle.map((browser) => this.#teardown(browser)));
|
|
2197
|
+
}
|
|
2198
|
+
[Symbol.asyncDispose]() {
|
|
2199
|
+
return this.close();
|
|
2200
|
+
}
|
|
2201
|
+
/** Run `fn` on a live browser, then hand the browser back. Waits while all are busy. */
|
|
2202
|
+
async lease(fn) {
|
|
2203
|
+
const slots = this.#requireOpen();
|
|
2204
|
+
await slots.acquire();
|
|
2205
|
+
try {
|
|
2206
|
+
if (this.#slots !== slots) throw new Error("the pool was closed while waiting for a slot");
|
|
2207
|
+
const browser = await this.#acquire();
|
|
2208
|
+
browser._lease();
|
|
2209
|
+
try {
|
|
2210
|
+
return await fn(browser);
|
|
2211
|
+
} finally {
|
|
2212
|
+
await this.#release(browser);
|
|
2213
|
+
}
|
|
2214
|
+
} finally {
|
|
2215
|
+
slots.release();
|
|
2216
|
+
}
|
|
2217
|
+
}
|
|
2218
|
+
#wait() {
|
|
2219
|
+
return new Promise((resolve) => this.#waiters.push(resolve));
|
|
2220
|
+
}
|
|
2221
|
+
#notifyAll() {
|
|
2222
|
+
const waiters = this.#waiters;
|
|
2223
|
+
this.#waiters = [];
|
|
2224
|
+
for (const wake of waiters) wake();
|
|
2225
|
+
}
|
|
2226
|
+
async #acquire() {
|
|
2227
|
+
while (true) {
|
|
2228
|
+
const dead = this.#idle.filter((browser) => browser.shouldRecycle);
|
|
2229
|
+
if (dead.length > 0) {
|
|
2230
|
+
this.#idle = this.#idle.filter((browser) => !browser.shouldRecycle);
|
|
2231
|
+
this.#owned -= dead.length;
|
|
2232
|
+
this.#planFull = false;
|
|
2233
|
+
this.#notifyAll();
|
|
2234
|
+
await Promise.all(dead.map((browser) => this.#teardown(browser)));
|
|
2235
|
+
continue;
|
|
2236
|
+
}
|
|
2237
|
+
const idle = this.#idle.pop();
|
|
2238
|
+
if (idle !== void 0) return idle;
|
|
2239
|
+
if (this.#planFull) {
|
|
2240
|
+
if (this.#owned === 0) throw this.#planFullError ?? new RateLimitError("parallel browser limit reached", { statusCode: 429 });
|
|
2241
|
+
await this.#wait();
|
|
2242
|
+
continue;
|
|
2243
|
+
}
|
|
2244
|
+
this.#owned += 1;
|
|
2245
|
+
let browser;
|
|
2246
|
+
try {
|
|
2247
|
+
browser = await this.#startBrowser();
|
|
2248
|
+
} catch (err) {
|
|
2249
|
+
const full = planIsFull(err);
|
|
2250
|
+
this.#owned -= 1;
|
|
2251
|
+
if (full) {
|
|
2252
|
+
this.#planFull = true;
|
|
2253
|
+
this.#planFullError = err;
|
|
2254
|
+
}
|
|
2255
|
+
const nothingLeft = this.#owned === 0;
|
|
2256
|
+
this.#notifyAll();
|
|
2257
|
+
if (!full || nothingLeft) throw err;
|
|
2258
|
+
this.#client.logger.info("plan is full, waiting for a browser of ours");
|
|
2259
|
+
continue;
|
|
2260
|
+
}
|
|
2261
|
+
this.#planFull = false;
|
|
2262
|
+
return browser;
|
|
2263
|
+
}
|
|
2264
|
+
}
|
|
2265
|
+
async #release(browser) {
|
|
2266
|
+
if (!browser.shouldRecycle) try {
|
|
2267
|
+
await withTimeout(browser._endLease(), 5e3, () => new BrowserTimeoutError("lease cleanup timed out"));
|
|
2268
|
+
} catch (err) {
|
|
2269
|
+
this.#client.logger.warn(`could not clean up ${browser.internalUuid}: ${String(err)}`);
|
|
2270
|
+
browser.retire();
|
|
2271
|
+
}
|
|
2272
|
+
const recycle = browser.shouldRecycle || this.#slots === void 0;
|
|
2273
|
+
if (recycle) await this.#teardown(browser);
|
|
2274
|
+
if (recycle) {
|
|
2275
|
+
this.#owned -= 1;
|
|
2276
|
+
this.#planFull = false;
|
|
2277
|
+
} else this.#idle.push(browser);
|
|
2278
|
+
this.#notifyAll();
|
|
2279
|
+
}
|
|
2280
|
+
/** Run every item through `handler` on a leased browser, `capacity` at a time. */
|
|
2281
|
+
async map(handler, items) {
|
|
2282
|
+
const work = items[Symbol.iterator]();
|
|
2283
|
+
const outcomes = [];
|
|
2284
|
+
let index = 0;
|
|
2285
|
+
let stopped = false;
|
|
2286
|
+
const worker = async () => {
|
|
2287
|
+
while (!stopped) {
|
|
2288
|
+
const next = work.next();
|
|
2289
|
+
if (next.done) return;
|
|
2290
|
+
const item = next.value;
|
|
2291
|
+
const at = index;
|
|
2292
|
+
index += 1;
|
|
2293
|
+
try {
|
|
2294
|
+
const value = await this.lease((browser) => handler(browser, item));
|
|
2295
|
+
outcomes.push({
|
|
2296
|
+
ok: true,
|
|
2297
|
+
item,
|
|
2298
|
+
index: at,
|
|
2299
|
+
value
|
|
2300
|
+
});
|
|
2301
|
+
} catch (error) {
|
|
2302
|
+
outcomes.push({
|
|
2303
|
+
ok: false,
|
|
2304
|
+
item,
|
|
2305
|
+
index: at,
|
|
2306
|
+
error
|
|
2307
|
+
});
|
|
2308
|
+
if (error instanceof StopRun) {
|
|
2309
|
+
stopped = true;
|
|
2310
|
+
return;
|
|
2311
|
+
}
|
|
2312
|
+
}
|
|
2313
|
+
}
|
|
2314
|
+
};
|
|
2315
|
+
await Promise.all(Array.from({ length: this.capacity }, worker));
|
|
2316
|
+
return outcomes.sort((a, b) => a.index - b.index);
|
|
2317
|
+
}
|
|
2318
|
+
async #startBrowser() {
|
|
2319
|
+
const session = await this.#client.profiles.startOneTime(this.sessionOptions);
|
|
2320
|
+
let browser;
|
|
2321
|
+
try {
|
|
2322
|
+
browser = new Browser(session, {
|
|
2323
|
+
blockResources: this.blockedResources,
|
|
2324
|
+
blockUrls: this.blockedUrls,
|
|
2325
|
+
createWebSocket: this.#createWebSocket,
|
|
2326
|
+
logger: this.#client.logger
|
|
2327
|
+
});
|
|
2328
|
+
await browser.connect();
|
|
2329
|
+
} catch (err) {
|
|
2330
|
+
await stopSession(this.#client, session.internal_uuid);
|
|
2331
|
+
throw err;
|
|
2332
|
+
}
|
|
2333
|
+
this.#client.logger.info(`pool browser ${browser.internalUuid} started`);
|
|
2334
|
+
return browser;
|
|
2335
|
+
}
|
|
2336
|
+
async #teardown(browser) {
|
|
2337
|
+
await browser.close();
|
|
2338
|
+
await stopSession(this.#client, browser.internalUuid);
|
|
2339
|
+
}
|
|
2340
|
+
};
|
|
2341
|
+
function sessionLimitsSpec() {
|
|
2342
|
+
return {
|
|
2343
|
+
method: "GET",
|
|
2344
|
+
path: "/users/session-limits"
|
|
2345
|
+
};
|
|
2346
|
+
}
|
|
2347
|
+
function browserLimitsSpec() {
|
|
2348
|
+
return {
|
|
2349
|
+
method: "GET",
|
|
2350
|
+
path: "/users/browser-limits"
|
|
2351
|
+
};
|
|
2352
|
+
}
|
|
2353
|
+
function envMaxBrowsers() {
|
|
2354
|
+
const value = process.env.SURFSKY_MAX_BROWSERS ?? "";
|
|
2355
|
+
return /^\d+$/.test(value) ? Math.max(1, Number(value)) : void 0;
|
|
2356
|
+
}
|
|
2357
|
+
var Account = class {
|
|
2358
|
+
client;
|
|
2359
|
+
constructor(client) {
|
|
2360
|
+
this.client = client;
|
|
2361
|
+
}
|
|
2362
|
+
async sessionLimits() {
|
|
2363
|
+
return this.client.call(sessionLimitsSpec());
|
|
2364
|
+
}
|
|
2365
|
+
async browserLimits() {
|
|
2366
|
+
return this.client.call(browserLimitsSpec());
|
|
2367
|
+
}
|
|
2368
|
+
/** The plan's parallel-browser cap; `SURFSKY_MAX_BROWSERS` overrides it. */
|
|
2369
|
+
async maxBrowsers() {
|
|
2370
|
+
return envMaxBrowsers() ?? ((await this.#limits()).parallel_browsers || 10);
|
|
2371
|
+
}
|
|
2372
|
+
async #limits() {
|
|
2373
|
+
try {
|
|
2374
|
+
return await this.browserLimits();
|
|
2375
|
+
} catch (err) {
|
|
2376
|
+
if (err instanceof NotFoundError) return {};
|
|
2377
|
+
throw err;
|
|
2378
|
+
}
|
|
2379
|
+
}
|
|
2380
|
+
};
|
|
2381
|
+
//#endregion
|
|
2382
|
+
//#region src/resources/extensions.ts
|
|
2383
|
+
async function readExtension(file) {
|
|
2384
|
+
if (typeof file === "string") {
|
|
2385
|
+
const [{ readFile }, { basename }] = await Promise.all([import("node:fs/promises"), import("node:path")]);
|
|
2386
|
+
const bytes = new Uint8Array(await readFile(file));
|
|
2387
|
+
return {
|
|
2388
|
+
name: basename(file),
|
|
2389
|
+
blob: new Blob([bytes], { type: "application/zip" })
|
|
2390
|
+
};
|
|
2391
|
+
}
|
|
2392
|
+
if (file instanceof Blob) return {
|
|
2393
|
+
name: "name" in file && typeof file.name === "string" ? file.name : "extension.zip",
|
|
2394
|
+
blob: file
|
|
2395
|
+
};
|
|
2396
|
+
return {
|
|
2397
|
+
name: "extension.zip",
|
|
2398
|
+
blob: new Blob([new Uint8Array(file)], { type: "application/zip" })
|
|
2399
|
+
};
|
|
2400
|
+
}
|
|
2401
|
+
function uploadSpec(name, filename, blob) {
|
|
2402
|
+
const form = new FormData();
|
|
2403
|
+
form.append("file", blob, filename);
|
|
2404
|
+
form.append("name", name);
|
|
2405
|
+
return {
|
|
2406
|
+
method: "POST",
|
|
2407
|
+
path: "/extensions",
|
|
2408
|
+
body: form
|
|
2409
|
+
};
|
|
2410
|
+
}
|
|
2411
|
+
function listAllSpec() {
|
|
2412
|
+
return {
|
|
2413
|
+
method: "GET",
|
|
2414
|
+
path: "/extensions",
|
|
2415
|
+
parse: (data) => isRecord(data) ? data.extensions ?? [] : []
|
|
2416
|
+
};
|
|
2417
|
+
}
|
|
2418
|
+
function getSpec(uuid) {
|
|
2419
|
+
return {
|
|
2420
|
+
method: "GET",
|
|
2421
|
+
path: `/extensions/${ref(uuid)}`
|
|
2422
|
+
};
|
|
2423
|
+
}
|
|
2424
|
+
function updateSpec(uuid, name) {
|
|
2425
|
+
return {
|
|
2426
|
+
method: "PATCH",
|
|
2427
|
+
path: `/extensions/${ref(uuid)}`,
|
|
2428
|
+
json: { name }
|
|
2429
|
+
};
|
|
2430
|
+
}
|
|
2431
|
+
function deleteSpec(uuid) {
|
|
2432
|
+
return {
|
|
2433
|
+
method: "DELETE",
|
|
2434
|
+
path: `/extensions/${ref(uuid)}`,
|
|
2435
|
+
parse: () => void 0
|
|
2436
|
+
};
|
|
2437
|
+
}
|
|
2438
|
+
var Extensions = class {
|
|
2439
|
+
client;
|
|
2440
|
+
constructor(client) {
|
|
2441
|
+
this.client = client;
|
|
2442
|
+
}
|
|
2443
|
+
/** Upload a zip (max 100 MB) given as a path, bytes or a Blob. */
|
|
2444
|
+
async upload(file, name) {
|
|
2445
|
+
const { name: filename, blob } = await readExtension(file);
|
|
2446
|
+
return this.client.call(uploadSpec(name, filename, blob));
|
|
2447
|
+
}
|
|
2448
|
+
async listAll() {
|
|
2449
|
+
return this.client.call(listAllSpec());
|
|
2450
|
+
}
|
|
2451
|
+
async get(uuid) {
|
|
2452
|
+
return this.client.call(getSpec(uuid));
|
|
2453
|
+
}
|
|
2454
|
+
async update(uuid, options) {
|
|
2455
|
+
return this.client.call(updateSpec(uuid, options.name));
|
|
2456
|
+
}
|
|
2457
|
+
async delete(uuid) {
|
|
2458
|
+
return this.client.call(deleteSpec(uuid));
|
|
2459
|
+
}
|
|
2460
|
+
};
|
|
2461
|
+
//#endregion
|
|
2462
|
+
//#region src/resources/fingerprints.ts
|
|
2463
|
+
const list$1 = (data) => data ?? [];
|
|
2464
|
+
function renderersSpec(os, osArch) {
|
|
2465
|
+
return {
|
|
2466
|
+
method: "GET",
|
|
2467
|
+
path: "/fingerprint/renderers",
|
|
2468
|
+
params: {
|
|
2469
|
+
os,
|
|
2470
|
+
os_arch: osArch
|
|
2471
|
+
},
|
|
2472
|
+
parse: list$1
|
|
2473
|
+
};
|
|
2474
|
+
}
|
|
2475
|
+
function screensSpec(os, osArch) {
|
|
2476
|
+
return {
|
|
2477
|
+
method: "GET",
|
|
2478
|
+
path: "/fingerprint/screens",
|
|
2479
|
+
params: {
|
|
2480
|
+
os,
|
|
2481
|
+
os_arch: osArch
|
|
2482
|
+
},
|
|
2483
|
+
parse: list$1
|
|
2484
|
+
};
|
|
2485
|
+
}
|
|
2486
|
+
function deviceModelsSpec(options) {
|
|
2487
|
+
return {
|
|
2488
|
+
method: "GET",
|
|
2489
|
+
path: "/fingerprint/device_models",
|
|
2490
|
+
params: {
|
|
2491
|
+
os: options.os,
|
|
2492
|
+
os_arch: options.os_arch,
|
|
2493
|
+
os_version: options.os_version,
|
|
2494
|
+
device_type: options.device_type
|
|
2495
|
+
},
|
|
2496
|
+
parse: list$1
|
|
2497
|
+
};
|
|
2498
|
+
}
|
|
2499
|
+
var Fingerprints = class {
|
|
2500
|
+
client;
|
|
2501
|
+
constructor(client) {
|
|
2502
|
+
this.client = client;
|
|
2503
|
+
}
|
|
2504
|
+
async renderers(os, osArch) {
|
|
2505
|
+
return this.client.call(renderersSpec(os, osArch));
|
|
2506
|
+
}
|
|
2507
|
+
async screens(os, osArch) {
|
|
2508
|
+
return this.client.call(screensSpec(os, osArch));
|
|
2509
|
+
}
|
|
2510
|
+
async deviceModels(options = {}) {
|
|
2511
|
+
return this.client.call(deviceModelsSpec(options));
|
|
2512
|
+
}
|
|
2513
|
+
};
|
|
2514
|
+
//#endregion
|
|
2515
|
+
//#region src/resources/proxies.ts
|
|
2516
|
+
const list = (data) => data ?? [];
|
|
2517
|
+
function countriesSpec() {
|
|
2518
|
+
return {
|
|
2519
|
+
method: "GET",
|
|
2520
|
+
path: "/proxies/countries",
|
|
2521
|
+
parse: list
|
|
2522
|
+
};
|
|
2523
|
+
}
|
|
2524
|
+
function regionsSpec(country) {
|
|
2525
|
+
return {
|
|
2526
|
+
method: "GET",
|
|
2527
|
+
path: `/proxies/regions/${ref(country)}`,
|
|
2528
|
+
parse: list
|
|
2529
|
+
};
|
|
2530
|
+
}
|
|
2531
|
+
function citiesSpec(country, region) {
|
|
2532
|
+
return {
|
|
2533
|
+
method: "GET",
|
|
2534
|
+
path: `/proxies/cities/${ref(country)}/${ref(region)}`,
|
|
2535
|
+
parse: list
|
|
2536
|
+
};
|
|
2537
|
+
}
|
|
2538
|
+
function quotaSpec() {
|
|
2539
|
+
return {
|
|
2540
|
+
method: "GET",
|
|
2541
|
+
path: "/proxies/quota"
|
|
2542
|
+
};
|
|
2543
|
+
}
|
|
2544
|
+
function premiumStatsSpec() {
|
|
2545
|
+
return {
|
|
2546
|
+
method: "GET",
|
|
2547
|
+
path: "/proxies/premium/stats"
|
|
2548
|
+
};
|
|
2549
|
+
}
|
|
2550
|
+
function sharedCountriesSpec() {
|
|
2551
|
+
return {
|
|
2552
|
+
method: "GET",
|
|
2553
|
+
path: "/proxies/shared/countries",
|
|
2554
|
+
parse: list
|
|
2555
|
+
};
|
|
2556
|
+
}
|
|
2557
|
+
function sharedQuotaSpec() {
|
|
2558
|
+
return {
|
|
2559
|
+
method: "GET",
|
|
2560
|
+
path: "/proxies/shared/quota"
|
|
2561
|
+
};
|
|
2562
|
+
}
|
|
2563
|
+
function sharedStatsSpec() {
|
|
2564
|
+
return {
|
|
2565
|
+
method: "GET",
|
|
2566
|
+
path: "/proxies/shared/stats"
|
|
2567
|
+
};
|
|
2568
|
+
}
|
|
2569
|
+
var Proxies = class {
|
|
2570
|
+
client;
|
|
2571
|
+
constructor(client) {
|
|
2572
|
+
this.client = client;
|
|
2573
|
+
}
|
|
2574
|
+
async countries() {
|
|
2575
|
+
return this.client.call(countriesSpec());
|
|
2576
|
+
}
|
|
2577
|
+
async regions(country) {
|
|
2578
|
+
return this.client.call(regionsSpec(country));
|
|
2579
|
+
}
|
|
2580
|
+
async cities(country, region) {
|
|
2581
|
+
return this.client.call(citiesSpec(country, region));
|
|
2582
|
+
}
|
|
2583
|
+
async quota() {
|
|
2584
|
+
return this.client.call(quotaSpec());
|
|
2585
|
+
}
|
|
2586
|
+
async premiumStats() {
|
|
2587
|
+
return this.client.call(premiumStatsSpec());
|
|
2588
|
+
}
|
|
2589
|
+
async sharedCountries() {
|
|
2590
|
+
return this.client.call(sharedCountriesSpec());
|
|
2591
|
+
}
|
|
2592
|
+
async sharedQuota() {
|
|
2593
|
+
return this.client.call(sharedQuotaSpec());
|
|
2594
|
+
}
|
|
2595
|
+
async sharedStats() {
|
|
2596
|
+
return this.client.call(sharedStatsSpec());
|
|
2597
|
+
}
|
|
2598
|
+
};
|
|
2599
|
+
//#endregion
|
|
2600
|
+
//#region src/version.ts
|
|
2601
|
+
const VERSION = "0.0.1";
|
|
2602
|
+
//#endregion
|
|
2603
|
+
//#region src/client.ts
|
|
2604
|
+
function userAgent() {
|
|
2605
|
+
const bun = globalThis.Bun;
|
|
2606
|
+
const runtime = bun ? `bun/${bun.version}` : `node/${process.versions.node}`;
|
|
2607
|
+
return `surfsky-js/${VERSION} ${runtime}`;
|
|
2608
|
+
}
|
|
2609
|
+
function connection(apiToken, baseUrl) {
|
|
2610
|
+
const token = apiToken || process.env.SURFSKY_API_TOKEN;
|
|
2611
|
+
if (!token) throw new ConfigurationError("pass apiToken or set SURFSKY_API_TOKEN");
|
|
2612
|
+
const url = baseUrl || process.env.SURFSKY_API_BASE_URL;
|
|
2613
|
+
if (!url) throw new ConfigurationError("pass baseUrl or set SURFSKY_API_BASE_URL");
|
|
2614
|
+
return {
|
|
2615
|
+
baseUrl: url.replace(/\/+$/, ""),
|
|
2616
|
+
headers: {
|
|
2617
|
+
[AUTH_HEADER]: token,
|
|
2618
|
+
Accept: "application/json",
|
|
2619
|
+
"User-Agent": userAgent()
|
|
2620
|
+
}
|
|
2621
|
+
};
|
|
2622
|
+
}
|
|
2623
|
+
var Surfsky = class Surfsky {
|
|
2624
|
+
baseUrl;
|
|
2625
|
+
headers;
|
|
2626
|
+
timeout;
|
|
2627
|
+
maxRetries;
|
|
2628
|
+
backoff;
|
|
2629
|
+
logger;
|
|
2630
|
+
fetch;
|
|
2631
|
+
profiles;
|
|
2632
|
+
proxies;
|
|
2633
|
+
fingerprints;
|
|
2634
|
+
extensions;
|
|
2635
|
+
account;
|
|
2636
|
+
constructor(options = {}) {
|
|
2637
|
+
const { baseUrl, headers } = connection(options.apiToken, options.baseUrl);
|
|
2638
|
+
this.baseUrl = baseUrl;
|
|
2639
|
+
this.headers = {
|
|
2640
|
+
...headers,
|
|
2641
|
+
...options.headers
|
|
2642
|
+
};
|
|
2643
|
+
this.timeout = options.timeout ?? 3e4;
|
|
2644
|
+
this.maxRetries = options.maxRetries ?? 3;
|
|
2645
|
+
this.backoff = options.backoff ?? 500;
|
|
2646
|
+
this.logger = makeLogger(options.logger);
|
|
2647
|
+
this.fetch = options.fetch ?? ((input, init) => globalThis.fetch(input, init));
|
|
2648
|
+
this.profiles = new Profiles(this);
|
|
2649
|
+
this.proxies = new Proxies(this);
|
|
2650
|
+
this.fingerprints = new Fingerprints(this);
|
|
2651
|
+
this.extensions = new Extensions(this);
|
|
2652
|
+
this.account = new Account(this);
|
|
2653
|
+
}
|
|
2654
|
+
withOptions(options) {
|
|
2655
|
+
return new Surfsky({
|
|
2656
|
+
apiToken: this.headers[AUTH_HEADER],
|
|
2657
|
+
baseUrl: this.baseUrl,
|
|
2658
|
+
timeout: options.timeout ?? this.timeout,
|
|
2659
|
+
maxRetries: options.maxRetries ?? this.maxRetries,
|
|
2660
|
+
backoff: this.backoff,
|
|
2661
|
+
headers: {
|
|
2662
|
+
...this.headers,
|
|
2663
|
+
...options.headers
|
|
2664
|
+
},
|
|
2665
|
+
logger: this.logger,
|
|
2666
|
+
fetch: this.fetch
|
|
2667
|
+
});
|
|
2668
|
+
}
|
|
2669
|
+
async call(spec) {
|
|
2670
|
+
return result(spec, await this.#send(spec));
|
|
2671
|
+
}
|
|
2672
|
+
/** A raw call for endpoints the SDK does not cover. Never throws on status. */
|
|
2673
|
+
request(method, path, options = {}) {
|
|
2674
|
+
const spec = {
|
|
2675
|
+
method,
|
|
2676
|
+
path,
|
|
2677
|
+
json: options.json,
|
|
2678
|
+
params: options.params,
|
|
2679
|
+
body: options.body,
|
|
2680
|
+
timeout: options.timeout
|
|
2681
|
+
};
|
|
2682
|
+
return this.#send(spec, options.headers);
|
|
2683
|
+
}
|
|
2684
|
+
#send(spec, headers) {
|
|
2685
|
+
return send(spec, {
|
|
2686
|
+
fetch: this.fetch,
|
|
2687
|
+
baseUrl: this.baseUrl,
|
|
2688
|
+
headers: {
|
|
2689
|
+
...this.headers,
|
|
2690
|
+
...headers
|
|
2691
|
+
},
|
|
2692
|
+
retries: this.maxRetries,
|
|
2693
|
+
backoff: this.backoff,
|
|
2694
|
+
timeout: this.timeout,
|
|
2695
|
+
logger: this.logger
|
|
2696
|
+
});
|
|
2697
|
+
}
|
|
2698
|
+
/** Start a session; `stop()` it (or `await using`) so it does not keep billing. */
|
|
2699
|
+
async session(options = {}) {
|
|
2700
|
+
const { profileUuid, ...rest } = options;
|
|
2701
|
+
const session = profileUuid === void 0 ? await this.profiles.startOneTime(rest) : await this.profiles.start(profileUuid, rest);
|
|
2702
|
+
let stopping;
|
|
2703
|
+
const stop = () => {
|
|
2704
|
+
stopping ??= stopSession(this, session.internal_uuid);
|
|
2705
|
+
return stopping;
|
|
2706
|
+
};
|
|
2707
|
+
return {
|
|
2708
|
+
...session,
|
|
2709
|
+
stop,
|
|
2710
|
+
[Symbol.asyncDispose]: stop
|
|
2711
|
+
};
|
|
2712
|
+
}
|
|
2713
|
+
/** Start a session and connect a `Browser` to it. `close()` stops the session. */
|
|
2714
|
+
async browser(options = {}) {
|
|
2715
|
+
const { profileUuid, blockResources, blockUrls, connectTimeout, commandTimeout, createWebSocket, ...sessionOptions } = options;
|
|
2716
|
+
const blocked = normalizeBlocked(blockResources);
|
|
2717
|
+
const urls = normalizeUrls(blockUrls);
|
|
2718
|
+
const session = await this.session({
|
|
2719
|
+
profileUuid,
|
|
2720
|
+
...sessionOptions
|
|
2721
|
+
});
|
|
2722
|
+
const browser = new Browser(session, {
|
|
2723
|
+
blockResources: blocked,
|
|
2724
|
+
blockUrls: urls,
|
|
2725
|
+
connectTimeout,
|
|
2726
|
+
commandTimeout,
|
|
2727
|
+
createWebSocket,
|
|
2728
|
+
logger: this.logger,
|
|
2729
|
+
onClose: () => session.stop()
|
|
2730
|
+
});
|
|
2731
|
+
await browser.connect();
|
|
2732
|
+
return browser;
|
|
2733
|
+
}
|
|
2734
|
+
/** An open `BrowserPool`. `close()` stops every browser in it. */
|
|
2735
|
+
async browsers(options = {}) {
|
|
2736
|
+
return new BrowserPool(this, options).open();
|
|
2737
|
+
}
|
|
2738
|
+
/** `browsers()` and `pool.map()` in 1 call. */
|
|
2739
|
+
async map(handler, items, options = {}) {
|
|
2740
|
+
const pool = await this.browsers(options);
|
|
2741
|
+
try {
|
|
2742
|
+
return await pool.map(handler, items);
|
|
2743
|
+
} finally {
|
|
2744
|
+
await pool.close();
|
|
2745
|
+
}
|
|
2746
|
+
}
|
|
2747
|
+
/** Nothing to release with fetch; here for `await using` symmetry. */
|
|
2748
|
+
async close() {}
|
|
2749
|
+
[Symbol.asyncDispose]() {
|
|
2750
|
+
return this.close();
|
|
2751
|
+
}
|
|
2752
|
+
};
|
|
2753
|
+
//#endregion
|
|
2754
|
+
export { APIConnectionError, APIError, APITimeoutError, Account, Actions, AuthenticationError, BadRequestError, Browser, BrowserPool, BrowserTimeoutError, CDPClient, CDPError, CapturedResponse, ConfigurationError, ConflictError, Extensions, Fingerprints, ForbiddenError, Keyboard, MonthlySessionLimitError, Mouse, NotFoundError, Page, PageClosedError, PaymentRequiredError, PremiumTrafficLimitError, Profiles, Proxies, ProxyCycle, ProxyRandom, ProxyTemplate, RESOURCE_TYPES, RateLimitError, ServerError, SharedTrafficLimitError, StopRun, Surfsky, SurfskyError, VERSION, ValidationError };
|