rezo 1.0.11 → 1.0.13
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/adapters/curl.cjs +73 -10
- package/dist/adapters/curl.js +73 -10
- package/dist/adapters/entries/curl.d.ts +66 -1
- package/dist/adapters/entries/fetch.d.ts +66 -1
- package/dist/adapters/entries/http.d.ts +66 -1
- package/dist/adapters/entries/http2.d.ts +66 -1
- package/dist/adapters/entries/react-native.d.ts +66 -1
- package/dist/adapters/entries/xhr.d.ts +66 -1
- package/dist/adapters/fetch.cjs +106 -59
- package/dist/adapters/fetch.js +106 -59
- package/dist/adapters/http.cjs +28 -15
- package/dist/adapters/http.js +28 -15
- package/dist/adapters/http2.cjs +114 -55
- package/dist/adapters/http2.js +114 -55
- package/dist/adapters/index.cjs +6 -6
- package/dist/cache/index.cjs +13 -13
- package/dist/crawler.d.ts +66 -1
- package/dist/entries/crawler.cjs +5 -5
- package/dist/index.cjs +24 -24
- package/dist/index.d.ts +66 -1
- package/dist/platform/browser.d.ts +66 -1
- package/dist/platform/bun.d.ts +66 -1
- package/dist/platform/deno.d.ts +66 -1
- package/dist/platform/node.d.ts +66 -1
- package/dist/platform/react-native.d.ts +66 -1
- package/dist/platform/worker.d.ts +66 -1
- package/dist/plugin/index.cjs +36 -36
- package/dist/proxy/index.cjs +2 -2
- package/dist/queue/index.cjs +8 -8
- package/dist/responses/buildError.cjs +5 -1
- package/dist/responses/buildError.js +5 -1
- package/dist/responses/buildResponse.cjs +30 -3
- package/dist/responses/buildResponse.js +30 -3
- package/dist/utils/compression.cjs +6 -6
- package/dist/utils/compression.js +6 -6
- package/dist/utils/headers.cjs +17 -0
- package/dist/utils/headers.js +17 -1
- package/dist/utils/http-config.cjs +47 -5
- package/dist/utils/http-config.js +47 -5
- package/package.json +1 -1
|
@@ -2502,8 +2502,73 @@ export interface RezoRequestConfig<D = any> {
|
|
|
2502
2502
|
maxRedirects?: number;
|
|
2503
2503
|
/** Whether to automatically decompress response data */
|
|
2504
2504
|
decompress?: boolean;
|
|
2505
|
-
/**
|
|
2505
|
+
/**
|
|
2506
|
+
* Whether to keep TCP connections alive for reuse across multiple requests.
|
|
2507
|
+
*
|
|
2508
|
+
* When enabled, the underlying TCP connection is kept open after a request completes,
|
|
2509
|
+
* allowing subsequent requests to the same host to reuse the connection. This reduces
|
|
2510
|
+
* latency by avoiding the overhead of establishing new connections (TCP handshake,
|
|
2511
|
+
* TLS negotiation for HTTPS).
|
|
2512
|
+
*
|
|
2513
|
+
* **Behavior:**
|
|
2514
|
+
* - `false` (default) - Connection closes after each request. Process exits immediately.
|
|
2515
|
+
* - `true` - Connection stays open for reuse. Idle connections close after `keepAliveMsecs`.
|
|
2516
|
+
*
|
|
2517
|
+
* **When to use `keepAlive: true`:**
|
|
2518
|
+
* - Making multiple requests to the same host in sequence
|
|
2519
|
+
* - Long-running applications (servers, bots, scrapers)
|
|
2520
|
+
* - Performance-critical applications where connection overhead matters
|
|
2521
|
+
*
|
|
2522
|
+
* **When to use `keepAlive: false` (default):**
|
|
2523
|
+
* - Single requests or scripts that should exit immediately
|
|
2524
|
+
* - CLI tools that make one-off requests
|
|
2525
|
+
* - When you need predictable process termination
|
|
2526
|
+
*
|
|
2527
|
+
* @example
|
|
2528
|
+
* ```typescript
|
|
2529
|
+
* // Default: process exits immediately after request
|
|
2530
|
+
* const { data } = await rezo.get('https://api.example.com/data');
|
|
2531
|
+
*
|
|
2532
|
+
* // Keep connection alive for 1 minute (default) for subsequent requests
|
|
2533
|
+
* const client = new Rezo({ keepAlive: true });
|
|
2534
|
+
* await client.get('https://api.example.com/users');
|
|
2535
|
+
* await client.get('https://api.example.com/posts'); // Reuses connection
|
|
2536
|
+
*
|
|
2537
|
+
* // Custom keep-alive timeout (30 seconds)
|
|
2538
|
+
* const client = new Rezo({ keepAlive: true, keepAliveMsecs: 30000 });
|
|
2539
|
+
* ```
|
|
2540
|
+
*
|
|
2541
|
+
* @default false
|
|
2542
|
+
*/
|
|
2506
2543
|
keepAlive?: boolean;
|
|
2544
|
+
/**
|
|
2545
|
+
* How long to keep idle connections alive in milliseconds.
|
|
2546
|
+
*
|
|
2547
|
+
* Only applies when `keepAlive: true`. After this duration of inactivity,
|
|
2548
|
+
* the connection is closed automatically. This prevents resource leaks
|
|
2549
|
+
* from connections that are no longer needed.
|
|
2550
|
+
*
|
|
2551
|
+
* **Note:** Even with keep-alive enabled, the Node.js process can still exit
|
|
2552
|
+
* cleanly when there's no other work to do, thanks to socket unreferencing.
|
|
2553
|
+
*
|
|
2554
|
+
* @example
|
|
2555
|
+
* ```typescript
|
|
2556
|
+
* // Keep connections alive for 30 seconds
|
|
2557
|
+
* const client = new Rezo({
|
|
2558
|
+
* keepAlive: true,
|
|
2559
|
+
* keepAliveMsecs: 30000
|
|
2560
|
+
* });
|
|
2561
|
+
*
|
|
2562
|
+
* // Keep connections alive for 2 minutes
|
|
2563
|
+
* const client = new Rezo({
|
|
2564
|
+
* keepAlive: true,
|
|
2565
|
+
* keepAliveMsecs: 120000
|
|
2566
|
+
* });
|
|
2567
|
+
* ```
|
|
2568
|
+
*
|
|
2569
|
+
* @default 60000 (1 minute)
|
|
2570
|
+
*/
|
|
2571
|
+
keepAliveMsecs?: number;
|
|
2507
2572
|
withoutBodyOnRedirect?: boolean;
|
|
2508
2573
|
autoSetReferer?: boolean;
|
|
2509
2574
|
autoSetOrigin?: boolean;
|
|
@@ -2502,8 +2502,73 @@ export interface RezoRequestConfig<D = any> {
|
|
|
2502
2502
|
maxRedirects?: number;
|
|
2503
2503
|
/** Whether to automatically decompress response data */
|
|
2504
2504
|
decompress?: boolean;
|
|
2505
|
-
/**
|
|
2505
|
+
/**
|
|
2506
|
+
* Whether to keep TCP connections alive for reuse across multiple requests.
|
|
2507
|
+
*
|
|
2508
|
+
* When enabled, the underlying TCP connection is kept open after a request completes,
|
|
2509
|
+
* allowing subsequent requests to the same host to reuse the connection. This reduces
|
|
2510
|
+
* latency by avoiding the overhead of establishing new connections (TCP handshake,
|
|
2511
|
+
* TLS negotiation for HTTPS).
|
|
2512
|
+
*
|
|
2513
|
+
* **Behavior:**
|
|
2514
|
+
* - `false` (default) - Connection closes after each request. Process exits immediately.
|
|
2515
|
+
* - `true` - Connection stays open for reuse. Idle connections close after `keepAliveMsecs`.
|
|
2516
|
+
*
|
|
2517
|
+
* **When to use `keepAlive: true`:**
|
|
2518
|
+
* - Making multiple requests to the same host in sequence
|
|
2519
|
+
* - Long-running applications (servers, bots, scrapers)
|
|
2520
|
+
* - Performance-critical applications where connection overhead matters
|
|
2521
|
+
*
|
|
2522
|
+
* **When to use `keepAlive: false` (default):**
|
|
2523
|
+
* - Single requests or scripts that should exit immediately
|
|
2524
|
+
* - CLI tools that make one-off requests
|
|
2525
|
+
* - When you need predictable process termination
|
|
2526
|
+
*
|
|
2527
|
+
* @example
|
|
2528
|
+
* ```typescript
|
|
2529
|
+
* // Default: process exits immediately after request
|
|
2530
|
+
* const { data } = await rezo.get('https://api.example.com/data');
|
|
2531
|
+
*
|
|
2532
|
+
* // Keep connection alive for 1 minute (default) for subsequent requests
|
|
2533
|
+
* const client = new Rezo({ keepAlive: true });
|
|
2534
|
+
* await client.get('https://api.example.com/users');
|
|
2535
|
+
* await client.get('https://api.example.com/posts'); // Reuses connection
|
|
2536
|
+
*
|
|
2537
|
+
* // Custom keep-alive timeout (30 seconds)
|
|
2538
|
+
* const client = new Rezo({ keepAlive: true, keepAliveMsecs: 30000 });
|
|
2539
|
+
* ```
|
|
2540
|
+
*
|
|
2541
|
+
* @default false
|
|
2542
|
+
*/
|
|
2506
2543
|
keepAlive?: boolean;
|
|
2544
|
+
/**
|
|
2545
|
+
* How long to keep idle connections alive in milliseconds.
|
|
2546
|
+
*
|
|
2547
|
+
* Only applies when `keepAlive: true`. After this duration of inactivity,
|
|
2548
|
+
* the connection is closed automatically. This prevents resource leaks
|
|
2549
|
+
* from connections that are no longer needed.
|
|
2550
|
+
*
|
|
2551
|
+
* **Note:** Even with keep-alive enabled, the Node.js process can still exit
|
|
2552
|
+
* cleanly when there's no other work to do, thanks to socket unreferencing.
|
|
2553
|
+
*
|
|
2554
|
+
* @example
|
|
2555
|
+
* ```typescript
|
|
2556
|
+
* // Keep connections alive for 30 seconds
|
|
2557
|
+
* const client = new Rezo({
|
|
2558
|
+
* keepAlive: true,
|
|
2559
|
+
* keepAliveMsecs: 30000
|
|
2560
|
+
* });
|
|
2561
|
+
*
|
|
2562
|
+
* // Keep connections alive for 2 minutes
|
|
2563
|
+
* const client = new Rezo({
|
|
2564
|
+
* keepAlive: true,
|
|
2565
|
+
* keepAliveMsecs: 120000
|
|
2566
|
+
* });
|
|
2567
|
+
* ```
|
|
2568
|
+
*
|
|
2569
|
+
* @default 60000 (1 minute)
|
|
2570
|
+
*/
|
|
2571
|
+
keepAliveMsecs?: number;
|
|
2507
2572
|
withoutBodyOnRedirect?: boolean;
|
|
2508
2573
|
autoSetReferer?: boolean;
|
|
2509
2574
|
autoSetOrigin?: boolean;
|
package/dist/adapters/fetch.cjs
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
const { URL } = require("node:url");
|
|
2
2
|
const { RezoError } = require('../errors/rezo-error.cjs');
|
|
3
3
|
const { buildSmartError, builErrorFromResponse, buildDownloadError } = require('../responses/buildError.cjs');
|
|
4
|
-
const {
|
|
4
|
+
const { RezoCookieJar } = require('../utils/cookies.cjs');
|
|
5
5
|
const RezoFormData = require('../utils/form-data.cjs');
|
|
6
6
|
const { getDefaultConfig, prepareHTTPOptions } = require('../utils/http-config.cjs');
|
|
7
7
|
const { RezoHeaders } = require('../utils/headers.cjs');
|
|
@@ -105,39 +105,69 @@ function sanitizeConfig(config) {
|
|
|
105
105
|
delete sanitized.data;
|
|
106
106
|
return sanitized;
|
|
107
107
|
}
|
|
108
|
-
function parseCookiesFromHeaders(headers, url) {
|
|
109
|
-
|
|
108
|
+
function parseCookiesFromHeaders(headers, url, config) {
|
|
109
|
+
let setCookieHeaders = [];
|
|
110
|
+
if (typeof headers.getSetCookie === "function") {
|
|
111
|
+
setCookieHeaders = headers.getSetCookie() || [];
|
|
112
|
+
} else {
|
|
113
|
+
const setCookieRaw = headers.get("set-cookie");
|
|
114
|
+
if (setCookieRaw) {
|
|
115
|
+
const splitPattern = /,(?=\s*[A-Za-z0-9_-]+=)/;
|
|
116
|
+
setCookieHeaders = setCookieRaw.split(splitPattern).map((s) => s.trim()).filter(Boolean);
|
|
117
|
+
}
|
|
118
|
+
}
|
|
119
|
+
if (setCookieHeaders.length === 0) {
|
|
120
|
+
return {
|
|
121
|
+
array: [],
|
|
122
|
+
serialized: [],
|
|
123
|
+
netscape: "",
|
|
124
|
+
string: "",
|
|
125
|
+
setCookiesString: []
|
|
126
|
+
};
|
|
127
|
+
}
|
|
128
|
+
const jar = new RezoCookieJar;
|
|
129
|
+
jar.setCookiesSync(setCookieHeaders, url);
|
|
130
|
+
if (config?.enableCookieJar && config?.cookieJar) {
|
|
131
|
+
config.cookieJar.setCookiesSync(setCookieHeaders, url);
|
|
132
|
+
}
|
|
133
|
+
const cookies = jar.cookies();
|
|
134
|
+
cookies.setCookiesString = setCookieHeaders;
|
|
135
|
+
return cookies;
|
|
136
|
+
}
|
|
137
|
+
function mergeRequestAndResponseCookies(config, responseCookies, url) {
|
|
138
|
+
const mergedCookiesArray = [];
|
|
139
|
+
const cookieKeyDomainMap = new Map;
|
|
140
|
+
if (config.requestCookies && config.requestCookies.length > 0) {
|
|
141
|
+
for (const cookie of config.requestCookies) {
|
|
142
|
+
const key = `${cookie.key}|${cookie.domain || ""}`;
|
|
143
|
+
mergedCookiesArray.push(cookie);
|
|
144
|
+
cookieKeyDomainMap.set(key, mergedCookiesArray.length - 1);
|
|
145
|
+
}
|
|
146
|
+
}
|
|
147
|
+
for (const cookie of responseCookies.array) {
|
|
148
|
+
const key = `${cookie.key}|${cookie.domain || ""}`;
|
|
149
|
+
const existingIndex = cookieKeyDomainMap.get(key);
|
|
150
|
+
if (existingIndex !== undefined) {
|
|
151
|
+
mergedCookiesArray[existingIndex] = cookie;
|
|
152
|
+
} else {
|
|
153
|
+
mergedCookiesArray.push(cookie);
|
|
154
|
+
cookieKeyDomainMap.set(key, mergedCookiesArray.length - 1);
|
|
155
|
+
}
|
|
156
|
+
}
|
|
157
|
+
if (mergedCookiesArray.length > 0) {
|
|
158
|
+
const mergedJar = new RezoCookieJar(mergedCookiesArray, url);
|
|
159
|
+
return mergedJar.cookies();
|
|
160
|
+
}
|
|
161
|
+
return {
|
|
110
162
|
array: [],
|
|
111
163
|
serialized: [],
|
|
112
|
-
netscape:
|
|
164
|
+
netscape: `# Netscape HTTP Cookie File
|
|
165
|
+
# This file was generated by Rezo HTTP client
|
|
166
|
+
# Based on uniqhtt cookie implementation
|
|
167
|
+
`,
|
|
113
168
|
string: "",
|
|
114
169
|
setCookiesString: []
|
|
115
170
|
};
|
|
116
|
-
const setCookieHeaders = headers.getSetCookie?.() || [];
|
|
117
|
-
for (const cookieStr of setCookieHeaders) {
|
|
118
|
-
cookies.setCookiesString.push(cookieStr);
|
|
119
|
-
const parts = cookieStr.split(";");
|
|
120
|
-
const [nameValue] = parts;
|
|
121
|
-
const [name, ...valueParts] = nameValue.split("=");
|
|
122
|
-
const value = valueParts.join("=");
|
|
123
|
-
if (name && value !== undefined) {
|
|
124
|
-
const cookie = new Cookie({
|
|
125
|
-
key: name.trim(),
|
|
126
|
-
value: value.trim(),
|
|
127
|
-
domain: new URL(url).hostname,
|
|
128
|
-
path: "/",
|
|
129
|
-
httpOnly: cookieStr.toLowerCase().includes("httponly"),
|
|
130
|
-
secure: cookieStr.toLowerCase().includes("secure"),
|
|
131
|
-
sameSite: "lax"
|
|
132
|
-
});
|
|
133
|
-
cookies.array.push(cookie);
|
|
134
|
-
}
|
|
135
|
-
}
|
|
136
|
-
cookies.string = cookies.array.map((c) => `${c.key}=${c.value}`).join("; ");
|
|
137
|
-
cookies.serialized = cookies.array.map((c) => c.toJSON());
|
|
138
|
-
cookies.netscape = cookies.array.map((c) => c.toNetscapeFormat()).join(`
|
|
139
|
-
`);
|
|
140
|
-
return cookies;
|
|
141
171
|
}
|
|
142
172
|
function toFetchHeaders(headers) {
|
|
143
173
|
const fetchHeaders = new Headers;
|
|
@@ -181,7 +211,7 @@ async function prepareFetchBody(body) {
|
|
|
181
211
|
return new Uint8Array(buffer);
|
|
182
212
|
}
|
|
183
213
|
}
|
|
184
|
-
const nativeForm = body.toNativeFormData();
|
|
214
|
+
const nativeForm = await body.toNativeFormData();
|
|
185
215
|
if (nativeForm) {
|
|
186
216
|
return nativeForm;
|
|
187
217
|
}
|
|
@@ -306,6 +336,8 @@ async function executeFetchRequest(fetchOptions, config, options, perform, strea
|
|
|
306
336
|
const response = await executeSingleFetchRequest(config, fetchOptions, requestCount, timing, _stats, streamResult, downloadResult, uploadResult);
|
|
307
337
|
const statusOnNext = _stats.statusOnNext;
|
|
308
338
|
if (response instanceof RezoError) {
|
|
339
|
+
if (!config.errors)
|
|
340
|
+
config.errors = [];
|
|
309
341
|
config.errors.push({
|
|
310
342
|
attempt: config.retryAttempts + 1,
|
|
311
343
|
error: response,
|
|
@@ -316,28 +348,25 @@ async function executeFetchRequest(fetchOptions, config, options, perform, strea
|
|
|
316
348
|
throw response;
|
|
317
349
|
}
|
|
318
350
|
if (config.retry) {
|
|
351
|
+
let shouldRetry = false;
|
|
319
352
|
if (config.retry.condition) {
|
|
320
353
|
const isPassed = await config.retry.condition(response);
|
|
321
|
-
|
|
322
|
-
throw response;
|
|
323
|
-
}
|
|
354
|
+
shouldRetry = isPassed === true;
|
|
324
355
|
} else {
|
|
325
|
-
|
|
326
|
-
throw response;
|
|
327
|
-
}
|
|
328
|
-
if (maxRetries <= retries) {
|
|
329
|
-
throw response;
|
|
330
|
-
}
|
|
331
|
-
retries++;
|
|
332
|
-
if (retryDelay > 0) {
|
|
333
|
-
await new Promise((resolve) => setTimeout(resolve, incrementDelay ? retryDelay * retries : retryDelay));
|
|
334
|
-
}
|
|
356
|
+
shouldRetry = statusCodes ? statusCodes.includes(response.status || 0) : true;
|
|
335
357
|
}
|
|
358
|
+
if (!shouldRetry || retries >= maxRetries) {
|
|
359
|
+
throw response;
|
|
360
|
+
}
|
|
361
|
+
retries++;
|
|
336
362
|
config.retryAttempts++;
|
|
363
|
+
if (retryDelay > 0) {
|
|
364
|
+
await new Promise((resolve) => setTimeout(resolve, incrementDelay ? retryDelay * retries : retryDelay));
|
|
365
|
+
}
|
|
337
366
|
}
|
|
338
367
|
continue;
|
|
339
368
|
}
|
|
340
|
-
if (statusOnNext === "success") {
|
|
369
|
+
if (statusOnNext === "success" || statusOnNext === "error") {
|
|
341
370
|
return response;
|
|
342
371
|
}
|
|
343
372
|
if (statusOnNext === "redirect") {
|
|
@@ -418,7 +447,13 @@ async function executeSingleFetchRequest(config, fetchOptions, requestCount, tim
|
|
|
418
447
|
config.isSecure = isSecure;
|
|
419
448
|
config.finalUrl = url.href;
|
|
420
449
|
config.network.protocol = isSecure ? "https" : "http";
|
|
450
|
+
config.network.httpVersion = undefined;
|
|
421
451
|
config.timing.startTimestamp = timing.startTimestamp;
|
|
452
|
+
if (!config.transfer) {
|
|
453
|
+
config.transfer = { requestSize: 0, responseSize: 0, headerSize: 0, bodySize: 0 };
|
|
454
|
+
} else if (config.transfer.requestSize === undefined) {
|
|
455
|
+
config.transfer.requestSize = 0;
|
|
456
|
+
}
|
|
422
457
|
}
|
|
423
458
|
const reqHeaders = fetchOptions.headers instanceof RezoHeaders ? fetchOptions.headers.toObject() : fetchOptions.headers || {};
|
|
424
459
|
const headers = toFetchHeaders(reqHeaders);
|
|
@@ -448,7 +483,26 @@ async function executeSingleFetchRequest(config, fetchOptions, requestCount, tim
|
|
|
448
483
|
clearTimeout(timeoutId);
|
|
449
484
|
});
|
|
450
485
|
}
|
|
486
|
+
if (body instanceof RezoFormData) {
|
|
487
|
+
const contentType = body.getContentType();
|
|
488
|
+
if (contentType && !headers.has("content-type")) {
|
|
489
|
+
headers.set("content-type", contentType);
|
|
490
|
+
}
|
|
491
|
+
}
|
|
451
492
|
const preparedBody = await prepareFetchBody(body);
|
|
493
|
+
if (config.transfer && body) {
|
|
494
|
+
if (typeof body === "string") {
|
|
495
|
+
config.transfer.requestSize = body.length;
|
|
496
|
+
} else if (body instanceof ArrayBuffer || body instanceof Uint8Array) {
|
|
497
|
+
config.transfer.requestSize = body.byteLength || body.length;
|
|
498
|
+
} else if (body instanceof URLSearchParams || body instanceof RezoURLSearchParams) {
|
|
499
|
+
config.transfer.requestSize = body.toString().length;
|
|
500
|
+
} else if (body instanceof RezoFormData) {
|
|
501
|
+
config.transfer.requestSize = body.getLengthSync();
|
|
502
|
+
} else if (typeof body === "object" && !(body instanceof Blob) && !(body instanceof ReadableStream)) {
|
|
503
|
+
config.transfer.requestSize = JSON.stringify(body).length;
|
|
504
|
+
}
|
|
505
|
+
}
|
|
452
506
|
const fetchInit = {
|
|
453
507
|
method: fetchOptions.method.toUpperCase(),
|
|
454
508
|
headers,
|
|
@@ -482,7 +536,7 @@ async function executeSingleFetchRequest(config, fetchOptions, requestCount, tim
|
|
|
482
536
|
const responseHeaders = fromFetchHeaders(response.headers);
|
|
483
537
|
const contentType = response.headers.get("content-type") || "";
|
|
484
538
|
const contentLength = response.headers.get("content-length");
|
|
485
|
-
const cookies = parseCookiesFromHeaders(response.headers, url.href);
|
|
539
|
+
const cookies = parseCookiesFromHeaders(response.headers, url.href, config);
|
|
486
540
|
config.responseCookies = cookies;
|
|
487
541
|
const location = response.headers.get("location");
|
|
488
542
|
const isRedirect = status >= 300 && status < 400 && location;
|
|
@@ -564,23 +618,16 @@ async function executeSingleFetchRequest(config, fetchOptions, requestCount, tim
|
|
|
564
618
|
const bodySize = bodyBuffer?.byteLength || (typeof responseData === "string" ? responseData.length : 0);
|
|
565
619
|
config.transfer.bodySize = bodySize;
|
|
566
620
|
config.transfer.responseSize = bodySize;
|
|
567
|
-
|
|
568
|
-
|
|
569
|
-
|
|
570
|
-
|
|
571
|
-
headers: responseHeaders,
|
|
572
|
-
data: responseData
|
|
573
|
-
}, config, fetchOptions);
|
|
574
|
-
_stats.statusOnNext = "error";
|
|
575
|
-
return error;
|
|
576
|
-
}
|
|
577
|
-
_stats.statusOnNext = "success";
|
|
621
|
+
config.status = status;
|
|
622
|
+
config.statusText = statusText;
|
|
623
|
+
_stats.statusOnNext = status >= 400 ? "error" : "success";
|
|
624
|
+
const mergedCookies = mergeRequestAndResponseCookies(config, cookies, url.href);
|
|
578
625
|
const finalResponse = {
|
|
579
626
|
data: responseData,
|
|
580
627
|
status,
|
|
581
628
|
statusText,
|
|
582
629
|
headers: responseHeaders,
|
|
583
|
-
cookies,
|
|
630
|
+
cookies: mergedCookies,
|
|
584
631
|
config,
|
|
585
632
|
contentType,
|
|
586
633
|
contentLength: bodySize,
|
|
@@ -599,7 +646,7 @@ async function executeSingleFetchRequest(config, fetchOptions, requestCount, tim
|
|
|
599
646
|
contentType,
|
|
600
647
|
contentLength: buffer.length,
|
|
601
648
|
finalUrl: url.href,
|
|
602
|
-
cookies,
|
|
649
|
+
cookies: mergedCookies,
|
|
603
650
|
urls: buildUrlTree(config, url.href),
|
|
604
651
|
fileName: config.fileName,
|
|
605
652
|
fileSize: buffer.length,
|
|
@@ -646,7 +693,7 @@ async function executeSingleFetchRequest(config, fetchOptions, requestCount, tim
|
|
|
646
693
|
contentLength: bodySize
|
|
647
694
|
},
|
|
648
695
|
finalUrl: url.href,
|
|
649
|
-
cookies,
|
|
696
|
+
cookies: mergedCookies,
|
|
650
697
|
urls: buildUrlTree(config, url.href),
|
|
651
698
|
uploadSize: config.transfer.requestSize || 0,
|
|
652
699
|
timing: {
|
package/dist/adapters/fetch.js
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import { URL } from "node:url";
|
|
2
2
|
import { RezoError } from '../errors/rezo-error.js';
|
|
3
3
|
import { buildSmartError, builErrorFromResponse, buildDownloadError } from '../responses/buildError.js';
|
|
4
|
-
import {
|
|
4
|
+
import { RezoCookieJar } from '../utils/cookies.js';
|
|
5
5
|
import RezoFormData from '../utils/form-data.js';
|
|
6
6
|
import { getDefaultConfig, prepareHTTPOptions } from '../utils/http-config.js';
|
|
7
7
|
import { RezoHeaders } from '../utils/headers.js';
|
|
@@ -105,39 +105,69 @@ function sanitizeConfig(config) {
|
|
|
105
105
|
delete sanitized.data;
|
|
106
106
|
return sanitized;
|
|
107
107
|
}
|
|
108
|
-
function parseCookiesFromHeaders(headers, url) {
|
|
109
|
-
|
|
108
|
+
function parseCookiesFromHeaders(headers, url, config) {
|
|
109
|
+
let setCookieHeaders = [];
|
|
110
|
+
if (typeof headers.getSetCookie === "function") {
|
|
111
|
+
setCookieHeaders = headers.getSetCookie() || [];
|
|
112
|
+
} else {
|
|
113
|
+
const setCookieRaw = headers.get("set-cookie");
|
|
114
|
+
if (setCookieRaw) {
|
|
115
|
+
const splitPattern = /,(?=\s*[A-Za-z0-9_-]+=)/;
|
|
116
|
+
setCookieHeaders = setCookieRaw.split(splitPattern).map((s) => s.trim()).filter(Boolean);
|
|
117
|
+
}
|
|
118
|
+
}
|
|
119
|
+
if (setCookieHeaders.length === 0) {
|
|
120
|
+
return {
|
|
121
|
+
array: [],
|
|
122
|
+
serialized: [],
|
|
123
|
+
netscape: "",
|
|
124
|
+
string: "",
|
|
125
|
+
setCookiesString: []
|
|
126
|
+
};
|
|
127
|
+
}
|
|
128
|
+
const jar = new RezoCookieJar;
|
|
129
|
+
jar.setCookiesSync(setCookieHeaders, url);
|
|
130
|
+
if (config?.enableCookieJar && config?.cookieJar) {
|
|
131
|
+
config.cookieJar.setCookiesSync(setCookieHeaders, url);
|
|
132
|
+
}
|
|
133
|
+
const cookies = jar.cookies();
|
|
134
|
+
cookies.setCookiesString = setCookieHeaders;
|
|
135
|
+
return cookies;
|
|
136
|
+
}
|
|
137
|
+
function mergeRequestAndResponseCookies(config, responseCookies, url) {
|
|
138
|
+
const mergedCookiesArray = [];
|
|
139
|
+
const cookieKeyDomainMap = new Map;
|
|
140
|
+
if (config.requestCookies && config.requestCookies.length > 0) {
|
|
141
|
+
for (const cookie of config.requestCookies) {
|
|
142
|
+
const key = `${cookie.key}|${cookie.domain || ""}`;
|
|
143
|
+
mergedCookiesArray.push(cookie);
|
|
144
|
+
cookieKeyDomainMap.set(key, mergedCookiesArray.length - 1);
|
|
145
|
+
}
|
|
146
|
+
}
|
|
147
|
+
for (const cookie of responseCookies.array) {
|
|
148
|
+
const key = `${cookie.key}|${cookie.domain || ""}`;
|
|
149
|
+
const existingIndex = cookieKeyDomainMap.get(key);
|
|
150
|
+
if (existingIndex !== undefined) {
|
|
151
|
+
mergedCookiesArray[existingIndex] = cookie;
|
|
152
|
+
} else {
|
|
153
|
+
mergedCookiesArray.push(cookie);
|
|
154
|
+
cookieKeyDomainMap.set(key, mergedCookiesArray.length - 1);
|
|
155
|
+
}
|
|
156
|
+
}
|
|
157
|
+
if (mergedCookiesArray.length > 0) {
|
|
158
|
+
const mergedJar = new RezoCookieJar(mergedCookiesArray, url);
|
|
159
|
+
return mergedJar.cookies();
|
|
160
|
+
}
|
|
161
|
+
return {
|
|
110
162
|
array: [],
|
|
111
163
|
serialized: [],
|
|
112
|
-
netscape:
|
|
164
|
+
netscape: `# Netscape HTTP Cookie File
|
|
165
|
+
# This file was generated by Rezo HTTP client
|
|
166
|
+
# Based on uniqhtt cookie implementation
|
|
167
|
+
`,
|
|
113
168
|
string: "",
|
|
114
169
|
setCookiesString: []
|
|
115
170
|
};
|
|
116
|
-
const setCookieHeaders = headers.getSetCookie?.() || [];
|
|
117
|
-
for (const cookieStr of setCookieHeaders) {
|
|
118
|
-
cookies.setCookiesString.push(cookieStr);
|
|
119
|
-
const parts = cookieStr.split(";");
|
|
120
|
-
const [nameValue] = parts;
|
|
121
|
-
const [name, ...valueParts] = nameValue.split("=");
|
|
122
|
-
const value = valueParts.join("=");
|
|
123
|
-
if (name && value !== undefined) {
|
|
124
|
-
const cookie = new Cookie({
|
|
125
|
-
key: name.trim(),
|
|
126
|
-
value: value.trim(),
|
|
127
|
-
domain: new URL(url).hostname,
|
|
128
|
-
path: "/",
|
|
129
|
-
httpOnly: cookieStr.toLowerCase().includes("httponly"),
|
|
130
|
-
secure: cookieStr.toLowerCase().includes("secure"),
|
|
131
|
-
sameSite: "lax"
|
|
132
|
-
});
|
|
133
|
-
cookies.array.push(cookie);
|
|
134
|
-
}
|
|
135
|
-
}
|
|
136
|
-
cookies.string = cookies.array.map((c) => `${c.key}=${c.value}`).join("; ");
|
|
137
|
-
cookies.serialized = cookies.array.map((c) => c.toJSON());
|
|
138
|
-
cookies.netscape = cookies.array.map((c) => c.toNetscapeFormat()).join(`
|
|
139
|
-
`);
|
|
140
|
-
return cookies;
|
|
141
171
|
}
|
|
142
172
|
function toFetchHeaders(headers) {
|
|
143
173
|
const fetchHeaders = new Headers;
|
|
@@ -181,7 +211,7 @@ async function prepareFetchBody(body) {
|
|
|
181
211
|
return new Uint8Array(buffer);
|
|
182
212
|
}
|
|
183
213
|
}
|
|
184
|
-
const nativeForm = body.toNativeFormData();
|
|
214
|
+
const nativeForm = await body.toNativeFormData();
|
|
185
215
|
if (nativeForm) {
|
|
186
216
|
return nativeForm;
|
|
187
217
|
}
|
|
@@ -306,6 +336,8 @@ async function executeFetchRequest(fetchOptions, config, options, perform, strea
|
|
|
306
336
|
const response = await executeSingleFetchRequest(config, fetchOptions, requestCount, timing, _stats, streamResult, downloadResult, uploadResult);
|
|
307
337
|
const statusOnNext = _stats.statusOnNext;
|
|
308
338
|
if (response instanceof RezoError) {
|
|
339
|
+
if (!config.errors)
|
|
340
|
+
config.errors = [];
|
|
309
341
|
config.errors.push({
|
|
310
342
|
attempt: config.retryAttempts + 1,
|
|
311
343
|
error: response,
|
|
@@ -316,28 +348,25 @@ async function executeFetchRequest(fetchOptions, config, options, perform, strea
|
|
|
316
348
|
throw response;
|
|
317
349
|
}
|
|
318
350
|
if (config.retry) {
|
|
351
|
+
let shouldRetry = false;
|
|
319
352
|
if (config.retry.condition) {
|
|
320
353
|
const isPassed = await config.retry.condition(response);
|
|
321
|
-
|
|
322
|
-
throw response;
|
|
323
|
-
}
|
|
354
|
+
shouldRetry = isPassed === true;
|
|
324
355
|
} else {
|
|
325
|
-
|
|
326
|
-
throw response;
|
|
327
|
-
}
|
|
328
|
-
if (maxRetries <= retries) {
|
|
329
|
-
throw response;
|
|
330
|
-
}
|
|
331
|
-
retries++;
|
|
332
|
-
if (retryDelay > 0) {
|
|
333
|
-
await new Promise((resolve) => setTimeout(resolve, incrementDelay ? retryDelay * retries : retryDelay));
|
|
334
|
-
}
|
|
356
|
+
shouldRetry = statusCodes ? statusCodes.includes(response.status || 0) : true;
|
|
335
357
|
}
|
|
358
|
+
if (!shouldRetry || retries >= maxRetries) {
|
|
359
|
+
throw response;
|
|
360
|
+
}
|
|
361
|
+
retries++;
|
|
336
362
|
config.retryAttempts++;
|
|
363
|
+
if (retryDelay > 0) {
|
|
364
|
+
await new Promise((resolve) => setTimeout(resolve, incrementDelay ? retryDelay * retries : retryDelay));
|
|
365
|
+
}
|
|
337
366
|
}
|
|
338
367
|
continue;
|
|
339
368
|
}
|
|
340
|
-
if (statusOnNext === "success") {
|
|
369
|
+
if (statusOnNext === "success" || statusOnNext === "error") {
|
|
341
370
|
return response;
|
|
342
371
|
}
|
|
343
372
|
if (statusOnNext === "redirect") {
|
|
@@ -418,7 +447,13 @@ async function executeSingleFetchRequest(config, fetchOptions, requestCount, tim
|
|
|
418
447
|
config.isSecure = isSecure;
|
|
419
448
|
config.finalUrl = url.href;
|
|
420
449
|
config.network.protocol = isSecure ? "https" : "http";
|
|
450
|
+
config.network.httpVersion = undefined;
|
|
421
451
|
config.timing.startTimestamp = timing.startTimestamp;
|
|
452
|
+
if (!config.transfer) {
|
|
453
|
+
config.transfer = { requestSize: 0, responseSize: 0, headerSize: 0, bodySize: 0 };
|
|
454
|
+
} else if (config.transfer.requestSize === undefined) {
|
|
455
|
+
config.transfer.requestSize = 0;
|
|
456
|
+
}
|
|
422
457
|
}
|
|
423
458
|
const reqHeaders = fetchOptions.headers instanceof RezoHeaders ? fetchOptions.headers.toObject() : fetchOptions.headers || {};
|
|
424
459
|
const headers = toFetchHeaders(reqHeaders);
|
|
@@ -448,7 +483,26 @@ async function executeSingleFetchRequest(config, fetchOptions, requestCount, tim
|
|
|
448
483
|
clearTimeout(timeoutId);
|
|
449
484
|
});
|
|
450
485
|
}
|
|
486
|
+
if (body instanceof RezoFormData) {
|
|
487
|
+
const contentType = body.getContentType();
|
|
488
|
+
if (contentType && !headers.has("content-type")) {
|
|
489
|
+
headers.set("content-type", contentType);
|
|
490
|
+
}
|
|
491
|
+
}
|
|
451
492
|
const preparedBody = await prepareFetchBody(body);
|
|
493
|
+
if (config.transfer && body) {
|
|
494
|
+
if (typeof body === "string") {
|
|
495
|
+
config.transfer.requestSize = body.length;
|
|
496
|
+
} else if (body instanceof ArrayBuffer || body instanceof Uint8Array) {
|
|
497
|
+
config.transfer.requestSize = body.byteLength || body.length;
|
|
498
|
+
} else if (body instanceof URLSearchParams || body instanceof RezoURLSearchParams) {
|
|
499
|
+
config.transfer.requestSize = body.toString().length;
|
|
500
|
+
} else if (body instanceof RezoFormData) {
|
|
501
|
+
config.transfer.requestSize = body.getLengthSync();
|
|
502
|
+
} else if (typeof body === "object" && !(body instanceof Blob) && !(body instanceof ReadableStream)) {
|
|
503
|
+
config.transfer.requestSize = JSON.stringify(body).length;
|
|
504
|
+
}
|
|
505
|
+
}
|
|
452
506
|
const fetchInit = {
|
|
453
507
|
method: fetchOptions.method.toUpperCase(),
|
|
454
508
|
headers,
|
|
@@ -482,7 +536,7 @@ async function executeSingleFetchRequest(config, fetchOptions, requestCount, tim
|
|
|
482
536
|
const responseHeaders = fromFetchHeaders(response.headers);
|
|
483
537
|
const contentType = response.headers.get("content-type") || "";
|
|
484
538
|
const contentLength = response.headers.get("content-length");
|
|
485
|
-
const cookies = parseCookiesFromHeaders(response.headers, url.href);
|
|
539
|
+
const cookies = parseCookiesFromHeaders(response.headers, url.href, config);
|
|
486
540
|
config.responseCookies = cookies;
|
|
487
541
|
const location = response.headers.get("location");
|
|
488
542
|
const isRedirect = status >= 300 && status < 400 && location;
|
|
@@ -564,23 +618,16 @@ async function executeSingleFetchRequest(config, fetchOptions, requestCount, tim
|
|
|
564
618
|
const bodySize = bodyBuffer?.byteLength || (typeof responseData === "string" ? responseData.length : 0);
|
|
565
619
|
config.transfer.bodySize = bodySize;
|
|
566
620
|
config.transfer.responseSize = bodySize;
|
|
567
|
-
|
|
568
|
-
|
|
569
|
-
|
|
570
|
-
|
|
571
|
-
headers: responseHeaders,
|
|
572
|
-
data: responseData
|
|
573
|
-
}, config, fetchOptions);
|
|
574
|
-
_stats.statusOnNext = "error";
|
|
575
|
-
return error;
|
|
576
|
-
}
|
|
577
|
-
_stats.statusOnNext = "success";
|
|
621
|
+
config.status = status;
|
|
622
|
+
config.statusText = statusText;
|
|
623
|
+
_stats.statusOnNext = status >= 400 ? "error" : "success";
|
|
624
|
+
const mergedCookies = mergeRequestAndResponseCookies(config, cookies, url.href);
|
|
578
625
|
const finalResponse = {
|
|
579
626
|
data: responseData,
|
|
580
627
|
status,
|
|
581
628
|
statusText,
|
|
582
629
|
headers: responseHeaders,
|
|
583
|
-
cookies,
|
|
630
|
+
cookies: mergedCookies,
|
|
584
631
|
config,
|
|
585
632
|
contentType,
|
|
586
633
|
contentLength: bodySize,
|
|
@@ -599,7 +646,7 @@ async function executeSingleFetchRequest(config, fetchOptions, requestCount, tim
|
|
|
599
646
|
contentType,
|
|
600
647
|
contentLength: buffer.length,
|
|
601
648
|
finalUrl: url.href,
|
|
602
|
-
cookies,
|
|
649
|
+
cookies: mergedCookies,
|
|
603
650
|
urls: buildUrlTree(config, url.href),
|
|
604
651
|
fileName: config.fileName,
|
|
605
652
|
fileSize: buffer.length,
|
|
@@ -646,7 +693,7 @@ async function executeSingleFetchRequest(config, fetchOptions, requestCount, tim
|
|
|
646
693
|
contentLength: bodySize
|
|
647
694
|
},
|
|
648
695
|
finalUrl: url.href,
|
|
649
|
-
cookies,
|
|
696
|
+
cookies: mergedCookies,
|
|
650
697
|
urls: buildUrlTree(config, url.href),
|
|
651
698
|
uploadSize: config.transfer.requestSize || 0,
|
|
652
699
|
timing: {
|