rezo 1.0.12 → 1.0.13
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/adapters/curl.cjs +73 -10
- package/dist/adapters/curl.js +73 -10
- package/dist/adapters/fetch.cjs +106 -59
- package/dist/adapters/fetch.js +106 -59
- package/dist/adapters/http.cjs +20 -13
- package/dist/adapters/http.js +20 -13
- package/dist/adapters/http2.cjs +114 -59
- package/dist/adapters/http2.js +114 -59
- package/dist/adapters/index.cjs +6 -6
- package/dist/cache/index.cjs +13 -13
- package/dist/entries/crawler.cjs +5 -5
- package/dist/index.cjs +24 -24
- package/dist/plugin/index.cjs +36 -36
- package/dist/proxy/index.cjs +2 -2
- package/dist/queue/index.cjs +8 -8
- package/dist/responses/buildError.cjs +5 -1
- package/dist/responses/buildError.js +5 -1
- package/dist/responses/buildResponse.cjs +2 -0
- package/dist/responses/buildResponse.js +2 -0
- package/dist/utils/compression.cjs +6 -6
- package/dist/utils/compression.js +6 -6
- package/dist/utils/headers.cjs +17 -0
- package/dist/utils/headers.js +17 -1
- package/dist/utils/http-config.cjs +39 -4
- package/dist/utils/http-config.js +39 -4
- package/package.json +1 -1
package/dist/adapters/fetch.js
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import { URL } from "node:url";
|
|
2
2
|
import { RezoError } from '../errors/rezo-error.js';
|
|
3
3
|
import { buildSmartError, builErrorFromResponse, buildDownloadError } from '../responses/buildError.js';
|
|
4
|
-
import {
|
|
4
|
+
import { RezoCookieJar } from '../utils/cookies.js';
|
|
5
5
|
import RezoFormData from '../utils/form-data.js';
|
|
6
6
|
import { getDefaultConfig, prepareHTTPOptions } from '../utils/http-config.js';
|
|
7
7
|
import { RezoHeaders } from '../utils/headers.js';
|
|
@@ -105,39 +105,69 @@ function sanitizeConfig(config) {
|
|
|
105
105
|
delete sanitized.data;
|
|
106
106
|
return sanitized;
|
|
107
107
|
}
|
|
108
|
-
function parseCookiesFromHeaders(headers, url) {
|
|
109
|
-
|
|
108
|
+
function parseCookiesFromHeaders(headers, url, config) {
|
|
109
|
+
let setCookieHeaders = [];
|
|
110
|
+
if (typeof headers.getSetCookie === "function") {
|
|
111
|
+
setCookieHeaders = headers.getSetCookie() || [];
|
|
112
|
+
} else {
|
|
113
|
+
const setCookieRaw = headers.get("set-cookie");
|
|
114
|
+
if (setCookieRaw) {
|
|
115
|
+
const splitPattern = /,(?=\s*[A-Za-z0-9_-]+=)/;
|
|
116
|
+
setCookieHeaders = setCookieRaw.split(splitPattern).map((s) => s.trim()).filter(Boolean);
|
|
117
|
+
}
|
|
118
|
+
}
|
|
119
|
+
if (setCookieHeaders.length === 0) {
|
|
120
|
+
return {
|
|
121
|
+
array: [],
|
|
122
|
+
serialized: [],
|
|
123
|
+
netscape: "",
|
|
124
|
+
string: "",
|
|
125
|
+
setCookiesString: []
|
|
126
|
+
};
|
|
127
|
+
}
|
|
128
|
+
const jar = new RezoCookieJar;
|
|
129
|
+
jar.setCookiesSync(setCookieHeaders, url);
|
|
130
|
+
if (config?.enableCookieJar && config?.cookieJar) {
|
|
131
|
+
config.cookieJar.setCookiesSync(setCookieHeaders, url);
|
|
132
|
+
}
|
|
133
|
+
const cookies = jar.cookies();
|
|
134
|
+
cookies.setCookiesString = setCookieHeaders;
|
|
135
|
+
return cookies;
|
|
136
|
+
}
|
|
137
|
+
function mergeRequestAndResponseCookies(config, responseCookies, url) {
|
|
138
|
+
const mergedCookiesArray = [];
|
|
139
|
+
const cookieKeyDomainMap = new Map;
|
|
140
|
+
if (config.requestCookies && config.requestCookies.length > 0) {
|
|
141
|
+
for (const cookie of config.requestCookies) {
|
|
142
|
+
const key = `${cookie.key}|${cookie.domain || ""}`;
|
|
143
|
+
mergedCookiesArray.push(cookie);
|
|
144
|
+
cookieKeyDomainMap.set(key, mergedCookiesArray.length - 1);
|
|
145
|
+
}
|
|
146
|
+
}
|
|
147
|
+
for (const cookie of responseCookies.array) {
|
|
148
|
+
const key = `${cookie.key}|${cookie.domain || ""}`;
|
|
149
|
+
const existingIndex = cookieKeyDomainMap.get(key);
|
|
150
|
+
if (existingIndex !== undefined) {
|
|
151
|
+
mergedCookiesArray[existingIndex] = cookie;
|
|
152
|
+
} else {
|
|
153
|
+
mergedCookiesArray.push(cookie);
|
|
154
|
+
cookieKeyDomainMap.set(key, mergedCookiesArray.length - 1);
|
|
155
|
+
}
|
|
156
|
+
}
|
|
157
|
+
if (mergedCookiesArray.length > 0) {
|
|
158
|
+
const mergedJar = new RezoCookieJar(mergedCookiesArray, url);
|
|
159
|
+
return mergedJar.cookies();
|
|
160
|
+
}
|
|
161
|
+
return {
|
|
110
162
|
array: [],
|
|
111
163
|
serialized: [],
|
|
112
|
-
netscape:
|
|
164
|
+
netscape: `# Netscape HTTP Cookie File
|
|
165
|
+
# This file was generated by Rezo HTTP client
|
|
166
|
+
# Based on uniqhtt cookie implementation
|
|
167
|
+
`,
|
|
113
168
|
string: "",
|
|
114
169
|
setCookiesString: []
|
|
115
170
|
};
|
|
116
|
-
const setCookieHeaders = headers.getSetCookie?.() || [];
|
|
117
|
-
for (const cookieStr of setCookieHeaders) {
|
|
118
|
-
cookies.setCookiesString.push(cookieStr);
|
|
119
|
-
const parts = cookieStr.split(";");
|
|
120
|
-
const [nameValue] = parts;
|
|
121
|
-
const [name, ...valueParts] = nameValue.split("=");
|
|
122
|
-
const value = valueParts.join("=");
|
|
123
|
-
if (name && value !== undefined) {
|
|
124
|
-
const cookie = new Cookie({
|
|
125
|
-
key: name.trim(),
|
|
126
|
-
value: value.trim(),
|
|
127
|
-
domain: new URL(url).hostname,
|
|
128
|
-
path: "/",
|
|
129
|
-
httpOnly: cookieStr.toLowerCase().includes("httponly"),
|
|
130
|
-
secure: cookieStr.toLowerCase().includes("secure"),
|
|
131
|
-
sameSite: "lax"
|
|
132
|
-
});
|
|
133
|
-
cookies.array.push(cookie);
|
|
134
|
-
}
|
|
135
|
-
}
|
|
136
|
-
cookies.string = cookies.array.map((c) => `${c.key}=${c.value}`).join("; ");
|
|
137
|
-
cookies.serialized = cookies.array.map((c) => c.toJSON());
|
|
138
|
-
cookies.netscape = cookies.array.map((c) => c.toNetscapeFormat()).join(`
|
|
139
|
-
`);
|
|
140
|
-
return cookies;
|
|
141
171
|
}
|
|
142
172
|
function toFetchHeaders(headers) {
|
|
143
173
|
const fetchHeaders = new Headers;
|
|
@@ -181,7 +211,7 @@ async function prepareFetchBody(body) {
|
|
|
181
211
|
return new Uint8Array(buffer);
|
|
182
212
|
}
|
|
183
213
|
}
|
|
184
|
-
const nativeForm = body.toNativeFormData();
|
|
214
|
+
const nativeForm = await body.toNativeFormData();
|
|
185
215
|
if (nativeForm) {
|
|
186
216
|
return nativeForm;
|
|
187
217
|
}
|
|
@@ -306,6 +336,8 @@ async function executeFetchRequest(fetchOptions, config, options, perform, strea
|
|
|
306
336
|
const response = await executeSingleFetchRequest(config, fetchOptions, requestCount, timing, _stats, streamResult, downloadResult, uploadResult);
|
|
307
337
|
const statusOnNext = _stats.statusOnNext;
|
|
308
338
|
if (response instanceof RezoError) {
|
|
339
|
+
if (!config.errors)
|
|
340
|
+
config.errors = [];
|
|
309
341
|
config.errors.push({
|
|
310
342
|
attempt: config.retryAttempts + 1,
|
|
311
343
|
error: response,
|
|
@@ -316,28 +348,25 @@ async function executeFetchRequest(fetchOptions, config, options, perform, strea
|
|
|
316
348
|
throw response;
|
|
317
349
|
}
|
|
318
350
|
if (config.retry) {
|
|
351
|
+
let shouldRetry = false;
|
|
319
352
|
if (config.retry.condition) {
|
|
320
353
|
const isPassed = await config.retry.condition(response);
|
|
321
|
-
|
|
322
|
-
throw response;
|
|
323
|
-
}
|
|
354
|
+
shouldRetry = isPassed === true;
|
|
324
355
|
} else {
|
|
325
|
-
|
|
326
|
-
throw response;
|
|
327
|
-
}
|
|
328
|
-
if (maxRetries <= retries) {
|
|
329
|
-
throw response;
|
|
330
|
-
}
|
|
331
|
-
retries++;
|
|
332
|
-
if (retryDelay > 0) {
|
|
333
|
-
await new Promise((resolve) => setTimeout(resolve, incrementDelay ? retryDelay * retries : retryDelay));
|
|
334
|
-
}
|
|
356
|
+
shouldRetry = statusCodes ? statusCodes.includes(response.status || 0) : true;
|
|
335
357
|
}
|
|
358
|
+
if (!shouldRetry || retries >= maxRetries) {
|
|
359
|
+
throw response;
|
|
360
|
+
}
|
|
361
|
+
retries++;
|
|
336
362
|
config.retryAttempts++;
|
|
363
|
+
if (retryDelay > 0) {
|
|
364
|
+
await new Promise((resolve) => setTimeout(resolve, incrementDelay ? retryDelay * retries : retryDelay));
|
|
365
|
+
}
|
|
337
366
|
}
|
|
338
367
|
continue;
|
|
339
368
|
}
|
|
340
|
-
if (statusOnNext === "success") {
|
|
369
|
+
if (statusOnNext === "success" || statusOnNext === "error") {
|
|
341
370
|
return response;
|
|
342
371
|
}
|
|
343
372
|
if (statusOnNext === "redirect") {
|
|
@@ -418,7 +447,13 @@ async function executeSingleFetchRequest(config, fetchOptions, requestCount, tim
|
|
|
418
447
|
config.isSecure = isSecure;
|
|
419
448
|
config.finalUrl = url.href;
|
|
420
449
|
config.network.protocol = isSecure ? "https" : "http";
|
|
450
|
+
config.network.httpVersion = undefined;
|
|
421
451
|
config.timing.startTimestamp = timing.startTimestamp;
|
|
452
|
+
if (!config.transfer) {
|
|
453
|
+
config.transfer = { requestSize: 0, responseSize: 0, headerSize: 0, bodySize: 0 };
|
|
454
|
+
} else if (config.transfer.requestSize === undefined) {
|
|
455
|
+
config.transfer.requestSize = 0;
|
|
456
|
+
}
|
|
422
457
|
}
|
|
423
458
|
const reqHeaders = fetchOptions.headers instanceof RezoHeaders ? fetchOptions.headers.toObject() : fetchOptions.headers || {};
|
|
424
459
|
const headers = toFetchHeaders(reqHeaders);
|
|
@@ -448,7 +483,26 @@ async function executeSingleFetchRequest(config, fetchOptions, requestCount, tim
|
|
|
448
483
|
clearTimeout(timeoutId);
|
|
449
484
|
});
|
|
450
485
|
}
|
|
486
|
+
if (body instanceof RezoFormData) {
|
|
487
|
+
const contentType = body.getContentType();
|
|
488
|
+
if (contentType && !headers.has("content-type")) {
|
|
489
|
+
headers.set("content-type", contentType);
|
|
490
|
+
}
|
|
491
|
+
}
|
|
451
492
|
const preparedBody = await prepareFetchBody(body);
|
|
493
|
+
if (config.transfer && body) {
|
|
494
|
+
if (typeof body === "string") {
|
|
495
|
+
config.transfer.requestSize = body.length;
|
|
496
|
+
} else if (body instanceof ArrayBuffer || body instanceof Uint8Array) {
|
|
497
|
+
config.transfer.requestSize = body.byteLength || body.length;
|
|
498
|
+
} else if (body instanceof URLSearchParams || body instanceof RezoURLSearchParams) {
|
|
499
|
+
config.transfer.requestSize = body.toString().length;
|
|
500
|
+
} else if (body instanceof RezoFormData) {
|
|
501
|
+
config.transfer.requestSize = body.getLengthSync();
|
|
502
|
+
} else if (typeof body === "object" && !(body instanceof Blob) && !(body instanceof ReadableStream)) {
|
|
503
|
+
config.transfer.requestSize = JSON.stringify(body).length;
|
|
504
|
+
}
|
|
505
|
+
}
|
|
452
506
|
const fetchInit = {
|
|
453
507
|
method: fetchOptions.method.toUpperCase(),
|
|
454
508
|
headers,
|
|
@@ -482,7 +536,7 @@ async function executeSingleFetchRequest(config, fetchOptions, requestCount, tim
|
|
|
482
536
|
const responseHeaders = fromFetchHeaders(response.headers);
|
|
483
537
|
const contentType = response.headers.get("content-type") || "";
|
|
484
538
|
const contentLength = response.headers.get("content-length");
|
|
485
|
-
const cookies = parseCookiesFromHeaders(response.headers, url.href);
|
|
539
|
+
const cookies = parseCookiesFromHeaders(response.headers, url.href, config);
|
|
486
540
|
config.responseCookies = cookies;
|
|
487
541
|
const location = response.headers.get("location");
|
|
488
542
|
const isRedirect = status >= 300 && status < 400 && location;
|
|
@@ -564,23 +618,16 @@ async function executeSingleFetchRequest(config, fetchOptions, requestCount, tim
|
|
|
564
618
|
const bodySize = bodyBuffer?.byteLength || (typeof responseData === "string" ? responseData.length : 0);
|
|
565
619
|
config.transfer.bodySize = bodySize;
|
|
566
620
|
config.transfer.responseSize = bodySize;
|
|
567
|
-
|
|
568
|
-
|
|
569
|
-
|
|
570
|
-
|
|
571
|
-
headers: responseHeaders,
|
|
572
|
-
data: responseData
|
|
573
|
-
}, config, fetchOptions);
|
|
574
|
-
_stats.statusOnNext = "error";
|
|
575
|
-
return error;
|
|
576
|
-
}
|
|
577
|
-
_stats.statusOnNext = "success";
|
|
621
|
+
config.status = status;
|
|
622
|
+
config.statusText = statusText;
|
|
623
|
+
_stats.statusOnNext = status >= 400 ? "error" : "success";
|
|
624
|
+
const mergedCookies = mergeRequestAndResponseCookies(config, cookies, url.href);
|
|
578
625
|
const finalResponse = {
|
|
579
626
|
data: responseData,
|
|
580
627
|
status,
|
|
581
628
|
statusText,
|
|
582
629
|
headers: responseHeaders,
|
|
583
|
-
cookies,
|
|
630
|
+
cookies: mergedCookies,
|
|
584
631
|
config,
|
|
585
632
|
contentType,
|
|
586
633
|
contentLength: bodySize,
|
|
@@ -599,7 +646,7 @@ async function executeSingleFetchRequest(config, fetchOptions, requestCount, tim
|
|
|
599
646
|
contentType,
|
|
600
647
|
contentLength: buffer.length,
|
|
601
648
|
finalUrl: url.href,
|
|
602
|
-
cookies,
|
|
649
|
+
cookies: mergedCookies,
|
|
603
650
|
urls: buildUrlTree(config, url.href),
|
|
604
651
|
fileName: config.fileName,
|
|
605
652
|
fileSize: buffer.length,
|
|
@@ -646,7 +693,7 @@ async function executeSingleFetchRequest(config, fetchOptions, requestCount, tim
|
|
|
646
693
|
contentLength: bodySize
|
|
647
694
|
},
|
|
648
695
|
finalUrl: url.href,
|
|
649
|
-
cookies,
|
|
696
|
+
cookies: mergedCookies,
|
|
650
697
|
urls: buildUrlTree(config, url.href),
|
|
651
698
|
uploadSize: config.transfer.requestSize || 0,
|
|
652
699
|
timing: {
|
package/dist/adapters/http.cjs
CHANGED
|
@@ -821,10 +821,23 @@ async function request(config, fetchOptions, requestCount, timing, _stats, respo
|
|
|
821
821
|
}
|
|
822
822
|
});
|
|
823
823
|
});
|
|
824
|
+
req.on("error", (error) => {
|
|
825
|
+
_stats.statusOnNext = "error";
|
|
826
|
+
updateTiming(config, timing, "", 0);
|
|
827
|
+
const e = buildSmartError(config, fetchOptions, error);
|
|
828
|
+
const eventEmitter = streamResult || downloadResult || uploadResult;
|
|
829
|
+
if (eventEmitter) {
|
|
830
|
+
eventEmitter.emit("error", e);
|
|
831
|
+
}
|
|
832
|
+
resolve(e);
|
|
833
|
+
return;
|
|
834
|
+
});
|
|
835
|
+
let bodyPiped = false;
|
|
824
836
|
if (body) {
|
|
825
837
|
if (body instanceof URLSearchParams || body instanceof RezoURLSearchParams) {
|
|
826
838
|
req.write(body.toString());
|
|
827
839
|
} else if (body instanceof FormData || body instanceof RezoFormData) {
|
|
840
|
+
bodyPiped = true;
|
|
828
841
|
if (body instanceof RezoFormData) {
|
|
829
842
|
req.setHeader("Content-Type", `multipart/form-data; boundary=${body.getBoundary()}`);
|
|
830
843
|
body.pipe(req);
|
|
@@ -833,24 +846,18 @@ async function request(config, fetchOptions, requestCount, timing, _stats, respo
|
|
|
833
846
|
req.setHeader("Content-Type", `multipart/form-data; boundary=${form.getBoundary()}`);
|
|
834
847
|
form.pipe(req);
|
|
835
848
|
}
|
|
836
|
-
} else if (
|
|
849
|
+
} else if (body instanceof Readable) {
|
|
850
|
+
bodyPiped = true;
|
|
851
|
+
body.pipe(req);
|
|
852
|
+
} else if (typeof body === "object" && !(body instanceof Buffer) && !(body instanceof Uint8Array)) {
|
|
837
853
|
req.write(JSON.stringify(body));
|
|
838
854
|
} else {
|
|
839
855
|
req.write(body);
|
|
840
856
|
}
|
|
841
857
|
}
|
|
842
|
-
|
|
843
|
-
|
|
844
|
-
|
|
845
|
-
updateTiming(config, timing, "", 0);
|
|
846
|
-
const e = buildSmartError(config, fetchOptions, error);
|
|
847
|
-
const eventEmitter = streamResult || downloadResult || uploadResult;
|
|
848
|
-
if (eventEmitter) {
|
|
849
|
-
eventEmitter.emit("error", e);
|
|
850
|
-
}
|
|
851
|
-
resolve(e);
|
|
852
|
-
return;
|
|
853
|
-
});
|
|
858
|
+
if (!bodyPiped) {
|
|
859
|
+
req.end();
|
|
860
|
+
}
|
|
854
861
|
} catch (error) {
|
|
855
862
|
_stats.statusOnNext = "error";
|
|
856
863
|
updateTiming(config, timing, "", 0);
|
package/dist/adapters/http.js
CHANGED
|
@@ -821,10 +821,23 @@ async function request(config, fetchOptions, requestCount, timing, _stats, respo
|
|
|
821
821
|
}
|
|
822
822
|
});
|
|
823
823
|
});
|
|
824
|
+
req.on("error", (error) => {
|
|
825
|
+
_stats.statusOnNext = "error";
|
|
826
|
+
updateTiming(config, timing, "", 0);
|
|
827
|
+
const e = buildSmartError(config, fetchOptions, error);
|
|
828
|
+
const eventEmitter = streamResult || downloadResult || uploadResult;
|
|
829
|
+
if (eventEmitter) {
|
|
830
|
+
eventEmitter.emit("error", e);
|
|
831
|
+
}
|
|
832
|
+
resolve(e);
|
|
833
|
+
return;
|
|
834
|
+
});
|
|
835
|
+
let bodyPiped = false;
|
|
824
836
|
if (body) {
|
|
825
837
|
if (body instanceof URLSearchParams || body instanceof RezoURLSearchParams) {
|
|
826
838
|
req.write(body.toString());
|
|
827
839
|
} else if (body instanceof FormData || body instanceof RezoFormData) {
|
|
840
|
+
bodyPiped = true;
|
|
828
841
|
if (body instanceof RezoFormData) {
|
|
829
842
|
req.setHeader("Content-Type", `multipart/form-data; boundary=${body.getBoundary()}`);
|
|
830
843
|
body.pipe(req);
|
|
@@ -833,24 +846,18 @@ async function request(config, fetchOptions, requestCount, timing, _stats, respo
|
|
|
833
846
|
req.setHeader("Content-Type", `multipart/form-data; boundary=${form.getBoundary()}`);
|
|
834
847
|
form.pipe(req);
|
|
835
848
|
}
|
|
836
|
-
} else if (
|
|
849
|
+
} else if (body instanceof Readable) {
|
|
850
|
+
bodyPiped = true;
|
|
851
|
+
body.pipe(req);
|
|
852
|
+
} else if (typeof body === "object" && !(body instanceof Buffer) && !(body instanceof Uint8Array)) {
|
|
837
853
|
req.write(JSON.stringify(body));
|
|
838
854
|
} else {
|
|
839
855
|
req.write(body);
|
|
840
856
|
}
|
|
841
857
|
}
|
|
842
|
-
|
|
843
|
-
|
|
844
|
-
|
|
845
|
-
updateTiming(config, timing, "", 0);
|
|
846
|
-
const e = buildSmartError(config, fetchOptions, error);
|
|
847
|
-
const eventEmitter = streamResult || downloadResult || uploadResult;
|
|
848
|
-
if (eventEmitter) {
|
|
849
|
-
eventEmitter.emit("error", e);
|
|
850
|
-
}
|
|
851
|
-
resolve(e);
|
|
852
|
-
return;
|
|
853
|
-
});
|
|
858
|
+
if (!bodyPiped) {
|
|
859
|
+
req.end();
|
|
860
|
+
}
|
|
854
861
|
} catch (error) {
|
|
855
862
|
_stats.statusOnNext = "error";
|
|
856
863
|
updateTiming(config, timing, "", 0);
|