rezo 1.0.12 → 1.0.13
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/adapters/curl.cjs +73 -10
- package/dist/adapters/curl.js +73 -10
- package/dist/adapters/fetch.cjs +106 -59
- package/dist/adapters/fetch.js +106 -59
- package/dist/adapters/http.cjs +20 -13
- package/dist/adapters/http.js +20 -13
- package/dist/adapters/http2.cjs +114 -59
- package/dist/adapters/http2.js +114 -59
- package/dist/adapters/index.cjs +6 -6
- package/dist/cache/index.cjs +13 -13
- package/dist/entries/crawler.cjs +5 -5
- package/dist/index.cjs +24 -24
- package/dist/plugin/index.cjs +36 -36
- package/dist/proxy/index.cjs +2 -2
- package/dist/queue/index.cjs +8 -8
- package/dist/responses/buildError.cjs +5 -1
- package/dist/responses/buildError.js +5 -1
- package/dist/responses/buildResponse.cjs +2 -0
- package/dist/responses/buildResponse.js +2 -0
- package/dist/utils/compression.cjs +6 -6
- package/dist/utils/compression.js +6 -6
- package/dist/utils/headers.cjs +17 -0
- package/dist/utils/headers.js +17 -1
- package/dist/utils/http-config.cjs +39 -4
- package/dist/utils/http-config.js +39 -4
- package/package.json +1 -1
package/dist/adapters/curl.cjs
CHANGED
|
@@ -7,7 +7,7 @@ const { Readable } = require("node:stream");
|
|
|
7
7
|
const { EventEmitter } = require("node:events");
|
|
8
8
|
const { RezoError } = require('../errors/rezo-error.cjs');
|
|
9
9
|
const { buildSmartError } = require('../responses/buildError.cjs');
|
|
10
|
-
const { Cookie } = require('../utils/cookies.cjs');
|
|
10
|
+
const { RezoCookieJar, Cookie } = require('../utils/cookies.cjs');
|
|
11
11
|
const RezoFormData = require('../utils/form-data.cjs');
|
|
12
12
|
const { existsSync } = require("node:fs");
|
|
13
13
|
const { getDefaultConfig, prepareHTTPOptions } = require('../utils/http-config.cjs');
|
|
@@ -16,6 +16,24 @@ const { StreamResponse } = require('../responses/stream.cjs');
|
|
|
16
16
|
const { DownloadResponse } = require('../responses/download.cjs');
|
|
17
17
|
const { UploadResponse } = require('../responses/upload.cjs');
|
|
18
18
|
const { RezoPerformance } = require('../utils/tools.cjs');
|
|
19
|
+
function mergeRequestAndResponseCookies(requestCookies, responseCookies) {
|
|
20
|
+
if (!requestCookies || requestCookies.length === 0) {
|
|
21
|
+
return responseCookies;
|
|
22
|
+
}
|
|
23
|
+
if (responseCookies.length === 0) {
|
|
24
|
+
return requestCookies;
|
|
25
|
+
}
|
|
26
|
+
const cookieMap = new Map;
|
|
27
|
+
for (const cookie of requestCookies) {
|
|
28
|
+
const key = `${cookie.key}|${cookie.domain || ""}`;
|
|
29
|
+
cookieMap.set(key, cookie);
|
|
30
|
+
}
|
|
31
|
+
for (const cookie of responseCookies) {
|
|
32
|
+
const key = `${cookie.key}|${cookie.domain || ""}`;
|
|
33
|
+
cookieMap.set(key, cookie);
|
|
34
|
+
}
|
|
35
|
+
return Array.from(cookieMap.values());
|
|
36
|
+
}
|
|
19
37
|
|
|
20
38
|
class CurlCapabilities {
|
|
21
39
|
static instance;
|
|
@@ -261,7 +279,7 @@ class CurlCommandBuilder {
|
|
|
261
279
|
this.buildDownloadOptions(config, originalRequest, createdTempFiles);
|
|
262
280
|
this.buildHeaders(config);
|
|
263
281
|
this.buildRedirectOptions(config, originalRequest);
|
|
264
|
-
this.buildRequestBody(config, createdTempFiles);
|
|
282
|
+
this.buildRequestBody(config, originalRequest, createdTempFiles);
|
|
265
283
|
if (originalRequest.onUploadProgress || originalRequest.onDownloadProgress) {
|
|
266
284
|
this.addArg("--progress-bar");
|
|
267
285
|
}
|
|
@@ -1290,11 +1308,11 @@ class CurlCommandBuilder {
|
|
|
1290
1308
|
this.addArg("--max-redirs", "0");
|
|
1291
1309
|
}
|
|
1292
1310
|
}
|
|
1293
|
-
buildRequestBody(config, tempFiles) {
|
|
1294
|
-
|
|
1311
|
+
buildRequestBody(config, originalRequest, tempFiles) {
|
|
1312
|
+
const data = originalRequest.body ?? config.data;
|
|
1313
|
+
if (!data) {
|
|
1295
1314
|
return;
|
|
1296
1315
|
}
|
|
1297
|
-
const data = config.data;
|
|
1298
1316
|
if (typeof data === "string") {
|
|
1299
1317
|
this.addArg("-d", data);
|
|
1300
1318
|
} else if (Buffer.isBuffer(data)) {
|
|
@@ -1304,7 +1322,6 @@ class CurlCommandBuilder {
|
|
|
1304
1322
|
this.addArg("--data-binary", `@${dataFile}`);
|
|
1305
1323
|
} else if (data instanceof RezoFormData) {
|
|
1306
1324
|
const formData = data;
|
|
1307
|
-
const knownLength = formData.getLengthSync?.() || 0;
|
|
1308
1325
|
const formDataFile = this.tempFiles.createTempFile("formdata", ".txt");
|
|
1309
1326
|
const formBuffer = formData.getBuffer?.();
|
|
1310
1327
|
if (formBuffer) {
|
|
@@ -1315,6 +1332,7 @@ class CurlCommandBuilder {
|
|
|
1315
1332
|
if (boundary) {
|
|
1316
1333
|
this.addArg("-H", `Content-Type: multipart/form-data; boundary=${boundary}`);
|
|
1317
1334
|
}
|
|
1335
|
+
this.addArg("-H", `Content-Length: ${formBuffer.length}`);
|
|
1318
1336
|
}
|
|
1319
1337
|
} else if (data instanceof Readable) {
|
|
1320
1338
|
this.addArg("-d", "@-");
|
|
@@ -1324,7 +1342,8 @@ class CurlCommandBuilder {
|
|
|
1324
1342
|
}
|
|
1325
1343
|
buildWriteOutFormat() {
|
|
1326
1344
|
return [
|
|
1327
|
-
|
|
1345
|
+
`
|
|
1346
|
+
---CURL_STATS_START---`,
|
|
1328
1347
|
"http_code:%{http_code}",
|
|
1329
1348
|
"time_namelookup:%{time_namelookup}",
|
|
1330
1349
|
"time_connect:%{time_connect}",
|
|
@@ -1344,7 +1363,8 @@ class CurlCommandBuilder {
|
|
|
1344
1363
|
"ssl_verify_result:%{ssl_verify_result}",
|
|
1345
1364
|
"content_type:%{content_type}",
|
|
1346
1365
|
"---CURL_STATS_END---"
|
|
1347
|
-
].join(
|
|
1366
|
+
].join(`
|
|
1367
|
+
`);
|
|
1348
1368
|
}
|
|
1349
1369
|
}
|
|
1350
1370
|
|
|
@@ -1381,7 +1401,24 @@ class CurlResponseParser {
|
|
|
1381
1401
|
const statusText = this.getStatusText(status);
|
|
1382
1402
|
const headers = this.parseHeaders(headerSection);
|
|
1383
1403
|
const rezoHeaders = new RezoHeaders(headers);
|
|
1384
|
-
const
|
|
1404
|
+
const responseCookies = this.parseCookies(headers);
|
|
1405
|
+
const mergedCookieArray = mergeRequestAndResponseCookies(config.requestCookies, responseCookies.array);
|
|
1406
|
+
let cookies;
|
|
1407
|
+
if (mergedCookieArray.length > 0) {
|
|
1408
|
+
const mergedJar = new RezoCookieJar(mergedCookieArray, config.url || "");
|
|
1409
|
+
cookies = mergedJar.cookies();
|
|
1410
|
+
} else {
|
|
1411
|
+
cookies = {
|
|
1412
|
+
array: [],
|
|
1413
|
+
serialized: [],
|
|
1414
|
+
netscape: `# Netscape HTTP Cookie File
|
|
1415
|
+
# This file was generated by Rezo HTTP client
|
|
1416
|
+
# Based on uniqhtt cookie implementation
|
|
1417
|
+
`,
|
|
1418
|
+
string: "",
|
|
1419
|
+
setCookiesString: []
|
|
1420
|
+
};
|
|
1421
|
+
}
|
|
1385
1422
|
let data;
|
|
1386
1423
|
const contentType = stats["content_type"] || rezoHeaders.get("content-type") || "";
|
|
1387
1424
|
const responseType = config.responseType || originalRequest.responseType || "auto";
|
|
@@ -1408,6 +1445,25 @@ class CurlResponseParser {
|
|
|
1408
1445
|
durationMs: totalMs
|
|
1409
1446
|
};
|
|
1410
1447
|
config.timing = timing;
|
|
1448
|
+
config.status = status;
|
|
1449
|
+
config.statusText = statusText;
|
|
1450
|
+
const isSecure = config.url?.startsWith("https") || false;
|
|
1451
|
+
config.adapterUsed = "curl";
|
|
1452
|
+
config.isSecure = isSecure;
|
|
1453
|
+
config.finalUrl = stats["redirect_url"] || config.url || "";
|
|
1454
|
+
if (!config.network) {
|
|
1455
|
+
config.network = {};
|
|
1456
|
+
}
|
|
1457
|
+
config.network.protocol = isSecure ? "https" : "http";
|
|
1458
|
+
config.network.httpVersion = headerSection.match(/HTTP\/([\d.]+)/)?.[1] || "1.1";
|
|
1459
|
+
if (!config.transfer) {
|
|
1460
|
+
config.transfer = { requestSize: 0, responseSize: 0, headerSize: 0, bodySize: 0 };
|
|
1461
|
+
}
|
|
1462
|
+
config.transfer.requestSize = parseInt(stats["size_upload"]) || 0;
|
|
1463
|
+
config.transfer.responseSize = parseInt(stats["size_download"]) || responseBody.length;
|
|
1464
|
+
config.transfer.bodySize = responseBody.length;
|
|
1465
|
+
config.transfer.headerSize = headerSection.length;
|
|
1466
|
+
config.responseCookies = cookies;
|
|
1411
1467
|
const urls = [config.url];
|
|
1412
1468
|
if (stats["redirect_url"]) {
|
|
1413
1469
|
urls.push(stats["redirect_url"]);
|
|
@@ -1507,7 +1563,7 @@ class CurlExecutor {
|
|
|
1507
1563
|
}
|
|
1508
1564
|
buildFinalUrl(config) {
|
|
1509
1565
|
let url = config.url;
|
|
1510
|
-
if (config.baseURL) {
|
|
1566
|
+
if (config.baseURL && !url.startsWith("http://") && !url.startsWith("https://")) {
|
|
1511
1567
|
url = config.baseURL.replace(/\/$/, "") + "/" + url.replace(/^\//, "");
|
|
1512
1568
|
}
|
|
1513
1569
|
if (config.params && Object.keys(config.params).length > 0) {
|
|
@@ -1596,6 +1652,13 @@ class CurlExecutor {
|
|
|
1596
1652
|
curl.on("close", (code) => {
|
|
1597
1653
|
try {
|
|
1598
1654
|
if (code !== 0 && code !== null) {
|
|
1655
|
+
if (code === 22 && stdout) {
|
|
1656
|
+
try {
|
|
1657
|
+
const response = CurlResponseParser.parse(stdout, stderr, config, originalRequest);
|
|
1658
|
+
resolve(response);
|
|
1659
|
+
return;
|
|
1660
|
+
} catch {}
|
|
1661
|
+
}
|
|
1599
1662
|
const errorCode = this.mapCurlErrorCode(code);
|
|
1600
1663
|
const errorMessage = this.buildDetailedErrorMessage(code, stderr, config);
|
|
1601
1664
|
const rezoError = new RezoError(errorMessage, config, errorCode);
|
package/dist/adapters/curl.js
CHANGED
|
@@ -7,7 +7,7 @@ import { Readable } from "node:stream";
|
|
|
7
7
|
import { EventEmitter } from "node:events";
|
|
8
8
|
import { RezoError } from '../errors/rezo-error.js';
|
|
9
9
|
import { buildSmartError } from '../responses/buildError.js';
|
|
10
|
-
import { Cookie } from '../utils/cookies.js';
|
|
10
|
+
import { RezoCookieJar, Cookie } from '../utils/cookies.js';
|
|
11
11
|
import RezoFormData from '../utils/form-data.js';
|
|
12
12
|
import { existsSync } from "node:fs";
|
|
13
13
|
import { getDefaultConfig, prepareHTTPOptions } from '../utils/http-config.js';
|
|
@@ -16,6 +16,24 @@ import { StreamResponse } from '../responses/stream.js';
|
|
|
16
16
|
import { DownloadResponse } from '../responses/download.js';
|
|
17
17
|
import { UploadResponse } from '../responses/upload.js';
|
|
18
18
|
import { RezoPerformance } from '../utils/tools.js';
|
|
19
|
+
function mergeRequestAndResponseCookies(requestCookies, responseCookies) {
|
|
20
|
+
if (!requestCookies || requestCookies.length === 0) {
|
|
21
|
+
return responseCookies;
|
|
22
|
+
}
|
|
23
|
+
if (responseCookies.length === 0) {
|
|
24
|
+
return requestCookies;
|
|
25
|
+
}
|
|
26
|
+
const cookieMap = new Map;
|
|
27
|
+
for (const cookie of requestCookies) {
|
|
28
|
+
const key = `${cookie.key}|${cookie.domain || ""}`;
|
|
29
|
+
cookieMap.set(key, cookie);
|
|
30
|
+
}
|
|
31
|
+
for (const cookie of responseCookies) {
|
|
32
|
+
const key = `${cookie.key}|${cookie.domain || ""}`;
|
|
33
|
+
cookieMap.set(key, cookie);
|
|
34
|
+
}
|
|
35
|
+
return Array.from(cookieMap.values());
|
|
36
|
+
}
|
|
19
37
|
|
|
20
38
|
class CurlCapabilities {
|
|
21
39
|
static instance;
|
|
@@ -261,7 +279,7 @@ class CurlCommandBuilder {
|
|
|
261
279
|
this.buildDownloadOptions(config, originalRequest, createdTempFiles);
|
|
262
280
|
this.buildHeaders(config);
|
|
263
281
|
this.buildRedirectOptions(config, originalRequest);
|
|
264
|
-
this.buildRequestBody(config, createdTempFiles);
|
|
282
|
+
this.buildRequestBody(config, originalRequest, createdTempFiles);
|
|
265
283
|
if (originalRequest.onUploadProgress || originalRequest.onDownloadProgress) {
|
|
266
284
|
this.addArg("--progress-bar");
|
|
267
285
|
}
|
|
@@ -1290,11 +1308,11 @@ class CurlCommandBuilder {
|
|
|
1290
1308
|
this.addArg("--max-redirs", "0");
|
|
1291
1309
|
}
|
|
1292
1310
|
}
|
|
1293
|
-
buildRequestBody(config, tempFiles) {
|
|
1294
|
-
|
|
1311
|
+
buildRequestBody(config, originalRequest, tempFiles) {
|
|
1312
|
+
const data = originalRequest.body ?? config.data;
|
|
1313
|
+
if (!data) {
|
|
1295
1314
|
return;
|
|
1296
1315
|
}
|
|
1297
|
-
const data = config.data;
|
|
1298
1316
|
if (typeof data === "string") {
|
|
1299
1317
|
this.addArg("-d", data);
|
|
1300
1318
|
} else if (Buffer.isBuffer(data)) {
|
|
@@ -1304,7 +1322,6 @@ class CurlCommandBuilder {
|
|
|
1304
1322
|
this.addArg("--data-binary", `@${dataFile}`);
|
|
1305
1323
|
} else if (data instanceof RezoFormData) {
|
|
1306
1324
|
const formData = data;
|
|
1307
|
-
const knownLength = formData.getLengthSync?.() || 0;
|
|
1308
1325
|
const formDataFile = this.tempFiles.createTempFile("formdata", ".txt");
|
|
1309
1326
|
const formBuffer = formData.getBuffer?.();
|
|
1310
1327
|
if (formBuffer) {
|
|
@@ -1315,6 +1332,7 @@ class CurlCommandBuilder {
|
|
|
1315
1332
|
if (boundary) {
|
|
1316
1333
|
this.addArg("-H", `Content-Type: multipart/form-data; boundary=${boundary}`);
|
|
1317
1334
|
}
|
|
1335
|
+
this.addArg("-H", `Content-Length: ${formBuffer.length}`);
|
|
1318
1336
|
}
|
|
1319
1337
|
} else if (data instanceof Readable) {
|
|
1320
1338
|
this.addArg("-d", "@-");
|
|
@@ -1324,7 +1342,8 @@ class CurlCommandBuilder {
|
|
|
1324
1342
|
}
|
|
1325
1343
|
buildWriteOutFormat() {
|
|
1326
1344
|
return [
|
|
1327
|
-
|
|
1345
|
+
`
|
|
1346
|
+
---CURL_STATS_START---`,
|
|
1328
1347
|
"http_code:%{http_code}",
|
|
1329
1348
|
"time_namelookup:%{time_namelookup}",
|
|
1330
1349
|
"time_connect:%{time_connect}",
|
|
@@ -1344,7 +1363,8 @@ class CurlCommandBuilder {
|
|
|
1344
1363
|
"ssl_verify_result:%{ssl_verify_result}",
|
|
1345
1364
|
"content_type:%{content_type}",
|
|
1346
1365
|
"---CURL_STATS_END---"
|
|
1347
|
-
].join(
|
|
1366
|
+
].join(`
|
|
1367
|
+
`);
|
|
1348
1368
|
}
|
|
1349
1369
|
}
|
|
1350
1370
|
|
|
@@ -1381,7 +1401,24 @@ class CurlResponseParser {
|
|
|
1381
1401
|
const statusText = this.getStatusText(status);
|
|
1382
1402
|
const headers = this.parseHeaders(headerSection);
|
|
1383
1403
|
const rezoHeaders = new RezoHeaders(headers);
|
|
1384
|
-
const
|
|
1404
|
+
const responseCookies = this.parseCookies(headers);
|
|
1405
|
+
const mergedCookieArray = mergeRequestAndResponseCookies(config.requestCookies, responseCookies.array);
|
|
1406
|
+
let cookies;
|
|
1407
|
+
if (mergedCookieArray.length > 0) {
|
|
1408
|
+
const mergedJar = new RezoCookieJar(mergedCookieArray, config.url || "");
|
|
1409
|
+
cookies = mergedJar.cookies();
|
|
1410
|
+
} else {
|
|
1411
|
+
cookies = {
|
|
1412
|
+
array: [],
|
|
1413
|
+
serialized: [],
|
|
1414
|
+
netscape: `# Netscape HTTP Cookie File
|
|
1415
|
+
# This file was generated by Rezo HTTP client
|
|
1416
|
+
# Based on uniqhtt cookie implementation
|
|
1417
|
+
`,
|
|
1418
|
+
string: "",
|
|
1419
|
+
setCookiesString: []
|
|
1420
|
+
};
|
|
1421
|
+
}
|
|
1385
1422
|
let data;
|
|
1386
1423
|
const contentType = stats["content_type"] || rezoHeaders.get("content-type") || "";
|
|
1387
1424
|
const responseType = config.responseType || originalRequest.responseType || "auto";
|
|
@@ -1408,6 +1445,25 @@ class CurlResponseParser {
|
|
|
1408
1445
|
durationMs: totalMs
|
|
1409
1446
|
};
|
|
1410
1447
|
config.timing = timing;
|
|
1448
|
+
config.status = status;
|
|
1449
|
+
config.statusText = statusText;
|
|
1450
|
+
const isSecure = config.url?.startsWith("https") || false;
|
|
1451
|
+
config.adapterUsed = "curl";
|
|
1452
|
+
config.isSecure = isSecure;
|
|
1453
|
+
config.finalUrl = stats["redirect_url"] || config.url || "";
|
|
1454
|
+
if (!config.network) {
|
|
1455
|
+
config.network = {};
|
|
1456
|
+
}
|
|
1457
|
+
config.network.protocol = isSecure ? "https" : "http";
|
|
1458
|
+
config.network.httpVersion = headerSection.match(/HTTP\/([\d.]+)/)?.[1] || "1.1";
|
|
1459
|
+
if (!config.transfer) {
|
|
1460
|
+
config.transfer = { requestSize: 0, responseSize: 0, headerSize: 0, bodySize: 0 };
|
|
1461
|
+
}
|
|
1462
|
+
config.transfer.requestSize = parseInt(stats["size_upload"]) || 0;
|
|
1463
|
+
config.transfer.responseSize = parseInt(stats["size_download"]) || responseBody.length;
|
|
1464
|
+
config.transfer.bodySize = responseBody.length;
|
|
1465
|
+
config.transfer.headerSize = headerSection.length;
|
|
1466
|
+
config.responseCookies = cookies;
|
|
1411
1467
|
const urls = [config.url];
|
|
1412
1468
|
if (stats["redirect_url"]) {
|
|
1413
1469
|
urls.push(stats["redirect_url"]);
|
|
@@ -1507,7 +1563,7 @@ class CurlExecutor {
|
|
|
1507
1563
|
}
|
|
1508
1564
|
buildFinalUrl(config) {
|
|
1509
1565
|
let url = config.url;
|
|
1510
|
-
if (config.baseURL) {
|
|
1566
|
+
if (config.baseURL && !url.startsWith("http://") && !url.startsWith("https://")) {
|
|
1511
1567
|
url = config.baseURL.replace(/\/$/, "") + "/" + url.replace(/^\//, "");
|
|
1512
1568
|
}
|
|
1513
1569
|
if (config.params && Object.keys(config.params).length > 0) {
|
|
@@ -1596,6 +1652,13 @@ class CurlExecutor {
|
|
|
1596
1652
|
curl.on("close", (code) => {
|
|
1597
1653
|
try {
|
|
1598
1654
|
if (code !== 0 && code !== null) {
|
|
1655
|
+
if (code === 22 && stdout) {
|
|
1656
|
+
try {
|
|
1657
|
+
const response = CurlResponseParser.parse(stdout, stderr, config, originalRequest);
|
|
1658
|
+
resolve(response);
|
|
1659
|
+
return;
|
|
1660
|
+
} catch {}
|
|
1661
|
+
}
|
|
1599
1662
|
const errorCode = this.mapCurlErrorCode(code);
|
|
1600
1663
|
const errorMessage = this.buildDetailedErrorMessage(code, stderr, config);
|
|
1601
1664
|
const rezoError = new RezoError(errorMessage, config, errorCode);
|
package/dist/adapters/fetch.cjs
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
const { URL } = require("node:url");
|
|
2
2
|
const { RezoError } = require('../errors/rezo-error.cjs');
|
|
3
3
|
const { buildSmartError, builErrorFromResponse, buildDownloadError } = require('../responses/buildError.cjs');
|
|
4
|
-
const {
|
|
4
|
+
const { RezoCookieJar } = require('../utils/cookies.cjs');
|
|
5
5
|
const RezoFormData = require('../utils/form-data.cjs');
|
|
6
6
|
const { getDefaultConfig, prepareHTTPOptions } = require('../utils/http-config.cjs');
|
|
7
7
|
const { RezoHeaders } = require('../utils/headers.cjs');
|
|
@@ -105,39 +105,69 @@ function sanitizeConfig(config) {
|
|
|
105
105
|
delete sanitized.data;
|
|
106
106
|
return sanitized;
|
|
107
107
|
}
|
|
108
|
-
function parseCookiesFromHeaders(headers, url) {
|
|
109
|
-
|
|
108
|
+
function parseCookiesFromHeaders(headers, url, config) {
|
|
109
|
+
let setCookieHeaders = [];
|
|
110
|
+
if (typeof headers.getSetCookie === "function") {
|
|
111
|
+
setCookieHeaders = headers.getSetCookie() || [];
|
|
112
|
+
} else {
|
|
113
|
+
const setCookieRaw = headers.get("set-cookie");
|
|
114
|
+
if (setCookieRaw) {
|
|
115
|
+
const splitPattern = /,(?=\s*[A-Za-z0-9_-]+=)/;
|
|
116
|
+
setCookieHeaders = setCookieRaw.split(splitPattern).map((s) => s.trim()).filter(Boolean);
|
|
117
|
+
}
|
|
118
|
+
}
|
|
119
|
+
if (setCookieHeaders.length === 0) {
|
|
120
|
+
return {
|
|
121
|
+
array: [],
|
|
122
|
+
serialized: [],
|
|
123
|
+
netscape: "",
|
|
124
|
+
string: "",
|
|
125
|
+
setCookiesString: []
|
|
126
|
+
};
|
|
127
|
+
}
|
|
128
|
+
const jar = new RezoCookieJar;
|
|
129
|
+
jar.setCookiesSync(setCookieHeaders, url);
|
|
130
|
+
if (config?.enableCookieJar && config?.cookieJar) {
|
|
131
|
+
config.cookieJar.setCookiesSync(setCookieHeaders, url);
|
|
132
|
+
}
|
|
133
|
+
const cookies = jar.cookies();
|
|
134
|
+
cookies.setCookiesString = setCookieHeaders;
|
|
135
|
+
return cookies;
|
|
136
|
+
}
|
|
137
|
+
function mergeRequestAndResponseCookies(config, responseCookies, url) {
|
|
138
|
+
const mergedCookiesArray = [];
|
|
139
|
+
const cookieKeyDomainMap = new Map;
|
|
140
|
+
if (config.requestCookies && config.requestCookies.length > 0) {
|
|
141
|
+
for (const cookie of config.requestCookies) {
|
|
142
|
+
const key = `${cookie.key}|${cookie.domain || ""}`;
|
|
143
|
+
mergedCookiesArray.push(cookie);
|
|
144
|
+
cookieKeyDomainMap.set(key, mergedCookiesArray.length - 1);
|
|
145
|
+
}
|
|
146
|
+
}
|
|
147
|
+
for (const cookie of responseCookies.array) {
|
|
148
|
+
const key = `${cookie.key}|${cookie.domain || ""}`;
|
|
149
|
+
const existingIndex = cookieKeyDomainMap.get(key);
|
|
150
|
+
if (existingIndex !== undefined) {
|
|
151
|
+
mergedCookiesArray[existingIndex] = cookie;
|
|
152
|
+
} else {
|
|
153
|
+
mergedCookiesArray.push(cookie);
|
|
154
|
+
cookieKeyDomainMap.set(key, mergedCookiesArray.length - 1);
|
|
155
|
+
}
|
|
156
|
+
}
|
|
157
|
+
if (mergedCookiesArray.length > 0) {
|
|
158
|
+
const mergedJar = new RezoCookieJar(mergedCookiesArray, url);
|
|
159
|
+
return mergedJar.cookies();
|
|
160
|
+
}
|
|
161
|
+
return {
|
|
110
162
|
array: [],
|
|
111
163
|
serialized: [],
|
|
112
|
-
netscape:
|
|
164
|
+
netscape: `# Netscape HTTP Cookie File
|
|
165
|
+
# This file was generated by Rezo HTTP client
|
|
166
|
+
# Based on uniqhtt cookie implementation
|
|
167
|
+
`,
|
|
113
168
|
string: "",
|
|
114
169
|
setCookiesString: []
|
|
115
170
|
};
|
|
116
|
-
const setCookieHeaders = headers.getSetCookie?.() || [];
|
|
117
|
-
for (const cookieStr of setCookieHeaders) {
|
|
118
|
-
cookies.setCookiesString.push(cookieStr);
|
|
119
|
-
const parts = cookieStr.split(";");
|
|
120
|
-
const [nameValue] = parts;
|
|
121
|
-
const [name, ...valueParts] = nameValue.split("=");
|
|
122
|
-
const value = valueParts.join("=");
|
|
123
|
-
if (name && value !== undefined) {
|
|
124
|
-
const cookie = new Cookie({
|
|
125
|
-
key: name.trim(),
|
|
126
|
-
value: value.trim(),
|
|
127
|
-
domain: new URL(url).hostname,
|
|
128
|
-
path: "/",
|
|
129
|
-
httpOnly: cookieStr.toLowerCase().includes("httponly"),
|
|
130
|
-
secure: cookieStr.toLowerCase().includes("secure"),
|
|
131
|
-
sameSite: "lax"
|
|
132
|
-
});
|
|
133
|
-
cookies.array.push(cookie);
|
|
134
|
-
}
|
|
135
|
-
}
|
|
136
|
-
cookies.string = cookies.array.map((c) => `${c.key}=${c.value}`).join("; ");
|
|
137
|
-
cookies.serialized = cookies.array.map((c) => c.toJSON());
|
|
138
|
-
cookies.netscape = cookies.array.map((c) => c.toNetscapeFormat()).join(`
|
|
139
|
-
`);
|
|
140
|
-
return cookies;
|
|
141
171
|
}
|
|
142
172
|
function toFetchHeaders(headers) {
|
|
143
173
|
const fetchHeaders = new Headers;
|
|
@@ -181,7 +211,7 @@ async function prepareFetchBody(body) {
|
|
|
181
211
|
return new Uint8Array(buffer);
|
|
182
212
|
}
|
|
183
213
|
}
|
|
184
|
-
const nativeForm = body.toNativeFormData();
|
|
214
|
+
const nativeForm = await body.toNativeFormData();
|
|
185
215
|
if (nativeForm) {
|
|
186
216
|
return nativeForm;
|
|
187
217
|
}
|
|
@@ -306,6 +336,8 @@ async function executeFetchRequest(fetchOptions, config, options, perform, strea
|
|
|
306
336
|
const response = await executeSingleFetchRequest(config, fetchOptions, requestCount, timing, _stats, streamResult, downloadResult, uploadResult);
|
|
307
337
|
const statusOnNext = _stats.statusOnNext;
|
|
308
338
|
if (response instanceof RezoError) {
|
|
339
|
+
if (!config.errors)
|
|
340
|
+
config.errors = [];
|
|
309
341
|
config.errors.push({
|
|
310
342
|
attempt: config.retryAttempts + 1,
|
|
311
343
|
error: response,
|
|
@@ -316,28 +348,25 @@ async function executeFetchRequest(fetchOptions, config, options, perform, strea
|
|
|
316
348
|
throw response;
|
|
317
349
|
}
|
|
318
350
|
if (config.retry) {
|
|
351
|
+
let shouldRetry = false;
|
|
319
352
|
if (config.retry.condition) {
|
|
320
353
|
const isPassed = await config.retry.condition(response);
|
|
321
|
-
|
|
322
|
-
throw response;
|
|
323
|
-
}
|
|
354
|
+
shouldRetry = isPassed === true;
|
|
324
355
|
} else {
|
|
325
|
-
|
|
326
|
-
throw response;
|
|
327
|
-
}
|
|
328
|
-
if (maxRetries <= retries) {
|
|
329
|
-
throw response;
|
|
330
|
-
}
|
|
331
|
-
retries++;
|
|
332
|
-
if (retryDelay > 0) {
|
|
333
|
-
await new Promise((resolve) => setTimeout(resolve, incrementDelay ? retryDelay * retries : retryDelay));
|
|
334
|
-
}
|
|
356
|
+
shouldRetry = statusCodes ? statusCodes.includes(response.status || 0) : true;
|
|
335
357
|
}
|
|
358
|
+
if (!shouldRetry || retries >= maxRetries) {
|
|
359
|
+
throw response;
|
|
360
|
+
}
|
|
361
|
+
retries++;
|
|
336
362
|
config.retryAttempts++;
|
|
363
|
+
if (retryDelay > 0) {
|
|
364
|
+
await new Promise((resolve) => setTimeout(resolve, incrementDelay ? retryDelay * retries : retryDelay));
|
|
365
|
+
}
|
|
337
366
|
}
|
|
338
367
|
continue;
|
|
339
368
|
}
|
|
340
|
-
if (statusOnNext === "success") {
|
|
369
|
+
if (statusOnNext === "success" || statusOnNext === "error") {
|
|
341
370
|
return response;
|
|
342
371
|
}
|
|
343
372
|
if (statusOnNext === "redirect") {
|
|
@@ -418,7 +447,13 @@ async function executeSingleFetchRequest(config, fetchOptions, requestCount, tim
|
|
|
418
447
|
config.isSecure = isSecure;
|
|
419
448
|
config.finalUrl = url.href;
|
|
420
449
|
config.network.protocol = isSecure ? "https" : "http";
|
|
450
|
+
config.network.httpVersion = undefined;
|
|
421
451
|
config.timing.startTimestamp = timing.startTimestamp;
|
|
452
|
+
if (!config.transfer) {
|
|
453
|
+
config.transfer = { requestSize: 0, responseSize: 0, headerSize: 0, bodySize: 0 };
|
|
454
|
+
} else if (config.transfer.requestSize === undefined) {
|
|
455
|
+
config.transfer.requestSize = 0;
|
|
456
|
+
}
|
|
422
457
|
}
|
|
423
458
|
const reqHeaders = fetchOptions.headers instanceof RezoHeaders ? fetchOptions.headers.toObject() : fetchOptions.headers || {};
|
|
424
459
|
const headers = toFetchHeaders(reqHeaders);
|
|
@@ -448,7 +483,26 @@ async function executeSingleFetchRequest(config, fetchOptions, requestCount, tim
|
|
|
448
483
|
clearTimeout(timeoutId);
|
|
449
484
|
});
|
|
450
485
|
}
|
|
486
|
+
if (body instanceof RezoFormData) {
|
|
487
|
+
const contentType = body.getContentType();
|
|
488
|
+
if (contentType && !headers.has("content-type")) {
|
|
489
|
+
headers.set("content-type", contentType);
|
|
490
|
+
}
|
|
491
|
+
}
|
|
451
492
|
const preparedBody = await prepareFetchBody(body);
|
|
493
|
+
if (config.transfer && body) {
|
|
494
|
+
if (typeof body === "string") {
|
|
495
|
+
config.transfer.requestSize = body.length;
|
|
496
|
+
} else if (body instanceof ArrayBuffer || body instanceof Uint8Array) {
|
|
497
|
+
config.transfer.requestSize = body.byteLength || body.length;
|
|
498
|
+
} else if (body instanceof URLSearchParams || body instanceof RezoURLSearchParams) {
|
|
499
|
+
config.transfer.requestSize = body.toString().length;
|
|
500
|
+
} else if (body instanceof RezoFormData) {
|
|
501
|
+
config.transfer.requestSize = body.getLengthSync();
|
|
502
|
+
} else if (typeof body === "object" && !(body instanceof Blob) && !(body instanceof ReadableStream)) {
|
|
503
|
+
config.transfer.requestSize = JSON.stringify(body).length;
|
|
504
|
+
}
|
|
505
|
+
}
|
|
452
506
|
const fetchInit = {
|
|
453
507
|
method: fetchOptions.method.toUpperCase(),
|
|
454
508
|
headers,
|
|
@@ -482,7 +536,7 @@ async function executeSingleFetchRequest(config, fetchOptions, requestCount, tim
|
|
|
482
536
|
const responseHeaders = fromFetchHeaders(response.headers);
|
|
483
537
|
const contentType = response.headers.get("content-type") || "";
|
|
484
538
|
const contentLength = response.headers.get("content-length");
|
|
485
|
-
const cookies = parseCookiesFromHeaders(response.headers, url.href);
|
|
539
|
+
const cookies = parseCookiesFromHeaders(response.headers, url.href, config);
|
|
486
540
|
config.responseCookies = cookies;
|
|
487
541
|
const location = response.headers.get("location");
|
|
488
542
|
const isRedirect = status >= 300 && status < 400 && location;
|
|
@@ -564,23 +618,16 @@ async function executeSingleFetchRequest(config, fetchOptions, requestCount, tim
|
|
|
564
618
|
const bodySize = bodyBuffer?.byteLength || (typeof responseData === "string" ? responseData.length : 0);
|
|
565
619
|
config.transfer.bodySize = bodySize;
|
|
566
620
|
config.transfer.responseSize = bodySize;
|
|
567
|
-
|
|
568
|
-
|
|
569
|
-
|
|
570
|
-
|
|
571
|
-
headers: responseHeaders,
|
|
572
|
-
data: responseData
|
|
573
|
-
}, config, fetchOptions);
|
|
574
|
-
_stats.statusOnNext = "error";
|
|
575
|
-
return error;
|
|
576
|
-
}
|
|
577
|
-
_stats.statusOnNext = "success";
|
|
621
|
+
config.status = status;
|
|
622
|
+
config.statusText = statusText;
|
|
623
|
+
_stats.statusOnNext = status >= 400 ? "error" : "success";
|
|
624
|
+
const mergedCookies = mergeRequestAndResponseCookies(config, cookies, url.href);
|
|
578
625
|
const finalResponse = {
|
|
579
626
|
data: responseData,
|
|
580
627
|
status,
|
|
581
628
|
statusText,
|
|
582
629
|
headers: responseHeaders,
|
|
583
|
-
cookies,
|
|
630
|
+
cookies: mergedCookies,
|
|
584
631
|
config,
|
|
585
632
|
contentType,
|
|
586
633
|
contentLength: bodySize,
|
|
@@ -599,7 +646,7 @@ async function executeSingleFetchRequest(config, fetchOptions, requestCount, tim
|
|
|
599
646
|
contentType,
|
|
600
647
|
contentLength: buffer.length,
|
|
601
648
|
finalUrl: url.href,
|
|
602
|
-
cookies,
|
|
649
|
+
cookies: mergedCookies,
|
|
603
650
|
urls: buildUrlTree(config, url.href),
|
|
604
651
|
fileName: config.fileName,
|
|
605
652
|
fileSize: buffer.length,
|
|
@@ -646,7 +693,7 @@ async function executeSingleFetchRequest(config, fetchOptions, requestCount, tim
|
|
|
646
693
|
contentLength: bodySize
|
|
647
694
|
},
|
|
648
695
|
finalUrl: url.href,
|
|
649
|
-
cookies,
|
|
696
|
+
cookies: mergedCookies,
|
|
650
697
|
urls: buildUrlTree(config, url.href),
|
|
651
698
|
uploadSize: config.transfer.requestSize || 0,
|
|
652
699
|
timing: {
|