rezo 1.0.36 → 1.0.38
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/adapters/curl.cjs +320 -9
- package/dist/adapters/curl.js +320 -9
- package/dist/adapters/entries/curl.d.ts +20 -2
- package/dist/adapters/entries/fetch.d.ts +20 -2
- package/dist/adapters/entries/http.d.ts +20 -2
- package/dist/adapters/entries/http2.d.ts +20 -2
- package/dist/adapters/entries/react-native.d.ts +20 -2
- package/dist/adapters/entries/xhr.d.ts +20 -2
- package/dist/adapters/fetch.cjs +10 -2
- package/dist/adapters/fetch.js +10 -2
- package/dist/adapters/http.cjs +206 -35
- package/dist/adapters/http.js +206 -35
- package/dist/adapters/http2.cjs +10 -2
- package/dist/adapters/http2.js +10 -2
- package/dist/adapters/index.cjs +6 -6
- package/dist/adapters/react-native.cjs +10 -2
- package/dist/adapters/react-native.js +10 -2
- package/dist/adapters/xhr.cjs +9 -1
- package/dist/adapters/xhr.js +9 -1
- package/dist/cache/index.cjs +13 -13
- package/dist/crawler.d.ts +20 -2
- package/dist/entries/crawler.cjs +5 -5
- package/dist/index.cjs +24 -24
- package/dist/index.d.ts +20 -2
- package/dist/platform/browser.d.ts +20 -2
- package/dist/platform/bun.d.ts +20 -2
- package/dist/platform/deno.d.ts +20 -2
- package/dist/platform/node.d.ts +20 -2
- package/dist/platform/react-native.d.ts +20 -2
- package/dist/platform/worker.d.ts +20 -2
- package/dist/plugin/index.cjs +36 -36
- package/dist/proxy/index.cjs +4 -4
- package/dist/queue/index.cjs +8 -8
- package/dist/responses/universal/index.cjs +11 -11
- package/dist/utils/agent-pool.cjs +204 -0
- package/dist/utils/agent-pool.js +201 -0
- package/dist/utils/http-config.cjs +24 -7
- package/dist/utils/http-config.js +24 -7
- package/dist/utils/index.cjs +2 -0
- package/dist/utils/index.js +2 -0
- package/dist/utils/staged-timeout.cjs +143 -0
- package/dist/utils/staged-timeout.js +139 -0
- package/package.json +1 -1
package/dist/adapters/http.js
CHANGED
|
@@ -19,6 +19,8 @@ import { buildDownloadError, buildDecompressionError, buildSmartError, builError
|
|
|
19
19
|
import { isSameDomain, RezoPerformance } from '../utils/tools.js';
|
|
20
20
|
import { getGlobalDNSCache } from '../cache/dns-cache.js';
|
|
21
21
|
import { ResponseCache } from '../cache/response-cache.js';
|
|
22
|
+
import { getGlobalAgentPool } from '../utils/agent-pool.js';
|
|
23
|
+
import { StagedTimeoutManager, parseStagedTimeouts } from '../utils/staged-timeout.js';
|
|
22
24
|
import dns from "dns";
|
|
23
25
|
const debugLog = {
|
|
24
26
|
requestStart: (config, url, method) => {
|
|
@@ -185,7 +187,15 @@ function buildUrlTree(config, finalUrl) {
|
|
|
185
187
|
const urlStr = typeof config.url === "string" ? config.url : config.url.toString();
|
|
186
188
|
urls.push(urlStr);
|
|
187
189
|
}
|
|
188
|
-
if (
|
|
190
|
+
if (config.redirectHistory && config.redirectHistory.length > 0) {
|
|
191
|
+
for (const redirect of config.redirectHistory) {
|
|
192
|
+
const redirectUrl = typeof redirect.url === "string" ? redirect.url : redirect.url?.toString?.() || "";
|
|
193
|
+
if (redirectUrl && urls[urls.length - 1] !== redirectUrl) {
|
|
194
|
+
urls.push(redirectUrl);
|
|
195
|
+
}
|
|
196
|
+
}
|
|
197
|
+
}
|
|
198
|
+
if (finalUrl && (urls.length === 0 || urls[urls.length - 1] !== finalUrl)) {
|
|
189
199
|
urls.push(finalUrl);
|
|
190
200
|
}
|
|
191
201
|
return urls.length > 0 ? urls : [finalUrl];
|
|
@@ -466,7 +476,8 @@ async function executeHttp1Request(fetchOptions, config, options, perform, fs, s
|
|
|
466
476
|
status: response.status,
|
|
467
477
|
headers: response.headers,
|
|
468
478
|
sameDomain: isSameDomain(fetchOptions.fullUrl, _stats.redirectUrl),
|
|
469
|
-
method: fetchOptions.method.toUpperCase()
|
|
479
|
+
method: fetchOptions.method.toUpperCase(),
|
|
480
|
+
body: config.originalBody
|
|
470
481
|
}) : undefined;
|
|
471
482
|
if (typeof onRedirect !== "undefined") {
|
|
472
483
|
if (typeof onRedirect === "boolean") {
|
|
@@ -478,7 +489,7 @@ async function executeHttp1Request(fetchOptions, config, options, perform, fs, s
|
|
|
478
489
|
config.errors.push({ attempt: config.retryAttempts + 1, error: redirectError, duration: perform.now() });
|
|
479
490
|
throw redirectError;
|
|
480
491
|
}
|
|
481
|
-
} else if (!onRedirect.redirect) {
|
|
492
|
+
} else if (!onRedirect.redirect && !onRedirect.withoutBody && !("body" in onRedirect)) {
|
|
482
493
|
const redirectError = builErrorFromResponse("Redirect denied by user", response, config, fetchOptions);
|
|
483
494
|
_stats.statusOnNext = "error";
|
|
484
495
|
if (!config.errors)
|
|
@@ -517,24 +528,55 @@ async function executeHttp1Request(fetchOptions, config, options, perform, fs, s
|
|
|
517
528
|
delete options.params;
|
|
518
529
|
const fromUrl = fetchOptions.fullUrl;
|
|
519
530
|
fetchOptions.fullUrl = location;
|
|
520
|
-
|
|
521
|
-
|
|
531
|
+
const normalizedRedirect = typeof onRedirect === "object" ? onRedirect.redirect || onRedirect.withoutBody || "body" in onRedirect : undefined;
|
|
532
|
+
if (typeof onRedirect === "object" && normalizedRedirect) {
|
|
533
|
+
const method = onRedirect.redirect ? onRedirect.method || fetchOptions.method : fetchOptions.method;
|
|
522
534
|
config.method = method;
|
|
523
|
-
|
|
524
|
-
|
|
535
|
+
if (onRedirect.redirect && onRedirect.url) {
|
|
536
|
+
options.fullUrl = onRedirect.url;
|
|
537
|
+
fetchOptions.fullUrl = onRedirect.url;
|
|
538
|
+
}
|
|
525
539
|
if (onRedirect.withoutBody) {
|
|
526
540
|
delete options.body;
|
|
527
|
-
|
|
541
|
+
delete fetchOptions.body;
|
|
542
|
+
config.originalBody = undefined;
|
|
543
|
+
if (fetchOptions.headers instanceof RezoHeaders) {
|
|
544
|
+
fetchOptions.headers.delete("Content-Type");
|
|
545
|
+
fetchOptions.headers.delete("Content-Length");
|
|
546
|
+
}
|
|
547
|
+
} else if ("body" in onRedirect) {
|
|
528
548
|
options.body = onRedirect.body;
|
|
549
|
+
fetchOptions.body = onRedirect.body;
|
|
550
|
+
config.originalBody = onRedirect.body;
|
|
551
|
+
} else if (redirectCode === 307 || redirectCode === 308) {
|
|
552
|
+
const methodUpper = method.toUpperCase();
|
|
553
|
+
if ((methodUpper === "POST" || methodUpper === "PUT" || methodUpper === "PATCH") && config.originalBody !== undefined) {
|
|
554
|
+
options.body = config.originalBody;
|
|
555
|
+
fetchOptions.body = config.originalBody;
|
|
556
|
+
}
|
|
557
|
+
} else {
|
|
558
|
+
delete options.body;
|
|
559
|
+
delete fetchOptions.body;
|
|
560
|
+
if (fetchOptions.headers instanceof RezoHeaders) {
|
|
561
|
+
fetchOptions.headers.delete("Content-Type");
|
|
562
|
+
fetchOptions.headers.delete("Content-Length");
|
|
563
|
+
}
|
|
529
564
|
}
|
|
530
565
|
debugLog.redirect(config, fromUrl, fetchOptions.fullUrl, redirectCode, method);
|
|
531
|
-
if (onRedirect.setHeaders) {
|
|
566
|
+
if (onRedirect.redirect && onRedirect.setHeaders) {
|
|
532
567
|
addedOptions.customHeaders = onRedirect.setHeaders;
|
|
533
568
|
}
|
|
534
569
|
} else if (response.status === 301 || response.status === 302 || response.status === 303) {
|
|
535
570
|
debugLog.redirect(config, fromUrl, fetchOptions.fullUrl, redirectCode, "GET");
|
|
536
571
|
options.method = "GET";
|
|
572
|
+
fetchOptions.method = "GET";
|
|
573
|
+
config.method = "GET";
|
|
537
574
|
delete options.body;
|
|
575
|
+
delete fetchOptions.body;
|
|
576
|
+
if (fetchOptions.headers instanceof RezoHeaders) {
|
|
577
|
+
fetchOptions.headers.delete("Content-Type");
|
|
578
|
+
fetchOptions.headers.delete("Content-Length");
|
|
579
|
+
}
|
|
538
580
|
} else {
|
|
539
581
|
debugLog.redirect(config, fromUrl, fetchOptions.fullUrl, redirectCode, fetchOptions.method);
|
|
540
582
|
}
|
|
@@ -603,8 +645,17 @@ async function request(config, fetchOptions, requestCount, timing, _stats, respo
|
|
|
603
645
|
eventEmitter.emit("start", startEvent);
|
|
604
646
|
}
|
|
605
647
|
const requestOptions = buildHTTPOptions(fetchOptions, isSecure, url);
|
|
648
|
+
const stagedTimeoutConfig = parseStagedTimeouts(fetchOptions.timeout);
|
|
649
|
+
const timeoutManager = new StagedTimeoutManager(stagedTimeoutConfig, config, fetchOptions);
|
|
650
|
+
if (timeoutManager.hasPhase("total")) {
|
|
651
|
+
timeoutManager.startPhase("total");
|
|
652
|
+
}
|
|
606
653
|
try {
|
|
607
654
|
const req = httpModule.request(requestOptions, async (res) => {
|
|
655
|
+
timeoutManager.clearPhase("headers");
|
|
656
|
+
if (timeoutManager.hasPhase("body")) {
|
|
657
|
+
timeoutManager.startPhase("body");
|
|
658
|
+
}
|
|
608
659
|
if (!timing.firstByteTime) {
|
|
609
660
|
timing.firstByteTime = performance.now();
|
|
610
661
|
config.timing.responseStart = timing.firstByteTime;
|
|
@@ -896,9 +947,11 @@ async function request(config, fetchOptions, requestCount, timing, _stats, respo
|
|
|
896
947
|
uploadResult.emit("done", uploadFinishEvent);
|
|
897
948
|
uploadResult._markFinished();
|
|
898
949
|
}
|
|
950
|
+
timeoutManager.clearAll();
|
|
899
951
|
resolve(finalResponse);
|
|
900
952
|
});
|
|
901
953
|
decompressedStream.on("error", (err) => {
|
|
954
|
+
timeoutManager.clearAll();
|
|
902
955
|
_stats.statusOnNext = "error";
|
|
903
956
|
updateTiming(config, timing, contentLength || "", contentLengthCounter, res.rawHeaders);
|
|
904
957
|
if (_stats.redirectUrl) {
|
|
@@ -924,12 +977,36 @@ async function request(config, fetchOptions, requestCount, timing, _stats, respo
|
|
|
924
977
|
}
|
|
925
978
|
});
|
|
926
979
|
req.on("error", (err) => {
|
|
980
|
+
timeoutManager.clearAll();
|
|
927
981
|
_stats.statusOnNext = "error";
|
|
928
982
|
const error = buildSmartError(config, fetchOptions, err);
|
|
929
983
|
resolve(error);
|
|
930
984
|
});
|
|
931
985
|
req.on("socket", (socket) => {
|
|
986
|
+
timeoutManager.setSocket(socket);
|
|
987
|
+
timeoutManager.setRequest(req);
|
|
988
|
+
timeoutManager.setTimeoutCallback((phase, elapsed) => {
|
|
989
|
+
_stats.statusOnNext = "error";
|
|
990
|
+
const error = timeoutManager.createTimeoutError(phase, elapsed);
|
|
991
|
+
const eventEmitter = streamResult || downloadResult || uploadResult;
|
|
992
|
+
if (eventEmitter) {
|
|
993
|
+
eventEmitter.emit("error", error);
|
|
994
|
+
}
|
|
995
|
+
resolve(error);
|
|
996
|
+
});
|
|
997
|
+
const isAlreadyConnected = !socket.connecting && (socket.readyState === "open" || socket.writable === true);
|
|
998
|
+
if (isAlreadyConnected) {
|
|
999
|
+
timeoutManager.clearPhase("connect");
|
|
1000
|
+
if (timeoutManager.hasPhase("headers")) {
|
|
1001
|
+
timeoutManager.startPhase("headers");
|
|
1002
|
+
}
|
|
1003
|
+
} else {
|
|
1004
|
+
if (timeoutManager.hasPhase("connect")) {
|
|
1005
|
+
timeoutManager.startPhase("connect");
|
|
1006
|
+
}
|
|
1007
|
+
}
|
|
932
1008
|
socket.on("error", (err) => {
|
|
1009
|
+
timeoutManager.clearAll();
|
|
933
1010
|
_stats.statusOnNext = "error";
|
|
934
1011
|
const error = buildSmartError(config, fetchOptions, err);
|
|
935
1012
|
resolve(error);
|
|
@@ -963,6 +1040,10 @@ async function request(config, fetchOptions, requestCount, timing, _stats, respo
|
|
|
963
1040
|
}
|
|
964
1041
|
});
|
|
965
1042
|
socket.on("secureConnect", () => {
|
|
1043
|
+
timeoutManager.clearPhase("connect");
|
|
1044
|
+
if (timeoutManager.hasPhase("headers")) {
|
|
1045
|
+
timeoutManager.startPhase("headers");
|
|
1046
|
+
}
|
|
966
1047
|
if (!timing.tlsEnd && timing.tlsStart) {
|
|
967
1048
|
timing.tlsEnd = performance.now();
|
|
968
1049
|
config.timing.connectEnd = timing.tlsEnd;
|
|
@@ -1015,6 +1096,12 @@ async function request(config, fetchOptions, requestCount, timing, _stats, respo
|
|
|
1015
1096
|
}
|
|
1016
1097
|
});
|
|
1017
1098
|
socket.on("connect", () => {
|
|
1099
|
+
if (!isSecure) {
|
|
1100
|
+
timeoutManager.clearPhase("connect");
|
|
1101
|
+
if (timeoutManager.hasPhase("headers")) {
|
|
1102
|
+
timeoutManager.startPhase("headers");
|
|
1103
|
+
}
|
|
1104
|
+
}
|
|
1018
1105
|
if (!timing.tcpEnd) {
|
|
1019
1106
|
timing.tcpEnd = performance.now();
|
|
1020
1107
|
config.timing.connectEnd = timing.tcpEnd;
|
|
@@ -1052,6 +1139,7 @@ async function request(config, fetchOptions, requestCount, timing, _stats, respo
|
|
|
1052
1139
|
});
|
|
1053
1140
|
});
|
|
1054
1141
|
req.on("error", (error) => {
|
|
1142
|
+
timeoutManager.clearAll();
|
|
1055
1143
|
_stats.statusOnNext = "error";
|
|
1056
1144
|
updateTiming(config, timing, "", 0);
|
|
1057
1145
|
const e = buildSmartError(config, fetchOptions, error);
|
|
@@ -1198,20 +1286,42 @@ function buildHTTPOptions(fetchOptions, isSecure, url) {
|
|
|
1198
1286
|
useSecureContext = true,
|
|
1199
1287
|
auth,
|
|
1200
1288
|
dnsCache: dnsCacheOption,
|
|
1201
|
-
keepAlive =
|
|
1202
|
-
keepAliveMsecs = 60000
|
|
1289
|
+
keepAlive = true,
|
|
1290
|
+
keepAliveMsecs = 60000,
|
|
1291
|
+
useAgentPool = true
|
|
1203
1292
|
} = fetchOptions;
|
|
1204
|
-
|
|
1205
|
-
|
|
1206
|
-
|
|
1207
|
-
|
|
1208
|
-
|
|
1209
|
-
|
|
1210
|
-
|
|
1211
|
-
|
|
1212
|
-
|
|
1293
|
+
let agent;
|
|
1294
|
+
if (httpAgent || httpsAgent) {
|
|
1295
|
+
agent = isSecure ? httpsAgent : httpAgent;
|
|
1296
|
+
} else if (proxy) {
|
|
1297
|
+
agent = parseProxy(proxy, isSecure, rejectUnauthorized);
|
|
1298
|
+
} else if (useAgentPool) {
|
|
1299
|
+
const agentPool = getGlobalAgentPool({
|
|
1300
|
+
keepAlive: true,
|
|
1301
|
+
keepAliveMsecs,
|
|
1302
|
+
maxSockets: 256,
|
|
1303
|
+
maxFreeSockets: 64,
|
|
1304
|
+
dnsCache: dnsCacheOption !== false
|
|
1305
|
+
});
|
|
1306
|
+
if (isSecure) {
|
|
1307
|
+
agent = agentPool.getHttpsAgent({
|
|
1308
|
+
rejectUnauthorized,
|
|
1309
|
+
servername: url.hostname
|
|
1310
|
+
});
|
|
1311
|
+
} else {
|
|
1312
|
+
agent = agentPool.getHttpAgent();
|
|
1313
|
+
}
|
|
1314
|
+
} else if (isSecure && useSecureContext) {
|
|
1315
|
+
agent = new https.Agent({
|
|
1316
|
+
secureContext: createSecureContext(),
|
|
1317
|
+
servername: url.hostname,
|
|
1318
|
+
rejectUnauthorized,
|
|
1319
|
+
keepAlive,
|
|
1320
|
+
keepAliveMsecs: keepAlive ? keepAliveMsecs : undefined
|
|
1321
|
+
});
|
|
1322
|
+
}
|
|
1213
1323
|
let lookup;
|
|
1214
|
-
if (dnsCacheOption) {
|
|
1324
|
+
if (dnsCacheOption !== false && !useAgentPool) {
|
|
1215
1325
|
if (!dnsCache) {
|
|
1216
1326
|
const cacheOptions = typeof dnsCacheOption === "object" ? {
|
|
1217
1327
|
enable: true,
|
|
@@ -1275,7 +1385,7 @@ async function setInitialConfig(config, fetchOptions, isSecure, url, httpModule,
|
|
|
1275
1385
|
http1: true,
|
|
1276
1386
|
http2: config.http2,
|
|
1277
1387
|
compression: true,
|
|
1278
|
-
cookies: config.
|
|
1388
|
+
cookies: !config.disableCookieJar,
|
|
1279
1389
|
redirects: config.maxRedirects > 0,
|
|
1280
1390
|
proxy: !!proxy,
|
|
1281
1391
|
timeout: !!timeout,
|
|
@@ -1285,7 +1395,7 @@ async function setInitialConfig(config, fetchOptions, isSecure, url, httpModule,
|
|
|
1285
1395
|
config.features = {
|
|
1286
1396
|
http2: !!config.http2,
|
|
1287
1397
|
compression: !!config.compression?.enabled,
|
|
1288
|
-
cookies:
|
|
1398
|
+
cookies: !config.disableCookieJar,
|
|
1289
1399
|
redirects: config.maxRedirects > 0,
|
|
1290
1400
|
proxy: !!proxy,
|
|
1291
1401
|
timeout: !!timeout,
|
|
@@ -1383,10 +1493,34 @@ function emitRedirect(emitter, headers, status, statusText, sourceUri, destinati
|
|
|
1383
1493
|
}
|
|
1384
1494
|
function createSecureContext() {
|
|
1385
1495
|
return tls.createSecureContext({
|
|
1386
|
-
ecdhCurve: "X25519:prime256v1:secp384r1
|
|
1387
|
-
|
|
1388
|
-
|
|
1389
|
-
|
|
1496
|
+
ecdhCurve: "X25519:prime256v1:secp384r1",
|
|
1497
|
+
ciphers: [
|
|
1498
|
+
"TLS_AES_128_GCM_SHA256",
|
|
1499
|
+
"TLS_AES_256_GCM_SHA384",
|
|
1500
|
+
"TLS_CHACHA20_POLY1305_SHA256",
|
|
1501
|
+
"ECDHE-ECDSA-AES128-GCM-SHA256",
|
|
1502
|
+
"ECDHE-RSA-AES128-GCM-SHA256",
|
|
1503
|
+
"ECDHE-ECDSA-AES256-GCM-SHA384",
|
|
1504
|
+
"ECDHE-RSA-AES256-GCM-SHA384",
|
|
1505
|
+
"ECDHE-ECDSA-CHACHA20-POLY1305",
|
|
1506
|
+
"ECDHE-RSA-CHACHA20-POLY1305",
|
|
1507
|
+
"ECDHE-RSA-AES128-SHA",
|
|
1508
|
+
"ECDHE-RSA-AES256-SHA",
|
|
1509
|
+
"AES128-GCM-SHA256",
|
|
1510
|
+
"AES256-GCM-SHA384",
|
|
1511
|
+
"AES128-SHA",
|
|
1512
|
+
"AES256-SHA"
|
|
1513
|
+
].join(":"),
|
|
1514
|
+
sigalgs: [
|
|
1515
|
+
"ecdsa_secp256r1_sha256",
|
|
1516
|
+
"ecdsa_secp384r1_sha384",
|
|
1517
|
+
"rsa_pss_rsae_sha256",
|
|
1518
|
+
"rsa_pss_rsae_sha384",
|
|
1519
|
+
"rsa_pss_rsae_sha512",
|
|
1520
|
+
"rsa_pkcs1_sha256",
|
|
1521
|
+
"rsa_pkcs1_sha384",
|
|
1522
|
+
"rsa_pkcs1_sha512"
|
|
1523
|
+
].join(":"),
|
|
1390
1524
|
minVersion: "TLSv1.2",
|
|
1391
1525
|
maxVersion: "TLSv1.3",
|
|
1392
1526
|
sessionTimeout: 3600
|
|
@@ -1421,25 +1555,62 @@ function generateSessionId() {
|
|
|
1421
1555
|
function generateTraceId() {
|
|
1422
1556
|
return `trc_${Date.now()}_${Math.random().toString(36).substring(2, 15)}`;
|
|
1423
1557
|
}
|
|
1558
|
+
const proxyAgentCache = new Map;
|
|
1559
|
+
const PROXY_AGENT_EVICTION_MS = 60000;
|
|
1560
|
+
function buildProxyAgentKey(proxy, isSecure, rejectUnauthorized) {
|
|
1561
|
+
if (typeof proxy === "string") {
|
|
1562
|
+
return `str:${proxy}:${isSecure}:${rejectUnauthorized}`;
|
|
1563
|
+
}
|
|
1564
|
+
const p = proxy;
|
|
1565
|
+
const authKey = p.auth ? `${p.auth.username}:${p.auth.password}` : "";
|
|
1566
|
+
return `obj:${p.protocol}://${p.host}:${p.port}:${authKey}:${isSecure}:${rejectUnauthorized}`;
|
|
1567
|
+
}
|
|
1568
|
+
function evictStaleProxyAgents() {
|
|
1569
|
+
const now = Date.now();
|
|
1570
|
+
for (const [key, entry] of proxyAgentCache) {
|
|
1571
|
+
if (now - entry.lastUsed > PROXY_AGENT_EVICTION_MS) {
|
|
1572
|
+
try {
|
|
1573
|
+
entry.agent.destroy();
|
|
1574
|
+
} catch {}
|
|
1575
|
+
proxyAgentCache.delete(key);
|
|
1576
|
+
}
|
|
1577
|
+
}
|
|
1578
|
+
}
|
|
1579
|
+
let lastProxyEviction = 0;
|
|
1424
1580
|
function parseProxy(proxy, isScure = true, rejectUnauthorized = false) {
|
|
1425
1581
|
if (!proxy) {
|
|
1426
1582
|
return;
|
|
1427
1583
|
}
|
|
1584
|
+
const now = Date.now();
|
|
1585
|
+
if (now - lastProxyEviction > PROXY_AGENT_EVICTION_MS / 2) {
|
|
1586
|
+
evictStaleProxyAgents();
|
|
1587
|
+
lastProxyEviction = now;
|
|
1588
|
+
}
|
|
1589
|
+
const cacheKey = buildProxyAgentKey(proxy, isScure, rejectUnauthorized);
|
|
1590
|
+
const cached = proxyAgentCache.get(cacheKey);
|
|
1591
|
+
if (cached) {
|
|
1592
|
+
cached.lastUsed = now;
|
|
1593
|
+
return cached.agent;
|
|
1594
|
+
}
|
|
1595
|
+
let agent;
|
|
1428
1596
|
if (typeof proxy === "string") {
|
|
1429
1597
|
if (proxy.startsWith("http://")) {
|
|
1430
|
-
|
|
1598
|
+
agent = rezoProxy(`http://${proxy.slice(7)}`, "http");
|
|
1431
1599
|
} else if (proxy.startsWith("https://")) {
|
|
1432
|
-
|
|
1600
|
+
agent = rezoProxy(`https://${proxy.slice(8)}`, "https");
|
|
1601
|
+
} else {
|
|
1602
|
+
agent = rezoProxy(proxy);
|
|
1433
1603
|
}
|
|
1434
|
-
|
|
1435
|
-
|
|
1436
|
-
if (proxy.protocol === "http" || proxy.protocol === "https") {
|
|
1437
|
-
return rezoProxy({
|
|
1604
|
+
} else if (proxy.protocol === "http" || proxy.protocol === "https") {
|
|
1605
|
+
agent = rezoProxy({
|
|
1438
1606
|
...proxy,
|
|
1439
1607
|
client: !isScure ? "http" : "https"
|
|
1440
1608
|
});
|
|
1609
|
+
} else {
|
|
1610
|
+
agent = rezoProxy(proxy);
|
|
1441
1611
|
}
|
|
1442
|
-
|
|
1612
|
+
proxyAgentCache.set(cacheKey, { agent, lastUsed: now });
|
|
1613
|
+
return agent;
|
|
1443
1614
|
}
|
|
1444
1615
|
async function updateCookies(config, headers, url) {
|
|
1445
1616
|
const cookies = headers["set-cookie"];
|
|
@@ -1480,7 +1651,7 @@ async function updateCookies(config, headers, url) {
|
|
|
1480
1651
|
acceptedCookies.push(...parsedCookies.array);
|
|
1481
1652
|
}
|
|
1482
1653
|
const acceptedCookieStrings = acceptedCookies.map((c) => c.toSetCookieString());
|
|
1483
|
-
if (config.
|
|
1654
|
+
if (!config.disableCookieJar && config.cookieJar) {
|
|
1484
1655
|
config.cookieJar.setCookiesSync(acceptedCookieStrings, url);
|
|
1485
1656
|
}
|
|
1486
1657
|
jar.setCookiesSync(acceptedCookieStrings, url);
|
package/dist/adapters/http2.cjs
CHANGED
|
@@ -291,7 +291,15 @@ function buildUrlTree(config, finalUrl) {
|
|
|
291
291
|
const urlStr = typeof config.url === "string" ? config.url : config.url.toString();
|
|
292
292
|
urls.push(urlStr);
|
|
293
293
|
}
|
|
294
|
-
if (
|
|
294
|
+
if (config.redirectHistory && config.redirectHistory.length > 0) {
|
|
295
|
+
for (const redirect of config.redirectHistory) {
|
|
296
|
+
const redirectUrl = typeof redirect.url === "string" ? redirect.url : redirect.url?.toString?.() || "";
|
|
297
|
+
if (redirectUrl && urls[urls.length - 1] !== redirectUrl) {
|
|
298
|
+
urls.push(redirectUrl);
|
|
299
|
+
}
|
|
300
|
+
}
|
|
301
|
+
}
|
|
302
|
+
if (finalUrl && (urls.length === 0 || urls[urls.length - 1] !== finalUrl)) {
|
|
295
303
|
urls.push(finalUrl);
|
|
296
304
|
}
|
|
297
305
|
return urls.length > 0 ? urls : [finalUrl];
|
|
@@ -344,7 +352,7 @@ async function updateCookies(config, headers, url) {
|
|
|
344
352
|
const acceptedCookieStrings = acceptedCookies.map((c) => c.toSetCookieString());
|
|
345
353
|
const jar = new RezoCookieJar;
|
|
346
354
|
jar.setCookiesSync(acceptedCookieStrings, url);
|
|
347
|
-
if (config.
|
|
355
|
+
if (!config.disableCookieJar && config.cookieJar) {
|
|
348
356
|
config.cookieJar.setCookiesSync(acceptedCookieStrings, url);
|
|
349
357
|
}
|
|
350
358
|
const cookies = jar.cookies();
|
package/dist/adapters/http2.js
CHANGED
|
@@ -291,7 +291,15 @@ function buildUrlTree(config, finalUrl) {
|
|
|
291
291
|
const urlStr = typeof config.url === "string" ? config.url : config.url.toString();
|
|
292
292
|
urls.push(urlStr);
|
|
293
293
|
}
|
|
294
|
-
if (
|
|
294
|
+
if (config.redirectHistory && config.redirectHistory.length > 0) {
|
|
295
|
+
for (const redirect of config.redirectHistory) {
|
|
296
|
+
const redirectUrl = typeof redirect.url === "string" ? redirect.url : redirect.url?.toString?.() || "";
|
|
297
|
+
if (redirectUrl && urls[urls.length - 1] !== redirectUrl) {
|
|
298
|
+
urls.push(redirectUrl);
|
|
299
|
+
}
|
|
300
|
+
}
|
|
301
|
+
}
|
|
302
|
+
if (finalUrl && (urls.length === 0 || urls[urls.length - 1] !== finalUrl)) {
|
|
295
303
|
urls.push(finalUrl);
|
|
296
304
|
}
|
|
297
305
|
return urls.length > 0 ? urls : [finalUrl];
|
|
@@ -344,7 +352,7 @@ async function updateCookies(config, headers, url) {
|
|
|
344
352
|
const acceptedCookieStrings = acceptedCookies.map((c) => c.toSetCookieString());
|
|
345
353
|
const jar = new RezoCookieJar;
|
|
346
354
|
jar.setCookiesSync(acceptedCookieStrings, url);
|
|
347
|
-
if (config.
|
|
355
|
+
if (!config.disableCookieJar && config.cookieJar) {
|
|
348
356
|
config.cookieJar.setCookiesSync(acceptedCookieStrings, url);
|
|
349
357
|
}
|
|
350
358
|
const cookies = jar.cookies();
|
package/dist/adapters/index.cjs
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
|
-
const
|
|
2
|
-
exports.detectRuntime =
|
|
3
|
-
exports.getAdapterCapabilities =
|
|
4
|
-
exports.buildAdapterContext =
|
|
5
|
-
exports.getAvailableAdapters =
|
|
6
|
-
exports.selectAdapter =
|
|
1
|
+
const _mod_qcqku6 = require('./picker.cjs');
|
|
2
|
+
exports.detectRuntime = _mod_qcqku6.detectRuntime;
|
|
3
|
+
exports.getAdapterCapabilities = _mod_qcqku6.getAdapterCapabilities;
|
|
4
|
+
exports.buildAdapterContext = _mod_qcqku6.buildAdapterContext;
|
|
5
|
+
exports.getAvailableAdapters = _mod_qcqku6.getAvailableAdapters;
|
|
6
|
+
exports.selectAdapter = _mod_qcqku6.selectAdapter;;
|
|
@@ -188,7 +188,15 @@ function buildUrlTree(config, finalUrl) {
|
|
|
188
188
|
const urlStr = typeof config.url === "string" ? config.url : String(config.url);
|
|
189
189
|
urls.push(urlStr);
|
|
190
190
|
}
|
|
191
|
-
if (
|
|
191
|
+
if (config.redirectHistory && config.redirectHistory.length > 0) {
|
|
192
|
+
for (const redirect of config.redirectHistory) {
|
|
193
|
+
const redirectUrl = typeof redirect.url === "string" ? redirect.url : redirect.url?.toString?.() || "";
|
|
194
|
+
if (redirectUrl && urls[urls.length - 1] !== redirectUrl) {
|
|
195
|
+
urls.push(redirectUrl);
|
|
196
|
+
}
|
|
197
|
+
}
|
|
198
|
+
}
|
|
199
|
+
if (finalUrl && (urls.length === 0 || urls[urls.length - 1] !== finalUrl)) {
|
|
192
200
|
urls.push(finalUrl);
|
|
193
201
|
}
|
|
194
202
|
return urls.length > 0 ? urls : [finalUrl];
|
|
@@ -520,7 +528,7 @@ async function executeSingleRequest(config, fetchOptions, timing, streamResult,
|
|
|
520
528
|
acceptedCookies.push(...parsedCookies.array);
|
|
521
529
|
}
|
|
522
530
|
const acceptedCookieStrings = acceptedCookies.map((c) => c.toSetCookieString());
|
|
523
|
-
if (config.
|
|
531
|
+
if (!config.disableCookieJar && config.cookieJar) {
|
|
524
532
|
config.cookieJar.setCookiesSync(acceptedCookieStrings, url);
|
|
525
533
|
}
|
|
526
534
|
const cookieJar = new RezoCookieJar(acceptedCookies, url);
|
|
@@ -188,7 +188,15 @@ function buildUrlTree(config, finalUrl) {
|
|
|
188
188
|
const urlStr = typeof config.url === "string" ? config.url : String(config.url);
|
|
189
189
|
urls.push(urlStr);
|
|
190
190
|
}
|
|
191
|
-
if (
|
|
191
|
+
if (config.redirectHistory && config.redirectHistory.length > 0) {
|
|
192
|
+
for (const redirect of config.redirectHistory) {
|
|
193
|
+
const redirectUrl = typeof redirect.url === "string" ? redirect.url : redirect.url?.toString?.() || "";
|
|
194
|
+
if (redirectUrl && urls[urls.length - 1] !== redirectUrl) {
|
|
195
|
+
urls.push(redirectUrl);
|
|
196
|
+
}
|
|
197
|
+
}
|
|
198
|
+
}
|
|
199
|
+
if (finalUrl && (urls.length === 0 || urls[urls.length - 1] !== finalUrl)) {
|
|
192
200
|
urls.push(finalUrl);
|
|
193
201
|
}
|
|
194
202
|
return urls.length > 0 ? urls : [finalUrl];
|
|
@@ -520,7 +528,7 @@ async function executeSingleRequest(config, fetchOptions, timing, streamResult,
|
|
|
520
528
|
acceptedCookies.push(...parsedCookies.array);
|
|
521
529
|
}
|
|
522
530
|
const acceptedCookieStrings = acceptedCookies.map((c) => c.toSetCookieString());
|
|
523
|
-
if (config.
|
|
531
|
+
if (!config.disableCookieJar && config.cookieJar) {
|
|
524
532
|
config.cookieJar.setCookiesSync(acceptedCookieStrings, url);
|
|
525
533
|
}
|
|
526
534
|
const cookieJar = new RezoCookieJar(acceptedCookies, url);
|
package/dist/adapters/xhr.cjs
CHANGED
|
@@ -107,7 +107,15 @@ function buildUrlTree(config, finalUrl) {
|
|
|
107
107
|
const urlStr = typeof config.url === "string" ? config.url : String(config.url);
|
|
108
108
|
urls.push(urlStr);
|
|
109
109
|
}
|
|
110
|
-
if (
|
|
110
|
+
if (config.redirectHistory && config.redirectHistory.length > 0) {
|
|
111
|
+
for (const redirect of config.redirectHistory) {
|
|
112
|
+
const redirectUrl = typeof redirect.url === "string" ? redirect.url : redirect.url?.toString?.() || "";
|
|
113
|
+
if (redirectUrl && urls[urls.length - 1] !== redirectUrl) {
|
|
114
|
+
urls.push(redirectUrl);
|
|
115
|
+
}
|
|
116
|
+
}
|
|
117
|
+
}
|
|
118
|
+
if (finalUrl && (urls.length === 0 || urls[urls.length - 1] !== finalUrl)) {
|
|
111
119
|
urls.push(finalUrl);
|
|
112
120
|
}
|
|
113
121
|
return urls.length > 0 ? urls : [finalUrl];
|
package/dist/adapters/xhr.js
CHANGED
|
@@ -107,7 +107,15 @@ function buildUrlTree(config, finalUrl) {
|
|
|
107
107
|
const urlStr = typeof config.url === "string" ? config.url : String(config.url);
|
|
108
108
|
urls.push(urlStr);
|
|
109
109
|
}
|
|
110
|
-
if (
|
|
110
|
+
if (config.redirectHistory && config.redirectHistory.length > 0) {
|
|
111
|
+
for (const redirect of config.redirectHistory) {
|
|
112
|
+
const redirectUrl = typeof redirect.url === "string" ? redirect.url : redirect.url?.toString?.() || "";
|
|
113
|
+
if (redirectUrl && urls[urls.length - 1] !== redirectUrl) {
|
|
114
|
+
urls.push(redirectUrl);
|
|
115
|
+
}
|
|
116
|
+
}
|
|
117
|
+
}
|
|
118
|
+
if (finalUrl && (urls.length === 0 || urls[urls.length - 1] !== finalUrl)) {
|
|
111
119
|
urls.push(finalUrl);
|
|
112
120
|
}
|
|
113
121
|
return urls.length > 0 ? urls : [finalUrl];
|
package/dist/cache/index.cjs
CHANGED
|
@@ -1,13 +1,13 @@
|
|
|
1
|
-
const
|
|
2
|
-
exports.LRUCache =
|
|
3
|
-
const
|
|
4
|
-
exports.DNSCache =
|
|
5
|
-
exports.getGlobalDNSCache =
|
|
6
|
-
exports.resetGlobalDNSCache =
|
|
7
|
-
const
|
|
8
|
-
exports.ResponseCache =
|
|
9
|
-
exports.normalizeResponseCacheConfig =
|
|
10
|
-
const
|
|
11
|
-
exports.FileCacher =
|
|
12
|
-
const
|
|
13
|
-
exports.UrlStore =
|
|
1
|
+
const _mod_rzjeap = require('./lru-cache.cjs');
|
|
2
|
+
exports.LRUCache = _mod_rzjeap.LRUCache;;
|
|
3
|
+
const _mod_u0cvom = require('./dns-cache.cjs');
|
|
4
|
+
exports.DNSCache = _mod_u0cvom.DNSCache;
|
|
5
|
+
exports.getGlobalDNSCache = _mod_u0cvom.getGlobalDNSCache;
|
|
6
|
+
exports.resetGlobalDNSCache = _mod_u0cvom.resetGlobalDNSCache;;
|
|
7
|
+
const _mod_32vqjj = require('./response-cache.cjs');
|
|
8
|
+
exports.ResponseCache = _mod_32vqjj.ResponseCache;
|
|
9
|
+
exports.normalizeResponseCacheConfig = _mod_32vqjj.normalizeResponseCacheConfig;;
|
|
10
|
+
const _mod_1wlgog = require('./file-cacher.cjs');
|
|
11
|
+
exports.FileCacher = _mod_1wlgog.FileCacher;;
|
|
12
|
+
const _mod_o2go7a = require('./url-store.cjs');
|
|
13
|
+
exports.UrlStore = _mod_o2go7a.UrlStore;;
|
package/dist/crawler.d.ts
CHANGED
|
@@ -1769,7 +1769,14 @@ export interface RezoConfig {
|
|
|
1769
1769
|
/** @description Supported compression algorithms */
|
|
1770
1770
|
algorithms?: string[];
|
|
1771
1771
|
};
|
|
1772
|
-
/**
|
|
1772
|
+
/**
|
|
1773
|
+
* @description Disable cookie jar for session management.
|
|
1774
|
+
* When false (default), cookies are automatically managed.
|
|
1775
|
+
* Set to true to disable automatic cookie handling.
|
|
1776
|
+
* @default false
|
|
1777
|
+
*/
|
|
1778
|
+
disableCookieJar?: boolean;
|
|
1779
|
+
/** @deprecated Use `disableCookieJar` instead */
|
|
1773
1780
|
enableCookieJar?: boolean;
|
|
1774
1781
|
/** @description Send cookies with cross-origin requests (matches Axios withCredentials). Default: false */
|
|
1775
1782
|
withCredentials?: boolean;
|
|
@@ -1880,6 +1887,8 @@ export interface RezoConfig {
|
|
|
1880
1887
|
hooks: Partial<RezoHooks> | null;
|
|
1881
1888
|
/** @description Snapshot of the original request configuration */
|
|
1882
1889
|
originalRequest: RezoRequestConfig;
|
|
1890
|
+
/** @description Original request body, preserved for POST body retention during redirects */
|
|
1891
|
+
originalBody?: RezoRequestConfig["body"];
|
|
1883
1892
|
/** @description Final resolved URL after redirects and processing */
|
|
1884
1893
|
finalUrl: string;
|
|
1885
1894
|
/** @description HTTP adapter used for the request */
|
|
@@ -2814,6 +2823,8 @@ export interface OnRedirectOptions {
|
|
|
2814
2823
|
headers: RezoHeaders;
|
|
2815
2824
|
sameDomain: boolean;
|
|
2816
2825
|
method: string;
|
|
2826
|
+
/** The current request body (RezoFormData, string, object, etc.) - allows user to inspect/modify */
|
|
2827
|
+
body?: any;
|
|
2817
2828
|
}
|
|
2818
2829
|
export type OnRedirectResponse = boolean | ToRedirectOptions | undefined;
|
|
2819
2830
|
export type ToRedirectOptions = {
|
|
@@ -3164,7 +3175,14 @@ export interface RezoDefaultOptions {
|
|
|
3164
3175
|
baseURL?: string;
|
|
3165
3176
|
/** Hooks for request/response lifecycle */
|
|
3166
3177
|
hooks?: Partial<RezoHooks>;
|
|
3167
|
-
/**
|
|
3178
|
+
/**
|
|
3179
|
+
* Whether to disable automatic cookie handling.
|
|
3180
|
+
* When false (default), cookies are automatically managed via the jar.
|
|
3181
|
+
* Set to true to disable automatic cookie management.
|
|
3182
|
+
* @default false
|
|
3183
|
+
*/
|
|
3184
|
+
disableCookieJar?: boolean;
|
|
3185
|
+
/** @deprecated Use `disableCookieJar` instead. Will be removed in next major version. */
|
|
3168
3186
|
enableCookieJar?: boolean;
|
|
3169
3187
|
/**
|
|
3170
3188
|
* Custom cookie jar for managing cookies.
|
package/dist/entries/crawler.cjs
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
const
|
|
2
|
-
exports.Crawler =
|
|
3
|
-
const
|
|
4
|
-
exports.CrawlerOptions =
|
|
5
|
-
exports.Domain =
|
|
1
|
+
const _mod_yx52v2 = require('../plugin/crawler.cjs');
|
|
2
|
+
exports.Crawler = _mod_yx52v2.Crawler;;
|
|
3
|
+
const _mod_kknzs3 = require('../plugin/crawler-options.cjs');
|
|
4
|
+
exports.CrawlerOptions = _mod_kknzs3.CrawlerOptions;
|
|
5
|
+
exports.Domain = _mod_kknzs3.Domain;;
|