rezo 1.0.35 → 1.0.37
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/adapters/curl.cjs +320 -9
- package/dist/adapters/curl.js +320 -9
- package/dist/adapters/entries/curl.d.ts +20 -2
- package/dist/adapters/entries/fetch.d.ts +20 -2
- package/dist/adapters/entries/http.d.ts +20 -2
- package/dist/adapters/entries/http2.d.ts +20 -2
- package/dist/adapters/entries/react-native.d.ts +20 -2
- package/dist/adapters/entries/xhr.d.ts +20 -2
- package/dist/adapters/fetch.cjs +10 -2
- package/dist/adapters/fetch.js +10 -2
- package/dist/adapters/http.cjs +204 -35
- package/dist/adapters/http.js +204 -35
- package/dist/adapters/http2.cjs +10 -2
- package/dist/adapters/http2.js +10 -2
- package/dist/adapters/index.cjs +6 -6
- package/dist/adapters/react-native.cjs +10 -2
- package/dist/adapters/react-native.js +10 -2
- package/dist/adapters/xhr.cjs +9 -1
- package/dist/adapters/xhr.js +9 -1
- package/dist/cache/index.cjs +13 -13
- package/dist/crawler.d.ts +20 -2
- package/dist/entries/crawler.cjs +5 -5
- package/dist/index.cjs +24 -24
- package/dist/index.d.ts +20 -2
- package/dist/platform/browser.d.ts +20 -2
- package/dist/platform/bun.d.ts +20 -2
- package/dist/platform/deno.d.ts +20 -2
- package/dist/platform/node.d.ts +20 -2
- package/dist/platform/react-native.d.ts +20 -2
- package/dist/platform/worker.d.ts +20 -2
- package/dist/plugin/index.cjs +36 -36
- package/dist/proxy/index.cjs +4 -4
- package/dist/queue/index.cjs +8 -8
- package/dist/responses/universal/index.cjs +11 -11
- package/dist/utils/agent-pool.cjs +204 -0
- package/dist/utils/agent-pool.js +201 -0
- package/dist/utils/http-config.cjs +24 -7
- package/dist/utils/http-config.js +24 -7
- package/dist/utils/index.cjs +2 -0
- package/dist/utils/index.js +2 -0
- package/dist/utils/staged-timeout.cjs +143 -0
- package/dist/utils/staged-timeout.js +139 -0
- package/package.json +1 -1
package/dist/adapters/http.js
CHANGED
|
@@ -19,6 +19,8 @@ import { buildDownloadError, buildDecompressionError, buildSmartError, builError
|
|
|
19
19
|
import { isSameDomain, RezoPerformance } from '../utils/tools.js';
|
|
20
20
|
import { getGlobalDNSCache } from '../cache/dns-cache.js';
|
|
21
21
|
import { ResponseCache } from '../cache/response-cache.js';
|
|
22
|
+
import { getGlobalAgentPool } from '../utils/agent-pool.js';
|
|
23
|
+
import { StagedTimeoutManager, parseStagedTimeouts } from '../utils/staged-timeout.js';
|
|
22
24
|
import dns from "dns";
|
|
23
25
|
const debugLog = {
|
|
24
26
|
requestStart: (config, url, method) => {
|
|
@@ -185,7 +187,15 @@ function buildUrlTree(config, finalUrl) {
|
|
|
185
187
|
const urlStr = typeof config.url === "string" ? config.url : config.url.toString();
|
|
186
188
|
urls.push(urlStr);
|
|
187
189
|
}
|
|
188
|
-
if (
|
|
190
|
+
if (config.redirectHistory && config.redirectHistory.length > 0) {
|
|
191
|
+
for (const redirect of config.redirectHistory) {
|
|
192
|
+
const redirectUrl = typeof redirect.url === "string" ? redirect.url : redirect.url?.toString?.() || "";
|
|
193
|
+
if (redirectUrl && urls[urls.length - 1] !== redirectUrl) {
|
|
194
|
+
urls.push(redirectUrl);
|
|
195
|
+
}
|
|
196
|
+
}
|
|
197
|
+
}
|
|
198
|
+
if (finalUrl && (urls.length === 0 || urls[urls.length - 1] !== finalUrl)) {
|
|
189
199
|
urls.push(finalUrl);
|
|
190
200
|
}
|
|
191
201
|
return urls.length > 0 ? urls : [finalUrl];
|
|
@@ -466,7 +476,8 @@ async function executeHttp1Request(fetchOptions, config, options, perform, fs, s
|
|
|
466
476
|
status: response.status,
|
|
467
477
|
headers: response.headers,
|
|
468
478
|
sameDomain: isSameDomain(fetchOptions.fullUrl, _stats.redirectUrl),
|
|
469
|
-
method: fetchOptions.method.toUpperCase()
|
|
479
|
+
method: fetchOptions.method.toUpperCase(),
|
|
480
|
+
body: config.originalBody
|
|
470
481
|
}) : undefined;
|
|
471
482
|
if (typeof onRedirect !== "undefined") {
|
|
472
483
|
if (typeof onRedirect === "boolean") {
|
|
@@ -478,7 +489,7 @@ async function executeHttp1Request(fetchOptions, config, options, perform, fs, s
|
|
|
478
489
|
config.errors.push({ attempt: config.retryAttempts + 1, error: redirectError, duration: perform.now() });
|
|
479
490
|
throw redirectError;
|
|
480
491
|
}
|
|
481
|
-
} else if (!onRedirect.redirect) {
|
|
492
|
+
} else if (!onRedirect.redirect && !onRedirect.withoutBody && !("body" in onRedirect)) {
|
|
482
493
|
const redirectError = builErrorFromResponse("Redirect denied by user", response, config, fetchOptions);
|
|
483
494
|
_stats.statusOnNext = "error";
|
|
484
495
|
if (!config.errors)
|
|
@@ -517,24 +528,53 @@ async function executeHttp1Request(fetchOptions, config, options, perform, fs, s
|
|
|
517
528
|
delete options.params;
|
|
518
529
|
const fromUrl = fetchOptions.fullUrl;
|
|
519
530
|
fetchOptions.fullUrl = location;
|
|
520
|
-
|
|
521
|
-
|
|
531
|
+
const normalizedRedirect = typeof onRedirect === "object" ? onRedirect.redirect || onRedirect.withoutBody || "body" in onRedirect : undefined;
|
|
532
|
+
if (typeof onRedirect === "object" && normalizedRedirect) {
|
|
533
|
+
const method = onRedirect.redirect ? onRedirect.method || fetchOptions.method : fetchOptions.method;
|
|
522
534
|
config.method = method;
|
|
523
|
-
|
|
524
|
-
|
|
535
|
+
if (onRedirect.redirect && onRedirect.url) {
|
|
536
|
+
options.fullUrl = onRedirect.url;
|
|
537
|
+
fetchOptions.fullUrl = onRedirect.url;
|
|
538
|
+
}
|
|
525
539
|
if (onRedirect.withoutBody) {
|
|
526
540
|
delete options.body;
|
|
527
|
-
|
|
541
|
+
delete fetchOptions.body;
|
|
542
|
+
config.originalBody = undefined;
|
|
543
|
+
if (fetchOptions.headers instanceof RezoHeaders) {
|
|
544
|
+
fetchOptions.headers.delete("Content-Type");
|
|
545
|
+
fetchOptions.headers.delete("Content-Length");
|
|
546
|
+
}
|
|
547
|
+
} else if ("body" in onRedirect) {
|
|
528
548
|
options.body = onRedirect.body;
|
|
549
|
+
fetchOptions.body = onRedirect.body;
|
|
550
|
+
config.originalBody = onRedirect.body;
|
|
551
|
+
} else if (redirectCode === 307 || redirectCode === 308) {
|
|
552
|
+
const methodUpper = method.toUpperCase();
|
|
553
|
+
if ((methodUpper === "POST" || methodUpper === "PUT" || methodUpper === "PATCH") && config.originalBody !== undefined) {
|
|
554
|
+
options.body = config.originalBody;
|
|
555
|
+
fetchOptions.body = config.originalBody;
|
|
556
|
+
}
|
|
557
|
+
} else {
|
|
558
|
+
delete options.body;
|
|
559
|
+
delete fetchOptions.body;
|
|
560
|
+
if (fetchOptions.headers instanceof RezoHeaders) {
|
|
561
|
+
fetchOptions.headers.delete("Content-Type");
|
|
562
|
+
fetchOptions.headers.delete("Content-Length");
|
|
563
|
+
}
|
|
529
564
|
}
|
|
530
565
|
debugLog.redirect(config, fromUrl, fetchOptions.fullUrl, redirectCode, method);
|
|
531
|
-
if (onRedirect.setHeaders) {
|
|
566
|
+
if (onRedirect.redirect && onRedirect.setHeaders) {
|
|
532
567
|
addedOptions.customHeaders = onRedirect.setHeaders;
|
|
533
568
|
}
|
|
534
569
|
} else if (response.status === 301 || response.status === 302 || response.status === 303) {
|
|
535
570
|
debugLog.redirect(config, fromUrl, fetchOptions.fullUrl, redirectCode, "GET");
|
|
536
571
|
options.method = "GET";
|
|
537
572
|
delete options.body;
|
|
573
|
+
delete fetchOptions.body;
|
|
574
|
+
if (fetchOptions.headers instanceof RezoHeaders) {
|
|
575
|
+
fetchOptions.headers.delete("Content-Type");
|
|
576
|
+
fetchOptions.headers.delete("Content-Length");
|
|
577
|
+
}
|
|
538
578
|
} else {
|
|
539
579
|
debugLog.redirect(config, fromUrl, fetchOptions.fullUrl, redirectCode, fetchOptions.method);
|
|
540
580
|
}
|
|
@@ -603,8 +643,17 @@ async function request(config, fetchOptions, requestCount, timing, _stats, respo
|
|
|
603
643
|
eventEmitter.emit("start", startEvent);
|
|
604
644
|
}
|
|
605
645
|
const requestOptions = buildHTTPOptions(fetchOptions, isSecure, url);
|
|
646
|
+
const stagedTimeoutConfig = parseStagedTimeouts(fetchOptions.timeout);
|
|
647
|
+
const timeoutManager = new StagedTimeoutManager(stagedTimeoutConfig, config, fetchOptions);
|
|
648
|
+
if (timeoutManager.hasPhase("total")) {
|
|
649
|
+
timeoutManager.startPhase("total");
|
|
650
|
+
}
|
|
606
651
|
try {
|
|
607
652
|
const req = httpModule.request(requestOptions, async (res) => {
|
|
653
|
+
timeoutManager.clearPhase("headers");
|
|
654
|
+
if (timeoutManager.hasPhase("body")) {
|
|
655
|
+
timeoutManager.startPhase("body");
|
|
656
|
+
}
|
|
608
657
|
if (!timing.firstByteTime) {
|
|
609
658
|
timing.firstByteTime = performance.now();
|
|
610
659
|
config.timing.responseStart = timing.firstByteTime;
|
|
@@ -896,9 +945,11 @@ async function request(config, fetchOptions, requestCount, timing, _stats, respo
|
|
|
896
945
|
uploadResult.emit("done", uploadFinishEvent);
|
|
897
946
|
uploadResult._markFinished();
|
|
898
947
|
}
|
|
948
|
+
timeoutManager.clearAll();
|
|
899
949
|
resolve(finalResponse);
|
|
900
950
|
});
|
|
901
951
|
decompressedStream.on("error", (err) => {
|
|
952
|
+
timeoutManager.clearAll();
|
|
902
953
|
_stats.statusOnNext = "error";
|
|
903
954
|
updateTiming(config, timing, contentLength || "", contentLengthCounter, res.rawHeaders);
|
|
904
955
|
if (_stats.redirectUrl) {
|
|
@@ -924,12 +975,36 @@ async function request(config, fetchOptions, requestCount, timing, _stats, respo
|
|
|
924
975
|
}
|
|
925
976
|
});
|
|
926
977
|
req.on("error", (err) => {
|
|
978
|
+
timeoutManager.clearAll();
|
|
927
979
|
_stats.statusOnNext = "error";
|
|
928
980
|
const error = buildSmartError(config, fetchOptions, err);
|
|
929
981
|
resolve(error);
|
|
930
982
|
});
|
|
931
983
|
req.on("socket", (socket) => {
|
|
984
|
+
timeoutManager.setSocket(socket);
|
|
985
|
+
timeoutManager.setRequest(req);
|
|
986
|
+
timeoutManager.setTimeoutCallback((phase, elapsed) => {
|
|
987
|
+
_stats.statusOnNext = "error";
|
|
988
|
+
const error = timeoutManager.createTimeoutError(phase, elapsed);
|
|
989
|
+
const eventEmitter = streamResult || downloadResult || uploadResult;
|
|
990
|
+
if (eventEmitter) {
|
|
991
|
+
eventEmitter.emit("error", error);
|
|
992
|
+
}
|
|
993
|
+
resolve(error);
|
|
994
|
+
});
|
|
995
|
+
const isAlreadyConnected = !socket.connecting && (socket.readyState === "open" || socket.writable === true);
|
|
996
|
+
if (isAlreadyConnected) {
|
|
997
|
+
timeoutManager.clearPhase("connect");
|
|
998
|
+
if (timeoutManager.hasPhase("headers")) {
|
|
999
|
+
timeoutManager.startPhase("headers");
|
|
1000
|
+
}
|
|
1001
|
+
} else {
|
|
1002
|
+
if (timeoutManager.hasPhase("connect")) {
|
|
1003
|
+
timeoutManager.startPhase("connect");
|
|
1004
|
+
}
|
|
1005
|
+
}
|
|
932
1006
|
socket.on("error", (err) => {
|
|
1007
|
+
timeoutManager.clearAll();
|
|
933
1008
|
_stats.statusOnNext = "error";
|
|
934
1009
|
const error = buildSmartError(config, fetchOptions, err);
|
|
935
1010
|
resolve(error);
|
|
@@ -963,6 +1038,10 @@ async function request(config, fetchOptions, requestCount, timing, _stats, respo
|
|
|
963
1038
|
}
|
|
964
1039
|
});
|
|
965
1040
|
socket.on("secureConnect", () => {
|
|
1041
|
+
timeoutManager.clearPhase("connect");
|
|
1042
|
+
if (timeoutManager.hasPhase("headers")) {
|
|
1043
|
+
timeoutManager.startPhase("headers");
|
|
1044
|
+
}
|
|
966
1045
|
if (!timing.tlsEnd && timing.tlsStart) {
|
|
967
1046
|
timing.tlsEnd = performance.now();
|
|
968
1047
|
config.timing.connectEnd = timing.tlsEnd;
|
|
@@ -1015,6 +1094,12 @@ async function request(config, fetchOptions, requestCount, timing, _stats, respo
|
|
|
1015
1094
|
}
|
|
1016
1095
|
});
|
|
1017
1096
|
socket.on("connect", () => {
|
|
1097
|
+
if (!isSecure) {
|
|
1098
|
+
timeoutManager.clearPhase("connect");
|
|
1099
|
+
if (timeoutManager.hasPhase("headers")) {
|
|
1100
|
+
timeoutManager.startPhase("headers");
|
|
1101
|
+
}
|
|
1102
|
+
}
|
|
1018
1103
|
if (!timing.tcpEnd) {
|
|
1019
1104
|
timing.tcpEnd = performance.now();
|
|
1020
1105
|
config.timing.connectEnd = timing.tcpEnd;
|
|
@@ -1052,6 +1137,7 @@ async function request(config, fetchOptions, requestCount, timing, _stats, respo
|
|
|
1052
1137
|
});
|
|
1053
1138
|
});
|
|
1054
1139
|
req.on("error", (error) => {
|
|
1140
|
+
timeoutManager.clearAll();
|
|
1055
1141
|
_stats.statusOnNext = "error";
|
|
1056
1142
|
updateTiming(config, timing, "", 0);
|
|
1057
1143
|
const e = buildSmartError(config, fetchOptions, error);
|
|
@@ -1198,20 +1284,42 @@ function buildHTTPOptions(fetchOptions, isSecure, url) {
|
|
|
1198
1284
|
useSecureContext = true,
|
|
1199
1285
|
auth,
|
|
1200
1286
|
dnsCache: dnsCacheOption,
|
|
1201
|
-
keepAlive =
|
|
1202
|
-
keepAliveMsecs = 60000
|
|
1287
|
+
keepAlive = true,
|
|
1288
|
+
keepAliveMsecs = 60000,
|
|
1289
|
+
useAgentPool = true
|
|
1203
1290
|
} = fetchOptions;
|
|
1204
|
-
|
|
1205
|
-
|
|
1206
|
-
|
|
1207
|
-
|
|
1208
|
-
|
|
1209
|
-
|
|
1210
|
-
|
|
1211
|
-
|
|
1212
|
-
|
|
1291
|
+
let agent;
|
|
1292
|
+
if (httpAgent || httpsAgent) {
|
|
1293
|
+
agent = isSecure ? httpsAgent : httpAgent;
|
|
1294
|
+
} else if (proxy) {
|
|
1295
|
+
agent = parseProxy(proxy, isSecure, rejectUnauthorized);
|
|
1296
|
+
} else if (useAgentPool) {
|
|
1297
|
+
const agentPool = getGlobalAgentPool({
|
|
1298
|
+
keepAlive: true,
|
|
1299
|
+
keepAliveMsecs,
|
|
1300
|
+
maxSockets: 256,
|
|
1301
|
+
maxFreeSockets: 64,
|
|
1302
|
+
dnsCache: dnsCacheOption !== false
|
|
1303
|
+
});
|
|
1304
|
+
if (isSecure) {
|
|
1305
|
+
agent = agentPool.getHttpsAgent({
|
|
1306
|
+
rejectUnauthorized,
|
|
1307
|
+
servername: url.hostname
|
|
1308
|
+
});
|
|
1309
|
+
} else {
|
|
1310
|
+
agent = agentPool.getHttpAgent();
|
|
1311
|
+
}
|
|
1312
|
+
} else if (isSecure && useSecureContext) {
|
|
1313
|
+
agent = new https.Agent({
|
|
1314
|
+
secureContext: createSecureContext(),
|
|
1315
|
+
servername: url.hostname,
|
|
1316
|
+
rejectUnauthorized,
|
|
1317
|
+
keepAlive,
|
|
1318
|
+
keepAliveMsecs: keepAlive ? keepAliveMsecs : undefined
|
|
1319
|
+
});
|
|
1320
|
+
}
|
|
1213
1321
|
let lookup;
|
|
1214
|
-
if (dnsCacheOption) {
|
|
1322
|
+
if (dnsCacheOption !== false && !useAgentPool) {
|
|
1215
1323
|
if (!dnsCache) {
|
|
1216
1324
|
const cacheOptions = typeof dnsCacheOption === "object" ? {
|
|
1217
1325
|
enable: true,
|
|
@@ -1275,7 +1383,7 @@ async function setInitialConfig(config, fetchOptions, isSecure, url, httpModule,
|
|
|
1275
1383
|
http1: true,
|
|
1276
1384
|
http2: config.http2,
|
|
1277
1385
|
compression: true,
|
|
1278
|
-
cookies: config.
|
|
1386
|
+
cookies: !config.disableCookieJar,
|
|
1279
1387
|
redirects: config.maxRedirects > 0,
|
|
1280
1388
|
proxy: !!proxy,
|
|
1281
1389
|
timeout: !!timeout,
|
|
@@ -1285,7 +1393,7 @@ async function setInitialConfig(config, fetchOptions, isSecure, url, httpModule,
|
|
|
1285
1393
|
config.features = {
|
|
1286
1394
|
http2: !!config.http2,
|
|
1287
1395
|
compression: !!config.compression?.enabled,
|
|
1288
|
-
cookies:
|
|
1396
|
+
cookies: !config.disableCookieJar,
|
|
1289
1397
|
redirects: config.maxRedirects > 0,
|
|
1290
1398
|
proxy: !!proxy,
|
|
1291
1399
|
timeout: !!timeout,
|
|
@@ -1383,10 +1491,34 @@ function emitRedirect(emitter, headers, status, statusText, sourceUri, destinati
|
|
|
1383
1491
|
}
|
|
1384
1492
|
function createSecureContext() {
|
|
1385
1493
|
return tls.createSecureContext({
|
|
1386
|
-
ecdhCurve: "X25519:prime256v1:secp384r1
|
|
1387
|
-
|
|
1388
|
-
|
|
1389
|
-
|
|
1494
|
+
ecdhCurve: "X25519:prime256v1:secp384r1",
|
|
1495
|
+
ciphers: [
|
|
1496
|
+
"TLS_AES_128_GCM_SHA256",
|
|
1497
|
+
"TLS_AES_256_GCM_SHA384",
|
|
1498
|
+
"TLS_CHACHA20_POLY1305_SHA256",
|
|
1499
|
+
"ECDHE-ECDSA-AES128-GCM-SHA256",
|
|
1500
|
+
"ECDHE-RSA-AES128-GCM-SHA256",
|
|
1501
|
+
"ECDHE-ECDSA-AES256-GCM-SHA384",
|
|
1502
|
+
"ECDHE-RSA-AES256-GCM-SHA384",
|
|
1503
|
+
"ECDHE-ECDSA-CHACHA20-POLY1305",
|
|
1504
|
+
"ECDHE-RSA-CHACHA20-POLY1305",
|
|
1505
|
+
"ECDHE-RSA-AES128-SHA",
|
|
1506
|
+
"ECDHE-RSA-AES256-SHA",
|
|
1507
|
+
"AES128-GCM-SHA256",
|
|
1508
|
+
"AES256-GCM-SHA384",
|
|
1509
|
+
"AES128-SHA",
|
|
1510
|
+
"AES256-SHA"
|
|
1511
|
+
].join(":"),
|
|
1512
|
+
sigalgs: [
|
|
1513
|
+
"ecdsa_secp256r1_sha256",
|
|
1514
|
+
"ecdsa_secp384r1_sha384",
|
|
1515
|
+
"rsa_pss_rsae_sha256",
|
|
1516
|
+
"rsa_pss_rsae_sha384",
|
|
1517
|
+
"rsa_pss_rsae_sha512",
|
|
1518
|
+
"rsa_pkcs1_sha256",
|
|
1519
|
+
"rsa_pkcs1_sha384",
|
|
1520
|
+
"rsa_pkcs1_sha512"
|
|
1521
|
+
].join(":"),
|
|
1390
1522
|
minVersion: "TLSv1.2",
|
|
1391
1523
|
maxVersion: "TLSv1.3",
|
|
1392
1524
|
sessionTimeout: 3600
|
|
@@ -1421,25 +1553,62 @@ function generateSessionId() {
|
|
|
1421
1553
|
function generateTraceId() {
|
|
1422
1554
|
return `trc_${Date.now()}_${Math.random().toString(36).substring(2, 15)}`;
|
|
1423
1555
|
}
|
|
1556
|
+
const proxyAgentCache = new Map;
|
|
1557
|
+
const PROXY_AGENT_EVICTION_MS = 60000;
|
|
1558
|
+
function buildProxyAgentKey(proxy, isSecure, rejectUnauthorized) {
|
|
1559
|
+
if (typeof proxy === "string") {
|
|
1560
|
+
return `str:${proxy}:${isSecure}:${rejectUnauthorized}`;
|
|
1561
|
+
}
|
|
1562
|
+
const p = proxy;
|
|
1563
|
+
const authKey = p.auth ? `${p.auth.username}:${p.auth.password}` : "";
|
|
1564
|
+
return `obj:${p.protocol}://${p.host}:${p.port}:${authKey}:${isSecure}:${rejectUnauthorized}`;
|
|
1565
|
+
}
|
|
1566
|
+
function evictStaleProxyAgents() {
|
|
1567
|
+
const now = Date.now();
|
|
1568
|
+
for (const [key, entry] of proxyAgentCache) {
|
|
1569
|
+
if (now - entry.lastUsed > PROXY_AGENT_EVICTION_MS) {
|
|
1570
|
+
try {
|
|
1571
|
+
entry.agent.destroy();
|
|
1572
|
+
} catch {}
|
|
1573
|
+
proxyAgentCache.delete(key);
|
|
1574
|
+
}
|
|
1575
|
+
}
|
|
1576
|
+
}
|
|
1577
|
+
let lastProxyEviction = 0;
|
|
1424
1578
|
function parseProxy(proxy, isScure = true, rejectUnauthorized = false) {
|
|
1425
1579
|
if (!proxy) {
|
|
1426
1580
|
return;
|
|
1427
1581
|
}
|
|
1582
|
+
const now = Date.now();
|
|
1583
|
+
if (now - lastProxyEviction > PROXY_AGENT_EVICTION_MS / 2) {
|
|
1584
|
+
evictStaleProxyAgents();
|
|
1585
|
+
lastProxyEviction = now;
|
|
1586
|
+
}
|
|
1587
|
+
const cacheKey = buildProxyAgentKey(proxy, isScure, rejectUnauthorized);
|
|
1588
|
+
const cached = proxyAgentCache.get(cacheKey);
|
|
1589
|
+
if (cached) {
|
|
1590
|
+
cached.lastUsed = now;
|
|
1591
|
+
return cached.agent;
|
|
1592
|
+
}
|
|
1593
|
+
let agent;
|
|
1428
1594
|
if (typeof proxy === "string") {
|
|
1429
1595
|
if (proxy.startsWith("http://")) {
|
|
1430
|
-
|
|
1596
|
+
agent = rezoProxy(`http://${proxy.slice(7)}`, "http");
|
|
1431
1597
|
} else if (proxy.startsWith("https://")) {
|
|
1432
|
-
|
|
1598
|
+
agent = rezoProxy(`https://${proxy.slice(8)}`, "https");
|
|
1599
|
+
} else {
|
|
1600
|
+
agent = rezoProxy(proxy);
|
|
1433
1601
|
}
|
|
1434
|
-
|
|
1435
|
-
|
|
1436
|
-
if (proxy.protocol === "http" || proxy.protocol === "https") {
|
|
1437
|
-
return rezoProxy({
|
|
1602
|
+
} else if (proxy.protocol === "http" || proxy.protocol === "https") {
|
|
1603
|
+
agent = rezoProxy({
|
|
1438
1604
|
...proxy,
|
|
1439
1605
|
client: !isScure ? "http" : "https"
|
|
1440
1606
|
});
|
|
1607
|
+
} else {
|
|
1608
|
+
agent = rezoProxy(proxy);
|
|
1441
1609
|
}
|
|
1442
|
-
|
|
1610
|
+
proxyAgentCache.set(cacheKey, { agent, lastUsed: now });
|
|
1611
|
+
return agent;
|
|
1443
1612
|
}
|
|
1444
1613
|
async function updateCookies(config, headers, url) {
|
|
1445
1614
|
const cookies = headers["set-cookie"];
|
|
@@ -1480,7 +1649,7 @@ async function updateCookies(config, headers, url) {
|
|
|
1480
1649
|
acceptedCookies.push(...parsedCookies.array);
|
|
1481
1650
|
}
|
|
1482
1651
|
const acceptedCookieStrings = acceptedCookies.map((c) => c.toSetCookieString());
|
|
1483
|
-
if (config.
|
|
1652
|
+
if (!config.disableCookieJar && config.cookieJar) {
|
|
1484
1653
|
config.cookieJar.setCookiesSync(acceptedCookieStrings, url);
|
|
1485
1654
|
}
|
|
1486
1655
|
jar.setCookiesSync(acceptedCookieStrings, url);
|
package/dist/adapters/http2.cjs
CHANGED
|
@@ -291,7 +291,15 @@ function buildUrlTree(config, finalUrl) {
|
|
|
291
291
|
const urlStr = typeof config.url === "string" ? config.url : config.url.toString();
|
|
292
292
|
urls.push(urlStr);
|
|
293
293
|
}
|
|
294
|
-
if (
|
|
294
|
+
if (config.redirectHistory && config.redirectHistory.length > 0) {
|
|
295
|
+
for (const redirect of config.redirectHistory) {
|
|
296
|
+
const redirectUrl = typeof redirect.url === "string" ? redirect.url : redirect.url?.toString?.() || "";
|
|
297
|
+
if (redirectUrl && urls[urls.length - 1] !== redirectUrl) {
|
|
298
|
+
urls.push(redirectUrl);
|
|
299
|
+
}
|
|
300
|
+
}
|
|
301
|
+
}
|
|
302
|
+
if (finalUrl && (urls.length === 0 || urls[urls.length - 1] !== finalUrl)) {
|
|
295
303
|
urls.push(finalUrl);
|
|
296
304
|
}
|
|
297
305
|
return urls.length > 0 ? urls : [finalUrl];
|
|
@@ -344,7 +352,7 @@ async function updateCookies(config, headers, url) {
|
|
|
344
352
|
const acceptedCookieStrings = acceptedCookies.map((c) => c.toSetCookieString());
|
|
345
353
|
const jar = new RezoCookieJar;
|
|
346
354
|
jar.setCookiesSync(acceptedCookieStrings, url);
|
|
347
|
-
if (config.
|
|
355
|
+
if (!config.disableCookieJar && config.cookieJar) {
|
|
348
356
|
config.cookieJar.setCookiesSync(acceptedCookieStrings, url);
|
|
349
357
|
}
|
|
350
358
|
const cookies = jar.cookies();
|
package/dist/adapters/http2.js
CHANGED
|
@@ -291,7 +291,15 @@ function buildUrlTree(config, finalUrl) {
|
|
|
291
291
|
const urlStr = typeof config.url === "string" ? config.url : config.url.toString();
|
|
292
292
|
urls.push(urlStr);
|
|
293
293
|
}
|
|
294
|
-
if (
|
|
294
|
+
if (config.redirectHistory && config.redirectHistory.length > 0) {
|
|
295
|
+
for (const redirect of config.redirectHistory) {
|
|
296
|
+
const redirectUrl = typeof redirect.url === "string" ? redirect.url : redirect.url?.toString?.() || "";
|
|
297
|
+
if (redirectUrl && urls[urls.length - 1] !== redirectUrl) {
|
|
298
|
+
urls.push(redirectUrl);
|
|
299
|
+
}
|
|
300
|
+
}
|
|
301
|
+
}
|
|
302
|
+
if (finalUrl && (urls.length === 0 || urls[urls.length - 1] !== finalUrl)) {
|
|
295
303
|
urls.push(finalUrl);
|
|
296
304
|
}
|
|
297
305
|
return urls.length > 0 ? urls : [finalUrl];
|
|
@@ -344,7 +352,7 @@ async function updateCookies(config, headers, url) {
|
|
|
344
352
|
const acceptedCookieStrings = acceptedCookies.map((c) => c.toSetCookieString());
|
|
345
353
|
const jar = new RezoCookieJar;
|
|
346
354
|
jar.setCookiesSync(acceptedCookieStrings, url);
|
|
347
|
-
if (config.
|
|
355
|
+
if (!config.disableCookieJar && config.cookieJar) {
|
|
348
356
|
config.cookieJar.setCookiesSync(acceptedCookieStrings, url);
|
|
349
357
|
}
|
|
350
358
|
const cookies = jar.cookies();
|
package/dist/adapters/index.cjs
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
|
-
const
|
|
2
|
-
exports.detectRuntime =
|
|
3
|
-
exports.getAdapterCapabilities =
|
|
4
|
-
exports.buildAdapterContext =
|
|
5
|
-
exports.getAvailableAdapters =
|
|
6
|
-
exports.selectAdapter =
|
|
1
|
+
const _mod_tvbgk1 = require('./picker.cjs');
|
|
2
|
+
exports.detectRuntime = _mod_tvbgk1.detectRuntime;
|
|
3
|
+
exports.getAdapterCapabilities = _mod_tvbgk1.getAdapterCapabilities;
|
|
4
|
+
exports.buildAdapterContext = _mod_tvbgk1.buildAdapterContext;
|
|
5
|
+
exports.getAvailableAdapters = _mod_tvbgk1.getAvailableAdapters;
|
|
6
|
+
exports.selectAdapter = _mod_tvbgk1.selectAdapter;;
|
|
@@ -188,7 +188,15 @@ function buildUrlTree(config, finalUrl) {
|
|
|
188
188
|
const urlStr = typeof config.url === "string" ? config.url : String(config.url);
|
|
189
189
|
urls.push(urlStr);
|
|
190
190
|
}
|
|
191
|
-
if (
|
|
191
|
+
if (config.redirectHistory && config.redirectHistory.length > 0) {
|
|
192
|
+
for (const redirect of config.redirectHistory) {
|
|
193
|
+
const redirectUrl = typeof redirect.url === "string" ? redirect.url : redirect.url?.toString?.() || "";
|
|
194
|
+
if (redirectUrl && urls[urls.length - 1] !== redirectUrl) {
|
|
195
|
+
urls.push(redirectUrl);
|
|
196
|
+
}
|
|
197
|
+
}
|
|
198
|
+
}
|
|
199
|
+
if (finalUrl && (urls.length === 0 || urls[urls.length - 1] !== finalUrl)) {
|
|
192
200
|
urls.push(finalUrl);
|
|
193
201
|
}
|
|
194
202
|
return urls.length > 0 ? urls : [finalUrl];
|
|
@@ -520,7 +528,7 @@ async function executeSingleRequest(config, fetchOptions, timing, streamResult,
|
|
|
520
528
|
acceptedCookies.push(...parsedCookies.array);
|
|
521
529
|
}
|
|
522
530
|
const acceptedCookieStrings = acceptedCookies.map((c) => c.toSetCookieString());
|
|
523
|
-
if (config.
|
|
531
|
+
if (!config.disableCookieJar && config.cookieJar) {
|
|
524
532
|
config.cookieJar.setCookiesSync(acceptedCookieStrings, url);
|
|
525
533
|
}
|
|
526
534
|
const cookieJar = new RezoCookieJar(acceptedCookies, url);
|
|
@@ -188,7 +188,15 @@ function buildUrlTree(config, finalUrl) {
|
|
|
188
188
|
const urlStr = typeof config.url === "string" ? config.url : String(config.url);
|
|
189
189
|
urls.push(urlStr);
|
|
190
190
|
}
|
|
191
|
-
if (
|
|
191
|
+
if (config.redirectHistory && config.redirectHistory.length > 0) {
|
|
192
|
+
for (const redirect of config.redirectHistory) {
|
|
193
|
+
const redirectUrl = typeof redirect.url === "string" ? redirect.url : redirect.url?.toString?.() || "";
|
|
194
|
+
if (redirectUrl && urls[urls.length - 1] !== redirectUrl) {
|
|
195
|
+
urls.push(redirectUrl);
|
|
196
|
+
}
|
|
197
|
+
}
|
|
198
|
+
}
|
|
199
|
+
if (finalUrl && (urls.length === 0 || urls[urls.length - 1] !== finalUrl)) {
|
|
192
200
|
urls.push(finalUrl);
|
|
193
201
|
}
|
|
194
202
|
return urls.length > 0 ? urls : [finalUrl];
|
|
@@ -520,7 +528,7 @@ async function executeSingleRequest(config, fetchOptions, timing, streamResult,
|
|
|
520
528
|
acceptedCookies.push(...parsedCookies.array);
|
|
521
529
|
}
|
|
522
530
|
const acceptedCookieStrings = acceptedCookies.map((c) => c.toSetCookieString());
|
|
523
|
-
if (config.
|
|
531
|
+
if (!config.disableCookieJar && config.cookieJar) {
|
|
524
532
|
config.cookieJar.setCookiesSync(acceptedCookieStrings, url);
|
|
525
533
|
}
|
|
526
534
|
const cookieJar = new RezoCookieJar(acceptedCookies, url);
|
package/dist/adapters/xhr.cjs
CHANGED
|
@@ -107,7 +107,15 @@ function buildUrlTree(config, finalUrl) {
|
|
|
107
107
|
const urlStr = typeof config.url === "string" ? config.url : String(config.url);
|
|
108
108
|
urls.push(urlStr);
|
|
109
109
|
}
|
|
110
|
-
if (
|
|
110
|
+
if (config.redirectHistory && config.redirectHistory.length > 0) {
|
|
111
|
+
for (const redirect of config.redirectHistory) {
|
|
112
|
+
const redirectUrl = typeof redirect.url === "string" ? redirect.url : redirect.url?.toString?.() || "";
|
|
113
|
+
if (redirectUrl && urls[urls.length - 1] !== redirectUrl) {
|
|
114
|
+
urls.push(redirectUrl);
|
|
115
|
+
}
|
|
116
|
+
}
|
|
117
|
+
}
|
|
118
|
+
if (finalUrl && (urls.length === 0 || urls[urls.length - 1] !== finalUrl)) {
|
|
111
119
|
urls.push(finalUrl);
|
|
112
120
|
}
|
|
113
121
|
return urls.length > 0 ? urls : [finalUrl];
|
package/dist/adapters/xhr.js
CHANGED
|
@@ -107,7 +107,15 @@ function buildUrlTree(config, finalUrl) {
|
|
|
107
107
|
const urlStr = typeof config.url === "string" ? config.url : String(config.url);
|
|
108
108
|
urls.push(urlStr);
|
|
109
109
|
}
|
|
110
|
-
if (
|
|
110
|
+
if (config.redirectHistory && config.redirectHistory.length > 0) {
|
|
111
|
+
for (const redirect of config.redirectHistory) {
|
|
112
|
+
const redirectUrl = typeof redirect.url === "string" ? redirect.url : redirect.url?.toString?.() || "";
|
|
113
|
+
if (redirectUrl && urls[urls.length - 1] !== redirectUrl) {
|
|
114
|
+
urls.push(redirectUrl);
|
|
115
|
+
}
|
|
116
|
+
}
|
|
117
|
+
}
|
|
118
|
+
if (finalUrl && (urls.length === 0 || urls[urls.length - 1] !== finalUrl)) {
|
|
111
119
|
urls.push(finalUrl);
|
|
112
120
|
}
|
|
113
121
|
return urls.length > 0 ? urls : [finalUrl];
|
package/dist/cache/index.cjs
CHANGED
|
@@ -1,13 +1,13 @@
|
|
|
1
|
-
const
|
|
2
|
-
exports.LRUCache =
|
|
3
|
-
const
|
|
4
|
-
exports.DNSCache =
|
|
5
|
-
exports.getGlobalDNSCache =
|
|
6
|
-
exports.resetGlobalDNSCache =
|
|
7
|
-
const
|
|
8
|
-
exports.ResponseCache =
|
|
9
|
-
exports.normalizeResponseCacheConfig =
|
|
10
|
-
const
|
|
11
|
-
exports.FileCacher =
|
|
12
|
-
const
|
|
13
|
-
exports.UrlStore =
|
|
1
|
+
const _mod_5sm27m = require('./lru-cache.cjs');
|
|
2
|
+
exports.LRUCache = _mod_5sm27m.LRUCache;;
|
|
3
|
+
const _mod_9h56ez = require('./dns-cache.cjs');
|
|
4
|
+
exports.DNSCache = _mod_9h56ez.DNSCache;
|
|
5
|
+
exports.getGlobalDNSCache = _mod_9h56ez.getGlobalDNSCache;
|
|
6
|
+
exports.resetGlobalDNSCache = _mod_9h56ez.resetGlobalDNSCache;;
|
|
7
|
+
const _mod_fo0wc9 = require('./response-cache.cjs');
|
|
8
|
+
exports.ResponseCache = _mod_fo0wc9.ResponseCache;
|
|
9
|
+
exports.normalizeResponseCacheConfig = _mod_fo0wc9.normalizeResponseCacheConfig;;
|
|
10
|
+
const _mod_qslx02 = require('./file-cacher.cjs');
|
|
11
|
+
exports.FileCacher = _mod_qslx02.FileCacher;;
|
|
12
|
+
const _mod_ur6epm = require('./url-store.cjs');
|
|
13
|
+
exports.UrlStore = _mod_ur6epm.UrlStore;;
|
package/dist/crawler.d.ts
CHANGED
|
@@ -1769,7 +1769,14 @@ export interface RezoConfig {
|
|
|
1769
1769
|
/** @description Supported compression algorithms */
|
|
1770
1770
|
algorithms?: string[];
|
|
1771
1771
|
};
|
|
1772
|
-
/**
|
|
1772
|
+
/**
|
|
1773
|
+
* @description Disable cookie jar for session management.
|
|
1774
|
+
* When false (default), cookies are automatically managed.
|
|
1775
|
+
* Set to true to disable automatic cookie handling.
|
|
1776
|
+
* @default false
|
|
1777
|
+
*/
|
|
1778
|
+
disableCookieJar?: boolean;
|
|
1779
|
+
/** @deprecated Use `disableCookieJar` instead */
|
|
1773
1780
|
enableCookieJar?: boolean;
|
|
1774
1781
|
/** @description Send cookies with cross-origin requests (matches Axios withCredentials). Default: false */
|
|
1775
1782
|
withCredentials?: boolean;
|
|
@@ -1880,6 +1887,8 @@ export interface RezoConfig {
|
|
|
1880
1887
|
hooks: Partial<RezoHooks> | null;
|
|
1881
1888
|
/** @description Snapshot of the original request configuration */
|
|
1882
1889
|
originalRequest: RezoRequestConfig;
|
|
1890
|
+
/** @description Original request body, preserved for POST body retention during redirects */
|
|
1891
|
+
originalBody?: RezoRequestConfig["body"];
|
|
1883
1892
|
/** @description Final resolved URL after redirects and processing */
|
|
1884
1893
|
finalUrl: string;
|
|
1885
1894
|
/** @description HTTP adapter used for the request */
|
|
@@ -2814,6 +2823,8 @@ export interface OnRedirectOptions {
|
|
|
2814
2823
|
headers: RezoHeaders;
|
|
2815
2824
|
sameDomain: boolean;
|
|
2816
2825
|
method: string;
|
|
2826
|
+
/** The current request body (RezoFormData, string, object, etc.) - allows user to inspect/modify */
|
|
2827
|
+
body?: any;
|
|
2817
2828
|
}
|
|
2818
2829
|
export type OnRedirectResponse = boolean | ToRedirectOptions | undefined;
|
|
2819
2830
|
export type ToRedirectOptions = {
|
|
@@ -3164,7 +3175,14 @@ export interface RezoDefaultOptions {
|
|
|
3164
3175
|
baseURL?: string;
|
|
3165
3176
|
/** Hooks for request/response lifecycle */
|
|
3166
3177
|
hooks?: Partial<RezoHooks>;
|
|
3167
|
-
/**
|
|
3178
|
+
/**
|
|
3179
|
+
* Whether to disable automatic cookie handling.
|
|
3180
|
+
* When false (default), cookies are automatically managed via the jar.
|
|
3181
|
+
* Set to true to disable automatic cookie management.
|
|
3182
|
+
* @default false
|
|
3183
|
+
*/
|
|
3184
|
+
disableCookieJar?: boolean;
|
|
3185
|
+
/** @deprecated Use `disableCookieJar` instead. Will be removed in next major version. */
|
|
3168
3186
|
enableCookieJar?: boolean;
|
|
3169
3187
|
/**
|
|
3170
3188
|
* Custom cookie jar for managing cookies.
|
package/dist/entries/crawler.cjs
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
const
|
|
2
|
-
exports.Crawler =
|
|
3
|
-
const
|
|
4
|
-
exports.CrawlerOptions =
|
|
5
|
-
exports.Domain =
|
|
1
|
+
const _mod_bxvm04 = require('../plugin/crawler.cjs');
|
|
2
|
+
exports.Crawler = _mod_bxvm04.Crawler;;
|
|
3
|
+
const _mod_gzxwe7 = require('../plugin/crawler-options.cjs');
|
|
4
|
+
exports.CrawlerOptions = _mod_gzxwe7.CrawlerOptions;
|
|
5
|
+
exports.Domain = _mod_gzxwe7.Domain;;
|