rezo 1.0.22 → 1.0.24
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/adapters/fetch.cjs +62 -6
- package/dist/adapters/fetch.js +62 -6
- package/dist/adapters/http.cjs +185 -16
- package/dist/adapters/http.js +185 -16
- package/dist/adapters/http2.cjs +72 -8
- package/dist/adapters/http2.js +72 -8
- package/dist/adapters/index.cjs +6 -6
- package/dist/cache/index.cjs +13 -13
- package/dist/core/rezo.cjs +48 -2
- package/dist/core/rezo.js +48 -2
- package/dist/entries/crawler.cjs +5 -5
- package/dist/index.cjs +24 -24
- package/dist/plugin/index.cjs +36 -36
- package/dist/proxy/index.cjs +2 -2
- package/dist/queue/index.cjs +8 -8
- package/dist/utils/http-config.cjs +0 -3
- package/dist/utils/http-config.js +0 -3
- package/package.json +1 -1
package/dist/adapters/http.js
CHANGED
|
@@ -290,10 +290,19 @@ export async function executeRequest(options, defaultOptions, jar) {
|
|
|
290
290
|
try {
|
|
291
291
|
const res = executeHttp1Request(config.fetchOptions, mainConfig, config.options, perform, d_options.fs, streamResponse, downloadResponse, uploadResponse);
|
|
292
292
|
if (streamResponse) {
|
|
293
|
+
res.catch((err) => {
|
|
294
|
+
streamResponse.emit("error", err);
|
|
295
|
+
});
|
|
293
296
|
return streamResponse;
|
|
294
297
|
} else if (downloadResponse) {
|
|
298
|
+
res.catch((err) => {
|
|
299
|
+
downloadResponse.emit("error", err);
|
|
300
|
+
});
|
|
295
301
|
return downloadResponse;
|
|
296
302
|
} else if (uploadResponse) {
|
|
303
|
+
res.catch((err) => {
|
|
304
|
+
uploadResponse.emit("error", err);
|
|
305
|
+
});
|
|
297
306
|
return uploadResponse;
|
|
298
307
|
}
|
|
299
308
|
const response = await res;
|
|
@@ -386,6 +395,11 @@ async function executeHttp1Request(fetchOptions, config, options, perform, fs, s
|
|
|
386
395
|
retries++;
|
|
387
396
|
const currentDelay = incrementDelay ? retryDelay * retries : retryDelay;
|
|
388
397
|
debugLog.retry(config, retries, maxRetries, responseStatusCode, currentDelay);
|
|
398
|
+
if (config.hooks?.beforeRetry && config.hooks.beforeRetry.length > 0) {
|
|
399
|
+
for (const hook of config.hooks.beforeRetry) {
|
|
400
|
+
await hook(config, response, retries);
|
|
401
|
+
}
|
|
402
|
+
}
|
|
389
403
|
if (retryDelay > 0) {
|
|
390
404
|
await new Promise((resolve) => setTimeout(resolve, currentDelay));
|
|
391
405
|
}
|
|
@@ -411,17 +425,32 @@ async function executeHttp1Request(fetchOptions, config, options, perform, fs, s
|
|
|
411
425
|
const addedOptions = {};
|
|
412
426
|
const location = _stats.redirectUrl;
|
|
413
427
|
if (!location || !_stats.redirectUrl) {
|
|
414
|
-
|
|
428
|
+
const redirectError = builErrorFromResponse("Redirect location not found", response, config, fetchOptions);
|
|
429
|
+
_stats.statusOnNext = "error";
|
|
430
|
+
if (!config.errors)
|
|
431
|
+
config.errors = [];
|
|
432
|
+
config.errors.push({ attempt: config.retryAttempts + 1, error: redirectError, duration: perform.now() });
|
|
433
|
+
throw redirectError;
|
|
415
434
|
}
|
|
416
435
|
if (config.maxRedirects === 0) {
|
|
417
436
|
config.maxRedirectsReached = true;
|
|
418
|
-
|
|
437
|
+
const redirectError = builErrorFromResponse(`Redirects are disabled (maxRedirects=0)`, response, config, fetchOptions);
|
|
438
|
+
_stats.statusOnNext = "error";
|
|
439
|
+
if (!config.errors)
|
|
440
|
+
config.errors = [];
|
|
441
|
+
config.errors.push({ attempt: config.retryAttempts + 1, error: redirectError, duration: perform.now() });
|
|
442
|
+
throw redirectError;
|
|
419
443
|
}
|
|
420
444
|
const enableCycleDetection = config.enableRedirectCycleDetection === true;
|
|
421
445
|
if (enableCycleDetection) {
|
|
422
446
|
const normalizedRedirectUrl = _stats.redirectUrl.toLowerCase();
|
|
423
447
|
if (visitedUrls.has(normalizedRedirectUrl)) {
|
|
424
|
-
|
|
448
|
+
const redirectError = builErrorFromResponse(`Redirect cycle detected: attempting to revisit ${_stats.redirectUrl}`, response, config, fetchOptions);
|
|
449
|
+
_stats.statusOnNext = "error";
|
|
450
|
+
if (!config.errors)
|
|
451
|
+
config.errors = [];
|
|
452
|
+
config.errors.push({ attempt: config.retryAttempts + 1, error: redirectError, duration: perform.now() });
|
|
453
|
+
throw redirectError;
|
|
425
454
|
}
|
|
426
455
|
visitedUrls.add(normalizedRedirectUrl);
|
|
427
456
|
}
|
|
@@ -438,15 +467,30 @@ async function executeHttp1Request(fetchOptions, config, options, perform, fs, s
|
|
|
438
467
|
if (typeof onRedirect !== "undefined") {
|
|
439
468
|
if (typeof onRedirect === "boolean") {
|
|
440
469
|
if (!onRedirect) {
|
|
441
|
-
|
|
470
|
+
const redirectError = builErrorFromResponse("Redirect denied by user", response, config, fetchOptions);
|
|
471
|
+
_stats.statusOnNext = "error";
|
|
472
|
+
if (!config.errors)
|
|
473
|
+
config.errors = [];
|
|
474
|
+
config.errors.push({ attempt: config.retryAttempts + 1, error: redirectError, duration: perform.now() });
|
|
475
|
+
throw redirectError;
|
|
442
476
|
}
|
|
443
477
|
} else if (!onRedirect.redirect) {
|
|
444
|
-
|
|
478
|
+
const redirectError = builErrorFromResponse("Redirect denied by user", response, config, fetchOptions);
|
|
479
|
+
_stats.statusOnNext = "error";
|
|
480
|
+
if (!config.errors)
|
|
481
|
+
config.errors = [];
|
|
482
|
+
config.errors.push({ attempt: config.retryAttempts + 1, error: redirectError, duration: perform.now() });
|
|
483
|
+
throw redirectError;
|
|
445
484
|
}
|
|
446
485
|
}
|
|
447
486
|
if (config.redirectCount >= config.maxRedirects && config.maxRedirects > 0) {
|
|
448
487
|
config.maxRedirectsReached = true;
|
|
449
|
-
|
|
488
|
+
const redirectError = builErrorFromResponse(`Max redirects (${config.maxRedirects}) reached`, response, config, fetchOptions);
|
|
489
|
+
_stats.statusOnNext = "error";
|
|
490
|
+
if (!config.errors)
|
|
491
|
+
config.errors = [];
|
|
492
|
+
config.errors.push({ attempt: config.retryAttempts + 1, error: redirectError, duration: perform.now() });
|
|
493
|
+
throw redirectError;
|
|
450
494
|
}
|
|
451
495
|
config.redirectHistory.push({
|
|
452
496
|
url: fetchOptions.fullUrl,
|
|
@@ -574,7 +618,7 @@ async function request(config, fetchOptions, requestCount, timing, _stats, respo
|
|
|
574
618
|
const location = headers["location"] || headers["Location"];
|
|
575
619
|
const contentLength = headers["content-length"];
|
|
576
620
|
const cookies = headers["set-cookie"];
|
|
577
|
-
updateCookies(config, headers, url.href);
|
|
621
|
+
await updateCookies(config, headers, url.href);
|
|
578
622
|
const cookieArray = config.responseCookies?.array || [];
|
|
579
623
|
delete headers["set-cookie"];
|
|
580
624
|
_stats.redirectUrl = undefined;
|
|
@@ -641,7 +685,11 @@ async function request(config, fetchOptions, requestCount, timing, _stats, respo
|
|
|
641
685
|
if (isRedirected)
|
|
642
686
|
_stats.statusOnNext = "redirect";
|
|
643
687
|
if (isRedirected && location) {
|
|
644
|
-
|
|
688
|
+
const redirectUrlObj = new URL(location, url);
|
|
689
|
+
if (!redirectUrlObj.hash && url.hash) {
|
|
690
|
+
redirectUrlObj.hash = url.hash;
|
|
691
|
+
}
|
|
692
|
+
_stats.redirectUrl = redirectUrlObj.href;
|
|
645
693
|
if (config.redirectCount) {
|
|
646
694
|
config.redirectCount++;
|
|
647
695
|
} else {
|
|
@@ -871,19 +919,52 @@ async function request(config, fetchOptions, requestCount, timing, _stats, respo
|
|
|
871
919
|
});
|
|
872
920
|
}
|
|
873
921
|
});
|
|
922
|
+
req.on("error", (err) => {
|
|
923
|
+
_stats.statusOnNext = "error";
|
|
924
|
+
const error = buildSmartError(config, fetchOptions, err);
|
|
925
|
+
resolve(error);
|
|
926
|
+
});
|
|
874
927
|
req.on("socket", (socket) => {
|
|
875
|
-
if (socket && typeof socket.
|
|
876
|
-
socket.
|
|
928
|
+
if (socket && typeof socket.ref === "function") {
|
|
929
|
+
socket.ref();
|
|
877
930
|
}
|
|
931
|
+
socket.on("error", (err) => {
|
|
932
|
+
_stats.statusOnNext = "error";
|
|
933
|
+
const error = buildSmartError(config, fetchOptions, err);
|
|
934
|
+
resolve(error);
|
|
935
|
+
});
|
|
936
|
+
socket.on("close", () => {
|
|
937
|
+
if (socket && typeof socket.unref === "function") {
|
|
938
|
+
socket.unref();
|
|
939
|
+
}
|
|
940
|
+
});
|
|
878
941
|
timing.dnsStart = performance.now();
|
|
879
942
|
config.timing.domainLookupStart = timing.dnsStart;
|
|
880
|
-
socket.on("lookup", () => {
|
|
943
|
+
socket.on("lookup", (err, address, family) => {
|
|
881
944
|
if (!timing.dnsEnd) {
|
|
882
945
|
timing.dnsEnd = performance.now();
|
|
883
946
|
config.timing.domainLookupEnd = timing.dnsEnd;
|
|
884
947
|
timing.tcpStart = performance.now();
|
|
885
948
|
config.timing.connectStart = timing.tcpStart;
|
|
886
949
|
}
|
|
950
|
+
if (config.hooks?.onDns && config.hooks.onDns.length > 0) {
|
|
951
|
+
const familyNum = typeof family === "number" ? family : family === "IPv6" ? 6 : 4;
|
|
952
|
+
for (const hook of config.hooks.onDns) {
|
|
953
|
+
try {
|
|
954
|
+
hook({
|
|
955
|
+
hostname: url.hostname,
|
|
956
|
+
address: address || "",
|
|
957
|
+
family: familyNum,
|
|
958
|
+
duration: timing.dnsEnd - timing.dnsStart,
|
|
959
|
+
timestamp: Date.now()
|
|
960
|
+
}, config);
|
|
961
|
+
} catch (err) {
|
|
962
|
+
if (config.debug) {
|
|
963
|
+
console.log("[Rezo Debug] onDns hook error:", err);
|
|
964
|
+
}
|
|
965
|
+
}
|
|
966
|
+
}
|
|
967
|
+
}
|
|
887
968
|
});
|
|
888
969
|
socket.on("secureConnect", () => {
|
|
889
970
|
if (!timing.tlsEnd && timing.tlsStart) {
|
|
@@ -910,6 +991,31 @@ async function request(config, fetchOptions, requestCount, timing, _stats, respo
|
|
|
910
991
|
hostnameMatch: cert.subject?.CN === url.hostname,
|
|
911
992
|
chainValid: true
|
|
912
993
|
};
|
|
994
|
+
if (config.hooks?.onTls && config.hooks.onTls.length > 0) {
|
|
995
|
+
for (const hook of config.hooks.onTls) {
|
|
996
|
+
try {
|
|
997
|
+
hook({
|
|
998
|
+
protocol: tlsVersion,
|
|
999
|
+
cipher: cipher?.name || "",
|
|
1000
|
+
authorized: !socket.authorizationError,
|
|
1001
|
+
authorizationError: socket.authorizationError,
|
|
1002
|
+
certificate: cert ? {
|
|
1003
|
+
subject: cert.subject?.CN || "",
|
|
1004
|
+
issuer: cert.issuer?.CN || "",
|
|
1005
|
+
validFrom: cert.valid_from || "",
|
|
1006
|
+
validTo: cert.valid_to || "",
|
|
1007
|
+
fingerprint: cert.fingerprint || ""
|
|
1008
|
+
} : undefined,
|
|
1009
|
+
duration: timing.tlsEnd - timing.tlsStart,
|
|
1010
|
+
timestamp: Date.now()
|
|
1011
|
+
}, config);
|
|
1012
|
+
} catch (err) {
|
|
1013
|
+
if (config.debug) {
|
|
1014
|
+
console.log("[Rezo Debug] onTls hook error:", err);
|
|
1015
|
+
}
|
|
1016
|
+
}
|
|
1017
|
+
}
|
|
1018
|
+
}
|
|
913
1019
|
});
|
|
914
1020
|
socket.on("connect", () => {
|
|
915
1021
|
if (!timing.tcpEnd) {
|
|
@@ -928,6 +1034,24 @@ async function request(config, fetchOptions, requestCount, timing, _stats, respo
|
|
|
928
1034
|
config.network.localPort = localPort;
|
|
929
1035
|
config.network.family = remoteFamily;
|
|
930
1036
|
}
|
|
1037
|
+
if (config.hooks?.onSocket && config.hooks.onSocket.length > 0) {
|
|
1038
|
+
for (const hook of config.hooks.onSocket) {
|
|
1039
|
+
try {
|
|
1040
|
+
hook({
|
|
1041
|
+
type: "connect",
|
|
1042
|
+
localAddress,
|
|
1043
|
+
localPort,
|
|
1044
|
+
remoteAddress,
|
|
1045
|
+
remotePort,
|
|
1046
|
+
timestamp: Date.now()
|
|
1047
|
+
}, socket);
|
|
1048
|
+
} catch (err) {
|
|
1049
|
+
if (config.debug) {
|
|
1050
|
+
console.log("[Rezo Debug] onSocket hook error:", err);
|
|
1051
|
+
}
|
|
1052
|
+
}
|
|
1053
|
+
}
|
|
1054
|
+
}
|
|
931
1055
|
});
|
|
932
1056
|
});
|
|
933
1057
|
req.on("error", (error) => {
|
|
@@ -1323,18 +1447,52 @@ function parseProxy(proxy, isScure = true, rejectUnauthorized = false) {
|
|
|
1323
1447
|
}
|
|
1324
1448
|
return rezoProxy(proxy);
|
|
1325
1449
|
}
|
|
1326
|
-
function updateCookies(config, headers, url) {
|
|
1450
|
+
async function updateCookies(config, headers, url) {
|
|
1327
1451
|
const cookies = headers["set-cookie"];
|
|
1328
1452
|
if (cookies) {
|
|
1329
1453
|
const jar = new RezoCookieJar;
|
|
1454
|
+
const tempJar = new RezoCookieJar;
|
|
1455
|
+
tempJar.setCookiesSync(cookies, url);
|
|
1456
|
+
const parsedCookies = tempJar.cookies();
|
|
1457
|
+
const acceptedCookies = [];
|
|
1458
|
+
let hookError = null;
|
|
1459
|
+
if (config.hooks?.beforeCookie && config.hooks.beforeCookie.length > 0) {
|
|
1460
|
+
for (const cookie of parsedCookies.array) {
|
|
1461
|
+
let shouldAccept = true;
|
|
1462
|
+
for (const hook of config.hooks.beforeCookie) {
|
|
1463
|
+
try {
|
|
1464
|
+
const result = await hook({
|
|
1465
|
+
cookie,
|
|
1466
|
+
source: "response",
|
|
1467
|
+
url,
|
|
1468
|
+
isValid: true
|
|
1469
|
+
}, config);
|
|
1470
|
+
if (result === false) {
|
|
1471
|
+
shouldAccept = false;
|
|
1472
|
+
break;
|
|
1473
|
+
}
|
|
1474
|
+
} catch (err) {
|
|
1475
|
+
hookError = err;
|
|
1476
|
+
if (config.debug) {
|
|
1477
|
+
console.log("[Rezo Debug] beforeCookie hook error:", err);
|
|
1478
|
+
}
|
|
1479
|
+
}
|
|
1480
|
+
}
|
|
1481
|
+
if (shouldAccept) {
|
|
1482
|
+
acceptedCookies.push(cookie);
|
|
1483
|
+
}
|
|
1484
|
+
}
|
|
1485
|
+
} else {
|
|
1486
|
+
acceptedCookies.push(...parsedCookies.array);
|
|
1487
|
+
}
|
|
1488
|
+
const acceptedCookieStrings = acceptedCookies.map((c) => c.cookieString());
|
|
1330
1489
|
if (config.enableCookieJar && config.cookieJar) {
|
|
1331
|
-
config.cookieJar.setCookiesSync(
|
|
1490
|
+
config.cookieJar.setCookiesSync(acceptedCookieStrings, url);
|
|
1332
1491
|
}
|
|
1333
|
-
jar.setCookiesSync(
|
|
1492
|
+
jar.setCookiesSync(acceptedCookieStrings, url);
|
|
1334
1493
|
if (config.useCookies) {
|
|
1335
|
-
const parsedCookies = jar.cookies();
|
|
1336
1494
|
const existingArray = config.responseCookies?.array || [];
|
|
1337
|
-
for (const cookie of
|
|
1495
|
+
for (const cookie of acceptedCookies) {
|
|
1338
1496
|
const existingIndex = existingArray.findIndex((c) => c.key === cookie.key && c.domain === cookie.domain);
|
|
1339
1497
|
if (existingIndex >= 0) {
|
|
1340
1498
|
existingArray[existingIndex] = cookie;
|
|
@@ -1345,5 +1503,16 @@ function updateCookies(config, headers, url) {
|
|
|
1345
1503
|
const mergedJar = new RezoCookieJar(existingArray, url);
|
|
1346
1504
|
config.responseCookies = mergedJar.cookies();
|
|
1347
1505
|
}
|
|
1506
|
+
if (!hookError && config.hooks?.afterCookie && config.hooks.afterCookie.length > 0) {
|
|
1507
|
+
for (const hook of config.hooks.afterCookie) {
|
|
1508
|
+
try {
|
|
1509
|
+
await hook(acceptedCookies, config);
|
|
1510
|
+
} catch (err) {
|
|
1511
|
+
if (config.debug) {
|
|
1512
|
+
console.log("[Rezo Debug] afterCookie hook error:", err);
|
|
1513
|
+
}
|
|
1514
|
+
}
|
|
1515
|
+
}
|
|
1516
|
+
}
|
|
1348
1517
|
}
|
|
1349
1518
|
}
|
package/dist/adapters/http2.cjs
CHANGED
|
@@ -300,23 +300,58 @@ function sanitizeConfig(config) {
|
|
|
300
300
|
const { data: _data, ...sanitized } = config;
|
|
301
301
|
return sanitized;
|
|
302
302
|
}
|
|
303
|
-
function updateCookies(config, headers, url) {
|
|
303
|
+
async function updateCookies(config, headers, url) {
|
|
304
304
|
const setCookieHeaders = headers["set-cookie"];
|
|
305
305
|
if (!setCookieHeaders)
|
|
306
306
|
return;
|
|
307
307
|
const cookieHeaderArray = Array.isArray(setCookieHeaders) ? setCookieHeaders : [setCookieHeaders];
|
|
308
308
|
if (cookieHeaderArray.length === 0)
|
|
309
309
|
return;
|
|
310
|
+
const tempJar = new RezoCookieJar;
|
|
311
|
+
tempJar.setCookiesSync(cookieHeaderArray, url);
|
|
312
|
+
const parsedCookies = tempJar.cookies();
|
|
313
|
+
const acceptedCookies = [];
|
|
314
|
+
let hookError = null;
|
|
315
|
+
if (config.hooks?.beforeCookie && config.hooks.beforeCookie.length > 0) {
|
|
316
|
+
for (const cookie of parsedCookies.array) {
|
|
317
|
+
let shouldAccept = true;
|
|
318
|
+
for (const hook of config.hooks.beforeCookie) {
|
|
319
|
+
try {
|
|
320
|
+
const result = await hook({
|
|
321
|
+
cookie,
|
|
322
|
+
source: "response",
|
|
323
|
+
url,
|
|
324
|
+
isValid: true
|
|
325
|
+
}, config);
|
|
326
|
+
if (result === false) {
|
|
327
|
+
shouldAccept = false;
|
|
328
|
+
break;
|
|
329
|
+
}
|
|
330
|
+
} catch (err) {
|
|
331
|
+
hookError = err;
|
|
332
|
+
if (config.debug) {
|
|
333
|
+
console.log("[Rezo Debug] beforeCookie hook error:", err);
|
|
334
|
+
}
|
|
335
|
+
}
|
|
336
|
+
}
|
|
337
|
+
if (shouldAccept) {
|
|
338
|
+
acceptedCookies.push(cookie);
|
|
339
|
+
}
|
|
340
|
+
}
|
|
341
|
+
} else {
|
|
342
|
+
acceptedCookies.push(...parsedCookies.array);
|
|
343
|
+
}
|
|
344
|
+
const acceptedCookieStrings = acceptedCookies.map((c) => c.cookieString());
|
|
310
345
|
const jar = new RezoCookieJar;
|
|
311
|
-
jar.setCookiesSync(
|
|
346
|
+
jar.setCookiesSync(acceptedCookieStrings, url);
|
|
312
347
|
if (config.enableCookieJar && config.cookieJar) {
|
|
313
|
-
config.cookieJar.setCookiesSync(
|
|
348
|
+
config.cookieJar.setCookiesSync(acceptedCookieStrings, url);
|
|
314
349
|
}
|
|
315
350
|
const cookies = jar.cookies();
|
|
316
351
|
cookies.setCookiesString = cookieHeaderArray;
|
|
317
352
|
if (config.useCookies) {
|
|
318
353
|
const existingArray = config.responseCookies?.array || [];
|
|
319
|
-
for (const cookie of
|
|
354
|
+
for (const cookie of acceptedCookies) {
|
|
320
355
|
const existingIndex = existingArray.findIndex((c) => c.key === cookie.key && c.domain === cookie.domain);
|
|
321
356
|
if (existingIndex >= 0) {
|
|
322
357
|
existingArray[existingIndex] = cookie;
|
|
@@ -330,6 +365,17 @@ function updateCookies(config, headers, url) {
|
|
|
330
365
|
} else {
|
|
331
366
|
config.responseCookies = cookies;
|
|
332
367
|
}
|
|
368
|
+
if (!hookError && config.hooks?.afterCookie && config.hooks.afterCookie.length > 0) {
|
|
369
|
+
for (const hook of config.hooks.afterCookie) {
|
|
370
|
+
try {
|
|
371
|
+
await hook(acceptedCookies, config);
|
|
372
|
+
} catch (err) {
|
|
373
|
+
if (config.debug) {
|
|
374
|
+
console.log("[Rezo Debug] afterCookie hook error:", err);
|
|
375
|
+
}
|
|
376
|
+
}
|
|
377
|
+
}
|
|
378
|
+
}
|
|
333
379
|
}
|
|
334
380
|
function mergeRequestAndResponseCookies(config, responseCookies, url) {
|
|
335
381
|
const mergedCookiesArray = [];
|
|
@@ -571,11 +617,17 @@ async function executeHttp2Request(fetchOptions, config, options, perform, fs, s
|
|
|
571
617
|
throw response;
|
|
572
618
|
}
|
|
573
619
|
retries++;
|
|
620
|
+
const currentDelay = incrementDelay ? retryDelay * retries : retryDelay;
|
|
574
621
|
if (config.debug) {
|
|
575
|
-
console.log(`Retrying... ${retryDelay > 0 ? "in " +
|
|
622
|
+
console.log(`Retrying... ${retryDelay > 0 ? "in " + currentDelay + "ms" : ""}`);
|
|
623
|
+
}
|
|
624
|
+
if (config.hooks?.beforeRetry && config.hooks.beforeRetry.length > 0) {
|
|
625
|
+
for (const hook of config.hooks.beforeRetry) {
|
|
626
|
+
await hook(config, response, retries);
|
|
627
|
+
}
|
|
576
628
|
}
|
|
577
629
|
if (retryDelay > 0) {
|
|
578
|
-
await new Promise((resolve) => setTimeout(resolve,
|
|
630
|
+
await new Promise((resolve) => setTimeout(resolve, currentDelay));
|
|
579
631
|
}
|
|
580
632
|
}
|
|
581
633
|
config.retryAttempts++;
|
|
@@ -772,10 +824,22 @@ async function executeHttp2Stream(config, fetchOptions, requestCount, timing, _s
|
|
|
772
824
|
const isRedirect = status >= 300 && status < 400 && location;
|
|
773
825
|
if (isRedirect) {
|
|
774
826
|
_stats.statusOnNext = "redirect";
|
|
775
|
-
|
|
827
|
+
const redirectUrlObj = new URL(location, url);
|
|
828
|
+
if (!redirectUrlObj.hash && url.hash) {
|
|
829
|
+
redirectUrlObj.hash = url.hash;
|
|
830
|
+
}
|
|
831
|
+
_stats.redirectUrl = redirectUrlObj.href;
|
|
776
832
|
}
|
|
777
833
|
config.network.httpVersion = "h2";
|
|
778
|
-
|
|
834
|
+
(async () => {
|
|
835
|
+
try {
|
|
836
|
+
await updateCookies(config, headers, url.href);
|
|
837
|
+
} catch (err) {
|
|
838
|
+
if (config.debug) {
|
|
839
|
+
console.log("[Rezo Debug] Cookie hook error:", err);
|
|
840
|
+
}
|
|
841
|
+
}
|
|
842
|
+
})();
|
|
779
843
|
if (eventEmitter && !isRedirect) {
|
|
780
844
|
const headersEvent = {
|
|
781
845
|
status,
|
package/dist/adapters/http2.js
CHANGED
|
@@ -300,23 +300,58 @@ function sanitizeConfig(config) {
|
|
|
300
300
|
const { data: _data, ...sanitized } = config;
|
|
301
301
|
return sanitized;
|
|
302
302
|
}
|
|
303
|
-
function updateCookies(config, headers, url) {
|
|
303
|
+
async function updateCookies(config, headers, url) {
|
|
304
304
|
const setCookieHeaders = headers["set-cookie"];
|
|
305
305
|
if (!setCookieHeaders)
|
|
306
306
|
return;
|
|
307
307
|
const cookieHeaderArray = Array.isArray(setCookieHeaders) ? setCookieHeaders : [setCookieHeaders];
|
|
308
308
|
if (cookieHeaderArray.length === 0)
|
|
309
309
|
return;
|
|
310
|
+
const tempJar = new RezoCookieJar;
|
|
311
|
+
tempJar.setCookiesSync(cookieHeaderArray, url);
|
|
312
|
+
const parsedCookies = tempJar.cookies();
|
|
313
|
+
const acceptedCookies = [];
|
|
314
|
+
let hookError = null;
|
|
315
|
+
if (config.hooks?.beforeCookie && config.hooks.beforeCookie.length > 0) {
|
|
316
|
+
for (const cookie of parsedCookies.array) {
|
|
317
|
+
let shouldAccept = true;
|
|
318
|
+
for (const hook of config.hooks.beforeCookie) {
|
|
319
|
+
try {
|
|
320
|
+
const result = await hook({
|
|
321
|
+
cookie,
|
|
322
|
+
source: "response",
|
|
323
|
+
url,
|
|
324
|
+
isValid: true
|
|
325
|
+
}, config);
|
|
326
|
+
if (result === false) {
|
|
327
|
+
shouldAccept = false;
|
|
328
|
+
break;
|
|
329
|
+
}
|
|
330
|
+
} catch (err) {
|
|
331
|
+
hookError = err;
|
|
332
|
+
if (config.debug) {
|
|
333
|
+
console.log("[Rezo Debug] beforeCookie hook error:", err);
|
|
334
|
+
}
|
|
335
|
+
}
|
|
336
|
+
}
|
|
337
|
+
if (shouldAccept) {
|
|
338
|
+
acceptedCookies.push(cookie);
|
|
339
|
+
}
|
|
340
|
+
}
|
|
341
|
+
} else {
|
|
342
|
+
acceptedCookies.push(...parsedCookies.array);
|
|
343
|
+
}
|
|
344
|
+
const acceptedCookieStrings = acceptedCookies.map((c) => c.cookieString());
|
|
310
345
|
const jar = new RezoCookieJar;
|
|
311
|
-
jar.setCookiesSync(
|
|
346
|
+
jar.setCookiesSync(acceptedCookieStrings, url);
|
|
312
347
|
if (config.enableCookieJar && config.cookieJar) {
|
|
313
|
-
config.cookieJar.setCookiesSync(
|
|
348
|
+
config.cookieJar.setCookiesSync(acceptedCookieStrings, url);
|
|
314
349
|
}
|
|
315
350
|
const cookies = jar.cookies();
|
|
316
351
|
cookies.setCookiesString = cookieHeaderArray;
|
|
317
352
|
if (config.useCookies) {
|
|
318
353
|
const existingArray = config.responseCookies?.array || [];
|
|
319
|
-
for (const cookie of
|
|
354
|
+
for (const cookie of acceptedCookies) {
|
|
320
355
|
const existingIndex = existingArray.findIndex((c) => c.key === cookie.key && c.domain === cookie.domain);
|
|
321
356
|
if (existingIndex >= 0) {
|
|
322
357
|
existingArray[existingIndex] = cookie;
|
|
@@ -330,6 +365,17 @@ function updateCookies(config, headers, url) {
|
|
|
330
365
|
} else {
|
|
331
366
|
config.responseCookies = cookies;
|
|
332
367
|
}
|
|
368
|
+
if (!hookError && config.hooks?.afterCookie && config.hooks.afterCookie.length > 0) {
|
|
369
|
+
for (const hook of config.hooks.afterCookie) {
|
|
370
|
+
try {
|
|
371
|
+
await hook(acceptedCookies, config);
|
|
372
|
+
} catch (err) {
|
|
373
|
+
if (config.debug) {
|
|
374
|
+
console.log("[Rezo Debug] afterCookie hook error:", err);
|
|
375
|
+
}
|
|
376
|
+
}
|
|
377
|
+
}
|
|
378
|
+
}
|
|
333
379
|
}
|
|
334
380
|
function mergeRequestAndResponseCookies(config, responseCookies, url) {
|
|
335
381
|
const mergedCookiesArray = [];
|
|
@@ -571,11 +617,17 @@ async function executeHttp2Request(fetchOptions, config, options, perform, fs, s
|
|
|
571
617
|
throw response;
|
|
572
618
|
}
|
|
573
619
|
retries++;
|
|
620
|
+
const currentDelay = incrementDelay ? retryDelay * retries : retryDelay;
|
|
574
621
|
if (config.debug) {
|
|
575
|
-
console.log(`Retrying... ${retryDelay > 0 ? "in " +
|
|
622
|
+
console.log(`Retrying... ${retryDelay > 0 ? "in " + currentDelay + "ms" : ""}`);
|
|
623
|
+
}
|
|
624
|
+
if (config.hooks?.beforeRetry && config.hooks.beforeRetry.length > 0) {
|
|
625
|
+
for (const hook of config.hooks.beforeRetry) {
|
|
626
|
+
await hook(config, response, retries);
|
|
627
|
+
}
|
|
576
628
|
}
|
|
577
629
|
if (retryDelay > 0) {
|
|
578
|
-
await new Promise((resolve) => setTimeout(resolve,
|
|
630
|
+
await new Promise((resolve) => setTimeout(resolve, currentDelay));
|
|
579
631
|
}
|
|
580
632
|
}
|
|
581
633
|
config.retryAttempts++;
|
|
@@ -772,10 +824,22 @@ async function executeHttp2Stream(config, fetchOptions, requestCount, timing, _s
|
|
|
772
824
|
const isRedirect = status >= 300 && status < 400 && location;
|
|
773
825
|
if (isRedirect) {
|
|
774
826
|
_stats.statusOnNext = "redirect";
|
|
775
|
-
|
|
827
|
+
const redirectUrlObj = new URL(location, url);
|
|
828
|
+
if (!redirectUrlObj.hash && url.hash) {
|
|
829
|
+
redirectUrlObj.hash = url.hash;
|
|
830
|
+
}
|
|
831
|
+
_stats.redirectUrl = redirectUrlObj.href;
|
|
776
832
|
}
|
|
777
833
|
config.network.httpVersion = "h2";
|
|
778
|
-
|
|
834
|
+
(async () => {
|
|
835
|
+
try {
|
|
836
|
+
await updateCookies(config, headers, url.href);
|
|
837
|
+
} catch (err) {
|
|
838
|
+
if (config.debug) {
|
|
839
|
+
console.log("[Rezo Debug] Cookie hook error:", err);
|
|
840
|
+
}
|
|
841
|
+
}
|
|
842
|
+
})();
|
|
779
843
|
if (eventEmitter && !isRedirect) {
|
|
780
844
|
const headersEvent = {
|
|
781
845
|
status,
|
package/dist/adapters/index.cjs
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
|
-
const
|
|
2
|
-
exports.detectRuntime =
|
|
3
|
-
exports.getAdapterCapabilities =
|
|
4
|
-
exports.buildAdapterContext =
|
|
5
|
-
exports.getAvailableAdapters =
|
|
6
|
-
exports.selectAdapter =
|
|
1
|
+
const _mod_ncjrtt = require('./picker.cjs');
|
|
2
|
+
exports.detectRuntime = _mod_ncjrtt.detectRuntime;
|
|
3
|
+
exports.getAdapterCapabilities = _mod_ncjrtt.getAdapterCapabilities;
|
|
4
|
+
exports.buildAdapterContext = _mod_ncjrtt.buildAdapterContext;
|
|
5
|
+
exports.getAvailableAdapters = _mod_ncjrtt.getAvailableAdapters;
|
|
6
|
+
exports.selectAdapter = _mod_ncjrtt.selectAdapter;;
|
package/dist/cache/index.cjs
CHANGED
|
@@ -1,13 +1,13 @@
|
|
|
1
|
-
const
|
|
2
|
-
exports.LRUCache =
|
|
3
|
-
const
|
|
4
|
-
exports.DNSCache =
|
|
5
|
-
exports.getGlobalDNSCache =
|
|
6
|
-
exports.resetGlobalDNSCache =
|
|
7
|
-
const
|
|
8
|
-
exports.ResponseCache =
|
|
9
|
-
exports.normalizeResponseCacheConfig =
|
|
10
|
-
const
|
|
11
|
-
exports.FileCacher =
|
|
12
|
-
const
|
|
13
|
-
exports.UrlStore =
|
|
1
|
+
const _mod_l7ow4q = require('./lru-cache.cjs');
|
|
2
|
+
exports.LRUCache = _mod_l7ow4q.LRUCache;;
|
|
3
|
+
const _mod_tpblqv = require('./dns-cache.cjs');
|
|
4
|
+
exports.DNSCache = _mod_tpblqv.DNSCache;
|
|
5
|
+
exports.getGlobalDNSCache = _mod_tpblqv.getGlobalDNSCache;
|
|
6
|
+
exports.resetGlobalDNSCache = _mod_tpblqv.resetGlobalDNSCache;;
|
|
7
|
+
const _mod_4rqtce = require('./response-cache.cjs');
|
|
8
|
+
exports.ResponseCache = _mod_4rqtce.ResponseCache;
|
|
9
|
+
exports.normalizeResponseCacheConfig = _mod_4rqtce.normalizeResponseCacheConfig;;
|
|
10
|
+
const _mod_uyhauf = require('./file-cacher.cjs');
|
|
11
|
+
exports.FileCacher = _mod_uyhauf.FileCacher;;
|
|
12
|
+
const _mod_vrr530 = require('./url-store.cjs');
|
|
13
|
+
exports.UrlStore = _mod_vrr530.UrlStore;;
|
package/dist/core/rezo.cjs
CHANGED
|
@@ -328,13 +328,59 @@ class Rezo {
|
|
|
328
328
|
}
|
|
329
329
|
}
|
|
330
330
|
const executeWithHooks = async () => {
|
|
331
|
-
const
|
|
332
|
-
const
|
|
331
|
+
const startTime = Date.now();
|
|
332
|
+
const beforeRequestContext = {
|
|
333
|
+
retryCount: 0,
|
|
334
|
+
isRedirect: false,
|
|
335
|
+
redirectCount: 0,
|
|
336
|
+
startTime
|
|
337
|
+
};
|
|
338
|
+
if (requestHooks.beforeRequest.length > 0) {
|
|
339
|
+
for (const hook of requestHooks.beforeRequest) {
|
|
340
|
+
const result = await hook(options, beforeRequestContext);
|
|
341
|
+
if (result && typeof result === "object" && "status" in result) {
|
|
342
|
+
return result;
|
|
343
|
+
}
|
|
344
|
+
}
|
|
345
|
+
}
|
|
346
|
+
const mergedDefaults = {
|
|
347
|
+
...defaultOptions,
|
|
348
|
+
_proxyManager: this._proxyManager || null,
|
|
349
|
+
_hooks: requestHooks
|
|
350
|
+
};
|
|
351
|
+
let response;
|
|
352
|
+
try {
|
|
353
|
+
response = await this.adapter(options, mergedDefaults, jar);
|
|
354
|
+
} catch (error) {
|
|
355
|
+
if (requestHooks.beforeError.length > 0) {
|
|
356
|
+
let transformedError = error;
|
|
357
|
+
for (const hook of requestHooks.beforeError) {
|
|
358
|
+
transformedError = await hook(transformedError);
|
|
359
|
+
}
|
|
360
|
+
throw transformedError;
|
|
361
|
+
}
|
|
362
|
+
throw error;
|
|
363
|
+
}
|
|
333
364
|
if (jar.cookieFile) {
|
|
334
365
|
try {
|
|
335
366
|
jar.saveToFile();
|
|
336
367
|
} catch (e) {}
|
|
337
368
|
}
|
|
369
|
+
if (requestHooks.beforeCache.length > 0 && response && typeof response === "object" && "data" in response) {
|
|
370
|
+
const cacheEvent = {
|
|
371
|
+
url: fullUrl,
|
|
372
|
+
status: response.status,
|
|
373
|
+
headers: response.headers,
|
|
374
|
+
cacheKey: `${method}:${fullUrl}`,
|
|
375
|
+
isCacheable: ["GET", "HEAD"].includes(method) && response.status >= 200 && response.status < 300
|
|
376
|
+
};
|
|
377
|
+
for (const hook of requestHooks.beforeCache) {
|
|
378
|
+
const shouldCache = hook(cacheEvent);
|
|
379
|
+
if (shouldCache === false) {
|
|
380
|
+
return response;
|
|
381
|
+
}
|
|
382
|
+
}
|
|
383
|
+
}
|
|
338
384
|
if (this.responseCache && cacheMode !== "no-store" && response && typeof response === "object" && "data" in response && !(options._isStream || options._isDownload || options._isUpload)) {
|
|
339
385
|
this.responseCache.set(method, fullUrl, response, requestHeaders);
|
|
340
386
|
}
|