rezo 1.0.21 → 1.0.23
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/adapters/entries/curl.d.ts +9 -0
- package/dist/adapters/entries/fetch.d.ts +9 -0
- package/dist/adapters/entries/http.d.ts +9 -0
- package/dist/adapters/entries/http2.d.ts +9 -0
- package/dist/adapters/entries/react-native.d.ts +9 -0
- package/dist/adapters/entries/xhr.d.ts +9 -0
- package/dist/adapters/fetch.cjs +59 -6
- package/dist/adapters/fetch.js +59 -6
- package/dist/adapters/http.cjs +120 -8
- package/dist/adapters/http.js +120 -8
- package/dist/adapters/http2.cjs +69 -8
- package/dist/adapters/http2.js +69 -8
- package/dist/adapters/index.cjs +6 -6
- package/dist/cache/index.cjs +13 -13
- package/dist/core/rezo.cjs +48 -2
- package/dist/core/rezo.js +48 -2
- package/dist/crawler.d.ts +9 -0
- package/dist/entries/crawler.cjs +5 -5
- package/dist/errors/rezo-error.cjs +2 -2
- package/dist/errors/rezo-error.js +2 -2
- package/dist/index.cjs +24 -24
- package/dist/index.d.ts +9 -0
- package/dist/platform/browser.d.ts +9 -0
- package/dist/platform/bun.d.ts +9 -0
- package/dist/platform/deno.d.ts +9 -0
- package/dist/platform/node.d.ts +9 -0
- package/dist/platform/react-native.d.ts +9 -0
- package/dist/platform/worker.d.ts +9 -0
- package/dist/plugin/index.cjs +36 -36
- package/dist/proxy/index.cjs +2 -2
- package/dist/queue/index.cjs +8 -8
- package/dist/utils/http-config.cjs +3 -0
- package/dist/utils/http-config.js +3 -0
- package/package.json +1 -1
package/dist/adapters/http.js
CHANGED
|
@@ -386,6 +386,11 @@ async function executeHttp1Request(fetchOptions, config, options, perform, fs, s
|
|
|
386
386
|
retries++;
|
|
387
387
|
const currentDelay = incrementDelay ? retryDelay * retries : retryDelay;
|
|
388
388
|
debugLog.retry(config, retries, maxRetries, responseStatusCode, currentDelay);
|
|
389
|
+
if (config.hooks?.beforeRetry && config.hooks.beforeRetry.length > 0) {
|
|
390
|
+
for (const hook of config.hooks.beforeRetry) {
|
|
391
|
+
await hook(config, response, retries);
|
|
392
|
+
}
|
|
393
|
+
}
|
|
389
394
|
if (retryDelay > 0) {
|
|
390
395
|
await new Promise((resolve) => setTimeout(resolve, currentDelay));
|
|
391
396
|
}
|
|
@@ -427,7 +432,8 @@ async function executeHttp1Request(fetchOptions, config, options, perform, fs, s
|
|
|
427
432
|
}
|
|
428
433
|
const redirectCode = response.status;
|
|
429
434
|
const customHeaders = undefined;
|
|
430
|
-
const
|
|
435
|
+
const redirectCallback = config.beforeRedirect || config.onRedirect;
|
|
436
|
+
const onRedirect = redirectCallback ? redirectCallback({
|
|
431
437
|
url: new URL(_stats.redirectUrl),
|
|
432
438
|
status: response.status,
|
|
433
439
|
headers: response.headers,
|
|
@@ -573,7 +579,7 @@ async function request(config, fetchOptions, requestCount, timing, _stats, respo
|
|
|
573
579
|
const location = headers["location"] || headers["Location"];
|
|
574
580
|
const contentLength = headers["content-length"];
|
|
575
581
|
const cookies = headers["set-cookie"];
|
|
576
|
-
updateCookies(config, headers, url.href);
|
|
582
|
+
await updateCookies(config, headers, url.href);
|
|
577
583
|
const cookieArray = config.responseCookies?.array || [];
|
|
578
584
|
delete headers["set-cookie"];
|
|
579
585
|
_stats.redirectUrl = undefined;
|
|
@@ -876,13 +882,31 @@ async function request(config, fetchOptions, requestCount, timing, _stats, respo
|
|
|
876
882
|
}
|
|
877
883
|
timing.dnsStart = performance.now();
|
|
878
884
|
config.timing.domainLookupStart = timing.dnsStart;
|
|
879
|
-
socket.on("lookup", () => {
|
|
885
|
+
socket.on("lookup", (err, address, family) => {
|
|
880
886
|
if (!timing.dnsEnd) {
|
|
881
887
|
timing.dnsEnd = performance.now();
|
|
882
888
|
config.timing.domainLookupEnd = timing.dnsEnd;
|
|
883
889
|
timing.tcpStart = performance.now();
|
|
884
890
|
config.timing.connectStart = timing.tcpStart;
|
|
885
891
|
}
|
|
892
|
+
if (config.hooks?.onDns && config.hooks.onDns.length > 0) {
|
|
893
|
+
const familyNum = typeof family === "number" ? family : family === "IPv6" ? 6 : 4;
|
|
894
|
+
for (const hook of config.hooks.onDns) {
|
|
895
|
+
try {
|
|
896
|
+
hook({
|
|
897
|
+
hostname: url.hostname,
|
|
898
|
+
address: address || "",
|
|
899
|
+
family: familyNum,
|
|
900
|
+
duration: timing.dnsEnd - timing.dnsStart,
|
|
901
|
+
timestamp: Date.now()
|
|
902
|
+
}, config);
|
|
903
|
+
} catch (err) {
|
|
904
|
+
if (config.debug) {
|
|
905
|
+
console.log("[Rezo Debug] onDns hook error:", err);
|
|
906
|
+
}
|
|
907
|
+
}
|
|
908
|
+
}
|
|
909
|
+
}
|
|
886
910
|
});
|
|
887
911
|
socket.on("secureConnect", () => {
|
|
888
912
|
if (!timing.tlsEnd && timing.tlsStart) {
|
|
@@ -909,6 +933,31 @@ async function request(config, fetchOptions, requestCount, timing, _stats, respo
|
|
|
909
933
|
hostnameMatch: cert.subject?.CN === url.hostname,
|
|
910
934
|
chainValid: true
|
|
911
935
|
};
|
|
936
|
+
if (config.hooks?.onTls && config.hooks.onTls.length > 0) {
|
|
937
|
+
for (const hook of config.hooks.onTls) {
|
|
938
|
+
try {
|
|
939
|
+
hook({
|
|
940
|
+
protocol: tlsVersion,
|
|
941
|
+
cipher: cipher?.name || "",
|
|
942
|
+
authorized: !socket.authorizationError,
|
|
943
|
+
authorizationError: socket.authorizationError,
|
|
944
|
+
certificate: cert ? {
|
|
945
|
+
subject: cert.subject?.CN || "",
|
|
946
|
+
issuer: cert.issuer?.CN || "",
|
|
947
|
+
validFrom: cert.valid_from || "",
|
|
948
|
+
validTo: cert.valid_to || "",
|
|
949
|
+
fingerprint: cert.fingerprint || ""
|
|
950
|
+
} : undefined,
|
|
951
|
+
duration: timing.tlsEnd - timing.tlsStart,
|
|
952
|
+
timestamp: Date.now()
|
|
953
|
+
}, config);
|
|
954
|
+
} catch (err) {
|
|
955
|
+
if (config.debug) {
|
|
956
|
+
console.log("[Rezo Debug] onTls hook error:", err);
|
|
957
|
+
}
|
|
958
|
+
}
|
|
959
|
+
}
|
|
960
|
+
}
|
|
912
961
|
});
|
|
913
962
|
socket.on("connect", () => {
|
|
914
963
|
if (!timing.tcpEnd) {
|
|
@@ -927,6 +976,24 @@ async function request(config, fetchOptions, requestCount, timing, _stats, respo
|
|
|
927
976
|
config.network.localPort = localPort;
|
|
928
977
|
config.network.family = remoteFamily;
|
|
929
978
|
}
|
|
979
|
+
if (config.hooks?.onSocket && config.hooks.onSocket.length > 0) {
|
|
980
|
+
for (const hook of config.hooks.onSocket) {
|
|
981
|
+
try {
|
|
982
|
+
hook({
|
|
983
|
+
type: "connect",
|
|
984
|
+
localAddress,
|
|
985
|
+
localPort,
|
|
986
|
+
remoteAddress,
|
|
987
|
+
remotePort,
|
|
988
|
+
timestamp: Date.now()
|
|
989
|
+
}, socket);
|
|
990
|
+
} catch (err) {
|
|
991
|
+
if (config.debug) {
|
|
992
|
+
console.log("[Rezo Debug] onSocket hook error:", err);
|
|
993
|
+
}
|
|
994
|
+
}
|
|
995
|
+
}
|
|
996
|
+
}
|
|
930
997
|
});
|
|
931
998
|
});
|
|
932
999
|
req.on("error", (error) => {
|
|
@@ -1322,18 +1389,52 @@ function parseProxy(proxy, isScure = true, rejectUnauthorized = false) {
|
|
|
1322
1389
|
}
|
|
1323
1390
|
return rezoProxy(proxy);
|
|
1324
1391
|
}
|
|
1325
|
-
function updateCookies(config, headers, url) {
|
|
1392
|
+
async function updateCookies(config, headers, url) {
|
|
1326
1393
|
const cookies = headers["set-cookie"];
|
|
1327
1394
|
if (cookies) {
|
|
1328
1395
|
const jar = new RezoCookieJar;
|
|
1396
|
+
const tempJar = new RezoCookieJar;
|
|
1397
|
+
tempJar.setCookiesSync(cookies, url);
|
|
1398
|
+
const parsedCookies = tempJar.cookies();
|
|
1399
|
+
const acceptedCookies = [];
|
|
1400
|
+
let hookError = null;
|
|
1401
|
+
if (config.hooks?.beforeCookie && config.hooks.beforeCookie.length > 0) {
|
|
1402
|
+
for (const cookie of parsedCookies.array) {
|
|
1403
|
+
let shouldAccept = true;
|
|
1404
|
+
for (const hook of config.hooks.beforeCookie) {
|
|
1405
|
+
try {
|
|
1406
|
+
const result = await hook({
|
|
1407
|
+
cookie,
|
|
1408
|
+
source: "response",
|
|
1409
|
+
url,
|
|
1410
|
+
isValid: true
|
|
1411
|
+
}, config);
|
|
1412
|
+
if (result === false) {
|
|
1413
|
+
shouldAccept = false;
|
|
1414
|
+
break;
|
|
1415
|
+
}
|
|
1416
|
+
} catch (err) {
|
|
1417
|
+
hookError = err;
|
|
1418
|
+
if (config.debug) {
|
|
1419
|
+
console.log("[Rezo Debug] beforeCookie hook error:", err);
|
|
1420
|
+
}
|
|
1421
|
+
}
|
|
1422
|
+
}
|
|
1423
|
+
if (shouldAccept) {
|
|
1424
|
+
acceptedCookies.push(cookie);
|
|
1425
|
+
}
|
|
1426
|
+
}
|
|
1427
|
+
} else {
|
|
1428
|
+
acceptedCookies.push(...parsedCookies.array);
|
|
1429
|
+
}
|
|
1430
|
+
const acceptedCookieStrings = acceptedCookies.map((c) => c.cookieString());
|
|
1329
1431
|
if (config.enableCookieJar && config.cookieJar) {
|
|
1330
|
-
config.cookieJar.setCookiesSync(
|
|
1432
|
+
config.cookieJar.setCookiesSync(acceptedCookieStrings, url);
|
|
1331
1433
|
}
|
|
1332
|
-
jar.setCookiesSync(
|
|
1434
|
+
jar.setCookiesSync(acceptedCookieStrings, url);
|
|
1333
1435
|
if (config.useCookies) {
|
|
1334
|
-
const parsedCookies = jar.cookies();
|
|
1335
1436
|
const existingArray = config.responseCookies?.array || [];
|
|
1336
|
-
for (const cookie of
|
|
1437
|
+
for (const cookie of acceptedCookies) {
|
|
1337
1438
|
const existingIndex = existingArray.findIndex((c) => c.key === cookie.key && c.domain === cookie.domain);
|
|
1338
1439
|
if (existingIndex >= 0) {
|
|
1339
1440
|
existingArray[existingIndex] = cookie;
|
|
@@ -1344,5 +1445,16 @@ function updateCookies(config, headers, url) {
|
|
|
1344
1445
|
const mergedJar = new RezoCookieJar(existingArray, url);
|
|
1345
1446
|
config.responseCookies = mergedJar.cookies();
|
|
1346
1447
|
}
|
|
1448
|
+
if (!hookError && config.hooks?.afterCookie && config.hooks.afterCookie.length > 0) {
|
|
1449
|
+
for (const hook of config.hooks.afterCookie) {
|
|
1450
|
+
try {
|
|
1451
|
+
await hook(acceptedCookies, config);
|
|
1452
|
+
} catch (err) {
|
|
1453
|
+
if (config.debug) {
|
|
1454
|
+
console.log("[Rezo Debug] afterCookie hook error:", err);
|
|
1455
|
+
}
|
|
1456
|
+
}
|
|
1457
|
+
}
|
|
1458
|
+
}
|
|
1347
1459
|
}
|
|
1348
1460
|
}
|
package/dist/adapters/http2.cjs
CHANGED
|
@@ -300,23 +300,58 @@ function sanitizeConfig(config) {
|
|
|
300
300
|
const { data: _data, ...sanitized } = config;
|
|
301
301
|
return sanitized;
|
|
302
302
|
}
|
|
303
|
-
function updateCookies(config, headers, url) {
|
|
303
|
+
async function updateCookies(config, headers, url) {
|
|
304
304
|
const setCookieHeaders = headers["set-cookie"];
|
|
305
305
|
if (!setCookieHeaders)
|
|
306
306
|
return;
|
|
307
307
|
const cookieHeaderArray = Array.isArray(setCookieHeaders) ? setCookieHeaders : [setCookieHeaders];
|
|
308
308
|
if (cookieHeaderArray.length === 0)
|
|
309
309
|
return;
|
|
310
|
+
const tempJar = new RezoCookieJar;
|
|
311
|
+
tempJar.setCookiesSync(cookieHeaderArray, url);
|
|
312
|
+
const parsedCookies = tempJar.cookies();
|
|
313
|
+
const acceptedCookies = [];
|
|
314
|
+
let hookError = null;
|
|
315
|
+
if (config.hooks?.beforeCookie && config.hooks.beforeCookie.length > 0) {
|
|
316
|
+
for (const cookie of parsedCookies.array) {
|
|
317
|
+
let shouldAccept = true;
|
|
318
|
+
for (const hook of config.hooks.beforeCookie) {
|
|
319
|
+
try {
|
|
320
|
+
const result = await hook({
|
|
321
|
+
cookie,
|
|
322
|
+
source: "response",
|
|
323
|
+
url,
|
|
324
|
+
isValid: true
|
|
325
|
+
}, config);
|
|
326
|
+
if (result === false) {
|
|
327
|
+
shouldAccept = false;
|
|
328
|
+
break;
|
|
329
|
+
}
|
|
330
|
+
} catch (err) {
|
|
331
|
+
hookError = err;
|
|
332
|
+
if (config.debug) {
|
|
333
|
+
console.log("[Rezo Debug] beforeCookie hook error:", err);
|
|
334
|
+
}
|
|
335
|
+
}
|
|
336
|
+
}
|
|
337
|
+
if (shouldAccept) {
|
|
338
|
+
acceptedCookies.push(cookie);
|
|
339
|
+
}
|
|
340
|
+
}
|
|
341
|
+
} else {
|
|
342
|
+
acceptedCookies.push(...parsedCookies.array);
|
|
343
|
+
}
|
|
344
|
+
const acceptedCookieStrings = acceptedCookies.map((c) => c.cookieString());
|
|
310
345
|
const jar = new RezoCookieJar;
|
|
311
|
-
jar.setCookiesSync(
|
|
346
|
+
jar.setCookiesSync(acceptedCookieStrings, url);
|
|
312
347
|
if (config.enableCookieJar && config.cookieJar) {
|
|
313
|
-
config.cookieJar.setCookiesSync(
|
|
348
|
+
config.cookieJar.setCookiesSync(acceptedCookieStrings, url);
|
|
314
349
|
}
|
|
315
350
|
const cookies = jar.cookies();
|
|
316
351
|
cookies.setCookiesString = cookieHeaderArray;
|
|
317
352
|
if (config.useCookies) {
|
|
318
353
|
const existingArray = config.responseCookies?.array || [];
|
|
319
|
-
for (const cookie of
|
|
354
|
+
for (const cookie of acceptedCookies) {
|
|
320
355
|
const existingIndex = existingArray.findIndex((c) => c.key === cookie.key && c.domain === cookie.domain);
|
|
321
356
|
if (existingIndex >= 0) {
|
|
322
357
|
existingArray[existingIndex] = cookie;
|
|
@@ -330,6 +365,17 @@ function updateCookies(config, headers, url) {
|
|
|
330
365
|
} else {
|
|
331
366
|
config.responseCookies = cookies;
|
|
332
367
|
}
|
|
368
|
+
if (!hookError && config.hooks?.afterCookie && config.hooks.afterCookie.length > 0) {
|
|
369
|
+
for (const hook of config.hooks.afterCookie) {
|
|
370
|
+
try {
|
|
371
|
+
await hook(acceptedCookies, config);
|
|
372
|
+
} catch (err) {
|
|
373
|
+
if (config.debug) {
|
|
374
|
+
console.log("[Rezo Debug] afterCookie hook error:", err);
|
|
375
|
+
}
|
|
376
|
+
}
|
|
377
|
+
}
|
|
378
|
+
}
|
|
333
379
|
}
|
|
334
380
|
function mergeRequestAndResponseCookies(config, responseCookies, url) {
|
|
335
381
|
const mergedCookiesArray = [];
|
|
@@ -571,11 +617,17 @@ async function executeHttp2Request(fetchOptions, config, options, perform, fs, s
|
|
|
571
617
|
throw response;
|
|
572
618
|
}
|
|
573
619
|
retries++;
|
|
620
|
+
const currentDelay = incrementDelay ? retryDelay * retries : retryDelay;
|
|
574
621
|
if (config.debug) {
|
|
575
|
-
console.log(`Retrying... ${retryDelay > 0 ? "in " +
|
|
622
|
+
console.log(`Retrying... ${retryDelay > 0 ? "in " + currentDelay + "ms" : ""}`);
|
|
623
|
+
}
|
|
624
|
+
if (config.hooks?.beforeRetry && config.hooks.beforeRetry.length > 0) {
|
|
625
|
+
for (const hook of config.hooks.beforeRetry) {
|
|
626
|
+
await hook(config, response, retries);
|
|
627
|
+
}
|
|
576
628
|
}
|
|
577
629
|
if (retryDelay > 0) {
|
|
578
|
-
await new Promise((resolve) => setTimeout(resolve,
|
|
630
|
+
await new Promise((resolve) => setTimeout(resolve, currentDelay));
|
|
579
631
|
}
|
|
580
632
|
}
|
|
581
633
|
config.retryAttempts++;
|
|
@@ -626,7 +678,8 @@ async function executeHttp2Request(fetchOptions, config, options, perform, fs, s
|
|
|
626
678
|
visitedUrls.add(normalizedRedirectUrl);
|
|
627
679
|
}
|
|
628
680
|
const redirectCode = response.status;
|
|
629
|
-
const
|
|
681
|
+
const redirectCallback = config.beforeRedirect || config.onRedirect;
|
|
682
|
+
const onRedirect = redirectCallback ? redirectCallback({
|
|
630
683
|
url: new URL(location),
|
|
631
684
|
status: response.status,
|
|
632
685
|
headers: response.headers,
|
|
@@ -774,7 +827,15 @@ async function executeHttp2Stream(config, fetchOptions, requestCount, timing, _s
|
|
|
774
827
|
_stats.redirectUrl = new URL(location, url).href;
|
|
775
828
|
}
|
|
776
829
|
config.network.httpVersion = "h2";
|
|
777
|
-
|
|
830
|
+
(async () => {
|
|
831
|
+
try {
|
|
832
|
+
await updateCookies(config, headers, url.href);
|
|
833
|
+
} catch (err) {
|
|
834
|
+
if (config.debug) {
|
|
835
|
+
console.log("[Rezo Debug] Cookie hook error:", err);
|
|
836
|
+
}
|
|
837
|
+
}
|
|
838
|
+
})();
|
|
778
839
|
if (eventEmitter && !isRedirect) {
|
|
779
840
|
const headersEvent = {
|
|
780
841
|
status,
|
package/dist/adapters/http2.js
CHANGED
|
@@ -300,23 +300,58 @@ function sanitizeConfig(config) {
|
|
|
300
300
|
const { data: _data, ...sanitized } = config;
|
|
301
301
|
return sanitized;
|
|
302
302
|
}
|
|
303
|
-
function updateCookies(config, headers, url) {
|
|
303
|
+
async function updateCookies(config, headers, url) {
|
|
304
304
|
const setCookieHeaders = headers["set-cookie"];
|
|
305
305
|
if (!setCookieHeaders)
|
|
306
306
|
return;
|
|
307
307
|
const cookieHeaderArray = Array.isArray(setCookieHeaders) ? setCookieHeaders : [setCookieHeaders];
|
|
308
308
|
if (cookieHeaderArray.length === 0)
|
|
309
309
|
return;
|
|
310
|
+
const tempJar = new RezoCookieJar;
|
|
311
|
+
tempJar.setCookiesSync(cookieHeaderArray, url);
|
|
312
|
+
const parsedCookies = tempJar.cookies();
|
|
313
|
+
const acceptedCookies = [];
|
|
314
|
+
let hookError = null;
|
|
315
|
+
if (config.hooks?.beforeCookie && config.hooks.beforeCookie.length > 0) {
|
|
316
|
+
for (const cookie of parsedCookies.array) {
|
|
317
|
+
let shouldAccept = true;
|
|
318
|
+
for (const hook of config.hooks.beforeCookie) {
|
|
319
|
+
try {
|
|
320
|
+
const result = await hook({
|
|
321
|
+
cookie,
|
|
322
|
+
source: "response",
|
|
323
|
+
url,
|
|
324
|
+
isValid: true
|
|
325
|
+
}, config);
|
|
326
|
+
if (result === false) {
|
|
327
|
+
shouldAccept = false;
|
|
328
|
+
break;
|
|
329
|
+
}
|
|
330
|
+
} catch (err) {
|
|
331
|
+
hookError = err;
|
|
332
|
+
if (config.debug) {
|
|
333
|
+
console.log("[Rezo Debug] beforeCookie hook error:", err);
|
|
334
|
+
}
|
|
335
|
+
}
|
|
336
|
+
}
|
|
337
|
+
if (shouldAccept) {
|
|
338
|
+
acceptedCookies.push(cookie);
|
|
339
|
+
}
|
|
340
|
+
}
|
|
341
|
+
} else {
|
|
342
|
+
acceptedCookies.push(...parsedCookies.array);
|
|
343
|
+
}
|
|
344
|
+
const acceptedCookieStrings = acceptedCookies.map((c) => c.cookieString());
|
|
310
345
|
const jar = new RezoCookieJar;
|
|
311
|
-
jar.setCookiesSync(
|
|
346
|
+
jar.setCookiesSync(acceptedCookieStrings, url);
|
|
312
347
|
if (config.enableCookieJar && config.cookieJar) {
|
|
313
|
-
config.cookieJar.setCookiesSync(
|
|
348
|
+
config.cookieJar.setCookiesSync(acceptedCookieStrings, url);
|
|
314
349
|
}
|
|
315
350
|
const cookies = jar.cookies();
|
|
316
351
|
cookies.setCookiesString = cookieHeaderArray;
|
|
317
352
|
if (config.useCookies) {
|
|
318
353
|
const existingArray = config.responseCookies?.array || [];
|
|
319
|
-
for (const cookie of
|
|
354
|
+
for (const cookie of acceptedCookies) {
|
|
320
355
|
const existingIndex = existingArray.findIndex((c) => c.key === cookie.key && c.domain === cookie.domain);
|
|
321
356
|
if (existingIndex >= 0) {
|
|
322
357
|
existingArray[existingIndex] = cookie;
|
|
@@ -330,6 +365,17 @@ function updateCookies(config, headers, url) {
|
|
|
330
365
|
} else {
|
|
331
366
|
config.responseCookies = cookies;
|
|
332
367
|
}
|
|
368
|
+
if (!hookError && config.hooks?.afterCookie && config.hooks.afterCookie.length > 0) {
|
|
369
|
+
for (const hook of config.hooks.afterCookie) {
|
|
370
|
+
try {
|
|
371
|
+
await hook(acceptedCookies, config);
|
|
372
|
+
} catch (err) {
|
|
373
|
+
if (config.debug) {
|
|
374
|
+
console.log("[Rezo Debug] afterCookie hook error:", err);
|
|
375
|
+
}
|
|
376
|
+
}
|
|
377
|
+
}
|
|
378
|
+
}
|
|
333
379
|
}
|
|
334
380
|
function mergeRequestAndResponseCookies(config, responseCookies, url) {
|
|
335
381
|
const mergedCookiesArray = [];
|
|
@@ -571,11 +617,17 @@ async function executeHttp2Request(fetchOptions, config, options, perform, fs, s
|
|
|
571
617
|
throw response;
|
|
572
618
|
}
|
|
573
619
|
retries++;
|
|
620
|
+
const currentDelay = incrementDelay ? retryDelay * retries : retryDelay;
|
|
574
621
|
if (config.debug) {
|
|
575
|
-
console.log(`Retrying... ${retryDelay > 0 ? "in " +
|
|
622
|
+
console.log(`Retrying... ${retryDelay > 0 ? "in " + currentDelay + "ms" : ""}`);
|
|
623
|
+
}
|
|
624
|
+
if (config.hooks?.beforeRetry && config.hooks.beforeRetry.length > 0) {
|
|
625
|
+
for (const hook of config.hooks.beforeRetry) {
|
|
626
|
+
await hook(config, response, retries);
|
|
627
|
+
}
|
|
576
628
|
}
|
|
577
629
|
if (retryDelay > 0) {
|
|
578
|
-
await new Promise((resolve) => setTimeout(resolve,
|
|
630
|
+
await new Promise((resolve) => setTimeout(resolve, currentDelay));
|
|
579
631
|
}
|
|
580
632
|
}
|
|
581
633
|
config.retryAttempts++;
|
|
@@ -626,7 +678,8 @@ async function executeHttp2Request(fetchOptions, config, options, perform, fs, s
|
|
|
626
678
|
visitedUrls.add(normalizedRedirectUrl);
|
|
627
679
|
}
|
|
628
680
|
const redirectCode = response.status;
|
|
629
|
-
const
|
|
681
|
+
const redirectCallback = config.beforeRedirect || config.onRedirect;
|
|
682
|
+
const onRedirect = redirectCallback ? redirectCallback({
|
|
630
683
|
url: new URL(location),
|
|
631
684
|
status: response.status,
|
|
632
685
|
headers: response.headers,
|
|
@@ -774,7 +827,15 @@ async function executeHttp2Stream(config, fetchOptions, requestCount, timing, _s
|
|
|
774
827
|
_stats.redirectUrl = new URL(location, url).href;
|
|
775
828
|
}
|
|
776
829
|
config.network.httpVersion = "h2";
|
|
777
|
-
|
|
830
|
+
(async () => {
|
|
831
|
+
try {
|
|
832
|
+
await updateCookies(config, headers, url.href);
|
|
833
|
+
} catch (err) {
|
|
834
|
+
if (config.debug) {
|
|
835
|
+
console.log("[Rezo Debug] Cookie hook error:", err);
|
|
836
|
+
}
|
|
837
|
+
}
|
|
838
|
+
})();
|
|
778
839
|
if (eventEmitter && !isRedirect) {
|
|
779
840
|
const headersEvent = {
|
|
780
841
|
status,
|
package/dist/adapters/index.cjs
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
|
-
const
|
|
2
|
-
exports.detectRuntime =
|
|
3
|
-
exports.getAdapterCapabilities =
|
|
4
|
-
exports.buildAdapterContext =
|
|
5
|
-
exports.getAvailableAdapters =
|
|
6
|
-
exports.selectAdapter =
|
|
1
|
+
const _mod_nbovud = require('./picker.cjs');
|
|
2
|
+
exports.detectRuntime = _mod_nbovud.detectRuntime;
|
|
3
|
+
exports.getAdapterCapabilities = _mod_nbovud.getAdapterCapabilities;
|
|
4
|
+
exports.buildAdapterContext = _mod_nbovud.buildAdapterContext;
|
|
5
|
+
exports.getAvailableAdapters = _mod_nbovud.getAvailableAdapters;
|
|
6
|
+
exports.selectAdapter = _mod_nbovud.selectAdapter;;
|
package/dist/cache/index.cjs
CHANGED
|
@@ -1,13 +1,13 @@
|
|
|
1
|
-
const
|
|
2
|
-
exports.LRUCache =
|
|
3
|
-
const
|
|
4
|
-
exports.DNSCache =
|
|
5
|
-
exports.getGlobalDNSCache =
|
|
6
|
-
exports.resetGlobalDNSCache =
|
|
7
|
-
const
|
|
8
|
-
exports.ResponseCache =
|
|
9
|
-
exports.normalizeResponseCacheConfig =
|
|
10
|
-
const
|
|
11
|
-
exports.FileCacher =
|
|
12
|
-
const
|
|
13
|
-
exports.UrlStore =
|
|
1
|
+
const _mod_gcibom = require('./lru-cache.cjs');
|
|
2
|
+
exports.LRUCache = _mod_gcibom.LRUCache;;
|
|
3
|
+
const _mod_1nnd9l = require('./dns-cache.cjs');
|
|
4
|
+
exports.DNSCache = _mod_1nnd9l.DNSCache;
|
|
5
|
+
exports.getGlobalDNSCache = _mod_1nnd9l.getGlobalDNSCache;
|
|
6
|
+
exports.resetGlobalDNSCache = _mod_1nnd9l.resetGlobalDNSCache;;
|
|
7
|
+
const _mod_lmofxo = require('./response-cache.cjs');
|
|
8
|
+
exports.ResponseCache = _mod_lmofxo.ResponseCache;
|
|
9
|
+
exports.normalizeResponseCacheConfig = _mod_lmofxo.normalizeResponseCacheConfig;;
|
|
10
|
+
const _mod_xgo0i6 = require('./file-cacher.cjs');
|
|
11
|
+
exports.FileCacher = _mod_xgo0i6.FileCacher;;
|
|
12
|
+
const _mod_9ouiu9 = require('./url-store.cjs');
|
|
13
|
+
exports.UrlStore = _mod_9ouiu9.UrlStore;;
|
package/dist/core/rezo.cjs
CHANGED
|
@@ -328,13 +328,59 @@ class Rezo {
|
|
|
328
328
|
}
|
|
329
329
|
}
|
|
330
330
|
const executeWithHooks = async () => {
|
|
331
|
-
const
|
|
332
|
-
const
|
|
331
|
+
const startTime = Date.now();
|
|
332
|
+
const beforeRequestContext = {
|
|
333
|
+
retryCount: 0,
|
|
334
|
+
isRedirect: false,
|
|
335
|
+
redirectCount: 0,
|
|
336
|
+
startTime
|
|
337
|
+
};
|
|
338
|
+
if (requestHooks.beforeRequest.length > 0) {
|
|
339
|
+
for (const hook of requestHooks.beforeRequest) {
|
|
340
|
+
const result = await hook(options, beforeRequestContext);
|
|
341
|
+
if (result && typeof result === "object" && "status" in result) {
|
|
342
|
+
return result;
|
|
343
|
+
}
|
|
344
|
+
}
|
|
345
|
+
}
|
|
346
|
+
const mergedDefaults = {
|
|
347
|
+
...defaultOptions,
|
|
348
|
+
_proxyManager: this._proxyManager || null,
|
|
349
|
+
_hooks: requestHooks
|
|
350
|
+
};
|
|
351
|
+
let response;
|
|
352
|
+
try {
|
|
353
|
+
response = await this.adapter(options, mergedDefaults, jar);
|
|
354
|
+
} catch (error) {
|
|
355
|
+
if (requestHooks.beforeError.length > 0) {
|
|
356
|
+
let transformedError = error;
|
|
357
|
+
for (const hook of requestHooks.beforeError) {
|
|
358
|
+
transformedError = await hook(transformedError);
|
|
359
|
+
}
|
|
360
|
+
throw transformedError;
|
|
361
|
+
}
|
|
362
|
+
throw error;
|
|
363
|
+
}
|
|
333
364
|
if (jar.cookieFile) {
|
|
334
365
|
try {
|
|
335
366
|
jar.saveToFile();
|
|
336
367
|
} catch (e) {}
|
|
337
368
|
}
|
|
369
|
+
if (requestHooks.beforeCache.length > 0 && response && typeof response === "object" && "data" in response) {
|
|
370
|
+
const cacheEvent = {
|
|
371
|
+
url: fullUrl,
|
|
372
|
+
status: response.status,
|
|
373
|
+
headers: response.headers,
|
|
374
|
+
cacheKey: `${method}:${fullUrl}`,
|
|
375
|
+
isCacheable: ["GET", "HEAD"].includes(method) && response.status >= 200 && response.status < 300
|
|
376
|
+
};
|
|
377
|
+
for (const hook of requestHooks.beforeCache) {
|
|
378
|
+
const shouldCache = hook(cacheEvent);
|
|
379
|
+
if (shouldCache === false) {
|
|
380
|
+
return response;
|
|
381
|
+
}
|
|
382
|
+
}
|
|
383
|
+
}
|
|
338
384
|
if (this.responseCache && cacheMode !== "no-store" && response && typeof response === "object" && "data" in response && !(options._isStream || options._isDownload || options._isUpload)) {
|
|
339
385
|
this.responseCache.set(method, fullUrl, response, requestHeaders);
|
|
340
386
|
}
|
package/dist/core/rezo.js
CHANGED
|
@@ -328,13 +328,59 @@ export class Rezo {
|
|
|
328
328
|
}
|
|
329
329
|
}
|
|
330
330
|
const executeWithHooks = async () => {
|
|
331
|
-
const
|
|
332
|
-
const
|
|
331
|
+
const startTime = Date.now();
|
|
332
|
+
const beforeRequestContext = {
|
|
333
|
+
retryCount: 0,
|
|
334
|
+
isRedirect: false,
|
|
335
|
+
redirectCount: 0,
|
|
336
|
+
startTime
|
|
337
|
+
};
|
|
338
|
+
if (requestHooks.beforeRequest.length > 0) {
|
|
339
|
+
for (const hook of requestHooks.beforeRequest) {
|
|
340
|
+
const result = await hook(options, beforeRequestContext);
|
|
341
|
+
if (result && typeof result === "object" && "status" in result) {
|
|
342
|
+
return result;
|
|
343
|
+
}
|
|
344
|
+
}
|
|
345
|
+
}
|
|
346
|
+
const mergedDefaults = {
|
|
347
|
+
...defaultOptions,
|
|
348
|
+
_proxyManager: this._proxyManager || null,
|
|
349
|
+
_hooks: requestHooks
|
|
350
|
+
};
|
|
351
|
+
let response;
|
|
352
|
+
try {
|
|
353
|
+
response = await this.adapter(options, mergedDefaults, jar);
|
|
354
|
+
} catch (error) {
|
|
355
|
+
if (requestHooks.beforeError.length > 0) {
|
|
356
|
+
let transformedError = error;
|
|
357
|
+
for (const hook of requestHooks.beforeError) {
|
|
358
|
+
transformedError = await hook(transformedError);
|
|
359
|
+
}
|
|
360
|
+
throw transformedError;
|
|
361
|
+
}
|
|
362
|
+
throw error;
|
|
363
|
+
}
|
|
333
364
|
if (jar.cookieFile) {
|
|
334
365
|
try {
|
|
335
366
|
jar.saveToFile();
|
|
336
367
|
} catch (e) {}
|
|
337
368
|
}
|
|
369
|
+
if (requestHooks.beforeCache.length > 0 && response && typeof response === "object" && "data" in response) {
|
|
370
|
+
const cacheEvent = {
|
|
371
|
+
url: fullUrl,
|
|
372
|
+
status: response.status,
|
|
373
|
+
headers: response.headers,
|
|
374
|
+
cacheKey: `${method}:${fullUrl}`,
|
|
375
|
+
isCacheable: ["GET", "HEAD"].includes(method) && response.status >= 200 && response.status < 300
|
|
376
|
+
};
|
|
377
|
+
for (const hook of requestHooks.beforeCache) {
|
|
378
|
+
const shouldCache = hook(cacheEvent);
|
|
379
|
+
if (shouldCache === false) {
|
|
380
|
+
return response;
|
|
381
|
+
}
|
|
382
|
+
}
|
|
383
|
+
}
|
|
338
384
|
if (this.responseCache && cacheMode !== "no-store" && response && typeof response === "object" && "data" in response && !(options._isStream || options._isDownload || options._isUpload)) {
|
|
339
385
|
this.responseCache.set(method, fullUrl, response, requestHeaders);
|
|
340
386
|
}
|
package/dist/crawler.d.ts
CHANGED
|
@@ -2148,6 +2148,8 @@ export interface RezoConfig {
|
|
|
2148
2148
|
* ```
|
|
2149
2149
|
*/
|
|
2150
2150
|
beforeRedirect?: RezoRequestConfig["beforeRedirect"];
|
|
2151
|
+
/** Alias for beforeRedirect */
|
|
2152
|
+
onRedirect?: RezoRequestConfig["beforeRedirect"];
|
|
2151
2153
|
/** Character encoding for request body and response data */
|
|
2152
2154
|
encoding?: BufferEncoding;
|
|
2153
2155
|
/**
|
|
@@ -2802,6 +2804,11 @@ export interface RezoRequestConfig<D = any> {
|
|
|
2802
2804
|
* ```
|
|
2803
2805
|
*/
|
|
2804
2806
|
beforeRedirect?: (options: OnRedirectOptions) => OnRedirectResponse;
|
|
2807
|
+
/**
|
|
2808
|
+
* Alias for beforeRedirect - callback invoked when a redirect response is received.
|
|
2809
|
+
* @see beforeRedirect
|
|
2810
|
+
*/
|
|
2811
|
+
onRedirect?: (options: OnRedirectOptions) => OnRedirectResponse;
|
|
2805
2812
|
/** Whether to send cookies and authorization headers with cross-origin requests */
|
|
2806
2813
|
withCredentials?: boolean;
|
|
2807
2814
|
/** Proxy configuration (URL string or detailed options) */
|
|
@@ -3351,6 +3358,8 @@ export interface RezoDefaultOptions {
|
|
|
3351
3358
|
* ```
|
|
3352
3359
|
*/
|
|
3353
3360
|
beforeRedirect?: RezoHttpRequest["beforeRedirect"];
|
|
3361
|
+
/** Alias for beforeRedirect */
|
|
3362
|
+
onRedirect?: RezoHttpRequest["onRedirect"];
|
|
3354
3363
|
/** Array of functions to transform request data */
|
|
3355
3364
|
transformRequest?: RezoHttpRequest["transformRequest"];
|
|
3356
3365
|
/** Array of functions to transform response data */
|
package/dist/entries/crawler.cjs
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
const
|
|
2
|
-
exports.Crawler =
|
|
3
|
-
const
|
|
4
|
-
exports.CrawlerOptions =
|
|
5
|
-
exports.Domain =
|
|
1
|
+
const _mod_j6t9ay = require('../plugin/crawler.cjs');
|
|
2
|
+
exports.Crawler = _mod_j6t9ay.Crawler;;
|
|
3
|
+
const _mod_jqrm0t = require('../plugin/crawler-options.cjs');
|
|
4
|
+
exports.CrawlerOptions = _mod_jqrm0t.CrawlerOptions;
|
|
5
|
+
exports.Domain = _mod_jqrm0t.Domain;;
|
|
@@ -585,8 +585,8 @@ class RezoError extends Error {
|
|
|
585
585
|
Object.defineProperty(this, "suggestion", { value: "Check the error for more information.", enumerable: false });
|
|
586
586
|
}
|
|
587
587
|
if (response) {
|
|
588
|
-
Object.defineProperty(this, "status", { value: response.status, enumerable: false });
|
|
589
|
-
Object.defineProperty(this, "statusText", { value: response.statusText, enumerable: false });
|
|
588
|
+
Object.defineProperty(this, "status", { value: response.status, enumerable: false, configurable: true });
|
|
589
|
+
Object.defineProperty(this, "statusText", { value: response.statusText, enumerable: false, configurable: true });
|
|
590
590
|
}
|
|
591
591
|
this.name = this.constructor.name;
|
|
592
592
|
Object.setPrototypeOf(this, RezoError.prototype);
|