rezo 1.0.41 → 1.0.42
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/adapters/curl.cjs +12 -3
- package/dist/adapters/curl.js +12 -3
- package/dist/adapters/fetch.cjs +98 -12
- package/dist/adapters/fetch.js +98 -12
- package/dist/adapters/http.cjs +26 -14
- package/dist/adapters/http.js +26 -14
- package/dist/adapters/http2.cjs +553 -211
- package/dist/adapters/http2.js +553 -211
- package/dist/adapters/index.cjs +6 -6
- package/dist/adapters/xhr.cjs +94 -2
- package/dist/adapters/xhr.js +94 -2
- package/dist/cache/dns-cache.cjs +5 -3
- package/dist/cache/dns-cache.js +5 -3
- package/dist/cache/index.cjs +13 -13
- package/dist/entries/crawler.cjs +5 -5
- package/dist/index.cjs +24 -24
- package/dist/plugin/index.cjs +36 -36
- package/dist/proxy/index.cjs +4 -4
- package/dist/queue/index.cjs +8 -8
- package/dist/responses/universal/index.cjs +11 -11
- package/dist/utils/agent-pool.cjs +1 -17
- package/dist/utils/agent-pool.js +1 -17
- package/package.json +1 -1
package/dist/adapters/curl.cjs
CHANGED
|
@@ -195,7 +195,7 @@ function buildUrlTree(config, finalUrl) {
|
|
|
195
195
|
}
|
|
196
196
|
return urls.length > 0 ? urls : [finalUrl];
|
|
197
197
|
}
|
|
198
|
-
async function updateCookies(config, cookieStrings, url) {
|
|
198
|
+
async function updateCookies(config, cookieStrings, url, rootJar) {
|
|
199
199
|
if (!cookieStrings || cookieStrings.length === 0)
|
|
200
200
|
return;
|
|
201
201
|
const tempJar = new RezoCookieJar;
|
|
@@ -233,8 +233,9 @@ async function updateCookies(config, cookieStrings, url) {
|
|
|
233
233
|
acceptedCookies.push(...parsedCookies.array);
|
|
234
234
|
}
|
|
235
235
|
const acceptedCookieStrings = acceptedCookies.map((c) => c.toSetCookieString());
|
|
236
|
-
|
|
237
|
-
|
|
236
|
+
const jarToUpdate = rootJar || config.cookieJar;
|
|
237
|
+
if (!config.disableCookieJar && jarToUpdate) {
|
|
238
|
+
jarToUpdate.setCookiesSync(acceptedCookieStrings, url);
|
|
238
239
|
}
|
|
239
240
|
if (config.useCookies) {
|
|
240
241
|
const existingArray = config.responseCookies?.array || [];
|
|
@@ -2148,6 +2149,9 @@ async function executeRequest(options, defaultOptions, jar) {
|
|
|
2148
2149
|
}
|
|
2149
2150
|
}
|
|
2150
2151
|
}
|
|
2152
|
+
if (!config.requestId) {
|
|
2153
|
+
config.requestId = `req_${Date.now()}_${Math.random().toString(36).substring(2, 11)}`;
|
|
2154
|
+
}
|
|
2151
2155
|
debugLog.requestStart(config, requestUrl, method);
|
|
2152
2156
|
let streamResponse;
|
|
2153
2157
|
let downloadResponse;
|
|
@@ -2201,6 +2205,11 @@ async function executeRequest(options, defaultOptions, jar) {
|
|
|
2201
2205
|
const duration = perform.now();
|
|
2202
2206
|
debugLog.response(config, response.status, response.statusText, duration);
|
|
2203
2207
|
debugLog.cookies(config, response.cookies?.array?.length || 0);
|
|
2208
|
+
if (response.cookies?.setCookiesString?.length > 0 && jar) {
|
|
2209
|
+
try {
|
|
2210
|
+
jar.setCookiesSync(response.cookies.setCookiesString, response.finalUrl || requestUrl);
|
|
2211
|
+
} catch (e) {}
|
|
2212
|
+
}
|
|
2204
2213
|
if (cache) {
|
|
2205
2214
|
if (response.status === 304 && cachedEntry) {
|
|
2206
2215
|
const responseHeaders = response.headers instanceof RezoHeaders ? Object.fromEntries(response.headers.entries()) : response.headers;
|
package/dist/adapters/curl.js
CHANGED
|
@@ -195,7 +195,7 @@ function buildUrlTree(config, finalUrl) {
|
|
|
195
195
|
}
|
|
196
196
|
return urls.length > 0 ? urls : [finalUrl];
|
|
197
197
|
}
|
|
198
|
-
async function updateCookies(config, cookieStrings, url) {
|
|
198
|
+
async function updateCookies(config, cookieStrings, url, rootJar) {
|
|
199
199
|
if (!cookieStrings || cookieStrings.length === 0)
|
|
200
200
|
return;
|
|
201
201
|
const tempJar = new RezoCookieJar;
|
|
@@ -233,8 +233,9 @@ async function updateCookies(config, cookieStrings, url) {
|
|
|
233
233
|
acceptedCookies.push(...parsedCookies.array);
|
|
234
234
|
}
|
|
235
235
|
const acceptedCookieStrings = acceptedCookies.map((c) => c.toSetCookieString());
|
|
236
|
-
|
|
237
|
-
|
|
236
|
+
const jarToUpdate = rootJar || config.cookieJar;
|
|
237
|
+
if (!config.disableCookieJar && jarToUpdate) {
|
|
238
|
+
jarToUpdate.setCookiesSync(acceptedCookieStrings, url);
|
|
238
239
|
}
|
|
239
240
|
if (config.useCookies) {
|
|
240
241
|
const existingArray = config.responseCookies?.array || [];
|
|
@@ -2148,6 +2149,9 @@ export async function executeRequest(options, defaultOptions, jar) {
|
|
|
2148
2149
|
}
|
|
2149
2150
|
}
|
|
2150
2151
|
}
|
|
2152
|
+
if (!config.requestId) {
|
|
2153
|
+
config.requestId = `req_${Date.now()}_${Math.random().toString(36).substring(2, 11)}`;
|
|
2154
|
+
}
|
|
2151
2155
|
debugLog.requestStart(config, requestUrl, method);
|
|
2152
2156
|
let streamResponse;
|
|
2153
2157
|
let downloadResponse;
|
|
@@ -2201,6 +2205,11 @@ export async function executeRequest(options, defaultOptions, jar) {
|
|
|
2201
2205
|
const duration = perform.now();
|
|
2202
2206
|
debugLog.response(config, response.status, response.statusText, duration);
|
|
2203
2207
|
debugLog.cookies(config, response.cookies?.array?.length || 0);
|
|
2208
|
+
if (response.cookies?.setCookiesString?.length > 0 && jar) {
|
|
2209
|
+
try {
|
|
2210
|
+
jar.setCookiesSync(response.cookies.setCookiesString, response.finalUrl || requestUrl);
|
|
2211
|
+
} catch (e) {}
|
|
2212
|
+
}
|
|
2204
2213
|
if (cache) {
|
|
2205
2214
|
if (response.status === 304 && cachedEntry) {
|
|
2206
2215
|
const responseHeaders = response.headers instanceof RezoHeaders ? Object.fromEntries(response.headers.entries()) : response.headers;
|
package/dist/adapters/fetch.cjs
CHANGED
|
@@ -90,9 +90,24 @@ const debugLog = {
|
|
|
90
90
|
}
|
|
91
91
|
}
|
|
92
92
|
},
|
|
93
|
-
|
|
93
|
+
redirect: (config, fromUrl, toUrl, statusCode, method) => {
|
|
94
94
|
if (config.debug) {
|
|
95
|
-
console.log(`[Rezo Debug]
|
|
95
|
+
console.log(`[Rezo Debug] Redirect ${statusCode}: ${fromUrl}`);
|
|
96
|
+
console.log(`[Rezo Debug] → ${toUrl} (${method})`);
|
|
97
|
+
}
|
|
98
|
+
if (config.trackUrl) {
|
|
99
|
+
console.log(`[Rezo Track] ↳ ${statusCode} → ${toUrl}`);
|
|
100
|
+
}
|
|
101
|
+
},
|
|
102
|
+
complete: (config, url, redirectCount, duration) => {
|
|
103
|
+
if (config.debug) {
|
|
104
|
+
console.log(`[Rezo Debug] Complete: ${url}`);
|
|
105
|
+
if (redirectCount && redirectCount > 0) {
|
|
106
|
+
console.log(`[Rezo Debug] Redirects: ${redirectCount}`);
|
|
107
|
+
}
|
|
108
|
+
if (duration) {
|
|
109
|
+
console.log(`[Rezo Debug] Total Duration: ${duration.toFixed(2)}ms`);
|
|
110
|
+
}
|
|
96
111
|
console.log(`[Rezo Debug] ─────────────────────────────────────
|
|
97
112
|
`);
|
|
98
113
|
}
|
|
@@ -435,7 +450,7 @@ async function executeRequest(options, defaultOptions, jar) {
|
|
|
435
450
|
uploadResponse = new UploadResponse(url);
|
|
436
451
|
}
|
|
437
452
|
try {
|
|
438
|
-
const res = executeFetchRequest(fetchOptions, mainConfig, options, perform, streamResponse, downloadResponse, uploadResponse);
|
|
453
|
+
const res = executeFetchRequest(fetchOptions, mainConfig, options, perform, streamResponse, downloadResponse, uploadResponse, jar);
|
|
439
454
|
if (streamResponse) {
|
|
440
455
|
return streamResponse;
|
|
441
456
|
} else if (downloadResponse) {
|
|
@@ -462,7 +477,7 @@ async function executeRequest(options, defaultOptions, jar) {
|
|
|
462
477
|
throw error;
|
|
463
478
|
}
|
|
464
479
|
}
|
|
465
|
-
async function executeFetchRequest(fetchOptions, config, options, perform, streamResult, downloadResult, uploadResult) {
|
|
480
|
+
async function executeFetchRequest(fetchOptions, config, options, perform, streamResult, downloadResult, uploadResult, rootJar) {
|
|
466
481
|
let requestCount = 0;
|
|
467
482
|
const _stats = { statusOnNext: "abort" };
|
|
468
483
|
let retries = 0;
|
|
@@ -479,6 +494,11 @@ async function executeFetchRequest(fetchOptions, config, options, perform, strea
|
|
|
479
494
|
const visitedUrls = new Set;
|
|
480
495
|
let totalAttempts = 0;
|
|
481
496
|
config.setSignal();
|
|
497
|
+
if (!config.requestId) {
|
|
498
|
+
config.requestId = `req_${Date.now()}_${Math.random().toString(36).substring(2, 11)}`;
|
|
499
|
+
}
|
|
500
|
+
const requestUrl = fetchOptions.fullUrl ? String(fetchOptions.fullUrl) : "";
|
|
501
|
+
debugLog.requestStart(config, requestUrl, fetchOptions.method || "GET");
|
|
482
502
|
const eventEmitter = streamResult || downloadResult || uploadResult;
|
|
483
503
|
if (eventEmitter) {
|
|
484
504
|
eventEmitter.emit("initiated");
|
|
@@ -530,6 +550,8 @@ async function executeFetchRequest(fetchOptions, config, options, perform, strea
|
|
|
530
550
|
continue;
|
|
531
551
|
}
|
|
532
552
|
if (statusOnNext === "success") {
|
|
553
|
+
const totalDuration = performance.now() - timing.startTime;
|
|
554
|
+
debugLog.complete(config, response.finalUrl || requestUrl, config.redirectCount, totalDuration);
|
|
533
555
|
return response;
|
|
534
556
|
}
|
|
535
557
|
if (statusOnNext === "error") {
|
|
@@ -579,12 +601,13 @@ async function executeFetchRequest(fetchOptions, config, options, perform, strea
|
|
|
579
601
|
status: response.status,
|
|
580
602
|
headers: response.headers,
|
|
581
603
|
sameDomain: isSameDomain(fetchOptions.fullUrl, location),
|
|
582
|
-
method: fetchOptions.method.toUpperCase()
|
|
604
|
+
method: fetchOptions.method.toUpperCase(),
|
|
605
|
+
body: config.originalBody
|
|
583
606
|
}) : undefined;
|
|
584
607
|
if (typeof onRedirect !== "undefined") {
|
|
585
608
|
if (typeof onRedirect === "boolean" && !onRedirect) {
|
|
586
609
|
throw builErrorFromResponse("Redirect denied by user", response, config, fetchOptions);
|
|
587
|
-
} else if (typeof onRedirect === "object" && !onRedirect.redirect) {
|
|
610
|
+
} else if (typeof onRedirect === "object" && !onRedirect.redirect && !onRedirect.withoutBody && !("body" in onRedirect)) {
|
|
588
611
|
throw builErrorFromResponse("Redirect denied by user", response, config, fetchOptions);
|
|
589
612
|
}
|
|
590
613
|
}
|
|
@@ -604,13 +627,78 @@ async function executeFetchRequest(fetchOptions, config, options, perform, strea
|
|
|
604
627
|
});
|
|
605
628
|
perform.reset();
|
|
606
629
|
config.redirectCount++;
|
|
607
|
-
|
|
608
|
-
|
|
609
|
-
|
|
630
|
+
const fromUrl = fetchOptions.fullUrl;
|
|
631
|
+
fetchOptions.fullUrl = location;
|
|
632
|
+
const normalizedRedirect = typeof onRedirect === "object" ? onRedirect.redirect || onRedirect.withoutBody || "body" in onRedirect : undefined;
|
|
633
|
+
if (typeof onRedirect === "object" && normalizedRedirect) {
|
|
634
|
+
let method;
|
|
635
|
+
const userMethod = onRedirect.method;
|
|
636
|
+
if (redirectCode === 301 || redirectCode === 302 || redirectCode === 303) {
|
|
637
|
+
method = userMethod || "GET";
|
|
638
|
+
} else {
|
|
639
|
+
method = userMethod || fetchOptions.method;
|
|
640
|
+
}
|
|
641
|
+
config.method = method;
|
|
642
|
+
options.method = method;
|
|
643
|
+
fetchOptions.method = method;
|
|
644
|
+
if (onRedirect.redirect && onRedirect.url) {
|
|
645
|
+
options.fullUrl = onRedirect.url;
|
|
646
|
+
fetchOptions.fullUrl = onRedirect.url;
|
|
647
|
+
}
|
|
648
|
+
if (onRedirect.withoutBody) {
|
|
649
|
+
delete options.body;
|
|
650
|
+
delete fetchOptions.body;
|
|
651
|
+
config.originalBody = undefined;
|
|
652
|
+
if (fetchOptions.headers instanceof RezoHeaders) {
|
|
653
|
+
fetchOptions.headers.delete("Content-Type");
|
|
654
|
+
fetchOptions.headers.delete("Content-Length");
|
|
655
|
+
}
|
|
656
|
+
} else if ("body" in onRedirect) {
|
|
657
|
+
options.body = onRedirect.body;
|
|
658
|
+
fetchOptions.body = onRedirect.body;
|
|
659
|
+
config.originalBody = onRedirect.body;
|
|
660
|
+
} else if (redirectCode === 307 || redirectCode === 308) {
|
|
661
|
+
const methodUpper = method.toUpperCase();
|
|
662
|
+
if ((methodUpper === "POST" || methodUpper === "PUT" || methodUpper === "PATCH") && config.originalBody !== undefined) {
|
|
663
|
+
options.body = config.originalBody;
|
|
664
|
+
fetchOptions.body = config.originalBody;
|
|
665
|
+
}
|
|
666
|
+
} else {
|
|
610
667
|
delete options.body;
|
|
668
|
+
delete fetchOptions.body;
|
|
669
|
+
if (fetchOptions.headers instanceof RezoHeaders) {
|
|
670
|
+
fetchOptions.headers.delete("Content-Type");
|
|
671
|
+
fetchOptions.headers.delete("Content-Length");
|
|
672
|
+
}
|
|
611
673
|
}
|
|
674
|
+
debugLog.redirect(config, fromUrl, fetchOptions.fullUrl, redirectCode, method);
|
|
675
|
+
if (onRedirect.redirect && onRedirect.setHeaders) {
|
|
676
|
+
if (fetchOptions.headers instanceof RezoHeaders) {
|
|
677
|
+
for (const [key, value] of Object.entries(onRedirect.setHeaders)) {
|
|
678
|
+
fetchOptions.headers.set(key, value);
|
|
679
|
+
}
|
|
680
|
+
}
|
|
681
|
+
}
|
|
682
|
+
} else if (redirectCode === 301 || redirectCode === 302 || redirectCode === 303) {
|
|
683
|
+
debugLog.redirect(config, fromUrl, fetchOptions.fullUrl, redirectCode, "GET");
|
|
684
|
+
options.method = "GET";
|
|
685
|
+
fetchOptions.method = "GET";
|
|
686
|
+
config.method = "GET";
|
|
687
|
+
delete options.body;
|
|
688
|
+
delete fetchOptions.body;
|
|
689
|
+
if (fetchOptions.headers instanceof RezoHeaders) {
|
|
690
|
+
fetchOptions.headers.delete("Content-Type");
|
|
691
|
+
fetchOptions.headers.delete("Content-Length");
|
|
692
|
+
}
|
|
693
|
+
} else {
|
|
694
|
+
debugLog.redirect(config, fromUrl, fetchOptions.fullUrl, redirectCode, fetchOptions.method);
|
|
695
|
+
}
|
|
696
|
+
const jarToSync = rootJar || config.cookieJar;
|
|
697
|
+
if (response.cookies?.array?.length > 0 && jarToSync) {
|
|
698
|
+
try {
|
|
699
|
+
jarToSync.setCookiesSync(response.cookies.array, fromUrl);
|
|
700
|
+
} catch (e) {}
|
|
612
701
|
}
|
|
613
|
-
fetchOptions.fullUrl = location;
|
|
614
702
|
delete options.params;
|
|
615
703
|
requestCount++;
|
|
616
704
|
continue;
|
|
@@ -640,7 +728,6 @@ async function executeSingleFetchRequest(config, fetchOptions, requestCount, tim
|
|
|
640
728
|
} else if (config.transfer.requestSize === undefined) {
|
|
641
729
|
config.transfer.requestSize = 0;
|
|
642
730
|
}
|
|
643
|
-
debugLog.requestStart(config, url.href, fetchOptions.method?.toUpperCase() || "GET");
|
|
644
731
|
}
|
|
645
732
|
const reqHeaders = fetchOptions.headers instanceof RezoHeaders ? fetchOptions.headers.toObject() : fetchOptions.headers || {};
|
|
646
733
|
const headers = toFetchHeaders(reqHeaders);
|
|
@@ -903,7 +990,6 @@ async function executeSingleFetchRequest(config, fetchOptions, requestCount, tim
|
|
|
903
990
|
uploadResult.emit("done", uploadFinishEvent);
|
|
904
991
|
uploadResult._markFinished();
|
|
905
992
|
}
|
|
906
|
-
debugLog.complete(config, url.href);
|
|
907
993
|
return finalResponse;
|
|
908
994
|
} catch (error) {
|
|
909
995
|
_stats.statusOnNext = "error";
|
package/dist/adapters/fetch.js
CHANGED
|
@@ -90,9 +90,24 @@ const debugLog = {
|
|
|
90
90
|
}
|
|
91
91
|
}
|
|
92
92
|
},
|
|
93
|
-
|
|
93
|
+
redirect: (config, fromUrl, toUrl, statusCode, method) => {
|
|
94
94
|
if (config.debug) {
|
|
95
|
-
console.log(`[Rezo Debug]
|
|
95
|
+
console.log(`[Rezo Debug] Redirect ${statusCode}: ${fromUrl}`);
|
|
96
|
+
console.log(`[Rezo Debug] → ${toUrl} (${method})`);
|
|
97
|
+
}
|
|
98
|
+
if (config.trackUrl) {
|
|
99
|
+
console.log(`[Rezo Track] ↳ ${statusCode} → ${toUrl}`);
|
|
100
|
+
}
|
|
101
|
+
},
|
|
102
|
+
complete: (config, url, redirectCount, duration) => {
|
|
103
|
+
if (config.debug) {
|
|
104
|
+
console.log(`[Rezo Debug] Complete: ${url}`);
|
|
105
|
+
if (redirectCount && redirectCount > 0) {
|
|
106
|
+
console.log(`[Rezo Debug] Redirects: ${redirectCount}`);
|
|
107
|
+
}
|
|
108
|
+
if (duration) {
|
|
109
|
+
console.log(`[Rezo Debug] Total Duration: ${duration.toFixed(2)}ms`);
|
|
110
|
+
}
|
|
96
111
|
console.log(`[Rezo Debug] ─────────────────────────────────────
|
|
97
112
|
`);
|
|
98
113
|
}
|
|
@@ -435,7 +450,7 @@ export async function executeRequest(options, defaultOptions, jar) {
|
|
|
435
450
|
uploadResponse = new UploadResponse(url);
|
|
436
451
|
}
|
|
437
452
|
try {
|
|
438
|
-
const res = executeFetchRequest(fetchOptions, mainConfig, options, perform, streamResponse, downloadResponse, uploadResponse);
|
|
453
|
+
const res = executeFetchRequest(fetchOptions, mainConfig, options, perform, streamResponse, downloadResponse, uploadResponse, jar);
|
|
439
454
|
if (streamResponse) {
|
|
440
455
|
return streamResponse;
|
|
441
456
|
} else if (downloadResponse) {
|
|
@@ -462,7 +477,7 @@ export async function executeRequest(options, defaultOptions, jar) {
|
|
|
462
477
|
throw error;
|
|
463
478
|
}
|
|
464
479
|
}
|
|
465
|
-
async function executeFetchRequest(fetchOptions, config, options, perform, streamResult, downloadResult, uploadResult) {
|
|
480
|
+
async function executeFetchRequest(fetchOptions, config, options, perform, streamResult, downloadResult, uploadResult, rootJar) {
|
|
466
481
|
let requestCount = 0;
|
|
467
482
|
const _stats = { statusOnNext: "abort" };
|
|
468
483
|
let retries = 0;
|
|
@@ -479,6 +494,11 @@ async function executeFetchRequest(fetchOptions, config, options, perform, strea
|
|
|
479
494
|
const visitedUrls = new Set;
|
|
480
495
|
let totalAttempts = 0;
|
|
481
496
|
config.setSignal();
|
|
497
|
+
if (!config.requestId) {
|
|
498
|
+
config.requestId = `req_${Date.now()}_${Math.random().toString(36).substring(2, 11)}`;
|
|
499
|
+
}
|
|
500
|
+
const requestUrl = fetchOptions.fullUrl ? String(fetchOptions.fullUrl) : "";
|
|
501
|
+
debugLog.requestStart(config, requestUrl, fetchOptions.method || "GET");
|
|
482
502
|
const eventEmitter = streamResult || downloadResult || uploadResult;
|
|
483
503
|
if (eventEmitter) {
|
|
484
504
|
eventEmitter.emit("initiated");
|
|
@@ -530,6 +550,8 @@ async function executeFetchRequest(fetchOptions, config, options, perform, strea
|
|
|
530
550
|
continue;
|
|
531
551
|
}
|
|
532
552
|
if (statusOnNext === "success") {
|
|
553
|
+
const totalDuration = performance.now() - timing.startTime;
|
|
554
|
+
debugLog.complete(config, response.finalUrl || requestUrl, config.redirectCount, totalDuration);
|
|
533
555
|
return response;
|
|
534
556
|
}
|
|
535
557
|
if (statusOnNext === "error") {
|
|
@@ -579,12 +601,13 @@ async function executeFetchRequest(fetchOptions, config, options, perform, strea
|
|
|
579
601
|
status: response.status,
|
|
580
602
|
headers: response.headers,
|
|
581
603
|
sameDomain: isSameDomain(fetchOptions.fullUrl, location),
|
|
582
|
-
method: fetchOptions.method.toUpperCase()
|
|
604
|
+
method: fetchOptions.method.toUpperCase(),
|
|
605
|
+
body: config.originalBody
|
|
583
606
|
}) : undefined;
|
|
584
607
|
if (typeof onRedirect !== "undefined") {
|
|
585
608
|
if (typeof onRedirect === "boolean" && !onRedirect) {
|
|
586
609
|
throw builErrorFromResponse("Redirect denied by user", response, config, fetchOptions);
|
|
587
|
-
} else if (typeof onRedirect === "object" && !onRedirect.redirect) {
|
|
610
|
+
} else if (typeof onRedirect === "object" && !onRedirect.redirect && !onRedirect.withoutBody && !("body" in onRedirect)) {
|
|
588
611
|
throw builErrorFromResponse("Redirect denied by user", response, config, fetchOptions);
|
|
589
612
|
}
|
|
590
613
|
}
|
|
@@ -604,13 +627,78 @@ async function executeFetchRequest(fetchOptions, config, options, perform, strea
|
|
|
604
627
|
});
|
|
605
628
|
perform.reset();
|
|
606
629
|
config.redirectCount++;
|
|
607
|
-
|
|
608
|
-
|
|
609
|
-
|
|
630
|
+
const fromUrl = fetchOptions.fullUrl;
|
|
631
|
+
fetchOptions.fullUrl = location;
|
|
632
|
+
const normalizedRedirect = typeof onRedirect === "object" ? onRedirect.redirect || onRedirect.withoutBody || "body" in onRedirect : undefined;
|
|
633
|
+
if (typeof onRedirect === "object" && normalizedRedirect) {
|
|
634
|
+
let method;
|
|
635
|
+
const userMethod = onRedirect.method;
|
|
636
|
+
if (redirectCode === 301 || redirectCode === 302 || redirectCode === 303) {
|
|
637
|
+
method = userMethod || "GET";
|
|
638
|
+
} else {
|
|
639
|
+
method = userMethod || fetchOptions.method;
|
|
640
|
+
}
|
|
641
|
+
config.method = method;
|
|
642
|
+
options.method = method;
|
|
643
|
+
fetchOptions.method = method;
|
|
644
|
+
if (onRedirect.redirect && onRedirect.url) {
|
|
645
|
+
options.fullUrl = onRedirect.url;
|
|
646
|
+
fetchOptions.fullUrl = onRedirect.url;
|
|
647
|
+
}
|
|
648
|
+
if (onRedirect.withoutBody) {
|
|
649
|
+
delete options.body;
|
|
650
|
+
delete fetchOptions.body;
|
|
651
|
+
config.originalBody = undefined;
|
|
652
|
+
if (fetchOptions.headers instanceof RezoHeaders) {
|
|
653
|
+
fetchOptions.headers.delete("Content-Type");
|
|
654
|
+
fetchOptions.headers.delete("Content-Length");
|
|
655
|
+
}
|
|
656
|
+
} else if ("body" in onRedirect) {
|
|
657
|
+
options.body = onRedirect.body;
|
|
658
|
+
fetchOptions.body = onRedirect.body;
|
|
659
|
+
config.originalBody = onRedirect.body;
|
|
660
|
+
} else if (redirectCode === 307 || redirectCode === 308) {
|
|
661
|
+
const methodUpper = method.toUpperCase();
|
|
662
|
+
if ((methodUpper === "POST" || methodUpper === "PUT" || methodUpper === "PATCH") && config.originalBody !== undefined) {
|
|
663
|
+
options.body = config.originalBody;
|
|
664
|
+
fetchOptions.body = config.originalBody;
|
|
665
|
+
}
|
|
666
|
+
} else {
|
|
610
667
|
delete options.body;
|
|
668
|
+
delete fetchOptions.body;
|
|
669
|
+
if (fetchOptions.headers instanceof RezoHeaders) {
|
|
670
|
+
fetchOptions.headers.delete("Content-Type");
|
|
671
|
+
fetchOptions.headers.delete("Content-Length");
|
|
672
|
+
}
|
|
611
673
|
}
|
|
674
|
+
debugLog.redirect(config, fromUrl, fetchOptions.fullUrl, redirectCode, method);
|
|
675
|
+
if (onRedirect.redirect && onRedirect.setHeaders) {
|
|
676
|
+
if (fetchOptions.headers instanceof RezoHeaders) {
|
|
677
|
+
for (const [key, value] of Object.entries(onRedirect.setHeaders)) {
|
|
678
|
+
fetchOptions.headers.set(key, value);
|
|
679
|
+
}
|
|
680
|
+
}
|
|
681
|
+
}
|
|
682
|
+
} else if (redirectCode === 301 || redirectCode === 302 || redirectCode === 303) {
|
|
683
|
+
debugLog.redirect(config, fromUrl, fetchOptions.fullUrl, redirectCode, "GET");
|
|
684
|
+
options.method = "GET";
|
|
685
|
+
fetchOptions.method = "GET";
|
|
686
|
+
config.method = "GET";
|
|
687
|
+
delete options.body;
|
|
688
|
+
delete fetchOptions.body;
|
|
689
|
+
if (fetchOptions.headers instanceof RezoHeaders) {
|
|
690
|
+
fetchOptions.headers.delete("Content-Type");
|
|
691
|
+
fetchOptions.headers.delete("Content-Length");
|
|
692
|
+
}
|
|
693
|
+
} else {
|
|
694
|
+
debugLog.redirect(config, fromUrl, fetchOptions.fullUrl, redirectCode, fetchOptions.method);
|
|
695
|
+
}
|
|
696
|
+
const jarToSync = rootJar || config.cookieJar;
|
|
697
|
+
if (response.cookies?.array?.length > 0 && jarToSync) {
|
|
698
|
+
try {
|
|
699
|
+
jarToSync.setCookiesSync(response.cookies.array, fromUrl);
|
|
700
|
+
} catch (e) {}
|
|
612
701
|
}
|
|
613
|
-
fetchOptions.fullUrl = location;
|
|
614
702
|
delete options.params;
|
|
615
703
|
requestCount++;
|
|
616
704
|
continue;
|
|
@@ -640,7 +728,6 @@ async function executeSingleFetchRequest(config, fetchOptions, requestCount, tim
|
|
|
640
728
|
} else if (config.transfer.requestSize === undefined) {
|
|
641
729
|
config.transfer.requestSize = 0;
|
|
642
730
|
}
|
|
643
|
-
debugLog.requestStart(config, url.href, fetchOptions.method?.toUpperCase() || "GET");
|
|
644
731
|
}
|
|
645
732
|
const reqHeaders = fetchOptions.headers instanceof RezoHeaders ? fetchOptions.headers.toObject() : fetchOptions.headers || {};
|
|
646
733
|
const headers = toFetchHeaders(reqHeaders);
|
|
@@ -903,7 +990,6 @@ async function executeSingleFetchRequest(config, fetchOptions, requestCount, tim
|
|
|
903
990
|
uploadResult.emit("done", uploadFinishEvent);
|
|
904
991
|
uploadResult._markFinished();
|
|
905
992
|
}
|
|
906
|
-
debugLog.complete(config, url.href);
|
|
907
993
|
return finalResponse;
|
|
908
994
|
} catch (error) {
|
|
909
995
|
_stats.statusOnNext = "error";
|
package/dist/adapters/http.cjs
CHANGED
|
@@ -21,7 +21,7 @@ const { getGlobalDNSCache } = require('../cache/dns-cache.cjs');
|
|
|
21
21
|
const { ResponseCache } = require('../cache/response-cache.cjs');
|
|
22
22
|
const { getGlobalAgentPool } = require('../utils/agent-pool.cjs');
|
|
23
23
|
const { StagedTimeoutManager, parseStagedTimeouts } = require('../utils/staged-timeout.cjs');
|
|
24
|
-
const dns = require("dns");
|
|
24
|
+
const dns = require("node:dns");
|
|
25
25
|
const debugLog = {
|
|
26
26
|
requestStart: (config, url, method) => {
|
|
27
27
|
if (config.debug) {
|
|
@@ -302,7 +302,7 @@ async function executeRequest(options, defaultOptions, jar) {
|
|
|
302
302
|
}
|
|
303
303
|
}
|
|
304
304
|
try {
|
|
305
|
-
const res = executeHttp1Request(config.fetchOptions, mainConfig, config.options, perform, d_options.fs, streamResponse, downloadResponse, uploadResponse);
|
|
305
|
+
const res = executeHttp1Request(config.fetchOptions, mainConfig, config.options, perform, d_options.fs, streamResponse, downloadResponse, uploadResponse, jar);
|
|
306
306
|
if (streamResponse) {
|
|
307
307
|
res.catch((err) => {
|
|
308
308
|
streamResponse.emit("error", err);
|
|
@@ -344,7 +344,7 @@ async function executeRequest(options, defaultOptions, jar) {
|
|
|
344
344
|
throw error;
|
|
345
345
|
}
|
|
346
346
|
}
|
|
347
|
-
async function executeHttp1Request(fetchOptions, config, options, perform, fs, streamResult, downloadResult, uploadResult) {
|
|
347
|
+
async function executeHttp1Request(fetchOptions, config, options, perform, fs, streamResult, downloadResult, uploadResult, rootJar) {
|
|
348
348
|
let requestCount = 0;
|
|
349
349
|
const _stats = { statusOnNext: "abort" };
|
|
350
350
|
let responseStatusCode;
|
|
@@ -363,6 +363,9 @@ async function executeHttp1Request(fetchOptions, config, options, perform, fs, s
|
|
|
363
363
|
config.setSignal();
|
|
364
364
|
const timeoutClearInstanse = config.timeoutClearInstanse;
|
|
365
365
|
delete config.timeoutClearInstanse;
|
|
366
|
+
if (!config.requestId) {
|
|
367
|
+
config.requestId = `req_${Date.now()}_${Math.random().toString(36).substring(2, 11)}`;
|
|
368
|
+
}
|
|
366
369
|
const requestUrl = fetchOptions.fullUrl ? String(fetchOptions.fullUrl) : "";
|
|
367
370
|
debugLog.requestStart(config, requestUrl, fetchOptions.method || "GET");
|
|
368
371
|
const eventEmitter = streamResult || downloadResult || uploadResult;
|
|
@@ -376,7 +379,7 @@ async function executeHttp1Request(fetchOptions, config, options, perform, fs, s
|
|
|
376
379
|
throw error;
|
|
377
380
|
}
|
|
378
381
|
try {
|
|
379
|
-
const response = await request(config, fetchOptions, requestCount, timing, _stats, responseStatusCode, fs, streamResult, downloadResult, uploadResult);
|
|
382
|
+
const response = await request(config, fetchOptions, requestCount, timing, _stats, responseStatusCode, fs, streamResult, downloadResult, uploadResult, rootJar);
|
|
380
383
|
const statusOnNext = _stats.statusOnNext;
|
|
381
384
|
if (response instanceof RezoError) {
|
|
382
385
|
const fileName = config.fileName;
|
|
@@ -530,8 +533,16 @@ async function executeHttp1Request(fetchOptions, config, options, perform, fs, s
|
|
|
530
533
|
fetchOptions.fullUrl = location;
|
|
531
534
|
const normalizedRedirect = typeof onRedirect === "object" ? onRedirect.redirect || onRedirect.withoutBody || "body" in onRedirect : undefined;
|
|
532
535
|
if (typeof onRedirect === "object" && normalizedRedirect) {
|
|
533
|
-
|
|
536
|
+
let method;
|
|
537
|
+
const userMethod = onRedirect.method;
|
|
538
|
+
if (redirectCode === 301 || redirectCode === 302 || redirectCode === 303) {
|
|
539
|
+
method = userMethod || "GET";
|
|
540
|
+
} else {
|
|
541
|
+
method = userMethod || fetchOptions.method;
|
|
542
|
+
}
|
|
534
543
|
config.method = method;
|
|
544
|
+
options.method = method;
|
|
545
|
+
fetchOptions.method = method;
|
|
535
546
|
if (onRedirect.redirect && onRedirect.url) {
|
|
536
547
|
options.fullUrl = onRedirect.url;
|
|
537
548
|
fetchOptions.fullUrl = onRedirect.url;
|
|
@@ -580,13 +591,13 @@ async function executeHttp1Request(fetchOptions, config, options, perform, fs, s
|
|
|
580
591
|
} else {
|
|
581
592
|
debugLog.redirect(config, fromUrl, fetchOptions.fullUrl, redirectCode, fetchOptions.method);
|
|
582
593
|
}
|
|
583
|
-
|
|
584
|
-
|
|
594
|
+
const jarToSync = rootJar || config.cookieJar;
|
|
595
|
+
if (response.cookies?.array?.length > 0 && jarToSync) {
|
|
585
596
|
try {
|
|
586
|
-
|
|
597
|
+
jarToSync.setCookiesSync(response.cookies.array, fromUrl);
|
|
587
598
|
} catch (e) {}
|
|
588
599
|
}
|
|
589
|
-
const __ = prepareHTTPOptions(fetchOptions,
|
|
600
|
+
const __ = prepareHTTPOptions(fetchOptions, jarToSync, addedOptions, config);
|
|
590
601
|
fetchOptions = __.fetchOptions;
|
|
591
602
|
config = __.config;
|
|
592
603
|
options = __.options;
|
|
@@ -625,7 +636,7 @@ async function executeHttp1Request(fetchOptions, config, options, perform, fs, s
|
|
|
625
636
|
}
|
|
626
637
|
}
|
|
627
638
|
}
|
|
628
|
-
async function request(config, fetchOptions, requestCount, timing, _stats, responseStatusCode, fs, streamResult, downloadResult, uploadResult) {
|
|
639
|
+
async function request(config, fetchOptions, requestCount, timing, _stats, responseStatusCode, fs, streamResult, downloadResult, uploadResult, rootJar) {
|
|
629
640
|
return await new Promise(async (resolve) => {
|
|
630
641
|
try {
|
|
631
642
|
const { fullUrl, body, fileName: filename } = fetchOptions;
|
|
@@ -679,7 +690,7 @@ async function request(config, fetchOptions, requestCount, timing, _stats, respo
|
|
|
679
690
|
const location = headers["location"] || headers["Location"];
|
|
680
691
|
const contentLength = headers["content-length"];
|
|
681
692
|
const cookies = headers["set-cookie"];
|
|
682
|
-
await updateCookies(config, headers, url.href);
|
|
693
|
+
await updateCookies(config, headers, url.href, rootJar);
|
|
683
694
|
const cookieArray = config.responseCookies?.array || [];
|
|
684
695
|
delete headers["set-cookie"];
|
|
685
696
|
_stats.redirectUrl = undefined;
|
|
@@ -1618,7 +1629,7 @@ function parseProxy(proxy, isScure = true, rejectUnauthorized = false) {
|
|
|
1618
1629
|
proxyAgentCache.set(cacheKey, { agent, lastUsed: now });
|
|
1619
1630
|
return agent;
|
|
1620
1631
|
}
|
|
1621
|
-
async function updateCookies(config, headers, url) {
|
|
1632
|
+
async function updateCookies(config, headers, url, rootJar) {
|
|
1622
1633
|
const cookies = headers["set-cookie"];
|
|
1623
1634
|
if (cookies) {
|
|
1624
1635
|
const jar = new RezoCookieJar;
|
|
@@ -1657,8 +1668,9 @@ async function updateCookies(config, headers, url) {
|
|
|
1657
1668
|
acceptedCookies.push(...parsedCookies.array);
|
|
1658
1669
|
}
|
|
1659
1670
|
const acceptedCookieStrings = acceptedCookies.map((c) => c.toSetCookieString());
|
|
1660
|
-
|
|
1661
|
-
|
|
1671
|
+
const jarToUpdate = rootJar || config.cookieJar;
|
|
1672
|
+
if (!config.disableCookieJar && jarToUpdate) {
|
|
1673
|
+
jarToUpdate.setCookiesSync(acceptedCookieStrings, url);
|
|
1662
1674
|
}
|
|
1663
1675
|
jar.setCookiesSync(acceptedCookieStrings, url);
|
|
1664
1676
|
if (config.useCookies) {
|