rezo 1.0.12 → 1.0.13
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/adapters/curl.cjs +73 -10
- package/dist/adapters/curl.js +73 -10
- package/dist/adapters/fetch.cjs +106 -59
- package/dist/adapters/fetch.js +106 -59
- package/dist/adapters/http.cjs +20 -13
- package/dist/adapters/http.js +20 -13
- package/dist/adapters/http2.cjs +114 -59
- package/dist/adapters/http2.js +114 -59
- package/dist/adapters/index.cjs +6 -6
- package/dist/cache/index.cjs +13 -13
- package/dist/entries/crawler.cjs +5 -5
- package/dist/index.cjs +24 -24
- package/dist/plugin/index.cjs +36 -36
- package/dist/proxy/index.cjs +2 -2
- package/dist/queue/index.cjs +8 -8
- package/dist/responses/buildError.cjs +5 -1
- package/dist/responses/buildError.js +5 -1
- package/dist/responses/buildResponse.cjs +2 -0
- package/dist/responses/buildResponse.js +2 -0
- package/dist/utils/compression.cjs +6 -6
- package/dist/utils/compression.js +6 -6
- package/dist/utils/headers.cjs +17 -0
- package/dist/utils/headers.js +17 -1
- package/dist/utils/http-config.cjs +39 -4
- package/dist/utils/http-config.js +39 -4
- package/package.json +1 -1
package/dist/adapters/http2.cjs
CHANGED
|
@@ -4,14 +4,15 @@ const { URL } = require("node:url");
|
|
|
4
4
|
const { Readable } = require("node:stream");
|
|
5
5
|
const { RezoError } = require('../errors/rezo-error.cjs');
|
|
6
6
|
const { buildSmartError, buildDecompressionError, builErrorFromResponse, buildDownloadError } = require('../responses/buildError.cjs');
|
|
7
|
-
const {
|
|
7
|
+
const { RezoCookieJar } = require('../utils/cookies.cjs');
|
|
8
8
|
const RezoFormData = require('../utils/form-data.cjs');
|
|
9
9
|
const { getDefaultConfig, prepareHTTPOptions } = require('../utils/http-config.cjs');
|
|
10
|
-
const { RezoHeaders } = require('../utils/headers.cjs');
|
|
10
|
+
const { RezoHeaders, sanitizeHttp2Headers } = require('../utils/headers.cjs');
|
|
11
11
|
const { RezoURLSearchParams } = require('../utils/data-operations.cjs');
|
|
12
12
|
const { StreamResponse } = require('../responses/stream.cjs');
|
|
13
13
|
const { DownloadResponse } = require('../responses/download.cjs');
|
|
14
14
|
const { UploadResponse } = require('../responses/upload.cjs');
|
|
15
|
+
const { CompressionUtil } = require('../utils/compression.cjs');
|
|
15
16
|
const { isSameDomain, RezoPerformance } = require('../utils/tools.cjs');
|
|
16
17
|
const { ResponseCache } = require('../cache/response-cache.cjs');
|
|
17
18
|
let zstdDecompressSync = null;
|
|
@@ -142,15 +143,14 @@ class Http2SessionPool {
|
|
|
142
143
|
reject(new Error(`HTTP/2 connection timeout after ${timeout}ms`));
|
|
143
144
|
}
|
|
144
145
|
}, timeout) : null;
|
|
146
|
+
if (timeoutId && typeof timeoutId === "object" && "unref" in timeoutId) {
|
|
147
|
+
timeoutId.unref();
|
|
148
|
+
}
|
|
145
149
|
session.on("connect", () => {
|
|
146
150
|
if (!settled) {
|
|
147
151
|
settled = true;
|
|
148
152
|
if (timeoutId)
|
|
149
153
|
clearTimeout(timeoutId);
|
|
150
|
-
const socket = session.socket;
|
|
151
|
-
if (socket && typeof socket.unref === "function") {
|
|
152
|
-
socket.unref();
|
|
153
|
-
}
|
|
154
154
|
resolve(session);
|
|
155
155
|
}
|
|
156
156
|
});
|
|
@@ -170,6 +170,12 @@ class Http2SessionPool {
|
|
|
170
170
|
if (entry) {
|
|
171
171
|
entry.refCount = Math.max(0, entry.refCount - 1);
|
|
172
172
|
entry.lastUsed = Date.now();
|
|
173
|
+
if (entry.refCount === 0) {
|
|
174
|
+
const socket = entry.session.socket;
|
|
175
|
+
if (socket && typeof socket.unref === "function") {
|
|
176
|
+
socket.unref();
|
|
177
|
+
}
|
|
178
|
+
}
|
|
173
179
|
}
|
|
174
180
|
}
|
|
175
181
|
closeSession(url) {
|
|
@@ -275,38 +281,66 @@ function updateCookies(config, headers, url) {
|
|
|
275
281
|
if (!setCookieHeaders)
|
|
276
282
|
return;
|
|
277
283
|
const cookieHeaderArray = Array.isArray(setCookieHeaders) ? setCookieHeaders : [setCookieHeaders];
|
|
278
|
-
if (
|
|
279
|
-
|
|
280
|
-
|
|
281
|
-
|
|
282
|
-
|
|
283
|
-
|
|
284
|
-
setCookiesString: []
|
|
285
|
-
};
|
|
284
|
+
if (cookieHeaderArray.length === 0)
|
|
285
|
+
return;
|
|
286
|
+
const jar = new RezoCookieJar;
|
|
287
|
+
jar.setCookiesSync(cookieHeaderArray, url);
|
|
288
|
+
if (config.enableCookieJar && config.cookieJar) {
|
|
289
|
+
config.cookieJar.setCookiesSync(cookieHeaderArray, url);
|
|
286
290
|
}
|
|
287
|
-
|
|
288
|
-
|
|
289
|
-
|
|
290
|
-
const
|
|
291
|
-
const
|
|
292
|
-
|
|
293
|
-
|
|
294
|
-
|
|
295
|
-
|
|
296
|
-
|
|
297
|
-
|
|
298
|
-
|
|
299
|
-
|
|
300
|
-
|
|
301
|
-
|
|
302
|
-
|
|
303
|
-
|
|
291
|
+
const cookies = jar.cookies();
|
|
292
|
+
cookies.setCookiesString = cookieHeaderArray;
|
|
293
|
+
if (config.useCookies) {
|
|
294
|
+
const existingArray = config.responseCookies?.array || [];
|
|
295
|
+
for (const cookie of cookies.array) {
|
|
296
|
+
const existingIndex = existingArray.findIndex((c) => c.key === cookie.key && c.domain === cookie.domain);
|
|
297
|
+
if (existingIndex >= 0) {
|
|
298
|
+
existingArray[existingIndex] = cookie;
|
|
299
|
+
} else {
|
|
300
|
+
existingArray.push(cookie);
|
|
301
|
+
}
|
|
302
|
+
}
|
|
303
|
+
const mergedJar = new RezoCookieJar(existingArray, url);
|
|
304
|
+
config.responseCookies = mergedJar.cookies();
|
|
305
|
+
config.responseCookies.setCookiesString = cookieHeaderArray;
|
|
306
|
+
} else {
|
|
307
|
+
config.responseCookies = cookies;
|
|
308
|
+
}
|
|
309
|
+
}
|
|
310
|
+
function mergeRequestAndResponseCookies(config, responseCookies, url) {
|
|
311
|
+
const mergedCookiesArray = [];
|
|
312
|
+
const cookieKeyDomainMap = new Map;
|
|
313
|
+
if (config.requestCookies && config.requestCookies.length > 0) {
|
|
314
|
+
for (const cookie of config.requestCookies) {
|
|
315
|
+
const key = `${cookie.key}|${cookie.domain || ""}`;
|
|
316
|
+
mergedCookiesArray.push(cookie);
|
|
317
|
+
cookieKeyDomainMap.set(key, mergedCookiesArray.length - 1);
|
|
304
318
|
}
|
|
305
319
|
}
|
|
306
|
-
|
|
307
|
-
|
|
308
|
-
|
|
309
|
-
|
|
320
|
+
for (const cookie of responseCookies.array) {
|
|
321
|
+
const key = `${cookie.key}|${cookie.domain || ""}`;
|
|
322
|
+
const existingIndex = cookieKeyDomainMap.get(key);
|
|
323
|
+
if (existingIndex !== undefined) {
|
|
324
|
+
mergedCookiesArray[existingIndex] = cookie;
|
|
325
|
+
} else {
|
|
326
|
+
mergedCookiesArray.push(cookie);
|
|
327
|
+
cookieKeyDomainMap.set(key, mergedCookiesArray.length - 1);
|
|
328
|
+
}
|
|
329
|
+
}
|
|
330
|
+
if (mergedCookiesArray.length > 0) {
|
|
331
|
+
const mergedJar = new RezoCookieJar(mergedCookiesArray, url);
|
|
332
|
+
return mergedJar.cookies();
|
|
333
|
+
}
|
|
334
|
+
return {
|
|
335
|
+
array: [],
|
|
336
|
+
serialized: [],
|
|
337
|
+
netscape: `# Netscape HTTP Cookie File
|
|
338
|
+
# This file was generated by Rezo HTTP client
|
|
339
|
+
# Based on uniqhtt cookie implementation
|
|
340
|
+
`,
|
|
341
|
+
string: "",
|
|
342
|
+
setCookiesString: []
|
|
343
|
+
};
|
|
310
344
|
}
|
|
311
345
|
async function executeRequest(options, defaultOptions, jar) {
|
|
312
346
|
if (!options.responseType) {
|
|
@@ -484,6 +518,8 @@ async function executeHttp2Request(fetchOptions, config, options, perform, fs, s
|
|
|
484
518
|
if (fileName && fs && fs.existsSync(fileName)) {
|
|
485
519
|
fs.unlinkSync(fileName);
|
|
486
520
|
}
|
|
521
|
+
if (!config.errors)
|
|
522
|
+
config.errors = [];
|
|
487
523
|
config.errors.push({
|
|
488
524
|
attempt: config.retryAttempts + 1,
|
|
489
525
|
error: response,
|
|
@@ -521,7 +557,7 @@ async function executeHttp2Request(fetchOptions, config, options, perform, fs, s
|
|
|
521
557
|
}
|
|
522
558
|
continue;
|
|
523
559
|
}
|
|
524
|
-
if (statusOnNext === "success") {
|
|
560
|
+
if (statusOnNext === "success" || statusOnNext === "error") {
|
|
525
561
|
return response;
|
|
526
562
|
}
|
|
527
563
|
if (statusOnNext === "redirect") {
|
|
@@ -620,6 +656,13 @@ async function executeHttp2Stream(config, fetchOptions, requestCount, timing, _s
|
|
|
620
656
|
if (!headers["accept-encoding"]) {
|
|
621
657
|
headers["accept-encoding"] = "gzip, deflate, br";
|
|
622
658
|
}
|
|
659
|
+
if (body instanceof RezoFormData) {
|
|
660
|
+
headers["content-type"] = `multipart/form-data; boundary=${body.getBoundary()}`;
|
|
661
|
+
} else if (body instanceof FormData) {
|
|
662
|
+
const tempForm = await RezoFormData.fromNativeFormData(body);
|
|
663
|
+
headers["content-type"] = `multipart/form-data; boundary=${tempForm.getBoundary()}`;
|
|
664
|
+
fetchOptions._convertedFormData = tempForm;
|
|
665
|
+
}
|
|
623
666
|
const eventEmitter = streamResult || downloadResult || uploadResult;
|
|
624
667
|
if (eventEmitter && requestCount === 0) {
|
|
625
668
|
const startEvent = {
|
|
@@ -735,6 +778,25 @@ async function executeHttp2Stream(config, fetchOptions, requestCount, timing, _s
|
|
|
735
778
|
config.timing.endTimestamp = Date.now();
|
|
736
779
|
config.timing.durationMs = performance.now() - timing.startTime;
|
|
737
780
|
config.timing.transferMs = timing.firstByteTime ? performance.now() - timing.firstByteTime : config.timing.durationMs;
|
|
781
|
+
if (!config.transfer) {
|
|
782
|
+
config.transfer = { requestSize: 0, responseSize: 0, headerSize: 0, bodySize: 0 };
|
|
783
|
+
}
|
|
784
|
+
if (config.transfer.requestSize === undefined) {
|
|
785
|
+
config.transfer.requestSize = 0;
|
|
786
|
+
}
|
|
787
|
+
if (config.transfer.requestSize === 0 && body) {
|
|
788
|
+
if (typeof body === "string") {
|
|
789
|
+
config.transfer.requestSize = Buffer.byteLength(body, "utf8");
|
|
790
|
+
} else if (body instanceof Buffer || body instanceof Uint8Array) {
|
|
791
|
+
config.transfer.requestSize = body.length;
|
|
792
|
+
} else if (body instanceof URLSearchParams || body instanceof RezoURLSearchParams) {
|
|
793
|
+
config.transfer.requestSize = Buffer.byteLength(body.toString(), "utf8");
|
|
794
|
+
} else if (body instanceof RezoFormData) {
|
|
795
|
+
config.transfer.requestSize = body.getLengthSync();
|
|
796
|
+
} else if (typeof body === "object") {
|
|
797
|
+
config.transfer.requestSize = Buffer.byteLength(JSON.stringify(body), "utf8");
|
|
798
|
+
}
|
|
799
|
+
}
|
|
738
800
|
config.transfer.bodySize = contentLengthCounter;
|
|
739
801
|
config.transfer.responseSize = contentLengthCounter;
|
|
740
802
|
(sessionPool || Http2SessionPool.getInstance()).releaseSession(url);
|
|
@@ -743,7 +805,7 @@ async function executeHttp2Stream(config, fetchOptions, requestCount, timing, _s
|
|
|
743
805
|
data: "",
|
|
744
806
|
status,
|
|
745
807
|
statusText,
|
|
746
|
-
headers: new RezoHeaders(responseHeaders),
|
|
808
|
+
headers: new RezoHeaders(sanitizeHttp2Headers(responseHeaders)),
|
|
747
809
|
cookies: config.responseCookies || { array: [], serialized: [], netscape: "", string: "", setCookiesString: [] },
|
|
748
810
|
config,
|
|
749
811
|
contentType: responseHeaders["content-type"],
|
|
@@ -756,14 +818,14 @@ async function executeHttp2Stream(config, fetchOptions, requestCount, timing, _s
|
|
|
756
818
|
}
|
|
757
819
|
let responseBody = Buffer.concat(chunks);
|
|
758
820
|
const contentEncoding = responseHeaders["content-encoding"];
|
|
759
|
-
if (contentEncoding && contentLengthCounter > 0) {
|
|
821
|
+
if (contentEncoding && contentLengthCounter > 0 && CompressionUtil.shouldDecompress(contentEncoding, config)) {
|
|
760
822
|
try {
|
|
761
823
|
const decompressed = await decompressBuffer(responseBody, contentEncoding);
|
|
762
824
|
responseBody = decompressed;
|
|
763
825
|
} catch (err) {
|
|
764
826
|
const error = buildDecompressionError({
|
|
765
827
|
statusCode: status,
|
|
766
|
-
headers: responseHeaders,
|
|
828
|
+
headers: sanitizeHttp2Headers(responseHeaders),
|
|
767
829
|
contentType: responseHeaders["content-type"],
|
|
768
830
|
contentLength: String(contentLengthCounter),
|
|
769
831
|
cookies: config.responseCookies?.setCookiesString || [],
|
|
@@ -803,24 +865,17 @@ async function executeHttp2Stream(config, fetchOptions, requestCount, timing, _s
|
|
|
803
865
|
data = responseBody.toString("utf-8");
|
|
804
866
|
}
|
|
805
867
|
}
|
|
806
|
-
|
|
807
|
-
|
|
808
|
-
|
|
809
|
-
|
|
810
|
-
|
|
811
|
-
data
|
|
812
|
-
}, config, fetchOptions);
|
|
813
|
-
_stats.statusOnNext = "error";
|
|
814
|
-
resolve(error);
|
|
815
|
-
return;
|
|
816
|
-
}
|
|
817
|
-
_stats.statusOnNext = "success";
|
|
868
|
+
config.status = status;
|
|
869
|
+
config.statusText = statusText;
|
|
870
|
+
_stats.statusOnNext = status >= 400 ? "error" : "success";
|
|
871
|
+
const responseCookies = config.responseCookies || { array: [], serialized: [], netscape: "", string: "", setCookiesString: [] };
|
|
872
|
+
const mergedCookies = mergeRequestAndResponseCookies(config, responseCookies, url.href);
|
|
818
873
|
const finalResponse = {
|
|
819
874
|
data,
|
|
820
875
|
status,
|
|
821
876
|
statusText,
|
|
822
|
-
headers: new RezoHeaders(responseHeaders),
|
|
823
|
-
cookies:
|
|
877
|
+
headers: new RezoHeaders(sanitizeHttp2Headers(responseHeaders)),
|
|
878
|
+
cookies: mergedCookies,
|
|
824
879
|
config,
|
|
825
880
|
contentType,
|
|
826
881
|
contentLength: contentLengthCounter,
|
|
@@ -833,11 +888,11 @@ async function executeHttp2Stream(config, fetchOptions, requestCount, timing, _s
|
|
|
833
888
|
const downloadFinishEvent = {
|
|
834
889
|
status,
|
|
835
890
|
statusText,
|
|
836
|
-
headers: new RezoHeaders(responseHeaders),
|
|
891
|
+
headers: new RezoHeaders(sanitizeHttp2Headers(responseHeaders)),
|
|
837
892
|
contentType,
|
|
838
893
|
contentLength: responseBody.length,
|
|
839
894
|
finalUrl: url.href,
|
|
840
|
-
cookies:
|
|
895
|
+
cookies: mergedCookies,
|
|
841
896
|
urls: buildUrlTree(config, url.href),
|
|
842
897
|
fileName: config.fileName,
|
|
843
898
|
fileSize: responseBody.length,
|
|
@@ -858,7 +913,7 @@ async function executeHttp2Stream(config, fetchOptions, requestCount, timing, _s
|
|
|
858
913
|
} catch (err) {
|
|
859
914
|
const error = buildDownloadError({
|
|
860
915
|
statusCode: status,
|
|
861
|
-
headers: responseHeaders,
|
|
916
|
+
headers: sanitizeHttp2Headers(responseHeaders),
|
|
862
917
|
contentType,
|
|
863
918
|
contentLength: String(contentLengthCounter),
|
|
864
919
|
cookies: config.responseCookies?.setCookiesString || [],
|
|
@@ -878,7 +933,7 @@ async function executeHttp2Stream(config, fetchOptions, requestCount, timing, _s
|
|
|
878
933
|
const streamFinishEvent = {
|
|
879
934
|
status,
|
|
880
935
|
statusText,
|
|
881
|
-
headers: new RezoHeaders(responseHeaders),
|
|
936
|
+
headers: new RezoHeaders(sanitizeHttp2Headers(responseHeaders)),
|
|
882
937
|
contentType,
|
|
883
938
|
contentLength: contentLengthCounter,
|
|
884
939
|
finalUrl: url.href,
|
|
@@ -904,7 +959,7 @@ async function executeHttp2Stream(config, fetchOptions, requestCount, timing, _s
|
|
|
904
959
|
response: {
|
|
905
960
|
status,
|
|
906
961
|
statusText,
|
|
907
|
-
headers: new RezoHeaders(responseHeaders),
|
|
962
|
+
headers: new RezoHeaders(sanitizeHttp2Headers(responseHeaders)),
|
|
908
963
|
data,
|
|
909
964
|
contentType,
|
|
910
965
|
contentLength: contentLengthCounter
|
|
@@ -949,7 +1004,7 @@ async function executeHttp2Stream(config, fetchOptions, requestCount, timing, _s
|
|
|
949
1004
|
body.pipe(req);
|
|
950
1005
|
return;
|
|
951
1006
|
} else {
|
|
952
|
-
const form = await RezoFormData.fromNativeFormData(body);
|
|
1007
|
+
const form = fetchOptions._convertedFormData || await RezoFormData.fromNativeFormData(body);
|
|
953
1008
|
form.pipe(req);
|
|
954
1009
|
return;
|
|
955
1010
|
}
|
package/dist/adapters/http2.js
CHANGED
|
@@ -4,14 +4,15 @@ import { URL } from "node:url";
|
|
|
4
4
|
import { Readable } from "node:stream";
|
|
5
5
|
import { RezoError } from '../errors/rezo-error.js';
|
|
6
6
|
import { buildSmartError, buildDecompressionError, builErrorFromResponse, buildDownloadError } from '../responses/buildError.js';
|
|
7
|
-
import {
|
|
7
|
+
import { RezoCookieJar } from '../utils/cookies.js';
|
|
8
8
|
import RezoFormData from '../utils/form-data.js';
|
|
9
9
|
import { getDefaultConfig, prepareHTTPOptions } from '../utils/http-config.js';
|
|
10
|
-
import { RezoHeaders } from '../utils/headers.js';
|
|
10
|
+
import { RezoHeaders, sanitizeHttp2Headers } from '../utils/headers.js';
|
|
11
11
|
import { RezoURLSearchParams } from '../utils/data-operations.js';
|
|
12
12
|
import { StreamResponse } from '../responses/stream.js';
|
|
13
13
|
import { DownloadResponse } from '../responses/download.js';
|
|
14
14
|
import { UploadResponse } from '../responses/upload.js';
|
|
15
|
+
import { CompressionUtil } from '../utils/compression.js';
|
|
15
16
|
import { isSameDomain, RezoPerformance } from '../utils/tools.js';
|
|
16
17
|
import { ResponseCache } from '../cache/response-cache.js';
|
|
17
18
|
let zstdDecompressSync = null;
|
|
@@ -142,15 +143,14 @@ class Http2SessionPool {
|
|
|
142
143
|
reject(new Error(`HTTP/2 connection timeout after ${timeout}ms`));
|
|
143
144
|
}
|
|
144
145
|
}, timeout) : null;
|
|
146
|
+
if (timeoutId && typeof timeoutId === "object" && "unref" in timeoutId) {
|
|
147
|
+
timeoutId.unref();
|
|
148
|
+
}
|
|
145
149
|
session.on("connect", () => {
|
|
146
150
|
if (!settled) {
|
|
147
151
|
settled = true;
|
|
148
152
|
if (timeoutId)
|
|
149
153
|
clearTimeout(timeoutId);
|
|
150
|
-
const socket = session.socket;
|
|
151
|
-
if (socket && typeof socket.unref === "function") {
|
|
152
|
-
socket.unref();
|
|
153
|
-
}
|
|
154
154
|
resolve(session);
|
|
155
155
|
}
|
|
156
156
|
});
|
|
@@ -170,6 +170,12 @@ class Http2SessionPool {
|
|
|
170
170
|
if (entry) {
|
|
171
171
|
entry.refCount = Math.max(0, entry.refCount - 1);
|
|
172
172
|
entry.lastUsed = Date.now();
|
|
173
|
+
if (entry.refCount === 0) {
|
|
174
|
+
const socket = entry.session.socket;
|
|
175
|
+
if (socket && typeof socket.unref === "function") {
|
|
176
|
+
socket.unref();
|
|
177
|
+
}
|
|
178
|
+
}
|
|
173
179
|
}
|
|
174
180
|
}
|
|
175
181
|
closeSession(url) {
|
|
@@ -275,38 +281,66 @@ function updateCookies(config, headers, url) {
|
|
|
275
281
|
if (!setCookieHeaders)
|
|
276
282
|
return;
|
|
277
283
|
const cookieHeaderArray = Array.isArray(setCookieHeaders) ? setCookieHeaders : [setCookieHeaders];
|
|
278
|
-
if (
|
|
279
|
-
|
|
280
|
-
|
|
281
|
-
|
|
282
|
-
|
|
283
|
-
|
|
284
|
-
setCookiesString: []
|
|
285
|
-
};
|
|
284
|
+
if (cookieHeaderArray.length === 0)
|
|
285
|
+
return;
|
|
286
|
+
const jar = new RezoCookieJar;
|
|
287
|
+
jar.setCookiesSync(cookieHeaderArray, url);
|
|
288
|
+
if (config.enableCookieJar && config.cookieJar) {
|
|
289
|
+
config.cookieJar.setCookiesSync(cookieHeaderArray, url);
|
|
286
290
|
}
|
|
287
|
-
|
|
288
|
-
|
|
289
|
-
|
|
290
|
-
const
|
|
291
|
-
const
|
|
292
|
-
|
|
293
|
-
|
|
294
|
-
|
|
295
|
-
|
|
296
|
-
|
|
297
|
-
|
|
298
|
-
|
|
299
|
-
|
|
300
|
-
|
|
301
|
-
|
|
302
|
-
|
|
303
|
-
|
|
291
|
+
const cookies = jar.cookies();
|
|
292
|
+
cookies.setCookiesString = cookieHeaderArray;
|
|
293
|
+
if (config.useCookies) {
|
|
294
|
+
const existingArray = config.responseCookies?.array || [];
|
|
295
|
+
for (const cookie of cookies.array) {
|
|
296
|
+
const existingIndex = existingArray.findIndex((c) => c.key === cookie.key && c.domain === cookie.domain);
|
|
297
|
+
if (existingIndex >= 0) {
|
|
298
|
+
existingArray[existingIndex] = cookie;
|
|
299
|
+
} else {
|
|
300
|
+
existingArray.push(cookie);
|
|
301
|
+
}
|
|
302
|
+
}
|
|
303
|
+
const mergedJar = new RezoCookieJar(existingArray, url);
|
|
304
|
+
config.responseCookies = mergedJar.cookies();
|
|
305
|
+
config.responseCookies.setCookiesString = cookieHeaderArray;
|
|
306
|
+
} else {
|
|
307
|
+
config.responseCookies = cookies;
|
|
308
|
+
}
|
|
309
|
+
}
|
|
310
|
+
function mergeRequestAndResponseCookies(config, responseCookies, url) {
|
|
311
|
+
const mergedCookiesArray = [];
|
|
312
|
+
const cookieKeyDomainMap = new Map;
|
|
313
|
+
if (config.requestCookies && config.requestCookies.length > 0) {
|
|
314
|
+
for (const cookie of config.requestCookies) {
|
|
315
|
+
const key = `${cookie.key}|${cookie.domain || ""}`;
|
|
316
|
+
mergedCookiesArray.push(cookie);
|
|
317
|
+
cookieKeyDomainMap.set(key, mergedCookiesArray.length - 1);
|
|
304
318
|
}
|
|
305
319
|
}
|
|
306
|
-
|
|
307
|
-
|
|
308
|
-
|
|
309
|
-
|
|
320
|
+
for (const cookie of responseCookies.array) {
|
|
321
|
+
const key = `${cookie.key}|${cookie.domain || ""}`;
|
|
322
|
+
const existingIndex = cookieKeyDomainMap.get(key);
|
|
323
|
+
if (existingIndex !== undefined) {
|
|
324
|
+
mergedCookiesArray[existingIndex] = cookie;
|
|
325
|
+
} else {
|
|
326
|
+
mergedCookiesArray.push(cookie);
|
|
327
|
+
cookieKeyDomainMap.set(key, mergedCookiesArray.length - 1);
|
|
328
|
+
}
|
|
329
|
+
}
|
|
330
|
+
if (mergedCookiesArray.length > 0) {
|
|
331
|
+
const mergedJar = new RezoCookieJar(mergedCookiesArray, url);
|
|
332
|
+
return mergedJar.cookies();
|
|
333
|
+
}
|
|
334
|
+
return {
|
|
335
|
+
array: [],
|
|
336
|
+
serialized: [],
|
|
337
|
+
netscape: `# Netscape HTTP Cookie File
|
|
338
|
+
# This file was generated by Rezo HTTP client
|
|
339
|
+
# Based on uniqhtt cookie implementation
|
|
340
|
+
`,
|
|
341
|
+
string: "",
|
|
342
|
+
setCookiesString: []
|
|
343
|
+
};
|
|
310
344
|
}
|
|
311
345
|
export async function executeRequest(options, defaultOptions, jar) {
|
|
312
346
|
if (!options.responseType) {
|
|
@@ -484,6 +518,8 @@ async function executeHttp2Request(fetchOptions, config, options, perform, fs, s
|
|
|
484
518
|
if (fileName && fs && fs.existsSync(fileName)) {
|
|
485
519
|
fs.unlinkSync(fileName);
|
|
486
520
|
}
|
|
521
|
+
if (!config.errors)
|
|
522
|
+
config.errors = [];
|
|
487
523
|
config.errors.push({
|
|
488
524
|
attempt: config.retryAttempts + 1,
|
|
489
525
|
error: response,
|
|
@@ -521,7 +557,7 @@ async function executeHttp2Request(fetchOptions, config, options, perform, fs, s
|
|
|
521
557
|
}
|
|
522
558
|
continue;
|
|
523
559
|
}
|
|
524
|
-
if (statusOnNext === "success") {
|
|
560
|
+
if (statusOnNext === "success" || statusOnNext === "error") {
|
|
525
561
|
return response;
|
|
526
562
|
}
|
|
527
563
|
if (statusOnNext === "redirect") {
|
|
@@ -620,6 +656,13 @@ async function executeHttp2Stream(config, fetchOptions, requestCount, timing, _s
|
|
|
620
656
|
if (!headers["accept-encoding"]) {
|
|
621
657
|
headers["accept-encoding"] = "gzip, deflate, br";
|
|
622
658
|
}
|
|
659
|
+
if (body instanceof RezoFormData) {
|
|
660
|
+
headers["content-type"] = `multipart/form-data; boundary=${body.getBoundary()}`;
|
|
661
|
+
} else if (body instanceof FormData) {
|
|
662
|
+
const tempForm = await RezoFormData.fromNativeFormData(body);
|
|
663
|
+
headers["content-type"] = `multipart/form-data; boundary=${tempForm.getBoundary()}`;
|
|
664
|
+
fetchOptions._convertedFormData = tempForm;
|
|
665
|
+
}
|
|
623
666
|
const eventEmitter = streamResult || downloadResult || uploadResult;
|
|
624
667
|
if (eventEmitter && requestCount === 0) {
|
|
625
668
|
const startEvent = {
|
|
@@ -735,6 +778,25 @@ async function executeHttp2Stream(config, fetchOptions, requestCount, timing, _s
|
|
|
735
778
|
config.timing.endTimestamp = Date.now();
|
|
736
779
|
config.timing.durationMs = performance.now() - timing.startTime;
|
|
737
780
|
config.timing.transferMs = timing.firstByteTime ? performance.now() - timing.firstByteTime : config.timing.durationMs;
|
|
781
|
+
if (!config.transfer) {
|
|
782
|
+
config.transfer = { requestSize: 0, responseSize: 0, headerSize: 0, bodySize: 0 };
|
|
783
|
+
}
|
|
784
|
+
if (config.transfer.requestSize === undefined) {
|
|
785
|
+
config.transfer.requestSize = 0;
|
|
786
|
+
}
|
|
787
|
+
if (config.transfer.requestSize === 0 && body) {
|
|
788
|
+
if (typeof body === "string") {
|
|
789
|
+
config.transfer.requestSize = Buffer.byteLength(body, "utf8");
|
|
790
|
+
} else if (body instanceof Buffer || body instanceof Uint8Array) {
|
|
791
|
+
config.transfer.requestSize = body.length;
|
|
792
|
+
} else if (body instanceof URLSearchParams || body instanceof RezoURLSearchParams) {
|
|
793
|
+
config.transfer.requestSize = Buffer.byteLength(body.toString(), "utf8");
|
|
794
|
+
} else if (body instanceof RezoFormData) {
|
|
795
|
+
config.transfer.requestSize = body.getLengthSync();
|
|
796
|
+
} else if (typeof body === "object") {
|
|
797
|
+
config.transfer.requestSize = Buffer.byteLength(JSON.stringify(body), "utf8");
|
|
798
|
+
}
|
|
799
|
+
}
|
|
738
800
|
config.transfer.bodySize = contentLengthCounter;
|
|
739
801
|
config.transfer.responseSize = contentLengthCounter;
|
|
740
802
|
(sessionPool || Http2SessionPool.getInstance()).releaseSession(url);
|
|
@@ -743,7 +805,7 @@ async function executeHttp2Stream(config, fetchOptions, requestCount, timing, _s
|
|
|
743
805
|
data: "",
|
|
744
806
|
status,
|
|
745
807
|
statusText,
|
|
746
|
-
headers: new RezoHeaders(responseHeaders),
|
|
808
|
+
headers: new RezoHeaders(sanitizeHttp2Headers(responseHeaders)),
|
|
747
809
|
cookies: config.responseCookies || { array: [], serialized: [], netscape: "", string: "", setCookiesString: [] },
|
|
748
810
|
config,
|
|
749
811
|
contentType: responseHeaders["content-type"],
|
|
@@ -756,14 +818,14 @@ async function executeHttp2Stream(config, fetchOptions, requestCount, timing, _s
|
|
|
756
818
|
}
|
|
757
819
|
let responseBody = Buffer.concat(chunks);
|
|
758
820
|
const contentEncoding = responseHeaders["content-encoding"];
|
|
759
|
-
if (contentEncoding && contentLengthCounter > 0) {
|
|
821
|
+
if (contentEncoding && contentLengthCounter > 0 && CompressionUtil.shouldDecompress(contentEncoding, config)) {
|
|
760
822
|
try {
|
|
761
823
|
const decompressed = await decompressBuffer(responseBody, contentEncoding);
|
|
762
824
|
responseBody = decompressed;
|
|
763
825
|
} catch (err) {
|
|
764
826
|
const error = buildDecompressionError({
|
|
765
827
|
statusCode: status,
|
|
766
|
-
headers: responseHeaders,
|
|
828
|
+
headers: sanitizeHttp2Headers(responseHeaders),
|
|
767
829
|
contentType: responseHeaders["content-type"],
|
|
768
830
|
contentLength: String(contentLengthCounter),
|
|
769
831
|
cookies: config.responseCookies?.setCookiesString || [],
|
|
@@ -803,24 +865,17 @@ async function executeHttp2Stream(config, fetchOptions, requestCount, timing, _s
|
|
|
803
865
|
data = responseBody.toString("utf-8");
|
|
804
866
|
}
|
|
805
867
|
}
|
|
806
|
-
|
|
807
|
-
|
|
808
|
-
|
|
809
|
-
|
|
810
|
-
|
|
811
|
-
data
|
|
812
|
-
}, config, fetchOptions);
|
|
813
|
-
_stats.statusOnNext = "error";
|
|
814
|
-
resolve(error);
|
|
815
|
-
return;
|
|
816
|
-
}
|
|
817
|
-
_stats.statusOnNext = "success";
|
|
868
|
+
config.status = status;
|
|
869
|
+
config.statusText = statusText;
|
|
870
|
+
_stats.statusOnNext = status >= 400 ? "error" : "success";
|
|
871
|
+
const responseCookies = config.responseCookies || { array: [], serialized: [], netscape: "", string: "", setCookiesString: [] };
|
|
872
|
+
const mergedCookies = mergeRequestAndResponseCookies(config, responseCookies, url.href);
|
|
818
873
|
const finalResponse = {
|
|
819
874
|
data,
|
|
820
875
|
status,
|
|
821
876
|
statusText,
|
|
822
|
-
headers: new RezoHeaders(responseHeaders),
|
|
823
|
-
cookies:
|
|
877
|
+
headers: new RezoHeaders(sanitizeHttp2Headers(responseHeaders)),
|
|
878
|
+
cookies: mergedCookies,
|
|
824
879
|
config,
|
|
825
880
|
contentType,
|
|
826
881
|
contentLength: contentLengthCounter,
|
|
@@ -833,11 +888,11 @@ async function executeHttp2Stream(config, fetchOptions, requestCount, timing, _s
|
|
|
833
888
|
const downloadFinishEvent = {
|
|
834
889
|
status,
|
|
835
890
|
statusText,
|
|
836
|
-
headers: new RezoHeaders(responseHeaders),
|
|
891
|
+
headers: new RezoHeaders(sanitizeHttp2Headers(responseHeaders)),
|
|
837
892
|
contentType,
|
|
838
893
|
contentLength: responseBody.length,
|
|
839
894
|
finalUrl: url.href,
|
|
840
|
-
cookies:
|
|
895
|
+
cookies: mergedCookies,
|
|
841
896
|
urls: buildUrlTree(config, url.href),
|
|
842
897
|
fileName: config.fileName,
|
|
843
898
|
fileSize: responseBody.length,
|
|
@@ -858,7 +913,7 @@ async function executeHttp2Stream(config, fetchOptions, requestCount, timing, _s
|
|
|
858
913
|
} catch (err) {
|
|
859
914
|
const error = buildDownloadError({
|
|
860
915
|
statusCode: status,
|
|
861
|
-
headers: responseHeaders,
|
|
916
|
+
headers: sanitizeHttp2Headers(responseHeaders),
|
|
862
917
|
contentType,
|
|
863
918
|
contentLength: String(contentLengthCounter),
|
|
864
919
|
cookies: config.responseCookies?.setCookiesString || [],
|
|
@@ -878,7 +933,7 @@ async function executeHttp2Stream(config, fetchOptions, requestCount, timing, _s
|
|
|
878
933
|
const streamFinishEvent = {
|
|
879
934
|
status,
|
|
880
935
|
statusText,
|
|
881
|
-
headers: new RezoHeaders(responseHeaders),
|
|
936
|
+
headers: new RezoHeaders(sanitizeHttp2Headers(responseHeaders)),
|
|
882
937
|
contentType,
|
|
883
938
|
contentLength: contentLengthCounter,
|
|
884
939
|
finalUrl: url.href,
|
|
@@ -904,7 +959,7 @@ async function executeHttp2Stream(config, fetchOptions, requestCount, timing, _s
|
|
|
904
959
|
response: {
|
|
905
960
|
status,
|
|
906
961
|
statusText,
|
|
907
|
-
headers: new RezoHeaders(responseHeaders),
|
|
962
|
+
headers: new RezoHeaders(sanitizeHttp2Headers(responseHeaders)),
|
|
908
963
|
data,
|
|
909
964
|
contentType,
|
|
910
965
|
contentLength: contentLengthCounter
|
|
@@ -949,7 +1004,7 @@ async function executeHttp2Stream(config, fetchOptions, requestCount, timing, _s
|
|
|
949
1004
|
body.pipe(req);
|
|
950
1005
|
return;
|
|
951
1006
|
} else {
|
|
952
|
-
const form = await RezoFormData.fromNativeFormData(body);
|
|
1007
|
+
const form = fetchOptions._convertedFormData || await RezoFormData.fromNativeFormData(body);
|
|
953
1008
|
form.pipe(req);
|
|
954
1009
|
return;
|
|
955
1010
|
}
|
package/dist/adapters/index.cjs
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
|
-
const
|
|
2
|
-
exports.detectRuntime =
|
|
3
|
-
exports.getAdapterCapabilities =
|
|
4
|
-
exports.buildAdapterContext =
|
|
5
|
-
exports.getAvailableAdapters =
|
|
6
|
-
exports.selectAdapter =
|
|
1
|
+
const _mod_h6yyuw = require('./picker.cjs');
|
|
2
|
+
exports.detectRuntime = _mod_h6yyuw.detectRuntime;
|
|
3
|
+
exports.getAdapterCapabilities = _mod_h6yyuw.getAdapterCapabilities;
|
|
4
|
+
exports.buildAdapterContext = _mod_h6yyuw.buildAdapterContext;
|
|
5
|
+
exports.getAvailableAdapters = _mod_h6yyuw.getAvailableAdapters;
|
|
6
|
+
exports.selectAdapter = _mod_h6yyuw.selectAdapter;;
|
package/dist/cache/index.cjs
CHANGED
|
@@ -1,13 +1,13 @@
|
|
|
1
|
-
const
|
|
2
|
-
exports.LRUCache =
|
|
3
|
-
const
|
|
4
|
-
exports.DNSCache =
|
|
5
|
-
exports.getGlobalDNSCache =
|
|
6
|
-
exports.resetGlobalDNSCache =
|
|
7
|
-
const
|
|
8
|
-
exports.ResponseCache =
|
|
9
|
-
exports.normalizeResponseCacheConfig =
|
|
10
|
-
const
|
|
11
|
-
exports.FileCacher =
|
|
12
|
-
const
|
|
13
|
-
exports.UrlStore =
|
|
1
|
+
const _mod_vgyhxh = require('./lru-cache.cjs');
|
|
2
|
+
exports.LRUCache = _mod_vgyhxh.LRUCache;;
|
|
3
|
+
const _mod_57d30y = require('./dns-cache.cjs');
|
|
4
|
+
exports.DNSCache = _mod_57d30y.DNSCache;
|
|
5
|
+
exports.getGlobalDNSCache = _mod_57d30y.getGlobalDNSCache;
|
|
6
|
+
exports.resetGlobalDNSCache = _mod_57d30y.resetGlobalDNSCache;;
|
|
7
|
+
const _mod_qx55fg = require('./response-cache.cjs');
|
|
8
|
+
exports.ResponseCache = _mod_qx55fg.ResponseCache;
|
|
9
|
+
exports.normalizeResponseCacheConfig = _mod_qx55fg.normalizeResponseCacheConfig;;
|
|
10
|
+
const _mod_clz0cm = require('./file-cacher.cjs');
|
|
11
|
+
exports.FileCacher = _mod_clz0cm.FileCacher;;
|
|
12
|
+
const _mod_39vswi = require('./url-store.cjs');
|
|
13
|
+
exports.UrlStore = _mod_39vswi.UrlStore;;
|