rezo 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +202 -0
- package/README.md +1507 -0
- package/assets/icon.svg +37 -0
- package/assets/logo-dark.svg +47 -0
- package/assets/logo.svg +58 -0
- package/dist/adapters/curl.cjs +1034 -0
- package/dist/adapters/curl.js +1031 -0
- package/dist/adapters/entries/curl.cjs +4 -0
- package/dist/adapters/entries/curl.d.ts +2136 -0
- package/dist/adapters/entries/curl.js +2 -0
- package/dist/adapters/entries/fetch.cjs +2 -0
- package/dist/adapters/entries/fetch.d.ts +2127 -0
- package/dist/adapters/entries/fetch.js +1 -0
- package/dist/adapters/entries/http.cjs +2 -0
- package/dist/adapters/entries/http.d.ts +2126 -0
- package/dist/adapters/entries/http.js +1 -0
- package/dist/adapters/entries/http2.cjs +4 -0
- package/dist/adapters/entries/http2.d.ts +2136 -0
- package/dist/adapters/entries/http2.js +2 -0
- package/dist/adapters/entries/react-native.cjs +2 -0
- package/dist/adapters/entries/react-native.d.ts +2126 -0
- package/dist/adapters/entries/react-native.js +1 -0
- package/dist/adapters/entries/xhr.cjs +2 -0
- package/dist/adapters/entries/xhr.d.ts +2127 -0
- package/dist/adapters/entries/xhr.js +1 -0
- package/dist/adapters/fetch.cjs +740 -0
- package/dist/adapters/fetch.js +739 -0
- package/dist/adapters/http.cjs +1153 -0
- package/dist/adapters/http.js +1151 -0
- package/dist/adapters/http2.cjs +957 -0
- package/dist/adapters/http2.js +956 -0
- package/dist/adapters/index.cjs +6 -0
- package/dist/adapters/index.js +7 -0
- package/dist/adapters/picker.cjs +342 -0
- package/dist/adapters/picker.js +331 -0
- package/dist/adapters/react-native.cjs +545 -0
- package/dist/adapters/react-native.js +544 -0
- package/dist/adapters/xhr.cjs +622 -0
- package/dist/adapters/xhr.js +621 -0
- package/dist/cache/dns-cache.cjs +118 -0
- package/dist/cache/dns-cache.js +113 -0
- package/dist/cache/file-cacher.cjs +264 -0
- package/dist/cache/file-cacher.js +261 -0
- package/dist/cache/index.cjs +13 -0
- package/dist/cache/index.js +5 -0
- package/dist/cache/lru-cache.cjs +96 -0
- package/dist/cache/lru-cache.js +93 -0
- package/dist/cache/response-cache.cjs +314 -0
- package/dist/cache/response-cache.js +310 -0
- package/dist/cache/url-store.cjs +288 -0
- package/dist/cache/url-store.js +285 -0
- package/dist/core/hooks.cjs +133 -0
- package/dist/core/hooks.js +120 -0
- package/dist/core/rezo.cjs +464 -0
- package/dist/core/rezo.js +458 -0
- package/dist/crawler.d.ts +6255 -0
- package/dist/dom/index.cjs +1 -0
- package/dist/dom/index.d.ts +23 -0
- package/dist/dom/index.js +1 -0
- package/dist/entries/crawler.cjs +5 -0
- package/dist/entries/crawler.js +2 -0
- package/dist/errors/rezo-error.cjs +722 -0
- package/dist/errors/rezo-error.js +716 -0
- package/dist/index.cjs +34 -0
- package/dist/index.d.ts +3335 -0
- package/dist/index.js +26 -0
- package/dist/platform/browser.cjs +9 -0
- package/dist/platform/browser.d.ts +3203 -0
- package/dist/platform/browser.js +7 -0
- package/dist/platform/bun.cjs +9 -0
- package/dist/platform/bun.d.ts +3203 -0
- package/dist/platform/bun.js +7 -0
- package/dist/platform/deno.cjs +9 -0
- package/dist/platform/deno.d.ts +3203 -0
- package/dist/platform/deno.js +7 -0
- package/dist/platform/node.cjs +9 -0
- package/dist/platform/node.d.ts +3203 -0
- package/dist/platform/node.js +7 -0
- package/dist/platform/react-native.cjs +9 -0
- package/dist/platform/react-native.d.ts +3203 -0
- package/dist/platform/react-native.js +7 -0
- package/dist/platform/worker.cjs +9 -0
- package/dist/platform/worker.d.ts +3203 -0
- package/dist/platform/worker.js +7 -0
- package/dist/plugin/addon/decodo/index.cjs +1 -0
- package/dist/plugin/addon/decodo/index.js +1 -0
- package/dist/plugin/addon/decodo/options.cjs +1 -0
- package/dist/plugin/addon/decodo/options.js +1 -0
- package/dist/plugin/addon/oxylabs/index.cjs +1 -0
- package/dist/plugin/addon/oxylabs/index.js +1 -0
- package/dist/plugin/addon/oxylabs/options.cjs +1 -0
- package/dist/plugin/addon/oxylabs/options.js +1 -0
- package/dist/plugin/crawler-options.cjs +1 -0
- package/dist/plugin/crawler-options.js +1 -0
- package/dist/plugin/crawler.cjs +519 -0
- package/dist/plugin/crawler.js +517 -0
- package/dist/plugin/index.cjs +36 -0
- package/dist/plugin/index.js +32 -0
- package/dist/proxy/index.cjs +142 -0
- package/dist/proxy/index.js +139 -0
- package/dist/responses/buildError.cjs +452 -0
- package/dist/responses/buildError.js +441 -0
- package/dist/responses/buildResponse.cjs +365 -0
- package/dist/responses/buildResponse.js +361 -0
- package/dist/responses/download.cjs +54 -0
- package/dist/responses/download.js +52 -0
- package/dist/responses/stream.cjs +60 -0
- package/dist/responses/stream.js +58 -0
- package/dist/responses/upload.cjs +54 -0
- package/dist/responses/upload.js +52 -0
- package/dist/types/cookies.cjs +394 -0
- package/dist/types/cookies.js +391 -0
- package/dist/types/download.cjs +10 -0
- package/dist/types/download.js +10 -0
- package/dist/types/rezo-request.cjs +131 -0
- package/dist/types/rezo-request.js +131 -0
- package/dist/utils/agent-merger.cjs +111 -0
- package/dist/utils/agent-merger.js +108 -0
- package/dist/utils/compression.cjs +84 -0
- package/dist/utils/compression.js +82 -0
- package/dist/utils/cookies.cjs +514 -0
- package/dist/utils/cookies.js +511 -0
- package/dist/utils/data-operations.cjs +75 -0
- package/dist/utils/data-operations.js +73 -0
- package/dist/utils/form-data.cjs +164 -0
- package/dist/utils/form-data.js +161 -0
- package/dist/utils/headers.cjs +162 -0
- package/dist/utils/headers.js +161 -0
- package/dist/utils/http-config.cjs +723 -0
- package/dist/utils/http-config.js +718 -0
- package/dist/utils/index.cjs +8 -0
- package/dist/utils/index.js +8 -0
- package/dist/utils/tools.cjs +18 -0
- package/dist/utils/tools.js +15 -0
- package/package.json +172 -0
|
@@ -0,0 +1,1153 @@
|
|
|
1
|
+
const http = require("node:http");
|
|
2
|
+
const https = require("node:https");
|
|
3
|
+
const tls = require("node:tls");
|
|
4
|
+
const { URL } = require("node:url");
|
|
5
|
+
const { Readable } = require("node:stream");
|
|
6
|
+
const { RezoError } = require('../errors/rezo-error.cjs');
|
|
7
|
+
const { RezoCookieJar } = require('../utils/cookies.cjs');
|
|
8
|
+
const RezoHeaders = require('../utils/headers.cjs');
|
|
9
|
+
const { getDefaultConfig, prepareHTTPOptions } = require('../utils/http-config.cjs');
|
|
10
|
+
const { RezoURLSearchParams } = require('../utils/data-operations.cjs');
|
|
11
|
+
const RezoFormData = require('../utils/form-data.cjs');
|
|
12
|
+
const { rezoProxy } = require('../proxy/index.cjs');
|
|
13
|
+
const { StreamResponse } = require('../responses/stream.cjs');
|
|
14
|
+
const { DownloadResponse } = require('../responses/download.cjs');
|
|
15
|
+
const { UploadResponse } = require('../responses/upload.cjs');
|
|
16
|
+
const { CompressionUtil } = require('../utils/compression.cjs');
|
|
17
|
+
const { buildResponseFromIncoming, buildDownloadResponse } = require('../responses/buildResponse.cjs');
|
|
18
|
+
const { buildDownloadError, buildDecompressionError, buildSmartError, builErrorFromResponse } = require('../responses/buildError.cjs');
|
|
19
|
+
const { isSameDomain, RezoPerformance } = require('../utils/tools.cjs');
|
|
20
|
+
const { getGlobalDNSCache } = require('../cache/dns-cache.cjs');
|
|
21
|
+
const { ResponseCache } = require('../cache/response-cache.cjs');
|
|
22
|
+
const dns = require("dns");
|
|
23
|
+
const responseCacheInstances = new Map;
|
|
24
|
+
function getCacheConfigKey(option) {
|
|
25
|
+
if (option === true)
|
|
26
|
+
return "default";
|
|
27
|
+
if (option === false)
|
|
28
|
+
return "disabled";
|
|
29
|
+
const cfg = option;
|
|
30
|
+
return JSON.stringify({
|
|
31
|
+
cacheDir: cfg.cacheDir || null,
|
|
32
|
+
ttl: cfg.ttl || 300000,
|
|
33
|
+
maxEntries: cfg.maxEntries || 500,
|
|
34
|
+
methods: cfg.methods || ["GET", "HEAD"],
|
|
35
|
+
respectHeaders: cfg.respectHeaders !== false
|
|
36
|
+
});
|
|
37
|
+
}
|
|
38
|
+
function getResponseCache(option) {
|
|
39
|
+
const key = getCacheConfigKey(option);
|
|
40
|
+
let cache = responseCacheInstances.get(key);
|
|
41
|
+
if (!cache) {
|
|
42
|
+
cache = new ResponseCache(option);
|
|
43
|
+
responseCacheInstances.set(key, cache);
|
|
44
|
+
}
|
|
45
|
+
return cache;
|
|
46
|
+
}
|
|
47
|
+
function parseCacheControlFromHeaders(headers) {
|
|
48
|
+
const cacheControl = headers["cache-control"] || "";
|
|
49
|
+
return {
|
|
50
|
+
noCache: cacheControl.includes("no-cache"),
|
|
51
|
+
mustRevalidate: cacheControl.includes("must-revalidate")
|
|
52
|
+
};
|
|
53
|
+
}
|
|
54
|
+
function buildCachedRezoResponse(cached, config) {
|
|
55
|
+
const headers = new RezoHeaders(cached.headers);
|
|
56
|
+
return {
|
|
57
|
+
data: cached.data,
|
|
58
|
+
status: cached.status,
|
|
59
|
+
statusText: cached.statusText,
|
|
60
|
+
headers,
|
|
61
|
+
finalUrl: cached.url,
|
|
62
|
+
urls: [cached.url],
|
|
63
|
+
contentType: cached.headers["content-type"],
|
|
64
|
+
contentLength: parseInt(cached.headers["content-length"] || "0", 10) || 0,
|
|
65
|
+
cookies: {
|
|
66
|
+
array: [],
|
|
67
|
+
serialized: [],
|
|
68
|
+
netscape: "",
|
|
69
|
+
string: "",
|
|
70
|
+
setCookiesString: []
|
|
71
|
+
},
|
|
72
|
+
config: {
|
|
73
|
+
...config,
|
|
74
|
+
url: cached.url,
|
|
75
|
+
method: "GET",
|
|
76
|
+
headers,
|
|
77
|
+
adapterUsed: "http",
|
|
78
|
+
fromCache: true
|
|
79
|
+
}
|
|
80
|
+
};
|
|
81
|
+
}
|
|
82
|
+
function buildUrlTree(config, finalUrl) {
|
|
83
|
+
const urls = [];
|
|
84
|
+
if (config.rawUrl) {
|
|
85
|
+
urls.push(config.rawUrl);
|
|
86
|
+
} else if (config.url) {
|
|
87
|
+
const urlStr = typeof config.url === "string" ? config.url : config.url.toString();
|
|
88
|
+
urls.push(urlStr);
|
|
89
|
+
}
|
|
90
|
+
if (finalUrl && (urls.length === 0 || urls[0] !== finalUrl)) {
|
|
91
|
+
urls.push(finalUrl);
|
|
92
|
+
}
|
|
93
|
+
return urls.length > 0 ? urls : [finalUrl];
|
|
94
|
+
}
|
|
95
|
+
async function executeRequest(options, defaultOptions, jar) {
|
|
96
|
+
if (!options.responseType) {
|
|
97
|
+
options.responseType = "auto";
|
|
98
|
+
}
|
|
99
|
+
const d_options = await getDefaultConfig(defaultOptions);
|
|
100
|
+
const config = prepareHTTPOptions(options, jar, { defaultOptions: d_options });
|
|
101
|
+
let mainConfig = config.config;
|
|
102
|
+
const perform = new RezoPerformance;
|
|
103
|
+
const cacheOption = options.cache;
|
|
104
|
+
const method = (options.method || "GET").toUpperCase();
|
|
105
|
+
const requestUrl = typeof config.fetchOptions.url === "string" ? config.fetchOptions.url : config.fetchOptions.url?.toString() || "";
|
|
106
|
+
let cache;
|
|
107
|
+
let requestHeaders;
|
|
108
|
+
let cachedEntry;
|
|
109
|
+
let needsRevalidation = false;
|
|
110
|
+
if (cacheOption) {
|
|
111
|
+
cache = getResponseCache(cacheOption);
|
|
112
|
+
requestHeaders = config.fetchOptions.headers instanceof RezoHeaders ? Object.fromEntries(config.fetchOptions.headers.entries()) : config.fetchOptions.headers;
|
|
113
|
+
cachedEntry = cache.get(method, requestUrl, requestHeaders);
|
|
114
|
+
if (cachedEntry) {
|
|
115
|
+
const cacheControl = parseCacheControlFromHeaders(cachedEntry.headers);
|
|
116
|
+
if (cacheControl.noCache || cacheControl.mustRevalidate) {
|
|
117
|
+
needsRevalidation = true;
|
|
118
|
+
} else {
|
|
119
|
+
return buildCachedRezoResponse(cachedEntry, mainConfig);
|
|
120
|
+
}
|
|
121
|
+
}
|
|
122
|
+
const conditionalHeaders = cache.getConditionalHeaders(method, requestUrl, requestHeaders);
|
|
123
|
+
if (conditionalHeaders) {
|
|
124
|
+
if (config.fetchOptions.headers instanceof RezoHeaders) {
|
|
125
|
+
for (const [key, value] of Object.entries(conditionalHeaders)) {
|
|
126
|
+
config.fetchOptions.headers.set(key, value);
|
|
127
|
+
}
|
|
128
|
+
} else {
|
|
129
|
+
config.fetchOptions.headers = {
|
|
130
|
+
...config.fetchOptions.headers,
|
|
131
|
+
...conditionalHeaders
|
|
132
|
+
};
|
|
133
|
+
}
|
|
134
|
+
}
|
|
135
|
+
}
|
|
136
|
+
const isStream = options.responseType === "stream" || options._isStream;
|
|
137
|
+
const isDownload = options.responseType === "download" || !!options.fileName || !!options.saveTo || options._isDownload;
|
|
138
|
+
const isUpload = options.responseType === "upload" || options._isUpload;
|
|
139
|
+
if (isUpload && !config.config.data) {
|
|
140
|
+
throw RezoError.fromError(new Error("Upload response type requires a request body (data or body)"), mainConfig, config.fetchOptions);
|
|
141
|
+
}
|
|
142
|
+
let streamResponse;
|
|
143
|
+
let downloadResponse;
|
|
144
|
+
let uploadResponse;
|
|
145
|
+
if (isStream) {
|
|
146
|
+
streamResponse = new StreamResponse;
|
|
147
|
+
} else if (isDownload) {
|
|
148
|
+
const fileName = options.fileName || options.saveTo;
|
|
149
|
+
const url = typeof config.fetchOptions.url === "string" ? config.fetchOptions.url : config.fetchOptions.url.toString();
|
|
150
|
+
downloadResponse = new DownloadResponse(fileName, url);
|
|
151
|
+
} else if (isUpload) {
|
|
152
|
+
const fileName = typeof options.body === "string" ? undefined : options.body?.name;
|
|
153
|
+
const url = typeof config.fetchOptions.url === "string" ? config.fetchOptions.url : config.fetchOptions.url.toString();
|
|
154
|
+
uploadResponse = new UploadResponse(url, fileName);
|
|
155
|
+
}
|
|
156
|
+
const res = executeHttp1Request(config.fetchOptions, mainConfig, config.options, perform, d_options.fs, streamResponse, downloadResponse, uploadResponse);
|
|
157
|
+
if (streamResponse) {
|
|
158
|
+
return streamResponse;
|
|
159
|
+
} else if (downloadResponse) {
|
|
160
|
+
return downloadResponse;
|
|
161
|
+
} else if (uploadResponse) {
|
|
162
|
+
return uploadResponse;
|
|
163
|
+
}
|
|
164
|
+
const response = await res;
|
|
165
|
+
if (cache && !isStream && !isDownload && !isUpload) {
|
|
166
|
+
if (response.status === 304 && cachedEntry) {
|
|
167
|
+
const responseHeaders = response.headers instanceof RezoHeaders ? Object.fromEntries(response.headers.entries()) : response.headers;
|
|
168
|
+
const updatedCached = cache.updateRevalidated(method, requestUrl, responseHeaders, requestHeaders);
|
|
169
|
+
if (updatedCached) {
|
|
170
|
+
return buildCachedRezoResponse(updatedCached, mainConfig);
|
|
171
|
+
}
|
|
172
|
+
return buildCachedRezoResponse(cachedEntry, mainConfig);
|
|
173
|
+
}
|
|
174
|
+
if (response.status >= 200 && response.status < 300) {
|
|
175
|
+
cache.set(method, requestUrl, response, requestHeaders);
|
|
176
|
+
}
|
|
177
|
+
}
|
|
178
|
+
return response;
|
|
179
|
+
}
|
|
180
|
+
async function executeHttp1Request(fetchOptions, config, options, perform, fs, streamResult, downloadResult, uploadResult) {
|
|
181
|
+
let requestCount = 0;
|
|
182
|
+
const _stats = { statusOnNext: "abort" };
|
|
183
|
+
let responseStatusCode;
|
|
184
|
+
let retries = 0;
|
|
185
|
+
const retryDelay = config?.retry?.retryDelay || 0;
|
|
186
|
+
const maxRetries = config?.retry?.maxRetries || 0;
|
|
187
|
+
const incrementDelay = config?.retry?.incrementDelay || false;
|
|
188
|
+
const statusCodes = config?.retry?.statusCodes;
|
|
189
|
+
const timing = {
|
|
190
|
+
startTime: performance.now(),
|
|
191
|
+
startTimestamp: Date.now()
|
|
192
|
+
};
|
|
193
|
+
const ABSOLUTE_MAX_ATTEMPTS = 50;
|
|
194
|
+
const visitedUrls = new Set;
|
|
195
|
+
let totalAttempts = 0;
|
|
196
|
+
config.setSignal();
|
|
197
|
+
const timeoutClearInstanse = config.timeoutClearInstanse;
|
|
198
|
+
delete config.timeoutClearInstanse;
|
|
199
|
+
const eventEmitter = streamResult || downloadResult || uploadResult;
|
|
200
|
+
if (eventEmitter) {
|
|
201
|
+
eventEmitter.emit("initiated");
|
|
202
|
+
}
|
|
203
|
+
while (true) {
|
|
204
|
+
totalAttempts++;
|
|
205
|
+
if (totalAttempts > ABSOLUTE_MAX_ATTEMPTS) {
|
|
206
|
+
const error = builErrorFromResponse(`Absolute maximum attempts (${ABSOLUTE_MAX_ATTEMPTS}) exceeded. This prevents infinite loops from retries and redirects.`, { status: 0, statusText: "Max Attempts Exceeded" }, config, fetchOptions);
|
|
207
|
+
throw error;
|
|
208
|
+
}
|
|
209
|
+
try {
|
|
210
|
+
const response = await request(config, fetchOptions, requestCount, timing, _stats, responseStatusCode, fs, streamResult, downloadResult, uploadResult);
|
|
211
|
+
const statusOnNext = _stats.statusOnNext;
|
|
212
|
+
if (response instanceof RezoError) {
|
|
213
|
+
const fileName = config.fileName;
|
|
214
|
+
if (fileName && fs && fs.existsSync(fileName)) {
|
|
215
|
+
fs.unlinkSync(fileName);
|
|
216
|
+
}
|
|
217
|
+
config.errors.push({
|
|
218
|
+
attempt: config.retryAttempts + 1,
|
|
219
|
+
error: response,
|
|
220
|
+
duration: perform.now()
|
|
221
|
+
});
|
|
222
|
+
perform.reset();
|
|
223
|
+
if (!responseStatusCode || !config.retry) {
|
|
224
|
+
throw response;
|
|
225
|
+
}
|
|
226
|
+
if (config.retry) {
|
|
227
|
+
if (config.retry.condition) {
|
|
228
|
+
const isPassed = await config.retry.condition(response);
|
|
229
|
+
if (typeof isPassed === "boolean" && isPassed === false) {
|
|
230
|
+
throw response;
|
|
231
|
+
}
|
|
232
|
+
} else {
|
|
233
|
+
if (!statusCodes.includes(responseStatusCode)) {
|
|
234
|
+
throw response;
|
|
235
|
+
}
|
|
236
|
+
if (maxRetries <= retries) {
|
|
237
|
+
if (config.debug) {
|
|
238
|
+
console.log(`Max retries (${maxRetries}) reached, throwing the last error`);
|
|
239
|
+
}
|
|
240
|
+
throw response;
|
|
241
|
+
}
|
|
242
|
+
retries++;
|
|
243
|
+
if (config.debug) {
|
|
244
|
+
console.log(`Request failed with status code ${responseStatusCode}, retrying...${retryDelay > 0 ? " in " + (incrementDelay ? retryDelay * retries : retryDelay) + "ms" : ""}`);
|
|
245
|
+
}
|
|
246
|
+
if (retryDelay > 0) {
|
|
247
|
+
await new Promise((resolve) => setTimeout(resolve, incrementDelay ? retryDelay * retries : retryDelay));
|
|
248
|
+
}
|
|
249
|
+
}
|
|
250
|
+
config.retryAttempts++;
|
|
251
|
+
}
|
|
252
|
+
continue;
|
|
253
|
+
}
|
|
254
|
+
if (statusOnNext === "success") {
|
|
255
|
+
return response;
|
|
256
|
+
}
|
|
257
|
+
if (statusOnNext === "redirect") {
|
|
258
|
+
const addedOptions = {};
|
|
259
|
+
const location = _stats.redirectUrl;
|
|
260
|
+
if (!location || !_stats.redirectUrl) {
|
|
261
|
+
throw builErrorFromResponse("Redirect location not found", response, config, fetchOptions);
|
|
262
|
+
}
|
|
263
|
+
if (config.maxRedirects === 0) {
|
|
264
|
+
config.maxRedirectsReached = true;
|
|
265
|
+
throw builErrorFromResponse(`Redirects are disabled (maxRedirects=0)`, response, config, fetchOptions);
|
|
266
|
+
}
|
|
267
|
+
const enableCycleDetection = config.enableRedirectCycleDetection === true;
|
|
268
|
+
if (enableCycleDetection) {
|
|
269
|
+
const normalizedRedirectUrl = _stats.redirectUrl.toLowerCase();
|
|
270
|
+
if (visitedUrls.has(normalizedRedirectUrl)) {
|
|
271
|
+
throw builErrorFromResponse(`Redirect cycle detected: attempting to revisit ${_stats.redirectUrl}`, response, config, fetchOptions);
|
|
272
|
+
}
|
|
273
|
+
visitedUrls.add(normalizedRedirectUrl);
|
|
274
|
+
}
|
|
275
|
+
const redirectCode = response.status;
|
|
276
|
+
const customHeaders = undefined;
|
|
277
|
+
const onRedirect = config.beforeRedirect ? config.beforeRedirect({
|
|
278
|
+
url: new URL(_stats.redirectUrl),
|
|
279
|
+
status: response.status,
|
|
280
|
+
headers: response.headers,
|
|
281
|
+
sameDomain: isSameDomain(fetchOptions.fullUrl, _stats.redirectUrl),
|
|
282
|
+
method: fetchOptions.method.toUpperCase()
|
|
283
|
+
}) : undefined;
|
|
284
|
+
if (typeof onRedirect !== "undefined") {
|
|
285
|
+
if (typeof onRedirect === "boolean") {
|
|
286
|
+
if (!onRedirect) {
|
|
287
|
+
throw builErrorFromResponse("Redirect denied by user", response, config, fetchOptions);
|
|
288
|
+
}
|
|
289
|
+
} else if (!onRedirect.redirect) {
|
|
290
|
+
throw builErrorFromResponse("Redirect denied by user", response, config, fetchOptions);
|
|
291
|
+
}
|
|
292
|
+
}
|
|
293
|
+
if (config.redirectCount >= config.maxRedirects && config.maxRedirects > 0) {
|
|
294
|
+
config.maxRedirectsReached = true;
|
|
295
|
+
throw builErrorFromResponse(`Max redirects (${config.maxRedirects}) reached`, response, config, fetchOptions);
|
|
296
|
+
}
|
|
297
|
+
config.redirectHistory.push({
|
|
298
|
+
url: fetchOptions.fullUrl,
|
|
299
|
+
statusCode: redirectCode,
|
|
300
|
+
statusText: response.statusText,
|
|
301
|
+
headers: response.headers,
|
|
302
|
+
method: fetchOptions.method.toUpperCase(),
|
|
303
|
+
cookies: response.cookies.array,
|
|
304
|
+
duration: perform.now(),
|
|
305
|
+
request: fetchOptions
|
|
306
|
+
});
|
|
307
|
+
perform.reset();
|
|
308
|
+
config.redirectCount++;
|
|
309
|
+
addedOptions.redirectedUrl = _stats.redirectUrl;
|
|
310
|
+
addedOptions.redirectCode = redirectCode;
|
|
311
|
+
addedOptions.isRedirected = true;
|
|
312
|
+
addedOptions.lastRedirectedUrl = fetchOptions.fullUrl;
|
|
313
|
+
addedOptions.customHeaders = customHeaders;
|
|
314
|
+
addedOptions.fullUrl = fetchOptions.fullUrl;
|
|
315
|
+
delete options.params;
|
|
316
|
+
fetchOptions.fullUrl = location;
|
|
317
|
+
let commented = false;
|
|
318
|
+
if (typeof onRedirect === "object" && onRedirect.redirect) {
|
|
319
|
+
const method = onRedirect.method || fetchOptions.method;
|
|
320
|
+
config.method = method;
|
|
321
|
+
options.fullUrl = onRedirect.url;
|
|
322
|
+
fetchOptions.fullUrl = onRedirect.url;
|
|
323
|
+
if (onRedirect.withoutBody) {
|
|
324
|
+
delete options.body;
|
|
325
|
+
} else if (onRedirect.body) {
|
|
326
|
+
options.body = onRedirect.body;
|
|
327
|
+
}
|
|
328
|
+
if (config.debug) {
|
|
329
|
+
commented = true;
|
|
330
|
+
console.log(`
|
|
331
|
+
Redirecting to: ${fetchOptions.fullUrl} using ${method} method`);
|
|
332
|
+
}
|
|
333
|
+
if (onRedirect.setHeaders) {
|
|
334
|
+
addedOptions.customHeaders = onRedirect.setHeaders;
|
|
335
|
+
}
|
|
336
|
+
} else if (response.status === 301 || response.status === 302 || response.status === 303) {
|
|
337
|
+
if (config.debug) {
|
|
338
|
+
commented = true;
|
|
339
|
+
console.log(`
|
|
340
|
+
Redirecting to: ${fetchOptions.fullUrl} using GET method`);
|
|
341
|
+
}
|
|
342
|
+
options.method = "GET";
|
|
343
|
+
delete options.body;
|
|
344
|
+
} else
|
|
345
|
+
commented = false;
|
|
346
|
+
if (config.debug && !commented) {
|
|
347
|
+
console.log(`Redirecting to: ${fetchOptions.fullUrl}`);
|
|
348
|
+
}
|
|
349
|
+
const __ = prepareHTTPOptions(fetchOptions, config.cookieJar, addedOptions, config);
|
|
350
|
+
fetchOptions = __.fetchOptions;
|
|
351
|
+
config = __.config;
|
|
352
|
+
options = __.options;
|
|
353
|
+
continue;
|
|
354
|
+
}
|
|
355
|
+
delete config.beforeRedirect;
|
|
356
|
+
config.setSignal = () => {};
|
|
357
|
+
return response;
|
|
358
|
+
} catch (error) {
|
|
359
|
+
throw error;
|
|
360
|
+
} finally {
|
|
361
|
+
if (timeoutClearInstanse)
|
|
362
|
+
clearTimeout(timeoutClearInstanse);
|
|
363
|
+
}
|
|
364
|
+
}
|
|
365
|
+
}
|
|
366
|
+
async function request(config, fetchOptions, requestCount, timing, _stats, responseStatusCode, fs, streamResult, downloadResult, uploadResult) {
|
|
367
|
+
return await new Promise(async (resolve) => {
|
|
368
|
+
try {
|
|
369
|
+
const { fullUrl, body, fileName: filename } = fetchOptions;
|
|
370
|
+
const url = new URL(fullUrl || fetchOptions.url);
|
|
371
|
+
const isSecure = url.protocol === "https:";
|
|
372
|
+
const httpModule = isSecure ? config.isSecure && config.adapter ? config.adapter : https : !config.isSecure && config.adapter ? config.adapter : http;
|
|
373
|
+
await setInitialConfig(config, fetchOptions, isSecure, url, httpModule, requestCount, timing.startTime, timing.startTimestamp);
|
|
374
|
+
const eventEmitter = streamResult || downloadResult || uploadResult;
|
|
375
|
+
if (eventEmitter && requestCount === 0) {
|
|
376
|
+
const startEvent = {
|
|
377
|
+
url: url.toString(),
|
|
378
|
+
method: fetchOptions.method.toUpperCase(),
|
|
379
|
+
headers: fetchOptions.headers instanceof RezoHeaders ? fetchOptions.headers : new RezoHeaders(fetchOptions.headers),
|
|
380
|
+
timestamp: timing.startTime,
|
|
381
|
+
timeout: fetchOptions.timeout,
|
|
382
|
+
maxRedirects: config.maxRedirects,
|
|
383
|
+
retry: config.retry ? {
|
|
384
|
+
maxRetries: config.retry.maxRetries,
|
|
385
|
+
delay: config.retry.retryDelay,
|
|
386
|
+
backoff: config.retry.incrementDelay ? config.retry.retryDelay : undefined
|
|
387
|
+
} : undefined
|
|
388
|
+
};
|
|
389
|
+
eventEmitter.emit("start", startEvent);
|
|
390
|
+
}
|
|
391
|
+
const requestOptions = buildHTTPOptions(fetchOptions, isSecure, url);
|
|
392
|
+
try {
|
|
393
|
+
const req = httpModule.request(requestOptions, async (res) => {
|
|
394
|
+
if (!config.timing.ttfbMs) {
|
|
395
|
+
timing.firstByteTime = performance.now();
|
|
396
|
+
config.timing.ttfbMs = timing.firstByteTime - timing.startTime;
|
|
397
|
+
}
|
|
398
|
+
const { statusCode, statusMessage, headers, httpVersion, socket } = res;
|
|
399
|
+
const { remoteAddress, remotePort, localAddress, localPort } = socket;
|
|
400
|
+
responseStatusCode = statusCode;
|
|
401
|
+
config.network.remoteAddress = remoteAddress;
|
|
402
|
+
config.network.remotePort = remotePort;
|
|
403
|
+
config.network.localAddress = localAddress;
|
|
404
|
+
config.network.localPort = localPort;
|
|
405
|
+
config.network.httpVersion = httpVersion;
|
|
406
|
+
config.network.family = socket.remoteFamily || undefined;
|
|
407
|
+
const contentType = headers["content-type"];
|
|
408
|
+
const location = headers["location"] || headers["Location"];
|
|
409
|
+
const contentLength = headers["content-length"];
|
|
410
|
+
const cookies = headers["set-cookie"];
|
|
411
|
+
updateCookies(config, headers, url.href);
|
|
412
|
+
const cookieArray = config.responseCookies?.array || [];
|
|
413
|
+
delete headers["set-cookie"];
|
|
414
|
+
_stats.redirectUrl = undefined;
|
|
415
|
+
const isRedirected = statusCode && statusCode >= 300 && statusCode < 400;
|
|
416
|
+
const eventEmitter = streamResult || downloadResult || uploadResult;
|
|
417
|
+
if (!isRedirected && eventEmitter) {
|
|
418
|
+
const headersEvent = {
|
|
419
|
+
status: statusCode || 200,
|
|
420
|
+
statusText: statusMessage || "",
|
|
421
|
+
headers: new RezoHeaders(headers),
|
|
422
|
+
contentType,
|
|
423
|
+
contentLength: contentLength ? parseInt(contentLength, 10) : undefined,
|
|
424
|
+
cookies: cookieArray,
|
|
425
|
+
timing: {
|
|
426
|
+
firstByte: config.timing.ttfbMs,
|
|
427
|
+
total: performance.now() - timing.startTime
|
|
428
|
+
}
|
|
429
|
+
};
|
|
430
|
+
eventEmitter.emit("headers", headersEvent);
|
|
431
|
+
eventEmitter.emit("status", statusCode, statusMessage);
|
|
432
|
+
eventEmitter.emit("cookies", cookieArray);
|
|
433
|
+
if (downloadResult) {
|
|
434
|
+
downloadResult.status = statusCode;
|
|
435
|
+
downloadResult.statusText = statusMessage;
|
|
436
|
+
} else if (uploadResult) {
|
|
437
|
+
uploadResult.status = statusCode;
|
|
438
|
+
uploadResult.statusText = statusMessage;
|
|
439
|
+
}
|
|
440
|
+
}
|
|
441
|
+
if (isSecure) {
|
|
442
|
+
const socket = res.socket || res.connection;
|
|
443
|
+
if (socket) {
|
|
444
|
+
try {
|
|
445
|
+
const hasTlsMethods = typeof socket.getCipher === "function" && typeof socket.getProtocol === "function";
|
|
446
|
+
if (hasTlsMethods) {
|
|
447
|
+
const cipher = socket.getCipher();
|
|
448
|
+
const cert = typeof socket.getPeerCertificate === "function" ? socket.getPeerCertificate() : null;
|
|
449
|
+
const tlsVersion = socket.getProtocol();
|
|
450
|
+
if (tlsVersion)
|
|
451
|
+
config.security.tlsVersion = tlsVersion;
|
|
452
|
+
if (cipher?.name)
|
|
453
|
+
config.security.cipher = cipher.name;
|
|
454
|
+
if (cert && cert.subject) {
|
|
455
|
+
config.security.certificateInfo = {
|
|
456
|
+
subject: cert.subject,
|
|
457
|
+
issuer: cert.issuer,
|
|
458
|
+
validFrom: cert.valid_from,
|
|
459
|
+
validTo: cert.valid_to,
|
|
460
|
+
fingerprint: cert.fingerprint
|
|
461
|
+
};
|
|
462
|
+
config.security.validationResults = {
|
|
463
|
+
certificateValid: !cert.fingerprint?.includes("error"),
|
|
464
|
+
hostnameMatch: cert.subject?.CN === url.hostname,
|
|
465
|
+
chainValid: true
|
|
466
|
+
};
|
|
467
|
+
}
|
|
468
|
+
} else if (socket.encrypted) {
|
|
469
|
+
config.security.encrypted = true;
|
|
470
|
+
config.security.tlsDetailsUnavailable = true;
|
|
471
|
+
}
|
|
472
|
+
} catch {}
|
|
473
|
+
}
|
|
474
|
+
}
|
|
475
|
+
if (isRedirected)
|
|
476
|
+
_stats.statusOnNext = "redirect";
|
|
477
|
+
if (isRedirected && location) {
|
|
478
|
+
_stats.redirectUrl = new URL(location, url).href;
|
|
479
|
+
if (config.redirectCount) {
|
|
480
|
+
config.redirectCount++;
|
|
481
|
+
} else {
|
|
482
|
+
config.redirectCount = 1;
|
|
483
|
+
}
|
|
484
|
+
if (eventEmitter) {
|
|
485
|
+
emitRedirect(eventEmitter, headers, statusCode || 302, statusMessage || "", url.toString(), _stats.redirectUrl, config.redirectCount, config.maxRedirects, fetchOptions.method.toUpperCase());
|
|
486
|
+
}
|
|
487
|
+
}
|
|
488
|
+
let contentLengthCounter = 0;
|
|
489
|
+
if (streamResult) {
|
|
490
|
+
if (isRedirected) {
|
|
491
|
+
resolve(null);
|
|
492
|
+
return;
|
|
493
|
+
}
|
|
494
|
+
if (streamResult.encoding) {
|
|
495
|
+
res.setEncoding(streamResult.encoding);
|
|
496
|
+
}
|
|
497
|
+
let streamedBytes = 0;
|
|
498
|
+
res.on("data", (chunk) => {
|
|
499
|
+
streamedBytes += chunk.length;
|
|
500
|
+
});
|
|
501
|
+
streamResult.on("finish", () => {
|
|
502
|
+
updateTiming(config, timing, contentLength || "", streamedBytes, res.rawHeaders);
|
|
503
|
+
const streamFinishEvent = {
|
|
504
|
+
status: statusCode || 200,
|
|
505
|
+
statusText: statusMessage || "OK",
|
|
506
|
+
headers: new RezoHeaders(headers),
|
|
507
|
+
contentType,
|
|
508
|
+
contentLength: streamedBytes,
|
|
509
|
+
finalUrl: url.toString(),
|
|
510
|
+
cookies: config.cookieJar?.cookies() || { array: [], map: {} },
|
|
511
|
+
urls: [url.toString()],
|
|
512
|
+
timing: {
|
|
513
|
+
total: config.timing.durationMs || 0,
|
|
514
|
+
dns: config.timing.dnsMs,
|
|
515
|
+
tcp: config.timing.tcpMs,
|
|
516
|
+
tls: config.timing.tlsMs,
|
|
517
|
+
firstByte: config.timing.ttfbMs,
|
|
518
|
+
download: config.timing.transferMs
|
|
519
|
+
},
|
|
520
|
+
config: sanitizeConfig(config)
|
|
521
|
+
};
|
|
522
|
+
streamResult.emit("done", streamFinishEvent);
|
|
523
|
+
streamResult._markFinished();
|
|
524
|
+
_stats.statusOnNext = "success";
|
|
525
|
+
const minimalResponse = buildResponseFromIncoming(res, Buffer.alloc(0), config, url.toString(), buildUrlTree(config, url.toString()));
|
|
526
|
+
resolve(minimalResponse);
|
|
527
|
+
});
|
|
528
|
+
res.pipe(streamResult);
|
|
529
|
+
} else if (downloadResult && filename && fs && statusCode && statusCode >= 200 && statusCode < 300) {
|
|
530
|
+
const writeStream = fs.createWriteStream(filename);
|
|
531
|
+
const totalBytes = contentLength ? parseInt(contentLength, 10) : 0;
|
|
532
|
+
let downloadedBytes = 0;
|
|
533
|
+
const downloadStartTime = performance.now();
|
|
534
|
+
let lastProgressTime = downloadStartTime;
|
|
535
|
+
res.on("data", (chunk) => {
|
|
536
|
+
downloadedBytes += chunk.length;
|
|
537
|
+
const now = performance.now();
|
|
538
|
+
const elapsed = now - downloadStartTime;
|
|
539
|
+
const timeSinceLastProgress = now - lastProgressTime;
|
|
540
|
+
if (timeSinceLastProgress >= 100 || downloadedBytes === totalBytes) {
|
|
541
|
+
const progressEvent = {
|
|
542
|
+
loaded: downloadedBytes,
|
|
543
|
+
total: totalBytes,
|
|
544
|
+
percentage: totalBytes > 0 ? downloadedBytes / totalBytes * 100 : 0,
|
|
545
|
+
speed: timeSinceLastProgress > 0 ? chunk.length / timeSinceLastProgress * 1000 : 0,
|
|
546
|
+
averageSpeed: elapsed > 0 ? downloadedBytes / elapsed * 1000 : 0,
|
|
547
|
+
estimatedTime: totalBytes > downloadedBytes && elapsed > 0 ? (totalBytes - downloadedBytes) / downloadedBytes * elapsed : 0,
|
|
548
|
+
timestamp: now
|
|
549
|
+
};
|
|
550
|
+
downloadResult.emit("progress", progressEvent);
|
|
551
|
+
lastProgressTime = now;
|
|
552
|
+
}
|
|
553
|
+
});
|
|
554
|
+
res.pipe(writeStream);
|
|
555
|
+
writeStream.on("finish", () => {
|
|
556
|
+
if (!contentLength) {
|
|
557
|
+
if (fs.existsSync(filename)) {
|
|
558
|
+
contentLengthCounter = fs.statSync(filename).size;
|
|
559
|
+
}
|
|
560
|
+
} else {
|
|
561
|
+
contentLengthCounter = downloadedBytes;
|
|
562
|
+
}
|
|
563
|
+
updateTiming(config, timing, contentLength || "", contentLengthCounter, res.rawHeaders);
|
|
564
|
+
const finishEvent = {
|
|
565
|
+
status: statusCode || 200,
|
|
566
|
+
statusText: statusMessage || "OK",
|
|
567
|
+
headers: new RezoHeaders(headers),
|
|
568
|
+
contentType,
|
|
569
|
+
contentLength: contentLengthCounter,
|
|
570
|
+
finalUrl: url.toString(),
|
|
571
|
+
cookies: config.cookieJar?.cookies() || { array: [], map: {} },
|
|
572
|
+
urls: [url.toString()],
|
|
573
|
+
fileName: filename,
|
|
574
|
+
fileSize: contentLengthCounter,
|
|
575
|
+
timing: {
|
|
576
|
+
total: config.timing.durationMs || 0,
|
|
577
|
+
dns: config.timing.dnsMs,
|
|
578
|
+
tcp: config.timing.tcpMs,
|
|
579
|
+
tls: config.timing.tlsMs,
|
|
580
|
+
firstByte: config.timing.ttfbMs,
|
|
581
|
+
download: config.timing.transferMs || 0
|
|
582
|
+
},
|
|
583
|
+
averageSpeed: config.timing.transferMs ? contentLengthCounter / config.timing.transferMs * 1000 : 0,
|
|
584
|
+
config: sanitizeConfig(config)
|
|
585
|
+
};
|
|
586
|
+
downloadResult.emit("finish", finishEvent);
|
|
587
|
+
downloadResult.emit("done", finishEvent);
|
|
588
|
+
downloadResult._markFinished();
|
|
589
|
+
const downloadResponse = buildDownloadResponse(res.statusCode ?? 200, res.statusMessage ?? "OK", headers, contentLengthCounter, cookies || [], res.url || url.toString(), url.toString(), [url.toString()], config);
|
|
590
|
+
_stats.statusOnNext = "success";
|
|
591
|
+
resolve(downloadResponse);
|
|
592
|
+
});
|
|
593
|
+
writeStream.on("error", (err) => {
|
|
594
|
+
updateTiming(config, timing, contentLength || "", contentLengthCounter, res.rawHeaders);
|
|
595
|
+
_stats.statusOnNext = "error";
|
|
596
|
+
const error = buildDownloadError({
|
|
597
|
+
statusCode: res.statusCode || 500,
|
|
598
|
+
headers,
|
|
599
|
+
contentType,
|
|
600
|
+
contentLength: contentLength || "0",
|
|
601
|
+
cookies: cookies || [],
|
|
602
|
+
statusText: err.message || "Download failed",
|
|
603
|
+
url: res.url || url.toString(),
|
|
604
|
+
body: null,
|
|
605
|
+
finalUrl: url.toString(),
|
|
606
|
+
config,
|
|
607
|
+
request: fetchOptions
|
|
608
|
+
});
|
|
609
|
+
downloadResult.emit("error", error);
|
|
610
|
+
resolve(error);
|
|
611
|
+
return;
|
|
612
|
+
});
|
|
613
|
+
} else if (filename && fs && statusCode && statusCode >= 200 && statusCode < 300) {
|
|
614
|
+
const writeStream = fs.createWriteStream(filename);
|
|
615
|
+
res.pipe(writeStream);
|
|
616
|
+
writeStream.on("finish", () => {
|
|
617
|
+
if (!contentLength) {
|
|
618
|
+
if (fs.existsSync(filename)) {
|
|
619
|
+
contentLengthCounter = fs.statSync(filename).size;
|
|
620
|
+
}
|
|
621
|
+
}
|
|
622
|
+
updateTiming(config, timing, contentLength || "", contentLengthCounter, res.rawHeaders);
|
|
623
|
+
const downloadResponse = buildDownloadResponse(res.statusCode ?? 200, res.statusMessage ?? "OK", headers, parseInt(contentLength || "0", 10) || contentLengthCounter, cookies || [], res.url || url.toString(), url.toString(), [url.toString()], config);
|
|
624
|
+
_stats.statusOnNext = "success";
|
|
625
|
+
resolve(downloadResponse);
|
|
626
|
+
});
|
|
627
|
+
writeStream.on("error", (err) => {
|
|
628
|
+
updateTiming(config, timing, contentLength || "", contentLengthCounter, res.rawHeaders);
|
|
629
|
+
_stats.statusOnNext = "error";
|
|
630
|
+
const error = buildDownloadError({
|
|
631
|
+
statusCode: res.statusCode || 500,
|
|
632
|
+
headers,
|
|
633
|
+
contentType,
|
|
634
|
+
contentLength: contentLength || "0",
|
|
635
|
+
cookies: cookies || [],
|
|
636
|
+
statusText: err.message || "Download failed",
|
|
637
|
+
url: res.url || url.toString(),
|
|
638
|
+
body: null,
|
|
639
|
+
finalUrl: url.toString(),
|
|
640
|
+
config,
|
|
641
|
+
request: fetchOptions
|
|
642
|
+
});
|
|
643
|
+
resolve(error);
|
|
644
|
+
return;
|
|
645
|
+
});
|
|
646
|
+
} else {
|
|
647
|
+
if (config.encoding) {
|
|
648
|
+
res.setEncoding(config.encoding);
|
|
649
|
+
}
|
|
650
|
+
const decompressedStream = CompressionUtil.decompressStream(res, res.headers["content-encoding"], config);
|
|
651
|
+
const chunks = [];
|
|
652
|
+
decompressedStream.on("data", (chunk) => {
|
|
653
|
+
contentLengthCounter += chunk.length;
|
|
654
|
+
chunks.push(chunk);
|
|
655
|
+
});
|
|
656
|
+
decompressedStream.on("end", () => {
|
|
657
|
+
_stats.statusOnNext = isRedirected ? "redirect" : statusCode && statusCode >= 200 && statusCode < 300 ? "success" : "error";
|
|
658
|
+
updateTiming(config, timing, contentLength || "", contentLengthCounter, res.rawHeaders);
|
|
659
|
+
const finalResponse = buildResponseFromIncoming(res, Buffer.concat(chunks), config, url.toString(), buildUrlTree(config, url.toString()), undefined, undefined, contentLengthCounter);
|
|
660
|
+
if (uploadResult && !isRedirected) {
|
|
661
|
+
const uploadFinishEvent = {
|
|
662
|
+
response: {
|
|
663
|
+
status: statusCode || 200,
|
|
664
|
+
statusText: statusMessage || "OK",
|
|
665
|
+
headers: new RezoHeaders(headers),
|
|
666
|
+
data: finalResponse.data,
|
|
667
|
+
contentType,
|
|
668
|
+
contentLength: contentLengthCounter
|
|
669
|
+
},
|
|
670
|
+
finalUrl: url.toString(),
|
|
671
|
+
cookies: config.cookieJar?.cookies() || { array: [], map: {} },
|
|
672
|
+
urls: [url.toString()],
|
|
673
|
+
uploadSize: config.transfer.requestSize || 0,
|
|
674
|
+
fileName: uploadResult.fileName,
|
|
675
|
+
timing: {
|
|
676
|
+
total: config.timing.durationMs || 0,
|
|
677
|
+
dns: config.timing.dnsMs,
|
|
678
|
+
tcp: config.timing.tcpMs,
|
|
679
|
+
tls: config.timing.tlsMs,
|
|
680
|
+
upload: config.timing.ttfbMs || 0,
|
|
681
|
+
waiting: config.timing.ttfbMs && config.timing.transferMs ? config.timing.transferMs - config.timing.ttfbMs : 0,
|
|
682
|
+
download: config.timing.transferMs
|
|
683
|
+
},
|
|
684
|
+
averageUploadSpeed: config.timing.ttfbMs && config.transfer.requestSize ? config.transfer.requestSize / config.timing.ttfbMs * 1000 : 0,
|
|
685
|
+
averageDownloadSpeed: config.timing.transferMs ? contentLengthCounter / config.timing.transferMs * 1000 : 0,
|
|
686
|
+
config: sanitizeConfig(config)
|
|
687
|
+
};
|
|
688
|
+
uploadResult.emit("finish", uploadFinishEvent);
|
|
689
|
+
uploadResult.emit("done", uploadFinishEvent);
|
|
690
|
+
uploadResult._markFinished();
|
|
691
|
+
}
|
|
692
|
+
resolve(finalResponse);
|
|
693
|
+
});
|
|
694
|
+
decompressedStream.on("error", (err) => {
|
|
695
|
+
_stats.statusOnNext = "error";
|
|
696
|
+
updateTiming(config, timing, contentLength || "", contentLengthCounter, res.rawHeaders);
|
|
697
|
+
if (_stats.redirectUrl) {
|
|
698
|
+
const partialResponse = buildResponseFromIncoming(res, Buffer.concat(chunks), config, url.toString(), buildUrlTree(config, url.toString()), undefined, undefined, contentLengthCounter);
|
|
699
|
+
resolve(partialResponse);
|
|
700
|
+
return;
|
|
701
|
+
}
|
|
702
|
+
const error = buildDecompressionError({
|
|
703
|
+
statusCode: res.statusCode || 500,
|
|
704
|
+
headers,
|
|
705
|
+
contentType,
|
|
706
|
+
contentLength: contentLength || contentLengthCounter.toString(),
|
|
707
|
+
cookies: cookies || [],
|
|
708
|
+
statusText: err.message || "Decompression failed",
|
|
709
|
+
url: res.url || url.toString(),
|
|
710
|
+
body: Buffer.concat(chunks),
|
|
711
|
+
finalUrl: url.toString(),
|
|
712
|
+
config,
|
|
713
|
+
request: fetchOptions
|
|
714
|
+
});
|
|
715
|
+
resolve(error);
|
|
716
|
+
});
|
|
717
|
+
}
|
|
718
|
+
});
|
|
719
|
+
req.on("socket", (socket) => {
|
|
720
|
+
timing.dnsStart = performance.now();
|
|
721
|
+
socket.on("lookup", () => {
|
|
722
|
+
if (!config.timing.dnsMs) {
|
|
723
|
+
if (timing.dnsStart)
|
|
724
|
+
config.timing.dnsMs = performance.now() - timing.dnsStart;
|
|
725
|
+
timing.tcpStart = performance.now();
|
|
726
|
+
}
|
|
727
|
+
});
|
|
728
|
+
socket.on("secureConnect", () => {
|
|
729
|
+
if (!config.timing.tlsMs) {
|
|
730
|
+
if (timing.tlsStart && !config.timing.tlsMs)
|
|
731
|
+
config.timing.tlsMs = performance.now() - timing.tlsStart;
|
|
732
|
+
}
|
|
733
|
+
const tls = {
|
|
734
|
+
cipher: socket.getCipher(),
|
|
735
|
+
cert: socket.getPeerCertificate(),
|
|
736
|
+
tlsVersion: socket.getProtocol()
|
|
737
|
+
};
|
|
738
|
+
const { cipher, cert, tlsVersion } = tls;
|
|
739
|
+
config.security.tlsVersion = tlsVersion;
|
|
740
|
+
config.security.cipher = cipher?.name;
|
|
741
|
+
config.security.certificateInfo = {
|
|
742
|
+
subject: cert.subject,
|
|
743
|
+
issuer: cert.issuer,
|
|
744
|
+
validFrom: cert.valid_from,
|
|
745
|
+
validTo: cert.valid_to,
|
|
746
|
+
fingerprint: cert.fingerprint
|
|
747
|
+
};
|
|
748
|
+
config.security.validationResults = {
|
|
749
|
+
certificateValid: !cert.fingerprint?.includes("error"),
|
|
750
|
+
hostnameMatch: cert.subject?.CN === url.hostname,
|
|
751
|
+
chainValid: true
|
|
752
|
+
};
|
|
753
|
+
});
|
|
754
|
+
socket.on("connect", () => {
|
|
755
|
+
if (!config.timing.tcpMs) {
|
|
756
|
+
if (timing.tcpStart)
|
|
757
|
+
config.timing.tcpMs = performance.now() - timing.tcpStart;
|
|
758
|
+
if (isSecure)
|
|
759
|
+
timing.tlsStart = performance.now();
|
|
760
|
+
}
|
|
761
|
+
const { remoteAddress, remotePort, localAddress, localPort, remoteFamily } = socket;
|
|
762
|
+
if (remoteAddress && !config.network.remoteAddress) {
|
|
763
|
+
config.network.remoteAddress = remoteAddress;
|
|
764
|
+
config.network.remotePort = remotePort;
|
|
765
|
+
config.network.localAddress = localAddress;
|
|
766
|
+
config.network.localPort = localPort;
|
|
767
|
+
config.network.family = remoteFamily;
|
|
768
|
+
}
|
|
769
|
+
});
|
|
770
|
+
});
|
|
771
|
+
if (body) {
|
|
772
|
+
if (body instanceof URLSearchParams || body instanceof RezoURLSearchParams) {
|
|
773
|
+
req.write(body.toString());
|
|
774
|
+
} else if (body instanceof FormData || body instanceof RezoFormData) {
|
|
775
|
+
if (body instanceof RezoFormData) {
|
|
776
|
+
req.setHeader("Content-Type", `multipart/form-data; boundary=${body.getBoundary()}`);
|
|
777
|
+
body.pipe(req);
|
|
778
|
+
} else {
|
|
779
|
+
const form = await RezoFormData.fromNativeFormData(body);
|
|
780
|
+
req.setHeader("Content-Type", `multipart/form-data; boundary=${form.getBoundary()}`);
|
|
781
|
+
form.pipe(req);
|
|
782
|
+
}
|
|
783
|
+
} else if (typeof body === "object" && !(body instanceof Buffer) && !(body instanceof Uint8Array) && !(body instanceof Readable)) {
|
|
784
|
+
req.write(JSON.stringify(body));
|
|
785
|
+
} else {
|
|
786
|
+
req.write(body);
|
|
787
|
+
}
|
|
788
|
+
}
|
|
789
|
+
req.end();
|
|
790
|
+
req.on("error", (error) => {
|
|
791
|
+
_stats.statusOnNext = "error";
|
|
792
|
+
updateTiming(config, timing, "", 0);
|
|
793
|
+
const e = buildSmartError(config, fetchOptions, error);
|
|
794
|
+
const eventEmitter = streamResult || downloadResult || uploadResult;
|
|
795
|
+
if (eventEmitter) {
|
|
796
|
+
eventEmitter.emit("error", e);
|
|
797
|
+
}
|
|
798
|
+
resolve(e);
|
|
799
|
+
return;
|
|
800
|
+
});
|
|
801
|
+
} catch (error) {
|
|
802
|
+
_stats.statusOnNext = "error";
|
|
803
|
+
updateTiming(config, timing, "", 0);
|
|
804
|
+
const e = buildSmartError(config, fetchOptions, error);
|
|
805
|
+
const eventEmitter = streamResult || downloadResult || uploadResult;
|
|
806
|
+
if (eventEmitter) {
|
|
807
|
+
eventEmitter.emit("error", e);
|
|
808
|
+
}
|
|
809
|
+
resolve(e);
|
|
810
|
+
return;
|
|
811
|
+
}
|
|
812
|
+
} catch (error) {
|
|
813
|
+
const rezoError = buildSmartError(config, fetchOptions, error);
|
|
814
|
+
const eventEmitter = streamResult || downloadResult || uploadResult;
|
|
815
|
+
if (eventEmitter) {
|
|
816
|
+
eventEmitter.emit("error", rezoError);
|
|
817
|
+
}
|
|
818
|
+
resolve(rezoError);
|
|
819
|
+
}
|
|
820
|
+
});
|
|
821
|
+
}
|
|
822
|
+
function updateTiming(config, timing, contentLength, contentLengthCounter, rawHeaders) {
|
|
823
|
+
config.timing.endTimestamp = Date.now();
|
|
824
|
+
const elapsedMs = performance.now() - timing.startTime;
|
|
825
|
+
config.timing.durationMs = elapsedMs;
|
|
826
|
+
config.timing.transferMs = timing.firstByteTime ? performance.now() - timing.firstByteTime : elapsedMs;
|
|
827
|
+
const bodySize = parseInt(contentLength || "0", 10) || contentLengthCounter;
|
|
828
|
+
config.transfer.bodySize = bodySize;
|
|
829
|
+
let headerSize = 0;
|
|
830
|
+
if (rawHeaders && rawHeaders.length > 0) {
|
|
831
|
+
for (let i = 0;i < rawHeaders.length; i += 2) {
|
|
832
|
+
const key = rawHeaders[i] || "";
|
|
833
|
+
const value = rawHeaders[i + 1] || "";
|
|
834
|
+
headerSize += Buffer.byteLength(key + ": " + value + `\r
|
|
835
|
+
`, "utf8");
|
|
836
|
+
}
|
|
837
|
+
headerSize += 2;
|
|
838
|
+
config.transfer.headerSize = headerSize;
|
|
839
|
+
}
|
|
840
|
+
config.transfer.responseSize = headerSize + bodySize;
|
|
841
|
+
if (contentLength && contentLengthCounter) {
|
|
842
|
+
const originalSize = parseInt(contentLength, 10);
|
|
843
|
+
if (originalSize > 0 && contentLengthCounter > 0) {
|
|
844
|
+
config.transfer.compressionRatio = contentLengthCounter / originalSize;
|
|
845
|
+
}
|
|
846
|
+
}
|
|
847
|
+
if (!config.trackingData || Object.keys(config.trackingData).length === 0) {
|
|
848
|
+
config.trackingData = {
|
|
849
|
+
redirectCount: config.redirectCount || 0,
|
|
850
|
+
method: config.method,
|
|
851
|
+
protocol: config.network?.protocol || "unknown",
|
|
852
|
+
httpVersion: config.network?.httpVersion,
|
|
853
|
+
compressed: !!config.transfer.compressionRatio && config.transfer.compressionRatio !== 1,
|
|
854
|
+
cached: false,
|
|
855
|
+
retried: (config.retryAttempts || 0) > 0
|
|
856
|
+
};
|
|
857
|
+
}
|
|
858
|
+
}
|
|
859
|
+
let dnsCache = null;
|
|
860
|
+
function createDNSLookup(cache) {
|
|
861
|
+
return (hostname, optionsOrCallback, callbackOrUndefined) => {
|
|
862
|
+
let options = {};
|
|
863
|
+
let callback;
|
|
864
|
+
if (typeof optionsOrCallback === "function") {
|
|
865
|
+
callback = optionsOrCallback;
|
|
866
|
+
} else if (typeof optionsOrCallback === "number") {
|
|
867
|
+
options = { family: optionsOrCallback };
|
|
868
|
+
callback = callbackOrUndefined;
|
|
869
|
+
} else {
|
|
870
|
+
options = optionsOrCallback || {};
|
|
871
|
+
callback = callbackOrUndefined;
|
|
872
|
+
}
|
|
873
|
+
const family = options.family;
|
|
874
|
+
cache.lookup(hostname, family).then((result) => {
|
|
875
|
+
if (result) {
|
|
876
|
+
callback(null, result.address, result.family);
|
|
877
|
+
} else {
|
|
878
|
+
dns.lookup(hostname, options, callback);
|
|
879
|
+
}
|
|
880
|
+
}).catch(() => {
|
|
881
|
+
dns.lookup(hostname, options, callback);
|
|
882
|
+
});
|
|
883
|
+
};
|
|
884
|
+
}
|
|
885
|
+
function buildHTTPOptions(fetchOptions, isSecure, url) {
|
|
886
|
+
const {
|
|
887
|
+
method,
|
|
888
|
+
headers,
|
|
889
|
+
timeout,
|
|
890
|
+
proxy,
|
|
891
|
+
httpAgent,
|
|
892
|
+
httpsAgent,
|
|
893
|
+
signal,
|
|
894
|
+
rejectUnauthorized,
|
|
895
|
+
useSecureContext = true,
|
|
896
|
+
auth,
|
|
897
|
+
dnsCache: dnsCacheOption
|
|
898
|
+
} = fetchOptions;
|
|
899
|
+
const secureContext = isSecure && useSecureContext ? new https.Agent({
|
|
900
|
+
secureContext: createSecureContext(),
|
|
901
|
+
servername: url.host,
|
|
902
|
+
rejectUnauthorized,
|
|
903
|
+
keepAlive: true
|
|
904
|
+
}) : undefined;
|
|
905
|
+
const customAgent = url.protocol === "https:" && httpsAgent ? httpsAgent : httpAgent ? httpAgent : undefined;
|
|
906
|
+
const agent = parseProxy(proxy) || customAgent || secureContext;
|
|
907
|
+
let lookup;
|
|
908
|
+
if (dnsCacheOption) {
|
|
909
|
+
if (!dnsCache) {
|
|
910
|
+
const cacheOptions = typeof dnsCacheOption === "object" ? {
|
|
911
|
+
enable: true,
|
|
912
|
+
ttl: dnsCacheOption.ttl,
|
|
913
|
+
maxEntries: dnsCacheOption.maxEntries
|
|
914
|
+
} : { enable: true };
|
|
915
|
+
dnsCache = getGlobalDNSCache(cacheOptions);
|
|
916
|
+
}
|
|
917
|
+
lookup = createDNSLookup(dnsCache);
|
|
918
|
+
}
|
|
919
|
+
const requestOptions = {
|
|
920
|
+
hostname: url.hostname,
|
|
921
|
+
port: url.port || (isSecure ? 443 : 80),
|
|
922
|
+
path: url.pathname + url.search,
|
|
923
|
+
method,
|
|
924
|
+
headers: headers.toObject(),
|
|
925
|
+
timeout: timeout || 0,
|
|
926
|
+
signal,
|
|
927
|
+
rejectUnauthorized,
|
|
928
|
+
agent,
|
|
929
|
+
auth: auth?.username && auth?.password ? `${auth.username}:${auth.password}` : undefined,
|
|
930
|
+
lookup
|
|
931
|
+
};
|
|
932
|
+
return requestOptions;
|
|
933
|
+
}
|
|
934
|
+
async function setInitialConfig(config, fetchOptions, isSecure, url, httpModule, requestCount, startTime, actualTimestamp) {
|
|
935
|
+
if (requestCount === 0) {
|
|
936
|
+
const { body, timeout, proxy, httpAgent, httpsAgent, fileName: filename, auth, signal } = fetchOptions;
|
|
937
|
+
config.adapterUsed = isSecure ? "https" : "http";
|
|
938
|
+
config.adapter = httpModule;
|
|
939
|
+
config.isSecure = isSecure;
|
|
940
|
+
config.finalUrl = url.href;
|
|
941
|
+
config.network.protocol = url.protocol.replace(":", "");
|
|
942
|
+
config.data = body ?? null;
|
|
943
|
+
config.auth = auth ?? null;
|
|
944
|
+
if (proxy !== undefined) {
|
|
945
|
+
config.proxy = proxy;
|
|
946
|
+
}
|
|
947
|
+
let normalizedResponseType = fetchOptions.responseType;
|
|
948
|
+
if (normalizedResponseType) {
|
|
949
|
+
const lowerType = normalizedResponseType.toLowerCase();
|
|
950
|
+
if (lowerType === "arraybuffer") {
|
|
951
|
+
normalizedResponseType = "arrayBuffer";
|
|
952
|
+
} else if (lowerType === "binary") {
|
|
953
|
+
normalizedResponseType = "buffer";
|
|
954
|
+
}
|
|
955
|
+
}
|
|
956
|
+
config.responseType = normalizedResponseType;
|
|
957
|
+
config.insecureHTTPParser = fetchOptions.insecureHTTPParser || false;
|
|
958
|
+
config.maxRate = fetchOptions.maxRate || 0;
|
|
959
|
+
config.cancelToken = fetchOptions.cancelToken ?? null;
|
|
960
|
+
config.signal = signal ?? null;
|
|
961
|
+
config.httpAgent = httpAgent ?? null;
|
|
962
|
+
config.httpsAgent = httpsAgent ?? null;
|
|
963
|
+
config.socketPath = fetchOptions.socketPath ?? null;
|
|
964
|
+
config.fileName = filename ?? null;
|
|
965
|
+
config.adapterMetadata = {
|
|
966
|
+
version: process.version || "1.0.0",
|
|
967
|
+
features: ["http1", "cookies", "redirects", "compression", "proxy", "timeout"],
|
|
968
|
+
capabilities: {
|
|
969
|
+
http1: true,
|
|
970
|
+
http2: config.http2,
|
|
971
|
+
compression: true,
|
|
972
|
+
cookies: config.enableCookieJar,
|
|
973
|
+
redirects: config.maxRedirects > 0,
|
|
974
|
+
proxy: !!proxy,
|
|
975
|
+
timeout: !!timeout,
|
|
976
|
+
ssl: isSecure
|
|
977
|
+
}
|
|
978
|
+
};
|
|
979
|
+
config.features = {
|
|
980
|
+
http2: !!config.http2,
|
|
981
|
+
compression: !!config.compression?.enabled,
|
|
982
|
+
cookies: !!config.enableCookieJar,
|
|
983
|
+
redirects: config.maxRedirects > 0,
|
|
984
|
+
proxy: !!proxy,
|
|
985
|
+
timeout: !!timeout,
|
|
986
|
+
retry: !!config.retry,
|
|
987
|
+
metrics: true,
|
|
988
|
+
events: true,
|
|
989
|
+
validation: true,
|
|
990
|
+
browser: false,
|
|
991
|
+
ssl: isSecure
|
|
992
|
+
};
|
|
993
|
+
const timestampToUse = actualTimestamp || Date.now();
|
|
994
|
+
config.timing = config.timing || { startTimestamp: timestampToUse, endTimestamp: 0, durationMs: 0 };
|
|
995
|
+
config.timing.startTimestamp = config.timing.startTimestamp || timestampToUse;
|
|
996
|
+
config.maxRedirectsReached = false;
|
|
997
|
+
config.responseCookies = {
|
|
998
|
+
array: [],
|
|
999
|
+
serialized: [],
|
|
1000
|
+
netscape: `# Netscape HTTP Cookie File
|
|
1001
|
+
# This file was generated by Rezo HTTP client
|
|
1002
|
+
# Based on uniqhtt cookie implementation
|
|
1003
|
+
`,
|
|
1004
|
+
string: "",
|
|
1005
|
+
setCookiesString: []
|
|
1006
|
+
};
|
|
1007
|
+
config.retryAttempts = 0;
|
|
1008
|
+
config.errors = [];
|
|
1009
|
+
config.debug = fetchOptions.debug || false;
|
|
1010
|
+
config.requestId = generateRequestId();
|
|
1011
|
+
config.sessionId = fetchOptions.sessionId || generateSessionId();
|
|
1012
|
+
config.traceId = generateTraceId();
|
|
1013
|
+
config.timestamp = config.timing.startTimestamp;
|
|
1014
|
+
config.trackingData = {};
|
|
1015
|
+
config.transfer = {
|
|
1016
|
+
requestSize: 0,
|
|
1017
|
+
responseSize: 0,
|
|
1018
|
+
headerSize: 0,
|
|
1019
|
+
bodySize: 0
|
|
1020
|
+
};
|
|
1021
|
+
config.security = {};
|
|
1022
|
+
}
|
|
1023
|
+
let requestBodySize = 0;
|
|
1024
|
+
if (fetchOptions.body) {
|
|
1025
|
+
if (typeof fetchOptions.body === "string") {
|
|
1026
|
+
requestBodySize = Buffer.byteLength(fetchOptions.body, "utf8");
|
|
1027
|
+
} else if (Buffer.isBuffer(fetchOptions.body)) {
|
|
1028
|
+
requestBodySize = fetchOptions.body.length;
|
|
1029
|
+
} else if (fetchOptions.body instanceof RezoFormData) {
|
|
1030
|
+
requestBodySize = fetchOptions.body.getLengthSync();
|
|
1031
|
+
} else if (fetchOptions.body instanceof FormData) {
|
|
1032
|
+
requestBodySize = (await RezoFormData.fromNativeFormData(fetchOptions.body)).getLengthSync();
|
|
1033
|
+
}
|
|
1034
|
+
}
|
|
1035
|
+
const headers = fetchOptions.headers instanceof RezoHeaders ? fetchOptions.headers : new RezoHeaders(fetchOptions.headers);
|
|
1036
|
+
const requestHeaderSize = calculateRequestHeaderSize(fetchOptions.method?.toUpperCase() || "GET", url, headers.toObject());
|
|
1037
|
+
config.transfer.requestSize = requestHeaderSize + requestBodySize;
|
|
1038
|
+
config.transfer.requestHeaderSize = requestHeaderSize;
|
|
1039
|
+
config.transfer.requestBodySize = requestBodySize;
|
|
1040
|
+
}
|
|
1041
|
+
function sanitizeConfig(config) {
|
|
1042
|
+
const { data, ...sanitized } = config;
|
|
1043
|
+
return sanitized;
|
|
1044
|
+
}
|
|
1045
|
+
function emitRedirect(emitter, headers, status, statusText, sourceUri, destinationUri, redirectCount, maxRedirects, method) {
|
|
1046
|
+
const jar = new RezoCookieJar;
|
|
1047
|
+
const newHeaders = new RezoHeaders(headers);
|
|
1048
|
+
const cookies = newHeaders.getSetCookie();
|
|
1049
|
+
newHeaders.delete("set-cookie");
|
|
1050
|
+
if (cookies && cookies.length > 0) {
|
|
1051
|
+
jar.setCookiesSync(cookies, sourceUri);
|
|
1052
|
+
}
|
|
1053
|
+
const redirectEvent = {
|
|
1054
|
+
sourceUrl: sourceUri,
|
|
1055
|
+
sourceStatus: status,
|
|
1056
|
+
sourceStatusText: statusText,
|
|
1057
|
+
destinationUrl: destinationUri,
|
|
1058
|
+
redirectCount,
|
|
1059
|
+
maxRedirects,
|
|
1060
|
+
headers: newHeaders,
|
|
1061
|
+
cookies: jar.cookies().array,
|
|
1062
|
+
method,
|
|
1063
|
+
timestamp: performance.now(),
|
|
1064
|
+
duration: 0
|
|
1065
|
+
};
|
|
1066
|
+
emitter.emit("redirect", redirectEvent);
|
|
1067
|
+
}
|
|
1068
|
+
function createSecureContext() {
|
|
1069
|
+
return tls.createSecureContext({
|
|
1070
|
+
ecdhCurve: "X25519:prime256v1:secp384r1:secp521r1",
|
|
1071
|
+
honorCipherOrder: true,
|
|
1072
|
+
ciphers: "TLS_AES_256_GCM_SHA384:TLS_CHACHA20_POLY1305_SHA256:TLS_AES_128_GCM_SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-AES256-GCM-SHA384:DHE-RSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-SHA256:DHE-RSA-AES128-SHA256:ECDHE-RSA-AES256-SHA384:DHE-RSA-AES256-SHA384:ECDHE-RSA-AES256-SHA256:DHE-RSA-AES256-SHA256:HIGH:!aNULL:!eNULL:!EXPORT:!DES:!RC4:!MD5:!PSK:!SRP:!CAMELLIA",
|
|
1073
|
+
sigalgs: "ecdsa_secp256r1_sha256:rsa_pss_rsae_sha256:rsa_pkcs1_sha256:ecdsa_secp384r1_sha384:rsa_pss_rsae_sha384:rsa_pkcs1_sha384:ecdsa_secp521r1_sha512:rsa_pss_rsae_sha512:rsa_pkcs1_sha512",
|
|
1074
|
+
minVersion: "TLSv1.2",
|
|
1075
|
+
maxVersion: "TLSv1.3",
|
|
1076
|
+
sessionTimeout: 3600
|
|
1077
|
+
});
|
|
1078
|
+
}
|
|
1079
|
+
function calculateRequestHeaderSize(method, url, headers) {
|
|
1080
|
+
const requestLine = `${method} ${url.pathname}${url.search} HTTP/1.1\r
|
|
1081
|
+
`;
|
|
1082
|
+
let size = Buffer.byteLength(requestLine, "utf8");
|
|
1083
|
+
size += Buffer.byteLength(`Host: ${url.host}\r
|
|
1084
|
+
`, "utf8");
|
|
1085
|
+
for (const [key, value] of Object.entries(headers)) {
|
|
1086
|
+
if (Array.isArray(value)) {
|
|
1087
|
+
for (const v of value) {
|
|
1088
|
+
size += Buffer.byteLength(`${key}: ${v}\r
|
|
1089
|
+
`, "utf8");
|
|
1090
|
+
}
|
|
1091
|
+
} else {
|
|
1092
|
+
size += Buffer.byteLength(`${key}: ${value}\r
|
|
1093
|
+
`, "utf8");
|
|
1094
|
+
}
|
|
1095
|
+
}
|
|
1096
|
+
size += 2;
|
|
1097
|
+
return size;
|
|
1098
|
+
}
|
|
1099
|
+
function generateRequestId() {
|
|
1100
|
+
return `req_${Date.now()}_${Math.random().toString(36).substring(2, 11)}`;
|
|
1101
|
+
}
|
|
1102
|
+
function generateSessionId() {
|
|
1103
|
+
return `ses_${Date.now()}_${Math.random().toString(36).substring(2, 11)}`;
|
|
1104
|
+
}
|
|
1105
|
+
function generateTraceId() {
|
|
1106
|
+
return `trc_${Date.now()}_${Math.random().toString(36).substring(2, 15)}`;
|
|
1107
|
+
}
|
|
1108
|
+
function parseProxy(proxy, isScure = true, rejectUnauthorized = false) {
|
|
1109
|
+
if (!proxy) {
|
|
1110
|
+
return;
|
|
1111
|
+
}
|
|
1112
|
+
if (typeof proxy === "string") {
|
|
1113
|
+
if (proxy.startsWith("http://")) {
|
|
1114
|
+
return rezoProxy(`http://${proxy.slice(7)}`, "http");
|
|
1115
|
+
} else if (proxy.startsWith("https://")) {
|
|
1116
|
+
return rezoProxy(`https://${proxy.slice(8)}`, "https");
|
|
1117
|
+
}
|
|
1118
|
+
return rezoProxy(proxy);
|
|
1119
|
+
}
|
|
1120
|
+
if (proxy.protocol === "http" || proxy.protocol === "https") {
|
|
1121
|
+
return rezoProxy({
|
|
1122
|
+
...proxy,
|
|
1123
|
+
client: !isScure ? "http" : "https"
|
|
1124
|
+
});
|
|
1125
|
+
}
|
|
1126
|
+
return rezoProxy(proxy);
|
|
1127
|
+
}
|
|
1128
|
+
function updateCookies(config, headers, url) {
|
|
1129
|
+
const cookies = headers["set-cookie"];
|
|
1130
|
+
if (cookies) {
|
|
1131
|
+
const jar = new RezoCookieJar;
|
|
1132
|
+
if (config.enableCookieJar && config.cookieJar) {
|
|
1133
|
+
config.cookieJar.setCookiesSync(cookies, url);
|
|
1134
|
+
}
|
|
1135
|
+
jar.setCookiesSync(cookies, url);
|
|
1136
|
+
if (config.useCookies) {
|
|
1137
|
+
const parsedCookies = jar.cookies();
|
|
1138
|
+
const existingArray = config.responseCookies?.array || [];
|
|
1139
|
+
for (const cookie of parsedCookies.array) {
|
|
1140
|
+
const existingIndex = existingArray.findIndex((c) => c.key === cookie.key && c.domain === cookie.domain);
|
|
1141
|
+
if (existingIndex >= 0) {
|
|
1142
|
+
existingArray[existingIndex] = cookie;
|
|
1143
|
+
} else {
|
|
1144
|
+
existingArray.push(cookie);
|
|
1145
|
+
}
|
|
1146
|
+
}
|
|
1147
|
+
const mergedJar = new RezoCookieJar(existingArray, url);
|
|
1148
|
+
config.responseCookies = mergedJar.cookies();
|
|
1149
|
+
}
|
|
1150
|
+
}
|
|
1151
|
+
}
|
|
1152
|
+
|
|
1153
|
+
exports.executeRequest = executeRequest;
|