rezo 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +202 -0
- package/README.md +1507 -0
- package/assets/icon.svg +37 -0
- package/assets/logo-dark.svg +47 -0
- package/assets/logo.svg +58 -0
- package/dist/adapters/curl.cjs +1034 -0
- package/dist/adapters/curl.js +1031 -0
- package/dist/adapters/entries/curl.cjs +4 -0
- package/dist/adapters/entries/curl.d.ts +2136 -0
- package/dist/adapters/entries/curl.js +2 -0
- package/dist/adapters/entries/fetch.cjs +2 -0
- package/dist/adapters/entries/fetch.d.ts +2127 -0
- package/dist/adapters/entries/fetch.js +1 -0
- package/dist/adapters/entries/http.cjs +2 -0
- package/dist/adapters/entries/http.d.ts +2126 -0
- package/dist/adapters/entries/http.js +1 -0
- package/dist/adapters/entries/http2.cjs +4 -0
- package/dist/adapters/entries/http2.d.ts +2136 -0
- package/dist/adapters/entries/http2.js +2 -0
- package/dist/adapters/entries/react-native.cjs +2 -0
- package/dist/adapters/entries/react-native.d.ts +2126 -0
- package/dist/adapters/entries/react-native.js +1 -0
- package/dist/adapters/entries/xhr.cjs +2 -0
- package/dist/adapters/entries/xhr.d.ts +2127 -0
- package/dist/adapters/entries/xhr.js +1 -0
- package/dist/adapters/fetch.cjs +740 -0
- package/dist/adapters/fetch.js +739 -0
- package/dist/adapters/http.cjs +1153 -0
- package/dist/adapters/http.js +1151 -0
- package/dist/adapters/http2.cjs +957 -0
- package/dist/adapters/http2.js +956 -0
- package/dist/adapters/index.cjs +6 -0
- package/dist/adapters/index.js +7 -0
- package/dist/adapters/picker.cjs +342 -0
- package/dist/adapters/picker.js +331 -0
- package/dist/adapters/react-native.cjs +545 -0
- package/dist/adapters/react-native.js +544 -0
- package/dist/adapters/xhr.cjs +622 -0
- package/dist/adapters/xhr.js +621 -0
- package/dist/cache/dns-cache.cjs +118 -0
- package/dist/cache/dns-cache.js +113 -0
- package/dist/cache/file-cacher.cjs +264 -0
- package/dist/cache/file-cacher.js +261 -0
- package/dist/cache/index.cjs +13 -0
- package/dist/cache/index.js +5 -0
- package/dist/cache/lru-cache.cjs +96 -0
- package/dist/cache/lru-cache.js +93 -0
- package/dist/cache/response-cache.cjs +314 -0
- package/dist/cache/response-cache.js +310 -0
- package/dist/cache/url-store.cjs +288 -0
- package/dist/cache/url-store.js +285 -0
- package/dist/core/hooks.cjs +133 -0
- package/dist/core/hooks.js +120 -0
- package/dist/core/rezo.cjs +464 -0
- package/dist/core/rezo.js +458 -0
- package/dist/crawler.d.ts +6255 -0
- package/dist/dom/index.cjs +1 -0
- package/dist/dom/index.d.ts +23 -0
- package/dist/dom/index.js +1 -0
- package/dist/entries/crawler.cjs +5 -0
- package/dist/entries/crawler.js +2 -0
- package/dist/errors/rezo-error.cjs +722 -0
- package/dist/errors/rezo-error.js +716 -0
- package/dist/index.cjs +34 -0
- package/dist/index.d.ts +3335 -0
- package/dist/index.js +26 -0
- package/dist/platform/browser.cjs +9 -0
- package/dist/platform/browser.d.ts +3203 -0
- package/dist/platform/browser.js +7 -0
- package/dist/platform/bun.cjs +9 -0
- package/dist/platform/bun.d.ts +3203 -0
- package/dist/platform/bun.js +7 -0
- package/dist/platform/deno.cjs +9 -0
- package/dist/platform/deno.d.ts +3203 -0
- package/dist/platform/deno.js +7 -0
- package/dist/platform/node.cjs +9 -0
- package/dist/platform/node.d.ts +3203 -0
- package/dist/platform/node.js +7 -0
- package/dist/platform/react-native.cjs +9 -0
- package/dist/platform/react-native.d.ts +3203 -0
- package/dist/platform/react-native.js +7 -0
- package/dist/platform/worker.cjs +9 -0
- package/dist/platform/worker.d.ts +3203 -0
- package/dist/platform/worker.js +7 -0
- package/dist/plugin/addon/decodo/index.cjs +1 -0
- package/dist/plugin/addon/decodo/index.js +1 -0
- package/dist/plugin/addon/decodo/options.cjs +1 -0
- package/dist/plugin/addon/decodo/options.js +1 -0
- package/dist/plugin/addon/oxylabs/index.cjs +1 -0
- package/dist/plugin/addon/oxylabs/index.js +1 -0
- package/dist/plugin/addon/oxylabs/options.cjs +1 -0
- package/dist/plugin/addon/oxylabs/options.js +1 -0
- package/dist/plugin/crawler-options.cjs +1 -0
- package/dist/plugin/crawler-options.js +1 -0
- package/dist/plugin/crawler.cjs +519 -0
- package/dist/plugin/crawler.js +517 -0
- package/dist/plugin/index.cjs +36 -0
- package/dist/plugin/index.js +32 -0
- package/dist/proxy/index.cjs +142 -0
- package/dist/proxy/index.js +139 -0
- package/dist/responses/buildError.cjs +452 -0
- package/dist/responses/buildError.js +441 -0
- package/dist/responses/buildResponse.cjs +365 -0
- package/dist/responses/buildResponse.js +361 -0
- package/dist/responses/download.cjs +54 -0
- package/dist/responses/download.js +52 -0
- package/dist/responses/stream.cjs +60 -0
- package/dist/responses/stream.js +58 -0
- package/dist/responses/upload.cjs +54 -0
- package/dist/responses/upload.js +52 -0
- package/dist/types/cookies.cjs +394 -0
- package/dist/types/cookies.js +391 -0
- package/dist/types/download.cjs +10 -0
- package/dist/types/download.js +10 -0
- package/dist/types/rezo-request.cjs +131 -0
- package/dist/types/rezo-request.js +131 -0
- package/dist/utils/agent-merger.cjs +111 -0
- package/dist/utils/agent-merger.js +108 -0
- package/dist/utils/compression.cjs +84 -0
- package/dist/utils/compression.js +82 -0
- package/dist/utils/cookies.cjs +514 -0
- package/dist/utils/cookies.js +511 -0
- package/dist/utils/data-operations.cjs +75 -0
- package/dist/utils/data-operations.js +73 -0
- package/dist/utils/form-data.cjs +164 -0
- package/dist/utils/form-data.js +161 -0
- package/dist/utils/headers.cjs +162 -0
- package/dist/utils/headers.js +161 -0
- package/dist/utils/http-config.cjs +723 -0
- package/dist/utils/http-config.js +718 -0
- package/dist/utils/index.cjs +8 -0
- package/dist/utils/index.js +8 -0
- package/dist/utils/tools.cjs +18 -0
- package/dist/utils/tools.js +15 -0
- package/package.json +172 -0
|
@@ -0,0 +1,740 @@
|
|
|
1
|
+
const { URL } = require("node:url");
|
|
2
|
+
const { RezoError } = require('../errors/rezo-error.cjs');
|
|
3
|
+
const { buildSmartError, builErrorFromResponse, buildDownloadError } = require('../responses/buildError.cjs');
|
|
4
|
+
const { Cookie } = require('../utils/cookies.cjs');
|
|
5
|
+
const RezoFormData = require('../utils/form-data.cjs');
|
|
6
|
+
const { getDefaultConfig, prepareHTTPOptions } = require('../utils/http-config.cjs');
|
|
7
|
+
const { RezoHeaders } = require('../utils/headers.cjs');
|
|
8
|
+
const { RezoURLSearchParams } = require('../utils/data-operations.cjs');
|
|
9
|
+
const { StreamResponse } = require('../responses/stream.cjs');
|
|
10
|
+
const { DownloadResponse } = require('../responses/download.cjs');
|
|
11
|
+
const { UploadResponse } = require('../responses/upload.cjs');
|
|
12
|
+
const { isSameDomain, RezoPerformance } = require('../utils/tools.cjs');
|
|
13
|
+
const { ResponseCache } = require('../cache/response-cache.cjs');
|
|
14
|
+
const Environment = {
|
|
15
|
+
isNode: typeof process !== "undefined" && process.versions?.node,
|
|
16
|
+
isBrowser: typeof window !== "undefined" && typeof document !== "undefined",
|
|
17
|
+
isWebWorker: typeof self !== "undefined" && typeof self.WorkerGlobalScope !== "undefined",
|
|
18
|
+
isDeno: typeof globalThis.Deno !== "undefined",
|
|
19
|
+
isBun: typeof globalThis.Bun !== "undefined",
|
|
20
|
+
isEdgeRuntime: typeof globalThis.EdgeRuntime !== "undefined" || typeof globalThis.caches !== "undefined",
|
|
21
|
+
get hasFetch() {
|
|
22
|
+
return typeof fetch !== "undefined";
|
|
23
|
+
},
|
|
24
|
+
get hasReadableStream() {
|
|
25
|
+
return typeof ReadableStream !== "undefined";
|
|
26
|
+
},
|
|
27
|
+
get hasAbortController() {
|
|
28
|
+
return typeof AbortController !== "undefined";
|
|
29
|
+
}
|
|
30
|
+
};
|
|
31
|
+
const responseCacheInstances = new Map;
|
|
32
|
+
function getCacheConfigKey(option) {
|
|
33
|
+
if (option === true)
|
|
34
|
+
return "default";
|
|
35
|
+
if (option === false)
|
|
36
|
+
return "disabled";
|
|
37
|
+
const cfg = option;
|
|
38
|
+
return JSON.stringify({
|
|
39
|
+
cacheDir: cfg.cacheDir || null,
|
|
40
|
+
ttl: cfg.ttl || 300000,
|
|
41
|
+
maxEntries: cfg.maxEntries || 500,
|
|
42
|
+
methods: cfg.methods || ["GET", "HEAD"],
|
|
43
|
+
respectHeaders: cfg.respectHeaders !== false
|
|
44
|
+
});
|
|
45
|
+
}
|
|
46
|
+
function getResponseCache(option) {
|
|
47
|
+
const key = getCacheConfigKey(option);
|
|
48
|
+
let cache = responseCacheInstances.get(key);
|
|
49
|
+
if (!cache) {
|
|
50
|
+
cache = new ResponseCache(option);
|
|
51
|
+
responseCacheInstances.set(key, cache);
|
|
52
|
+
}
|
|
53
|
+
return cache;
|
|
54
|
+
}
|
|
55
|
+
function parseCacheControlFromHeaders(headers) {
|
|
56
|
+
const cacheControl = headers["cache-control"] || "";
|
|
57
|
+
return {
|
|
58
|
+
noCache: cacheControl.includes("no-cache"),
|
|
59
|
+
mustRevalidate: cacheControl.includes("must-revalidate")
|
|
60
|
+
};
|
|
61
|
+
}
|
|
62
|
+
function buildCachedRezoResponse(cached, config) {
|
|
63
|
+
const headers = new RezoHeaders(cached.headers);
|
|
64
|
+
return {
|
|
65
|
+
data: cached.data,
|
|
66
|
+
status: cached.status,
|
|
67
|
+
statusText: cached.statusText,
|
|
68
|
+
headers,
|
|
69
|
+
finalUrl: cached.url,
|
|
70
|
+
urls: [cached.url],
|
|
71
|
+
contentType: cached.headers["content-type"],
|
|
72
|
+
contentLength: parseInt(cached.headers["content-length"] || "0", 10) || 0,
|
|
73
|
+
cookies: {
|
|
74
|
+
array: [],
|
|
75
|
+
serialized: [],
|
|
76
|
+
netscape: "",
|
|
77
|
+
string: "",
|
|
78
|
+
setCookiesString: []
|
|
79
|
+
},
|
|
80
|
+
config: {
|
|
81
|
+
...config,
|
|
82
|
+
url: cached.url,
|
|
83
|
+
method: "GET",
|
|
84
|
+
headers,
|
|
85
|
+
adapterUsed: "fetch",
|
|
86
|
+
fromCache: true
|
|
87
|
+
}
|
|
88
|
+
};
|
|
89
|
+
}
|
|
90
|
+
function buildUrlTree(config, finalUrl) {
|
|
91
|
+
const urls = [];
|
|
92
|
+
if (config.rawUrl) {
|
|
93
|
+
urls.push(config.rawUrl);
|
|
94
|
+
} else if (config.url) {
|
|
95
|
+
const urlStr = typeof config.url === "string" ? config.url : config.url.toString();
|
|
96
|
+
urls.push(urlStr);
|
|
97
|
+
}
|
|
98
|
+
if (finalUrl && (urls.length === 0 || urls[0] !== finalUrl)) {
|
|
99
|
+
urls.push(finalUrl);
|
|
100
|
+
}
|
|
101
|
+
return urls.length > 0 ? urls : [finalUrl];
|
|
102
|
+
}
|
|
103
|
+
function sanitizeConfig(config) {
|
|
104
|
+
const sanitized = { ...config };
|
|
105
|
+
delete sanitized.data;
|
|
106
|
+
return sanitized;
|
|
107
|
+
}
|
|
108
|
+
function parseCookiesFromHeaders(headers, url) {
|
|
109
|
+
const cookies = {
|
|
110
|
+
array: [],
|
|
111
|
+
serialized: [],
|
|
112
|
+
netscape: "",
|
|
113
|
+
string: "",
|
|
114
|
+
setCookiesString: []
|
|
115
|
+
};
|
|
116
|
+
const setCookieHeaders = headers.getSetCookie?.() || [];
|
|
117
|
+
for (const cookieStr of setCookieHeaders) {
|
|
118
|
+
cookies.setCookiesString.push(cookieStr);
|
|
119
|
+
const parts = cookieStr.split(";");
|
|
120
|
+
const [nameValue] = parts;
|
|
121
|
+
const [name, ...valueParts] = nameValue.split("=");
|
|
122
|
+
const value = valueParts.join("=");
|
|
123
|
+
if (name && value !== undefined) {
|
|
124
|
+
const cookie = new Cookie({
|
|
125
|
+
key: name.trim(),
|
|
126
|
+
value: value.trim(),
|
|
127
|
+
domain: new URL(url).hostname,
|
|
128
|
+
path: "/",
|
|
129
|
+
httpOnly: cookieStr.toLowerCase().includes("httponly"),
|
|
130
|
+
secure: cookieStr.toLowerCase().includes("secure"),
|
|
131
|
+
sameSite: "lax"
|
|
132
|
+
});
|
|
133
|
+
cookies.array.push(cookie);
|
|
134
|
+
}
|
|
135
|
+
}
|
|
136
|
+
cookies.string = cookies.array.map((c) => `${c.key}=${c.value}`).join("; ");
|
|
137
|
+
cookies.serialized = cookies.array.map((c) => c.toJSON());
|
|
138
|
+
cookies.netscape = cookies.array.map((c) => c.toNetscapeFormat()).join(`
|
|
139
|
+
`);
|
|
140
|
+
return cookies;
|
|
141
|
+
}
|
|
142
|
+
function toFetchHeaders(headers) {
|
|
143
|
+
const fetchHeaders = new Headers;
|
|
144
|
+
if (!headers)
|
|
145
|
+
return fetchHeaders;
|
|
146
|
+
if (headers instanceof RezoHeaders) {
|
|
147
|
+
for (const [key, value] of headers.entries()) {
|
|
148
|
+
if (value !== undefined && value !== null) {
|
|
149
|
+
fetchHeaders.set(key, String(value));
|
|
150
|
+
}
|
|
151
|
+
}
|
|
152
|
+
} else {
|
|
153
|
+
for (const [key, value] of Object.entries(headers)) {
|
|
154
|
+
if (value !== undefined && value !== null) {
|
|
155
|
+
fetchHeaders.set(key, String(value));
|
|
156
|
+
}
|
|
157
|
+
}
|
|
158
|
+
}
|
|
159
|
+
return fetchHeaders;
|
|
160
|
+
}
|
|
161
|
+
function fromFetchHeaders(headers) {
|
|
162
|
+
const record = {};
|
|
163
|
+
headers.forEach((value, key) => {
|
|
164
|
+
record[key.toLowerCase()] = value;
|
|
165
|
+
});
|
|
166
|
+
return new RezoHeaders(record);
|
|
167
|
+
}
|
|
168
|
+
async function prepareFetchBody(body) {
|
|
169
|
+
if (!body)
|
|
170
|
+
return;
|
|
171
|
+
if (body instanceof URLSearchParams || body instanceof RezoURLSearchParams) {
|
|
172
|
+
return body.toString();
|
|
173
|
+
}
|
|
174
|
+
if (body instanceof FormData) {
|
|
175
|
+
return body;
|
|
176
|
+
}
|
|
177
|
+
if (body instanceof RezoFormData) {
|
|
178
|
+
if (Environment.isNode) {
|
|
179
|
+
const buffer = body.getBuffer();
|
|
180
|
+
if (buffer) {
|
|
181
|
+
return new Uint8Array(buffer);
|
|
182
|
+
}
|
|
183
|
+
}
|
|
184
|
+
const nativeForm = body.toNativeFormData();
|
|
185
|
+
if (nativeForm) {
|
|
186
|
+
return nativeForm;
|
|
187
|
+
}
|
|
188
|
+
return;
|
|
189
|
+
}
|
|
190
|
+
if (typeof body === "object" && !(body instanceof ArrayBuffer) && !(body instanceof Uint8Array)) {
|
|
191
|
+
return JSON.stringify(body);
|
|
192
|
+
}
|
|
193
|
+
return body;
|
|
194
|
+
}
|
|
195
|
+
async function executeRequest(options, defaultOptions, jar) {
|
|
196
|
+
if (!Environment.hasFetch) {
|
|
197
|
+
throw new Error("Fetch API is not available in this environment");
|
|
198
|
+
}
|
|
199
|
+
if (!options.responseType) {
|
|
200
|
+
options.responseType = "auto";
|
|
201
|
+
}
|
|
202
|
+
const d_options = await getDefaultConfig(defaultOptions);
|
|
203
|
+
const configResult = prepareHTTPOptions(options, jar, { defaultOptions: d_options });
|
|
204
|
+
let mainConfig = configResult.config;
|
|
205
|
+
const fetchOptions = configResult.fetchOptions;
|
|
206
|
+
const perform = new RezoPerformance;
|
|
207
|
+
const cacheOption = options.cache;
|
|
208
|
+
const method = (options.method || "GET").toUpperCase();
|
|
209
|
+
const requestUrl = typeof fetchOptions.url === "string" ? fetchOptions.url : fetchOptions.url?.toString() || "";
|
|
210
|
+
let cache;
|
|
211
|
+
let requestHeaders;
|
|
212
|
+
let cachedEntry;
|
|
213
|
+
if (cacheOption) {
|
|
214
|
+
cache = getResponseCache(cacheOption);
|
|
215
|
+
requestHeaders = fetchOptions.headers instanceof RezoHeaders ? Object.fromEntries(fetchOptions.headers.entries()) : fetchOptions.headers;
|
|
216
|
+
cachedEntry = cache.get(method, requestUrl, requestHeaders);
|
|
217
|
+
if (cachedEntry) {
|
|
218
|
+
const cacheControl = parseCacheControlFromHeaders(cachedEntry.headers);
|
|
219
|
+
if (!cacheControl.noCache && !cacheControl.mustRevalidate) {
|
|
220
|
+
return buildCachedRezoResponse(cachedEntry, mainConfig);
|
|
221
|
+
}
|
|
222
|
+
}
|
|
223
|
+
const conditionalHeaders = cache.getConditionalHeaders(method, requestUrl, requestHeaders);
|
|
224
|
+
if (conditionalHeaders) {
|
|
225
|
+
const headers = fetchOptions.headers instanceof RezoHeaders ? fetchOptions.headers : new RezoHeaders(fetchOptions.headers || {});
|
|
226
|
+
if (conditionalHeaders.etag) {
|
|
227
|
+
headers.set("If-None-Match", conditionalHeaders.etag);
|
|
228
|
+
}
|
|
229
|
+
if (conditionalHeaders.lastModified) {
|
|
230
|
+
headers.set("If-Modified-Since", conditionalHeaders.lastModified);
|
|
231
|
+
}
|
|
232
|
+
fetchOptions.headers = headers;
|
|
233
|
+
}
|
|
234
|
+
}
|
|
235
|
+
const isStream = options._isStream;
|
|
236
|
+
const isDownload = options._isDownload || !!options.fileName || !!options.saveTo;
|
|
237
|
+
const isUpload = options._isUpload;
|
|
238
|
+
let streamResponse;
|
|
239
|
+
let downloadResponse;
|
|
240
|
+
let uploadResponse;
|
|
241
|
+
if (isStream) {
|
|
242
|
+
streamResponse = new StreamResponse;
|
|
243
|
+
} else if (isDownload) {
|
|
244
|
+
const fileName = options.fileName || options.saveTo || "";
|
|
245
|
+
const url = typeof fetchOptions.url === "string" ? fetchOptions.url : fetchOptions.url?.toString() || "";
|
|
246
|
+
downloadResponse = new DownloadResponse(fileName, url);
|
|
247
|
+
} else if (isUpload) {
|
|
248
|
+
const url = typeof fetchOptions.url === "string" ? fetchOptions.url : fetchOptions.url?.toString() || "";
|
|
249
|
+
uploadResponse = new UploadResponse(url);
|
|
250
|
+
}
|
|
251
|
+
const res = executeFetchRequest(fetchOptions, mainConfig, options, perform, streamResponse, downloadResponse, uploadResponse);
|
|
252
|
+
if (streamResponse) {
|
|
253
|
+
return streamResponse;
|
|
254
|
+
} else if (downloadResponse) {
|
|
255
|
+
return downloadResponse;
|
|
256
|
+
} else if (uploadResponse) {
|
|
257
|
+
return uploadResponse;
|
|
258
|
+
}
|
|
259
|
+
const response = await res;
|
|
260
|
+
if (cache && !isStream && !isDownload && !isUpload) {
|
|
261
|
+
if (response.status === 304 && cachedEntry) {
|
|
262
|
+
const responseHeaders = response.headers instanceof RezoHeaders ? Object.fromEntries(response.headers.entries()) : response.headers;
|
|
263
|
+
const updatedCached = cache.updateRevalidated(method, requestUrl, responseHeaders, requestHeaders);
|
|
264
|
+
if (updatedCached) {
|
|
265
|
+
return buildCachedRezoResponse(updatedCached, mainConfig);
|
|
266
|
+
}
|
|
267
|
+
return buildCachedRezoResponse(cachedEntry, mainConfig);
|
|
268
|
+
}
|
|
269
|
+
if (response.status >= 200 && response.status < 300) {
|
|
270
|
+
cache.set(method, requestUrl, response, requestHeaders);
|
|
271
|
+
}
|
|
272
|
+
}
|
|
273
|
+
return response;
|
|
274
|
+
}
|
|
275
|
+
async function executeFetchRequest(fetchOptions, config, options, perform, streamResult, downloadResult, uploadResult) {
|
|
276
|
+
let requestCount = 0;
|
|
277
|
+
const _stats = { statusOnNext: "abort" };
|
|
278
|
+
let retries = 0;
|
|
279
|
+
const retryDelay = config?.retry?.retryDelay || 0;
|
|
280
|
+
const maxRetries = config?.retry?.maxRetries || 0;
|
|
281
|
+
const incrementDelay = config?.retry?.incrementDelay || false;
|
|
282
|
+
const statusCodes = config?.retry?.statusCodes;
|
|
283
|
+
const timing = {
|
|
284
|
+
startTime: performance.now(),
|
|
285
|
+
startTimestamp: Date.now()
|
|
286
|
+
};
|
|
287
|
+
const ABSOLUTE_MAX_ATTEMPTS = 50;
|
|
288
|
+
const visitedUrls = new Set;
|
|
289
|
+
let totalAttempts = 0;
|
|
290
|
+
config.setSignal();
|
|
291
|
+
const eventEmitter = streamResult || downloadResult || uploadResult;
|
|
292
|
+
if (eventEmitter) {
|
|
293
|
+
eventEmitter.emit("initiated");
|
|
294
|
+
}
|
|
295
|
+
while (true) {
|
|
296
|
+
totalAttempts++;
|
|
297
|
+
if (totalAttempts > ABSOLUTE_MAX_ATTEMPTS) {
|
|
298
|
+
const error = builErrorFromResponse(`Absolute maximum attempts (${ABSOLUTE_MAX_ATTEMPTS}) exceeded.`, { status: 0, statusText: "Max Attempts Exceeded" }, config, fetchOptions);
|
|
299
|
+
throw error;
|
|
300
|
+
}
|
|
301
|
+
try {
|
|
302
|
+
const response = await executeSingleFetchRequest(config, fetchOptions, requestCount, timing, _stats, streamResult, downloadResult, uploadResult);
|
|
303
|
+
const statusOnNext = _stats.statusOnNext;
|
|
304
|
+
if (response instanceof RezoError) {
|
|
305
|
+
config.errors.push({
|
|
306
|
+
attempt: config.retryAttempts + 1,
|
|
307
|
+
error: response,
|
|
308
|
+
duration: perform.now()
|
|
309
|
+
});
|
|
310
|
+
perform.reset();
|
|
311
|
+
if (!config.retry) {
|
|
312
|
+
throw response;
|
|
313
|
+
}
|
|
314
|
+
if (config.retry) {
|
|
315
|
+
if (config.retry.condition) {
|
|
316
|
+
const isPassed = await config.retry.condition(response);
|
|
317
|
+
if (typeof isPassed === "boolean" && isPassed === false) {
|
|
318
|
+
throw response;
|
|
319
|
+
}
|
|
320
|
+
} else {
|
|
321
|
+
if (statusCodes && !statusCodes.includes(response.status || 0)) {
|
|
322
|
+
throw response;
|
|
323
|
+
}
|
|
324
|
+
if (maxRetries <= retries) {
|
|
325
|
+
throw response;
|
|
326
|
+
}
|
|
327
|
+
retries++;
|
|
328
|
+
if (retryDelay > 0) {
|
|
329
|
+
await new Promise((resolve) => setTimeout(resolve, incrementDelay ? retryDelay * retries : retryDelay));
|
|
330
|
+
}
|
|
331
|
+
}
|
|
332
|
+
config.retryAttempts++;
|
|
333
|
+
}
|
|
334
|
+
continue;
|
|
335
|
+
}
|
|
336
|
+
if (statusOnNext === "success") {
|
|
337
|
+
return response;
|
|
338
|
+
}
|
|
339
|
+
if (statusOnNext === "redirect") {
|
|
340
|
+
const location = _stats.redirectUrl;
|
|
341
|
+
if (!location) {
|
|
342
|
+
throw builErrorFromResponse("Redirect location not found", response, config, fetchOptions);
|
|
343
|
+
}
|
|
344
|
+
if (config.maxRedirects === 0) {
|
|
345
|
+
config.maxRedirectsReached = true;
|
|
346
|
+
throw builErrorFromResponse("Redirects are disabled (maxRedirects=0)", response, config, fetchOptions);
|
|
347
|
+
}
|
|
348
|
+
const enableCycleDetection = config.enableRedirectCycleDetection === true;
|
|
349
|
+
if (enableCycleDetection) {
|
|
350
|
+
const normalizedRedirectUrl = location.toLowerCase();
|
|
351
|
+
if (visitedUrls.has(normalizedRedirectUrl)) {
|
|
352
|
+
throw builErrorFromResponse(`Redirect cycle detected: ${location}`, response, config, fetchOptions);
|
|
353
|
+
}
|
|
354
|
+
visitedUrls.add(normalizedRedirectUrl);
|
|
355
|
+
}
|
|
356
|
+
const redirectCode = response.status;
|
|
357
|
+
const onRedirect = config.beforeRedirect ? config.beforeRedirect({
|
|
358
|
+
url: new URL(location),
|
|
359
|
+
status: response.status,
|
|
360
|
+
headers: response.headers,
|
|
361
|
+
sameDomain: isSameDomain(fetchOptions.fullUrl, location),
|
|
362
|
+
method: fetchOptions.method.toUpperCase()
|
|
363
|
+
}) : undefined;
|
|
364
|
+
if (typeof onRedirect !== "undefined") {
|
|
365
|
+
if (typeof onRedirect === "boolean" && !onRedirect) {
|
|
366
|
+
throw builErrorFromResponse("Redirect denied by user", response, config, fetchOptions);
|
|
367
|
+
} else if (typeof onRedirect === "object" && !onRedirect.redirect) {
|
|
368
|
+
throw builErrorFromResponse("Redirect denied by user", response, config, fetchOptions);
|
|
369
|
+
}
|
|
370
|
+
}
|
|
371
|
+
if (config.redirectCount >= config.maxRedirects && config.maxRedirects > 0) {
|
|
372
|
+
config.maxRedirectsReached = true;
|
|
373
|
+
throw builErrorFromResponse(`Max redirects (${config.maxRedirects}) reached`, response, config, fetchOptions);
|
|
374
|
+
}
|
|
375
|
+
config.redirectHistory.push({
|
|
376
|
+
url: fetchOptions.fullUrl,
|
|
377
|
+
statusCode: redirectCode,
|
|
378
|
+
statusText: response.statusText,
|
|
379
|
+
headers: response.headers,
|
|
380
|
+
method: fetchOptions.method.toUpperCase(),
|
|
381
|
+
cookies: response.cookies.array,
|
|
382
|
+
duration: perform.now(),
|
|
383
|
+
request: fetchOptions
|
|
384
|
+
});
|
|
385
|
+
perform.reset();
|
|
386
|
+
config.redirectCount++;
|
|
387
|
+
if (response.status === 301 || response.status === 302 || response.status === 303) {
|
|
388
|
+
if (config.treat302As303 !== false || response.status === 303) {
|
|
389
|
+
options.method = "GET";
|
|
390
|
+
delete options.body;
|
|
391
|
+
}
|
|
392
|
+
}
|
|
393
|
+
fetchOptions.fullUrl = location;
|
|
394
|
+
delete options.params;
|
|
395
|
+
requestCount++;
|
|
396
|
+
continue;
|
|
397
|
+
}
|
|
398
|
+
throw builErrorFromResponse("Unexpected state", response, config, fetchOptions);
|
|
399
|
+
} catch (error) {
|
|
400
|
+
if (error instanceof RezoError) {
|
|
401
|
+
throw error;
|
|
402
|
+
}
|
|
403
|
+
throw buildSmartError(config, fetchOptions, error);
|
|
404
|
+
}
|
|
405
|
+
}
|
|
406
|
+
}
|
|
407
|
+
async function executeSingleFetchRequest(config, fetchOptions, requestCount, timing, _stats, streamResult, downloadResult, uploadResult) {
|
|
408
|
+
try {
|
|
409
|
+
const { fullUrl, body } = fetchOptions;
|
|
410
|
+
const url = new URL(fullUrl || fetchOptions.url);
|
|
411
|
+
const isSecure = url.protocol === "https:";
|
|
412
|
+
if (requestCount === 0) {
|
|
413
|
+
config.adapterUsed = "fetch";
|
|
414
|
+
config.isSecure = isSecure;
|
|
415
|
+
config.finalUrl = url.href;
|
|
416
|
+
config.network.protocol = isSecure ? "https" : "http";
|
|
417
|
+
config.timing.startTimestamp = timing.startTimestamp;
|
|
418
|
+
}
|
|
419
|
+
const reqHeaders = fetchOptions.headers instanceof RezoHeaders ? fetchOptions.headers.toObject() : fetchOptions.headers || {};
|
|
420
|
+
const headers = toFetchHeaders(reqHeaders);
|
|
421
|
+
const eventEmitter = streamResult || downloadResult || uploadResult;
|
|
422
|
+
if (eventEmitter && requestCount === 0) {
|
|
423
|
+
const startEvent = {
|
|
424
|
+
url: url.toString(),
|
|
425
|
+
method: fetchOptions.method.toUpperCase(),
|
|
426
|
+
headers: new RezoHeaders(reqHeaders),
|
|
427
|
+
timestamp: timing.startTime,
|
|
428
|
+
timeout: fetchOptions.timeout,
|
|
429
|
+
maxRedirects: config.maxRedirects
|
|
430
|
+
};
|
|
431
|
+
eventEmitter.emit("start", startEvent);
|
|
432
|
+
}
|
|
433
|
+
const abortController = new AbortController;
|
|
434
|
+
let timeoutId;
|
|
435
|
+
if (config.timeout) {
|
|
436
|
+
timeoutId = setTimeout(() => {
|
|
437
|
+
abortController.abort();
|
|
438
|
+
}, config.timeout);
|
|
439
|
+
}
|
|
440
|
+
if (config.signal) {
|
|
441
|
+
config.signal.addEventListener("abort", () => {
|
|
442
|
+
abortController.abort();
|
|
443
|
+
if (timeoutId)
|
|
444
|
+
clearTimeout(timeoutId);
|
|
445
|
+
});
|
|
446
|
+
}
|
|
447
|
+
const preparedBody = await prepareFetchBody(body);
|
|
448
|
+
const fetchInit = {
|
|
449
|
+
method: fetchOptions.method.toUpperCase(),
|
|
450
|
+
headers,
|
|
451
|
+
body: preparedBody,
|
|
452
|
+
signal: abortController.signal,
|
|
453
|
+
redirect: "manual",
|
|
454
|
+
credentials: config.enableCookieJar ? "include" : "same-origin"
|
|
455
|
+
};
|
|
456
|
+
let response;
|
|
457
|
+
try {
|
|
458
|
+
response = await fetch(url.toString(), fetchInit);
|
|
459
|
+
} catch (err) {
|
|
460
|
+
if (timeoutId)
|
|
461
|
+
clearTimeout(timeoutId);
|
|
462
|
+
if (err.name === "AbortError") {
|
|
463
|
+
const error = buildSmartError(config, fetchOptions, new Error(`Request timeout after ${config.timeout}ms`));
|
|
464
|
+
_stats.statusOnNext = "error";
|
|
465
|
+
return error;
|
|
466
|
+
}
|
|
467
|
+
throw err;
|
|
468
|
+
} finally {
|
|
469
|
+
if (timeoutId)
|
|
470
|
+
clearTimeout(timeoutId);
|
|
471
|
+
}
|
|
472
|
+
if (!config.timing.ttfbMs) {
|
|
473
|
+
timing.firstByteTime = performance.now();
|
|
474
|
+
config.timing.ttfbMs = timing.firstByteTime - timing.startTime;
|
|
475
|
+
}
|
|
476
|
+
const status = response.status;
|
|
477
|
+
const statusText = response.statusText;
|
|
478
|
+
const responseHeaders = fromFetchHeaders(response.headers);
|
|
479
|
+
const contentType = response.headers.get("content-type") || "";
|
|
480
|
+
const contentLength = response.headers.get("content-length");
|
|
481
|
+
const cookies = parseCookiesFromHeaders(response.headers, url.href);
|
|
482
|
+
config.responseCookies = cookies;
|
|
483
|
+
const location = response.headers.get("location");
|
|
484
|
+
const isRedirect = status >= 300 && status < 400 && location;
|
|
485
|
+
if (isRedirect) {
|
|
486
|
+
_stats.statusOnNext = "redirect";
|
|
487
|
+
_stats.redirectUrl = new URL(location, url).href;
|
|
488
|
+
const partialResponse = {
|
|
489
|
+
data: "",
|
|
490
|
+
status,
|
|
491
|
+
statusText,
|
|
492
|
+
headers: responseHeaders,
|
|
493
|
+
cookies,
|
|
494
|
+
config,
|
|
495
|
+
contentType,
|
|
496
|
+
contentLength: 0,
|
|
497
|
+
finalUrl: url.href,
|
|
498
|
+
urls: buildUrlTree(config, url.href)
|
|
499
|
+
};
|
|
500
|
+
return partialResponse;
|
|
501
|
+
}
|
|
502
|
+
if (eventEmitter && !isRedirect) {
|
|
503
|
+
const headersEvent = {
|
|
504
|
+
status,
|
|
505
|
+
statusText,
|
|
506
|
+
headers: responseHeaders,
|
|
507
|
+
contentType,
|
|
508
|
+
contentLength: contentLength ? parseInt(contentLength, 10) : undefined,
|
|
509
|
+
cookies: cookies.array,
|
|
510
|
+
timing: {
|
|
511
|
+
firstByte: config.timing.ttfbMs,
|
|
512
|
+
total: performance.now() - timing.startTime
|
|
513
|
+
}
|
|
514
|
+
};
|
|
515
|
+
eventEmitter.emit("headers", headersEvent);
|
|
516
|
+
eventEmitter.emit("status", status, statusText);
|
|
517
|
+
eventEmitter.emit("cookies", cookies.array);
|
|
518
|
+
if (downloadResult) {
|
|
519
|
+
downloadResult.status = status;
|
|
520
|
+
downloadResult.statusText = statusText;
|
|
521
|
+
} else if (uploadResult) {
|
|
522
|
+
uploadResult.status = status;
|
|
523
|
+
uploadResult.statusText = statusText;
|
|
524
|
+
}
|
|
525
|
+
}
|
|
526
|
+
if (streamResult && response.body) {
|
|
527
|
+
handleStreamingResponse(response, config, timing, streamResult, url, status, statusText, responseHeaders, cookies);
|
|
528
|
+
return {};
|
|
529
|
+
}
|
|
530
|
+
let responseData;
|
|
531
|
+
let bodyBuffer;
|
|
532
|
+
const responseType = config.responseType || fetchOptions.responseType || "auto";
|
|
533
|
+
if (responseType === "buffer" || responseType === "arrayBuffer") {
|
|
534
|
+
bodyBuffer = await response.arrayBuffer();
|
|
535
|
+
responseData = Environment.isNode ? Buffer.from(bodyBuffer) : bodyBuffer;
|
|
536
|
+
} else if (responseType === "blob") {
|
|
537
|
+
responseData = await response.blob();
|
|
538
|
+
} else if (responseType === "text") {
|
|
539
|
+
responseData = await response.text();
|
|
540
|
+
} else if (responseType === "json") {
|
|
541
|
+
try {
|
|
542
|
+
responseData = await response.json();
|
|
543
|
+
} catch {
|
|
544
|
+
responseData = await response.text();
|
|
545
|
+
}
|
|
546
|
+
} else {
|
|
547
|
+
if (contentType.includes("application/json")) {
|
|
548
|
+
try {
|
|
549
|
+
responseData = await response.json();
|
|
550
|
+
} catch {
|
|
551
|
+
responseData = await response.text();
|
|
552
|
+
}
|
|
553
|
+
} else {
|
|
554
|
+
responseData = await response.text();
|
|
555
|
+
}
|
|
556
|
+
}
|
|
557
|
+
config.timing.endTimestamp = Date.now();
|
|
558
|
+
config.timing.durationMs = performance.now() - timing.startTime;
|
|
559
|
+
config.timing.transferMs = timing.firstByteTime ? performance.now() - timing.firstByteTime : config.timing.durationMs;
|
|
560
|
+
const bodySize = bodyBuffer?.byteLength || (typeof responseData === "string" ? responseData.length : 0);
|
|
561
|
+
config.transfer.bodySize = bodySize;
|
|
562
|
+
config.transfer.responseSize = bodySize;
|
|
563
|
+
if (status >= 400) {
|
|
564
|
+
const error = builErrorFromResponse(`HTTP Error ${status}: ${statusText}`, {
|
|
565
|
+
status,
|
|
566
|
+
statusText,
|
|
567
|
+
headers: responseHeaders,
|
|
568
|
+
data: responseData
|
|
569
|
+
}, config, fetchOptions);
|
|
570
|
+
_stats.statusOnNext = "error";
|
|
571
|
+
return error;
|
|
572
|
+
}
|
|
573
|
+
_stats.statusOnNext = "success";
|
|
574
|
+
const finalResponse = {
|
|
575
|
+
data: responseData,
|
|
576
|
+
status,
|
|
577
|
+
statusText,
|
|
578
|
+
headers: responseHeaders,
|
|
579
|
+
cookies,
|
|
580
|
+
config,
|
|
581
|
+
contentType,
|
|
582
|
+
contentLength: bodySize,
|
|
583
|
+
finalUrl: url.href,
|
|
584
|
+
urls: buildUrlTree(config, url.href)
|
|
585
|
+
};
|
|
586
|
+
if (downloadResult && config.fileName && Environment.isNode) {
|
|
587
|
+
try {
|
|
588
|
+
const fs = await import("node:fs");
|
|
589
|
+
const buffer = bodyBuffer ? Buffer.from(bodyBuffer) : Buffer.from(responseData);
|
|
590
|
+
fs.writeFileSync(config.fileName, buffer);
|
|
591
|
+
const downloadFinishEvent = {
|
|
592
|
+
status,
|
|
593
|
+
statusText,
|
|
594
|
+
headers: responseHeaders,
|
|
595
|
+
contentType,
|
|
596
|
+
contentLength: buffer.length,
|
|
597
|
+
finalUrl: url.href,
|
|
598
|
+
cookies,
|
|
599
|
+
urls: buildUrlTree(config, url.href),
|
|
600
|
+
fileName: config.fileName,
|
|
601
|
+
fileSize: buffer.length,
|
|
602
|
+
timing: {
|
|
603
|
+
total: config.timing.durationMs || 0,
|
|
604
|
+
dns: config.timing.dnsMs,
|
|
605
|
+
tcp: config.timing.tcpMs,
|
|
606
|
+
tls: config.timing.tlsMs,
|
|
607
|
+
firstByte: config.timing.ttfbMs,
|
|
608
|
+
download: config.timing.transferMs || 0
|
|
609
|
+
},
|
|
610
|
+
averageSpeed: config.timing.transferMs ? buffer.length / config.timing.transferMs * 1000 : 0,
|
|
611
|
+
config: sanitizeConfig(config)
|
|
612
|
+
};
|
|
613
|
+
downloadResult.emit("finish", downloadFinishEvent);
|
|
614
|
+
downloadResult.emit("done", downloadFinishEvent);
|
|
615
|
+
downloadResult._markFinished();
|
|
616
|
+
} catch (err) {
|
|
617
|
+
const error = buildDownloadError({
|
|
618
|
+
statusCode: status,
|
|
619
|
+
headers: Object.fromEntries(responseHeaders.entries()),
|
|
620
|
+
contentType,
|
|
621
|
+
contentLength: String(bodySize),
|
|
622
|
+
cookies: cookies.setCookiesString,
|
|
623
|
+
statusText: err.message,
|
|
624
|
+
url: url.href,
|
|
625
|
+
body: Buffer.from([]),
|
|
626
|
+
finalUrl: url.href,
|
|
627
|
+
config,
|
|
628
|
+
request: fetchOptions
|
|
629
|
+
});
|
|
630
|
+
downloadResult.emit("error", error);
|
|
631
|
+
return error;
|
|
632
|
+
}
|
|
633
|
+
}
|
|
634
|
+
if (uploadResult) {
|
|
635
|
+
const uploadFinishEvent = {
|
|
636
|
+
response: {
|
|
637
|
+
status,
|
|
638
|
+
statusText,
|
|
639
|
+
headers: responseHeaders,
|
|
640
|
+
data: responseData,
|
|
641
|
+
contentType,
|
|
642
|
+
contentLength: bodySize
|
|
643
|
+
},
|
|
644
|
+
finalUrl: url.href,
|
|
645
|
+
cookies,
|
|
646
|
+
urls: buildUrlTree(config, url.href),
|
|
647
|
+
uploadSize: config.transfer.requestSize || 0,
|
|
648
|
+
timing: {
|
|
649
|
+
total: config.timing.durationMs || 0,
|
|
650
|
+
dns: config.timing.dnsMs,
|
|
651
|
+
tcp: config.timing.tcpMs,
|
|
652
|
+
tls: config.timing.tlsMs,
|
|
653
|
+
upload: config.timing.transferMs || 0,
|
|
654
|
+
waiting: config.timing.ttfbMs || 0,
|
|
655
|
+
download: config.timing.transferMs
|
|
656
|
+
},
|
|
657
|
+
averageUploadSpeed: config.timing.transferMs ? (config.transfer.requestSize || 0) / config.timing.transferMs * 1000 : 0,
|
|
658
|
+
averageDownloadSpeed: config.timing.transferMs ? bodySize / config.timing.transferMs * 1000 : 0,
|
|
659
|
+
config: sanitizeConfig(config)
|
|
660
|
+
};
|
|
661
|
+
uploadResult.emit("finish", uploadFinishEvent);
|
|
662
|
+
uploadResult.emit("done", uploadFinishEvent);
|
|
663
|
+
uploadResult._markFinished();
|
|
664
|
+
}
|
|
665
|
+
return finalResponse;
|
|
666
|
+
} catch (error) {
|
|
667
|
+
_stats.statusOnNext = "error";
|
|
668
|
+
const rezoError = buildSmartError(config, fetchOptions, error);
|
|
669
|
+
const eventEmitter = streamResult || downloadResult || uploadResult;
|
|
670
|
+
if (eventEmitter) {
|
|
671
|
+
eventEmitter.emit("error", rezoError);
|
|
672
|
+
}
|
|
673
|
+
return rezoError;
|
|
674
|
+
}
|
|
675
|
+
}
|
|
676
|
+
async function handleStreamingResponse(response, config, timing, streamResult, url, status, statusText, headers, cookies) {
|
|
677
|
+
if (!response.body) {
|
|
678
|
+
streamResult.emit("error", new Error("Response body is not available for streaming"));
|
|
679
|
+
return;
|
|
680
|
+
}
|
|
681
|
+
const reader = response.body.getReader();
|
|
682
|
+
const contentLength = response.headers.get("content-length");
|
|
683
|
+
const totalBytes = contentLength ? parseInt(contentLength, 10) : 0;
|
|
684
|
+
let bytesReceived = 0;
|
|
685
|
+
try {
|
|
686
|
+
while (true) {
|
|
687
|
+
const { done, value } = await reader.read();
|
|
688
|
+
if (done)
|
|
689
|
+
break;
|
|
690
|
+
if (value) {
|
|
691
|
+
bytesReceived += value.length;
|
|
692
|
+
streamResult.emit("data", Buffer.from(value));
|
|
693
|
+
const progressEvent = {
|
|
694
|
+
loaded: bytesReceived,
|
|
695
|
+
total: totalBytes,
|
|
696
|
+
percentage: totalBytes ? Math.round(bytesReceived / totalBytes * 100) : 0,
|
|
697
|
+
speed: 0,
|
|
698
|
+
averageSpeed: 0,
|
|
699
|
+
estimatedTime: 0,
|
|
700
|
+
timestamp: Date.now()
|
|
701
|
+
};
|
|
702
|
+
streamResult.emit("progress", progressEvent);
|
|
703
|
+
streamResult.emit("download-progress", progressEvent);
|
|
704
|
+
}
|
|
705
|
+
}
|
|
706
|
+
config.timing.endTimestamp = Date.now();
|
|
707
|
+
config.timing.durationMs = performance.now() - timing.startTime;
|
|
708
|
+
config.timing.transferMs = timing.firstByteTime ? performance.now() - timing.firstByteTime : config.timing.durationMs;
|
|
709
|
+
config.transfer.bodySize = bytesReceived;
|
|
710
|
+
config.transfer.responseSize = bytesReceived;
|
|
711
|
+
const streamFinishEvent = {
|
|
712
|
+
status,
|
|
713
|
+
statusText,
|
|
714
|
+
headers,
|
|
715
|
+
contentType: response.headers.get("content-type") || undefined,
|
|
716
|
+
contentLength: bytesReceived,
|
|
717
|
+
finalUrl: url.href,
|
|
718
|
+
cookies,
|
|
719
|
+
urls: buildUrlTree(config, url.href),
|
|
720
|
+
timing: {
|
|
721
|
+
total: config.timing.durationMs || 0,
|
|
722
|
+
dns: config.timing.dnsMs,
|
|
723
|
+
tcp: config.timing.tcpMs,
|
|
724
|
+
tls: config.timing.tlsMs,
|
|
725
|
+
firstByte: config.timing.ttfbMs,
|
|
726
|
+
download: config.timing.transferMs
|
|
727
|
+
},
|
|
728
|
+
config: sanitizeConfig(config)
|
|
729
|
+
};
|
|
730
|
+
streamResult.emit("finish", streamFinishEvent);
|
|
731
|
+
streamResult.emit("done", streamFinishEvent);
|
|
732
|
+
streamResult.emit("end");
|
|
733
|
+
streamResult._markFinished();
|
|
734
|
+
} catch (err) {
|
|
735
|
+
streamResult.emit("error", buildSmartError(config, {}, err));
|
|
736
|
+
}
|
|
737
|
+
}
|
|
738
|
+
|
|
739
|
+
exports.Environment = Environment;
|
|
740
|
+
exports.executeRequest = executeRequest;
|