rezo 1.0.42 → 1.0.44

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (89) hide show
  1. package/dist/adapters/curl.cjs +131 -29
  2. package/dist/adapters/curl.js +131 -29
  3. package/dist/adapters/entries/curl.d.ts +65 -0
  4. package/dist/adapters/entries/fetch.d.ts +65 -0
  5. package/dist/adapters/entries/http.d.ts +65 -0
  6. package/dist/adapters/entries/http2.d.ts +65 -0
  7. package/dist/adapters/entries/react-native.d.ts +65 -0
  8. package/dist/adapters/entries/xhr.d.ts +65 -0
  9. package/dist/adapters/http2.cjs +209 -22
  10. package/dist/adapters/http2.js +209 -22
  11. package/dist/adapters/index.cjs +6 -6
  12. package/dist/cache/index.cjs +9 -13
  13. package/dist/cache/index.js +0 -2
  14. package/dist/core/rezo.cjs +7 -0
  15. package/dist/core/rezo.js +7 -0
  16. package/dist/crawler/addon/decodo/index.cjs +1 -0
  17. package/dist/crawler/addon/decodo/index.js +1 -0
  18. package/dist/crawler/crawler-options.cjs +1 -0
  19. package/dist/crawler/crawler-options.js +1 -0
  20. package/dist/crawler/crawler.cjs +1070 -0
  21. package/dist/crawler/crawler.js +1068 -0
  22. package/dist/crawler/index.cjs +40 -0
  23. package/dist/{plugin → crawler}/index.js +4 -2
  24. package/dist/crawler/plugin/file-cacher.cjs +19 -0
  25. package/dist/crawler/plugin/file-cacher.js +19 -0
  26. package/dist/crawler/plugin/index.cjs +1 -0
  27. package/dist/crawler/plugin/index.js +1 -0
  28. package/dist/crawler/plugin/navigation-history.cjs +43 -0
  29. package/dist/crawler/plugin/navigation-history.js +43 -0
  30. package/dist/crawler/plugin/robots-txt.cjs +2 -0
  31. package/dist/crawler/plugin/robots-txt.js +2 -0
  32. package/dist/crawler/plugin/url-store.cjs +18 -0
  33. package/dist/crawler/plugin/url-store.js +18 -0
  34. package/dist/crawler.d.ts +511 -183
  35. package/dist/entries/crawler.cjs +5 -5
  36. package/dist/entries/crawler.js +2 -2
  37. package/dist/index.cjs +27 -24
  38. package/dist/index.d.ts +73 -0
  39. package/dist/index.js +1 -0
  40. package/dist/internal/agents/base.cjs +113 -0
  41. package/dist/internal/agents/base.js +110 -0
  42. package/dist/internal/agents/http-proxy.cjs +89 -0
  43. package/dist/internal/agents/http-proxy.js +86 -0
  44. package/dist/internal/agents/https-proxy.cjs +176 -0
  45. package/dist/internal/agents/https-proxy.js +173 -0
  46. package/dist/internal/agents/index.cjs +10 -0
  47. package/dist/internal/agents/index.js +5 -0
  48. package/dist/internal/agents/socks-client.cjs +571 -0
  49. package/dist/internal/agents/socks-client.js +567 -0
  50. package/dist/internal/agents/socks-proxy.cjs +75 -0
  51. package/dist/internal/agents/socks-proxy.js +72 -0
  52. package/dist/platform/browser.d.ts +65 -0
  53. package/dist/platform/bun.d.ts +65 -0
  54. package/dist/platform/deno.d.ts +65 -0
  55. package/dist/platform/node.d.ts +65 -0
  56. package/dist/platform/react-native.d.ts +65 -0
  57. package/dist/platform/worker.d.ts +65 -0
  58. package/dist/proxy/index.cjs +18 -16
  59. package/dist/proxy/index.js +17 -12
  60. package/dist/queue/index.cjs +8 -8
  61. package/dist/responses/buildError.cjs +11 -2
  62. package/dist/responses/buildError.js +11 -2
  63. package/dist/responses/universal/index.cjs +11 -11
  64. package/dist/utils/curl.cjs +317 -0
  65. package/dist/utils/curl.js +314 -0
  66. package/package.json +2 -6
  67. package/dist/cache/file-cacher.cjs +0 -264
  68. package/dist/cache/file-cacher.js +0 -261
  69. package/dist/cache/url-store.cjs +0 -288
  70. package/dist/cache/url-store.js +0 -285
  71. package/dist/plugin/addon/decodo/index.cjs +0 -1
  72. package/dist/plugin/addon/decodo/index.js +0 -1
  73. package/dist/plugin/crawler-options.cjs +0 -1
  74. package/dist/plugin/crawler-options.js +0 -1
  75. package/dist/plugin/crawler.cjs +0 -519
  76. package/dist/plugin/crawler.js +0 -517
  77. package/dist/plugin/index.cjs +0 -36
  78. /package/dist/{plugin → crawler}/addon/decodo/options.cjs +0 -0
  79. /package/dist/{plugin → crawler}/addon/decodo/options.js +0 -0
  80. /package/dist/{plugin → crawler}/addon/decodo/types.cjs +0 -0
  81. /package/dist/{plugin → crawler}/addon/decodo/types.js +0 -0
  82. /package/dist/{plugin → crawler}/addon/oxylabs/index.cjs +0 -0
  83. /package/dist/{plugin → crawler}/addon/oxylabs/index.js +0 -0
  84. /package/dist/{plugin → crawler}/addon/oxylabs/options.cjs +0 -0
  85. /package/dist/{plugin → crawler}/addon/oxylabs/options.js +0 -0
  86. /package/dist/{plugin → crawler}/addon/oxylabs/types.cjs +0 -0
  87. /package/dist/{plugin → crawler}/addon/oxylabs/types.js +0 -0
  88. /package/dist/{plugin → crawler}/scraper.cjs +0 -0
  89. /package/dist/{plugin → crawler}/scraper.js +0 -0
@@ -146,14 +146,19 @@ function isTlsError(code, message) {
146
146
  "SELF_SIGNED_CERT_IN_CHAIN",
147
147
  "DEPTH_ZERO_SELF_SIGNED_CERT",
148
148
  "ERR_TLS_RENEGOTIATION_DISABLED",
149
- "ERR_TLS_CERT_SIGNATURE_ALGORITHM_UNSUPPORTED"
149
+ "ERR_TLS_CERT_SIGNATURE_ALGORITHM_UNSUPPORTED",
150
+ "ERR_SSL_TLSV1_ALERT_NO_APPLICATION_PROTOCOL"
150
151
  ];
151
152
  if (tlsCodes.includes(code))
152
153
  return true;
154
+ if (code.startsWith("ERR_SSL_"))
155
+ return true;
153
156
  const msg = message.toLowerCase();
154
- return msg.includes("certificate") || msg.includes("ssl") || msg.includes("tls") || msg.includes("handshake");
157
+ return msg.includes("certificate") || msg.includes("ssl") || msg.includes("tls") || msg.includes("handshake") || msg.includes("alpn");
155
158
  }
156
159
  function mapTlsCode(code, message) {
160
+ if (code && code.startsWith("ERR_SSL_"))
161
+ return code;
157
162
  if (code && code.startsWith("ERR_TLS") || code.startsWith("CERT_") || code === "EPROTO" || code.startsWith("UNABLE_TO") || code.startsWith("SELF_SIGNED") || code.startsWith("DEPTH_ZERO")) {
158
163
  return code;
159
164
  }
@@ -166,6 +171,10 @@ function mapTlsCode(code, message) {
166
171
  return "ERR_TLS_CERT_ALTNAME_INVALID";
167
172
  if (msg.includes("handshake") && msg.includes("timeout"))
168
173
  return "ERR_TLS_HANDSHAKE_TIMEOUT";
174
+ if (msg.includes("no application protocol") || msg.includes("alpn"))
175
+ return "ERR_SSL_NO_ALPN";
176
+ if (msg.includes("does not support http/2"))
177
+ return "ERR_HTTP2_NOT_SUPPORTED";
169
178
  return "EPROTO";
170
179
  }
171
180
  function isTimeoutError(code, message) {
@@ -146,14 +146,19 @@ function isTlsError(code, message) {
146
146
  "SELF_SIGNED_CERT_IN_CHAIN",
147
147
  "DEPTH_ZERO_SELF_SIGNED_CERT",
148
148
  "ERR_TLS_RENEGOTIATION_DISABLED",
149
- "ERR_TLS_CERT_SIGNATURE_ALGORITHM_UNSUPPORTED"
149
+ "ERR_TLS_CERT_SIGNATURE_ALGORITHM_UNSUPPORTED",
150
+ "ERR_SSL_TLSV1_ALERT_NO_APPLICATION_PROTOCOL"
150
151
  ];
151
152
  if (tlsCodes.includes(code))
152
153
  return true;
154
+ if (code.startsWith("ERR_SSL_"))
155
+ return true;
153
156
  const msg = message.toLowerCase();
154
- return msg.includes("certificate") || msg.includes("ssl") || msg.includes("tls") || msg.includes("handshake");
157
+ return msg.includes("certificate") || msg.includes("ssl") || msg.includes("tls") || msg.includes("handshake") || msg.includes("alpn");
155
158
  }
156
159
  function mapTlsCode(code, message) {
160
+ if (code && code.startsWith("ERR_SSL_"))
161
+ return code;
157
162
  if (code && code.startsWith("ERR_TLS") || code.startsWith("CERT_") || code === "EPROTO" || code.startsWith("UNABLE_TO") || code.startsWith("SELF_SIGNED") || code.startsWith("DEPTH_ZERO")) {
158
163
  return code;
159
164
  }
@@ -166,6 +171,10 @@ function mapTlsCode(code, message) {
166
171
  return "ERR_TLS_CERT_ALTNAME_INVALID";
167
172
  if (msg.includes("handshake") && msg.includes("timeout"))
168
173
  return "ERR_TLS_HANDSHAKE_TIMEOUT";
174
+ if (msg.includes("no application protocol") || msg.includes("alpn"))
175
+ return "ERR_SSL_NO_ALPN";
176
+ if (msg.includes("does not support http/2"))
177
+ return "ERR_HTTP2_NOT_SUPPORTED";
169
178
  return "EPROTO";
170
179
  }
171
180
  function isTimeoutError(code, message) {
@@ -1,11 +1,11 @@
1
- const _mod_m9gx68 = require('./event-emitter.cjs');
2
- exports.UniversalEventEmitter = _mod_m9gx68.UniversalEventEmitter;;
3
- const _mod_7gd4j6 = require('./stream.cjs');
4
- exports.UniversalStreamResponse = _mod_7gd4j6.UniversalStreamResponse;
5
- exports.StreamResponse = _mod_7gd4j6.StreamResponse;;
6
- const _mod_45jy3g = require('./download.cjs');
7
- exports.UniversalDownloadResponse = _mod_45jy3g.UniversalDownloadResponse;
8
- exports.DownloadResponse = _mod_45jy3g.DownloadResponse;;
9
- const _mod_ibw1bv = require('./upload.cjs');
10
- exports.UniversalUploadResponse = _mod_ibw1bv.UniversalUploadResponse;
11
- exports.UploadResponse = _mod_ibw1bv.UploadResponse;;
1
+ const _mod_gfdadc = require('./event-emitter.cjs');
2
+ exports.UniversalEventEmitter = _mod_gfdadc.UniversalEventEmitter;;
3
+ const _mod_9z6wit = require('./stream.cjs');
4
+ exports.UniversalStreamResponse = _mod_9z6wit.UniversalStreamResponse;
5
+ exports.StreamResponse = _mod_9z6wit.StreamResponse;;
6
+ const _mod_ybfqy6 = require('./download.cjs');
7
+ exports.UniversalDownloadResponse = _mod_ybfqy6.UniversalDownloadResponse;
8
+ exports.DownloadResponse = _mod_ybfqy6.DownloadResponse;;
9
+ const _mod_48lipr = require('./upload.cjs');
10
+ exports.UniversalUploadResponse = _mod_48lipr.UniversalUploadResponse;
11
+ exports.UploadResponse = _mod_48lipr.UploadResponse;;
@@ -0,0 +1,317 @@
1
+ const { RezoFormData } = require('./form-data.cjs');
2
+ function toCurl(config) {
3
+ const parts = ["curl"];
4
+ const method = (config.method || "GET").toUpperCase();
5
+ if (method !== "GET") {
6
+ parts.push("-X", method);
7
+ }
8
+ const headers = config.headers || {};
9
+ const normalizedHeaders = {};
10
+ if (headers && typeof headers === "object") {
11
+ const headersAny = headers;
12
+ if (typeof headersAny.entries === "function") {
13
+ for (const [key, value] of headersAny.entries()) {
14
+ normalizedHeaders[key.toLowerCase()] = value;
15
+ }
16
+ } else if (typeof headersAny.forEach === "function") {
17
+ headersAny.forEach((value, key) => {
18
+ normalizedHeaders[key.toLowerCase()] = value;
19
+ });
20
+ } else {
21
+ for (const [key, value] of Object.entries(headers)) {
22
+ if (value !== undefined && value !== null) {
23
+ normalizedHeaders[key.toLowerCase()] = String(value);
24
+ }
25
+ }
26
+ }
27
+ }
28
+ for (const [key, value] of Object.entries(normalizedHeaders)) {
29
+ parts.push("-H", escapeShellArg(`${key}: ${value}`));
30
+ }
31
+ const body = config.body ?? config.data;
32
+ if (body !== undefined && body !== null) {
33
+ let dataString;
34
+ if (typeof body === "string") {
35
+ dataString = body;
36
+ } else if (body instanceof RezoFormData) {
37
+ const entries = [];
38
+ const formData = body.toNativeFormData();
39
+ formData.forEach((value, key) => {
40
+ entries.push(`${encodeURIComponent(key)}=${encodeURIComponent(String(value))}`);
41
+ });
42
+ dataString = entries.join("&");
43
+ parts.push("--data-urlencode", escapeShellArg(dataString));
44
+ } else if (typeof body === "object") {
45
+ const contentType = normalizedHeaders["content-type"] || "";
46
+ if (contentType.includes("application/x-www-form-urlencoded")) {
47
+ const params = new URLSearchParams;
48
+ for (const [key, value] of Object.entries(body)) {
49
+ params.append(key, String(value));
50
+ }
51
+ dataString = params.toString();
52
+ } else {
53
+ dataString = JSON.stringify(body);
54
+ if (!normalizedHeaders["content-type"]) {
55
+ parts.push("-H", escapeShellArg("content-type: application/json"));
56
+ }
57
+ }
58
+ } else {
59
+ dataString = String(body);
60
+ }
61
+ if (!(body instanceof RezoFormData)) {
62
+ if (method === "GET" || method === "HEAD") {
63
+ parts.push("-G", "--data-raw", escapeShellArg(dataString));
64
+ } else {
65
+ parts.push("--data-raw", escapeShellArg(dataString));
66
+ }
67
+ }
68
+ }
69
+ if (config.timeout) {
70
+ const timeoutSeconds = Math.ceil(config.timeout / 1000);
71
+ parts.push("--max-time", String(timeoutSeconds));
72
+ }
73
+ if (config.maxRedirects !== undefined) {
74
+ if (config.maxRedirects === 0) {
75
+ parts.push("--no-location");
76
+ } else {
77
+ parts.push("-L", "--max-redirs", String(config.maxRedirects));
78
+ }
79
+ } else {
80
+ parts.push("-L");
81
+ }
82
+ if (config.auth) {
83
+ const auth = `${config.auth.username}:${config.auth.password}`;
84
+ parts.push("-u", escapeShellArg(auth));
85
+ }
86
+ if (config.proxy) {
87
+ let proxyUrl;
88
+ if (typeof config.proxy === "string") {
89
+ proxyUrl = config.proxy;
90
+ } else if (config.proxy && typeof config.proxy === "object" && "host" in config.proxy) {
91
+ const p = config.proxy;
92
+ const protocol = p.protocol || "http";
93
+ const authStr = p.auth ? `${p.auth.username}:${p.auth.password}@` : "";
94
+ proxyUrl = `${protocol}://${authStr}${p.host}:${p.port}`;
95
+ } else {
96
+ proxyUrl = String(config.proxy);
97
+ }
98
+ if (proxyUrl.startsWith("socks")) {
99
+ parts.push("--socks5", escapeShellArg(proxyUrl.replace(/^socks[45]a?:\/\//, "")));
100
+ } else {
101
+ parts.push("-x", escapeShellArg(proxyUrl));
102
+ }
103
+ }
104
+ if (config.rejectUnauthorized === false || config.insecure) {
105
+ parts.push("-k");
106
+ }
107
+ parts.push("--compressed");
108
+ const url = config.fullUrl || config.url || "";
109
+ parts.push(escapeShellArg(String(url)));
110
+ return parts.join(" ");
111
+ }
112
+ function fromCurl(curlCommand) {
113
+ const trimmed = curlCommand.trim();
114
+ if (!trimmed.toLowerCase().startsWith("curl ")) {
115
+ throw new Error('Invalid curl command: must start with "curl "');
116
+ }
117
+ const tokens = tokenize(trimmed.slice(5));
118
+ const result = {
119
+ url: "",
120
+ method: "GET",
121
+ headers: {}
122
+ };
123
+ let i = 0;
124
+ while (i < tokens.length) {
125
+ const token = tokens[i];
126
+ if (token === "-X" || token === "--request") {
127
+ result.method = tokens[++i]?.toUpperCase() || "GET";
128
+ } else if (token === "-H" || token === "--header") {
129
+ const header = tokens[++i];
130
+ if (header) {
131
+ const colonIndex = header.indexOf(":");
132
+ if (colonIndex > 0) {
133
+ const key = header.slice(0, colonIndex).trim().toLowerCase();
134
+ const value = header.slice(colonIndex + 1).trim();
135
+ result.headers[key] = value;
136
+ }
137
+ }
138
+ } else if (token === "-d" || token === "--data" || token === "--data-raw" || token === "--data-binary") {
139
+ const data = tokens[++i];
140
+ if (data) {
141
+ if (result.method === "GET") {
142
+ result.method = "POST";
143
+ }
144
+ if (data.startsWith("{") || data.startsWith("[")) {
145
+ try {
146
+ result.body = JSON.parse(data);
147
+ } catch {
148
+ result.body = data;
149
+ }
150
+ } else {
151
+ result.body = data;
152
+ }
153
+ }
154
+ } else if (token === "--data-urlencode") {
155
+ const data = tokens[++i];
156
+ if (data) {
157
+ if (result.method === "GET") {
158
+ result.method = "POST";
159
+ }
160
+ result.body = data;
161
+ if (!result.headers["content-type"]) {
162
+ result.headers["content-type"] = "application/x-www-form-urlencoded";
163
+ }
164
+ }
165
+ } else if (token === "-G" || token === "--get") {
166
+ result.method = "GET";
167
+ } else if (token === "-b" || token === "--cookie") {
168
+ result.cookies = tokens[++i];
169
+ } else if (token === "-u" || token === "--user") {
170
+ const auth = tokens[++i];
171
+ if (auth) {
172
+ const colonIndex = auth.indexOf(":");
173
+ if (colonIndex > 0) {
174
+ result.auth = {
175
+ username: auth.slice(0, colonIndex),
176
+ password: auth.slice(colonIndex + 1)
177
+ };
178
+ } else {
179
+ result.auth = { username: auth, password: "" };
180
+ }
181
+ }
182
+ } else if (token === "-x" || token === "--proxy") {
183
+ result.proxy = tokens[++i];
184
+ } else if (token === "--socks5" || token === "--socks5-hostname" || token === "--socks4" || token === "--socks4a") {
185
+ const proxyHost = tokens[++i];
186
+ const protocol = token.replace("--", "").replace("-hostname", "");
187
+ result.proxy = `${protocol}://${proxyHost}`;
188
+ } else if (token === "-A" || token === "--user-agent") {
189
+ result.userAgent = tokens[++i];
190
+ if (result.userAgent) {
191
+ result.headers["user-agent"] = result.userAgent;
192
+ }
193
+ } else if (token === "-L" || token === "--location") {
194
+ result.followRedirects = true;
195
+ } else if (token === "--no-location") {
196
+ result.followRedirects = false;
197
+ } else if (token === "--max-redirs") {
198
+ result.maxRedirects = parseInt(tokens[++i] || "0", 10);
199
+ } else if (token === "--max-time" || token === "-m") {
200
+ result.timeout = parseInt(tokens[++i] || "0", 10) * 1000;
201
+ } else if (token === "--connect-timeout") {
202
+ i++;
203
+ } else if (token === "-k" || token === "--insecure") {
204
+ result.insecure = true;
205
+ } else if (token === "--compressed") {
206
+ result.compressed = true;
207
+ } else if (token === "-s" || token === "--silent" || token === "-S" || token === "--show-error") {} else if (token === "-o" || token === "--output") {
208
+ i++;
209
+ } else if (token === "-I" || token === "--head") {
210
+ result.method = "HEAD";
211
+ } else if (!token.startsWith("-")) {
212
+ if (!result.url) {
213
+ result.url = token;
214
+ }
215
+ }
216
+ i++;
217
+ }
218
+ const config = {
219
+ url: result.url,
220
+ method: result.method,
221
+ headers: result.headers
222
+ };
223
+ if (result.body) {
224
+ config.body = result.body;
225
+ }
226
+ if (result.auth) {
227
+ config.auth = result.auth;
228
+ }
229
+ if (result.proxy) {
230
+ config.proxy = result.proxy;
231
+ }
232
+ if (result.timeout) {
233
+ config.timeout = result.timeout;
234
+ }
235
+ if (result.maxRedirects !== undefined) {
236
+ config.maxRedirects = result.maxRedirects;
237
+ }
238
+ if (result.insecure) {
239
+ config.rejectUnauthorized = false;
240
+ }
241
+ if (result.cookies) {
242
+ config.headers = config.headers || {};
243
+ config.headers["cookie"] = result.cookies;
244
+ }
245
+ return config;
246
+ }
247
+ function escapeShellArg(arg) {
248
+ if (arg === "")
249
+ return "''";
250
+ if (!/[^a-zA-Z0-9_\-=:/.@]/.test(arg)) {
251
+ return arg;
252
+ }
253
+ return `'${arg.replace(/'/g, "'\\''")}'`;
254
+ }
255
+ function tokenize(input) {
256
+ const tokens = [];
257
+ let current = "";
258
+ let inSingleQuote = false;
259
+ let inDoubleQuote = false;
260
+ let escaped = false;
261
+ let i = 0;
262
+ while (i < input.length) {
263
+ const char = input[i];
264
+ if (escaped) {
265
+ current += char;
266
+ escaped = false;
267
+ i++;
268
+ continue;
269
+ }
270
+ if (char === "\\" && !inSingleQuote) {
271
+ if (inDoubleQuote) {
272
+ const nextChar = input[i + 1];
273
+ if (nextChar === '"' || nextChar === "\\" || nextChar === "$" || nextChar === "`") {
274
+ escaped = true;
275
+ i++;
276
+ continue;
277
+ }
278
+ } else {
279
+ escaped = true;
280
+ i++;
281
+ continue;
282
+ }
283
+ }
284
+ if (char === "'" && !inDoubleQuote) {
285
+ inSingleQuote = !inSingleQuote;
286
+ i++;
287
+ continue;
288
+ }
289
+ if (char === '"' && !inSingleQuote) {
290
+ inDoubleQuote = !inDoubleQuote;
291
+ i++;
292
+ continue;
293
+ }
294
+ if ((char === " " || char === "\t" || char === `
295
+ `) && !inSingleQuote && !inDoubleQuote) {
296
+ if (current) {
297
+ tokens.push(current);
298
+ current = "";
299
+ }
300
+ while (i + 1 < input.length && (input[i + 1] === " " || input[i + 1] === "\t" || input[i + 1] === `
301
+ `)) {
302
+ i++;
303
+ }
304
+ i++;
305
+ continue;
306
+ }
307
+ current += char;
308
+ i++;
309
+ }
310
+ if (current) {
311
+ tokens.push(current);
312
+ }
313
+ return tokens;
314
+ }
315
+
316
+ exports.toCurl = toCurl;
317
+ exports.fromCurl = fromCurl;