rezo 1.0.5 → 1.0.7

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (49) hide show
  1. package/README.md +352 -9
  2. package/dist/adapters/curl.cjs +796 -0
  3. package/dist/adapters/curl.js +796 -0
  4. package/dist/adapters/entries/curl.d.ts +2407 -20
  5. package/dist/adapters/entries/fetch.d.ts +364 -20
  6. package/dist/adapters/entries/http.d.ts +364 -20
  7. package/dist/adapters/entries/http2.d.ts +364 -20
  8. package/dist/adapters/entries/react-native.d.ts +364 -20
  9. package/dist/adapters/entries/xhr.d.ts +364 -20
  10. package/dist/adapters/index.cjs +6 -6
  11. package/dist/adapters/picker.cjs +2 -2
  12. package/dist/adapters/picker.js +2 -2
  13. package/dist/cache/index.cjs +13 -13
  14. package/dist/core/hooks.cjs +2 -0
  15. package/dist/core/hooks.js +2 -0
  16. package/dist/core/rezo.cjs +2 -2
  17. package/dist/core/rezo.js +2 -2
  18. package/dist/crawler.d.ts +366 -22
  19. package/dist/entries/crawler.cjs +5 -5
  20. package/dist/index.cjs +23 -18
  21. package/dist/index.d.ts +631 -20
  22. package/dist/index.js +1 -0
  23. package/dist/platform/browser.d.ts +364 -20
  24. package/dist/platform/bun.d.ts +364 -20
  25. package/dist/platform/deno.d.ts +364 -20
  26. package/dist/platform/node.d.ts +364 -20
  27. package/dist/platform/react-native.d.ts +364 -20
  28. package/dist/platform/worker.d.ts +364 -20
  29. package/dist/plugin/crawler-options.cjs +1 -1
  30. package/dist/plugin/crawler-options.js +1 -1
  31. package/dist/plugin/crawler.cjs +2 -2
  32. package/dist/plugin/crawler.js +2 -2
  33. package/dist/plugin/index.cjs +36 -36
  34. package/dist/proxy/index.cjs +2 -2
  35. package/dist/proxy/manager.cjs +57 -2
  36. package/dist/proxy/manager.js +57 -2
  37. package/dist/queue/http-queue.cjs +313 -0
  38. package/dist/queue/http-queue.js +312 -0
  39. package/dist/queue/index.cjs +8 -0
  40. package/dist/queue/index.js +6 -0
  41. package/dist/queue/queue.cjs +346 -0
  42. package/dist/queue/queue.js +344 -0
  43. package/dist/queue/types.cjs +17 -0
  44. package/dist/queue/types.js +17 -0
  45. package/dist/types/curl-options.cjs +25 -0
  46. package/dist/types/curl-options.js +25 -0
  47. package/dist/utils/http-config.cjs +0 -15
  48. package/dist/utils/http-config.js +0 -15
  49. package/package.json +1 -2
@@ -0,0 +1,344 @@
1
+ function generateId() {
2
+ return `${Date.now().toString(36)}-${Math.random().toString(36).slice(2, 9)}`;
3
+ }
4
+
5
+ export class RezoQueue {
6
+ queue = [];
7
+ pendingCount = 0;
8
+ isPausedFlag = false;
9
+ intervalId;
10
+ intervalCount = 0;
11
+ intervalStart = 0;
12
+ eventHandlers = new Map;
13
+ statsData = {
14
+ added: 0,
15
+ processed: 0,
16
+ completed: 0,
17
+ failed: 0,
18
+ timedOut: 0,
19
+ cancelled: 0,
20
+ averageDuration: 0,
21
+ throughput: 0
22
+ };
23
+ totalDuration = 0;
24
+ throughputWindow = [];
25
+ throughputWindowSize = 60;
26
+ idlePromise;
27
+ emptyPromise;
28
+ config;
29
+ constructor(config = {}) {
30
+ this.config = {
31
+ concurrency: config.concurrency ?? 1 / 0,
32
+ autoStart: config.autoStart ?? true,
33
+ timeout: config.timeout ?? 0,
34
+ throwOnTimeout: config.throwOnTimeout ?? true,
35
+ interval: config.interval ?? 0,
36
+ intervalCap: config.intervalCap ?? 1 / 0,
37
+ carryoverConcurrencyCount: config.carryoverConcurrencyCount ?? false
38
+ };
39
+ if (!this.config.autoStart) {
40
+ this.isPausedFlag = true;
41
+ }
42
+ if (this.config.interval > 0) {
43
+ this.startInterval();
44
+ }
45
+ }
46
+ get state() {
47
+ return {
48
+ pending: this.pendingCount,
49
+ size: this.queue.length,
50
+ total: this.pendingCount + this.queue.length,
51
+ isPaused: this.isPausedFlag,
52
+ isIdle: this.pendingCount === 0 && this.queue.length === 0
53
+ };
54
+ }
55
+ get stats() {
56
+ return { ...this.statsData };
57
+ }
58
+ get concurrency() {
59
+ return this.config.concurrency;
60
+ }
61
+ set concurrency(value) {
62
+ this.config.concurrency = value;
63
+ this.tryRunNext();
64
+ }
65
+ get pending() {
66
+ return this.pendingCount;
67
+ }
68
+ get size() {
69
+ return this.queue.length;
70
+ }
71
+ get isPaused() {
72
+ return this.isPausedFlag;
73
+ }
74
+ add(fn, options = {}) {
75
+ return new Promise((resolve, reject) => {
76
+ const task = {
77
+ id: options.id ?? generateId(),
78
+ fn,
79
+ priority: options.priority ?? 0,
80
+ timeout: options.timeout ?? this.config.timeout,
81
+ signal: options.signal,
82
+ resolve,
83
+ reject,
84
+ addedAt: Date.now()
85
+ };
86
+ if (options.signal?.aborted) {
87
+ reject(new Error("Task was cancelled before starting"));
88
+ return;
89
+ }
90
+ options.signal?.addEventListener("abort", () => {
91
+ this.cancel(task.id);
92
+ });
93
+ this.insertByPriority(task);
94
+ this.statsData.added++;
95
+ this.emit("add", { id: task.id, priority: task.priority });
96
+ if (this.config.autoStart && !this.isPausedFlag) {
97
+ this.tryRunNext();
98
+ }
99
+ });
100
+ }
101
+ addAll(fns, options = {}) {
102
+ return Promise.all(fns.map((fn, i) => this.add(fn, { ...options, id: options.id ? `${options.id}-${i}` : undefined })));
103
+ }
104
+ pause() {
105
+ if (!this.isPausedFlag) {
106
+ this.isPausedFlag = true;
107
+ this.emit("paused", undefined);
108
+ }
109
+ }
110
+ start() {
111
+ if (this.isPausedFlag) {
112
+ this.isPausedFlag = false;
113
+ this.emit("resumed", undefined);
114
+ this.tryRunNext();
115
+ }
116
+ }
117
+ clear() {
118
+ const tasks = [...this.queue];
119
+ this.queue = [];
120
+ for (const task of tasks) {
121
+ task.reject(new Error("Queue was cleared"));
122
+ this.statsData.cancelled++;
123
+ this.emit("cancelled", { id: task.id });
124
+ }
125
+ this.checkEmpty();
126
+ }
127
+ cancel(id) {
128
+ const index = this.queue.findIndex((t) => t.id === id);
129
+ if (index !== -1) {
130
+ const [task] = this.queue.splice(index, 1);
131
+ task.reject(new Error("Task was cancelled"));
132
+ this.statsData.cancelled++;
133
+ this.emit("cancelled", { id });
134
+ this.checkEmpty();
135
+ return true;
136
+ }
137
+ return false;
138
+ }
139
+ cancelBy(predicate) {
140
+ let count = 0;
141
+ const remaining = [];
142
+ for (const task of this.queue) {
143
+ if (predicate({ id: task.id, priority: task.priority })) {
144
+ task.reject(new Error("Task was cancelled"));
145
+ this.statsData.cancelled++;
146
+ this.emit("cancelled", { id: task.id });
147
+ count++;
148
+ } else {
149
+ remaining.push(task);
150
+ }
151
+ }
152
+ this.queue = remaining;
153
+ if (count > 0) {
154
+ this.checkEmpty();
155
+ }
156
+ return count;
157
+ }
158
+ onIdle() {
159
+ if (this.state.isIdle) {
160
+ return Promise.resolve();
161
+ }
162
+ if (!this.idlePromise) {
163
+ let resolvePromise;
164
+ const promise = new Promise((resolve) => {
165
+ resolvePromise = resolve;
166
+ });
167
+ this.idlePromise = { promise, resolve: resolvePromise };
168
+ }
169
+ return this.idlePromise.promise;
170
+ }
171
+ onEmpty() {
172
+ if (this.queue.length === 0) {
173
+ return Promise.resolve();
174
+ }
175
+ if (!this.emptyPromise) {
176
+ let resolvePromise;
177
+ const promise = new Promise((resolve) => {
178
+ resolvePromise = resolve;
179
+ });
180
+ this.emptyPromise = { promise, resolve: resolvePromise };
181
+ }
182
+ return this.emptyPromise.promise;
183
+ }
184
+ onSizeLessThan(limit) {
185
+ if (this.queue.length < limit) {
186
+ return Promise.resolve();
187
+ }
188
+ return new Promise((resolve) => {
189
+ const check = () => {
190
+ if (this.queue.length < limit) {
191
+ this.off("completed", check);
192
+ this.off("cancelled", check);
193
+ this.off("error", check);
194
+ resolve();
195
+ }
196
+ };
197
+ this.on("completed", check);
198
+ this.on("cancelled", check);
199
+ this.on("error", check);
200
+ });
201
+ }
202
+ on(event, handler) {
203
+ if (!this.eventHandlers.has(event)) {
204
+ this.eventHandlers.set(event, new Set);
205
+ }
206
+ this.eventHandlers.get(event).add(handler);
207
+ }
208
+ off(event, handler) {
209
+ this.eventHandlers.get(event)?.delete(handler);
210
+ }
211
+ destroy() {
212
+ this.clear();
213
+ if (this.intervalId) {
214
+ clearInterval(this.intervalId);
215
+ this.intervalId = undefined;
216
+ }
217
+ this.eventHandlers.clear();
218
+ }
219
+ insertByPriority(task) {
220
+ let insertIndex = this.queue.length;
221
+ for (let i = 0;i < this.queue.length; i++) {
222
+ if (task.priority > this.queue[i].priority) {
223
+ insertIndex = i;
224
+ break;
225
+ }
226
+ }
227
+ this.queue.splice(insertIndex, 0, task);
228
+ }
229
+ tryRunNext() {
230
+ if (this.isPausedFlag)
231
+ return;
232
+ if (this.queue.length === 0)
233
+ return;
234
+ if (this.pendingCount >= this.config.concurrency)
235
+ return;
236
+ if (this.config.interval > 0) {
237
+ if (this.intervalCount >= this.config.intervalCap)
238
+ return;
239
+ this.intervalCount++;
240
+ }
241
+ const task = this.queue.shift();
242
+ this.runTask(task);
243
+ this.tryRunNext();
244
+ }
245
+ async runTask(task) {
246
+ this.pendingCount++;
247
+ this.statsData.processed++;
248
+ const wasIdle = this.pendingCount === 1 && this.queue.length === 0;
249
+ if (wasIdle) {
250
+ this.emit("active", undefined);
251
+ }
252
+ this.emit("start", { id: task.id });
253
+ this.emit("next", undefined);
254
+ const startTime = Date.now();
255
+ let timeoutId;
256
+ try {
257
+ let result;
258
+ if (task.timeout && task.timeout > 0) {
259
+ result = await Promise.race([
260
+ task.fn(),
261
+ new Promise((_, reject) => {
262
+ timeoutId = setTimeout(() => {
263
+ this.statsData.timedOut++;
264
+ this.emit("timeout", { id: task.id });
265
+ if (this.config.throwOnTimeout) {
266
+ reject(new Error(`Task ${task.id} timed out after ${task.timeout}ms`));
267
+ }
268
+ }, task.timeout);
269
+ })
270
+ ]);
271
+ } else {
272
+ result = await task.fn();
273
+ }
274
+ if (timeoutId)
275
+ clearTimeout(timeoutId);
276
+ const duration = Date.now() - startTime;
277
+ this.recordDuration(duration);
278
+ this.statsData.completed++;
279
+ this.emit("completed", { id: task.id, result, duration });
280
+ task.resolve(result);
281
+ } catch (error) {
282
+ if (timeoutId)
283
+ clearTimeout(timeoutId);
284
+ this.statsData.failed++;
285
+ this.emit("error", { id: task.id, error });
286
+ task.reject(error);
287
+ } finally {
288
+ this.pendingCount--;
289
+ this.checkEmpty();
290
+ this.checkIdle();
291
+ this.tryRunNext();
292
+ }
293
+ }
294
+ recordDuration(duration) {
295
+ this.totalDuration += duration;
296
+ this.statsData.averageDuration = this.totalDuration / this.statsData.completed;
297
+ const now = Date.now();
298
+ this.throughputWindow.push(now);
299
+ const windowStart = now - this.throughputWindowSize * 1000;
300
+ this.throughputWindow = this.throughputWindow.filter((t) => t > windowStart);
301
+ this.statsData.throughput = this.throughputWindow.length / this.throughputWindowSize;
302
+ }
303
+ startInterval() {
304
+ this.intervalStart = Date.now();
305
+ this.intervalCount = 0;
306
+ this.intervalId = setInterval(() => {
307
+ if (!this.config.carryoverConcurrencyCount) {
308
+ this.intervalCount = 0;
309
+ } else {
310
+ this.intervalCount = Math.max(0, this.intervalCount - this.config.intervalCap);
311
+ }
312
+ this.intervalStart = Date.now();
313
+ this.tryRunNext();
314
+ }, this.config.interval);
315
+ }
316
+ emit(event, data) {
317
+ const handlers = this.eventHandlers.get(event);
318
+ if (handlers) {
319
+ for (const handler of handlers) {
320
+ try {
321
+ handler(data);
322
+ } catch {}
323
+ }
324
+ }
325
+ }
326
+ checkEmpty() {
327
+ if (this.queue.length === 0) {
328
+ this.emit("empty", undefined);
329
+ if (this.emptyPromise) {
330
+ this.emptyPromise.resolve();
331
+ this.emptyPromise = undefined;
332
+ }
333
+ }
334
+ }
335
+ checkIdle() {
336
+ if (this.state.isIdle) {
337
+ this.emit("idle", undefined);
338
+ if (this.idlePromise) {
339
+ this.idlePromise.resolve();
340
+ this.idlePromise = undefined;
341
+ }
342
+ }
343
+ }
344
+ }
@@ -0,0 +1,17 @@
1
+ const Priority = exports.Priority = {
2
+ LOWEST: 0,
3
+ LOW: 25,
4
+ NORMAL: 50,
5
+ HIGH: 75,
6
+ HIGHEST: 100,
7
+ CRITICAL: 1000
8
+ };
9
+ const HttpMethodPriority = exports.HttpMethodPriority = {
10
+ HEAD: 100,
11
+ GET: 75,
12
+ OPTIONS: 50,
13
+ POST: 50,
14
+ PUT: 50,
15
+ PATCH: 50,
16
+ DELETE: 25
17
+ };
@@ -0,0 +1,17 @@
1
+ export const Priority = {
2
+ LOWEST: 0,
3
+ LOW: 25,
4
+ NORMAL: 50,
5
+ HIGH: 75,
6
+ HIGHEST: 100,
7
+ CRITICAL: 1000
8
+ };
9
+ export const HttpMethodPriority = {
10
+ HEAD: 100,
11
+ GET: 75,
12
+ OPTIONS: 50,
13
+ POST: 50,
14
+ PUT: 50,
15
+ PATCH: 50,
16
+ DELETE: 25
17
+ };
@@ -0,0 +1,25 @@
1
+ const CURL_OPTION_DEFAULTS = exports.CURL_OPTION_DEFAULTS = {
2
+ tcpFastOpen: false,
3
+ tcpNodelay: false,
4
+ noKeepalive: false,
5
+ pathAsIs: false,
6
+ globOff: false,
7
+ certStatus: false,
8
+ noAlpn: false,
9
+ sessionId: true,
10
+ locationTrusted: false,
11
+ junkSessionCookies: false,
12
+ fail: false,
13
+ verbose: false,
14
+ traceTime: false,
15
+ raw: false,
16
+ noCompressed: false,
17
+ noAltSvc: false,
18
+ negotiate: false,
19
+ saslIr: false,
20
+ compressedSsh: false,
21
+ insecure: false,
22
+ maxRedirs: 50,
23
+ happyEyeballsTimeout: 200
24
+ };
25
+ const CURL_OPTIONS_COUNT = exports.CURL_OPTIONS_COUNT = 120;
@@ -0,0 +1,25 @@
1
+ export const CURL_OPTION_DEFAULTS = {
2
+ tcpFastOpen: false,
3
+ tcpNodelay: false,
4
+ noKeepalive: false,
5
+ pathAsIs: false,
6
+ globOff: false,
7
+ certStatus: false,
8
+ noAlpn: false,
9
+ sessionId: true,
10
+ locationTrusted: false,
11
+ junkSessionCookies: false,
12
+ fail: false,
13
+ verbose: false,
14
+ traceTime: false,
15
+ raw: false,
16
+ noCompressed: false,
17
+ noAltSvc: false,
18
+ negotiate: false,
19
+ saslIr: false,
20
+ compressedSsh: false,
21
+ insecure: false,
22
+ maxRedirs: 50,
23
+ happyEyeballsTimeout: 200
24
+ };
25
+ export const CURL_OPTIONS_COUNT = 120;
@@ -175,10 +175,7 @@ function setSignal() {
175
175
  }
176
176
  }
177
177
  async function getDefaultConfig(config = {}, proxyManager) {
178
- const curl = await checkCurl();
179
178
  return {
180
- useCurl: config.curl === true,
181
- isCurl: curl.status || false,
182
179
  baseURL: config.baseURL,
183
180
  headers: config.headers,
184
181
  rejectUnauthorized: config.rejectUnauthorized,
@@ -192,7 +189,6 @@ async function getDefaultConfig(config = {}, proxyManager) {
192
189
  useCookies: config.enableCookieJar,
193
190
  fs: await getFS(),
194
191
  timeout: config.timeout ?? config.requestTimeout,
195
- http2: config.http2 === true,
196
192
  hooks: config.hooks,
197
193
  cookieFile: config.cookieFile,
198
194
  encoding: config.encoding,
@@ -523,8 +519,6 @@ As a workaround, process.env.NODE_TLS_REJECT_UNAUTHORIZED is being set to '0'.
523
519
  headers: requestOptions.headers,
524
520
  maxRedirects,
525
521
  retryAttempts: 0,
526
- http2: requestOptions.http2 === true,
527
- curl: requestOptions.curl === true,
528
522
  timeout: typeof requestOptions.timeout === "number" ? requestOptions.timeout : null,
529
523
  enableCookieJar: typeof defaultOptions.enableCookieJar === "boolean" ? defaultOptions.enableCookieJar : true,
530
524
  useCookies: typeof requestOptions.useCookies === "boolean" ? requestOptions.useCookies : true,
@@ -583,15 +577,6 @@ As a workaround, process.env.NODE_TLS_REJECT_UNAUTHORIZED is being set to '0'.
583
577
  config.httpsAgent = options.httpsAgent;
584
578
  }
585
579
  const isSupportedRuntime = type === "node" || type === "bun" || type === "deno";
586
- if (requestOptions.curl && !defaultOptions.isCurl && debug) {
587
- const recommendations = isSupportedRuntime ? `Recommendations:
588
- ` + `• Only enable 'curl' mode when absolutely necessary
589
- ` + `• Verify curl is installed and up-to-date on your system
590
- ` + `• For enhanced security, consider using native Rezo HTTP clients instead` : "";
591
- console.warn(`⚠️ [WARNING]: ${isSupportedRuntime ? `Curl binary not detected in your environment while 'curl' mode is enabled` : `'curl' mode is not supported in '${type}' environment`}
592
-
593
- ` + recommendations);
594
- }
595
580
  if (saveTo) {
596
581
  if (!isSupportedRuntime) {
597
582
  throw new Error(`You can only use this feature in Node.js, Deno or Bun and not available in Edge or Browser.`);
@@ -175,10 +175,7 @@ function setSignal() {
175
175
  }
176
176
  }
177
177
  export async function getDefaultConfig(config = {}, proxyManager) {
178
- const curl = await checkCurl();
179
178
  return {
180
- useCurl: config.curl === true,
181
- isCurl: curl.status || false,
182
179
  baseURL: config.baseURL,
183
180
  headers: config.headers,
184
181
  rejectUnauthorized: config.rejectUnauthorized,
@@ -192,7 +189,6 @@ export async function getDefaultConfig(config = {}, proxyManager) {
192
189
  useCookies: config.enableCookieJar,
193
190
  fs: await getFS(),
194
191
  timeout: config.timeout ?? config.requestTimeout,
195
- http2: config.http2 === true,
196
192
  hooks: config.hooks,
197
193
  cookieFile: config.cookieFile,
198
194
  encoding: config.encoding,
@@ -523,8 +519,6 @@ As a workaround, process.env.NODE_TLS_REJECT_UNAUTHORIZED is being set to '0'.
523
519
  headers: requestOptions.headers,
524
520
  maxRedirects,
525
521
  retryAttempts: 0,
526
- http2: requestOptions.http2 === true,
527
- curl: requestOptions.curl === true,
528
522
  timeout: typeof requestOptions.timeout === "number" ? requestOptions.timeout : null,
529
523
  enableCookieJar: typeof defaultOptions.enableCookieJar === "boolean" ? defaultOptions.enableCookieJar : true,
530
524
  useCookies: typeof requestOptions.useCookies === "boolean" ? requestOptions.useCookies : true,
@@ -583,15 +577,6 @@ As a workaround, process.env.NODE_TLS_REJECT_UNAUTHORIZED is being set to '0'.
583
577
  config.httpsAgent = options.httpsAgent;
584
578
  }
585
579
  const isSupportedRuntime = type === "node" || type === "bun" || type === "deno";
586
- if (requestOptions.curl && !defaultOptions.isCurl && debug) {
587
- const recommendations = isSupportedRuntime ? `Recommendations:
588
- ` + `• Only enable 'curl' mode when absolutely necessary
589
- ` + `• Verify curl is installed and up-to-date on your system
590
- ` + `• For enhanced security, consider using native Rezo HTTP clients instead` : "";
591
- console.warn(`⚠️ [WARNING]: ${isSupportedRuntime ? `Curl binary not detected in your environment while 'curl' mode is enabled` : `'curl' mode is not supported in '${type}' environment`}
592
-
593
- ` + recommendations);
594
- }
595
580
  if (saveTo) {
596
581
  if (!isSupportedRuntime) {
597
582
  throw new Error(`You can only use this feature in Node.js, Deno or Bun and not available in Edge or Browser.`);
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "rezo",
3
- "version": "1.0.5",
3
+ "version": "1.0.7",
4
4
  "description": "Lightning-fast, enterprise-grade HTTP client for modern JavaScript. Full HTTP/2 support, intelligent cookie management, multiple adapters (HTTP, Fetch, cURL, XHR), streaming, proxy support (HTTP/HTTPS/SOCKS), and cross-environment compatibility.",
5
5
  "main": "dist/index.cjs",
6
6
  "module": "dist/index.js",
@@ -66,7 +66,6 @@
66
66
  "http-proxy-agent": "^7.0.2",
67
67
  "https-proxy-agent": "^7.0.6",
68
68
  "socks-proxy-agent": "^8.0.5",
69
- "p-queue": "^8.1.0",
70
69
  "linkedom": "^0.18.11",
71
70
  "agent-base": "^7.1.4"
72
71
  },