rezo 1.0.5 → 1.0.7

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (49) hide show
  1. package/README.md +352 -9
  2. package/dist/adapters/curl.cjs +796 -0
  3. package/dist/adapters/curl.js +796 -0
  4. package/dist/adapters/entries/curl.d.ts +2407 -20
  5. package/dist/adapters/entries/fetch.d.ts +364 -20
  6. package/dist/adapters/entries/http.d.ts +364 -20
  7. package/dist/adapters/entries/http2.d.ts +364 -20
  8. package/dist/adapters/entries/react-native.d.ts +364 -20
  9. package/dist/adapters/entries/xhr.d.ts +364 -20
  10. package/dist/adapters/index.cjs +6 -6
  11. package/dist/adapters/picker.cjs +2 -2
  12. package/dist/adapters/picker.js +2 -2
  13. package/dist/cache/index.cjs +13 -13
  14. package/dist/core/hooks.cjs +2 -0
  15. package/dist/core/hooks.js +2 -0
  16. package/dist/core/rezo.cjs +2 -2
  17. package/dist/core/rezo.js +2 -2
  18. package/dist/crawler.d.ts +366 -22
  19. package/dist/entries/crawler.cjs +5 -5
  20. package/dist/index.cjs +23 -18
  21. package/dist/index.d.ts +631 -20
  22. package/dist/index.js +1 -0
  23. package/dist/platform/browser.d.ts +364 -20
  24. package/dist/platform/bun.d.ts +364 -20
  25. package/dist/platform/deno.d.ts +364 -20
  26. package/dist/platform/node.d.ts +364 -20
  27. package/dist/platform/react-native.d.ts +364 -20
  28. package/dist/platform/worker.d.ts +364 -20
  29. package/dist/plugin/crawler-options.cjs +1 -1
  30. package/dist/plugin/crawler-options.js +1 -1
  31. package/dist/plugin/crawler.cjs +2 -2
  32. package/dist/plugin/crawler.js +2 -2
  33. package/dist/plugin/index.cjs +36 -36
  34. package/dist/proxy/index.cjs +2 -2
  35. package/dist/proxy/manager.cjs +57 -2
  36. package/dist/proxy/manager.js +57 -2
  37. package/dist/queue/http-queue.cjs +313 -0
  38. package/dist/queue/http-queue.js +312 -0
  39. package/dist/queue/index.cjs +8 -0
  40. package/dist/queue/index.js +6 -0
  41. package/dist/queue/queue.cjs +346 -0
  42. package/dist/queue/queue.js +344 -0
  43. package/dist/queue/types.cjs +17 -0
  44. package/dist/queue/types.js +17 -0
  45. package/dist/types/curl-options.cjs +25 -0
  46. package/dist/types/curl-options.js +25 -0
  47. package/dist/utils/http-config.cjs +0 -15
  48. package/dist/utils/http-config.js +0 -15
  49. package/package.json +1 -2
@@ -6,9 +6,6 @@ import { Agent as HttpsAgent } from 'node:https';
6
6
  import { Socket } from 'node:net';
7
7
  import { Readable, Writable, WritableOptions } from 'node:stream';
8
8
  import { SecureContext, TLSSocket } from 'node:tls';
9
- import PQueue from 'p-queue';
10
- import { Options as Options$1, QueueAddOptions } from 'p-queue';
11
- import PriorityQueue from 'p-queue/dist/priority-queue';
12
9
  import { Cookie as TouchCookie, CookieJar as TouchCookieJar, CreateCookieOptions } from 'tough-cookie';
13
10
 
14
11
  export interface RezoHttpHeaders {
@@ -972,8 +969,17 @@ export interface ProxyCooldownConfig {
972
969
  * Complete configuration for proxy rotation, filtering, and failure handling
973
970
  */
974
971
  export interface ProxyManagerBaseConfig {
975
- /** Array of proxies to manage */
976
- proxies: ProxyInfo[];
972
+ /**
973
+ * Array of proxies to manage
974
+ * Accepts ProxyInfo objects or proxy URL strings (parsed via parseProxyString)
975
+ * @example
976
+ * proxies: [
977
+ * { protocol: 'http', host: '127.0.0.1', port: 8080 },
978
+ * 'socks5://user:pass@proxy.example.com:1080',
979
+ * 'http://proxy.example.com:3128'
980
+ * ]
981
+ */
982
+ proxies: (ProxyInfo | string)[];
977
983
  /**
978
984
  * Whitelist patterns for URLs that should use proxy
979
985
  * - String: exact domain match (e.g., 'api.example.com') or subdomain match (e.g., 'example.com' matches '*.example.com')
@@ -1185,6 +1191,35 @@ export interface AfterProxyEnableContext {
1185
1191
  /** Reason for enabling */
1186
1192
  reason: "cooldown-expired" | "manual";
1187
1193
  }
1194
+ /**
1195
+ * Context for onNoProxiesAvailable hook
1196
+ * Triggered when no proxies are available and an error would be thrown
1197
+ */
1198
+ export interface OnNoProxiesAvailableContext {
1199
+ /** Request URL that needed a proxy */
1200
+ url: string;
1201
+ /** The error that will be thrown */
1202
+ error: Error;
1203
+ /** All proxies (including disabled ones) */
1204
+ allProxies: ProxyState[];
1205
+ /** Number of active proxies (should be 0) */
1206
+ activeCount: number;
1207
+ /** Number of disabled proxies */
1208
+ disabledCount: number;
1209
+ /** Number of proxies in cooldown */
1210
+ cooldownCount: number;
1211
+ /** Reasons why proxies are unavailable */
1212
+ disabledReasons: {
1213
+ /** Proxies disabled due to failures */
1214
+ dead: number;
1215
+ /** Proxies disabled due to request limit */
1216
+ limitReached: number;
1217
+ /** Proxies manually disabled */
1218
+ manual: number;
1219
+ };
1220
+ /** Timestamp when this event occurred */
1221
+ timestamp: number;
1222
+ }
1188
1223
  /**
1189
1224
  * Context provided to beforeRequest hook
1190
1225
  * Contains metadata about the current request state
@@ -1503,6 +1538,16 @@ export type AfterProxyRotateHook = (context: AfterProxyRotateContext) => void |
1503
1538
  * Use for notifications, logging
1504
1539
  */
1505
1540
  export type AfterProxyEnableHook = (context: AfterProxyEnableContext) => void | Promise<void>;
1541
+ /**
1542
+ * Hook called when no proxies are available and an error is about to be thrown
1543
+ * Use for alerting, logging exhausted proxy pools, or triggering proxy refresh
1544
+ * This hook is called just before the error is thrown, allowing you to:
1545
+ * - Log the exhaustion event for monitoring
1546
+ * - Trigger external proxy pool refresh
1547
+ * - Send alerts to monitoring systems
1548
+ * - Record statistics about proxy pool health
1549
+ */
1550
+ export type OnNoProxiesAvailableHook = (context: OnNoProxiesAvailableContext) => void | Promise<void>;
1506
1551
  /**
1507
1552
  * Collection of all hook types
1508
1553
  * All hooks are arrays to allow multiple handlers
@@ -1527,6 +1572,7 @@ export interface RezoHooks {
1527
1572
  afterProxyDisable: AfterProxyDisableHook[];
1528
1573
  afterProxyRotate: AfterProxyRotateHook[];
1529
1574
  afterProxyEnable: AfterProxyEnableHook[];
1575
+ onNoProxiesAvailable: OnNoProxiesAvailableHook[];
1530
1576
  onSocket: OnSocketHook[];
1531
1577
  onDns: OnDnsHook[];
1532
1578
  onTls: OnTlsHook[];
@@ -1983,6 +2029,275 @@ export declare class RezoError<T = any> extends Error {
1983
2029
  toString(): string;
1984
2030
  getFullDetails(): string;
1985
2031
  }
2032
+ /**
2033
+ * Queue configuration options
2034
+ */
2035
+ export interface QueueConfig {
2036
+ /** Maximum concurrent tasks (default: Infinity) */
2037
+ concurrency?: number;
2038
+ /** Auto-start processing when tasks are added (default: true) */
2039
+ autoStart?: boolean;
2040
+ /** Timeout per task in milliseconds (default: none) */
2041
+ timeout?: number;
2042
+ /** Throw on timeout vs silently fail (default: true) */
2043
+ throwOnTimeout?: boolean;
2044
+ /** Interval between task starts in ms for rate limiting */
2045
+ interval?: number;
2046
+ /** Max tasks to start per interval (default: Infinity) */
2047
+ intervalCap?: number;
2048
+ /** Carry over unused interval capacity to next interval */
2049
+ carryoverConcurrencyCount?: boolean;
2050
+ }
2051
+ /**
2052
+ * Task options when adding to queue
2053
+ */
2054
+ export interface TaskOptions {
2055
+ /** Task priority (higher runs first, default: 0) */
2056
+ priority?: number;
2057
+ /** Task-specific timeout (overrides queue default) */
2058
+ timeout?: number;
2059
+ /** Unique ID for tracking/cancellation */
2060
+ id?: string;
2061
+ /** Signal for external cancellation */
2062
+ signal?: AbortSignal;
2063
+ }
2064
+ /**
2065
+ * Current queue state
2066
+ */
2067
+ export interface QueueState {
2068
+ /** Number of tasks currently running */
2069
+ pending: number;
2070
+ /** Number of tasks waiting in queue */
2071
+ size: number;
2072
+ /** Total tasks (pending + size) */
2073
+ total: number;
2074
+ /** Is queue paused */
2075
+ isPaused: boolean;
2076
+ /** Is queue idle (no tasks) */
2077
+ isIdle: boolean;
2078
+ }
2079
+ /**
2080
+ * Queue statistics
2081
+ */
2082
+ export interface QueueStats {
2083
+ /** Total tasks added since creation */
2084
+ added: number;
2085
+ /** Total tasks processed (started) */
2086
+ processed: number;
2087
+ /** Total successful completions */
2088
+ completed: number;
2089
+ /** Total failures */
2090
+ failed: number;
2091
+ /** Total timeouts */
2092
+ timedOut: number;
2093
+ /** Total cancellations */
2094
+ cancelled: number;
2095
+ /** Average task duration (ms) */
2096
+ averageDuration: number;
2097
+ /** Tasks per second (rolling average) */
2098
+ throughput: number;
2099
+ }
2100
+ /**
2101
+ * Queue event types
2102
+ */
2103
+ export interface QueueEvents {
2104
+ /** Task added to queue */
2105
+ add: {
2106
+ id: string;
2107
+ priority: number;
2108
+ };
2109
+ /** Task started executing */
2110
+ start: {
2111
+ id: string;
2112
+ };
2113
+ /** Task completed successfully */
2114
+ completed: {
2115
+ id: string;
2116
+ result: any;
2117
+ duration: number;
2118
+ };
2119
+ /** Task failed with error */
2120
+ error: {
2121
+ id: string;
2122
+ error: Error;
2123
+ };
2124
+ /** Task timed out */
2125
+ timeout: {
2126
+ id: string;
2127
+ };
2128
+ /** Task cancelled */
2129
+ cancelled: {
2130
+ id: string;
2131
+ };
2132
+ /** Queue became active (was idle, now processing) */
2133
+ active: undefined;
2134
+ /** Queue became idle (all tasks done) */
2135
+ idle: undefined;
2136
+ /** Queue was paused */
2137
+ paused: undefined;
2138
+ /** Queue was resumed */
2139
+ resumed: undefined;
2140
+ /** Next task about to run */
2141
+ next: undefined;
2142
+ /** Queue was emptied (no pending tasks) */
2143
+ empty: undefined;
2144
+ }
2145
+ /**
2146
+ * Event handler type
2147
+ */
2148
+ export type EventHandler<T> = (data: T) => void;
2149
+ /**
2150
+ * Task function type
2151
+ */
2152
+ export type TaskFunction<T> = () => Promise<T>;
2153
+ declare class RezoQueue<T = any> {
2154
+ private queue;
2155
+ private pendingCount;
2156
+ private isPausedFlag;
2157
+ private intervalId?;
2158
+ private intervalCount;
2159
+ private intervalStart;
2160
+ private eventHandlers;
2161
+ private statsData;
2162
+ private totalDuration;
2163
+ private throughputWindow;
2164
+ private readonly throughputWindowSize;
2165
+ private idlePromise?;
2166
+ private emptyPromise?;
2167
+ readonly config: Required<QueueConfig>;
2168
+ /**
2169
+ * Create a new RezoQueue
2170
+ * @param config - Queue configuration options
2171
+ */
2172
+ constructor(config?: QueueConfig);
2173
+ /**
2174
+ * Get current queue state
2175
+ */
2176
+ get state(): QueueState;
2177
+ /**
2178
+ * Get queue statistics
2179
+ */
2180
+ get stats(): QueueStats;
2181
+ /**
2182
+ * Get/set concurrency limit
2183
+ */
2184
+ get concurrency(): number;
2185
+ set concurrency(value: number);
2186
+ /**
2187
+ * Number of pending (running) tasks
2188
+ */
2189
+ get pending(): number;
2190
+ /**
2191
+ * Number of tasks waiting in queue
2192
+ */
2193
+ get size(): number;
2194
+ /**
2195
+ * Check if queue is paused
2196
+ */
2197
+ get isPaused(): boolean;
2198
+ /**
2199
+ * Add a task to the queue
2200
+ * @param fn - Async function to execute
2201
+ * @param options - Task options
2202
+ * @returns Promise resolving to task result
2203
+ */
2204
+ add<R = T>(fn: TaskFunction<R>, options?: TaskOptions): Promise<R>;
2205
+ /**
2206
+ * Add multiple tasks to the queue
2207
+ * @param fns - Array of async functions
2208
+ * @param options - Task options (applied to all)
2209
+ * @returns Promise resolving to array of results
2210
+ */
2211
+ addAll<R = T>(fns: TaskFunction<R>[], options?: TaskOptions): Promise<R[]>;
2212
+ /**
2213
+ * Pause queue processing (running tasks continue)
2214
+ */
2215
+ pause(): void;
2216
+ /**
2217
+ * Resume queue processing
2218
+ */
2219
+ start(): void;
2220
+ /**
2221
+ * Clear all pending tasks from queue
2222
+ */
2223
+ clear(): void;
2224
+ /**
2225
+ * Cancel a specific task by ID
2226
+ * @param id - Task ID to cancel
2227
+ * @returns true if task was found and cancelled
2228
+ */
2229
+ cancel(id: string): boolean;
2230
+ /**
2231
+ * Cancel all tasks matching a predicate
2232
+ * @param predicate - Function to test each task
2233
+ * @returns Number of tasks cancelled
2234
+ */
2235
+ cancelBy(predicate: (task: {
2236
+ id: string;
2237
+ priority: number;
2238
+ }) => boolean): number;
2239
+ /**
2240
+ * Wait for queue to become idle (no running or pending tasks)
2241
+ */
2242
+ onIdle(): Promise<void>;
2243
+ /**
2244
+ * Wait for queue to be empty (no pending tasks, but may have running)
2245
+ */
2246
+ onEmpty(): Promise<void>;
2247
+ /**
2248
+ * Wait for queue size to be less than limit
2249
+ * @param limit - Size threshold
2250
+ */
2251
+ onSizeLessThan(limit: number): Promise<void>;
2252
+ /**
2253
+ * Register an event handler
2254
+ * @param event - Event name
2255
+ * @param handler - Handler function
2256
+ */
2257
+ on<E extends keyof QueueEvents>(event: E, handler: EventHandler<QueueEvents[E]>): void;
2258
+ /**
2259
+ * Remove an event handler
2260
+ * @param event - Event name
2261
+ * @param handler - Handler function to remove
2262
+ */
2263
+ off<E extends keyof QueueEvents>(event: E, handler: EventHandler<QueueEvents[E]>): void;
2264
+ /**
2265
+ * Destroy the queue and cleanup resources
2266
+ */
2267
+ destroy(): void;
2268
+ /**
2269
+ * Insert task into queue maintaining priority order (highest first)
2270
+ */
2271
+ private insertByPriority;
2272
+ /**
2273
+ * Try to run next task if capacity available
2274
+ */
2275
+ private tryRunNext;
2276
+ /**
2277
+ * Execute a task
2278
+ */
2279
+ private runTask;
2280
+ /**
2281
+ * Record task duration for statistics
2282
+ */
2283
+ private recordDuration;
2284
+ /**
2285
+ * Start interval-based rate limiting
2286
+ */
2287
+ private startInterval;
2288
+ /**
2289
+ * Emit an event
2290
+ */
2291
+ protected emit<E extends keyof QueueEvents>(event: E, data: QueueEvents[E]): void;
2292
+ /**
2293
+ * Check if queue became empty
2294
+ */
2295
+ private checkEmpty;
2296
+ /**
2297
+ * Check if queue became idle
2298
+ */
2299
+ private checkIdle;
2300
+ }
1986
2301
  type ProxyProtocol$1 = "http" | "https" | "socks4" | "socks5";
1987
2302
  /**
1988
2303
  * Configuration options for proxy connections
@@ -2104,7 +2419,7 @@ export interface RezoRequestConfig<D = any> {
2104
2419
  /**
2105
2420
  * Queue to use for request execution
2106
2421
  */
2107
- queue?: PQueue | null;
2422
+ queue?: RezoQueue | null;
2108
2423
  /**
2109
2424
  * Controls how the response body is parsed and returned in `response.data`.
2110
2425
  *
@@ -2194,10 +2509,6 @@ export interface RezoRequestConfig<D = any> {
2194
2509
  autoSetOrigin?: boolean;
2195
2510
  treat302As303?: boolean;
2196
2511
  startNewRequest?: boolean;
2197
- /** Whether to use HTTP/2 protocol */
2198
- http2?: boolean;
2199
- /** Whether to use cURL adapter */
2200
- curl?: boolean;
2201
2512
  /**
2202
2513
  * DNS cache configuration for faster repeated requests.
2203
2514
  *
@@ -2301,6 +2612,12 @@ export interface RezoRequestConfig<D = any> {
2301
2612
  withCredentials?: boolean;
2302
2613
  /** Proxy configuration (URL string or detailed options) */
2303
2614
  proxy?: string | ProxyOptions;
2615
+ /**
2616
+ * Whether to use ProxyManager for this request
2617
+ * Set to false to bypass ProxyManager even when one is configured
2618
+ * @default true (uses ProxyManager if configured)
2619
+ */
2620
+ useProxyManager?: boolean;
2304
2621
  /** Whether to enable automatic cookie handling */
2305
2622
  useCookies?: boolean;
2306
2623
  /** Custom cookie jar for managing cookies */
@@ -2322,8 +2639,6 @@ export interface RezoRequestConfig<D = any> {
2322
2639
  transformRequest?: Array<(data: any, headers: RezoHeaders) => any>;
2323
2640
  /** Array of functions to transform response data */
2324
2641
  transformResponse?: Array<(data: any) => any>;
2325
- /** Adapter to use for the request (name or custom function) */
2326
- adapter?: string | ((config: RezoRequestConfig) => Promise<any>);
2327
2642
  /** AbortSignal to cancel the request */
2328
2643
  signal?: AbortSignal;
2329
2644
  /** File path to save the response to (for downloads) */
@@ -2540,6 +2855,7 @@ type BeforeProxyDisableHook$1 = (context: BeforeProxyDisableContext) => boolean
2540
2855
  type AfterProxyDisableHook$1 = (context: AfterProxyDisableContext) => void | Promise<void>;
2541
2856
  type AfterProxyRotateHook$1 = (context: AfterProxyRotateContext) => void | Promise<void>;
2542
2857
  type AfterProxyEnableHook$1 = (context: AfterProxyEnableContext) => void | Promise<void>;
2858
+ type OnNoProxiesAvailableHook$1 = (context: OnNoProxiesAvailableContext) => void | Promise<void>;
2543
2859
  /**
2544
2860
  * Proxy hooks collection for ProxyManager events
2545
2861
  */
@@ -2552,6 +2868,8 @@ export interface ProxyHooks {
2552
2868
  afterProxyDisable: AfterProxyDisableHook$1[];
2553
2869
  afterProxyRotate: AfterProxyRotateHook$1[];
2554
2870
  afterProxyEnable: AfterProxyEnableHook$1[];
2871
+ /** Hook triggered when no proxies are available */
2872
+ onNoProxiesAvailable: OnNoProxiesAvailableHook$1[];
2555
2873
  }
2556
2874
  declare class ProxyManager {
2557
2875
  /** Configuration for the proxy manager */
@@ -2687,8 +3005,40 @@ declare class ProxyManager {
2687
3005
  private runAfterProxyRotateHooks;
2688
3006
  private runAfterProxyDisableHooks;
2689
3007
  private runAfterProxyEnableHooks;
3008
+ /**
3009
+ * Run onNoProxiesAvailable hooks synchronously
3010
+ * Called when no proxies are available and an error is about to be thrown
3011
+ */
3012
+ private runOnNoProxiesAvailableHooksSync;
3013
+ /**
3014
+ * Run onNoProxiesAvailable hooks asynchronously
3015
+ * Called when no proxies are available and an error is about to be thrown
3016
+ */
3017
+ runOnNoProxiesAvailableHooks(context: OnNoProxiesAvailableContext): Promise<void>;
3018
+ /**
3019
+ * Notify that no proxies are available and trigger hooks
3020
+ * This method is called when proxy selection fails due to pool exhaustion
3021
+ *
3022
+ * @param url - The request URL that needed a proxy
3023
+ * @param error - The error that will be thrown
3024
+ * @returns The context object with detailed information about the proxy pool state
3025
+ *
3026
+ * @example
3027
+ * ```typescript
3028
+ * manager.hooks.onNoProxiesAvailable.push((context) => {
3029
+ * console.error(`No proxies available for ${context.url}`);
3030
+ * console.log(`Dead: ${context.disabledReasons.dead}, Limit: ${context.disabledReasons.limitReached}`);
3031
+ * // Trigger external alert or proxy refresh
3032
+ * alertSystem.notify('Proxy pool exhausted', context);
3033
+ * });
3034
+ *
3035
+ * // Called internally or by adapters when no proxies are available
3036
+ * const context = manager.notifyNoProxiesAvailable('https://api.example.com', new Error('No proxies'));
3037
+ * ```
3038
+ */
3039
+ notifyNoProxiesAvailable(url: string, error: Error): OnNoProxiesAvailableContext;
2690
3040
  }
2691
- export type queueOptions = Options$1<PriorityQueue, QueueAddOptions>;
3041
+ export type queueOptions = QueueConfig;
2692
3042
  export interface CacheConfig {
2693
3043
  /** Response cache configuration */
2694
3044
  response?: boolean | ResponseCacheConfig;
@@ -2747,10 +3097,6 @@ export interface RezoDefaultOptions {
2747
3097
  keepAlive?: boolean;
2748
3098
  /** Whether to detect and prevent redirect cycles */
2749
3099
  enableRedirectCycleDetection?: boolean;
2750
- /** Whether to use HTTP/2 protocol */
2751
- http2?: boolean;
2752
- /** Whether to use cURL adapter */
2753
- curl?: boolean;
2754
3100
  /** Whether to send cookies and authorization headers with cross-origin requests */
2755
3101
  withCredentials?: boolean;
2756
3102
  /** Proxy configuration (URL string or detailed options) */
@@ -2789,8 +3135,6 @@ export interface RezoDefaultOptions {
2789
3135
  transformRequest?: RezoHttpRequest["transformRequest"];
2790
3136
  /** Array of functions to transform response data */
2791
3137
  transformResponse?: RezoHttpRequest["transformResponse"];
2792
- /** Adapter to use for the request (name or custom function) */
2793
- adapter?: RezoHttpRequest["adapter"];
2794
3138
  /** Browser simulation configuration for user agent spoofing */
2795
3139
  browser?: RezoHttpRequest["browser"];
2796
3140
  /** Enable debug logging for the request */
@@ -3693,7 +4037,7 @@ export type AdapterFunction<T = any> = (options: RezoRequestConfig, defaultOptio
3693
4037
  * Main Rezo class - Enterprise-grade HTTP client with advanced features
3694
4038
  */
3695
4039
  export declare class Rezo {
3696
- protected queue: PQueue | null;
4040
+ protected queue: RezoQueue | null;
3697
4041
  protected isQueueEnabled: boolean;
3698
4042
  defaults: RezoDefaultOptions;
3699
4043
  hooks: RezoHooks;