rezo 1.0.5 → 1.0.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +352 -9
- package/dist/adapters/curl.cjs +796 -0
- package/dist/adapters/curl.js +796 -0
- package/dist/adapters/entries/curl.d.ts +2332 -20
- package/dist/adapters/entries/fetch.d.ts +289 -20
- package/dist/adapters/entries/http.d.ts +289 -20
- package/dist/adapters/entries/http2.d.ts +289 -20
- package/dist/adapters/entries/react-native.d.ts +289 -20
- package/dist/adapters/entries/xhr.d.ts +289 -20
- package/dist/adapters/index.cjs +6 -6
- package/dist/adapters/picker.cjs +2 -2
- package/dist/adapters/picker.js +2 -2
- package/dist/cache/index.cjs +13 -13
- package/dist/core/rezo.cjs +2 -2
- package/dist/core/rezo.js +2 -2
- package/dist/crawler.d.ts +291 -22
- package/dist/entries/crawler.cjs +5 -5
- package/dist/index.cjs +23 -18
- package/dist/index.d.ts +556 -20
- package/dist/index.js +1 -0
- package/dist/platform/browser.d.ts +289 -20
- package/dist/platform/bun.d.ts +289 -20
- package/dist/platform/deno.d.ts +289 -20
- package/dist/platform/node.d.ts +289 -20
- package/dist/platform/react-native.d.ts +289 -20
- package/dist/platform/worker.d.ts +289 -20
- package/dist/plugin/crawler-options.cjs +1 -1
- package/dist/plugin/crawler-options.js +1 -1
- package/dist/plugin/crawler.cjs +2 -2
- package/dist/plugin/crawler.js +2 -2
- package/dist/plugin/index.cjs +36 -36
- package/dist/proxy/index.cjs +2 -2
- package/dist/proxy/manager.cjs +14 -1
- package/dist/proxy/manager.js +14 -1
- package/dist/queue/http-queue.cjs +313 -0
- package/dist/queue/http-queue.js +312 -0
- package/dist/queue/index.cjs +8 -0
- package/dist/queue/index.js +6 -0
- package/dist/queue/queue.cjs +346 -0
- package/dist/queue/queue.js +344 -0
- package/dist/queue/types.cjs +17 -0
- package/dist/queue/types.js +17 -0
- package/dist/types/curl-options.cjs +25 -0
- package/dist/types/curl-options.js +25 -0
- package/dist/utils/http-config.cjs +0 -15
- package/dist/utils/http-config.js +0 -15
- package/package.json +1 -2
package/dist/platform/node.d.ts
CHANGED
|
@@ -6,9 +6,6 @@ import { Agent as HttpsAgent } from 'node:https';
|
|
|
6
6
|
import { Socket } from 'node:net';
|
|
7
7
|
import { Readable, Writable, WritableOptions } from 'node:stream';
|
|
8
8
|
import { SecureContext, TLSSocket } from 'node:tls';
|
|
9
|
-
import PQueue from 'p-queue';
|
|
10
|
-
import { Options as Options$1, QueueAddOptions } from 'p-queue';
|
|
11
|
-
import PriorityQueue from 'p-queue/dist/priority-queue';
|
|
12
9
|
import { Cookie as TouchCookie, CookieJar as TouchCookieJar, CreateCookieOptions } from 'tough-cookie';
|
|
13
10
|
|
|
14
11
|
export interface RezoHttpHeaders {
|
|
@@ -972,8 +969,17 @@ export interface ProxyCooldownConfig {
|
|
|
972
969
|
* Complete configuration for proxy rotation, filtering, and failure handling
|
|
973
970
|
*/
|
|
974
971
|
export interface ProxyManagerBaseConfig {
|
|
975
|
-
/**
|
|
976
|
-
|
|
972
|
+
/**
|
|
973
|
+
* Array of proxies to manage
|
|
974
|
+
* Accepts ProxyInfo objects or proxy URL strings (parsed via parseProxyString)
|
|
975
|
+
* @example
|
|
976
|
+
* proxies: [
|
|
977
|
+
* { protocol: 'http', host: '127.0.0.1', port: 8080 },
|
|
978
|
+
* 'socks5://user:pass@proxy.example.com:1080',
|
|
979
|
+
* 'http://proxy.example.com:3128'
|
|
980
|
+
* ]
|
|
981
|
+
*/
|
|
982
|
+
proxies: (ProxyInfo | string)[];
|
|
977
983
|
/**
|
|
978
984
|
* Whitelist patterns for URLs that should use proxy
|
|
979
985
|
* - String: exact domain match (e.g., 'api.example.com') or subdomain match (e.g., 'example.com' matches '*.example.com')
|
|
@@ -1983,6 +1989,275 @@ export declare class RezoError<T = any> extends Error {
|
|
|
1983
1989
|
toString(): string;
|
|
1984
1990
|
getFullDetails(): string;
|
|
1985
1991
|
}
|
|
1992
|
+
/**
|
|
1993
|
+
* Queue configuration options
|
|
1994
|
+
*/
|
|
1995
|
+
export interface QueueConfig {
|
|
1996
|
+
/** Maximum concurrent tasks (default: Infinity) */
|
|
1997
|
+
concurrency?: number;
|
|
1998
|
+
/** Auto-start processing when tasks are added (default: true) */
|
|
1999
|
+
autoStart?: boolean;
|
|
2000
|
+
/** Timeout per task in milliseconds (default: none) */
|
|
2001
|
+
timeout?: number;
|
|
2002
|
+
/** Throw on timeout vs silently fail (default: true) */
|
|
2003
|
+
throwOnTimeout?: boolean;
|
|
2004
|
+
/** Interval between task starts in ms for rate limiting */
|
|
2005
|
+
interval?: number;
|
|
2006
|
+
/** Max tasks to start per interval (default: Infinity) */
|
|
2007
|
+
intervalCap?: number;
|
|
2008
|
+
/** Carry over unused interval capacity to next interval */
|
|
2009
|
+
carryoverConcurrencyCount?: boolean;
|
|
2010
|
+
}
|
|
2011
|
+
/**
|
|
2012
|
+
* Task options when adding to queue
|
|
2013
|
+
*/
|
|
2014
|
+
export interface TaskOptions {
|
|
2015
|
+
/** Task priority (higher runs first, default: 0) */
|
|
2016
|
+
priority?: number;
|
|
2017
|
+
/** Task-specific timeout (overrides queue default) */
|
|
2018
|
+
timeout?: number;
|
|
2019
|
+
/** Unique ID for tracking/cancellation */
|
|
2020
|
+
id?: string;
|
|
2021
|
+
/** Signal for external cancellation */
|
|
2022
|
+
signal?: AbortSignal;
|
|
2023
|
+
}
|
|
2024
|
+
/**
|
|
2025
|
+
* Current queue state
|
|
2026
|
+
*/
|
|
2027
|
+
export interface QueueState {
|
|
2028
|
+
/** Number of tasks currently running */
|
|
2029
|
+
pending: number;
|
|
2030
|
+
/** Number of tasks waiting in queue */
|
|
2031
|
+
size: number;
|
|
2032
|
+
/** Total tasks (pending + size) */
|
|
2033
|
+
total: number;
|
|
2034
|
+
/** Is queue paused */
|
|
2035
|
+
isPaused: boolean;
|
|
2036
|
+
/** Is queue idle (no tasks) */
|
|
2037
|
+
isIdle: boolean;
|
|
2038
|
+
}
|
|
2039
|
+
/**
|
|
2040
|
+
* Queue statistics
|
|
2041
|
+
*/
|
|
2042
|
+
export interface QueueStats {
|
|
2043
|
+
/** Total tasks added since creation */
|
|
2044
|
+
added: number;
|
|
2045
|
+
/** Total tasks processed (started) */
|
|
2046
|
+
processed: number;
|
|
2047
|
+
/** Total successful completions */
|
|
2048
|
+
completed: number;
|
|
2049
|
+
/** Total failures */
|
|
2050
|
+
failed: number;
|
|
2051
|
+
/** Total timeouts */
|
|
2052
|
+
timedOut: number;
|
|
2053
|
+
/** Total cancellations */
|
|
2054
|
+
cancelled: number;
|
|
2055
|
+
/** Average task duration (ms) */
|
|
2056
|
+
averageDuration: number;
|
|
2057
|
+
/** Tasks per second (rolling average) */
|
|
2058
|
+
throughput: number;
|
|
2059
|
+
}
|
|
2060
|
+
/**
|
|
2061
|
+
* Queue event types
|
|
2062
|
+
*/
|
|
2063
|
+
export interface QueueEvents {
|
|
2064
|
+
/** Task added to queue */
|
|
2065
|
+
add: {
|
|
2066
|
+
id: string;
|
|
2067
|
+
priority: number;
|
|
2068
|
+
};
|
|
2069
|
+
/** Task started executing */
|
|
2070
|
+
start: {
|
|
2071
|
+
id: string;
|
|
2072
|
+
};
|
|
2073
|
+
/** Task completed successfully */
|
|
2074
|
+
completed: {
|
|
2075
|
+
id: string;
|
|
2076
|
+
result: any;
|
|
2077
|
+
duration: number;
|
|
2078
|
+
};
|
|
2079
|
+
/** Task failed with error */
|
|
2080
|
+
error: {
|
|
2081
|
+
id: string;
|
|
2082
|
+
error: Error;
|
|
2083
|
+
};
|
|
2084
|
+
/** Task timed out */
|
|
2085
|
+
timeout: {
|
|
2086
|
+
id: string;
|
|
2087
|
+
};
|
|
2088
|
+
/** Task cancelled */
|
|
2089
|
+
cancelled: {
|
|
2090
|
+
id: string;
|
|
2091
|
+
};
|
|
2092
|
+
/** Queue became active (was idle, now processing) */
|
|
2093
|
+
active: undefined;
|
|
2094
|
+
/** Queue became idle (all tasks done) */
|
|
2095
|
+
idle: undefined;
|
|
2096
|
+
/** Queue was paused */
|
|
2097
|
+
paused: undefined;
|
|
2098
|
+
/** Queue was resumed */
|
|
2099
|
+
resumed: undefined;
|
|
2100
|
+
/** Next task about to run */
|
|
2101
|
+
next: undefined;
|
|
2102
|
+
/** Queue was emptied (no pending tasks) */
|
|
2103
|
+
empty: undefined;
|
|
2104
|
+
}
|
|
2105
|
+
/**
|
|
2106
|
+
* Event handler type
|
|
2107
|
+
*/
|
|
2108
|
+
export type EventHandler<T> = (data: T) => void;
|
|
2109
|
+
/**
|
|
2110
|
+
* Task function type
|
|
2111
|
+
*/
|
|
2112
|
+
export type TaskFunction<T> = () => Promise<T>;
|
|
2113
|
+
declare class RezoQueue<T = any> {
|
|
2114
|
+
private queue;
|
|
2115
|
+
private pendingCount;
|
|
2116
|
+
private isPausedFlag;
|
|
2117
|
+
private intervalId?;
|
|
2118
|
+
private intervalCount;
|
|
2119
|
+
private intervalStart;
|
|
2120
|
+
private eventHandlers;
|
|
2121
|
+
private statsData;
|
|
2122
|
+
private totalDuration;
|
|
2123
|
+
private throughputWindow;
|
|
2124
|
+
private readonly throughputWindowSize;
|
|
2125
|
+
private idlePromise?;
|
|
2126
|
+
private emptyPromise?;
|
|
2127
|
+
readonly config: Required<QueueConfig>;
|
|
2128
|
+
/**
|
|
2129
|
+
* Create a new RezoQueue
|
|
2130
|
+
* @param config - Queue configuration options
|
|
2131
|
+
*/
|
|
2132
|
+
constructor(config?: QueueConfig);
|
|
2133
|
+
/**
|
|
2134
|
+
* Get current queue state
|
|
2135
|
+
*/
|
|
2136
|
+
get state(): QueueState;
|
|
2137
|
+
/**
|
|
2138
|
+
* Get queue statistics
|
|
2139
|
+
*/
|
|
2140
|
+
get stats(): QueueStats;
|
|
2141
|
+
/**
|
|
2142
|
+
* Get/set concurrency limit
|
|
2143
|
+
*/
|
|
2144
|
+
get concurrency(): number;
|
|
2145
|
+
set concurrency(value: number);
|
|
2146
|
+
/**
|
|
2147
|
+
* Number of pending (running) tasks
|
|
2148
|
+
*/
|
|
2149
|
+
get pending(): number;
|
|
2150
|
+
/**
|
|
2151
|
+
* Number of tasks waiting in queue
|
|
2152
|
+
*/
|
|
2153
|
+
get size(): number;
|
|
2154
|
+
/**
|
|
2155
|
+
* Check if queue is paused
|
|
2156
|
+
*/
|
|
2157
|
+
get isPaused(): boolean;
|
|
2158
|
+
/**
|
|
2159
|
+
* Add a task to the queue
|
|
2160
|
+
* @param fn - Async function to execute
|
|
2161
|
+
* @param options - Task options
|
|
2162
|
+
* @returns Promise resolving to task result
|
|
2163
|
+
*/
|
|
2164
|
+
add<R = T>(fn: TaskFunction<R>, options?: TaskOptions): Promise<R>;
|
|
2165
|
+
/**
|
|
2166
|
+
* Add multiple tasks to the queue
|
|
2167
|
+
* @param fns - Array of async functions
|
|
2168
|
+
* @param options - Task options (applied to all)
|
|
2169
|
+
* @returns Promise resolving to array of results
|
|
2170
|
+
*/
|
|
2171
|
+
addAll<R = T>(fns: TaskFunction<R>[], options?: TaskOptions): Promise<R[]>;
|
|
2172
|
+
/**
|
|
2173
|
+
* Pause queue processing (running tasks continue)
|
|
2174
|
+
*/
|
|
2175
|
+
pause(): void;
|
|
2176
|
+
/**
|
|
2177
|
+
* Resume queue processing
|
|
2178
|
+
*/
|
|
2179
|
+
start(): void;
|
|
2180
|
+
/**
|
|
2181
|
+
* Clear all pending tasks from queue
|
|
2182
|
+
*/
|
|
2183
|
+
clear(): void;
|
|
2184
|
+
/**
|
|
2185
|
+
* Cancel a specific task by ID
|
|
2186
|
+
* @param id - Task ID to cancel
|
|
2187
|
+
* @returns true if task was found and cancelled
|
|
2188
|
+
*/
|
|
2189
|
+
cancel(id: string): boolean;
|
|
2190
|
+
/**
|
|
2191
|
+
* Cancel all tasks matching a predicate
|
|
2192
|
+
* @param predicate - Function to test each task
|
|
2193
|
+
* @returns Number of tasks cancelled
|
|
2194
|
+
*/
|
|
2195
|
+
cancelBy(predicate: (task: {
|
|
2196
|
+
id: string;
|
|
2197
|
+
priority: number;
|
|
2198
|
+
}) => boolean): number;
|
|
2199
|
+
/**
|
|
2200
|
+
* Wait for queue to become idle (no running or pending tasks)
|
|
2201
|
+
*/
|
|
2202
|
+
onIdle(): Promise<void>;
|
|
2203
|
+
/**
|
|
2204
|
+
* Wait for queue to be empty (no pending tasks, but may have running)
|
|
2205
|
+
*/
|
|
2206
|
+
onEmpty(): Promise<void>;
|
|
2207
|
+
/**
|
|
2208
|
+
* Wait for queue size to be less than limit
|
|
2209
|
+
* @param limit - Size threshold
|
|
2210
|
+
*/
|
|
2211
|
+
onSizeLessThan(limit: number): Promise<void>;
|
|
2212
|
+
/**
|
|
2213
|
+
* Register an event handler
|
|
2214
|
+
* @param event - Event name
|
|
2215
|
+
* @param handler - Handler function
|
|
2216
|
+
*/
|
|
2217
|
+
on<E extends keyof QueueEvents>(event: E, handler: EventHandler<QueueEvents[E]>): void;
|
|
2218
|
+
/**
|
|
2219
|
+
* Remove an event handler
|
|
2220
|
+
* @param event - Event name
|
|
2221
|
+
* @param handler - Handler function to remove
|
|
2222
|
+
*/
|
|
2223
|
+
off<E extends keyof QueueEvents>(event: E, handler: EventHandler<QueueEvents[E]>): void;
|
|
2224
|
+
/**
|
|
2225
|
+
* Destroy the queue and cleanup resources
|
|
2226
|
+
*/
|
|
2227
|
+
destroy(): void;
|
|
2228
|
+
/**
|
|
2229
|
+
* Insert task into queue maintaining priority order (highest first)
|
|
2230
|
+
*/
|
|
2231
|
+
private insertByPriority;
|
|
2232
|
+
/**
|
|
2233
|
+
* Try to run next task if capacity available
|
|
2234
|
+
*/
|
|
2235
|
+
private tryRunNext;
|
|
2236
|
+
/**
|
|
2237
|
+
* Execute a task
|
|
2238
|
+
*/
|
|
2239
|
+
private runTask;
|
|
2240
|
+
/**
|
|
2241
|
+
* Record task duration for statistics
|
|
2242
|
+
*/
|
|
2243
|
+
private recordDuration;
|
|
2244
|
+
/**
|
|
2245
|
+
* Start interval-based rate limiting
|
|
2246
|
+
*/
|
|
2247
|
+
private startInterval;
|
|
2248
|
+
/**
|
|
2249
|
+
* Emit an event
|
|
2250
|
+
*/
|
|
2251
|
+
protected emit<E extends keyof QueueEvents>(event: E, data: QueueEvents[E]): void;
|
|
2252
|
+
/**
|
|
2253
|
+
* Check if queue became empty
|
|
2254
|
+
*/
|
|
2255
|
+
private checkEmpty;
|
|
2256
|
+
/**
|
|
2257
|
+
* Check if queue became idle
|
|
2258
|
+
*/
|
|
2259
|
+
private checkIdle;
|
|
2260
|
+
}
|
|
1986
2261
|
type ProxyProtocol$1 = "http" | "https" | "socks4" | "socks5";
|
|
1987
2262
|
/**
|
|
1988
2263
|
* Configuration options for proxy connections
|
|
@@ -2104,7 +2379,7 @@ export interface RezoRequestConfig<D = any> {
|
|
|
2104
2379
|
/**
|
|
2105
2380
|
* Queue to use for request execution
|
|
2106
2381
|
*/
|
|
2107
|
-
queue?:
|
|
2382
|
+
queue?: RezoQueue | null;
|
|
2108
2383
|
/**
|
|
2109
2384
|
* Controls how the response body is parsed and returned in `response.data`.
|
|
2110
2385
|
*
|
|
@@ -2194,10 +2469,6 @@ export interface RezoRequestConfig<D = any> {
|
|
|
2194
2469
|
autoSetOrigin?: boolean;
|
|
2195
2470
|
treat302As303?: boolean;
|
|
2196
2471
|
startNewRequest?: boolean;
|
|
2197
|
-
/** Whether to use HTTP/2 protocol */
|
|
2198
|
-
http2?: boolean;
|
|
2199
|
-
/** Whether to use cURL adapter */
|
|
2200
|
-
curl?: boolean;
|
|
2201
2472
|
/**
|
|
2202
2473
|
* DNS cache configuration for faster repeated requests.
|
|
2203
2474
|
*
|
|
@@ -2301,6 +2572,12 @@ export interface RezoRequestConfig<D = any> {
|
|
|
2301
2572
|
withCredentials?: boolean;
|
|
2302
2573
|
/** Proxy configuration (URL string or detailed options) */
|
|
2303
2574
|
proxy?: string | ProxyOptions;
|
|
2575
|
+
/**
|
|
2576
|
+
* Whether to use ProxyManager for this request
|
|
2577
|
+
* Set to false to bypass ProxyManager even when one is configured
|
|
2578
|
+
* @default true (uses ProxyManager if configured)
|
|
2579
|
+
*/
|
|
2580
|
+
useProxyManager?: boolean;
|
|
2304
2581
|
/** Whether to enable automatic cookie handling */
|
|
2305
2582
|
useCookies?: boolean;
|
|
2306
2583
|
/** Custom cookie jar for managing cookies */
|
|
@@ -2322,8 +2599,6 @@ export interface RezoRequestConfig<D = any> {
|
|
|
2322
2599
|
transformRequest?: Array<(data: any, headers: RezoHeaders) => any>;
|
|
2323
2600
|
/** Array of functions to transform response data */
|
|
2324
2601
|
transformResponse?: Array<(data: any) => any>;
|
|
2325
|
-
/** Adapter to use for the request (name or custom function) */
|
|
2326
|
-
adapter?: string | ((config: RezoRequestConfig) => Promise<any>);
|
|
2327
2602
|
/** AbortSignal to cancel the request */
|
|
2328
2603
|
signal?: AbortSignal;
|
|
2329
2604
|
/** File path to save the response to (for downloads) */
|
|
@@ -2688,7 +2963,7 @@ declare class ProxyManager {
|
|
|
2688
2963
|
private runAfterProxyDisableHooks;
|
|
2689
2964
|
private runAfterProxyEnableHooks;
|
|
2690
2965
|
}
|
|
2691
|
-
export type queueOptions =
|
|
2966
|
+
export type queueOptions = QueueConfig;
|
|
2692
2967
|
export interface CacheConfig {
|
|
2693
2968
|
/** Response cache configuration */
|
|
2694
2969
|
response?: boolean | ResponseCacheConfig;
|
|
@@ -2747,10 +3022,6 @@ export interface RezoDefaultOptions {
|
|
|
2747
3022
|
keepAlive?: boolean;
|
|
2748
3023
|
/** Whether to detect and prevent redirect cycles */
|
|
2749
3024
|
enableRedirectCycleDetection?: boolean;
|
|
2750
|
-
/** Whether to use HTTP/2 protocol */
|
|
2751
|
-
http2?: boolean;
|
|
2752
|
-
/** Whether to use cURL adapter */
|
|
2753
|
-
curl?: boolean;
|
|
2754
3025
|
/** Whether to send cookies and authorization headers with cross-origin requests */
|
|
2755
3026
|
withCredentials?: boolean;
|
|
2756
3027
|
/** Proxy configuration (URL string or detailed options) */
|
|
@@ -2789,8 +3060,6 @@ export interface RezoDefaultOptions {
|
|
|
2789
3060
|
transformRequest?: RezoHttpRequest["transformRequest"];
|
|
2790
3061
|
/** Array of functions to transform response data */
|
|
2791
3062
|
transformResponse?: RezoHttpRequest["transformResponse"];
|
|
2792
|
-
/** Adapter to use for the request (name or custom function) */
|
|
2793
|
-
adapter?: RezoHttpRequest["adapter"];
|
|
2794
3063
|
/** Browser simulation configuration for user agent spoofing */
|
|
2795
3064
|
browser?: RezoHttpRequest["browser"];
|
|
2796
3065
|
/** Enable debug logging for the request */
|
|
@@ -3693,7 +3962,7 @@ export type AdapterFunction<T = any> = (options: RezoRequestConfig, defaultOptio
|
|
|
3693
3962
|
* Main Rezo class - Enterprise-grade HTTP client with advanced features
|
|
3694
3963
|
*/
|
|
3695
3964
|
export declare class Rezo {
|
|
3696
|
-
protected queue:
|
|
3965
|
+
protected queue: RezoQueue | null;
|
|
3697
3966
|
protected isQueueEnabled: boolean;
|
|
3698
3967
|
defaults: RezoDefaultOptions;
|
|
3699
3968
|
hooks: RezoHooks;
|