rezo 1.0.5 → 1.0.7

package/dist/crawler.d.ts

@@ -6,9 +6,6 @@ import { Agent as HttpsAgent } from 'node:https';
 import { Socket } from 'node:net';
 import { Readable, Writable, WritableOptions } from 'node:stream';
 import { SecureContext, TLSSocket } from 'node:tls';
-import PQueue from 'p-queue';
-import { Options as Options$1, QueueAddOptions } from 'p-queue';
-import PriorityQueue from 'p-queue/dist/priority-queue';
 import { Cookie as TouchCookie, CookieJar as TouchCookieJar, CreateCookieOptions } from 'tough-cookie';
 
 /**
@@ -1195,8 +1192,17 @@ export interface ProxyCooldownConfig {
  * Complete configuration for proxy rotation, filtering, and failure handling
  */
 export interface ProxyManagerBaseConfig {
-	/** Array of proxies to manage */
-	proxies: ProxyInfo[];
+	/**
+	 * Array of proxies to manage
+	 * Accepts ProxyInfo objects or proxy URL strings (parsed via parseProxyString)
+	 * @example
+	 * proxies: [
+	 *   { protocol: 'http', host: '127.0.0.1', port: 8080 },
+	 *   'socks5://user:pass@proxy.example.com:1080',
+	 *   'http://proxy.example.com:3128'
+	 * ]
+	 */
+	proxies: (ProxyInfo | string)[];
 	/**
 	 * Whitelist patterns for URLs that should use proxy
 	 * - String: exact domain match (e.g., 'api.example.com') or subdomain match (e.g., 'example.com' matches '*.example.com')
@@ -1408,6 +1414,35 @@ export interface AfterProxyEnableContext {
 	/** Reason for enabling */
 	reason: "cooldown-expired" | "manual";
 }
+/**
+ * Context for onNoProxiesAvailable hook
+ * Triggered when no proxies are available and an error would be thrown
+ */
+export interface OnNoProxiesAvailableContext {
+	/** Request URL that needed a proxy */
+	url: string;
+	/** The error that will be thrown */
+	error: Error;
+	/** All proxies (including disabled ones) */
+	allProxies: ProxyState[];
+	/** Number of active proxies (should be 0) */
+	activeCount: number;
+	/** Number of disabled proxies */
+	disabledCount: number;
+	/** Number of proxies in cooldown */
+	cooldownCount: number;
+	/** Reasons why proxies are unavailable */
+	disabledReasons: {
+		/** Proxies disabled due to failures */
+		dead: number;
+		/** Proxies disabled due to request limit */
+		limitReached: number;
+		/** Proxies manually disabled */
+		manual: number;
+	};
+	/** Timestamp when this event occurred */
+	timestamp: number;
+}
 /**
  * Context provided to beforeRequest hook
  * Contains metadata about the current request state
@@ -1726,6 +1761,16 @@ export type AfterProxyRotateHook = (context: AfterProxyRotateContext) => void |
  * Use for notifications, logging
  */
 export type AfterProxyEnableHook = (context: AfterProxyEnableContext) => void | Promise<void>;
+/**
+ * Hook called when no proxies are available and an error is about to be thrown
+ * Use for alerting, logging exhausted proxy pools, or triggering proxy refresh
+ * This hook is called just before the error is thrown, allowing you to:
+ * - Log the exhaustion event for monitoring
+ * - Trigger external proxy pool refresh
+ * - Send alerts to monitoring systems
+ * - Record statistics about proxy pool health
+ */
+export type OnNoProxiesAvailableHook = (context: OnNoProxiesAvailableContext) => void | Promise<void>;
 /**
  * Collection of all hook types
  * All hooks are arrays to allow multiple handlers
@@ -1750,6 +1795,7 @@ export interface RezoHooks {
 	afterProxyDisable: AfterProxyDisableHook[];
 	afterProxyRotate: AfterProxyRotateHook[];
 	afterProxyEnable: AfterProxyEnableHook[];
+	onNoProxiesAvailable: OnNoProxiesAvailableHook[];
 	onSocket: OnSocketHook[];
 	onDns: OnDnsHook[];
 	onTls: OnTlsHook[];
@@ -2146,6 +2192,275 @@ declare class RezoError<T = any> extends Error {
 	toString(): string;
 	getFullDetails(): string;
 }
+/**
+ * Queue configuration options
+ */
+export interface QueueConfig {
+	/** Maximum concurrent tasks (default: Infinity) */
+	concurrency?: number;
+	/** Auto-start processing when tasks are added (default: true) */
+	autoStart?: boolean;
+	/** Timeout per task in milliseconds (default: none) */
+	timeout?: number;
+	/** Throw on timeout vs silently fail (default: true) */
+	throwOnTimeout?: boolean;
+	/** Interval between task starts in ms for rate limiting */
+	interval?: number;
+	/** Max tasks to start per interval (default: Infinity) */
+	intervalCap?: number;
+	/** Carry over unused interval capacity to next interval */
+	carryoverConcurrencyCount?: boolean;
+}
+/**
+ * Task options when adding to queue
+ */
+export interface TaskOptions {
+	/** Task priority (higher runs first, default: 0) */
+	priority?: number;
+	/** Task-specific timeout (overrides queue default) */
+	timeout?: number;
+	/** Unique ID for tracking/cancellation */
+	id?: string;
+	/** Signal for external cancellation */
+	signal?: AbortSignal;
+}
+/**
+ * Current queue state
+ */
+export interface QueueState {
+	/** Number of tasks currently running */
+	pending: number;
+	/** Number of tasks waiting in queue */
+	size: number;
+	/** Total tasks (pending + size) */
+	total: number;
+	/** Is queue paused */
+	isPaused: boolean;
+	/** Is queue idle (no tasks) */
+	isIdle: boolean;
+}
+/**
+ * Queue statistics
+ */
+export interface QueueStats {
+	/** Total tasks added since creation */
+	added: number;
+	/** Total tasks processed (started) */
+	processed: number;
+	/** Total successful completions */
+	completed: number;
+	/** Total failures */
+	failed: number;
+	/** Total timeouts */
+	timedOut: number;
+	/** Total cancellations */
+	cancelled: number;
+	/** Average task duration (ms) */
+	averageDuration: number;
+	/** Tasks per second (rolling average) */
+	throughput: number;
+}
+/**
+ * Queue event types
+ */
+export interface QueueEvents {
+	/** Task added to queue */
+	add: {
+		id: string;
+		priority: number;
+	};
+	/** Task started executing */
+	start: {
+		id: string;
+	};
+	/** Task completed successfully */
+	completed: {
+		id: string;
+		result: any;
+		duration: number;
+	};
+	/** Task failed with error */
+	error: {
+		id: string;
+		error: Error;
+	};
+	/** Task timed out */
+	timeout: {
+		id: string;
+	};
+	/** Task cancelled */
+	cancelled: {
+		id: string;
+	};
+	/** Queue became active (was idle, now processing) */
+	active: undefined;
+	/** Queue became idle (all tasks done) */
+	idle: undefined;
+	/** Queue was paused */
+	paused: undefined;
+	/** Queue was resumed */
+	resumed: undefined;
+	/** Next task about to run */
+	next: undefined;
+	/** Queue was emptied (no pending tasks) */
+	empty: undefined;
+}
+/**
+ * Event handler type
+ */
+export type EventHandler<T> = (data: T) => void;
+/**
+ * Task function type
+ */
+export type TaskFunction<T> = () => Promise<T>;
+declare class RezoQueue<T = any> {
+	private queue;
+	private pendingCount;
+	private isPausedFlag;
+	private intervalId?;
+	private intervalCount;
+	private intervalStart;
+	private eventHandlers;
+	private statsData;
+	private totalDuration;
+	private throughputWindow;
+	private readonly throughputWindowSize;
+	private idlePromise?;
+	private emptyPromise?;
+	readonly config: Required<QueueConfig>;
+	/**
+	 * Create a new RezoQueue
+	 * @param config - Queue configuration options
+	 */
+	constructor(config?: QueueConfig);
+	/**
+	 * Get current queue state
+	 */
+	get state(): QueueState;
+	/**
+	 * Get queue statistics
+	 */
+	get stats(): QueueStats;
+	/**
+	 * Get/set concurrency limit
+	 */
+	get concurrency(): number;
+	set concurrency(value: number);
+	/**
+	 * Number of pending (running) tasks
+	 */
+	get pending(): number;
+	/**
+	 * Number of tasks waiting in queue
+	 */
+	get size(): number;
+	/**
+	 * Check if queue is paused
+	 */
+	get isPaused(): boolean;
+	/**
+	 * Add a task to the queue
+	 * @param fn - Async function to execute
+	 * @param options - Task options
+	 * @returns Promise resolving to task result
+	 */
+	add<R = T>(fn: TaskFunction<R>, options?: TaskOptions): Promise<R>;
+	/**
+	 * Add multiple tasks to the queue
+	 * @param fns - Array of async functions
+	 * @param options - Task options (applied to all)
+	 * @returns Promise resolving to array of results
+	 */
+	addAll<R = T>(fns: TaskFunction<R>[], options?: TaskOptions): Promise<R[]>;
+	/**
+	 * Pause queue processing (running tasks continue)
+	 */
+	pause(): void;
+	/**
+	 * Resume queue processing
+	 */
+	start(): void;
+	/**
+	 * Clear all pending tasks from queue
+	 */
+	clear(): void;
+	/**
+	 * Cancel a specific task by ID
+	 * @param id - Task ID to cancel
+	 * @returns true if task was found and cancelled
+	 */
+	cancel(id: string): boolean;
+	/**
+	 * Cancel all tasks matching a predicate
+	 * @param predicate - Function to test each task
+	 * @returns Number of tasks cancelled
+	 */
+	cancelBy(predicate: (task: {
+		id: string;
+		priority: number;
+	}) => boolean): number;
+	/**
+	 * Wait for queue to become idle (no running or pending tasks)
+	 */
+	onIdle(): Promise<void>;
+	/**
+	 * Wait for queue to be empty (no pending tasks, but may have running)
+	 */
+	onEmpty(): Promise<void>;
+	/**
+	 * Wait for queue size to be less than limit
+	 * @param limit - Size threshold
+	 */
+	onSizeLessThan(limit: number): Promise<void>;
+	/**
+	 * Register an event handler
+	 * @param event - Event name
+	 * @param handler - Handler function
+	 */
+	on<E extends keyof QueueEvents>(event: E, handler: EventHandler<QueueEvents[E]>): void;
+	/**
+	 * Remove an event handler
+	 * @param event - Event name
+	 * @param handler - Handler function to remove
+	 */
+	off<E extends keyof QueueEvents>(event: E, handler: EventHandler<QueueEvents[E]>): void;
+	/**
+	 * Destroy the queue and cleanup resources
+	 */
+	destroy(): void;
+	/**
+	 * Insert task into queue maintaining priority order (highest first)
+	 */
+	private insertByPriority;
+	/**
+	 * Try to run next task if capacity available
+	 */
+	private tryRunNext;
+	/**
+	 * Execute a task
+	 */
+	private runTask;
+	/**
+	 * Record task duration for statistics
+	 */
+	private recordDuration;
+	/**
+	 * Start interval-based rate limiting
+	 */
+	private startInterval;
+	/**
+	 * Emit an event
+	 */
+	protected emit<E extends keyof QueueEvents>(event: E, data: QueueEvents[E]): void;
+	/**
+	 * Check if queue became empty
+	 */
+	private checkEmpty;
+	/**
+	 * Check if queue became idle
+	 */
+	private checkIdle;
+}
 type ProxyProtocol$1 = "http" | "https" | "socks4" | "socks5";
 /**
  * Configuration options for proxy connections
@@ -2222,7 +2537,7 @@ export interface RezoRequestConfig<D = any> {
 	/**
 	 * Queue to use for request execution
 	 */
-	queue?: PQueue | null;
+	queue?: RezoQueue | null;
 	/**
 	 * Controls how the response body is parsed and returned in `response.data`.
 	 *
@@ -2312,10 +2627,6 @@ export interface RezoRequestConfig<D = any> {
 	autoSetOrigin?: boolean;
 	treat302As303?: boolean;
 	startNewRequest?: boolean;
-	/** Whether to use HTTP/2 protocol */
-	http2?: boolean;
-	/** Whether to use cURL adapter */
-	curl?: boolean;
 	/**
 	 * DNS cache configuration for faster repeated requests.
 	 *
@@ -2419,6 +2730,12 @@ export interface RezoRequestConfig<D = any> {
 	withCredentials?: boolean;
 	/** Proxy configuration (URL string or detailed options) */
 	proxy?: string | ProxyOptions;
+	/**
+	 * Whether to use ProxyManager for this request
+	 * Set to false to bypass ProxyManager even when one is configured
+	 * @default true (uses ProxyManager if configured)
+	 */
+	useProxyManager?: boolean;
 	/** Whether to enable automatic cookie handling */
 	useCookies?: boolean;
 	/** Custom cookie jar for managing cookies */
@@ -2440,8 +2757,6 @@ export interface RezoRequestConfig<D = any> {
 	transformRequest?: Array<(data: any, headers: RezoHeaders) => any>;
 	/** Array of functions to transform response data */
 	transformResponse?: Array<(data: any) => any>;
-	/** Adapter to use for the request (name or custom function) */
-	adapter?: string | ((config: RezoRequestConfig) => Promise<any>);
 	/** AbortSignal to cancel the request */
 	signal?: AbortSignal;
 	/** File path to save the response to (for downloads) */
@@ -2658,6 +2973,7 @@ type BeforeProxyDisableHook$1 = (context: BeforeProxyDisableContext) => boolean
 type AfterProxyDisableHook$1 = (context: AfterProxyDisableContext) => void | Promise<void>;
 type AfterProxyRotateHook$1 = (context: AfterProxyRotateContext) => void | Promise<void>;
 type AfterProxyEnableHook$1 = (context: AfterProxyEnableContext) => void | Promise<void>;
+type OnNoProxiesAvailableHook$1 = (context: OnNoProxiesAvailableContext) => void | Promise<void>;
 /**
  * Proxy hooks collection for ProxyManager events
  */
@@ -2670,6 +2986,8 @@ export interface ProxyHooks {
 	afterProxyDisable: AfterProxyDisableHook$1[];
 	afterProxyRotate: AfterProxyRotateHook$1[];
 	afterProxyEnable: AfterProxyEnableHook$1[];
+	/** Hook triggered when no proxies are available */
+	onNoProxiesAvailable: OnNoProxiesAvailableHook$1[];
 }
 declare class ProxyManager {
 	/** Configuration for the proxy manager */
@@ -2805,8 +3123,40 @@ declare class ProxyManager {
 	private runAfterProxyRotateHooks;
 	private runAfterProxyDisableHooks;
 	private runAfterProxyEnableHooks;
+	/**
+	 * Run onNoProxiesAvailable hooks synchronously
+	 * Called when no proxies are available and an error is about to be thrown
+	 */
+	private runOnNoProxiesAvailableHooksSync;
+	/**
+	 * Run onNoProxiesAvailable hooks asynchronously
+	 * Called when no proxies are available and an error is about to be thrown
+	 */
+	runOnNoProxiesAvailableHooks(context: OnNoProxiesAvailableContext): Promise<void>;
+	/**
+	 * Notify that no proxies are available and trigger hooks
+	 * This method is called when proxy selection fails due to pool exhaustion
+	 *
+	 * @param url - The request URL that needed a proxy
+	 * @param error - The error that will be thrown
+	 * @returns The context object with detailed information about the proxy pool state
+	 *
+	 * @example
+	 * ```typescript
+	 * manager.hooks.onNoProxiesAvailable.push((context) => {
+	 *     console.error(`No proxies available for ${context.url}`);
+	 *     console.log(`Dead: ${context.disabledReasons.dead}, Limit: ${context.disabledReasons.limitReached}`);
+	 *     // Trigger external alert or proxy refresh
+	 *     alertSystem.notify('Proxy pool exhausted', context);
+	 * });
+	 *
+	 * // Called internally or by adapters when no proxies are available
+	 * const context = manager.notifyNoProxiesAvailable('https://api.example.com', new Error('No proxies'));
+	 * ```
+	 */
+	notifyNoProxiesAvailable(url: string, error: Error): OnNoProxiesAvailableContext;
 }
-export type queueOptions = Options$1<PriorityQueue, QueueAddOptions>;
+export type queueOptions = QueueConfig;
 export interface CacheConfig {
 	/** Response cache configuration */
 	response?: boolean | ResponseCacheConfig;
@@ -2865,10 +3215,6 @@ export interface RezoDefaultOptions {
 	keepAlive?: boolean;
 	/** Whether to detect and prevent redirect cycles */
 	enableRedirectCycleDetection?: boolean;
-	/** Whether to use HTTP/2 protocol */
-	http2?: boolean;
-	/** Whether to use cURL adapter */
-	curl?: boolean;
 	/** Whether to send cookies and authorization headers with cross-origin requests */
 	withCredentials?: boolean;
 	/** Proxy configuration (URL string or detailed options) */
@@ -2907,8 +3253,6 @@ export interface RezoDefaultOptions {
 	transformRequest?: RezoHttpRequest["transformRequest"];
 	/** Array of functions to transform response data */
 	transformResponse?: RezoHttpRequest["transformResponse"];
-	/** Adapter to use for the request (name or custom function) */
-	adapter?: RezoHttpRequest["adapter"];
 	/** Browser simulation configuration for user agent spoofing */
 	browser?: RezoHttpRequest["browser"];
 	/** Enable debug logging for the request */
@@ -3808,7 +4152,7 @@ export interface httpAdapterPutOverloads {
  */
 export type AdapterFunction<T = any> = (options: RezoRequestConfig, defaultOptions: RezoDefaultOptions, jar: RezoCookieJar) => Promise<RezoResponse<T> | RezoStreamResponse | RezoDownloadResponse | RezoUploadResponse>;
 declare class Rezo {
-	protected queue: PQueue | null;
+	protected queue: RezoQueue | null;
 	protected isQueueEnabled: boolean;
 	defaults: RezoDefaultOptions;
 	hooks: RezoHooks;
@@ -6041,7 +6385,7 @@ export declare class CrawlerOptions {
 	/**
 	 * Internal method to process and add rate limiter configurations
 	 * @param options - Limiter configuration object with enable flag and queue options
-	 * @description Validates and stores rate limiter configurations, creating PQueue instances
+	 * @description Validates and stores rate limiter configurations, creating RezoQueue instances
 	 * for each valid configuration. Supports domain-specific or global rate limiting.
 	 * @private
 	 */
@@ -6205,7 +6549,7 @@ export declare class CrawlerOptions {
 	 * ```
 	 */
 	getAdapter(url: string, type: "proxies", useGlobal?: boolean, random?: boolean): IProxy | null;
-	getAdapter(url: string, type: "limiters", useGlobal?: boolean, random?: boolean): PQueue | null;
+	getAdapter(url: string, type: "limiters", useGlobal?: boolean, random?: boolean): RezoQueue | null;
 	getAdapter(url: string, type: "oxylabs", useGlobal?: boolean, random?: boolean): Oxylabs | null;
 	getAdapter(url: string, type: "decodo", useGlobal?: boolean, random?: boolean): Decodo | null;
 	getAdapter(url: string, type: "headers", useGlobal?: boolean, random?: boolean): OutgoingHttpHeaders | null;

package/dist/entries/crawler.cjs

@@ -1,5 +1,5 @@
-const _mod_44yuyi = require('../plugin/crawler.cjs');
-exports.Crawler = _mod_44yuyi.Crawler;;
-const _mod_1br1ni = require('../plugin/crawler-options.cjs');
-exports.CrawlerOptions = _mod_1br1ni.CrawlerOptions;
-exports.Domain = _mod_1br1ni.Domain;;
+const _mod_juod8q = require('../plugin/crawler.cjs');
+exports.Crawler = _mod_juod8q.Crawler;;
+const _mod_lugvyb = require('../plugin/crawler-options.cjs');
+exports.CrawlerOptions = _mod_lugvyb.CrawlerOptions;
+exports.Domain = _mod_lugvyb.Domain;;

package/dist/index.cjs

@@ -1,21 +1,26 @@
-const _mod_s7u8s1 = require('./core/rezo.cjs');
-exports.Rezo = _mod_s7u8s1.Rezo;
-exports.createRezoInstance = _mod_s7u8s1.createRezoInstance;
-exports.createDefaultInstance = _mod_s7u8s1.createDefaultInstance;;
-const _mod_bb2vwi = require('./errors/rezo-error.cjs');
-exports.RezoError = _mod_bb2vwi.RezoError;
-exports.RezoErrorCode = _mod_bb2vwi.RezoErrorCode;;
-const _mod_1ja95d = require('./utils/headers.cjs');
-exports.RezoHeaders = _mod_1ja95d.RezoHeaders;;
-const _mod_5xfrse = require('./utils/form-data.cjs');
-exports.RezoFormData = _mod_5xfrse.RezoFormData;;
-const _mod_xg8slw = require('./utils/cookies.cjs');
-exports.RezoCookieJar = _mod_xg8slw.RezoCookieJar;;
-const _mod_rdxy2i = require('./core/hooks.cjs');
-exports.createDefaultHooks = _mod_rdxy2i.createDefaultHooks;
-exports.mergeHooks = _mod_rdxy2i.mergeHooks;;
-const _mod_lv3eam = require('./proxy/manager.cjs');
-exports.ProxyManager = _mod_lv3eam.ProxyManager;;
+const _mod_8g1zwb = require('./core/rezo.cjs');
+exports.Rezo = _mod_8g1zwb.Rezo;
+exports.createRezoInstance = _mod_8g1zwb.createRezoInstance;
+exports.createDefaultInstance = _mod_8g1zwb.createDefaultInstance;;
+const _mod_brpqyx = require('./errors/rezo-error.cjs');
+exports.RezoError = _mod_brpqyx.RezoError;
+exports.RezoErrorCode = _mod_brpqyx.RezoErrorCode;;
+const _mod_nxs9mv = require('./utils/headers.cjs');
+exports.RezoHeaders = _mod_nxs9mv.RezoHeaders;;
+const _mod_sp17k6 = require('./utils/form-data.cjs');
+exports.RezoFormData = _mod_sp17k6.RezoFormData;;
+const _mod_rt2h0v = require('./utils/cookies.cjs');
+exports.RezoCookieJar = _mod_rt2h0v.RezoCookieJar;;
+const _mod_pg0wwn = require('./core/hooks.cjs');
+exports.createDefaultHooks = _mod_pg0wwn.createDefaultHooks;
+exports.mergeHooks = _mod_pg0wwn.mergeHooks;;
+const _mod_qipnt4 = require('./proxy/manager.cjs');
+exports.ProxyManager = _mod_qipnt4.ProxyManager;;
+const _mod_z8k198 = require('./queue/index.cjs');
+exports.RezoQueue = _mod_z8k198.RezoQueue;
+exports.HttpQueue = _mod_z8k198.HttpQueue;
+exports.Priority = _mod_z8k198.Priority;
+exports.HttpMethodPriority = _mod_z8k198.HttpMethodPriority;;
 const { RezoError } = require('./errors/rezo-error.cjs');
 const isRezoError = exports.isRezoError = RezoError.isRezoError;
 const Cancel = exports.Cancel = RezoError;