rezo 1.0.43 → 1.0.45

package/dist/adapters/entries/xhr.d.ts

@@ -1416,6 +1416,35 @@ export type OnTimeoutHook = (event: TimeoutEvent, config: RezoConfig) => void;
  * Use for cleanup, logging
  */
 export type OnAbortHook = (event: AbortEvent, config: RezoConfig) => void;
+/**
+ * Rate limit wait event data - fired when waiting due to rate limiting
+ */
+export interface RateLimitWaitEvent {
+	/** HTTP status code that triggered the wait (e.g., 429, 503) */
+	status: number;
+	/** Time to wait in milliseconds */
+	waitTime: number;
+	/** Current wait attempt number (1-indexed) */
+	attempt: number;
+	/** Maximum wait attempts configured */
+	maxAttempts: number;
+	/** Where the wait time was extracted from */
+	source: "header" | "body" | "function" | "default";
+	/** The header or body path used (if applicable) */
+	sourcePath?: string;
+	/** URL being requested */
+	url: string;
+	/** HTTP method of the request */
+	method: string;
+	/** Timestamp when the wait started */
+	timestamp: number;
+}
+/**
+ * Hook called when rate limit wait occurs
+ * Informational only - cannot abort the wait
+ * Use for logging, monitoring, alerting
+ */
+export type OnRateLimitWaitHook = (event: RateLimitWaitEvent, config: RezoConfig) => void | Promise<void>;
 /**
  * Hook called before a proxy is selected
  * Can return a specific proxy to override selection
@@ -1496,6 +1525,7 @@ export interface RezoHooks {
 	onTls: OnTlsHook[];
 	onTimeout: OnTimeoutHook[];
 	onAbort: OnAbortHook[];
+	onRateLimitWait: OnRateLimitWaitHook[];
 }
 /**
  * Create empty hooks object with all arrays initialized
@@ -2426,6 +2456,91 @@ export interface RezoRequestConfig<D = any> {
 		/** Weather to stop or continue retry when certain condition is met*/
 		condition?: (error: RezoError) => boolean | Promise<boolean>;
 	};
+	/**
+	 * Rate limit wait configuration - wait and retry when receiving rate limit responses.
+	 *
+	 * This feature runs BEFORE the retry system. When a rate-limiting status code is received,
+	 * the client will wait for the specified time and automatically retry the request.
+	 *
+	 * **Basic Usage:**
+	 * - `waitOnStatus: true` - Enable waiting on 429 status (default behavior)
+	 * - `waitOnStatus: [429, 503]` - Enable waiting on specific status codes
+	 *
+	 * **Wait Time Sources:**
+	 * - `'retry-after'` - Use standard Retry-After header (default)
+	 * - `{ header: 'X-RateLimit-Reset' }` - Use custom header
+	 * - `{ body: 'retry_after' }` - Extract from JSON response body
+	 * - Custom function for complex logic
+	 *
+	 * @example
+	 * ```typescript
+	 * // Wait on 429 using Retry-After header
+	 * await rezo.get(url, { waitOnStatus: true });
+	 *
+	 * // Wait on 429 using custom header
+	 * await rezo.get(url, {
+	 *   waitOnStatus: true,
+	 *   waitTimeSource: { header: 'X-RateLimit-Reset' }
+	 * });
+	 *
+	 * // Wait on 429 extracting time from JSON body
+	 * await rezo.get(url, {
+	 *   waitOnStatus: true,
+	 *   waitTimeSource: { body: 'data.retry_after' }
+	 * });
+	 *
+	 * // Custom function for complex APIs
+	 * await rezo.get(url, {
+	 *   waitOnStatus: [429, 503],
+	 *   waitTimeSource: (response) => {
+	 *     const reset = response.headers.get('x-ratelimit-reset');
+	 *     return reset ? parseInt(reset) - Math.floor(Date.now() / 1000) : null;
+	 *   }
+	 * });
+	 * ```
+	 */
+	waitOnStatus?: boolean | number[];
+	/**
+	 * Where to extract the wait time from when rate-limited.
+	 *
+	 * - `'retry-after'` - Standard Retry-After header (default)
+	 * - `{ header: string }` - Custom header name (e.g., 'X-RateLimit-Reset')
+	 * - `{ body: string }` - JSON path in response body (e.g., 'data.retry_after', 'wait_seconds')
+	 * - Function - Custom logic receiving the response, return seconds to wait or null
+	 *
+	 * @default 'retry-after'
+	 */
+	waitTimeSource?: "retry-after" | {
+		header: string;
+	} | {
+		body: string;
+	} | ((response: {
+		status: number;
+		headers: RezoHeaders;
+		data?: any;
+	}) => number | null);
+	/**
+	 * Maximum time to wait for rate limit in milliseconds.
+	 * If the extracted wait time exceeds this, the request will fail instead of waiting.
+	 * Set to 0 for unlimited wait time.
+	 *
+	 * @default 60000 (60 seconds)
+	 */
+	maxWaitTime?: number;
+	/**
+	 * Default wait time in milliseconds if the wait time source returns nothing.
+	 * Used as fallback when Retry-After header or body path is not present.
+	 *
+	 * @default 1000 (1 second)
+	 */
+	defaultWaitTime?: number;
+	/**
+	 * Maximum number of wait attempts before giving up.
+	 * After this many waits, the request will proceed to retry logic or fail.
+	 *
+	 * @default 3
+	 */
+	maxWaitAttempts?: number;
 	/** Whether to use a secure context for HTTPS requests */
 	useSecureContext?: boolean;
 	/** Custom secure context for TLS connections */

package/dist/adapters/fetch.cjs

@@ -10,6 +10,7 @@ const { DownloadResponse } = require('../responses/universal/download.cjs');
 const { UploadResponse } = require('../responses/universal/upload.cjs');
 const { isSameDomain, RezoPerformance } = require('../utils/tools.cjs');
 const { ResponseCache } = require('../cache/universal-response-cache.cjs');
+const { handleRateLimitWait, shouldWaitOnStatus } = require('../utils/rate-limit-wait.cjs');
 const Environment = {
   isNode: typeof process !== "undefined" && process.versions?.node,
   isBrowser: typeof window !== "undefined" && typeof document !== "undefined",
@@ -555,6 +556,23 @@ async function executeFetchRequest(fetchOptions, config, options, perform, strea
         return response;
       }
       if (statusOnNext === "error") {
+        if (shouldWaitOnStatus(response.status, options.waitOnStatus)) {
+          const rateLimitWaitAttempt = config._rateLimitWaitAttempt || 0;
+          const waitResult = await handleRateLimitWait({
+            status: response.status,
+            headers: response.headers,
+            data: response.data,
+            url: fetchOptions.fullUrl || fetchOptions.url?.toString() || "",
+            method: fetchOptions.method || "GET",
+            config,
+            options,
+            currentWaitAttempt: rateLimitWaitAttempt
+          });
+          if (waitResult.shouldRetry) {
+            config._rateLimitWaitAttempt = waitResult.waitAttempt;
+            continue;
+          }
+        }
         const httpError = builErrorFromResponse(`Request failed with status code ${response.status}`, response, config, fetchOptions);
         if (config.retry && statusCodes?.includes(response.status)) {
           if (maxRetries > retries) {

package/dist/adapters/fetch.js

@@ -10,6 +10,7 @@ import { DownloadResponse } from '../responses/universal/download.js';
 import { UploadResponse } from '../responses/universal/upload.js';
 import { isSameDomain, RezoPerformance } from '../utils/tools.js';
 import { ResponseCache } from '../cache/universal-response-cache.js';
+import { handleRateLimitWait, shouldWaitOnStatus } from '../utils/rate-limit-wait.js';
 const Environment = {
   isNode: typeof process !== "undefined" && process.versions?.node,
   isBrowser: typeof window !== "undefined" && typeof document !== "undefined",
@@ -555,6 +556,23 @@ async function executeFetchRequest(fetchOptions, config, options, perform, strea
         return response;
       }
       if (statusOnNext === "error") {
+        if (shouldWaitOnStatus(response.status, options.waitOnStatus)) {
+          const rateLimitWaitAttempt = config._rateLimitWaitAttempt || 0;
+          const waitResult = await handleRateLimitWait({
+            status: response.status,
+            headers: response.headers,
+            data: response.data,
+            url: fetchOptions.fullUrl || fetchOptions.url?.toString() || "",
+            method: fetchOptions.method || "GET",
+            config,
+            options,
+            currentWaitAttempt: rateLimitWaitAttempt
+          });
+          if (waitResult.shouldRetry) {
+            config._rateLimitWaitAttempt = waitResult.waitAttempt;
+            continue;
+          }
+        }
         const httpError = builErrorFromResponse(`Request failed with status code ${response.status}`, response, config, fetchOptions);
         if (config.retry && statusCodes?.includes(response.status)) {
           if (maxRetries > retries) {

package/dist/adapters/http.cjs

@@ -21,6 +21,7 @@ const { getGlobalDNSCache } = require('../cache/dns-cache.cjs');
 const { ResponseCache } = require('../cache/response-cache.cjs');
 const { getGlobalAgentPool } = require('../utils/agent-pool.cjs');
 const { StagedTimeoutManager, parseStagedTimeouts } = require('../utils/staged-timeout.cjs');
+const { handleRateLimitWait, shouldWaitOnStatus } = require('../utils/rate-limit-wait.cjs');
 const dns = require("node:dns");
 const debugLog = {
   requestStart: (config, url, method) => {
@@ -604,6 +605,23 @@ async function executeHttp1Request(fetchOptions, config, options, perform, fs, s
         continue;
       }
       if (statusOnNext === "error") {
+        if (shouldWaitOnStatus(response.status, options.waitOnStatus)) {
+          const rateLimitWaitAttempt = config._rateLimitWaitAttempt || 0;
+          const waitResult = await handleRateLimitWait({
+            status: response.status,
+            headers: response.headers,
+            data: response.data,
+            url: fetchOptions.fullUrl || fetchOptions.url?.toString() || "",
+            method: fetchOptions.method || "GET",
+            config,
+            options,
+            currentWaitAttempt: rateLimitWaitAttempt
+          });
+          if (waitResult.shouldRetry) {
+            config._rateLimitWaitAttempt = waitResult.waitAttempt;
+            continue;
+          }
+        }
         const httpError = builErrorFromResponse(`Request failed with status code ${response.status}`, response, config, fetchOptions);
         if (config.retry && statusCodes?.includes(response.status)) {
           if (maxRetries > retries) {

package/dist/adapters/http.js

@@ -21,6 +21,7 @@ import { getGlobalDNSCache } from '../cache/dns-cache.js';
 import { ResponseCache } from '../cache/response-cache.js';
 import { getGlobalAgentPool } from '../utils/agent-pool.js';
 import { StagedTimeoutManager, parseStagedTimeouts } from '../utils/staged-timeout.js';
+import { handleRateLimitWait, shouldWaitOnStatus } from '../utils/rate-limit-wait.js';
 import dns from "node:dns";
 const debugLog = {
   requestStart: (config, url, method) => {
@@ -604,6 +605,23 @@ async function executeHttp1Request(fetchOptions, config, options, perform, fs, s
         continue;
       }
       if (statusOnNext === "error") {
+        if (shouldWaitOnStatus(response.status, options.waitOnStatus)) {
+          const rateLimitWaitAttempt = config._rateLimitWaitAttempt || 0;
+          const waitResult = await handleRateLimitWait({
+            status: response.status,
+            headers: response.headers,
+            data: response.data,
+            url: fetchOptions.fullUrl || fetchOptions.url?.toString() || "",
+            method: fetchOptions.method || "GET",
+            config,
+            options,
+            currentWaitAttempt: rateLimitWaitAttempt
+          });
+          if (waitResult.shouldRetry) {
+            config._rateLimitWaitAttempt = waitResult.waitAttempt;
+            continue;
+          }
+        }
         const httpError = builErrorFromResponse(`Request failed with status code ${response.status}`, response, config, fetchOptions);
         if (config.retry && statusCodes?.includes(response.status)) {
           if (maxRetries > retries) {

package/dist/adapters/http2.cjs

@@ -18,6 +18,7 @@ const { isSameDomain, RezoPerformance } = require('../utils/tools.cjs');
 const { SocksClient } = require('../internal/agents/socks-client.cjs');
 const net = require("node:net");
 const { ResponseCache } = require('../cache/response-cache.cjs');
+const { handleRateLimitWait, shouldWaitOnStatus } = require('../utils/rate-limit-wait.cjs');
 let zstdDecompressSync = null;
 let zstdChecked = false;
 const debugLog = {
@@ -960,6 +961,23 @@ async function executeHttp2Request(fetchOptions, config, options, perform, fs, s
         return response;
       }
       if (statusOnNext === "error") {
+        if (shouldWaitOnStatus(response.status, options.waitOnStatus)) {
+          const rateLimitWaitAttempt = config._rateLimitWaitAttempt || 0;
+          const waitResult = await handleRateLimitWait({
+            status: response.status,
+            headers: response.headers,
+            data: response.data,
+            url: fetchOptions.fullUrl || fetchOptions.url?.toString() || "",
+            method: fetchOptions.method || "GET",
+            config,
+            options,
+            currentWaitAttempt: rateLimitWaitAttempt
+          });
+          if (waitResult.shouldRetry) {
+            config._rateLimitWaitAttempt = waitResult.waitAttempt;
+            continue;
+          }
+        }
         const httpError = builErrorFromResponse(`Request failed with status code ${response.status}`, response, config, fetchOptions);
         if (config.retry && statusCodes?.includes(response.status)) {
           if (maxRetries > retries) {

package/dist/adapters/http2.js

@@ -18,6 +18,7 @@ import { isSameDomain, RezoPerformance } from '../utils/tools.js';
 import { SocksClient } from '../internal/agents/socks-client.js';
 import * as net from "node:net";
 import { ResponseCache } from '../cache/response-cache.js';
+import { handleRateLimitWait, shouldWaitOnStatus } from '../utils/rate-limit-wait.js';
 let zstdDecompressSync = null;
 let zstdChecked = false;
 const debugLog = {
@@ -960,6 +961,23 @@ async function executeHttp2Request(fetchOptions, config, options, perform, fs, s
         return response;
       }
       if (statusOnNext === "error") {
+        if (shouldWaitOnStatus(response.status, options.waitOnStatus)) {
+          const rateLimitWaitAttempt = config._rateLimitWaitAttempt || 0;
+          const waitResult = await handleRateLimitWait({
+            status: response.status,
+            headers: response.headers,
+            data: response.data,
+            url: fetchOptions.fullUrl || fetchOptions.url?.toString() || "",
+            method: fetchOptions.method || "GET",
+            config,
+            options,
+            currentWaitAttempt: rateLimitWaitAttempt
+          });
+          if (waitResult.shouldRetry) {
+            config._rateLimitWaitAttempt = waitResult.waitAttempt;
+            continue;
+          }
+        }
         const httpError = builErrorFromResponse(`Request failed with status code ${response.status}`, response, config, fetchOptions);
         if (config.retry && statusCodes?.includes(response.status)) {
           if (maxRetries > retries) {

package/dist/adapters/index.cjs

@@ -1,6 +1,6 @@
-const _mod_63iyz4 = require('./picker.cjs');
-exports.detectRuntime = _mod_63iyz4.detectRuntime;
-exports.getAdapterCapabilities = _mod_63iyz4.getAdapterCapabilities;
-exports.buildAdapterContext = _mod_63iyz4.buildAdapterContext;
-exports.getAvailableAdapters = _mod_63iyz4.getAvailableAdapters;
-exports.selectAdapter = _mod_63iyz4.selectAdapter;;
+const _mod_jrw0vw = require('./picker.cjs');
+exports.detectRuntime = _mod_jrw0vw.detectRuntime;
+exports.getAdapterCapabilities = _mod_jrw0vw.getAdapterCapabilities;
+exports.buildAdapterContext = _mod_jrw0vw.buildAdapterContext;
+exports.getAvailableAdapters = _mod_jrw0vw.getAvailableAdapters;
+exports.selectAdapter = _mod_jrw0vw.selectAdapter;;

package/dist/adapters/xhr.cjs

@@ -10,6 +10,7 @@ const { DownloadResponse } = require('../responses/universal/download.cjs');
 const { UploadResponse } = require('../responses/universal/upload.cjs');
 const { RezoPerformance } = require('../utils/tools.cjs');
 const { ResponseCache } = require('../cache/universal-response-cache.cjs');
+const { handleRateLimitWait, shouldWaitOnStatus } = require('../utils/rate-limit-wait.cjs');
 const Environment = {
   isBrowser: typeof window !== "undefined" && typeof document !== "undefined",
   hasXHR: typeof XMLHttpRequest !== "undefined",
@@ -390,6 +391,24 @@ async function executeXHRRequest(fetchOptions, config, options, perform, streamR
     try {
       const response = await executeSingleXHRRequest(config, fetchOptions, timing, streamResult, downloadResult, uploadResult);
       if (response instanceof RezoError) {
+        const errorStatus = response.status || 0;
+        if (shouldWaitOnStatus(errorStatus, options.waitOnStatus)) {
+          const rateLimitWaitAttempt = config._rateLimitWaitAttempt || 0;
+          const waitResult = await handleRateLimitWait({
+            status: errorStatus,
+            headers: response.response?.headers || new RezoHeaders,
+            data: response.response?.data,
+            url: fetchOptions.fullUrl || fetchOptions.url?.toString() || "",
+            method: fetchOptions.method || "GET",
+            config,
+            options,
+            currentWaitAttempt: rateLimitWaitAttempt
+          });
+          if (waitResult.shouldRetry) {
+            config._rateLimitWaitAttempt = waitResult.waitAttempt;
+            continue;
+          }
+        }
         config.errors.push({
           attempt: config.retryAttempts + 1,
           error: response,

package/dist/adapters/xhr.js

@@ -10,6 +10,7 @@ import { DownloadResponse } from '../responses/universal/download.js';
 import { UploadResponse } from '../responses/universal/upload.js';
 import { RezoPerformance } from '../utils/tools.js';
 import { ResponseCache } from '../cache/universal-response-cache.js';
+import { handleRateLimitWait, shouldWaitOnStatus } from '../utils/rate-limit-wait.js';
 const Environment = {
   isBrowser: typeof window !== "undefined" && typeof document !== "undefined",
   hasXHR: typeof XMLHttpRequest !== "undefined",
@@ -390,6 +391,24 @@ async function executeXHRRequest(fetchOptions, config, options, perform, streamR
     try {
       const response = await executeSingleXHRRequest(config, fetchOptions, timing, streamResult, downloadResult, uploadResult);
       if (response instanceof RezoError) {
+        const errorStatus = response.status || 0;
+        if (shouldWaitOnStatus(errorStatus, options.waitOnStatus)) {
+          const rateLimitWaitAttempt = config._rateLimitWaitAttempt || 0;
+          const waitResult = await handleRateLimitWait({
+            status: errorStatus,
+            headers: response.response?.headers || new RezoHeaders,
+            data: response.response?.data,
+            url: fetchOptions.fullUrl || fetchOptions.url?.toString() || "",
+            method: fetchOptions.method || "GET",
+            config,
+            options,
+            currentWaitAttempt: rateLimitWaitAttempt
+          });
+          if (waitResult.shouldRetry) {
+            config._rateLimitWaitAttempt = waitResult.waitAttempt;
+            continue;
+          }
+        }
         config.errors.push({
           attempt: config.retryAttempts + 1,
           error: response,

package/dist/cache/index.cjs

@@ -1,15 +1,9 @@
-const _mod_6dj0p0 = require('./lru-cache.cjs');
-exports.LRUCache = _mod_6dj0p0.LRUCache;;
-const _mod_9z4pm1 = require('./dns-cache.cjs');
-exports.DNSCache = _mod_9z4pm1.DNSCache;
-exports.getGlobalDNSCache = _mod_9z4pm1.getGlobalDNSCache;
-exports.resetGlobalDNSCache = _mod_9z4pm1.resetGlobalDNSCache;;
-const _mod_5ylf2b = require('./response-cache.cjs');
-exports.ResponseCache = _mod_5ylf2b.ResponseCache;
-exports.normalizeResponseCacheConfig = _mod_5ylf2b.normalizeResponseCacheConfig;;
-const _mod_fk65c2 = require('./file-cacher.cjs');
-exports.FileCacher = _mod_fk65c2.FileCacher;;
-const _mod_7tzsb5 = require('./url-store.cjs');
-exports.UrlStore = _mod_7tzsb5.UrlStore;;
-const _mod_04c8wb = require('./navigation-history.cjs');
-exports.NavigationHistory = _mod_04c8wb.NavigationHistory;;
+const _mod_0ku4z2 = require('./lru-cache.cjs');
+exports.LRUCache = _mod_0ku4z2.LRUCache;;
+const _mod_odvwch = require('./dns-cache.cjs');
+exports.DNSCache = _mod_odvwch.DNSCache;
+exports.getGlobalDNSCache = _mod_odvwch.getGlobalDNSCache;
+exports.resetGlobalDNSCache = _mod_odvwch.resetGlobalDNSCache;;
+const _mod_b9x6qw = require('./response-cache.cjs');
+exports.ResponseCache = _mod_b9x6qw.ResponseCache;
+exports.normalizeResponseCacheConfig = _mod_b9x6qw.normalizeResponseCacheConfig;;

package/dist/cache/index.js

@@ -1,6 +1,3 @@
 export { LRUCache } from './lru-cache.js';
 export { DNSCache, getGlobalDNSCache, resetGlobalDNSCache } from './dns-cache.js';
 export { ResponseCache, normalizeResponseCacheConfig } from './response-cache.js';
-export { FileCacher } from './file-cacher.js';
-export { UrlStore } from './url-store.js';
-export { NavigationHistory } from './navigation-history.js';

package/dist/core/hooks.cjs

@@ -24,7 +24,8 @@ function createDefaultHooks() {
     onDns: [],
     onTls: [],
     onTimeout: [],
-    onAbort: []
+    onAbort: [],
+    onRateLimitWait: []
   };
 }
 function mergeHooks(base, overrides) {
@@ -55,7 +56,8 @@ function mergeHooks(base, overrides) {
     onDns: [...base.onDns, ...overrides.onDns || []],
     onTls: [...base.onTls, ...overrides.onTls || []],
     onTimeout: [...base.onTimeout, ...overrides.onTimeout || []],
-    onAbort: [...base.onAbort, ...overrides.onAbort || []]
+    onAbort: [...base.onAbort, ...overrides.onAbort || []],
+    onRateLimitWait: [...base.onRateLimitWait, ...overrides.onRateLimitWait || []]
   };
 }
 function serializeHooks(hooks) {

package/dist/core/hooks.js

@@ -24,7 +24,8 @@ export function createDefaultHooks() {
     onDns: [],
     onTls: [],
     onTimeout: [],
-    onAbort: []
+    onAbort: [],
+    onRateLimitWait: []
   };
 }
 export function mergeHooks(base, overrides) {
@@ -55,7 +56,8 @@ export function mergeHooks(base, overrides) {
     onDns: [...base.onDns, ...overrides.onDns || []],
     onTls: [...base.onTls, ...overrides.onTls || []],
     onTimeout: [...base.onTimeout, ...overrides.onTimeout || []],
-    onAbort: [...base.onAbort, ...overrides.onAbort || []]
+    onAbort: [...base.onAbort, ...overrides.onAbort || []],
+    onRateLimitWait: [...base.onRateLimitWait, ...overrides.onRateLimitWait || []]
   };
 }
 export function serializeHooks(hooks) {

package/dist/crawler/addon/decodo/index.cjs

@@ -0,0 +1 @@
+var{Rezo:h}=require("../../../core/rezo.cjs"),r=require("./options.cjs");exports.DECODO_DEVICE_TYPES=r.DECODO_DEVICE_TYPES;exports.DECODO_HEADLESS_MODES=r.DECODO_HEADLESS_MODES;exports.DECODO_COMMON_LOCALES=r.DECODO_COMMON_LOCALES;exports.DECODO_COMMON_COUNTRIES=r.DECODO_COMMON_COUNTRIES;exports.DECODO_EUROPEAN_COUNTRIES=r.DECODO_EUROPEAN_COUNTRIES;exports.DECODO_ASIAN_COUNTRIES=r.DECODO_ASIAN_COUNTRIES;exports.DECODO_US_STATES=r.DECODO_US_STATES;exports.DECODO_COMMON_CITIES=r.DECODO_COMMON_CITIES;exports.getRandomDeviceType=r.getRandomDeviceType;exports.getRandomLocale=r.getRandomLocale;exports.getRandomCountry=r.getRandomCountry;exports.getRandomCity=r.getRandomCity;exports.generateSessionId=r.generateSessionId;var u="https://scraper-api.smartproxy.com/v2/scrape";class d{config;http;authHeader;constructor(e){let t="username"in e&&"password"in e&&e.username&&e.password,s="token"in e&&e.token;if(!t&&!s)throw Error("Decodo requires either username/password or token for authentication");if(this.config={username:e.username??"",password:e.password??"",deviceType:e.deviceType??"desktop",locale:e.locale??"en-US",country:e.country??"",state:e.state??"",city:e.city??"",headless:e.headless??void 0,headers:e.headers??{},sessionId:e.sessionId??"",sessionDuration:e.sessionDuration??0,javascript:e.javascript??"",javascriptWait:e.javascriptWait??0,waitForCss:e.waitForCss??"",timeout:e.timeout??120000},this.http=new h({baseURL:u,timeout:this.config.timeout}),s)this.authHeader=`Basic ${e.token}`;else this.authHeader=`Basic ${Buffer.from(`${e.username}:${e.password}`).toString("base64")}`}async scrape(e,t){let s={...this.config,...t,headers:{...this.config.headers,...t?.headers||{}}},n=this.buildRequestBody(e,s),a=(await this.http.postJson(u,n,{headers:{Authorization:this.authHeader,"Content-Type":"application/json"}})).data;if(a.error)throw Error(`Decodo API error: ${a.error} (${a.error_code||"unknown"})`);if(!a.results||a.results.length===0)throw Error("Decodo API returned no results");let o=a.results[0];return{statusCode:o.status_code,url:o.url,content:o.body,cookies:o.cookies||[],headers:o.headers||{},taskId:a.id,rendered:!!s.headless,country:s.country||void 0,city:s.city||void 0,state:s.state||void 0,deviceType:s.deviceType,contentType:o.content_type,contentLength:o.content_length,raw:a}}async scrapeMany(e,t,s=1000){let n=[];for(let i=0;i<e.length;i++){let a=await this.scrape(e[i],t);if(n.push(a),i<e.length-1&&s>0)await new Promise((o)=>setTimeout(o,s))}return n}async scrapeWithSession(e,t,s=10){let n=`session_${Date.now()}_${Math.random().toString(36).substring(2,11)}`;return this.scrapeMany(e,{...t,sessionId:n,sessionDuration:s},500)}buildRequestBody(e,t){let s={url:e,return_cookies:!0,return_headers:!0};if(t.deviceType)s.device_type=t.deviceType;if(t.headless)s.headless=t.headless;if(t.locale)s.locale=t.locale;if(t.country)s.country=t.country;if(t.state)s.state=t.state;if(t.city)s.city=t.city;if(t.sessionId){if(s.session=t.sessionId,t.sessionDuration)s.session_duration=t.sessionDuration}if(t.headers&&Object.keys(t.headers).length>0)s.headers=t.headers;if(t.javascript){if(s.javascript=t.javascript,t.javascriptWait)s.javascript_wait=t.javascriptWait}if(t.waitForCss)s.wait_for_css=t.waitForCss;return s}getConfig(){return{...this.config,password:"***"}}withConfig(e){return new d({...this.config,...e})}async testConnection(){try{return await this.scrape("https://httpbin.org/ip"),!0}catch(e){throw Error(`Decodo connection test failed: ${e.message}`)}}}exports.Decodo=d;exports.default=d;module.exports=Object.assign(d,exports);

package/dist/crawler/addon/decodo/index.js

@@ -0,0 +1 @@
+import{Rezo as u}from"../../../core/rezo.js";import{DECODO_DEVICE_TYPES as y,DECODO_HEADLESS_MODES as l,DECODO_COMMON_LOCALES as w,DECODO_COMMON_COUNTRIES as O,DECODO_EUROPEAN_COUNTRIES as _,DECODO_ASIAN_COUNTRIES as D,DECODO_US_STATES as C,DECODO_COMMON_CITIES as E,getRandomDeviceType as m,getRandomLocale as v,getRandomCountry as S,getRandomCity as T,generateSessionId as I}from"./options.js";var d="https://scraper-api.smartproxy.com/v2/scrape";class i{config;http;authHeader;constructor(e){let t="username"in e&&"password"in e&&e.username&&e.password,s="token"in e&&e.token;if(!t&&!s)throw Error("Decodo requires either username/password or token for authentication");if(this.config={username:e.username??"",password:e.password??"",deviceType:e.deviceType??"desktop",locale:e.locale??"en-US",country:e.country??"",state:e.state??"",city:e.city??"",headless:e.headless??void 0,headers:e.headers??{},sessionId:e.sessionId??"",sessionDuration:e.sessionDuration??0,javascript:e.javascript??"",javascriptWait:e.javascriptWait??0,waitForCss:e.waitForCss??"",timeout:e.timeout??120000},this.http=new u({baseURL:d,timeout:this.config.timeout}),s)this.authHeader=`Basic ${e.token}`;else this.authHeader=`Basic ${Buffer.from(`${e.username}:${e.password}`).toString("base64")}`}async scrape(e,t){let s={...this.config,...t,headers:{...this.config.headers,...t?.headers||{}}},o=this.buildRequestBody(e,s),r=(await this.http.postJson(d,o,{headers:{Authorization:this.authHeader,"Content-Type":"application/json"}})).data;if(r.error)throw Error(`Decodo API error: ${r.error} (${r.error_code||"unknown"})`);if(!r.results||r.results.length===0)throw Error("Decodo API returned no results");let a=r.results[0];return{statusCode:a.status_code,url:a.url,content:a.body,cookies:a.cookies||[],headers:a.headers||{},taskId:r.id,rendered:!!s.headless,country:s.country||void 0,city:s.city||void 0,state:s.state||void 0,deviceType:s.deviceType,contentType:a.content_type,contentLength:a.content_length,raw:r}}async scrapeMany(e,t,s=1000){let o=[];for(let n=0;n<e.length;n++){let r=await this.scrape(e[n],t);if(o.push(r),n<e.length-1&&s>0)await new Promise((a)=>setTimeout(a,s))}return o}async scrapeWithSession(e,t,s=10){let o=`session_${Date.now()}_${Math.random().toString(36).substring(2,11)}`;return this.scrapeMany(e,{...t,sessionId:o,sessionDuration:s},500)}buildRequestBody(e,t){let s={url:e,return_cookies:!0,return_headers:!0};if(t.deviceType)s.device_type=t.deviceType;if(t.headless)s.headless=t.headless;if(t.locale)s.locale=t.locale;if(t.country)s.country=t.country;if(t.state)s.state=t.state;if(t.city)s.city=t.city;if(t.sessionId){if(s.session=t.sessionId,t.sessionDuration)s.session_duration=t.sessionDuration}if(t.headers&&Object.keys(t.headers).length>0)s.headers=t.headers;if(t.javascript){if(s.javascript=t.javascript,t.javascriptWait)s.javascript_wait=t.javascriptWait}if(t.waitForCss)s.wait_for_css=t.waitForCss;return s}getConfig(){return{...this.config,password:"***"}}withConfig(e){return new i({...this.config,...e})}async testConnection(){try{return await this.scrape("https://httpbin.org/ip"),!0}catch(e){throw Error(`Decodo connection test failed: ${e.message}`)}}}var R=i;export{v as getRandomLocale,m as getRandomDeviceType,S as getRandomCountry,T as getRandomCity,I as generateSessionId,R as default,i as Decodo,C as DECODO_US_STATES,l as DECODO_HEADLESS_MODES,_ as DECODO_EUROPEAN_COUNTRIES,y as DECODO_DEVICE_TYPES,w as DECODO_COMMON_LOCALES,O as DECODO_COMMON_COUNTRIES,E as DECODO_COMMON_CITIES,D as DECODO_ASIAN_COUNTRIES};

package/dist/crawler/crawler-options.cjs

@@ -0,0 +1 @@
+var{defineProperty:f,getOwnPropertyNames:g,getOwnPropertyDescriptor:y}=Object,A=Object.prototype.hasOwnProperty;var m=new WeakMap,v=(e)=>{var t=m.get(e),s;if(t)return t;if(t=f({},"__esModule",{value:!0}),e&&typeof e==="object"||typeof e==="function")g(e).map((a)=>!A.call(t,a)&&f(t,a,{get:()=>e[a],enumerable:!(s=y(e,a))||s.enumerable}));return m.set(e,t),t};var b={};module.exports=v(b);var{RezoQueue:$}=require("../queue/queue.cjs"),{Oxylabs:M}=require("./addon/oxylabs/index.cjs"),w=require("node:path"),O=require("node:os"),{Decodo:p}=require("./addon/decodo/index.cjs");class x{baseUrl;adapter;enableNavigationHistory;sessionId;rejectUnauthorized;userAgent;useRndUserAgent;timeout;maxRedirects;maxRetryAttempts;retryDelay;retryOnStatusCode;forceRevisit;retryWithoutProxyOnStatusCode;retryOnProxyError;maxRetryOnProxyError;allowRevisiting;enableCache;cacheTTL;cacheDir;throwFatalError;debug;maxDepth;maxUrls;maxResponseSize;respectRobotsTxt;followNofollow;autoThrottle;autoThrottleTargetDelay;autoThrottleMinDelay;autoThrottleMaxDelay;maxWaitOn429;alwaysWaitOn429;oxylabs=[];decodo=[];proxies=[];limiters=[];requestHeaders=[];userAgents=D();constructor(e={}){this.baseUrl=e.baseUrl||"",this.adapter=e.adapter??"http",this.enableNavigationHistory=e.enableNavigationHistory??!1,this.sessionId=e.sessionId??`session_${Date.now()}_${Math.random().toString(36).slice(2,8)}`,this.rejectUnauthorized=e.rejectUnauthorized??!0,this.userAgent=e.userAgent,this.useRndUserAgent=e.useRndUserAgent??!1,this.timeout=e.timeout??30000,this.maxRedirects=e.maxRedirects??10,this.maxRetryAttempts=e.maxRetryAttempts??3,this.retryDelay=e.retryDelay??0,this.retryOnStatusCode=e.retryOnStatusCode??[408,429,500,502,503,504],this.forceRevisit=e.forceRevisit??!1,this.retryWithoutProxyOnStatusCode=e.retryWithoutProxyOnStatusCode??[407,403],this.retryOnProxyError=e.retryOnProxyError??!0,this.maxRetryOnProxyError=e.maxRetryOnProxyError??3,this.allowRevisiting=e.allowRevisiting??!1,this.enableCache=e.enableCache??!0,this.cacheTTL=e.cacheTTL??604800000,this.cacheDir=e.cacheDir??w.join(O.tmpdir(),"uiniqhtt_cache"),this.throwFatalError=e.throwFatalError??!1,this.debug=e.debug??!1,this.maxDepth=e.maxDepth??0,this.maxUrls=e.maxUrls??0,this.maxResponseSize=e.maxResponseSize??0,this.respectRobotsTxt=e.respectRobotsTxt??!1,this.followNofollow=e.followNofollow??!1,this.autoThrottle=e.autoThrottle??!0,this.autoThrottleTargetDelay=e.autoThrottleTargetDelay??1000,this.autoThrottleMinDelay=e.autoThrottleMinDelay??100,this.autoThrottleMaxDelay=e.autoThrottleMaxDelay??60000,this.maxWaitOn429=e.maxWaitOn429??1800000,this.alwaysWaitOn429=e.alwaysWaitOn429??!1,this._addHeaders(e.headers),this._addOxylabs(e.oxylabs),this._addDecodo(e.decodo),this._addProxies(e.proxy),this._addLimiters(e.limiter)}getConfiguredDomains(e){return(e==="headers"?this.requestHeaders:e==="limiters"?this.limiters:e==="oxylabs"?this.oxylabs:this.proxies).filter((s)=>s.domain).map((s)=>s.domain).filter((s,a,r)=>r.indexOf(s)===a)}removeDomain(e){return this.requestHeaders=this.requestHeaders.filter((t)=>!t.domain||!this._domainsEqual(t.domain,e)),this.proxies=this.proxies.filter((t)=>!t.domain||!this._domainsEqual(t.domain,e)),this.limiters=this.limiters.filter((t)=>!t.domain||!this._domainsEqual(t.domain,e)),this.oxylabs=this.oxylabs.filter((t)=>!t.domain||!this._domainsEqual(t.domain,e)),this}_domainsEqual(e,t){if(Array.isArray(e)&&Array.isArray(t))return e.length===t.length&&e.every((s,a)=>s===t[a]);return e===t}getConfigurationSummary(){let e=(t)=>({total:t.length,global:t.filter((s)=>s.isGlobal).length,domainSpecific:t.filter((s)=>!s.isGlobal&&s.domain).length});return{headers:e(this.requestHeaders),proxies:e(this.proxies),limiters:e(this.limiters),oxylabs:e(this.oxylabs)}}_addHeaders(e){if(!e||!e.enable)return;for(let t of e.httpHeaders){let{domain:s,isGlobal:a,headers:r}=t;if(!s&&!a)continue;if(r instanceof Headers){let i=Object.fromEntries(r.entries());if(Object.keys(i).length<1)continue;r=i}else if(!r||Object.keys(r).length<1)continue;this.requestHeaders.push({domain:s,isGlobal:a,headers:r})}}_addProxies(e){if(!e||!e.enable)return;for(let t of e.proxies){let{domain:s,isGlobal:a,proxy:r}=t;if(!s&&!a)continue;if(!r||Object.keys(r).length<1)continue;this.proxies.push({domain:s,isGlobal:a,proxy:r})}}_addLimiters(e){if(!e||!e.enable)return;for(let t of e.limiters){let{domain:s,isGlobal:a,options:r}=t;if(!s&&!a)continue;if(!r||Object.keys(r).length<1)continue;this.limiters.push({domain:s,isGlobal:a,pqueue:new $(r)})}}_addOxylabs(e){if(!e||!e.enable)return;for(let t of e.labs){let{domain:s,isGlobal:a,options:r,queueOptions:i}=t;if(!s&&!a)continue;if(!r||Object.keys(r).length<1)continue;this.oxylabs.push({domain:s,isGlobal:a,adaptar:new M(r)})}}_addDecodo(e){if(!e||!e.enable)return;for(let t of e.labs){let{domain:s,isGlobal:a,options:r,queueOptions:i}=t;if(!s&&!a)continue;if(!r||Object.keys(r).length<1)continue;this.decodo.push({domain:s,isGlobal:a,adaptar:new p(r)})}}addHeaders(e){return this._addHeaders({enable:!0,httpHeaders:[e]}),this}addProxy(e){return this._addProxies({enable:!0,proxies:[e]}),this}addLimiter(e){return this._addLimiters({enable:!0,limiters:[e]}),this}addOxylabs(e){return this._addOxylabs({enable:!0,labs:[e]}),this}addDecodo(e){return this._addDecodo({enable:!0,labs:[e]}),this}clearGlobalConfigs(){if(Array.isArray(this.requestHeaders))this.requestHeaders=this.requestHeaders.filter((e)=>!e.isGlobal);if(Array.isArray(this.oxylabs))this.oxylabs=this.oxylabs.filter((e)=>!e.isGlobal);if(Array.isArray(this.limiters))this.limiters=this.limiters.filter((e)=>!e.isGlobal);if(Array.isArray(this.proxies))this.proxies=this.proxies.filter((e)=>!e.isGlobal);return this}getAdapter(e,t,s,a){if(!this.getDomainName(e))return null;let i=[],n=t==="headers"?this.requestHeaders:t==="limiters"?this.limiters:t==="oxylabs"?this.oxylabs:t==="decodo"?this.decodo:this.proxies;for(let o=0;o<n.length;o++)if(this._hasDomain(e,n[o].domain))i.push(o);if(i.length){let o=a?i[this.rnd(0,i.length-1)]:i[0];return t==="headers"?this.requestHeaders[o].headers:t==="limiters"?this.limiters[o].pqueue:t==="oxylabs"?this.oxylabs[o].adaptar:t==="decodo"?this.decodo[o].adaptar:this.proxies[o].proxy}i.length=0;for(let o=0;o<n.length;o++)i.push(o);if(i.length){let o=a?i[this.rnd(0,i.length-1)]:i[0];if(n[o].isGlobal&&s)return t==="headers"?this.requestHeaders[o].headers:t==="limiters"?this.limiters[o].pqueue:t==="oxylabs"?this.oxylabs[o].adaptar:t==="decodo"?this.decodo[o].adaptar:this.proxies[o].proxy}return null}rnd(e=0,t=Number.MAX_VALUE){return Math.floor(Math.random()*(t-e+1))+e}hasDomain(e,t,s){if(!this.getDomainName(e))return!1;let r=t==="headers"?this.requestHeaders:t==="limiters"?this.limiters:t==="oxylabs"?this.oxylabs:t==="decodo"?this.decodo:this.proxies;for(let i=0;i<r.length;i++)if(this._hasDomain(e,r[i].domain))return!0;if(s){for(let i=0;i<r.length;i++)if(r[i].isGlobal)return!0}return!1}pickHeaders(e,t,s,a){let r=this.getAdapter(e,"headers",t),i=new Headers(r??{});if(s&&s instanceof Headers)for(let[n,o]of Object.entries(s.entries()))i.set(n,o);else if(s&&typeof s==="object"){for(let[n,o]of Object.entries(s))if(typeof o==="string")i.set(n,o)}if(a)i.set("user-agent",this.getRandomUserAgent());return Object.fromEntries(i.entries())}_hasDomain(e,t){if(!t)return!1;let s=this.getDomainName(e);if(!s)return!1;let a=(i)=>{return/[\^\$\*\+\?\{\}\[\]\(\)\|\\]/.test(i)||i.startsWith("/")||i.includes(".*")||i.includes(".+")},r=(i)=>{if(i instanceof RegExp)return i.test(s)||i.test(e);let n=i.toString().trim();if(s.toLowerCase()===n.toLowerCase())return!0;if(n.includes("*")){let l=n.replace(/[.*+?^${}()|[\]\\]/g,"\\$&").replace(/\\\*/g,".*"),h=new RegExp(`^${l}$`,"i");return h.test(s)||h.test(e)}if(a(n))try{let l=n,h="i",u=n.match(/^\/(.*)\/(\w*)$/);if(u)l=u[1],h=u[2]||"i";let c=new RegExp(l,h);return c.test(s)||c.test(e)}catch(l){return s.toLowerCase().includes(n.toLowerCase())}let o=s.toLowerCase(),d=n.toLowerCase();return o===d||o.endsWith("."+d)||d.endsWith("."+o)};if(Array.isArray(t)){for(let i of t)if(r(i))return!0;return!1}return r(t)}getDomainName(e){if(this.isValidUrl(e))return new URL(e).hostname;else if(this.isHostName(e))return e;return null}isHostName(e){if(!e)return!1;if(e.length>255)return!1;let t=/^(?:(?:[a-zA-Z0-9](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?\.)+ [a-zA-Z]{2,})$/;return e=e.trim().toLowerCase(),t.test(e)&&!e.startsWith("-")&&!e.endsWith("-")}isValidUrl(e){if(!e)return!1;e=e.trim();try{let t=new URL(e);if(!t.protocol||!["http:","https:"].includes(t.protocol.toLowerCase()))return!1;if(!t.hostname)return!1;if(!/^(?:(?:[a-zA-Z0-9](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?\.)+[a-zA-Z]{2,})$/.test(t.hostname))return!1;return!0}catch{return!1}}getRandomUserAgent(){return this.userAgents[Math.floor(Math.random()*this.userAgents.length)]}}function D(){let e=[{name:"Chrome",version:"91.0.4472.124",engine:"AppleWebKit/537.36"},{name:"Firefox",version:"89.0",engine:"Gecko/20100101"},{name:"Safari",version:"14.1.1",engine:"AppleWebKit/605.1.15"},{name:"Edge",version:"91.0.864.59",engine:"AppleWebKit/537.36"},{name:"Opera",version:"77.0.4054.277",engine:"AppleWebKit/537.36"},{name:"Vivaldi",version:"3.8.2259.42",engine:"AppleWebKit/537.36"},{name:"Brave",version:"1.26.74",engine:"AppleWebKit/537.36"},{name:"Chromium",version:"91.0.4472.101",engine:"AppleWebKit/537.36"},{name:"Yandex",version:"21.5.3.742",engine:"AppleWebKit/537.36"},{name:"Maxthon",version:"5.3.8.2000",engine:"AppleWebKit/537.36"}],t=["Windows NT 10.0","Windows NT 6.1","Macintosh; Intel Mac OS X 10_15_7","Macintosh; Intel Mac OS X 11_4_0","X11; Linux x86_64","X11; Ubuntu; Linux x86_64"],s=[];for(let a=0;a<200;a++){let r=e[Math.floor(Math.random()*e.length)],i=t[Math.floor(Math.random()*t.length)],n="";switch(r.name){case"Chrome":n=`Mozilla/5.0 (${i}) ${r.engine} (KHTML, like Gecko) Chrome/${r.version} Safari/537.36`;break;case"Firefox":n=`Mozilla/5.0 (${i}; rv:${r.version}) ${r.engine} Firefox/${r.version}`;break;case"Safari":n=`Mozilla/5.0 (${i}) ${r.engine} (KHTML, like Gecko) Version/${r.version} Safari/605.1.15`;break;case"Edge":n=`Mozilla/5.0 (${i}) ${r.engine} (KHTML, like Gecko) Chrome/${r.version} Safari/537.36 Edg/${r.version}`;break;case"Opera":n=`Mozilla/5.0 (${i}) ${r.engine} (KHTML, like Gecko) Chrome/${r.version} Safari/537.36 OPR/${r.version}`;break;case"Vivaldi":n=`Mozilla/5.0 (${i}) ${r.engine} (KHTML, like Gecko) Chrome/${r.version} Safari/537.36 Vivaldi/${r.version}`;break;case"Brave":n=`Mozilla/5.0 (${i}) ${r.engine} (KHTML, like Gecko) Chrome/${r.version} Safari/537.36 Brave/${r.version}`;break;case"Chromium":n=`Mozilla/5.0 (${i}) ${r.engine} (KHTML, like Gecko) Chromium/${r.version} Chrome/${r.version} Safari/537.36`;break;case"Yandex":n=`Mozilla/5.0 (${i}) ${r.engine} (KHTML, like Gecko) Chrome/${r.version} YaBrowser/${r.version} Safari/537.36`;break;case"Maxthon":n=`Mozilla/5.0 (${i}) ${r.engine} (KHTML, like Gecko) Chrome/${r.version} Safari/537.36 Maxthon/${r.version}`;break}s.push(n)}return s}b.CrawlerOptions=x;

package/dist/crawler/crawler-options.js

@@ -0,0 +1 @@
+import{RezoQueue as m}from"../queue/queue.js";import{Oxylabs as f}from"./addon/oxylabs/index.js";import x from"node:path";import b from"node:os";import{Decodo as g}from"./addon/decodo/index.js";class y{baseUrl;adapter;enableNavigationHistory;sessionId;rejectUnauthorized;userAgent;useRndUserAgent;timeout;maxRedirects;maxRetryAttempts;retryDelay;retryOnStatusCode;forceRevisit;retryWithoutProxyOnStatusCode;retryOnProxyError;maxRetryOnProxyError;allowRevisiting;enableCache;cacheTTL;cacheDir;throwFatalError;debug;maxDepth;maxUrls;maxResponseSize;respectRobotsTxt;followNofollow;autoThrottle;autoThrottleTargetDelay;autoThrottleMinDelay;autoThrottleMaxDelay;maxWaitOn429;alwaysWaitOn429;oxylabs=[];decodo=[];proxies=[];limiters=[];requestHeaders=[];userAgents=A();constructor(e={}){this.baseUrl=e.baseUrl||"",this.adapter=e.adapter??"http",this.enableNavigationHistory=e.enableNavigationHistory??!1,this.sessionId=e.sessionId??`session_${Date.now()}_${Math.random().toString(36).slice(2,8)}`,this.rejectUnauthorized=e.rejectUnauthorized??!0,this.userAgent=e.userAgent,this.useRndUserAgent=e.useRndUserAgent??!1,this.timeout=e.timeout??30000,this.maxRedirects=e.maxRedirects??10,this.maxRetryAttempts=e.maxRetryAttempts??3,this.retryDelay=e.retryDelay??0,this.retryOnStatusCode=e.retryOnStatusCode??[408,429,500,502,503,504],this.forceRevisit=e.forceRevisit??!1,this.retryWithoutProxyOnStatusCode=e.retryWithoutProxyOnStatusCode??[407,403],this.retryOnProxyError=e.retryOnProxyError??!0,this.maxRetryOnProxyError=e.maxRetryOnProxyError??3,this.allowRevisiting=e.allowRevisiting??!1,this.enableCache=e.enableCache??!0,this.cacheTTL=e.cacheTTL??604800000,this.cacheDir=e.cacheDir??x.join(b.tmpdir(),"uiniqhtt_cache"),this.throwFatalError=e.throwFatalError??!1,this.debug=e.debug??!1,this.maxDepth=e.maxDepth??0,this.maxUrls=e.maxUrls??0,this.maxResponseSize=e.maxResponseSize??0,this.respectRobotsTxt=e.respectRobotsTxt??!1,this.followNofollow=e.followNofollow??!1,this.autoThrottle=e.autoThrottle??!0,this.autoThrottleTargetDelay=e.autoThrottleTargetDelay??1000,this.autoThrottleMinDelay=e.autoThrottleMinDelay??100,this.autoThrottleMaxDelay=e.autoThrottleMaxDelay??60000,this.maxWaitOn429=e.maxWaitOn429??1800000,this.alwaysWaitOn429=e.alwaysWaitOn429??!1,this._addHeaders(e.headers),this._addOxylabs(e.oxylabs),this._addDecodo(e.decodo),this._addProxies(e.proxy),this._addLimiters(e.limiter)}getConfiguredDomains(e){return(e==="headers"?this.requestHeaders:e==="limiters"?this.limiters:e==="oxylabs"?this.oxylabs:this.proxies).filter((s)=>s.domain).map((s)=>s.domain).filter((s,n,r)=>r.indexOf(s)===n)}removeDomain(e){return this.requestHeaders=this.requestHeaders.filter((t)=>!t.domain||!this._domainsEqual(t.domain,e)),this.proxies=this.proxies.filter((t)=>!t.domain||!this._domainsEqual(t.domain,e)),this.limiters=this.limiters.filter((t)=>!t.domain||!this._domainsEqual(t.domain,e)),this.oxylabs=this.oxylabs.filter((t)=>!t.domain||!this._domainsEqual(t.domain,e)),this}_domainsEqual(e,t){if(Array.isArray(e)&&Array.isArray(t))return e.length===t.length&&e.every((s,n)=>s===t[n]);return e===t}getConfigurationSummary(){let e=(t)=>({total:t.length,global:t.filter((s)=>s.isGlobal).length,domainSpecific:t.filter((s)=>!s.isGlobal&&s.domain).length});return{headers:e(this.requestHeaders),proxies:e(this.proxies),limiters:e(this.limiters),oxylabs:e(this.oxylabs)}}_addHeaders(e){if(!e||!e.enable)return;for(let t of e.httpHeaders){let{domain:s,isGlobal:n,headers:r}=t;if(!s&&!n)continue;if(r instanceof Headers){let i=Object.fromEntries(r.entries());if(Object.keys(i).length<1)continue;r=i}else if(!r||Object.keys(r).length<1)continue;this.requestHeaders.push({domain:s,isGlobal:n,headers:r})}}_addProxies(e){if(!e||!e.enable)return;for(let t of e.proxies){let{domain:s,isGlobal:n,proxy:r}=t;if(!s&&!n)continue;if(!r||Object.keys(r).length<1)continue;this.proxies.push({domain:s,isGlobal:n,proxy:r})}}_addLimiters(e){if(!e||!e.enable)return;for(let t of e.limiters){let{domain:s,isGlobal:n,options:r}=t;if(!s&&!n)continue;if(!r||Object.keys(r).length<1)continue;this.limiters.push({domain:s,isGlobal:n,pqueue:new m(r)})}}_addOxylabs(e){if(!e||!e.enable)return;for(let t of e.labs){let{domain:s,isGlobal:n,options:r,queueOptions:i}=t;if(!s&&!n)continue;if(!r||Object.keys(r).length<1)continue;this.oxylabs.push({domain:s,isGlobal:n,adaptar:new f(r)})}}_addDecodo(e){if(!e||!e.enable)return;for(let t of e.labs){let{domain:s,isGlobal:n,options:r,queueOptions:i}=t;if(!s&&!n)continue;if(!r||Object.keys(r).length<1)continue;this.decodo.push({domain:s,isGlobal:n,adaptar:new g(r)})}}addHeaders(e){return this._addHeaders({enable:!0,httpHeaders:[e]}),this}addProxy(e){return this._addProxies({enable:!0,proxies:[e]}),this}addLimiter(e){return this._addLimiters({enable:!0,limiters:[e]}),this}addOxylabs(e){return this._addOxylabs({enable:!0,labs:[e]}),this}addDecodo(e){return this._addDecodo({enable:!0,labs:[e]}),this}clearGlobalConfigs(){if(Array.isArray(this.requestHeaders))this.requestHeaders=this.requestHeaders.filter((e)=>!e.isGlobal);if(Array.isArray(this.oxylabs))this.oxylabs=this.oxylabs.filter((e)=>!e.isGlobal);if(Array.isArray(this.limiters))this.limiters=this.limiters.filter((e)=>!e.isGlobal);if(Array.isArray(this.proxies))this.proxies=this.proxies.filter((e)=>!e.isGlobal);return this}getAdapter(e,t,s,n){if(!this.getDomainName(e))return null;let i=[],a=t==="headers"?this.requestHeaders:t==="limiters"?this.limiters:t==="oxylabs"?this.oxylabs:t==="decodo"?this.decodo:this.proxies;for(let o=0;o<a.length;o++)if(this._hasDomain(e,a[o].domain))i.push(o);if(i.length){let o=n?i[this.rnd(0,i.length-1)]:i[0];return t==="headers"?this.requestHeaders[o].headers:t==="limiters"?this.limiters[o].pqueue:t==="oxylabs"?this.oxylabs[o].adaptar:t==="decodo"?this.decodo[o].adaptar:this.proxies[o].proxy}i.length=0;for(let o=0;o<a.length;o++)i.push(o);if(i.length){let o=n?i[this.rnd(0,i.length-1)]:i[0];if(a[o].isGlobal&&s)return t==="headers"?this.requestHeaders[o].headers:t==="limiters"?this.limiters[o].pqueue:t==="oxylabs"?this.oxylabs[o].adaptar:t==="decodo"?this.decodo[o].adaptar:this.proxies[o].proxy}return null}rnd(e=0,t=Number.MAX_VALUE){return Math.floor(Math.random()*(t-e+1))+e}hasDomain(e,t,s){if(!this.getDomainName(e))return!1;let r=t==="headers"?this.requestHeaders:t==="limiters"?this.limiters:t==="oxylabs"?this.oxylabs:t==="decodo"?this.decodo:this.proxies;for(let i=0;i<r.length;i++)if(this._hasDomain(e,r[i].domain))return!0;if(s){for(let i=0;i<r.length;i++)if(r[i].isGlobal)return!0}return!1}pickHeaders(e,t,s,n){let r=this.getAdapter(e,"headers",t),i=new Headers(r??{});if(s&&s instanceof Headers)for(let[a,o]of Object.entries(s.entries()))i.set(a,o);else if(s&&typeof s==="object"){for(let[a,o]of Object.entries(s))if(typeof o==="string")i.set(a,o)}if(n)i.set("user-agent",this.getRandomUserAgent());return Object.fromEntries(i.entries())}_hasDomain(e,t){if(!t)return!1;let s=this.getDomainName(e);if(!s)return!1;let n=(i)=>{return/[\^\$\*\+\?\{\}\[\]\(\)\|\\]/.test(i)||i.startsWith("/")||i.includes(".*")||i.includes(".+")},r=(i)=>{if(i instanceof RegExp)return i.test(s)||i.test(e);let a=i.toString().trim();if(s.toLowerCase()===a.toLowerCase())return!0;if(a.includes("*")){let l=a.replace(/[.*+?^${}()|[\]\\]/g,"\\$&").replace(/\\\*/g,".*"),h=new RegExp(`^${l}$`,"i");return h.test(s)||h.test(e)}if(n(a))try{let l=a,h="i",u=a.match(/^\/(.*)\/(\w*)$/);if(u)l=u[1],h=u[2]||"i";let c=new RegExp(l,h);return c.test(s)||c.test(e)}catch(l){return s.toLowerCase().includes(a.toLowerCase())}let o=s.toLowerCase(),d=a.toLowerCase();return o===d||o.endsWith("."+d)||d.endsWith("."+o)};if(Array.isArray(t)){for(let i of t)if(r(i))return!0;return!1}return r(t)}getDomainName(e){if(this.isValidUrl(e))return new URL(e).hostname;else if(this.isHostName(e))return e;return null}isHostName(e){if(!e)return!1;if(e.length>255)return!1;let t=/^(?:(?:[a-zA-Z0-9](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?\.)+ [a-zA-Z]{2,})$/;return e=e.trim().toLowerCase(),t.test(e)&&!e.startsWith("-")&&!e.endsWith("-")}isValidUrl(e){if(!e)return!1;e=e.trim();try{let t=new URL(e);if(!t.protocol||!["http:","https:"].includes(t.protocol.toLowerCase()))return!1;if(!t.hostname)return!1;if(!/^(?:(?:[a-zA-Z0-9](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?\.)+[a-zA-Z]{2,})$/.test(t.hostname))return!1;return!0}catch{return!1}}getRandomUserAgent(){return this.userAgents[Math.floor(Math.random()*this.userAgents.length)]}}function A(){let e=[{name:"Chrome",version:"91.0.4472.124",engine:"AppleWebKit/537.36"},{name:"Firefox",version:"89.0",engine:"Gecko/20100101"},{name:"Safari",version:"14.1.1",engine:"AppleWebKit/605.1.15"},{name:"Edge",version:"91.0.864.59",engine:"AppleWebKit/537.36"},{name:"Opera",version:"77.0.4054.277",engine:"AppleWebKit/537.36"},{name:"Vivaldi",version:"3.8.2259.42",engine:"AppleWebKit/537.36"},{name:"Brave",version:"1.26.74",engine:"AppleWebKit/537.36"},{name:"Chromium",version:"91.0.4472.101",engine:"AppleWebKit/537.36"},{name:"Yandex",version:"21.5.3.742",engine:"AppleWebKit/537.36"},{name:"Maxthon",version:"5.3.8.2000",engine:"AppleWebKit/537.36"}],t=["Windows NT 10.0","Windows NT 6.1","Macintosh; Intel Mac OS X 10_15_7","Macintosh; Intel Mac OS X 11_4_0","X11; Linux x86_64","X11; Ubuntu; Linux x86_64"],s=[];for(let n=0;n<200;n++){let r=e[Math.floor(Math.random()*e.length)],i=t[Math.floor(Math.random()*t.length)],a="";switch(r.name){case"Chrome":a=`Mozilla/5.0 (${i}) ${r.engine} (KHTML, like Gecko) Chrome/${r.version} Safari/537.36`;break;case"Firefox":a=`Mozilla/5.0 (${i}; rv:${r.version}) ${r.engine} Firefox/${r.version}`;break;case"Safari":a=`Mozilla/5.0 (${i}) ${r.engine} (KHTML, like Gecko) Version/${r.version} Safari/605.1.15`;break;case"Edge":a=`Mozilla/5.0 (${i}) ${r.engine} (KHTML, like Gecko) Chrome/${r.version} Safari/537.36 Edg/${r.version}`;break;case"Opera":a=`Mozilla/5.0 (${i}) ${r.engine} (KHTML, like Gecko) Chrome/${r.version} Safari/537.36 OPR/${r.version}`;break;case"Vivaldi":a=`Mozilla/5.0 (${i}) ${r.engine} (KHTML, like Gecko) Chrome/${r.version} Safari/537.36 Vivaldi/${r.version}`;break;case"Brave":a=`Mozilla/5.0 (${i}) ${r.engine} (KHTML, like Gecko) Chrome/${r.version} Safari/537.36 Brave/${r.version}`;break;case"Chromium":a=`Mozilla/5.0 (${i}) ${r.engine} (KHTML, like Gecko) Chromium/${r.version} Chrome/${r.version} Safari/537.36`;break;case"Yandex":a=`Mozilla/5.0 (${i}) ${r.engine} (KHTML, like Gecko) Chrome/${r.version} YaBrowser/${r.version} Safari/537.36`;break;case"Maxthon":a=`Mozilla/5.0 (${i}) ${r.engine} (KHTML, like Gecko) Chrome/${r.version} Safari/537.36 Maxthon/${r.version}`;break}s.push(a)}return s}export{y as CrawlerOptions};