rezo 1.0.131 → 1.0.133

package/dist/adapters/entries/curl.d.ts

@@ -6027,7 +6027,7 @@ export interface RezoInstance extends Rezo, RezoCallable {
  *
  * IMPORTANT: Update these values when bumping package version.
  */
-export declare const VERSION = "1.0.131";
+export declare const VERSION = "1.0.133";
 /**
  * cURL Options Configuration
  *

package/dist/adapters/entries/fetch.d.ts

@@ -6027,7 +6027,7 @@ export interface RezoInstance extends Rezo, RezoCallable {
  *
  * IMPORTANT: Update these values when bumping package version.
  */
-export declare const VERSION = "1.0.131";
+export declare const VERSION = "1.0.133";
 export declare const isRezoError: typeof RezoError.isRezoError;
 export declare const Cancel: typeof RezoError;
 export declare const CancelToken: {

package/dist/adapters/entries/http.d.ts

@@ -6027,7 +6027,7 @@ export interface RezoInstance extends Rezo, RezoCallable {
  *
  * IMPORTANT: Update these values when bumping package version.
  */
-export declare const VERSION = "1.0.131";
+export declare const VERSION = "1.0.133";
 /**
  * Type guard to check if an error is a RezoError instance.
  */

package/dist/adapters/entries/http2.d.ts

@@ -6027,7 +6027,7 @@ export interface RezoInstance extends Rezo, RezoCallable {
  *
  * IMPORTANT: Update these values when bumping package version.
  */
-export declare const VERSION = "1.0.131";
+export declare const VERSION = "1.0.133";
 export declare const isRezoError: typeof RezoError.isRezoError;
 export declare const Cancel: typeof RezoError;
 export declare const CancelToken: {

package/dist/adapters/entries/react-native.cjs

@@ -6,12 +6,12 @@ const { RezoFormData } = require('../../utils/form-data.cjs');
 const { RezoCookieJar, Cookie } = require('../../cookies/cookie-jar.cjs');
 const { createDefaultHooks, mergeHooks } = require('../../core/hooks.cjs');
 const { VERSION } = require('../../version.cjs');
-const _mod_elhp6j = require('../../platform/react-native-providers.cjs');
-exports.createFetchStreamTransport = _mod_elhp6j.createFetchStreamTransport;
-exports.createExpoFileSystemAdapter = _mod_elhp6j.createExpoFileSystemAdapter;
-exports.createReactNativeFsAdapter = _mod_elhp6j.createReactNativeFsAdapter;
-exports.createNetInfoProvider = _mod_elhp6j.createNetInfoProvider;
-exports.createExpoBackgroundTaskProvider = _mod_elhp6j.createExpoBackgroundTaskProvider;;
+const _mod_6jezic = require('../../platform/react-native-providers.cjs');
+exports.createFetchStreamTransport = _mod_6jezic.createFetchStreamTransport;
+exports.createExpoFileSystemAdapter = _mod_6jezic.createExpoFileSystemAdapter;
+exports.createReactNativeFsAdapter = _mod_6jezic.createReactNativeFsAdapter;
+exports.createNetInfoProvider = _mod_6jezic.createNetInfoProvider;
+exports.createExpoBackgroundTaskProvider = _mod_6jezic.createExpoBackgroundTaskProvider;;
 exports.Rezo = Rezo;
 exports.RezoError = RezoError;
 exports.RezoErrorCode = RezoErrorCode;

package/dist/adapters/entries/react-native.d.ts

@@ -6027,7 +6027,7 @@ export interface RezoInstance extends Rezo, RezoCallable {
  *
  * IMPORTANT: Update these values when bumping package version.
  */
-export declare const VERSION = "1.0.131";
+export declare const VERSION = "1.0.133";
 export interface ExpoFileSystemFileLike {
 	uri?: string;
 	size?: number;

package/dist/adapters/entries/xhr.d.ts

@@ -6027,7 +6027,7 @@ export interface RezoInstance extends Rezo, RezoCallable {
  *
  * IMPORTANT: Update these values when bumping package version.
  */
-export declare const VERSION = "1.0.131";
+export declare const VERSION = "1.0.133";
 export declare const isRezoError: typeof RezoError.isRezoError;
 export declare const Cancel: typeof RezoError;
 export declare const CancelToken: {

package/dist/adapters/index.cjs

@@ -1,6 +1,6 @@
-const _mod_ta19rw = require('./picker.cjs');
-exports.detectRuntime = _mod_ta19rw.detectRuntime;
-exports.getAdapterCapabilities = _mod_ta19rw.getAdapterCapabilities;
-exports.buildAdapterContext = _mod_ta19rw.buildAdapterContext;
-exports.getAvailableAdapters = _mod_ta19rw.getAvailableAdapters;
-exports.selectAdapter = _mod_ta19rw.selectAdapter;;
+const _mod_wl45y9 = require('./picker.cjs');
+exports.detectRuntime = _mod_wl45y9.detectRuntime;
+exports.getAdapterCapabilities = _mod_wl45y9.getAdapterCapabilities;
+exports.buildAdapterContext = _mod_wl45y9.buildAdapterContext;
+exports.getAvailableAdapters = _mod_wl45y9.getAvailableAdapters;
+exports.selectAdapter = _mod_wl45y9.selectAdapter;;

package/dist/cache/index.cjs

@@ -1,9 +1,9 @@
-const _mod_2sir3t = require('./lru-cache.cjs');
-exports.LRUCache = _mod_2sir3t.LRUCache;;
-const _mod_h1s52p = require('./dns-cache.cjs');
-exports.DNSCache = _mod_h1s52p.DNSCache;
-exports.getGlobalDNSCache = _mod_h1s52p.getGlobalDNSCache;
-exports.resetGlobalDNSCache = _mod_h1s52p.resetGlobalDNSCache;;
-const _mod_a5smaj = require('./response-cache.cjs');
-exports.ResponseCache = _mod_a5smaj.ResponseCache;
-exports.normalizeResponseCacheConfig = _mod_a5smaj.normalizeResponseCacheConfig;;
+const _mod_pwv10q = require('./lru-cache.cjs');
+exports.LRUCache = _mod_pwv10q.LRUCache;;
+const _mod_r89cab = require('./dns-cache.cjs');
+exports.DNSCache = _mod_r89cab.DNSCache;
+exports.getGlobalDNSCache = _mod_r89cab.getGlobalDNSCache;
+exports.resetGlobalDNSCache = _mod_r89cab.resetGlobalDNSCache;;
+const _mod_vz9afj = require('./response-cache.cjs');
+exports.ResponseCache = _mod_vz9afj.ResponseCache;
+exports.normalizeResponseCacheConfig = _mod_vz9afj.normalizeResponseCacheConfig;;

package/dist/cookies/cookie-jar.cjs

@@ -1,8 +1,8 @@
 const { CookieJar: TouchCookieJar } = require("tough-cookie");
 const { Cookie } = require('./cookie.cjs');
 const { requireNodeModule } = require('../utils/node-runtime.cjs');
-const _mod_2xetkm = require('./cookie.cjs');
-exports.Cookie = _mod_2xetkm.Cookie;;
+const _mod_egd39a = require('./cookie.cjs');
+exports.Cookie = _mod_egd39a.Cookie;;
 
 class RezoCookieJar extends TouchCookieJar {
   constructor(store, options) {
@@ -481,7 +481,7 @@ class RezoCookieJar extends TouchCookieJar {
   }
 }
 const CookieJar = exports.CookieJar = RezoCookieJar;
-const _mod_o2fr98 = require("tough-cookie");
-exports.Store = _mod_o2fr98.Store;;
+const _mod_uokl3t = require("tough-cookie");
+exports.Store = _mod_uokl3t.Store;;
 
 exports.RezoCookieJar = RezoCookieJar;

package/dist/cookies/index.cjs

@@ -1,10 +1,10 @@
-const _mod_dnn0yd = require('./cookie.cjs');
-exports.Cookie = _mod_dnn0yd.Cookie;
-exports.RezoCookie = _mod_dnn0yd.RezoCookie;;
-const _mod_euh49t = require('./cookie-store.cjs');
-exports.RezoCookieStore = _mod_euh49t.RezoCookieStore;;
-const _mod_n9r664 = require('./cookie-jar.cjs');
-exports.RezoCookieJar = _mod_n9r664.RezoCookieJar;
-exports.CookieJar = _mod_n9r664.CookieJar;;
-const _mod_5ka56g = require("tough-cookie");
-exports.Store = _mod_5ka56g.Store;;
+const _mod_3anom3 = require('./cookie.cjs');
+exports.Cookie = _mod_3anom3.Cookie;
+exports.RezoCookie = _mod_3anom3.RezoCookie;;
+const _mod_p2lai5 = require('./cookie-store.cjs');
+exports.RezoCookieStore = _mod_p2lai5.RezoCookieStore;;
+const _mod_dze7qd = require('./cookie-jar.cjs');
+exports.RezoCookieJar = _mod_dze7qd.RezoCookieJar;
+exports.CookieJar = _mod_dze7qd.CookieJar;;
+const _mod_e0fpg1 = require("tough-cookie");
+exports.Store = _mod_e0fpg1.Store;;

package/dist/crawler/addon/oxylabs/index.cjs

@@ -1 +1 @@
-var{Rezo:h}=require("../../../core/rezo.cjs"),a=require("./options.cjs");exports.OXYLABS_BROWSER_TYPES=a.OXYLABS_BROWSER_TYPES;exports.OXYLABS_COMMON_LOCALES=a.OXYLABS_COMMON_LOCALES;exports.OXYLABS_COMMON_GEO_LOCATIONS=a.OXYLABS_COMMON_GEO_LOCATIONS;exports.OXYLABS_US_STATES=a.OXYLABS_US_STATES;exports.OXYLABS_EUROPEAN_COUNTRIES=a.OXYLABS_EUROPEAN_COUNTRIES;exports.OXYLABS_ASIAN_COUNTRIES=a.OXYLABS_ASIAN_COUNTRIES;exports.getRandomBrowserType=a.getRandomBrowserType;exports.getRandomLocale=a.getRandomLocale;exports.getRandomGeoLocation=a.getRandomGeoLocation;var i="https://realtime.oxylabs.io/v1/queries";class d{config;http;authHeader;constructor(s){if(!s.username||!s.password)throw Error("Oxylabs username and password are required");this.config={username:s.username,password:s.password,browserType:s.browserType??"desktop",locale:s.locale??"en-US",geoLocation:s.geoLocation??"",render:s.render??!1,context:s.context??{},timeout:s.timeout??120000},this.http=new h({baseURL:i,timeout:this.config.timeout}),this.authHeader=`Basic ${Buffer.from(`${s.username}:${s.password}`).toString("base64")}`}async scrape(s,e){let t={...this.config,...e},r=this.buildRequestBody(s,t),o=(await this.http.postJson(i,r,{headers:{Authorization:this.authHeader,"Content-Type":"application/json"}})).data;if(o.error)throw Error(`Oxylabs API error: ${o.error}`);if(!o.results||o.results.length===0)throw Error("Oxylabs API returned no results");let n=o.results[0],l=n._response?.cookies||[],_=n._response?.headers||{};return{statusCode:n.status_code,url:n.url,content:n.content,cookies:l,headers:_,jobId:o.job_id||n.job_id,rendered:t.render,geoLocation:t.geoLocation||void 0,locale:t.locale,browserType:t.browserType,raw:o}}async scrapeMany(s,e,t=1000){let r=[];for(let u=0;u<s.length;u++){let o=await this.scrape(s[u],e);if(r.push(o),u<s.length-1&&t>0)await new Promise((n)=>setTimeout(n,t))}return r}buildRequestBody(s,e){let t=[];if(t.push({key:"return_page_cookies",value:!0}),t.push({key:"return_page_headers",value:!0}),e.context)for(let[u,o]of Object.entries(e.context))t.push({key:u,value:o});if(e.headers&&Object.keys(e.headers).length>0)t.push({key:"headers",value:e.headers});if(e.cookies&&e.cookies.length>0)t.push({key:"cookies",value:e.cookies});if(e.session_id)t.push({key:"session_id",value:e.session_id});if(e.http_method)t.push({key:"http_method",value:e.http_method});if(e.base64Body)t.push({key:"content",value:e.base64Body});if(typeof e.follow_redirects==="boolean")t.push({key:"follow_redirects",value:e.follow_redirects});if(e.successful_status_codes&&e.successful_status_codes.length>0)t.push({key:"successful_status_codes",value:e.successful_status_codes});let r={source:"universal",url:s,context:t};if(e.render||e.javascript_rendering)r.render="html";if(e.browserType)r.user_agent_type=e.browserType;if(e.locale)r.locale=e.locale;if(e.geoLocation)r.geo_location=e.geoLocation;if(e.returnAsBase64)r.content_encoding="base64";return r}getConfig(){return{...this.config,password:"***"}}withConfig(s){return new d({...this.config,...s})}async testConnection(){try{return await this.scrape("https://httpbin.org/ip"),!0}catch(s){throw Error(`Oxylabs connection test failed: ${s.message}`)}}}exports.Oxylabs=d;exports.default=d;module.exports=Object.assign(d,exports);
+var{Rezo:_}=require("../../../core/rezo.cjs"),a=require("./options.cjs");exports.OXYLABS_BROWSER_TYPES=a.OXYLABS_BROWSER_TYPES;exports.OXYLABS_COMMON_LOCALES=a.OXYLABS_COMMON_LOCALES;exports.OXYLABS_COMMON_GEO_LOCATIONS=a.OXYLABS_COMMON_GEO_LOCATIONS;exports.OXYLABS_US_STATES=a.OXYLABS_US_STATES;exports.OXYLABS_EUROPEAN_COUNTRIES=a.OXYLABS_EUROPEAN_COUNTRIES;exports.OXYLABS_ASIAN_COUNTRIES=a.OXYLABS_ASIAN_COUNTRIES;exports.getRandomBrowserType=a.getRandomBrowserType;exports.getRandomLocale=a.getRandomLocale;exports.getRandomGeoLocation=a.getRandomGeoLocation;var i="https://realtime.oxylabs.io/v1/queries";class d{config;http;authHeader;constructor(s){if(!s.username||!s.password)throw Error("Oxylabs username and password are required");this.config={username:s.username,password:s.password,browserType:s.browserType??"desktop",locale:s.locale??"en-US",geoLocation:s.geoLocation??"",render:s.render??!1,context:s.context??{},timeout:s.timeout??120000},this.http=new _({baseURL:i,timeout:this.config.timeout}),this.authHeader=`Basic ${Buffer.from(`${s.username}:${s.password}`).toString("base64")}`}async scrape(s,e){let t={...this.config,...e},r=this.buildRequestBody(s,t),o=(await this.http.postJson(i,r,{headers:{Authorization:this.authHeader,"Content-Type":"application/json"}})).data;if(o.error)throw Error(`Oxylabs API error: ${o.error}`);if(!o.results||o.results.length===0)throw Error("Oxylabs API returned no results");let n=o.results[0],l=n._response?.cookies||[],h=n._response?.headers||{};return{statusCode:n.status_code,url:n.url,content:n.content,cookies:l,headers:h,jobId:o.job_id||n.job_id,rendered:t.render,geoLocation:t.geoLocation||void 0,locale:t.locale,browserType:t.browserType,raw:o}}async scrapeMany(s,e,t=1000){let r=[];for(let u=0;u<s.length;u++){let o=await this.scrape(s[u],e);if(r.push(o),u<s.length-1&&t>0)await new Promise((n)=>setTimeout(n,t))}return r}buildRequestBody(s,e){let t=[];if(t.push({key:"return_page_cookies",value:!0}),t.push({key:"return_page_headers",value:!0}),e.context)for(let[u,o]of Object.entries(e.context))t.push({key:u,value:o});if(e.headers&&Object.keys(e.headers).length>0)t.push({key:"headers",value:e.headers});if(e.cookies&&e.cookies.length>0)t.push({key:"cookies",value:e.cookies});if(e.session_id)t.push({key:"session_id",value:e.session_id});if(e.http_method)t.push({key:"http_method",value:e.http_method});if(e.base64Body)t.push({key:"content",value:e.base64Body});if(typeof e.follow_redirects==="boolean")t.push({key:"follow_redirects",value:e.follow_redirects});if(e.successful_status_codes&&e.successful_status_codes.length>0)t.push({key:"successful_status_codes",value:e.successful_status_codes});let r={source:"universal",url:s,context:t};if(e.render||e.javascript_rendering)r.render="html";if(e.browserType)r.user_agent_type=e.browserType;if(e.locale)r.locale=e.locale;if(e.geoLocation)r.geo_location=e.geoLocation;if(e.returnAsBase64)r.content_encoding="base64";return r}getConfig(){return{...this.config,password:"***"}}withConfig(s){return new d({...this.config,...s})}async testConnection(){try{return await this.scrape("https://httpbin.org/ip"),!0}catch(s){throw Error(`Oxylabs connection test failed: ${s.message}`)}}}exports.Oxylabs=d;exports.default=d;module.exports=Object.assign(d,exports);

package/dist/crawler/crawler-options.cjs

@@ -1 +1 @@
-var{RezoQueue:y}=require("../queue/queue.cjs"),{Oxylabs:x}=require("./addon/oxylabs/index.cjs"),v=require("node:path"),A=require("node:os"),{Decodo:p}=require("./addon/decodo/index.cjs"),{Rezo:D}=require("../core/rezo.cjs");class l{static destroyQueue(e){if(e&&typeof e.destroy==="function")e.destroy()}static destroyConfigQueues(e){for(let r of e)l.destroyQueue(r.pqueue)}static splitConfigs(e,r){let s=[],a=[];for(let t of e)if(r(t))a.push(t);else s.push(t);return{kept:s,removed:a}}static getHostname(e){try{return new URL(e).hostname||void 0}catch{return}}static createStableThroughputOptions(e){let{baseUrl:r,concurrency:s=40,scraperConcurrency:a=10,retryDelay:t=1000,maxRetryAttempts:i=2,retryOnStatusCode:o=[408,500,502,503,504],maxWaitOn429:d=15000,alwaysWaitOn429:n=!1,globalLimiter:h={concurrency:8},domain:u=l.getHostname(r),domainLimiter:c={concurrency:2,interval:1000,intervalCap:2,randomDelay:150},domainRetry:m={enable:!0,max429Retries:2,retryDelay:1000,maxRetryAttempts:2,backoff:!0},extraLimiters:g=[],overrides:b={}}=e,f=[];if(h)f.push({isGlobal:!0,options:h});if(u&&c)f.push({domain:u,options:c,retry:m||void 0});return f.push(...g),{...{baseUrl:r,autoThrottle:!1,concurrency:s,scraperConcurrency:a,retryDelay:t,maxRetryAttempts:i,retryOnStatusCode:o,maxWaitOn429:d,alwaysWaitOn429:n,limiter:f.length>0?{enable:!0,limiters:f}:!1},...b,baseUrl:r}}baseUrl;adapter;enableNavigationHistory;sessionId;rejectUnauthorized;userAgent;useRndUserAgent;timeout;maxRedirects;maxRetryAttempts;retryDelay;retryOnStatusCode;forceRevisit;retryWithoutProxyOnStatusCode;retryOnProxyError;maxRetryOnProxyError;allowRevisiting;enableCache;cacheTTL;cacheDir;throwFatalError;debug;enableSignalHandlers;concurrency;scraperConcurrency;maxDepth;maxUrls;maxResponseSize;respectRobotsTxt;followNofollow;autoThrottle;autoThrottleTargetDelay;autoThrottleMinDelay;autoThrottleMaxDelay;maxWaitOn429;alwaysWaitOn429;stealth;oxylabs=[];decodo=[];stealthProfiles=[];proxies=[];limiters=[];onLimiterAdded;onProviderAdded;requestHeaders=[];userAgents=$();constructor(e={}){this.baseUrl=e.baseUrl||"",this.adapter=e.adapter??"http",this.enableNavigationHistory=e.enableNavigationHistory??!1,this.sessionId=e.sessionId??`session_${Date.now()}_${Math.random().toString(36).slice(2,8)}`,this.rejectUnauthorized=e.rejectUnauthorized??!0,this.userAgent=e.userAgent,this.useRndUserAgent=e.useRndUserAgent??!1,this.timeout=e.timeout??30000,this.maxRedirects=e.maxRedirects??10,this.maxRetryAttempts=e.maxRetryAttempts??3,this.retryDelay=e.retryDelay??0,this.retryOnStatusCode=e.retryOnStatusCode??[408,429,500,502,503,504],this.forceRevisit=e.forceRevisit??!1,this.retryWithoutProxyOnStatusCode=e.retryWithoutProxyOnStatusCode??[407,403],this.retryOnProxyError=e.retryOnProxyError??!0,this.maxRetryOnProxyError=e.maxRetryOnProxyError??3,this.allowRevisiting=e.allowRevisiting??!1,this.enableCache=e.enableCache??!0,this.cacheTTL=e.cacheTTL??604800000,this.cacheDir=e.cacheDir??v.join(A.tmpdir(),"rezo_cache"),this.throwFatalError=e.throwFatalError??!1,this.debug=e.debug??!1,this.enableSignalHandlers=e.enableSignalHandlers??!1,this.concurrency=e.concurrency??100,this.scraperConcurrency=e.scraperConcurrency??this.concurrency,this.maxDepth=e.maxDepth??0,this.maxUrls=e.maxUrls??0,this.maxResponseSize=e.maxResponseSize??0,this.respectRobotsTxt=e.respectRobotsTxt??!1,this.followNofollow=e.followNofollow??!1,this.autoThrottle=e.autoThrottle??!0,this.autoThrottleTargetDelay=e.autoThrottleTargetDelay??1000,this.autoThrottleMinDelay=e.autoThrottleMinDelay??100,this.autoThrottleMaxDelay=e.autoThrottleMaxDelay??60000,this.maxWaitOn429=e.maxWaitOn429??1800000,this.alwaysWaitOn429=e.alwaysWaitOn429??!1,this._addHeaders(e.headers),this._addOxylabs(e.oxylabs),this._addDecodo(e.decodo),this._addProxies(e.proxy),this._addLimiters(e.limiter),this._addStealths(e.stealths),this.stealth=e.stealth}getConfiguredDomains(e){return(e==="headers"?this.requestHeaders:e==="limiters"?this.limiters:e==="oxylabs"?this.oxylabs:e==="decodo"?this.decodo:e==="stealth"?this.stealthProfiles:this.proxies).filter((s)=>s.domain).map((s)=>s.domain).filter((s,a,t)=>t.indexOf(s)===a)}removeDomain(e){this.requestHeaders=this.requestHeaders.filter((t)=>!t.domain||!this._domainsEqual(t.domain,e)),this.proxies=this.proxies.filter((t)=>!t.domain||!this._domainsEqual(t.domain,e));let r=l.splitConfigs(this.limiters,(t)=>!!t.domain&&this._domainsEqual(t.domain,e));l.destroyConfigQueues(r.removed),this.limiters=r.kept;let s=l.splitConfigs(this.oxylabs,(t)=>!!t.domain&&this._domainsEqual(t.domain,e));l.destroyConfigQueues(s.removed),this.oxylabs=s.kept;let a=l.splitConfigs(this.decodo,(t)=>!!t.domain&&this._domainsEqual(t.domain,e));return l.destroyConfigQueues(a.removed),this.decodo=a.kept,this.stealthProfiles=this.stealthProfiles.filter((t)=>!t.domain||!this._domainsEqual(t.domain,e)),this}_domainsEqual(e,r){if(Array.isArray(e)&&Array.isArray(r))return e.length===r.length&&e.every((s,a)=>s===r[a]);return e===r}getConfigurationSummary(){let e=(r)=>({total:r.length,global:r.filter((s)=>s.isGlobal).length,domainSpecific:r.filter((s)=>!s.isGlobal&&s.domain).length});return{headers:e(this.requestHeaders),proxies:e(this.proxies),limiters:e(this.limiters),oxylabs:e(this.oxylabs),decodo:e(this.decodo)}}_addHeaders(e){if(!e||!e.enable)return;for(let r of e.httpHeaders){let{domain:s,isGlobal:a,headers:t}=r;if(!s&&!a)continue;if(t instanceof Headers){let i=Object.fromEntries(t.entries());if(Object.keys(i).length<1)continue;t=i}else if(!t||Object.keys(t).length<1)continue;this.requestHeaders.push({domain:s,isGlobal:a,headers:t})}}_addProxies(e){if(!e||!e.enable)return;for(let r of e.proxies){let{domain:s,isGlobal:a,proxy:t,rotating:i}=r;if(!s&&!a)continue;if(!t||Object.keys(t).length<1)continue;this.proxies.push({domain:s,isGlobal:a,proxy:t,rotating:!!i})}}_addLimiters(e){if(!e||!e.enable)return;for(let r of e.limiters){let{domain:s,isGlobal:a,options:t,retry:i}=r;if(!s&&!a)continue;if(!t&&!i)continue;let o=t?new y(t):new y({name:"limiter"});if(this.limiters.push({domain:s,isGlobal:a,pqueue:o,randomDelay:t?.randomDelay,retry:i}),this.onLimiterAdded)this.onLimiterAdded(o)}}_addOxylabs(e){if(!e||!e.enable)return;for(let r of e.labs){let{domain:s,isGlobal:a,options:t,queueOptions:i}=r;if(!s&&!a)continue;if(!t||Object.keys(t).length<1)continue;let o=i?new y(i):void 0;if(this.oxylabs.push({domain:s,isGlobal:a,adaptar:new x(t),pqueue:o}),o&&this.onProviderAdded)this.onProviderAdded(o)}}_addDecodo(e){if(!e||!e.enable)return;for(let r of e.labs){let{domain:s,isGlobal:a,options:t,queueOptions:i}=r;if(!s&&!a)continue;if(!t||Object.keys(t).length<1)continue;let o=i?new y(i):void 0;if(this.decodo.push({domain:s,isGlobal:a,adaptar:new p(t),pqueue:o}),o&&this.onProviderAdded)this.onProviderAdded(o)}}_addStealths(e){if(!e||!e.enable)return;for(let r of e.profiles){let{domain:s,isGlobal:a,stealth:t}=r;if(!s&&!a)continue;if(!t)continue;this.stealthProfiles.push({domain:s,isGlobal:a,adaptar:new D({stealth:t})})}}addHeaders(e){return this._addHeaders({enable:!0,httpHeaders:[e]}),this}addProxy(e){return this._addProxies({enable:!0,proxies:[e]}),this}addProxies(e){return this._addProxies({enable:!0,proxies:e}),this}getProxyConfig(e,r,s){let a=this.selectConfigEntry(e,this.proxies,r,s);if(!a)return null;return{proxy:a.proxy,rotating:!!a.rotating}}addLimiter(e){return this._addLimiters({enable:!0,limiters:[e]}),this}addOxylabs(e){return this._addOxylabs({enable:!0,labs:[e]}),this}addDecodo(e){return this._addDecodo({enable:!0,labs:[e]}),this}addStealth(e){return this._addStealths({enable:!0,profiles:[e]}),this}destroyLimiters(){for(let e of this.limiters)l.destroyQueue(e.pqueue);this.limiters=[]}destroyProviderQueues(){for(let e of this.oxylabs)l.destroyQueue(e.pqueue),delete e.pqueue;for(let e of this.decodo)l.destroyQueue(e.pqueue),delete e.pqueue}getLimiters(){return this.limiters}getRandomDelay(e,r){if(!this.getDomainName(e))return;for(let a of this.limiters)if(this._hasDomain(e,a.domain)&&a.randomDelay!==void 0)return a.randomDelay;if(r){for(let a of this.limiters)if(a.isGlobal&&a.randomDelay!==void 0)return a.randomDelay}return}clearGlobalConfigs(){if(Array.isArray(this.requestHeaders))this.requestHeaders=this.requestHeaders.filter((e)=>!e.isGlobal);if(Array.isArray(this.oxylabs)){let e=l.splitConfigs(this.oxylabs,(r)=>!!r.isGlobal);l.destroyConfigQueues(e.removed),this.oxylabs=e.kept}if(Array.isArray(this.decodo)){let e=l.splitConfigs(this.decodo,(r)=>!!r.isGlobal);l.destroyConfigQueues(e.removed),this.decodo=e.kept}if(Array.isArray(this.limiters)){let e=l.splitConfigs(this.limiters,(r)=>!!r.isGlobal);l.destroyConfigQueues(e.removed),this.limiters=e.kept}if(Array.isArray(this.proxies))this.proxies=this.proxies.filter((e)=>!e.isGlobal);if(Array.isArray(this.stealthProfiles))this.stealthProfiles=this.stealthProfiles.filter((e)=>!e.isGlobal);return this}getAdapter(e,r,s,a){if(!this.getDomainName(e))return null;let i=r==="headers"?this.requestHeaders:r==="limiters"?this.limiters:r==="oxylabs"?this.oxylabs:r==="decodo"?this.decodo:r==="stealth"?this.stealthProfiles:this.proxies,o=this.selectConfigEntry(e,i,s,a);if(!o)return null;return r==="headers"?o.headers:r==="limiters"?o.pqueue:r==="oxylabs"?o.adaptar:r==="decodo"?o.adaptar:r==="stealth"?o.adaptar:o.proxy}getRetryOptions(e){if(!this.getDomainName(e))return null;for(let s=0;s<this.limiters.length;s++)if(this._hasDomain(e,this.limiters[s].domain))return this.limiters[s].retry||null;for(let s=0;s<this.limiters.length;s++)if(this.limiters[s].isGlobal&&this.limiters[s].retry)return this.limiters[s].retry;return null}rnd(e=0,r=Number.MAX_VALUE){return Math.floor(Math.random()*(r-e+1))+e}hasDomain(e,r,s){if(!this.getDomainName(e))return!1;let t=r==="headers"?this.requestHeaders:r==="limiters"?this.limiters:r==="oxylabs"?this.oxylabs:r==="decodo"?this.decodo:r==="stealth"?this.stealthProfiles:this.proxies;for(let i=0;i<t.length;i++)if(this._hasDomain(e,t[i].domain))return!0;if(s){for(let i=0;i<t.length;i++)if(t[i].isGlobal)return!0}return!1}pickHeaders(e,r,s,a){let t=this.getAdapter(e,"headers",r),i=new Headers(t??{}),o=i.count;if(s&&s instanceof Headers)for(let[d,n]of Object.entries(s.entries()))i.set(d,n);else if(s&&typeof s==="object"){for(let[d,n]of Object.entries(s))if(typeof n==="string")i.set(d,n)}if(a&&o===0&&!this.stealth)i.set("user-agent",this.getRandomUserAgent());return Object.fromEntries(i.entries())}_hasDomain(e,r){if(!r)return!1;let s=this.getDomainName(e);if(!s)return!1;let a=(i)=>{return/[\^\$\*\+\?\{\}\[\]\(\)\|\\]/.test(i)||i.startsWith("/")||i.includes(".*")||i.includes(".+")},t=(i)=>{if(i instanceof RegExp)return i.test(s)||i.test(e);let o=i.toString().trim();if(s.toLowerCase()===o.toLowerCase())return!0;if(o.includes("*")){let h=o.replace(/[.*+?^${}()|[\]\\]/g,"\\$&").replace(/\\\*/g,".*"),u=new RegExp(`^${h}$`,"i");return u.test(s)||u.test(e)}if(a(o))try{let h=o,u="i",c=o.match(/^\/(.*)\/(\w*)$/);if(c)h=c[1],u=c[2]||"i";let m=new RegExp(h,u);return m.test(s)||m.test(e)}catch(h){return s.toLowerCase().includes(o.toLowerCase())}let d=s.toLowerCase(),n=o.toLowerCase();return d===n||d.endsWith("."+n)||n.endsWith("."+d)};if(Array.isArray(r)){for(let i of r)if(t(i))return!0;return!1}return t(r)}selectConfigEntry(e,r,s,a){if(!this.getDomainName(e))return null;let i=(n)=>{if(!n.length)return null;let h=a?this.rnd(0,n.length-1):0;return n[h]},o=[];for(let n of r)if(this._hasDomain(e,n.domain))o.push(n);let d=i(o);if(d)return d;if(s){let n=[];for(let h of r)if(h.isGlobal)n.push(h);return i(n)}return null}getDomainName(e){if(this.isValidUrl(e))return new URL(e).hostname;else if(this.isHostName(e))return e;return null}isHostName(e){if(!e)return!1;if(e.length>255)return!1;let r=/^(?:(?:[a-zA-Z0-9](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?\.)+ [a-zA-Z]{2,})$/;return e=e.trim().toLowerCase(),r.test(e)&&!e.startsWith("-")&&!e.endsWith("-")}isValidUrl(e){if(!e)return!1;e=e.trim();try{let r=new URL(e);if(!r.protocol||!["http:","https:"].includes(r.protocol.toLowerCase()))return!1;if(!r.hostname)return!1;if(!/^(?:(?:[a-zA-Z0-9](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?\.)+[a-zA-Z]{2,})$/.test(r.hostname))return!1;return!0}catch{return!1}}getRandomUserAgent(){return this.userAgents[Math.floor(Math.random()*this.userAgents.length)]}}function $(){let e=[{name:"Chrome",version:"91.0.4472.124",engine:"AppleWebKit/537.36"},{name:"Firefox",version:"89.0",engine:"Gecko/20100101"},{name:"Safari",version:"14.1.1",engine:"AppleWebKit/605.1.15"},{name:"Edge",version:"91.0.864.59",engine:"AppleWebKit/537.36"},{name:"Opera",version:"77.0.4054.277",engine:"AppleWebKit/537.36"},{name:"Vivaldi",version:"3.8.2259.42",engine:"AppleWebKit/537.36"},{name:"Brave",version:"1.26.74",engine:"AppleWebKit/537.36"},{name:"Chromium",version:"91.0.4472.101",engine:"AppleWebKit/537.36"},{name:"Yandex",version:"21.5.3.742",engine:"AppleWebKit/537.36"},{name:"Maxthon",version:"5.3.8.2000",engine:"AppleWebKit/537.36"}],r=["Windows NT 10.0","Windows NT 6.1","Macintosh; Intel Mac OS X 10_15_7","Macintosh; Intel Mac OS X 11_4_0","X11; Linux x86_64","X11; Ubuntu; Linux x86_64"],s=[];for(let a=0;a<200;a++){let t=e[Math.floor(Math.random()*e.length)],i=r[Math.floor(Math.random()*r.length)],o="";switch(t.name){case"Chrome":o=`Mozilla/5.0 (${i}) ${t.engine} (KHTML, like Gecko) Chrome/${t.version} Safari/537.36`;break;case"Firefox":o=`Mozilla/5.0 (${i}; rv:${t.version}) ${t.engine} Firefox/${t.version}`;break;case"Safari":o=`Mozilla/5.0 (${i}) ${t.engine} (KHTML, like Gecko) Version/${t.version} Safari/605.1.15`;break;case"Edge":o=`Mozilla/5.0 (${i}) ${t.engine} (KHTML, like Gecko) Chrome/${t.version} Safari/537.36 Edg/${t.version}`;break;case"Opera":o=`Mozilla/5.0 (${i}) ${t.engine} (KHTML, like Gecko) Chrome/${t.version} Safari/537.36 OPR/${t.version}`;break;case"Vivaldi":o=`Mozilla/5.0 (${i}) ${t.engine} (KHTML, like Gecko) Chrome/${t.version} Safari/537.36 Vivaldi/${t.version}`;break;case"Brave":o=`Mozilla/5.0 (${i}) ${t.engine} (KHTML, like Gecko) Chrome/${t.version} Safari/537.36 Brave/${t.version}`;break;case"Chromium":o=`Mozilla/5.0 (${i}) ${t.engine} (KHTML, like Gecko) Chromium/${t.version} Chrome/${t.version} Safari/537.36`;break;case"Yandex":o=`Mozilla/5.0 (${i}) ${t.engine} (KHTML, like Gecko) Chrome/${t.version} YaBrowser/${t.version} Safari/537.36`;break;case"Maxthon":o=`Mozilla/5.0 (${i}) ${t.engine} (KHTML, like Gecko) Chrome/${t.version} Safari/537.36 Maxthon/${t.version}`;break}s.push(o)}return s}exports.CrawlerOptions=l;
+var{RezoQueue:y}=require("../queue/queue.cjs"),{Oxylabs:x}=require("./addon/oxylabs/index.cjs"),v=require("node:path"),A=require("node:os"),{Decodo:p}=require("./addon/decodo/index.cjs"),{Rezo:D}=require("../core/rezo.cjs");class l{static destroyQueue(e){if(e&&typeof e.destroy==="function")e.destroy()}static destroyConfigQueues(e){for(let r of e)l.destroyQueue(r.pqueue)}static splitConfigs(e,r){let s=[],a=[];for(let t of e)if(r(t))a.push(t);else s.push(t);return{kept:s,removed:a}}static getHostname(e){try{return new URL(e).hostname||void 0}catch{return}}static createStableThroughputOptions(e){let{baseUrl:r,concurrency:s=40,scraperConcurrency:a=10,retryDelay:t=1000,maxRetryAttempts:i=2,retryOnStatusCode:o=[408,500,502,503,504],maxWaitOn429:d=15000,alwaysWaitOn429:n=!1,globalLimiter:h={concurrency:8},domain:u=l.getHostname(r),domainLimiter:c={concurrency:2,interval:1000,intervalCap:2,randomDelay:150},domainRetry:m={enable:!0,max429Retries:2,retryDelay:1000,maxRetryAttempts:2,backoff:!0},extraLimiters:g=[],overrides:b={}}=e,f=[];if(h)f.push({isGlobal:!0,options:h});if(u&&c)f.push({domain:u,options:c,retry:m||void 0});return f.push(...g),{...{baseUrl:r,autoThrottle:!1,concurrency:s,scraperConcurrency:a,retryDelay:t,maxRetryAttempts:i,retryOnStatusCode:o,maxWaitOn429:d,alwaysWaitOn429:n,limiter:f.length>0?{enable:!0,limiters:f}:!1},...b,baseUrl:r}}baseUrl;adapter;enableNavigationHistory;sessionId;rejectUnauthorized;userAgent;useRndUserAgent;timeout;maxRedirects;maxRetryAttempts;retryDelay;retryOnStatusCode;forceRevisit;retryWithoutProxyOnStatusCode;retryOnProxyError;maxRetryOnProxyError;allowRevisiting;enableCache;cacheTTL;cacheDir;throwFatalError;debug;enableSignalHandlers;concurrency;scraperConcurrency;maxDepth;maxUrls;maxResponseSize;respectRobotsTxt;followNofollow;autoThrottle;autoThrottleTargetDelay;autoThrottleMinDelay;autoThrottleMaxDelay;maxWaitOn429;alwaysWaitOn429;stealth;oxylabs=[];decodo=[];stealthProfiles=[];proxies=[];limiters=[];onLimiterAdded;onProviderAdded;requestHeaders=[];userAgents=$();constructor(e={}){this.baseUrl=e.baseUrl||"",this.adapter=e.adapter??"http",this.enableNavigationHistory=e.enableNavigationHistory??!1,this.sessionId=e.sessionId??`session_${Date.now()}_${Math.random().toString(36).slice(2,8)}`,this.rejectUnauthorized=e.rejectUnauthorized??!0,this.userAgent=e.userAgent,this.useRndUserAgent=e.useRndUserAgent??!1,this.timeout=e.timeout??30000,this.maxRedirects=e.maxRedirects??10,this.maxRetryAttempts=e.maxRetryAttempts??3,this.retryDelay=e.retryDelay??0,this.retryOnStatusCode=e.retryOnStatusCode??[408,429,500,502,503,504],this.forceRevisit=e.forceRevisit??!1,this.retryWithoutProxyOnStatusCode=e.retryWithoutProxyOnStatusCode??[407,403],this.retryOnProxyError=e.retryOnProxyError??!0,this.maxRetryOnProxyError=e.maxRetryOnProxyError??3,this.allowRevisiting=e.allowRevisiting??!1,this.enableCache=e.enableCache??!0,this.cacheTTL=e.cacheTTL??604800000,this.cacheDir=e.cacheDir??v.join(A.tmpdir(),"rezo_cache"),this.throwFatalError=e.throwFatalError??!1,this.debug=e.debug??!1,this.enableSignalHandlers=e.enableSignalHandlers??!1,this.concurrency=e.concurrency??100,this.scraperConcurrency=e.scraperConcurrency??this.concurrency,this.maxDepth=e.maxDepth??0,this.maxUrls=e.maxUrls??0,this.maxResponseSize=e.maxResponseSize??0,this.respectRobotsTxt=e.respectRobotsTxt??!1,this.followNofollow=e.followNofollow??!1,this.autoThrottle=e.autoThrottle??!0,this.autoThrottleTargetDelay=e.autoThrottleTargetDelay??1000,this.autoThrottleMinDelay=e.autoThrottleMinDelay??100,this.autoThrottleMaxDelay=e.autoThrottleMaxDelay??60000,this.maxWaitOn429=e.maxWaitOn429??1800000,this.alwaysWaitOn429=e.alwaysWaitOn429??!1,this._addHeaders(e.headers),this._addOxylabs(e.oxylabs),this._addDecodo(e.decodo),this._addProxies(e.proxy),this._addLimiters(e.limiter),this._addStealths(e.stealths),this.stealth=e.stealth}getConfiguredDomains(e){return(e==="headers"?this.requestHeaders:e==="limiters"?this.limiters:e==="oxylabs"?this.oxylabs:e==="decodo"?this.decodo:e==="stealth"?this.stealthProfiles:this.proxies).filter((s)=>s.domain).map((s)=>s.domain).filter((s,a,t)=>t.indexOf(s)===a)}removeDomain(e){this.requestHeaders=this.requestHeaders.filter((t)=>!t.domain||!this._domainsEqual(t.domain,e)),this.proxies=this.proxies.filter((t)=>!t.domain||!this._domainsEqual(t.domain,e));let r=l.splitConfigs(this.limiters,(t)=>!!t.domain&&this._domainsEqual(t.domain,e));l.destroyConfigQueues(r.removed),this.limiters=r.kept;let s=l.splitConfigs(this.oxylabs,(t)=>!!t.domain&&this._domainsEqual(t.domain,e));l.destroyConfigQueues(s.removed),this.oxylabs=s.kept;let a=l.splitConfigs(this.decodo,(t)=>!!t.domain&&this._domainsEqual(t.domain,e));return l.destroyConfigQueues(a.removed),this.decodo=a.kept,this.stealthProfiles=this.stealthProfiles.filter((t)=>!t.domain||!this._domainsEqual(t.domain,e)),this}_domainsEqual(e,r){if(Array.isArray(e)&&Array.isArray(r))return e.length===r.length&&e.every((s,a)=>s===r[a]);return e===r}getConfigurationSummary(){let e=(r)=>({total:r.length,global:r.filter((s)=>s.isGlobal).length,domainSpecific:r.filter((s)=>!s.isGlobal&&s.domain).length});return{headers:e(this.requestHeaders),proxies:e(this.proxies),limiters:e(this.limiters),oxylabs:e(this.oxylabs),decodo:e(this.decodo)}}_addHeaders(e){if(!e||!e.enable)return;for(let r of e.httpHeaders){let{domain:s,isGlobal:a,headers:t}=r;if(!s&&!a)continue;if(t instanceof Headers){let i=Object.fromEntries(t.entries());if(Object.keys(i).length<1)continue;t=i}else if(!t||Object.keys(t).length<1)continue;this.requestHeaders.push({domain:s,isGlobal:a,headers:t})}}_addProxies(e){if(!e||!e.enable)return;for(let r of e.proxies){let{domain:s,isGlobal:a,proxy:t,rotating:i}=r;if(!s&&!a)continue;if(!t||Object.keys(t).length<1)continue;this.proxies.push({domain:s,isGlobal:a,proxy:t,rotating:!!i})}}_addLimiters(e){if(!e||!e.enable)return;for(let r of e.limiters){let{domain:s,isGlobal:a,options:t,retry:i}=r;if(!s&&!a)continue;if(!t&&!i)continue;let o=t?new y(t):new y({name:"limiter"});if(this.limiters.push({domain:s,isGlobal:a,pqueue:o,randomDelay:t?.randomDelay,retry:i}),this.onLimiterAdded)this.onLimiterAdded(o)}}_addOxylabs(e){if(!e||!e.enable)return;for(let r of e.labs){let{domain:s,isGlobal:a,options:t,queueOptions:i}=r;if(!s&&!a)continue;if(!t||Object.keys(t).length<1)continue;let o=i?new y(i):void 0;if(this.oxylabs.push({domain:s,isGlobal:a,adaptar:new x(t),pqueue:o}),o&&this.onProviderAdded)this.onProviderAdded(o)}}_addDecodo(e){if(!e||!e.enable)return;for(let r of e.labs){let{domain:s,isGlobal:a,options:t,queueOptions:i}=r;if(!s&&!a)continue;if(!t||Object.keys(t).length<1)continue;let o=i?new y(i):void 0;if(this.decodo.push({domain:s,isGlobal:a,adaptar:new p(t),pqueue:o}),o&&this.onProviderAdded)this.onProviderAdded(o)}}_addStealths(e){if(!e||!e.enable)return;for(let r of e.profiles){let{domain:s,isGlobal:a,stealth:t}=r;if(!s&&!a)continue;if(!t)continue;this.stealthProfiles.push({domain:s,isGlobal:a,adaptar:new D({stealth:t})})}}addHeaders(e){return this._addHeaders({enable:!0,httpHeaders:[e]}),this}addProxy(e){return this._addProxies({enable:!0,proxies:[e]}),this}addProxies(e){return this._addProxies({enable:!0,proxies:e}),this}getProxyConfig(e,r,s){let a=this.selectConfigEntry(e,this.proxies,r,s);if(!a)return null;return{proxy:a.proxy,rotating:!!a.rotating}}addLimiter(e){return this._addLimiters({enable:!0,limiters:[e]}),this}addOxylabs(e){return this._addOxylabs({enable:!0,labs:[e]}),this}addDecodo(e){return this._addDecodo({enable:!0,labs:[e]}),this}addStealth(e){return this._addStealths({enable:!0,profiles:[e]}),this}destroyLimiters(){for(let e of this.limiters)l.destroyQueue(e.pqueue);this.limiters=[]}destroyProviderQueues(){for(let e of this.oxylabs)l.destroyQueue(e.pqueue),delete e.pqueue;for(let e of this.decodo)l.destroyQueue(e.pqueue),delete e.pqueue}getLimiters(){return this.limiters}getRandomDelay(e,r){if(!this.getDomainName(e))return;for(let a of this.limiters)if(this._hasDomain(e,a.domain)&&a.randomDelay!==void 0)return a.randomDelay;if(r){for(let a of this.limiters)if(a.isGlobal&&a.randomDelay!==void 0)return a.randomDelay}return}clearGlobalConfigs(){if(Array.isArray(this.requestHeaders))this.requestHeaders=this.requestHeaders.filter((e)=>!e.isGlobal);if(Array.isArray(this.oxylabs)){let e=l.splitConfigs(this.oxylabs,(r)=>!!r.isGlobal);l.destroyConfigQueues(e.removed),this.oxylabs=e.kept}if(Array.isArray(this.decodo)){let e=l.splitConfigs(this.decodo,(r)=>!!r.isGlobal);l.destroyConfigQueues(e.removed),this.decodo=e.kept}if(Array.isArray(this.limiters)){let e=l.splitConfigs(this.limiters,(r)=>!!r.isGlobal);l.destroyConfigQueues(e.removed),this.limiters=e.kept}if(Array.isArray(this.proxies))this.proxies=this.proxies.filter((e)=>!e.isGlobal);if(Array.isArray(this.stealthProfiles))this.stealthProfiles=this.stealthProfiles.filter((e)=>!e.isGlobal);return this}getAdapter(e,r,s,a){if(!this.getDomainName(e))return null;let i=r==="headers"?this.requestHeaders:r==="limiters"?this.limiters:r==="oxylabs"?this.oxylabs:r==="decodo"?this.decodo:r==="stealth"?this.stealthProfiles:this.proxies,o=this.selectConfigEntry(e,i,s,a);if(!o)return null;return r==="headers"?o.headers:r==="limiters"?o.pqueue:r==="oxylabs"?o.adaptar:r==="decodo"?o.adaptar:r==="stealth"?o.adaptar:o.proxy}getRetryOptions(e){if(!this.getDomainName(e))return null;for(let s=0;s<this.limiters.length;s++)if(this._hasDomain(e,this.limiters[s].domain))return this.limiters[s].retry||null;for(let s=0;s<this.limiters.length;s++)if(this.limiters[s].isGlobal&&this.limiters[s].retry)return this.limiters[s].retry;return null}rnd(e=0,r=Number.MAX_VALUE){return Math.floor(Math.random()*(r-e+1))+e}hasDomain(e,r,s){if(!this.getDomainName(e))return!1;let t=r==="headers"?this.requestHeaders:r==="limiters"?this.limiters:r==="oxylabs"?this.oxylabs:r==="decodo"?this.decodo:r==="stealth"?this.stealthProfiles:this.proxies;for(let i=0;i<t.length;i++)if(this._hasDomain(e,t[i].domain))return!0;if(s){for(let i=0;i<t.length;i++)if(t[i].isGlobal)return!0}return!1}pickHeaders(e,r,s,a){let t=this.getAdapter(e,"headers",r),i=new Headers(t??{}),o=[...i.keys()].length;if(s&&s instanceof Headers)for(let[d,n]of Object.entries(s.entries()))i.set(d,n);else if(s&&typeof s==="object"){for(let[d,n]of Object.entries(s))if(typeof n==="string")i.set(d,n)}if(a&&o===0&&!this.stealth)i.set("user-agent",this.getRandomUserAgent());return Object.fromEntries(i.entries())}_hasDomain(e,r){if(!r)return!1;let s=this.getDomainName(e);if(!s)return!1;let a=(i)=>{return/[\^\$\*\+\?\{\}\[\]\(\)\|\\]/.test(i)||i.startsWith("/")||i.includes(".*")||i.includes(".+")},t=(i)=>{if(i instanceof RegExp)return i.test(s)||i.test(e);let o=i.toString().trim();if(s.toLowerCase()===o.toLowerCase())return!0;if(o.includes("*")){let h=o.replace(/[.*+?^${}()|[\]\\]/g,"\\$&").replace(/\\\*/g,".*"),u=new RegExp(`^${h}$`,"i");return u.test(s)||u.test(e)}if(a(o))try{let h=o,u="i",c=o.match(/^\/(.*)\/(\w*)$/);if(c)h=c[1],u=c[2]||"i";let m=new RegExp(h,u);return m.test(s)||m.test(e)}catch(h){return s.toLowerCase().includes(o.toLowerCase())}let d=s.toLowerCase(),n=o.toLowerCase();return d===n||d.endsWith("."+n)||n.endsWith("."+d)};if(Array.isArray(r)){for(let i of r)if(t(i))return!0;return!1}return t(r)}selectConfigEntry(e,r,s,a){if(!this.getDomainName(e))return null;let i=(n)=>{if(!n.length)return null;let h=a?this.rnd(0,n.length-1):0;return n[h]},o=[];for(let n of r)if(this._hasDomain(e,n.domain))o.push(n);let d=i(o);if(d)return d;if(s){let n=[];for(let h of r)if(h.isGlobal)n.push(h);return i(n)}return null}getDomainName(e){if(this.isValidUrl(e))return new URL(e).hostname;else if(this.isHostName(e))return e;return null}isHostName(e){if(!e)return!1;if(e.length>255)return!1;let r=/^(?:(?:[a-zA-Z0-9](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?\.)+ [a-zA-Z]{2,})$/;return e=e.trim().toLowerCase(),r.test(e)&&!e.startsWith("-")&&!e.endsWith("-")}isValidUrl(e){if(!e)return!1;e=e.trim();try{let r=new URL(e);if(!r.protocol||!["http:","https:"].includes(r.protocol.toLowerCase()))return!1;if(!r.hostname)return!1;if(!/^(?:(?:[a-zA-Z0-9](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?\.)+[a-zA-Z]{2,})$/.test(r.hostname))return!1;return!0}catch{return!1}}getRandomUserAgent(){return this.userAgents[Math.floor(Math.random()*this.userAgents.length)]}}function $(){let e=[{name:"Chrome",version:"91.0.4472.124",engine:"AppleWebKit/537.36"},{name:"Firefox",version:"89.0",engine:"Gecko/20100101"},{name:"Safari",version:"14.1.1",engine:"AppleWebKit/605.1.15"},{name:"Edge",version:"91.0.864.59",engine:"AppleWebKit/537.36"},{name:"Opera",version:"77.0.4054.277",engine:"AppleWebKit/537.36"},{name:"Vivaldi",version:"3.8.2259.42",engine:"AppleWebKit/537.36"},{name:"Brave",version:"1.26.74",engine:"AppleWebKit/537.36"},{name:"Chromium",version:"91.0.4472.101",engine:"AppleWebKit/537.36"},{name:"Yandex",version:"21.5.3.742",engine:"AppleWebKit/537.36"},{name:"Maxthon",version:"5.3.8.2000",engine:"AppleWebKit/537.36"}],r=["Windows NT 10.0","Windows NT 6.1","Macintosh; Intel Mac OS X 10_15_7","Macintosh; Intel Mac OS X 11_4_0","X11; Linux x86_64","X11; Ubuntu; Linux x86_64"],s=[];for(let a=0;a<200;a++){let t=e[Math.floor(Math.random()*e.length)],i=r[Math.floor(Math.random()*r.length)],o="";switch(t.name){case"Chrome":o=`Mozilla/5.0 (${i}) ${t.engine} (KHTML, like Gecko) Chrome/${t.version} Safari/537.36`;break;case"Firefox":o=`Mozilla/5.0 (${i}; rv:${t.version}) ${t.engine} Firefox/${t.version}`;break;case"Safari":o=`Mozilla/5.0 (${i}) ${t.engine} (KHTML, like Gecko) Version/${t.version} Safari/605.1.15`;break;case"Edge":o=`Mozilla/5.0 (${i}) ${t.engine} (KHTML, like Gecko) Chrome/${t.version} Safari/537.36 Edg/${t.version}`;break;case"Opera":o=`Mozilla/5.0 (${i}) ${t.engine} (KHTML, like Gecko) Chrome/${t.version} Safari/537.36 OPR/${t.version}`;break;case"Vivaldi":o=`Mozilla/5.0 (${i}) ${t.engine} (KHTML, like Gecko) Chrome/${t.version} Safari/537.36 Vivaldi/${t.version}`;break;case"Brave":o=`Mozilla/5.0 (${i}) ${t.engine} (KHTML, like Gecko) Chrome/${t.version} Safari/537.36 Brave/${t.version}`;break;case"Chromium":o=`Mozilla/5.0 (${i}) ${t.engine} (KHTML, like Gecko) Chromium/${t.version} Chrome/${t.version} Safari/537.36`;break;case"Yandex":o=`Mozilla/5.0 (${i}) ${t.engine} (KHTML, like Gecko) Chrome/${t.version} YaBrowser/${t.version} Safari/537.36`;break;case"Maxthon":o=`Mozilla/5.0 (${i}) ${t.engine} (KHTML, like Gecko) Chrome/${t.version} Safari/537.36 Maxthon/${t.version}`;break}s.push(o)}return s}exports.CrawlerOptions=l;

package/dist/crawler/crawler-options.js

@@ -1 +1 @@
-import{RezoQueue as y}from"../queue/queue.js";import{Oxylabs as x}from"./addon/oxylabs/index.js";import p from"node:path";import v from"node:os";import{Decodo as A}from"./addon/decodo/index.js";import{Rezo as D}from"../core/rezo.js";class l{static destroyQueue(e){if(e&&typeof e.destroy==="function")e.destroy()}static destroyConfigQueues(e){for(let r of e)l.destroyQueue(r.pqueue)}static splitConfigs(e,r){let i=[],a=[];for(let t of e)if(r(t))a.push(t);else i.push(t);return{kept:i,removed:a}}static getHostname(e){try{return new URL(e).hostname||void 0}catch{return}}static createStableThroughputOptions(e){let{baseUrl:r,concurrency:i=40,scraperConcurrency:a=10,retryDelay:t=1000,maxRetryAttempts:s=2,retryOnStatusCode:o=[408,500,502,503,504],maxWaitOn429:d=15000,alwaysWaitOn429:n=!1,globalLimiter:h={concurrency:8},domain:u=l.getHostname(r),domainLimiter:c={concurrency:2,interval:1000,intervalCap:2,randomDelay:150},domainRetry:m={enable:!0,max429Retries:2,retryDelay:1000,maxRetryAttempts:2,backoff:!0},extraLimiters:g=[],overrides:b={}}=e,f=[];if(h)f.push({isGlobal:!0,options:h});if(u&&c)f.push({domain:u,options:c,retry:m||void 0});return f.push(...g),{...{baseUrl:r,autoThrottle:!1,concurrency:i,scraperConcurrency:a,retryDelay:t,maxRetryAttempts:s,retryOnStatusCode:o,maxWaitOn429:d,alwaysWaitOn429:n,limiter:f.length>0?{enable:!0,limiters:f}:!1},...b,baseUrl:r}}baseUrl;adapter;enableNavigationHistory;sessionId;rejectUnauthorized;userAgent;useRndUserAgent;timeout;maxRedirects;maxRetryAttempts;retryDelay;retryOnStatusCode;forceRevisit;retryWithoutProxyOnStatusCode;retryOnProxyError;maxRetryOnProxyError;allowRevisiting;enableCache;cacheTTL;cacheDir;throwFatalError;debug;enableSignalHandlers;concurrency;scraperConcurrency;maxDepth;maxUrls;maxResponseSize;respectRobotsTxt;followNofollow;autoThrottle;autoThrottleTargetDelay;autoThrottleMinDelay;autoThrottleMaxDelay;maxWaitOn429;alwaysWaitOn429;stealth;oxylabs=[];decodo=[];stealthProfiles=[];proxies=[];limiters=[];onLimiterAdded;onProviderAdded;requestHeaders=[];userAgents=$();constructor(e={}){this.baseUrl=e.baseUrl||"",this.adapter=e.adapter??"http",this.enableNavigationHistory=e.enableNavigationHistory??!1,this.sessionId=e.sessionId??`session_${Date.now()}_${Math.random().toString(36).slice(2,8)}`,this.rejectUnauthorized=e.rejectUnauthorized??!0,this.userAgent=e.userAgent,this.useRndUserAgent=e.useRndUserAgent??!1,this.timeout=e.timeout??30000,this.maxRedirects=e.maxRedirects??10,this.maxRetryAttempts=e.maxRetryAttempts??3,this.retryDelay=e.retryDelay??0,this.retryOnStatusCode=e.retryOnStatusCode??[408,429,500,502,503,504],this.forceRevisit=e.forceRevisit??!1,this.retryWithoutProxyOnStatusCode=e.retryWithoutProxyOnStatusCode??[407,403],this.retryOnProxyError=e.retryOnProxyError??!0,this.maxRetryOnProxyError=e.maxRetryOnProxyError??3,this.allowRevisiting=e.allowRevisiting??!1,this.enableCache=e.enableCache??!0,this.cacheTTL=e.cacheTTL??604800000,this.cacheDir=e.cacheDir??p.join(v.tmpdir(),"rezo_cache"),this.throwFatalError=e.throwFatalError??!1,this.debug=e.debug??!1,this.enableSignalHandlers=e.enableSignalHandlers??!1,this.concurrency=e.concurrency??100,this.scraperConcurrency=e.scraperConcurrency??this.concurrency,this.maxDepth=e.maxDepth??0,this.maxUrls=e.maxUrls??0,this.maxResponseSize=e.maxResponseSize??0,this.respectRobotsTxt=e.respectRobotsTxt??!1,this.followNofollow=e.followNofollow??!1,this.autoThrottle=e.autoThrottle??!0,this.autoThrottleTargetDelay=e.autoThrottleTargetDelay??1000,this.autoThrottleMinDelay=e.autoThrottleMinDelay??100,this.autoThrottleMaxDelay=e.autoThrottleMaxDelay??60000,this.maxWaitOn429=e.maxWaitOn429??1800000,this.alwaysWaitOn429=e.alwaysWaitOn429??!1,this._addHeaders(e.headers),this._addOxylabs(e.oxylabs),this._addDecodo(e.decodo),this._addProxies(e.proxy),this._addLimiters(e.limiter),this._addStealths(e.stealths),this.stealth=e.stealth}getConfiguredDomains(e){return(e==="headers"?this.requestHeaders:e==="limiters"?this.limiters:e==="oxylabs"?this.oxylabs:e==="decodo"?this.decodo:e==="stealth"?this.stealthProfiles:this.proxies).filter((i)=>i.domain).map((i)=>i.domain).filter((i,a,t)=>t.indexOf(i)===a)}removeDomain(e){this.requestHeaders=this.requestHeaders.filter((t)=>!t.domain||!this._domainsEqual(t.domain,e)),this.proxies=this.proxies.filter((t)=>!t.domain||!this._domainsEqual(t.domain,e));let r=l.splitConfigs(this.limiters,(t)=>!!t.domain&&this._domainsEqual(t.domain,e));l.destroyConfigQueues(r.removed),this.limiters=r.kept;let i=l.splitConfigs(this.oxylabs,(t)=>!!t.domain&&this._domainsEqual(t.domain,e));l.destroyConfigQueues(i.removed),this.oxylabs=i.kept;let a=l.splitConfigs(this.decodo,(t)=>!!t.domain&&this._domainsEqual(t.domain,e));return l.destroyConfigQueues(a.removed),this.decodo=a.kept,this.stealthProfiles=this.stealthProfiles.filter((t)=>!t.domain||!this._domainsEqual(t.domain,e)),this}_domainsEqual(e,r){if(Array.isArray(e)&&Array.isArray(r))return e.length===r.length&&e.every((i,a)=>i===r[a]);return e===r}getConfigurationSummary(){let e=(r)=>({total:r.length,global:r.filter((i)=>i.isGlobal).length,domainSpecific:r.filter((i)=>!i.isGlobal&&i.domain).length});return{headers:e(this.requestHeaders),proxies:e(this.proxies),limiters:e(this.limiters),oxylabs:e(this.oxylabs),decodo:e(this.decodo)}}_addHeaders(e){if(!e||!e.enable)return;for(let r of e.httpHeaders){let{domain:i,isGlobal:a,headers:t}=r;if(!i&&!a)continue;if(t instanceof Headers){let s=Object.fromEntries(t.entries());if(Object.keys(s).length<1)continue;t=s}else if(!t||Object.keys(t).length<1)continue;this.requestHeaders.push({domain:i,isGlobal:a,headers:t})}}_addProxies(e){if(!e||!e.enable)return;for(let r of e.proxies){let{domain:i,isGlobal:a,proxy:t,rotating:s}=r;if(!i&&!a)continue;if(!t||Object.keys(t).length<1)continue;this.proxies.push({domain:i,isGlobal:a,proxy:t,rotating:!!s})}}_addLimiters(e){if(!e||!e.enable)return;for(let r of e.limiters){let{domain:i,isGlobal:a,options:t,retry:s}=r;if(!i&&!a)continue;if(!t&&!s)continue;let o=t?new y(t):new y({name:"limiter"});if(this.limiters.push({domain:i,isGlobal:a,pqueue:o,randomDelay:t?.randomDelay,retry:s}),this.onLimiterAdded)this.onLimiterAdded(o)}}_addOxylabs(e){if(!e||!e.enable)return;for(let r of e.labs){let{domain:i,isGlobal:a,options:t,queueOptions:s}=r;if(!i&&!a)continue;if(!t||Object.keys(t).length<1)continue;let o=s?new y(s):void 0;if(this.oxylabs.push({domain:i,isGlobal:a,adaptar:new x(t),pqueue:o}),o&&this.onProviderAdded)this.onProviderAdded(o)}}_addDecodo(e){if(!e||!e.enable)return;for(let r of e.labs){let{domain:i,isGlobal:a,options:t,queueOptions:s}=r;if(!i&&!a)continue;if(!t||Object.keys(t).length<1)continue;let o=s?new y(s):void 0;if(this.decodo.push({domain:i,isGlobal:a,adaptar:new A(t),pqueue:o}),o&&this.onProviderAdded)this.onProviderAdded(o)}}_addStealths(e){if(!e||!e.enable)return;for(let r of e.profiles){let{domain:i,isGlobal:a,stealth:t}=r;if(!i&&!a)continue;if(!t)continue;this.stealthProfiles.push({domain:i,isGlobal:a,adaptar:new D({stealth:t})})}}addHeaders(e){return this._addHeaders({enable:!0,httpHeaders:[e]}),this}addProxy(e){return this._addProxies({enable:!0,proxies:[e]}),this}addProxies(e){return this._addProxies({enable:!0,proxies:e}),this}getProxyConfig(e,r,i){let a=this.selectConfigEntry(e,this.proxies,r,i);if(!a)return null;return{proxy:a.proxy,rotating:!!a.rotating}}addLimiter(e){return this._addLimiters({enable:!0,limiters:[e]}),this}addOxylabs(e){return this._addOxylabs({enable:!0,labs:[e]}),this}addDecodo(e){return this._addDecodo({enable:!0,labs:[e]}),this}addStealth(e){return this._addStealths({enable:!0,profiles:[e]}),this}destroyLimiters(){for(let e of this.limiters)l.destroyQueue(e.pqueue);this.limiters=[]}destroyProviderQueues(){for(let e of this.oxylabs)l.destroyQueue(e.pqueue),delete e.pqueue;for(let e of this.decodo)l.destroyQueue(e.pqueue),delete e.pqueue}getLimiters(){return this.limiters}getRandomDelay(e,r){if(!this.getDomainName(e))return;for(let a of this.limiters)if(this._hasDomain(e,a.domain)&&a.randomDelay!==void 0)return a.randomDelay;if(r){for(let a of this.limiters)if(a.isGlobal&&a.randomDelay!==void 0)return a.randomDelay}return}clearGlobalConfigs(){if(Array.isArray(this.requestHeaders))this.requestHeaders=this.requestHeaders.filter((e)=>!e.isGlobal);if(Array.isArray(this.oxylabs)){let e=l.splitConfigs(this.oxylabs,(r)=>!!r.isGlobal);l.destroyConfigQueues(e.removed),this.oxylabs=e.kept}if(Array.isArray(this.decodo)){let e=l.splitConfigs(this.decodo,(r)=>!!r.isGlobal);l.destroyConfigQueues(e.removed),this.decodo=e.kept}if(Array.isArray(this.limiters)){let e=l.splitConfigs(this.limiters,(r)=>!!r.isGlobal);l.destroyConfigQueues(e.removed),this.limiters=e.kept}if(Array.isArray(this.proxies))this.proxies=this.proxies.filter((e)=>!e.isGlobal);if(Array.isArray(this.stealthProfiles))this.stealthProfiles=this.stealthProfiles.filter((e)=>!e.isGlobal);return this}getAdapter(e,r,i,a){if(!this.getDomainName(e))return null;let s=r==="headers"?this.requestHeaders:r==="limiters"?this.limiters:r==="oxylabs"?this.oxylabs:r==="decodo"?this.decodo:r==="stealth"?this.stealthProfiles:this.proxies,o=this.selectConfigEntry(e,s,i,a);if(!o)return null;return r==="headers"?o.headers:r==="limiters"?o.pqueue:r==="oxylabs"?o.adaptar:r==="decodo"?o.adaptar:r==="stealth"?o.adaptar:o.proxy}getRetryOptions(e){if(!this.getDomainName(e))return null;for(let i=0;i<this.limiters.length;i++)if(this._hasDomain(e,this.limiters[i].domain))return this.limiters[i].retry||null;for(let i=0;i<this.limiters.length;i++)if(this.limiters[i].isGlobal&&this.limiters[i].retry)return this.limiters[i].retry;return null}rnd(e=0,r=Number.MAX_VALUE){return Math.floor(Math.random()*(r-e+1))+e}hasDomain(e,r,i){if(!this.getDomainName(e))return!1;let t=r==="headers"?this.requestHeaders:r==="limiters"?this.limiters:r==="oxylabs"?this.oxylabs:r==="decodo"?this.decodo:r==="stealth"?this.stealthProfiles:this.proxies;for(let s=0;s<t.length;s++)if(this._hasDomain(e,t[s].domain))return!0;if(i){for(let s=0;s<t.length;s++)if(t[s].isGlobal)return!0}return!1}pickHeaders(e,r,i,a){let t=this.getAdapter(e,"headers",r),s=new Headers(t??{}),o=s.count;if(i&&i instanceof Headers)for(let[d,n]of Object.entries(i.entries()))s.set(d,n);else if(i&&typeof i==="object"){for(let[d,n]of Object.entries(i))if(typeof n==="string")s.set(d,n)}if(a&&o===0&&!this.stealth)s.set("user-agent",this.getRandomUserAgent());return Object.fromEntries(s.entries())}_hasDomain(e,r){if(!r)return!1;let i=this.getDomainName(e);if(!i)return!1;let a=(s)=>{return/[\^\$\*\+\?\{\}\[\]\(\)\|\\]/.test(s)||s.startsWith("/")||s.includes(".*")||s.includes(".+")},t=(s)=>{if(s instanceof RegExp)return s.test(i)||s.test(e);let o=s.toString().trim();if(i.toLowerCase()===o.toLowerCase())return!0;if(o.includes("*")){let h=o.replace(/[.*+?^${}()|[\]\\]/g,"\\$&").replace(/\\\*/g,".*"),u=new RegExp(`^${h}$`,"i");return u.test(i)||u.test(e)}if(a(o))try{let h=o,u="i",c=o.match(/^\/(.*)\/(\w*)$/);if(c)h=c[1],u=c[2]||"i";let m=new RegExp(h,u);return m.test(i)||m.test(e)}catch(h){return i.toLowerCase().includes(o.toLowerCase())}let d=i.toLowerCase(),n=o.toLowerCase();return d===n||d.endsWith("."+n)||n.endsWith("."+d)};if(Array.isArray(r)){for(let s of r)if(t(s))return!0;return!1}return t(r)}selectConfigEntry(e,r,i,a){if(!this.getDomainName(e))return null;let s=(n)=>{if(!n.length)return null;let h=a?this.rnd(0,n.length-1):0;return n[h]},o=[];for(let n of r)if(this._hasDomain(e,n.domain))o.push(n);let d=s(o);if(d)return d;if(i){let n=[];for(let h of r)if(h.isGlobal)n.push(h);return s(n)}return null}getDomainName(e){if(this.isValidUrl(e))return new URL(e).hostname;else if(this.isHostName(e))return e;return null}isHostName(e){if(!e)return!1;if(e.length>255)return!1;let r=/^(?:(?:[a-zA-Z0-9](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?\.)+ [a-zA-Z]{2,})$/;return e=e.trim().toLowerCase(),r.test(e)&&!e.startsWith("-")&&!e.endsWith("-")}isValidUrl(e){if(!e)return!1;e=e.trim();try{let r=new URL(e);if(!r.protocol||!["http:","https:"].includes(r.protocol.toLowerCase()))return!1;if(!r.hostname)return!1;if(!/^(?:(?:[a-zA-Z0-9](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?\.)+[a-zA-Z]{2,})$/.test(r.hostname))return!1;return!0}catch{return!1}}getRandomUserAgent(){return this.userAgents[Math.floor(Math.random()*this.userAgents.length)]}}function $(){let e=[{name:"Chrome",version:"91.0.4472.124",engine:"AppleWebKit/537.36"},{name:"Firefox",version:"89.0",engine:"Gecko/20100101"},{name:"Safari",version:"14.1.1",engine:"AppleWebKit/605.1.15"},{name:"Edge",version:"91.0.864.59",engine:"AppleWebKit/537.36"},{name:"Opera",version:"77.0.4054.277",engine:"AppleWebKit/537.36"},{name:"Vivaldi",version:"3.8.2259.42",engine:"AppleWebKit/537.36"},{name:"Brave",version:"1.26.74",engine:"AppleWebKit/537.36"},{name:"Chromium",version:"91.0.4472.101",engine:"AppleWebKit/537.36"},{name:"Yandex",version:"21.5.3.742",engine:"AppleWebKit/537.36"},{name:"Maxthon",version:"5.3.8.2000",engine:"AppleWebKit/537.36"}],r=["Windows NT 10.0","Windows NT 6.1","Macintosh; Intel Mac OS X 10_15_7","Macintosh; Intel Mac OS X 11_4_0","X11; Linux x86_64","X11; Ubuntu; Linux x86_64"],i=[];for(let a=0;a<200;a++){let t=e[Math.floor(Math.random()*e.length)],s=r[Math.floor(Math.random()*r.length)],o="";switch(t.name){case"Chrome":o=`Mozilla/5.0 (${s}) ${t.engine} (KHTML, like Gecko) Chrome/${t.version} Safari/537.36`;break;case"Firefox":o=`Mozilla/5.0 (${s}; rv:${t.version}) ${t.engine} Firefox/${t.version}`;break;case"Safari":o=`Mozilla/5.0 (${s}) ${t.engine} (KHTML, like Gecko) Version/${t.version} Safari/605.1.15`;break;case"Edge":o=`Mozilla/5.0 (${s}) ${t.engine} (KHTML, like Gecko) Chrome/${t.version} Safari/537.36 Edg/${t.version}`;break;case"Opera":o=`Mozilla/5.0 (${s}) ${t.engine} (KHTML, like Gecko) Chrome/${t.version} Safari/537.36 OPR/${t.version}`;break;case"Vivaldi":o=`Mozilla/5.0 (${s}) ${t.engine} (KHTML, like Gecko) Chrome/${t.version} Safari/537.36 Vivaldi/${t.version}`;break;case"Brave":o=`Mozilla/5.0 (${s}) ${t.engine} (KHTML, like Gecko) Chrome/${t.version} Safari/537.36 Brave/${t.version}`;break;case"Chromium":o=`Mozilla/5.0 (${s}) ${t.engine} (KHTML, like Gecko) Chromium/${t.version} Chrome/${t.version} Safari/537.36`;break;case"Yandex":o=`Mozilla/5.0 (${s}) ${t.engine} (KHTML, like Gecko) Chrome/${t.version} YaBrowser/${t.version} Safari/537.36`;break;case"Maxthon":o=`Mozilla/5.0 (${s}) ${t.engine} (KHTML, like Gecko) Chrome/${t.version} Safari/537.36 Maxthon/${t.version}`;break}i.push(o)}return i}export{l as CrawlerOptions};
+import{RezoQueue as y}from"../queue/queue.js";import{Oxylabs as x}from"./addon/oxylabs/index.js";import p from"node:path";import v from"node:os";import{Decodo as A}from"./addon/decodo/index.js";import{Rezo as D}from"../core/rezo.js";class l{static destroyQueue(e){if(e&&typeof e.destroy==="function")e.destroy()}static destroyConfigQueues(e){for(let r of e)l.destroyQueue(r.pqueue)}static splitConfigs(e,r){let i=[],a=[];for(let t of e)if(r(t))a.push(t);else i.push(t);return{kept:i,removed:a}}static getHostname(e){try{return new URL(e).hostname||void 0}catch{return}}static createStableThroughputOptions(e){let{baseUrl:r,concurrency:i=40,scraperConcurrency:a=10,retryDelay:t=1000,maxRetryAttempts:s=2,retryOnStatusCode:o=[408,500,502,503,504],maxWaitOn429:d=15000,alwaysWaitOn429:n=!1,globalLimiter:h={concurrency:8},domain:u=l.getHostname(r),domainLimiter:c={concurrency:2,interval:1000,intervalCap:2,randomDelay:150},domainRetry:m={enable:!0,max429Retries:2,retryDelay:1000,maxRetryAttempts:2,backoff:!0},extraLimiters:g=[],overrides:b={}}=e,f=[];if(h)f.push({isGlobal:!0,options:h});if(u&&c)f.push({domain:u,options:c,retry:m||void 0});return f.push(...g),{...{baseUrl:r,autoThrottle:!1,concurrency:i,scraperConcurrency:a,retryDelay:t,maxRetryAttempts:s,retryOnStatusCode:o,maxWaitOn429:d,alwaysWaitOn429:n,limiter:f.length>0?{enable:!0,limiters:f}:!1},...b,baseUrl:r}}baseUrl;adapter;enableNavigationHistory;sessionId;rejectUnauthorized;userAgent;useRndUserAgent;timeout;maxRedirects;maxRetryAttempts;retryDelay;retryOnStatusCode;forceRevisit;retryWithoutProxyOnStatusCode;retryOnProxyError;maxRetryOnProxyError;allowRevisiting;enableCache;cacheTTL;cacheDir;throwFatalError;debug;enableSignalHandlers;concurrency;scraperConcurrency;maxDepth;maxUrls;maxResponseSize;respectRobotsTxt;followNofollow;autoThrottle;autoThrottleTargetDelay;autoThrottleMinDelay;autoThrottleMaxDelay;maxWaitOn429;alwaysWaitOn429;stealth;oxylabs=[];decodo=[];stealthProfiles=[];proxies=[];limiters=[];onLimiterAdded;onProviderAdded;requestHeaders=[];userAgents=$();constructor(e={}){this.baseUrl=e.baseUrl||"",this.adapter=e.adapter??"http",this.enableNavigationHistory=e.enableNavigationHistory??!1,this.sessionId=e.sessionId??`session_${Date.now()}_${Math.random().toString(36).slice(2,8)}`,this.rejectUnauthorized=e.rejectUnauthorized??!0,this.userAgent=e.userAgent,this.useRndUserAgent=e.useRndUserAgent??!1,this.timeout=e.timeout??30000,this.maxRedirects=e.maxRedirects??10,this.maxRetryAttempts=e.maxRetryAttempts??3,this.retryDelay=e.retryDelay??0,this.retryOnStatusCode=e.retryOnStatusCode??[408,429,500,502,503,504],this.forceRevisit=e.forceRevisit??!1,this.retryWithoutProxyOnStatusCode=e.retryWithoutProxyOnStatusCode??[407,403],this.retryOnProxyError=e.retryOnProxyError??!0,this.maxRetryOnProxyError=e.maxRetryOnProxyError??3,this.allowRevisiting=e.allowRevisiting??!1,this.enableCache=e.enableCache??!0,this.cacheTTL=e.cacheTTL??604800000,this.cacheDir=e.cacheDir??p.join(v.tmpdir(),"rezo_cache"),this.throwFatalError=e.throwFatalError??!1,this.debug=e.debug??!1,this.enableSignalHandlers=e.enableSignalHandlers??!1,this.concurrency=e.concurrency??100,this.scraperConcurrency=e.scraperConcurrency??this.concurrency,this.maxDepth=e.maxDepth??0,this.maxUrls=e.maxUrls??0,this.maxResponseSize=e.maxResponseSize??0,this.respectRobotsTxt=e.respectRobotsTxt??!1,this.followNofollow=e.followNofollow??!1,this.autoThrottle=e.autoThrottle??!0,this.autoThrottleTargetDelay=e.autoThrottleTargetDelay??1000,this.autoThrottleMinDelay=e.autoThrottleMinDelay??100,this.autoThrottleMaxDelay=e.autoThrottleMaxDelay??60000,this.maxWaitOn429=e.maxWaitOn429??1800000,this.alwaysWaitOn429=e.alwaysWaitOn429??!1,this._addHeaders(e.headers),this._addOxylabs(e.oxylabs),this._addDecodo(e.decodo),this._addProxies(e.proxy),this._addLimiters(e.limiter),this._addStealths(e.stealths),this.stealth=e.stealth}getConfiguredDomains(e){return(e==="headers"?this.requestHeaders:e==="limiters"?this.limiters:e==="oxylabs"?this.oxylabs:e==="decodo"?this.decodo:e==="stealth"?this.stealthProfiles:this.proxies).filter((i)=>i.domain).map((i)=>i.domain).filter((i,a,t)=>t.indexOf(i)===a)}removeDomain(e){this.requestHeaders=this.requestHeaders.filter((t)=>!t.domain||!this._domainsEqual(t.domain,e)),this.proxies=this.proxies.filter((t)=>!t.domain||!this._domainsEqual(t.domain,e));let r=l.splitConfigs(this.limiters,(t)=>!!t.domain&&this._domainsEqual(t.domain,e));l.destroyConfigQueues(r.removed),this.limiters=r.kept;let i=l.splitConfigs(this.oxylabs,(t)=>!!t.domain&&this._domainsEqual(t.domain,e));l.destroyConfigQueues(i.removed),this.oxylabs=i.kept;let a=l.splitConfigs(this.decodo,(t)=>!!t.domain&&this._domainsEqual(t.domain,e));return l.destroyConfigQueues(a.removed),this.decodo=a.kept,this.stealthProfiles=this.stealthProfiles.filter((t)=>!t.domain||!this._domainsEqual(t.domain,e)),this}_domainsEqual(e,r){if(Array.isArray(e)&&Array.isArray(r))return e.length===r.length&&e.every((i,a)=>i===r[a]);return e===r}getConfigurationSummary(){let e=(r)=>({total:r.length,global:r.filter((i)=>i.isGlobal).length,domainSpecific:r.filter((i)=>!i.isGlobal&&i.domain).length});return{headers:e(this.requestHeaders),proxies:e(this.proxies),limiters:e(this.limiters),oxylabs:e(this.oxylabs),decodo:e(this.decodo)}}_addHeaders(e){if(!e||!e.enable)return;for(let r of e.httpHeaders){let{domain:i,isGlobal:a,headers:t}=r;if(!i&&!a)continue;if(t instanceof Headers){let s=Object.fromEntries(t.entries());if(Object.keys(s).length<1)continue;t=s}else if(!t||Object.keys(t).length<1)continue;this.requestHeaders.push({domain:i,isGlobal:a,headers:t})}}_addProxies(e){if(!e||!e.enable)return;for(let r of e.proxies){let{domain:i,isGlobal:a,proxy:t,rotating:s}=r;if(!i&&!a)continue;if(!t||Object.keys(t).length<1)continue;this.proxies.push({domain:i,isGlobal:a,proxy:t,rotating:!!s})}}_addLimiters(e){if(!e||!e.enable)return;for(let r of e.limiters){let{domain:i,isGlobal:a,options:t,retry:s}=r;if(!i&&!a)continue;if(!t&&!s)continue;let o=t?new y(t):new y({name:"limiter"});if(this.limiters.push({domain:i,isGlobal:a,pqueue:o,randomDelay:t?.randomDelay,retry:s}),this.onLimiterAdded)this.onLimiterAdded(o)}}_addOxylabs(e){if(!e||!e.enable)return;for(let r of e.labs){let{domain:i,isGlobal:a,options:t,queueOptions:s}=r;if(!i&&!a)continue;if(!t||Object.keys(t).length<1)continue;let o=s?new y(s):void 0;if(this.oxylabs.push({domain:i,isGlobal:a,adaptar:new x(t),pqueue:o}),o&&this.onProviderAdded)this.onProviderAdded(o)}}_addDecodo(e){if(!e||!e.enable)return;for(let r of e.labs){let{domain:i,isGlobal:a,options:t,queueOptions:s}=r;if(!i&&!a)continue;if(!t||Object.keys(t).length<1)continue;let o=s?new y(s):void 0;if(this.decodo.push({domain:i,isGlobal:a,adaptar:new A(t),pqueue:o}),o&&this.onProviderAdded)this.onProviderAdded(o)}}_addStealths(e){if(!e||!e.enable)return;for(let r of e.profiles){let{domain:i,isGlobal:a,stealth:t}=r;if(!i&&!a)continue;if(!t)continue;this.stealthProfiles.push({domain:i,isGlobal:a,adaptar:new D({stealth:t})})}}addHeaders(e){return this._addHeaders({enable:!0,httpHeaders:[e]}),this}addProxy(e){return this._addProxies({enable:!0,proxies:[e]}),this}addProxies(e){return this._addProxies({enable:!0,proxies:e}),this}getProxyConfig(e,r,i){let a=this.selectConfigEntry(e,this.proxies,r,i);if(!a)return null;return{proxy:a.proxy,rotating:!!a.rotating}}addLimiter(e){return this._addLimiters({enable:!0,limiters:[e]}),this}addOxylabs(e){return this._addOxylabs({enable:!0,labs:[e]}),this}addDecodo(e){return this._addDecodo({enable:!0,labs:[e]}),this}addStealth(e){return this._addStealths({enable:!0,profiles:[e]}),this}destroyLimiters(){for(let e of this.limiters)l.destroyQueue(e.pqueue);this.limiters=[]}destroyProviderQueues(){for(let e of this.oxylabs)l.destroyQueue(e.pqueue),delete e.pqueue;for(let e of this.decodo)l.destroyQueue(e.pqueue),delete e.pqueue}getLimiters(){return this.limiters}getRandomDelay(e,r){if(!this.getDomainName(e))return;for(let a of this.limiters)if(this._hasDomain(e,a.domain)&&a.randomDelay!==void 0)return a.randomDelay;if(r){for(let a of this.limiters)if(a.isGlobal&&a.randomDelay!==void 0)return a.randomDelay}return}clearGlobalConfigs(){if(Array.isArray(this.requestHeaders))this.requestHeaders=this.requestHeaders.filter((e)=>!e.isGlobal);if(Array.isArray(this.oxylabs)){let e=l.splitConfigs(this.oxylabs,(r)=>!!r.isGlobal);l.destroyConfigQueues(e.removed),this.oxylabs=e.kept}if(Array.isArray(this.decodo)){let e=l.splitConfigs(this.decodo,(r)=>!!r.isGlobal);l.destroyConfigQueues(e.removed),this.decodo=e.kept}if(Array.isArray(this.limiters)){let e=l.splitConfigs(this.limiters,(r)=>!!r.isGlobal);l.destroyConfigQueues(e.removed),this.limiters=e.kept}if(Array.isArray(this.proxies))this.proxies=this.proxies.filter((e)=>!e.isGlobal);if(Array.isArray(this.stealthProfiles))this.stealthProfiles=this.stealthProfiles.filter((e)=>!e.isGlobal);return this}getAdapter(e,r,i,a){if(!this.getDomainName(e))return null;let s=r==="headers"?this.requestHeaders:r==="limiters"?this.limiters:r==="oxylabs"?this.oxylabs:r==="decodo"?this.decodo:r==="stealth"?this.stealthProfiles:this.proxies,o=this.selectConfigEntry(e,s,i,a);if(!o)return null;return r==="headers"?o.headers:r==="limiters"?o.pqueue:r==="oxylabs"?o.adaptar:r==="decodo"?o.adaptar:r==="stealth"?o.adaptar:o.proxy}getRetryOptions(e){if(!this.getDomainName(e))return null;for(let i=0;i<this.limiters.length;i++)if(this._hasDomain(e,this.limiters[i].domain))return this.limiters[i].retry||null;for(let i=0;i<this.limiters.length;i++)if(this.limiters[i].isGlobal&&this.limiters[i].retry)return this.limiters[i].retry;return null}rnd(e=0,r=Number.MAX_VALUE){return Math.floor(Math.random()*(r-e+1))+e}hasDomain(e,r,i){if(!this.getDomainName(e))return!1;let t=r==="headers"?this.requestHeaders:r==="limiters"?this.limiters:r==="oxylabs"?this.oxylabs:r==="decodo"?this.decodo:r==="stealth"?this.stealthProfiles:this.proxies;for(let s=0;s<t.length;s++)if(this._hasDomain(e,t[s].domain))return!0;if(i){for(let s=0;s<t.length;s++)if(t[s].isGlobal)return!0}return!1}pickHeaders(e,r,i,a){let t=this.getAdapter(e,"headers",r),s=new Headers(t??{}),o=[...s.keys()].length;if(i&&i instanceof Headers)for(let[d,n]of Object.entries(i.entries()))s.set(d,n);else if(i&&typeof i==="object"){for(let[d,n]of Object.entries(i))if(typeof n==="string")s.set(d,n)}if(a&&o===0&&!this.stealth)s.set("user-agent",this.getRandomUserAgent());return Object.fromEntries(s.entries())}_hasDomain(e,r){if(!r)return!1;let i=this.getDomainName(e);if(!i)return!1;let a=(s)=>{return/[\^\$\*\+\?\{\}\[\]\(\)\|\\]/.test(s)||s.startsWith("/")||s.includes(".*")||s.includes(".+")},t=(s)=>{if(s instanceof RegExp)return s.test(i)||s.test(e);let o=s.toString().trim();if(i.toLowerCase()===o.toLowerCase())return!0;if(o.includes("*")){let h=o.replace(/[.*+?^${}()|[\]\\]/g,"\\$&").replace(/\\\*/g,".*"),u=new RegExp(`^${h}$`,"i");return u.test(i)||u.test(e)}if(a(o))try{let h=o,u="i",c=o.match(/^\/(.*)\/(\w*)$/);if(c)h=c[1],u=c[2]||"i";let m=new RegExp(h,u);return m.test(i)||m.test(e)}catch(h){return i.toLowerCase().includes(o.toLowerCase())}let d=i.toLowerCase(),n=o.toLowerCase();return d===n||d.endsWith("."+n)||n.endsWith("."+d)};if(Array.isArray(r)){for(let s of r)if(t(s))return!0;return!1}return t(r)}selectConfigEntry(e,r,i,a){if(!this.getDomainName(e))return null;let s=(n)=>{if(!n.length)return null;let h=a?this.rnd(0,n.length-1):0;return n[h]},o=[];for(let n of r)if(this._hasDomain(e,n.domain))o.push(n);let d=s(o);if(d)return d;if(i){let n=[];for(let h of r)if(h.isGlobal)n.push(h);return s(n)}return null}getDomainName(e){if(this.isValidUrl(e))return new URL(e).hostname;else if(this.isHostName(e))return e;return null}isHostName(e){if(!e)return!1;if(e.length>255)return!1;let r=/^(?:(?:[a-zA-Z0-9](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?\.)+ [a-zA-Z]{2,})$/;return e=e.trim().toLowerCase(),r.test(e)&&!e.startsWith("-")&&!e.endsWith("-")}isValidUrl(e){if(!e)return!1;e=e.trim();try{let r=new URL(e);if(!r.protocol||!["http:","https:"].includes(r.protocol.toLowerCase()))return!1;if(!r.hostname)return!1;if(!/^(?:(?:[a-zA-Z0-9](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?\.)+[a-zA-Z]{2,})$/.test(r.hostname))return!1;return!0}catch{return!1}}getRandomUserAgent(){return this.userAgents[Math.floor(Math.random()*this.userAgents.length)]}}function $(){let e=[{name:"Chrome",version:"91.0.4472.124",engine:"AppleWebKit/537.36"},{name:"Firefox",version:"89.0",engine:"Gecko/20100101"},{name:"Safari",version:"14.1.1",engine:"AppleWebKit/605.1.15"},{name:"Edge",version:"91.0.864.59",engine:"AppleWebKit/537.36"},{name:"Opera",version:"77.0.4054.277",engine:"AppleWebKit/537.36"},{name:"Vivaldi",version:"3.8.2259.42",engine:"AppleWebKit/537.36"},{name:"Brave",version:"1.26.74",engine:"AppleWebKit/537.36"},{name:"Chromium",version:"91.0.4472.101",engine:"AppleWebKit/537.36"},{name:"Yandex",version:"21.5.3.742",engine:"AppleWebKit/537.36"},{name:"Maxthon",version:"5.3.8.2000",engine:"AppleWebKit/537.36"}],r=["Windows NT 10.0","Windows NT 6.1","Macintosh; Intel Mac OS X 10_15_7","Macintosh; Intel Mac OS X 11_4_0","X11; Linux x86_64","X11; Ubuntu; Linux x86_64"],i=[];for(let a=0;a<200;a++){let t=e[Math.floor(Math.random()*e.length)],s=r[Math.floor(Math.random()*r.length)],o="";switch(t.name){case"Chrome":o=`Mozilla/5.0 (${s}) ${t.engine} (KHTML, like Gecko) Chrome/${t.version} Safari/537.36`;break;case"Firefox":o=`Mozilla/5.0 (${s}; rv:${t.version}) ${t.engine} Firefox/${t.version}`;break;case"Safari":o=`Mozilla/5.0 (${s}) ${t.engine} (KHTML, like Gecko) Version/${t.version} Safari/605.1.15`;break;case"Edge":o=`Mozilla/5.0 (${s}) ${t.engine} (KHTML, like Gecko) Chrome/${t.version} Safari/537.36 Edg/${t.version}`;break;case"Opera":o=`Mozilla/5.0 (${s}) ${t.engine} (KHTML, like Gecko) Chrome/${t.version} Safari/537.36 OPR/${t.version}`;break;case"Vivaldi":o=`Mozilla/5.0 (${s}) ${t.engine} (KHTML, like Gecko) Chrome/${t.version} Safari/537.36 Vivaldi/${t.version}`;break;case"Brave":o=`Mozilla/5.0 (${s}) ${t.engine} (KHTML, like Gecko) Chrome/${t.version} Safari/537.36 Brave/${t.version}`;break;case"Chromium":o=`Mozilla/5.0 (${s}) ${t.engine} (KHTML, like Gecko) Chromium/${t.version} Chrome/${t.version} Safari/537.36`;break;case"Yandex":o=`Mozilla/5.0 (${s}) ${t.engine} (KHTML, like Gecko) Chrome/${t.version} YaBrowser/${t.version} Safari/537.36`;break;case"Maxthon":o=`Mozilla/5.0 (${s}) ${t.engine} (KHTML, like Gecko) Chrome/${t.version} Safari/537.36 Maxthon/${t.version}`;break}i.push(o)}return i}export{l as CrawlerOptions};

package/dist/crawler/crawler.cjs

@@ -927,6 +927,18 @@ class Crawler {
       }
     }
   }
+  dispatchErrorEvents(err) {
+    for (let i = 0;i < this.errorEvents.length; i++) {
+      const event = this.errorEvents[i];
+      try {
+        this[event.handler](...event.attr, err);
+      } catch (handlerErr) {
+        if (this.config.debug) {
+          console.error("[Crawler] error-event handler threw:", handlerErr);
+        }
+      }
+    }
+  }
   _runHandler(handler, arg) {
     this.eventCount++;
     new Promise(async (resolve) => {
@@ -1410,8 +1422,10 @@ class Crawler {
         await this.waitForNavigationHistory();
       await this.executeHttp(method, url, body, options, proxyRotating, isEmail, forceRevisit, oxylabsOptions, oxylabsInstance, oxylabsQueue, decodoInstance, decodoOptions, decodoQueue, 0, undefined, skipCache, emailMetadata, stealthInstance);
     }).catch((err) => {
-      if (this.config.debug)
-        console.warn("[Crawler] execute() task error:", err?.message);
+      console.error(`[Crawler] execute() task error for ${url}:`, err?.message ?? err);
+      if (this.config.debug && err?.stack)
+        console.error(err.stack);
+      this.dispatchErrorEvents(err);
     });
   }
   async execute2(method, url, body, options = {}, forceRevisit, emailMetadata) {
@@ -1436,8 +1450,10 @@ class Crawler {
         emailMetadata
       }, forceRevisit, true);
     }).catch((err) => {
-      if (this.config.debug)
-        console.warn("[Crawler] execute2() task error:", err?.message);
+      console.error(`[Crawler] execute2() task error for ${url}:`, err?.message ?? err);
+      if (this.config.debug && err?.stack)
+        console.error(err.stack);
+      this.dispatchErrorEvents(err);
     });
   }
   async executeHttp(method, url, body, options = {}, proxyRotating = false, isEmail, forceRevisit, oxylabsOptions, oxylabsInstance, oxylabsQueue, decodoInstance, decodoOptions, decodoQueue, retryCount = 0, parentUrl, skipCache, emailMetadata, stealthInstance) {
@@ -1726,6 +1742,10 @@ class Crawler {
     this.rawResponseEvents.length = 0;
     this.emailDiscoveredEvents.length = 0;
     this.emailLeadsEvents.length = 0;
+    this.startHandlers.length = 0;
+    this.finishHandlers.length = 0;
+    this.redirectHandlers.length = 0;
+    this.queueChangeHandlers.length = 0;
     this.domainResponseTimes.clear();
     this.domainCurrentDelay.clear();
     this.urlDepthMap.clear();

package/dist/crawler/crawler.js

@@ -927,6 +927,18 @@ export class Crawler {
       }
     }
   }
+  dispatchErrorEvents(err) {
+    for (let i = 0;i < this.errorEvents.length; i++) {
+      const event = this.errorEvents[i];
+      try {
+        this[event.handler](...event.attr, err);
+      } catch (handlerErr) {
+        if (this.config.debug) {
+          console.error("[Crawler] error-event handler threw:", handlerErr);
+        }
+      }
+    }
+  }
   _runHandler(handler, arg) {
     this.eventCount++;
     new Promise(async (resolve) => {
@@ -1410,8 +1422,10 @@ export class Crawler {
         await this.waitForNavigationHistory();
       await this.executeHttp(method, url, body, options, proxyRotating, isEmail, forceRevisit, oxylabsOptions, oxylabsInstance, oxylabsQueue, decodoInstance, decodoOptions, decodoQueue, 0, undefined, skipCache, emailMetadata, stealthInstance);
     }).catch((err) => {
-      if (this.config.debug)
-        console.warn("[Crawler] execute() task error:", err?.message);
+      console.error(`[Crawler] execute() task error for ${url}:`, err?.message ?? err);
+      if (this.config.debug && err?.stack)
+        console.error(err.stack);
+      this.dispatchErrorEvents(err);
     });
   }
   async execute2(method, url, body, options = {}, forceRevisit, emailMetadata) {
@@ -1436,8 +1450,10 @@ export class Crawler {
         emailMetadata
       }, forceRevisit, true);
     }).catch((err) => {
-      if (this.config.debug)
-        console.warn("[Crawler] execute2() task error:", err?.message);
+      console.error(`[Crawler] execute2() task error for ${url}:`, err?.message ?? err);
+      if (this.config.debug && err?.stack)
+        console.error(err.stack);
+      this.dispatchErrorEvents(err);
     });
   }
   async executeHttp(method, url, body, options = {}, proxyRotating = false, isEmail, forceRevisit, oxylabsOptions, oxylabsInstance, oxylabsQueue, decodoInstance, decodoOptions, decodoQueue, retryCount = 0, parentUrl, skipCache, emailMetadata, stealthInstance) {
@@ -1726,6 +1742,10 @@ export class Crawler {
     this.rawResponseEvents.length = 0;
     this.emailDiscoveredEvents.length = 0;
     this.emailLeadsEvents.length = 0;
+    this.startHandlers.length = 0;
+    this.finishHandlers.length = 0;
+    this.redirectHandlers.length = 0;
+    this.queueChangeHandlers.length = 0;
     this.domainResponseTimes.clear();
     this.domainCurrentDelay.clear();
     this.urlDepthMap.clear();

package/dist/crawler/index.cjs

@@ -1,42 +1,42 @@
-const _mod_22900y = require('./crawler.cjs');
-exports.Crawler = _mod_22900y.Crawler;;
-const _mod_22ubyp = require('./crawler-options.cjs');
-exports.CrawlerOptions = _mod_22ubyp.CrawlerOptions;;
-const _mod_h84o2j = require('./plugin/robots-txt.cjs');
-exports.RobotsTxt = _mod_h84o2j.RobotsTxt;;
-const _mod_jjmsn2 = require('./plugin/file-cacher.cjs');
-exports.FileCacher = _mod_jjmsn2.FileCacher;;
-const _mod_538aj2 = require('./plugin/url-store.cjs');
-exports.UrlStore = _mod_538aj2.UrlStore;;
-const _mod_0mnqbz = require('./plugin/navigation-history.cjs');
-exports.NavigationHistory = _mod_0mnqbz.NavigationHistory;;
-const _mod_euwe7r = require('./addon/oxylabs/index.cjs');
-exports.Oxylabs = _mod_euwe7r.Oxylabs;;
-const _mod_gp9gpd = require('./addon/oxylabs/options.cjs');
-exports.OXYLABS_BROWSER_TYPES = _mod_gp9gpd.OXYLABS_BROWSER_TYPES;
-exports.OXYLABS_COMMON_LOCALES = _mod_gp9gpd.OXYLABS_COMMON_LOCALES;
-exports.OXYLABS_COMMON_GEO_LOCATIONS = _mod_gp9gpd.OXYLABS_COMMON_GEO_LOCATIONS;
-exports.OXYLABS_US_STATES = _mod_gp9gpd.OXYLABS_US_STATES;
-exports.OXYLABS_EUROPEAN_COUNTRIES = _mod_gp9gpd.OXYLABS_EUROPEAN_COUNTRIES;
-exports.OXYLABS_ASIAN_COUNTRIES = _mod_gp9gpd.OXYLABS_ASIAN_COUNTRIES;
-exports.getRandomOxylabsBrowserType = _mod_gp9gpd.getRandomBrowserType;
-exports.getRandomOxylabsLocale = _mod_gp9gpd.getRandomLocale;
-exports.getRandomOxylabsGeoLocation = _mod_gp9gpd.getRandomGeoLocation;;
-const _mod_d6ta58 = require('./scraper.cjs');
-exports.isRestrictedDomain = _mod_d6ta58.isRestrictedDomain;;
-const _mod_sdriey = require('./addon/decodo/index.cjs');
-exports.Decodo = _mod_sdriey.Decodo;;
-const _mod_udz2vx = require('./addon/decodo/options.cjs');
-exports.DECODO_DEVICE_TYPES = _mod_udz2vx.DECODO_DEVICE_TYPES;
-exports.DECODO_HEADLESS_MODES = _mod_udz2vx.DECODO_HEADLESS_MODES;
-exports.DECODO_COMMON_LOCALES = _mod_udz2vx.DECODO_COMMON_LOCALES;
-exports.DECODO_COMMON_COUNTRIES = _mod_udz2vx.DECODO_COMMON_COUNTRIES;
-exports.DECODO_EUROPEAN_COUNTRIES = _mod_udz2vx.DECODO_EUROPEAN_COUNTRIES;
-exports.DECODO_ASIAN_COUNTRIES = _mod_udz2vx.DECODO_ASIAN_COUNTRIES;
-exports.DECODO_US_STATES = _mod_udz2vx.DECODO_US_STATES;
-exports.DECODO_COMMON_CITIES = _mod_udz2vx.DECODO_COMMON_CITIES;
-exports.getRandomDecodoDeviceType = _mod_udz2vx.getRandomDeviceType;
-exports.getRandomDecodoLocale = _mod_udz2vx.getRandomLocale;
-exports.getRandomDecodoCountry = _mod_udz2vx.getRandomCountry;
-exports.getRandomDecodoCity = _mod_udz2vx.getRandomCity;
-exports.generateDecodoSessionId = _mod_udz2vx.generateSessionId;;
+const _mod_0d9h80 = require('./crawler.cjs');
+exports.Crawler = _mod_0d9h80.Crawler;;
+const _mod_flsl96 = require('./crawler-options.cjs');
+exports.CrawlerOptions = _mod_flsl96.CrawlerOptions;;
+const _mod_hshb8k = require('./plugin/robots-txt.cjs');
+exports.RobotsTxt = _mod_hshb8k.RobotsTxt;;
+const _mod_qmo1zd = require('./plugin/file-cacher.cjs');
+exports.FileCacher = _mod_qmo1zd.FileCacher;;
+const _mod_lktnzz = require('./plugin/url-store.cjs');
+exports.UrlStore = _mod_lktnzz.UrlStore;;
+const _mod_g62vuv = require('./plugin/navigation-history.cjs');
+exports.NavigationHistory = _mod_g62vuv.NavigationHistory;;
+const _mod_dfvupb = require('./addon/oxylabs/index.cjs');
+exports.Oxylabs = _mod_dfvupb.Oxylabs;;
+const _mod_lffstj = require('./addon/oxylabs/options.cjs');
+exports.OXYLABS_BROWSER_TYPES = _mod_lffstj.OXYLABS_BROWSER_TYPES;
+exports.OXYLABS_COMMON_LOCALES = _mod_lffstj.OXYLABS_COMMON_LOCALES;
+exports.OXYLABS_COMMON_GEO_LOCATIONS = _mod_lffstj.OXYLABS_COMMON_GEO_LOCATIONS;
+exports.OXYLABS_US_STATES = _mod_lffstj.OXYLABS_US_STATES;
+exports.OXYLABS_EUROPEAN_COUNTRIES = _mod_lffstj.OXYLABS_EUROPEAN_COUNTRIES;
+exports.OXYLABS_ASIAN_COUNTRIES = _mod_lffstj.OXYLABS_ASIAN_COUNTRIES;
+exports.getRandomOxylabsBrowserType = _mod_lffstj.getRandomBrowserType;
+exports.getRandomOxylabsLocale = _mod_lffstj.getRandomLocale;
+exports.getRandomOxylabsGeoLocation = _mod_lffstj.getRandomGeoLocation;;
+const _mod_6v3fut = require('./scraper.cjs');
+exports.isRestrictedDomain = _mod_6v3fut.isRestrictedDomain;;
+const _mod_jcv8ev = require('./addon/decodo/index.cjs');
+exports.Decodo = _mod_jcv8ev.Decodo;;
+const _mod_quo7cv = require('./addon/decodo/options.cjs');
+exports.DECODO_DEVICE_TYPES = _mod_quo7cv.DECODO_DEVICE_TYPES;
+exports.DECODO_HEADLESS_MODES = _mod_quo7cv.DECODO_HEADLESS_MODES;
+exports.DECODO_COMMON_LOCALES = _mod_quo7cv.DECODO_COMMON_LOCALES;
+exports.DECODO_COMMON_COUNTRIES = _mod_quo7cv.DECODO_COMMON_COUNTRIES;
+exports.DECODO_EUROPEAN_COUNTRIES = _mod_quo7cv.DECODO_EUROPEAN_COUNTRIES;
+exports.DECODO_ASIAN_COUNTRIES = _mod_quo7cv.DECODO_ASIAN_COUNTRIES;
+exports.DECODO_US_STATES = _mod_quo7cv.DECODO_US_STATES;
+exports.DECODO_COMMON_CITIES = _mod_quo7cv.DECODO_COMMON_CITIES;
+exports.getRandomDecodoDeviceType = _mod_quo7cv.getRandomDeviceType;
+exports.getRandomDecodoLocale = _mod_quo7cv.getRandomLocale;
+exports.getRandomDecodoCountry = _mod_quo7cv.getRandomCountry;
+exports.getRandomDecodoCity = _mod_quo7cv.getRandomCity;
+exports.generateDecodoSessionId = _mod_quo7cv.generateSessionId;;

package/dist/crawler/plugin/index.cjs

@@ -1 +1 @@
-var e=require("./file-cacher.cjs");exports.FileCacher=e.FileCacher;var r=require("./url-store.cjs");exports.UrlStore=r.UrlStore;var o=require("./result-stream.cjs");exports.ResultStream=o.ResultStream;var t=require("./memory-monitor.cjs");exports.MemoryMonitor=t.MemoryMonitor;var a=require("./health-metrics.cjs");exports.HealthMetrics=a.HealthMetrics;var i=require("./capped-map.cjs");exports.CappedMap=i.CappedMap;var m=require("./capped-array.cjs");exports.CappedArray=m.CappedArray;
+var e=require("./file-cacher.cjs");exports.FileCacher=e.FileCacher;var r=require("./url-store.cjs");exports.UrlStore=r.UrlStore;var o=require("./result-stream.cjs");exports.ResultStream=o.ResultStream;var t=require("./memory-monitor.cjs");exports.MemoryMonitor=t.MemoryMonitor;var a=require("./health-metrics.cjs");exports.HealthMetrics=a.HealthMetrics;var i=require("./capped-map.cjs");exports.CappedMap=i.CappedMap;var d=require("./capped-array.cjs");exports.CappedArray=d.CappedArray;

package/dist/crawler/plugin/sqlite-utils.cjs

@@ -1 +1 @@
-var y=exports.isBun=typeof globalThis.Bun<"u",h={maxRetries:5,initialDelayMs:50,maxDelayMs:1000};function f(s){if(!s)return!1;let t=s.message||s.toString();return t.includes("SQLITE_BUSY")||t.includes("database is locked")||t.includes("database table is locked")}function d(s){return new Promise((t)=>setTimeout(t,s))}function u(s,t={}){let r={...h,...t},l;for(let n=0;n<=r.maxRetries;n++)try{return s()}catch(i){if(l=i,!f(i)||n>=r.maxRetries)throw i;let e=Math.min(r.initialDelayMs*Math.pow(2,n)+Math.random()*50,r.maxDelayMs),a=Date.now();while(Date.now()-a<e);}throw l}async function g(s,t={}){let r={...h,...t},l;for(let n=0;n<=r.maxRetries;n++)try{return await s()}catch(i){if(l=i,!f(i)||n>=r.maxRetries)throw i;let e=Math.min(r.initialDelayMs*Math.pow(2,n)+Math.random()*50,r.maxDelayMs);await d(e)}throw l}async function M(s,t={}){if(y){let{Database:e}=await import("bun:sqlite"),a=new e(s),m=new Map,w=(c)=>{let o=m.get(c);if(!o)o=a.query(c),m.set(c,o);return o};return{run:(c,...o)=>u(()=>{if(o.length===0)a.run(c);else w(c).run(...o)},t),get:(c,...o)=>u(()=>w(c).get(...o),t),all:(c,...o)=>u(()=>w(c).all(...o),t),exec:(c)=>u(()=>a.exec(c),t),close:()=>{m.clear(),a.close()}}}let{DatabaseSync:r}=await import("node:sqlite"),l=new r(s),n=new Map,i=(e)=>{let a=n.get(e);if(!a)a=l.prepare(e),n.set(e,a);return a};return{run:(e,...a)=>u(()=>{if(a.length===0)l.exec(e);else i(e).run(...a)},t),get:(e,...a)=>u(()=>{return i(e).get(...a)},t),all:(e,...a)=>u(()=>{return i(e).all(...a)},t),exec:(e)=>u(()=>l.exec(e),t),close:()=>{n.clear(),l.close()}}}exports.withRetrySync=u;exports.withRetryAsync=g;exports.createDatabase=M;
+var d=exports.isBun=typeof globalThis.Bun<"u",f={maxRetries:5,initialDelayMs:50,maxDelayMs:1000};function y(a){if(!a)return!1;let e=a.message||a.toString();return e.includes("SQLITE_BUSY")||e.includes("database is locked")||e.includes("database table is locked")}function g(a){return new Promise((e)=>setTimeout(e,a))}function M(a){if(a<=0)return;try{let e=m??(m=new Int32Array(new SharedArrayBuffer(4)));Atomics.wait(e,0,0,a)}catch{let e=Date.now();while(Date.now()-e<a);}}var m=null;function u(a,e={}){let c={...f,...e},o;for(let r=0;r<=c.maxRetries;r++)try{return a()}catch(i){if(o=i,!y(i)||r>=c.maxRetries)throw i;let t=Math.min(c.initialDelayMs*Math.pow(2,r)+Math.random()*50,c.maxDelayMs);M(t)}throw o}async function x(a,e={}){let c={...f,...e},o;for(let r=0;r<=c.maxRetries;r++)try{return await a()}catch(i){if(o=i,!y(i)||r>=c.maxRetries)throw i;let t=Math.min(c.initialDelayMs*Math.pow(2,r)+Math.random()*50,c.maxDelayMs);await g(t)}throw o}async function D(a,e={}){if(d){let{Database:t}=await import("bun:sqlite"),n=new t(a),w=new Map,h=(s)=>{let l=w.get(s);if(!l)l=n.query(s),w.set(s,l);return l};return{run:(s,...l)=>u(()=>{if(l.length===0)n.run(s);else h(s).run(...l)},e),get:(s,...l)=>u(()=>h(s).get(...l),e),all:(s,...l)=>u(()=>h(s).all(...l),e),exec:(s)=>u(()=>n.exec(s),e),close:()=>{w.clear(),n.close()}}}let{DatabaseSync:c}=await import("node:sqlite"),o=new c(a),r=new Map,i=(t)=>{let n=r.get(t);if(!n)n=o.prepare(t),r.set(t,n);return n};return{run:(t,...n)=>u(()=>{if(n.length===0)o.exec(t);else i(t).run(...n)},e),get:(t,...n)=>u(()=>{return i(t).get(...n)},e),all:(t,...n)=>u(()=>{return i(t).all(...n)},e),exec:(t)=>u(()=>o.exec(t),e),close:()=>{r.clear(),o.close()}}}exports.withRetrySync=u;exports.withRetryAsync=x;exports.createDatabase=D;

package/dist/crawler/plugin/sqlite-utils.js

@@ -1 +1 @@
-import{createRequire as d}from"node:module";var h=d(import.meta.url);var g=typeof globalThis.Bun<"u",f={maxRetries:5,initialDelayMs:50,maxDelayMs:1000};function x(s){if(!s)return!1;let t=s.message||s.toString();return t.includes("SQLITE_BUSY")||t.includes("database is locked")||t.includes("database table is locked")}function M(s){return new Promise((t)=>setTimeout(t,s))}function u(s,t={}){let c={...f,...t},l;for(let n=0;n<=c.maxRetries;n++)try{return s()}catch(o){if(l=o,!x(o)||n>=c.maxRetries)throw o;let e=Math.min(c.initialDelayMs*Math.pow(2,n)+Math.random()*50,c.maxDelayMs),a=Date.now();while(Date.now()-a<e);}throw l}async function p(s,t={}){let c={...f,...t},l;for(let n=0;n<=c.maxRetries;n++)try{return await s()}catch(o){if(l=o,!x(o)||n>=c.maxRetries)throw o;let e=Math.min(c.initialDelayMs*Math.pow(2,n)+Math.random()*50,c.maxDelayMs);await M(e)}throw l}async function R(s,t={}){if(g){let{Database:e}=await import("bun:sqlite"),a=new e(s),m=new Map,w=(r)=>{let i=m.get(r);if(!i)i=a.query(r),m.set(r,i);return i};return{run:(r,...i)=>u(()=>{if(i.length===0)a.run(r);else w(r).run(...i)},t),get:(r,...i)=>u(()=>w(r).get(...i),t),all:(r,...i)=>u(()=>w(r).all(...i),t),exec:(r)=>u(()=>a.exec(r),t),close:()=>{m.clear(),a.close()}}}let{DatabaseSync:c}=await import("node:sqlite"),l=new c(s),n=new Map,o=(e)=>{let a=n.get(e);if(!a)a=l.prepare(e),n.set(e,a);return a};return{run:(e,...a)=>u(()=>{if(a.length===0)l.exec(e);else o(e).run(...a)},t),get:(e,...a)=>u(()=>{return o(e).get(...a)},t),all:(e,...a)=>u(()=>{return o(e).all(...a)},t),exec:(e)=>u(()=>l.exec(e),t),close:()=>{n.clear(),l.close()}}}export{u as withRetrySync,p as withRetryAsync,g as isBun,R as createDatabase};
+import{createRequire as d}from"node:module";var f=d(import.meta.url);var g=typeof globalThis.Bun<"u",x={maxRetries:5,initialDelayMs:50,maxDelayMs:1000};function y(a){if(!a)return!1;let e=a.message||a.toString();return e.includes("SQLITE_BUSY")||e.includes("database is locked")||e.includes("database table is locked")}function M(a){return new Promise((e)=>setTimeout(e,a))}function D(a){if(a<=0)return;try{let e=h??(h=new Int32Array(new SharedArrayBuffer(4)));Atomics.wait(e,0,0,a)}catch{let e=Date.now();while(Date.now()-e<a);}}var h=null;function u(a,e={}){let c={...x,...e},l;for(let r=0;r<=c.maxRetries;r++)try{return a()}catch(o){if(l=o,!y(o)||r>=c.maxRetries)throw o;let t=Math.min(c.initialDelayMs*Math.pow(2,r)+Math.random()*50,c.maxDelayMs);D(t)}throw l}async function R(a,e={}){let c={...x,...e},l;for(let r=0;r<=c.maxRetries;r++)try{return await a()}catch(o){if(l=o,!y(o)||r>=c.maxRetries)throw o;let t=Math.min(c.initialDelayMs*Math.pow(2,r)+Math.random()*50,c.maxDelayMs);await M(t)}throw l}async function b(a,e={}){if(g){let{Database:t}=await import("bun:sqlite"),n=new t(a),w=new Map,m=(s)=>{let i=w.get(s);if(!i)i=n.query(s),w.set(s,i);return i};return{run:(s,...i)=>u(()=>{if(i.length===0)n.run(s);else m(s).run(...i)},e),get:(s,...i)=>u(()=>m(s).get(...i),e),all:(s,...i)=>u(()=>m(s).all(...i),e),exec:(s)=>u(()=>n.exec(s),e),close:()=>{w.clear(),n.close()}}}let{DatabaseSync:c}=await import("node:sqlite"),l=new c(a),r=new Map,o=(t)=>{let n=r.get(t);if(!n)n=l.prepare(t),r.set(t,n);return n};return{run:(t,...n)=>u(()=>{if(n.length===0)l.exec(t);else o(t).run(...n)},e),get:(t,...n)=>u(()=>{return o(t).get(...n)},e),all:(t,...n)=>u(()=>{return o(t).all(...n)},e),exec:(t)=>u(()=>l.exec(t),e),close:()=>{r.clear(),l.close()}}}export{u as withRetrySync,R as withRetryAsync,g as isBun,b as createDatabase};

package/dist/crawler.d.ts

@@ -9372,6 +9372,13 @@ export declare class Crawler {
 	 * Called when onQueueChange handlers are registered to ensure limiter events are captured.
 	 */
 	private _subscribeToManagedQueues;
+	/**
+	 * Invokes every registered error-event handler with the given error.
+	 * Swallows per-handler throws so one misbehaving handler can't mask others.
+	 * Used from the fire-and-forget execute/execute2 catch blocks so those
+	 * errors reach the same handler chain as in-crawl errors.
+	 */
+	private dispatchErrorEvents;
 	/**
 	 * Run a handler with event tracking (not through queue).
 	 * Increments eventCount before running, decrements after completion.

package/dist/dom/index.cjs

@@ -1,23 +1 @@
-let _linkedom = null;
-function getLinkedom() {
-  if (_linkedom)
-    return _linkedom;
-  try {
-    _linkedom = require("linkedom");
-    return _linkedom;
-  } catch {
-    throw new Error(`linkedom is required for DOM parsing but is not installed.
-` + `Install it with: npm install linkedom
-` + "Or: bun add linkedom");
-  }
-}
-function parseHTML(html) {
-  return getLinkedom().parseHTML(html);
-}
-function createDOMParser() {
-  const Linkedom = getLinkedom();
-  return new Linkedom.DOMParser;
-}
-
-exports.parseHTML = parseHTML;
-exports.createDOMParser = createDOMParser;
+Object.assign(exports, require("linkedom"));