rezo 1.0.119 → 1.0.121

package/dist/adapters/entries/curl.d.ts

@@ -5671,7 +5671,7 @@ export interface RezoInstance extends Rezo, RezoCallable {
  *
  * IMPORTANT: Update these values when bumping package version.
  */
-export declare const VERSION = "1.0.119";
+export declare const VERSION = "1.0.121";
 /**
  * cURL Options Configuration
  *

package/dist/adapters/entries/fetch.d.ts

@@ -5671,7 +5671,7 @@ export interface RezoInstance extends Rezo, RezoCallable {
  *
  * IMPORTANT: Update these values when bumping package version.
  */
-export declare const VERSION = "1.0.119";
+export declare const VERSION = "1.0.121";
 export declare const isRezoError: typeof RezoError.isRezoError;
 export declare const Cancel: typeof RezoError;
 export declare const CancelToken: {

package/dist/adapters/entries/http.cjs

@@ -6,7 +6,9 @@ const { RezoFormData } = require('../../utils/form-data.cjs');
 const { RezoCookieJar, Cookie } = require('../../cookies/cookie-jar.cjs');
 const { createDefaultHooks, mergeHooks } = require('../../core/hooks.cjs');
 const { VERSION } = require('../../version.cjs');
+const { RezoStealth } = require('../../stealth/index.cjs');
 
+exports.RezoStealth = RezoStealth;
 exports.Rezo = Rezo;
 exports.RezoError = RezoError;
 exports.RezoErrorCode = RezoErrorCode;

package/dist/adapters/entries/http.d.ts

@@ -5671,7 +5671,7 @@ export interface RezoInstance extends Rezo, RezoCallable {
  *
  * IMPORTANT: Update these values when bumping package version.
  */
-export declare const VERSION = "1.0.119";
+export declare const VERSION = "1.0.121";
 /**
  * Type guard to check if an error is a RezoError instance.
  */

package/dist/adapters/entries/http.js

@@ -6,7 +6,9 @@ import { RezoFormData } from '../../utils/form-data.js';
 import { RezoCookieJar, Cookie } from '../../cookies/cookie-jar.js';
 import { createDefaultHooks, mergeHooks } from '../../core/hooks.js';
 import { VERSION } from '../../version.js';
+import { RezoStealth } from '../../stealth/index.js';
 
+export { RezoStealth };
 export { Rezo };
 export { RezoError };
 export { RezoErrorCode };

package/dist/adapters/entries/http2.cjs

@@ -6,7 +6,9 @@ const { RezoFormData } = require('../../utils/form-data.cjs');
 const { RezoCookieJar, Cookie } = require('../../cookies/cookie-jar.cjs');
 const { createDefaultHooks, mergeHooks } = require('../../core/hooks.cjs');
 const { VERSION } = require('../../version.cjs');
+const { RezoStealth } = require('../../stealth/index.cjs');
 
+exports.RezoStealth = RezoStealth;
 exports.Rezo = Rezo;
 exports.RezoError = RezoError;
 exports.RezoErrorCode = RezoErrorCode;

package/dist/adapters/entries/http2.d.ts

@@ -5671,7 +5671,7 @@ export interface RezoInstance extends Rezo, RezoCallable {
  *
  * IMPORTANT: Update these values when bumping package version.
  */
-export declare const VERSION = "1.0.119";
+export declare const VERSION = "1.0.121";
 export declare const isRezoError: typeof RezoError.isRezoError;
 export declare const Cancel: typeof RezoError;
 export declare const CancelToken: {

package/dist/adapters/entries/http2.js

@@ -6,7 +6,9 @@ import { RezoFormData } from '../../utils/form-data.js';
 import { RezoCookieJar, Cookie } from '../../cookies/cookie-jar.js';
 import { createDefaultHooks, mergeHooks } from '../../core/hooks.js';
 import { VERSION } from '../../version.js';
+import { RezoStealth } from '../../stealth/index.js';
 
+export { RezoStealth };
 export { Rezo };
 export { RezoError };
 export { RezoErrorCode };

package/dist/adapters/entries/react-native.d.ts

@@ -5671,7 +5671,7 @@ export interface RezoInstance extends Rezo, RezoCallable {
  *
  * IMPORTANT: Update these values when bumping package version.
  */
-export declare const VERSION = "1.0.119";
+export declare const VERSION = "1.0.121";
 export declare const isRezoError: typeof RezoError.isRezoError;
 export declare const Cancel: typeof RezoError;
 export declare const CancelToken: {

package/dist/adapters/entries/xhr.d.ts

@@ -5671,7 +5671,7 @@ export interface RezoInstance extends Rezo, RezoCallable {
  *
  * IMPORTANT: Update these values when bumping package version.
  */
-export declare const VERSION = "1.0.119";
+export declare const VERSION = "1.0.121";
 export declare const isRezoError: typeof RezoError.isRezoError;
 export declare const Cancel: typeof RezoError;
 export declare const CancelToken: {

package/dist/adapters/index.cjs

@@ -1,6 +1,6 @@
-const _mod_p8hwxi = require('./picker.cjs');
-exports.detectRuntime = _mod_p8hwxi.detectRuntime;
-exports.getAdapterCapabilities = _mod_p8hwxi.getAdapterCapabilities;
-exports.buildAdapterContext = _mod_p8hwxi.buildAdapterContext;
-exports.getAvailableAdapters = _mod_p8hwxi.getAvailableAdapters;
-exports.selectAdapter = _mod_p8hwxi.selectAdapter;;
+const _mod_fxwdnl = require('./picker.cjs');
+exports.detectRuntime = _mod_fxwdnl.detectRuntime;
+exports.getAdapterCapabilities = _mod_fxwdnl.getAdapterCapabilities;
+exports.buildAdapterContext = _mod_fxwdnl.buildAdapterContext;
+exports.getAvailableAdapters = _mod_fxwdnl.getAvailableAdapters;
+exports.selectAdapter = _mod_fxwdnl.selectAdapter;;

package/dist/cache/index.cjs

@@ -1,9 +1,9 @@
-const _mod_cfx9b7 = require('./lru-cache.cjs');
-exports.LRUCache = _mod_cfx9b7.LRUCache;;
-const _mod_ifvloh = require('./dns-cache.cjs');
-exports.DNSCache = _mod_ifvloh.DNSCache;
-exports.getGlobalDNSCache = _mod_ifvloh.getGlobalDNSCache;
-exports.resetGlobalDNSCache = _mod_ifvloh.resetGlobalDNSCache;;
-const _mod_barvl4 = require('./response-cache.cjs');
-exports.ResponseCache = _mod_barvl4.ResponseCache;
-exports.normalizeResponseCacheConfig = _mod_barvl4.normalizeResponseCacheConfig;;
+const _mod_g8y6rz = require('./lru-cache.cjs');
+exports.LRUCache = _mod_g8y6rz.LRUCache;;
+const _mod_gpx76w = require('./dns-cache.cjs');
+exports.DNSCache = _mod_gpx76w.DNSCache;
+exports.getGlobalDNSCache = _mod_gpx76w.getGlobalDNSCache;
+exports.resetGlobalDNSCache = _mod_gpx76w.resetGlobalDNSCache;;
+const _mod_ab58da = require('./response-cache.cjs');
+exports.ResponseCache = _mod_ab58da.ResponseCache;
+exports.normalizeResponseCacheConfig = _mod_ab58da.normalizeResponseCacheConfig;;

package/dist/cookies/cookie-jar.cjs

@@ -1,7 +1,7 @@
 const { CookieJar: TouchCookieJar } = require("tough-cookie");
 const { Cookie } = require('./cookie.cjs');
-const _mod_7j8bjx = require('./cookie.cjs');
-exports.Cookie = _mod_7j8bjx.Cookie;;
+const _mod_a6yd89 = require('./cookie.cjs');
+exports.Cookie = _mod_a6yd89.Cookie;;
 
 class RezoCookieJar extends TouchCookieJar {
   constructor(store, options) {
@@ -476,7 +476,7 @@ class RezoCookieJar extends TouchCookieJar {
   }
 }
 const CookieJar = exports.CookieJar = RezoCookieJar;
-const _mod_e1ly28 = require("tough-cookie");
-exports.Store = _mod_e1ly28.Store;;
+const _mod_pnd4u7 = require("tough-cookie");
+exports.Store = _mod_pnd4u7.Store;;
 
 exports.RezoCookieJar = RezoCookieJar;

package/dist/cookies/index.cjs

@@ -1,10 +1,10 @@
-const _mod_ho5j4l = require('./cookie.cjs');
-exports.Cookie = _mod_ho5j4l.Cookie;
-exports.RezoCookie = _mod_ho5j4l.RezoCookie;;
-const _mod_7i408t = require('./cookie-store.cjs');
-exports.RezoCookieStore = _mod_7i408t.RezoCookieStore;;
-const _mod_4hgn96 = require('./cookie-jar.cjs');
-exports.RezoCookieJar = _mod_4hgn96.RezoCookieJar;
-exports.CookieJar = _mod_4hgn96.CookieJar;;
-const _mod_uamjqy = require("tough-cookie");
-exports.Store = _mod_uamjqy.Store;;
+const _mod_3pfjjj = require('./cookie.cjs');
+exports.Cookie = _mod_3pfjjj.Cookie;
+exports.RezoCookie = _mod_3pfjjj.RezoCookie;;
+const _mod_0wvzgl = require('./cookie-store.cjs');
+exports.RezoCookieStore = _mod_0wvzgl.RezoCookieStore;;
+const _mod_fd9r9j = require('./cookie-jar.cjs');
+exports.RezoCookieJar = _mod_fd9r9j.RezoCookieJar;
+exports.CookieJar = _mod_fd9r9j.CookieJar;;
+const _mod_ca4i6g = require("tough-cookie");
+exports.Store = _mod_ca4i6g.Store;;

package/dist/crawler/addon/decodo/index.cjs

@@ -1 +1 @@
-var{Rezo:h}=require("../../../core/rezo.cjs"),r=require("./options.cjs");exports.DECODO_DEVICE_TYPES=r.DECODO_DEVICE_TYPES;exports.DECODO_HEADLESS_MODES=r.DECODO_HEADLESS_MODES;exports.DECODO_COMMON_LOCALES=r.DECODO_COMMON_LOCALES;exports.DECODO_COMMON_COUNTRIES=r.DECODO_COMMON_COUNTRIES;exports.DECODO_EUROPEAN_COUNTRIES=r.DECODO_EUROPEAN_COUNTRIES;exports.DECODO_ASIAN_COUNTRIES=r.DECODO_ASIAN_COUNTRIES;exports.DECODO_US_STATES=r.DECODO_US_STATES;exports.DECODO_COMMON_CITIES=r.DECODO_COMMON_CITIES;exports.getRandomDeviceType=r.getRandomDeviceType;exports.getRandomLocale=r.getRandomLocale;exports.getRandomCountry=r.getRandomCountry;exports.getRandomCity=r.getRandomCity;exports.generateSessionId=r.generateSessionId;var u="https://scraper-api.smartproxy.com/v2/scrape";class d{config;http;authHeader;constructor(e){let t="username"in e&&"password"in e&&e.username&&e.password,s="token"in e&&e.token;if(!t&&!s)throw Error("Decodo requires either username/password or token for authentication");if(this.config={username:e.username??"",password:e.password??"",deviceType:e.deviceType??"desktop",locale:e.locale??"en-US",country:e.country??"",state:e.state??"",city:e.city??"",headless:e.headless??void 0,headers:e.headers??{},sessionId:e.sessionId??"",sessionDuration:e.sessionDuration??0,javascript:e.javascript??"",javascriptWait:e.javascriptWait??0,waitForCss:e.waitForCss??"",timeout:e.timeout??120000},this.http=new h({baseURL:u,timeout:this.config.timeout}),s)this.authHeader=`Basic ${e.token}`;else this.authHeader=`Basic ${Buffer.from(`${e.username}:${e.password}`).toString("base64")}`}async scrape(e,t){let s={...this.config,...t,headers:{...this.config.headers,...t?.headers||{}}},n=this.buildRequestBody(e,s),a=(await this.http.postJson(u,n,{headers:{Authorization:this.authHeader,"Content-Type":"application/json"}})).data;if(a.error)throw Error(`Decodo API error: ${a.error} (${a.error_code||"unknown"})`);if(!a.results||a.results.length===0)throw Error("Decodo API returned no results");let o=a.results[0];return{statusCode:o.status_code,url:o.url,content:o.body,cookies:o.cookies||[],headers:o.headers||{},taskId:a.id,rendered:!!s.headless,country:s.country||void 0,city:s.city||void 0,state:s.state||void 0,deviceType:s.deviceType,contentType:o.content_type,contentLength:o.content_length,raw:a}}async scrapeMany(e,t,s=1000){let n=[];for(let i=0;i<e.length;i++){let a=await this.scrape(e[i],t);if(n.push(a),i<e.length-1&&s>0)await new Promise((o)=>setTimeout(o,s))}return n}async scrapeWithSession(e,t,s=10){let n=`session_${Date.now()}_${Math.random().toString(36).substring(2,11)}`;return this.scrapeMany(e,{...t,sessionId:n,sessionDuration:s},500)}buildRequestBody(e,t){let s={url:e,return_cookies:!0,return_headers:!0};if(t.deviceType)s.device_type=t.deviceType;if(t.headless)s.headless=t.headless;if(t.locale)s.locale=t.locale;if(t.country)s.country=t.country;if(t.state)s.state=t.state;if(t.city)s.city=t.city;if(t.sessionId){if(s.session=t.sessionId,t.sessionDuration)s.session_duration=t.sessionDuration}if(t.headers&&Object.keys(t.headers).length>0)s.headers=t.headers;if(t.javascript){if(s.javascript=t.javascript,t.javascriptWait)s.javascript_wait=t.javascriptWait}if(t.waitForCss)s.wait_for_css=t.waitForCss;return s}getConfig(){return{...this.config,password:"***"}}withConfig(e){return new d({...this.config,...e})}async testConnection(){try{return await this.scrape("https://httpbin.org/ip"),!0}catch(e){throw Error(`Decodo connection test failed: ${e.message}`)}}}exports.Decodo=d;exports.default=d;module.exports=Object.assign(d,exports);
+var{Rezo:h}=require("../../../core/rezo.cjs"),r=require("./options.cjs");exports.DECODO_DEVICE_TYPES=r.DECODO_DEVICE_TYPES;exports.DECODO_HEADLESS_MODES=r.DECODO_HEADLESS_MODES;exports.DECODO_COMMON_LOCALES=r.DECODO_COMMON_LOCALES;exports.DECODO_COMMON_COUNTRIES=r.DECODO_COMMON_COUNTRIES;exports.DECODO_EUROPEAN_COUNTRIES=r.DECODO_EUROPEAN_COUNTRIES;exports.DECODO_ASIAN_COUNTRIES=r.DECODO_ASIAN_COUNTRIES;exports.DECODO_US_STATES=r.DECODO_US_STATES;exports.DECODO_COMMON_CITIES=r.DECODO_COMMON_CITIES;exports.getRandomDeviceType=r.getRandomDeviceType;exports.getRandomLocale=r.getRandomLocale;exports.getRandomCountry=r.getRandomCountry;exports.getRandomCity=r.getRandomCity;exports.generateSessionId=r.generateSessionId;var u="https://scraper-api.smartproxy.com/v2/scrape";class d{config;http;authHeader;constructor(s){let e="username"in s&&"password"in s&&s.username&&s.password,t="token"in s&&s.token;if(!e&&!t)throw Error("Decodo requires either username/password or token for authentication");if(this.config={username:s.username??"",password:s.password??"",deviceType:s.deviceType??"desktop",locale:s.locale??"en-US",country:s.country??"",state:s.state??"",city:s.city??"",headless:s.headless??void 0,headers:s.headers??{},sessionId:s.sessionId??"",sessionDuration:s.sessionDuration??0,javascript:s.javascript??"",javascriptWait:s.javascriptWait??0,waitForCss:s.waitForCss??"",timeout:s.timeout??120000},this.http=new h({baseURL:u,timeout:this.config.timeout}),t)this.authHeader=`Basic ${s.token}`;else this.authHeader=`Basic ${Buffer.from(`${s.username}:${s.password}`).toString("base64")}`}async scrape(s,e){let t={...this.config,...e,headers:{...this.config.headers,...e?.headers||{}}},o=this.buildRequestBody(s,t),a=(await this.http.postJson(u,o,{headers:{Authorization:this.authHeader,"Content-Type":"application/json"}})).data;if(a.error)throw Error(`Decodo API error: ${a.error} (${a.error_code||"unknown"})`);if(!a.results||a.results.length===0)throw Error("Decodo API returned no results");let i=a.results[0];return{statusCode:i.status_code,url:i.url,content:i.body,cookies:i.cookies||[],headers:i.headers||{},taskId:a.id,rendered:!!t.headless,country:t.country||void 0,city:t.city||void 0,state:t.state||void 0,deviceType:t.deviceType,contentType:i.content_type,contentLength:i.content_length,raw:a}}async scrapeMany(s,e,t=1000){let o=[];for(let n=0;n<s.length;n++){let a=await this.scrape(s[n],e);if(o.push(a),n<s.length-1&&t>0)await new Promise((i)=>setTimeout(i,t))}return o}async scrapeWithSession(s,e,t=10){let o=`session_${Date.now()}_${Math.random().toString(36).substring(2,11)}`;return this.scrapeMany(s,{...e,sessionId:o,sessionDuration:t},500)}buildRequestBody(s,e){let t={target:"universal",url:s,return_cookies:!0,return_headers:!0};if(e.deviceType)t.device_type=e.deviceType;if(e.headless)t.headless=e.headless;else if(e.javascript_rendering)t.headless="html";if(e.locale)t.locale=e.locale;if(e.country)t.country=e.country;if(e.state)t.state=e.state;if(e.city)t.city=e.city;if(e.session_id||e.sessionId){if(t.session_id=e.session_id||e.sessionId,e.sessionDuration)t.session_duration=e.sessionDuration}if(e.headers&&Object.keys(e.headers).length>0)t.headers=e.headers,t.force_headers=!0;if(e.cookies&&e.cookies.length>0)t.cookies=e.cookies.map((o)=>`${o.key}=${o.value}`).join("; "),t.force_cookies=!0;if(e.http_method)t.http_method=e.http_method;if(e.base64Body)t.payload=e.base64Body;if(e.successful_status_codes&&e.successful_status_codes.length>0)t.successful_status_codes=e.successful_status_codes;if(e.javascript){if(t.javascript=e.javascript,e.javascriptWait)t.javascript_wait=e.javascriptWait}if(e.waitForCss)t.wait_for_css=e.waitForCss;return t}getConfig(){return{...this.config,password:"***"}}withConfig(s){return new d({...this.config,...s})}async testConnection(){try{return await this.scrape("https://httpbin.org/ip"),!0}catch(s){throw Error(`Decodo connection test failed: ${s.message}`)}}}exports.Decodo=d;exports.default=d;module.exports=Object.assign(d,exports);

package/dist/crawler/addon/decodo/index.js

@@ -1 +1 @@
-import{Rezo as u}from"../../../core/rezo.js";import{DECODO_DEVICE_TYPES as y,DECODO_HEADLESS_MODES as l,DECODO_COMMON_LOCALES as w,DECODO_COMMON_COUNTRIES as O,DECODO_EUROPEAN_COUNTRIES as _,DECODO_ASIAN_COUNTRIES as D,DECODO_US_STATES as C,DECODO_COMMON_CITIES as E,getRandomDeviceType as m,getRandomLocale as v,getRandomCountry as S,getRandomCity as T,generateSessionId as I}from"./options.js";var d="https://scraper-api.smartproxy.com/v2/scrape";class i{config;http;authHeader;constructor(e){let t="username"in e&&"password"in e&&e.username&&e.password,s="token"in e&&e.token;if(!t&&!s)throw Error("Decodo requires either username/password or token for authentication");if(this.config={username:e.username??"",password:e.password??"",deviceType:e.deviceType??"desktop",locale:e.locale??"en-US",country:e.country??"",state:e.state??"",city:e.city??"",headless:e.headless??void 0,headers:e.headers??{},sessionId:e.sessionId??"",sessionDuration:e.sessionDuration??0,javascript:e.javascript??"",javascriptWait:e.javascriptWait??0,waitForCss:e.waitForCss??"",timeout:e.timeout??120000},this.http=new u({baseURL:d,timeout:this.config.timeout}),s)this.authHeader=`Basic ${e.token}`;else this.authHeader=`Basic ${Buffer.from(`${e.username}:${e.password}`).toString("base64")}`}async scrape(e,t){let s={...this.config,...t,headers:{...this.config.headers,...t?.headers||{}}},o=this.buildRequestBody(e,s),r=(await this.http.postJson(d,o,{headers:{Authorization:this.authHeader,"Content-Type":"application/json"}})).data;if(r.error)throw Error(`Decodo API error: ${r.error} (${r.error_code||"unknown"})`);if(!r.results||r.results.length===0)throw Error("Decodo API returned no results");let a=r.results[0];return{statusCode:a.status_code,url:a.url,content:a.body,cookies:a.cookies||[],headers:a.headers||{},taskId:r.id,rendered:!!s.headless,country:s.country||void 0,city:s.city||void 0,state:s.state||void 0,deviceType:s.deviceType,contentType:a.content_type,contentLength:a.content_length,raw:r}}async scrapeMany(e,t,s=1000){let o=[];for(let n=0;n<e.length;n++){let r=await this.scrape(e[n],t);if(o.push(r),n<e.length-1&&s>0)await new Promise((a)=>setTimeout(a,s))}return o}async scrapeWithSession(e,t,s=10){let o=`session_${Date.now()}_${Math.random().toString(36).substring(2,11)}`;return this.scrapeMany(e,{...t,sessionId:o,sessionDuration:s},500)}buildRequestBody(e,t){let s={url:e,return_cookies:!0,return_headers:!0};if(t.deviceType)s.device_type=t.deviceType;if(t.headless)s.headless=t.headless;if(t.locale)s.locale=t.locale;if(t.country)s.country=t.country;if(t.state)s.state=t.state;if(t.city)s.city=t.city;if(t.sessionId){if(s.session=t.sessionId,t.sessionDuration)s.session_duration=t.sessionDuration}if(t.headers&&Object.keys(t.headers).length>0)s.headers=t.headers;if(t.javascript){if(s.javascript=t.javascript,t.javascriptWait)s.javascript_wait=t.javascriptWait}if(t.waitForCss)s.wait_for_css=t.waitForCss;return s}getConfig(){return{...this.config,password:"***"}}withConfig(e){return new i({...this.config,...e})}async testConnection(){try{return await this.scrape("https://httpbin.org/ip"),!0}catch(e){throw Error(`Decodo connection test failed: ${e.message}`)}}}var R=i;export{v as getRandomLocale,m as getRandomDeviceType,S as getRandomCountry,T as getRandomCity,I as generateSessionId,R as default,i as Decodo,C as DECODO_US_STATES,l as DECODO_HEADLESS_MODES,_ as DECODO_EUROPEAN_COUNTRIES,y as DECODO_DEVICE_TYPES,w as DECODO_COMMON_LOCALES,O as DECODO_COMMON_COUNTRIES,E as DECODO_COMMON_CITIES,D as DECODO_ASIAN_COUNTRIES};
+import{Rezo as u}from"../../../core/rezo.js";import{DECODO_DEVICE_TYPES as l,DECODO_HEADLESS_MODES as _,DECODO_COMMON_LOCALES as y,DECODO_COMMON_COUNTRIES as w,DECODO_EUROPEAN_COUNTRIES as O,DECODO_ASIAN_COUNTRIES as D,DECODO_US_STATES as m,DECODO_COMMON_CITIES as C,getRandomDeviceType as E,getRandomLocale as v,getRandomCountry as S,getRandomCity as T,generateSessionId as k}from"./options.js";var n="https://scraper-api.smartproxy.com/v2/scrape";class d{config;http;authHeader;constructor(s){let e="username"in s&&"password"in s&&s.username&&s.password,t="token"in s&&s.token;if(!e&&!t)throw Error("Decodo requires either username/password or token for authentication");if(this.config={username:s.username??"",password:s.password??"",deviceType:s.deviceType??"desktop",locale:s.locale??"en-US",country:s.country??"",state:s.state??"",city:s.city??"",headless:s.headless??void 0,headers:s.headers??{},sessionId:s.sessionId??"",sessionDuration:s.sessionDuration??0,javascript:s.javascript??"",javascriptWait:s.javascriptWait??0,waitForCss:s.waitForCss??"",timeout:s.timeout??120000},this.http=new u({baseURL:n,timeout:this.config.timeout}),t)this.authHeader=`Basic ${s.token}`;else this.authHeader=`Basic ${Buffer.from(`${s.username}:${s.password}`).toString("base64")}`}async scrape(s,e){let t={...this.config,...e,headers:{...this.config.headers,...e?.headers||{}}},a=this.buildRequestBody(s,t),r=(await this.http.postJson(n,a,{headers:{Authorization:this.authHeader,"Content-Type":"application/json"}})).data;if(r.error)throw Error(`Decodo API error: ${r.error} (${r.error_code||"unknown"})`);if(!r.results||r.results.length===0)throw Error("Decodo API returned no results");let o=r.results[0];return{statusCode:o.status_code,url:o.url,content:o.body,cookies:o.cookies||[],headers:o.headers||{},taskId:r.id,rendered:!!t.headless,country:t.country||void 0,city:t.city||void 0,state:t.state||void 0,deviceType:t.deviceType,contentType:o.content_type,contentLength:o.content_length,raw:r}}async scrapeMany(s,e,t=1000){let a=[];for(let i=0;i<s.length;i++){let r=await this.scrape(s[i],e);if(a.push(r),i<s.length-1&&t>0)await new Promise((o)=>setTimeout(o,t))}return a}async scrapeWithSession(s,e,t=10){let a=`session_${Date.now()}_${Math.random().toString(36).substring(2,11)}`;return this.scrapeMany(s,{...e,sessionId:a,sessionDuration:t},500)}buildRequestBody(s,e){let t={target:"universal",url:s,return_cookies:!0,return_headers:!0};if(e.deviceType)t.device_type=e.deviceType;if(e.headless)t.headless=e.headless;else if(e.javascript_rendering)t.headless="html";if(e.locale)t.locale=e.locale;if(e.country)t.country=e.country;if(e.state)t.state=e.state;if(e.city)t.city=e.city;if(e.session_id||e.sessionId){if(t.session_id=e.session_id||e.sessionId,e.sessionDuration)t.session_duration=e.sessionDuration}if(e.headers&&Object.keys(e.headers).length>0)t.headers=e.headers,t.force_headers=!0;if(e.cookies&&e.cookies.length>0)t.cookies=e.cookies.map((a)=>`${a.key}=${a.value}`).join("; "),t.force_cookies=!0;if(e.http_method)t.http_method=e.http_method;if(e.base64Body)t.payload=e.base64Body;if(e.successful_status_codes&&e.successful_status_codes.length>0)t.successful_status_codes=e.successful_status_codes;if(e.javascript){if(t.javascript=e.javascript,e.javascriptWait)t.javascript_wait=e.javascriptWait}if(e.waitForCss)t.wait_for_css=e.waitForCss;return t}getConfig(){return{...this.config,password:"***"}}withConfig(s){return new d({...this.config,...s})}async testConnection(){try{return await this.scrape("https://httpbin.org/ip"),!0}catch(s){throw Error(`Decodo connection test failed: ${s.message}`)}}}var I=d;export{v as getRandomLocale,E as getRandomDeviceType,S as getRandomCountry,T as getRandomCity,k as generateSessionId,I as default,d as Decodo,m as DECODO_US_STATES,_ as DECODO_HEADLESS_MODES,O as DECODO_EUROPEAN_COUNTRIES,l as DECODO_DEVICE_TYPES,y as DECODO_COMMON_LOCALES,w as DECODO_COMMON_COUNTRIES,C as DECODO_COMMON_CITIES,D as DECODO_ASIAN_COUNTRIES};

package/dist/crawler/addon/oxylabs/index.cjs

@@ -1 +1 @@
-var{Rezo:p}=require("../../../core/rezo.cjs"),a=require("./options.cjs");exports.OXYLABS_BROWSER_TYPES=a.OXYLABS_BROWSER_TYPES;exports.OXYLABS_COMMON_LOCALES=a.OXYLABS_COMMON_LOCALES;exports.OXYLABS_COMMON_GEO_LOCATIONS=a.OXYLABS_COMMON_GEO_LOCATIONS;exports.OXYLABS_US_STATES=a.OXYLABS_US_STATES;exports.OXYLABS_EUROPEAN_COUNTRIES=a.OXYLABS_EUROPEAN_COUNTRIES;exports.OXYLABS_ASIAN_COUNTRIES=a.OXYLABS_ASIAN_COUNTRIES;exports.getRandomBrowserType=a.getRandomBrowserType;exports.getRandomLocale=a.getRandomLocale;exports.getRandomGeoLocation=a.getRandomGeoLocation;var c="https://realtime.oxylabs.io/v1/queries";class u{config;http;authHeader;constructor(e){if(!e.username||!e.password)throw Error("Oxylabs username and password are required");this.config={username:e.username,password:e.password,browserType:e.browserType??"desktop",locale:e.locale??"en-US",geoLocation:e.geoLocation??"",render:e.render??!1,context:e.context??{},timeout:e.timeout??120000},this.http=new p({baseURL:c,timeout:this.config.timeout}),this.authHeader=`Basic ${Buffer.from(`${e.username}:${e.password}`).toString("base64")}`}async scrape(e,t){let r={...this.config,...t},s=this.buildRequestBody(e,r),o=(await this.http.postJson(c,s,{headers:{Authorization:this.authHeader,"Content-Type":"application/json"}})).data;if(o.error)throw Error(`Oxylabs API error: ${o.error}`);if(!o.results||o.results.length===0)throw Error("Oxylabs API returned no results");let n=o.results[0],d=n._response?.cookies||[],l=n._response?.headers||{};return{statusCode:n.status_code,url:n.url,content:n.content,cookies:d,headers:l,jobId:o.job_id||n.job_id,rendered:r.render,geoLocation:r.geoLocation||void 0,locale:r.locale,browserType:r.browserType,raw:o}}async scrapeMany(e,t,r=1000){let s=[];for(let i=0;i<e.length;i++){let o=await this.scrape(e[i],t);if(s.push(o),i<e.length-1&&r>0)await new Promise((n)=>setTimeout(n,r))}return s}buildRequestBody(e,t){let r=[];if(r.push({key:"return_page_cookies",value:!0}),r.push({key:"return_page_headers",value:!0}),t.context)for(let[i,o]of Object.entries(t.context))r.push({key:i,value:o});let s={source:"universal",url:e,context:r};if(t.render)s.render="html";if(t.browserType)s.user_agent_type=t.browserType;if(t.locale)s.locale=t.locale;if(t.geoLocation)s.geo_location=t.geoLocation;return s}getConfig(){return{...this.config,password:"***"}}withConfig(e){return new u({...this.config,...e})}async testConnection(){try{return await this.scrape("https://httpbin.org/ip"),!0}catch(e){throw Error(`Oxylabs connection test failed: ${e.message}`)}}}exports.Oxylabs=u;exports.default=u;module.exports=Object.assign(u,exports);
+var{Rezo:h}=require("../../../core/rezo.cjs"),a=require("./options.cjs");exports.OXYLABS_BROWSER_TYPES=a.OXYLABS_BROWSER_TYPES;exports.OXYLABS_COMMON_LOCALES=a.OXYLABS_COMMON_LOCALES;exports.OXYLABS_COMMON_GEO_LOCATIONS=a.OXYLABS_COMMON_GEO_LOCATIONS;exports.OXYLABS_US_STATES=a.OXYLABS_US_STATES;exports.OXYLABS_EUROPEAN_COUNTRIES=a.OXYLABS_EUROPEAN_COUNTRIES;exports.OXYLABS_ASIAN_COUNTRIES=a.OXYLABS_ASIAN_COUNTRIES;exports.getRandomBrowserType=a.getRandomBrowserType;exports.getRandomLocale=a.getRandomLocale;exports.getRandomGeoLocation=a.getRandomGeoLocation;var i="https://realtime.oxylabs.io/v1/queries";class d{config;http;authHeader;constructor(s){if(!s.username||!s.password)throw Error("Oxylabs username and password are required");this.config={username:s.username,password:s.password,browserType:s.browserType??"desktop",locale:s.locale??"en-US",geoLocation:s.geoLocation??"",render:s.render??!1,context:s.context??{},timeout:s.timeout??120000},this.http=new h({baseURL:i,timeout:this.config.timeout}),this.authHeader=`Basic ${Buffer.from(`${s.username}:${s.password}`).toString("base64")}`}async scrape(s,e){let t={...this.config,...e},r=this.buildRequestBody(s,t),o=(await this.http.postJson(i,r,{headers:{Authorization:this.authHeader,"Content-Type":"application/json"}})).data;if(o.error)throw Error(`Oxylabs API error: ${o.error}`);if(!o.results||o.results.length===0)throw Error("Oxylabs API returned no results");let n=o.results[0],l=n._response?.cookies||[],_=n._response?.headers||{};return{statusCode:n.status_code,url:n.url,content:n.content,cookies:l,headers:_,jobId:o.job_id||n.job_id,rendered:t.render,geoLocation:t.geoLocation||void 0,locale:t.locale,browserType:t.browserType,raw:o}}async scrapeMany(s,e,t=1000){let r=[];for(let u=0;u<s.length;u++){let o=await this.scrape(s[u],e);if(r.push(o),u<s.length-1&&t>0)await new Promise((n)=>setTimeout(n,t))}return r}buildRequestBody(s,e){let t=[];if(t.push({key:"return_page_cookies",value:!0}),t.push({key:"return_page_headers",value:!0}),e.context)for(let[u,o]of Object.entries(e.context))t.push({key:u,value:o});if(e.headers&&Object.keys(e.headers).length>0)t.push({key:"headers",value:e.headers});if(e.cookies&&e.cookies.length>0)t.push({key:"cookies",value:e.cookies});if(e.session_id)t.push({key:"session_id",value:e.session_id});if(e.http_method)t.push({key:"http_method",value:e.http_method});if(e.base64Body)t.push({key:"content",value:e.base64Body});if(typeof e.follow_redirects==="boolean")t.push({key:"follow_redirects",value:e.follow_redirects});if(e.successful_status_codes&&e.successful_status_codes.length>0)t.push({key:"successful_status_codes",value:e.successful_status_codes});let r={source:"universal",url:s,context:t};if(e.render||e.javascript_rendering)r.render="html";if(e.browserType)r.user_agent_type=e.browserType;if(e.locale)r.locale=e.locale;if(e.geoLocation)r.geo_location=e.geoLocation;if(e.returnAsBase64)r.content_encoding="base64";return r}getConfig(){return{...this.config,password:"***"}}withConfig(s){return new d({...this.config,...s})}async testConnection(){try{return await this.scrape("https://httpbin.org/ip"),!0}catch(s){throw Error(`Oxylabs connection test failed: ${s.message}`)}}}exports.Oxylabs=d;exports.default=d;module.exports=Object.assign(d,exports);

package/dist/crawler/addon/oxylabs/index.js

@@ -1 +1 @@
-import{Rezo as d}from"../../../core/rezo.js";import{OXYLABS_BROWSER_TYPES as _,OXYLABS_COMMON_LOCALES as y,OXYLABS_COMMON_GEO_LOCATIONS as O,OXYLABS_US_STATES as b,OXYLABS_EUROPEAN_COUNTRIES as m,OXYLABS_ASIAN_COUNTRIES as L,getRandomBrowserType as S,getRandomLocale as A,getRandomGeoLocation as T}from"./options.js";var u="https://realtime.oxylabs.io/v1/queries";class i{config;http;authHeader;constructor(e){if(!e.username||!e.password)throw Error("Oxylabs username and password are required");this.config={username:e.username,password:e.password,browserType:e.browserType??"desktop",locale:e.locale??"en-US",geoLocation:e.geoLocation??"",render:e.render??!1,context:e.context??{},timeout:e.timeout??120000},this.http=new d({baseURL:u,timeout:this.config.timeout}),this.authHeader=`Basic ${Buffer.from(`${e.username}:${e.password}`).toString("base64")}`}async scrape(e,t){let r={...this.config,...t},s=this.buildRequestBody(e,r),o=(await this.http.postJson(u,s,{headers:{Authorization:this.authHeader,"Content-Type":"application/json"}})).data;if(o.error)throw Error(`Oxylabs API error: ${o.error}`);if(!o.results||o.results.length===0)throw Error("Oxylabs API returned no results");let a=o.results[0],c=a._response?.cookies||[],p=a._response?.headers||{};return{statusCode:a.status_code,url:a.url,content:a.content,cookies:c,headers:p,jobId:o.job_id||a.job_id,rendered:r.render,geoLocation:r.geoLocation||void 0,locale:r.locale,browserType:r.browserType,raw:o}}async scrapeMany(e,t,r=1000){let s=[];for(let n=0;n<e.length;n++){let o=await this.scrape(e[n],t);if(s.push(o),n<e.length-1&&r>0)await new Promise((a)=>setTimeout(a,r))}return s}buildRequestBody(e,t){let r=[];if(r.push({key:"return_page_cookies",value:!0}),r.push({key:"return_page_headers",value:!0}),t.context)for(let[n,o]of Object.entries(t.context))r.push({key:n,value:o});let s={source:"universal",url:e,context:r};if(t.render)s.render="html";if(t.browserType)s.user_agent_type=t.browserType;if(t.locale)s.locale=t.locale;if(t.geoLocation)s.geo_location=t.geoLocation;return s}getConfig(){return{...this.config,password:"***"}}withConfig(e){return new i({...this.config,...e})}async testConnection(){try{return await this.scrape("https://httpbin.org/ip"),!0}catch(e){throw Error(`Oxylabs connection test failed: ${e.message}`)}}}var B=i;export{A as getRandomLocale,T as getRandomGeoLocation,S as getRandomBrowserType,B as default,i as Oxylabs,b as OXYLABS_US_STATES,m as OXYLABS_EUROPEAN_COUNTRIES,y as OXYLABS_COMMON_LOCALES,O as OXYLABS_COMMON_GEO_LOCATIONS,_ as OXYLABS_BROWSER_TYPES,L as OXYLABS_ASIAN_COUNTRIES};
+import{Rezo as h}from"../../../core/rezo.js";import{OXYLABS_BROWSER_TYPES as y,OXYLABS_COMMON_LOCALES as w,OXYLABS_COMMON_GEO_LOCATIONS as b,OXYLABS_US_STATES as m,OXYLABS_EUROPEAN_COUNTRIES as O,OXYLABS_ASIAN_COUNTRIES as L,getRandomBrowserType as k,getRandomLocale as S,getRandomGeoLocation as A}from"./options.js";var d="https://realtime.oxylabs.io/v1/queries";class n{config;http;authHeader;constructor(t){if(!t.username||!t.password)throw Error("Oxylabs username and password are required");this.config={username:t.username,password:t.password,browserType:t.browserType??"desktop",locale:t.locale??"en-US",geoLocation:t.geoLocation??"",render:t.render??!1,context:t.context??{},timeout:t.timeout??120000},this.http=new h({baseURL:d,timeout:this.config.timeout}),this.authHeader=`Basic ${Buffer.from(`${t.username}:${t.password}`).toString("base64")}`}async scrape(t,e){let s={...this.config,...e},r=this.buildRequestBody(t,s),o=(await this.http.postJson(d,r,{headers:{Authorization:this.authHeader,"Content-Type":"application/json"}})).data;if(o.error)throw Error(`Oxylabs API error: ${o.error}`);if(!o.results||o.results.length===0)throw Error("Oxylabs API returned no results");let a=o.results[0],l=a._response?.cookies||[],i=a._response?.headers||{};return{statusCode:a.status_code,url:a.url,content:a.content,cookies:l,headers:i,jobId:o.job_id||a.job_id,rendered:s.render,geoLocation:s.geoLocation||void 0,locale:s.locale,browserType:s.browserType,raw:o}}async scrapeMany(t,e,s=1000){let r=[];for(let u=0;u<t.length;u++){let o=await this.scrape(t[u],e);if(r.push(o),u<t.length-1&&s>0)await new Promise((a)=>setTimeout(a,s))}return r}buildRequestBody(t,e){let s=[];if(s.push({key:"return_page_cookies",value:!0}),s.push({key:"return_page_headers",value:!0}),e.context)for(let[u,o]of Object.entries(e.context))s.push({key:u,value:o});if(e.headers&&Object.keys(e.headers).length>0)s.push({key:"headers",value:e.headers});if(e.cookies&&e.cookies.length>0)s.push({key:"cookies",value:e.cookies});if(e.session_id)s.push({key:"session_id",value:e.session_id});if(e.http_method)s.push({key:"http_method",value:e.http_method});if(e.base64Body)s.push({key:"content",value:e.base64Body});if(typeof e.follow_redirects==="boolean")s.push({key:"follow_redirects",value:e.follow_redirects});if(e.successful_status_codes&&e.successful_status_codes.length>0)s.push({key:"successful_status_codes",value:e.successful_status_codes});let r={source:"universal",url:t,context:s};if(e.render||e.javascript_rendering)r.render="html";if(e.browserType)r.user_agent_type=e.browserType;if(e.locale)r.locale=e.locale;if(e.geoLocation)r.geo_location=e.geoLocation;if(e.returnAsBase64)r.content_encoding="base64";return r}getConfig(){return{...this.config,password:"***"}}withConfig(t){return new n({...this.config,...t})}async testConnection(){try{return await this.scrape("https://httpbin.org/ip"),!0}catch(t){throw Error(`Oxylabs connection test failed: ${t.message}`)}}}var B=n;export{S as getRandomLocale,A as getRandomGeoLocation,k as getRandomBrowserType,B as default,n as Oxylabs,m as OXYLABS_US_STATES,O as OXYLABS_EUROPEAN_COUNTRIES,w as OXYLABS_COMMON_LOCALES,b as OXYLABS_COMMON_GEO_LOCATIONS,y as OXYLABS_BROWSER_TYPES,L as OXYLABS_ASIAN_COUNTRIES};

package/dist/crawler/crawler-options.cjs

@@ -1 +1 @@
-var{RezoQueue:f}=require("../queue/queue.cjs"),{Oxylabs:b}=require("./addon/oxylabs/index.cjs"),g=require("node:path"),x=require("node:os"),{Decodo:y}=require("./addon/decodo/index.cjs"),{Rezo:A}=require("../core/rezo.cjs");class m{baseUrl;adapter;enableNavigationHistory;sessionId;rejectUnauthorized;userAgent;useRndUserAgent;timeout;maxRedirects;maxRetryAttempts;retryDelay;retryOnStatusCode;forceRevisit;retryWithoutProxyOnStatusCode;retryOnProxyError;maxRetryOnProxyError;allowRevisiting;enableCache;cacheTTL;cacheDir;throwFatalError;debug;enableSignalHandlers;concurrency;scraperConcurrency;maxDepth;maxUrls;maxResponseSize;respectRobotsTxt;followNofollow;autoThrottle;autoThrottleTargetDelay;autoThrottleMinDelay;autoThrottleMaxDelay;maxWaitOn429;alwaysWaitOn429;stealth;oxylabs=[];decodo=[];stealthProfiles=[];proxies=[];limiters=[];onLimiterAdded;requestHeaders=[];userAgents=D();constructor(e={}){this.baseUrl=e.baseUrl||"",this.adapter=e.adapter??"http",this.enableNavigationHistory=e.enableNavigationHistory??!1,this.sessionId=e.sessionId??`session_${Date.now()}_${Math.random().toString(36).slice(2,8)}`,this.rejectUnauthorized=e.rejectUnauthorized??!0,this.userAgent=e.userAgent,this.useRndUserAgent=e.useRndUserAgent??!1,this.timeout=e.timeout??30000,this.maxRedirects=e.maxRedirects??10,this.maxRetryAttempts=e.maxRetryAttempts??3,this.retryDelay=e.retryDelay??0,this.retryOnStatusCode=e.retryOnStatusCode??[408,429,500,502,503,504],this.forceRevisit=e.forceRevisit??!1,this.retryWithoutProxyOnStatusCode=e.retryWithoutProxyOnStatusCode??[407,403],this.retryOnProxyError=e.retryOnProxyError??!0,this.maxRetryOnProxyError=e.maxRetryOnProxyError??3,this.allowRevisiting=e.allowRevisiting??!1,this.enableCache=e.enableCache??!0,this.cacheTTL=e.cacheTTL??604800000,this.cacheDir=e.cacheDir??g.join(x.tmpdir(),"rezo_cache"),this.throwFatalError=e.throwFatalError??!1,this.debug=e.debug??!1,this.enableSignalHandlers=e.enableSignalHandlers??!1,this.concurrency=e.concurrency??100,this.scraperConcurrency=e.scraperConcurrency??this.concurrency,this.maxDepth=e.maxDepth??0,this.maxUrls=e.maxUrls??0,this.maxResponseSize=e.maxResponseSize??0,this.respectRobotsTxt=e.respectRobotsTxt??!1,this.followNofollow=e.followNofollow??!1,this.autoThrottle=e.autoThrottle??!0,this.autoThrottleTargetDelay=e.autoThrottleTargetDelay??1000,this.autoThrottleMinDelay=e.autoThrottleMinDelay??100,this.autoThrottleMaxDelay=e.autoThrottleMaxDelay??60000,this.maxWaitOn429=e.maxWaitOn429??1800000,this.alwaysWaitOn429=e.alwaysWaitOn429??!1,this._addHeaders(e.headers),this._addOxylabs(e.oxylabs),this._addDecodo(e.decodo),this._addProxies(e.proxy),this._addLimiters(e.limiter),this._addStealths(e.stealths),this.stealth=e.stealth}getConfiguredDomains(e){return(e==="headers"?this.requestHeaders:e==="limiters"?this.limiters:e==="oxylabs"?this.oxylabs:e==="stealth"?this.stealthProfiles:this.proxies).filter((i)=>i.domain).map((i)=>i.domain).filter((i,a,r)=>r.indexOf(i)===a)}removeDomain(e){return this.requestHeaders=this.requestHeaders.filter((t)=>!t.domain||!this._domainsEqual(t.domain,e)),this.proxies=this.proxies.filter((t)=>!t.domain||!this._domainsEqual(t.domain,e)),this.limiters=this.limiters.filter((t)=>!t.domain||!this._domainsEqual(t.domain,e)),this.oxylabs=this.oxylabs.filter((t)=>!t.domain||!this._domainsEqual(t.domain,e)),this.stealthProfiles=this.stealthProfiles.filter((t)=>!t.domain||!this._domainsEqual(t.domain,e)),this}_domainsEqual(e,t){if(Array.isArray(e)&&Array.isArray(t))return e.length===t.length&&e.every((i,a)=>i===t[a]);return e===t}getConfigurationSummary(){let e=(t)=>({total:t.length,global:t.filter((i)=>i.isGlobal).length,domainSpecific:t.filter((i)=>!i.isGlobal&&i.domain).length});return{headers:e(this.requestHeaders),proxies:e(this.proxies),limiters:e(this.limiters),oxylabs:e(this.oxylabs)}}_addHeaders(e){if(!e||!e.enable)return;for(let t of e.httpHeaders){let{domain:i,isGlobal:a,headers:r}=t;if(!i&&!a)continue;if(r instanceof Headers){let s=Object.fromEntries(r.entries());if(Object.keys(s).length<1)continue;r=s}else if(!r||Object.keys(r).length<1)continue;this.requestHeaders.push({domain:i,isGlobal:a,headers:r})}}_addProxies(e){if(!e||!e.enable)return;for(let t of e.proxies){let{domain:i,isGlobal:a,proxy:r}=t;if(!i&&!a)continue;if(!r||Object.keys(r).length<1)continue;this.proxies.push({domain:i,isGlobal:a,proxy:r})}}_addLimiters(e){if(!e||!e.enable)return;for(let t of e.limiters){let{domain:i,isGlobal:a,options:r,retry:s}=t;if(!i&&!a)continue;if(!r&&!s)continue;let n=r?new f(r):new f({name:"limiter"});if(this.limiters.push({domain:i,isGlobal:a,pqueue:n,randomDelay:r?.randomDelay,retry:s}),this.onLimiterAdded)this.onLimiterAdded(n)}}_addOxylabs(e){if(!e||!e.enable)return;for(let t of e.labs){let{domain:i,isGlobal:a,options:r,queueOptions:s}=t;if(!i&&!a)continue;if(!r||Object.keys(r).length<1)continue;this.oxylabs.push({domain:i,isGlobal:a,adaptar:new b(r)})}}_addDecodo(e){if(!e||!e.enable)return;for(let t of e.labs){let{domain:i,isGlobal:a,options:r,queueOptions:s}=t;if(!i&&!a)continue;if(!r||Object.keys(r).length<1)continue;this.decodo.push({domain:i,isGlobal:a,adaptar:new y(r)})}}_addStealths(e){if(!e||!e.enable)return;for(let t of e.profiles){let{domain:i,isGlobal:a,stealth:r}=t;if(!i&&!a)continue;if(!r)continue;this.stealthProfiles.push({domain:i,isGlobal:a,adaptar:new A({stealth:r})})}}addHeaders(e){return this._addHeaders({enable:!0,httpHeaders:[e]}),this}addProxy(e){return this._addProxies({enable:!0,proxies:[e]}),this}addLimiter(e){return this._addLimiters({enable:!0,limiters:[e]}),this}addOxylabs(e){return this._addOxylabs({enable:!0,labs:[e]}),this}addDecodo(e){return this._addDecodo({enable:!0,labs:[e]}),this}addStealth(e){return this._addStealths({enable:!0,profiles:[e]}),this}destroyLimiters(){for(let e of this.limiters)if(e.pqueue&&typeof e.pqueue.destroy==="function")e.pqueue.destroy();this.limiters=[]}getLimiters(){return this.limiters}getRandomDelay(e,t){if(!this.getDomainName(e))return;for(let a of this.limiters)if(this._hasDomain(e,a.domain)&&a.randomDelay!==void 0)return a.randomDelay;if(t){for(let a of this.limiters)if(a.isGlobal&&a.randomDelay!==void 0)return a.randomDelay}return}clearGlobalConfigs(){if(Array.isArray(this.requestHeaders))this.requestHeaders=this.requestHeaders.filter((e)=>!e.isGlobal);if(Array.isArray(this.oxylabs))this.oxylabs=this.oxylabs.filter((e)=>!e.isGlobal);if(Array.isArray(this.limiters))this.limiters=this.limiters.filter((e)=>!e.isGlobal);if(Array.isArray(this.proxies))this.proxies=this.proxies.filter((e)=>!e.isGlobal);if(Array.isArray(this.stealthProfiles))this.stealthProfiles=this.stealthProfiles.filter((e)=>!e.isGlobal);return this}getAdapter(e,t,i,a){if(!this.getDomainName(e))return null;let s=[],n=t==="headers"?this.requestHeaders:t==="limiters"?this.limiters:t==="oxylabs"?this.oxylabs:t==="decodo"?this.decodo:t==="stealth"?this.stealthProfiles:this.proxies;for(let o=0;o<n.length;o++)if(this._hasDomain(e,n[o].domain))s.push(o);if(s.length){let o=a?s[this.rnd(0,s.length-1)]:s[0];return t==="headers"?this.requestHeaders[o].headers:t==="limiters"?this.limiters[o].pqueue:t==="oxylabs"?this.oxylabs[o].adaptar:t==="decodo"?this.decodo[o].adaptar:t==="stealth"?this.stealthProfiles[o].adaptar:this.proxies[o].proxy}s.length=0;for(let o=0;o<n.length;o++)s.push(o);if(s.length){let o=a?s[this.rnd(0,s.length-1)]:s[0];if(n[o].isGlobal&&i)return t==="headers"?this.requestHeaders[o].headers:t==="limiters"?this.limiters[o].pqueue:t==="oxylabs"?this.oxylabs[o].adaptar:t==="decodo"?this.decodo[o].adaptar:t==="stealth"?this.stealthProfiles[o].adaptar:this.proxies[o].proxy}return null}getRetryOptions(e){if(!this.getDomainName(e))return null;for(let i=0;i<this.limiters.length;i++)if(this._hasDomain(e,this.limiters[i].domain))return this.limiters[i].retry||null;for(let i=0;i<this.limiters.length;i++)if(this.limiters[i].isGlobal&&this.limiters[i].retry)return this.limiters[i].retry;return null}rnd(e=0,t=Number.MAX_VALUE){return Math.floor(Math.random()*(t-e+1))+e}hasDomain(e,t,i){if(!this.getDomainName(e))return!1;let r=t==="headers"?this.requestHeaders:t==="limiters"?this.limiters:t==="oxylabs"?this.oxylabs:t==="decodo"?this.decodo:t==="stealth"?this.stealthProfiles:this.proxies;for(let s=0;s<r.length;s++)if(this._hasDomain(e,r[s].domain))return!0;if(i){for(let s=0;s<r.length;s++)if(r[s].isGlobal)return!0}return!1}pickHeaders(e,t,i,a){let r=this.getAdapter(e,"headers",t),s=new Headers(r??{}),n=s.count;if(i&&i instanceof Headers)for(let[o,l]of Object.entries(i.entries()))s.set(o,l);else if(i&&typeof i==="object"){for(let[o,l]of Object.entries(i))if(typeof l==="string")s.set(o,l)}if(a&&n===0&&!this.stealth)s.set("user-agent",this.getRandomUserAgent());return Object.fromEntries(s.entries())}_hasDomain(e,t){if(!t)return!1;let i=this.getDomainName(e);if(!i)return!1;let a=(s)=>{return/[\^\$\*\+\?\{\}\[\]\(\)\|\\]/.test(s)||s.startsWith("/")||s.includes(".*")||s.includes(".+")},r=(s)=>{if(s instanceof RegExp)return s.test(i)||s.test(e);let n=s.toString().trim();if(i.toLowerCase()===n.toLowerCase())return!0;if(n.includes("*")){let h=n.replace(/[.*+?^${}()|[\]\\]/g,"\\$&").replace(/\\\*/g,".*"),d=new RegExp(`^${h}$`,"i");return d.test(i)||d.test(e)}if(a(n))try{let h=n,d="i",u=n.match(/^\/(.*)\/(\w*)$/);if(u)h=u[1],d=u[2]||"i";let c=new RegExp(h,d);return c.test(i)||c.test(e)}catch(h){return i.toLowerCase().includes(n.toLowerCase())}let o=i.toLowerCase(),l=n.toLowerCase();return o===l||o.endsWith("."+l)||l.endsWith("."+o)};if(Array.isArray(t)){for(let s of t)if(r(s))return!0;return!1}return r(t)}getDomainName(e){if(this.isValidUrl(e))return new URL(e).hostname;else if(this.isHostName(e))return e;return null}isHostName(e){if(!e)return!1;if(e.length>255)return!1;let t=/^(?:(?:[a-zA-Z0-9](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?\.)+ [a-zA-Z]{2,})$/;return e=e.trim().toLowerCase(),t.test(e)&&!e.startsWith("-")&&!e.endsWith("-")}isValidUrl(e){if(!e)return!1;e=e.trim();try{let t=new URL(e);if(!t.protocol||!["http:","https:"].includes(t.protocol.toLowerCase()))return!1;if(!t.hostname)return!1;if(!/^(?:(?:[a-zA-Z0-9](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?\.)+[a-zA-Z]{2,})$/.test(t.hostname))return!1;return!0}catch{return!1}}getRandomUserAgent(){return this.userAgents[Math.floor(Math.random()*this.userAgents.length)]}}function D(){let e=[{name:"Chrome",version:"91.0.4472.124",engine:"AppleWebKit/537.36"},{name:"Firefox",version:"89.0",engine:"Gecko/20100101"},{name:"Safari",version:"14.1.1",engine:"AppleWebKit/605.1.15"},{name:"Edge",version:"91.0.864.59",engine:"AppleWebKit/537.36"},{name:"Opera",version:"77.0.4054.277",engine:"AppleWebKit/537.36"},{name:"Vivaldi",version:"3.8.2259.42",engine:"AppleWebKit/537.36"},{name:"Brave",version:"1.26.74",engine:"AppleWebKit/537.36"},{name:"Chromium",version:"91.0.4472.101",engine:"AppleWebKit/537.36"},{name:"Yandex",version:"21.5.3.742",engine:"AppleWebKit/537.36"},{name:"Maxthon",version:"5.3.8.2000",engine:"AppleWebKit/537.36"}],t=["Windows NT 10.0","Windows NT 6.1","Macintosh; Intel Mac OS X 10_15_7","Macintosh; Intel Mac OS X 11_4_0","X11; Linux x86_64","X11; Ubuntu; Linux x86_64"],i=[];for(let a=0;a<200;a++){let r=e[Math.floor(Math.random()*e.length)],s=t[Math.floor(Math.random()*t.length)],n="";switch(r.name){case"Chrome":n=`Mozilla/5.0 (${s}) ${r.engine} (KHTML, like Gecko) Chrome/${r.version} Safari/537.36`;break;case"Firefox":n=`Mozilla/5.0 (${s}; rv:${r.version}) ${r.engine} Firefox/${r.version}`;break;case"Safari":n=`Mozilla/5.0 (${s}) ${r.engine} (KHTML, like Gecko) Version/${r.version} Safari/605.1.15`;break;case"Edge":n=`Mozilla/5.0 (${s}) ${r.engine} (KHTML, like Gecko) Chrome/${r.version} Safari/537.36 Edg/${r.version}`;break;case"Opera":n=`Mozilla/5.0 (${s}) ${r.engine} (KHTML, like Gecko) Chrome/${r.version} Safari/537.36 OPR/${r.version}`;break;case"Vivaldi":n=`Mozilla/5.0 (${s}) ${r.engine} (KHTML, like Gecko) Chrome/${r.version} Safari/537.36 Vivaldi/${r.version}`;break;case"Brave":n=`Mozilla/5.0 (${s}) ${r.engine} (KHTML, like Gecko) Chrome/${r.version} Safari/537.36 Brave/${r.version}`;break;case"Chromium":n=`Mozilla/5.0 (${s}) ${r.engine} (KHTML, like Gecko) Chromium/${r.version} Chrome/${r.version} Safari/537.36`;break;case"Yandex":n=`Mozilla/5.0 (${s}) ${r.engine} (KHTML, like Gecko) Chrome/${r.version} YaBrowser/${r.version} Safari/537.36`;break;case"Maxthon":n=`Mozilla/5.0 (${s}) ${r.engine} (KHTML, like Gecko) Chrome/${r.version} Safari/537.36 Maxthon/${r.version}`;break}i.push(n)}return i}exports.CrawlerOptions=m;
+var{RezoQueue:y}=require("../queue/queue.cjs"),{Oxylabs:x}=require("./addon/oxylabs/index.cjs"),v=require("node:path"),A=require("node:os"),{Decodo:p}=require("./addon/decodo/index.cjs"),{Rezo:D}=require("../core/rezo.cjs");class l{static destroyQueue(e){if(e&&typeof e.destroy==="function")e.destroy()}static destroyConfigQueues(e){for(let r of e)l.destroyQueue(r.pqueue)}static splitConfigs(e,r){let s=[],a=[];for(let t of e)if(r(t))a.push(t);else s.push(t);return{kept:s,removed:a}}static getHostname(e){try{return new URL(e).hostname||void 0}catch{return}}static createStableThroughputOptions(e){let{baseUrl:r,concurrency:s=40,scraperConcurrency:a=10,retryDelay:t=1000,maxRetryAttempts:i=2,retryOnStatusCode:o=[408,500,502,503,504],maxWaitOn429:d=15000,alwaysWaitOn429:n=!1,globalLimiter:h={concurrency:8},domain:u=l.getHostname(r),domainLimiter:c={concurrency:2,interval:1000,intervalCap:2,randomDelay:150},domainRetry:m={enable:!0,max429Retries:2,retryDelay:1000,maxRetryAttempts:2,backoff:!0},extraLimiters:g=[],overrides:b={}}=e,f=[];if(h)f.push({isGlobal:!0,options:h});if(u&&c)f.push({domain:u,options:c,retry:m||void 0});return f.push(...g),{...{baseUrl:r,autoThrottle:!1,concurrency:s,scraperConcurrency:a,retryDelay:t,maxRetryAttempts:i,retryOnStatusCode:o,maxWaitOn429:d,alwaysWaitOn429:n,limiter:f.length>0?{enable:!0,limiters:f}:!1},...b,baseUrl:r}}baseUrl;adapter;enableNavigationHistory;sessionId;rejectUnauthorized;userAgent;useRndUserAgent;timeout;maxRedirects;maxRetryAttempts;retryDelay;retryOnStatusCode;forceRevisit;retryWithoutProxyOnStatusCode;retryOnProxyError;maxRetryOnProxyError;allowRevisiting;enableCache;cacheTTL;cacheDir;throwFatalError;debug;enableSignalHandlers;concurrency;scraperConcurrency;maxDepth;maxUrls;maxResponseSize;respectRobotsTxt;followNofollow;autoThrottle;autoThrottleTargetDelay;autoThrottleMinDelay;autoThrottleMaxDelay;maxWaitOn429;alwaysWaitOn429;stealth;oxylabs=[];decodo=[];stealthProfiles=[];proxies=[];limiters=[];onLimiterAdded;onProviderAdded;requestHeaders=[];userAgents=$();constructor(e={}){this.baseUrl=e.baseUrl||"",this.adapter=e.adapter??"http",this.enableNavigationHistory=e.enableNavigationHistory??!1,this.sessionId=e.sessionId??`session_${Date.now()}_${Math.random().toString(36).slice(2,8)}`,this.rejectUnauthorized=e.rejectUnauthorized??!0,this.userAgent=e.userAgent,this.useRndUserAgent=e.useRndUserAgent??!1,this.timeout=e.timeout??30000,this.maxRedirects=e.maxRedirects??10,this.maxRetryAttempts=e.maxRetryAttempts??3,this.retryDelay=e.retryDelay??0,this.retryOnStatusCode=e.retryOnStatusCode??[408,429,500,502,503,504],this.forceRevisit=e.forceRevisit??!1,this.retryWithoutProxyOnStatusCode=e.retryWithoutProxyOnStatusCode??[407,403],this.retryOnProxyError=e.retryOnProxyError??!0,this.maxRetryOnProxyError=e.maxRetryOnProxyError??3,this.allowRevisiting=e.allowRevisiting??!1,this.enableCache=e.enableCache??!0,this.cacheTTL=e.cacheTTL??604800000,this.cacheDir=e.cacheDir??v.join(A.tmpdir(),"rezo_cache"),this.throwFatalError=e.throwFatalError??!1,this.debug=e.debug??!1,this.enableSignalHandlers=e.enableSignalHandlers??!1,this.concurrency=e.concurrency??100,this.scraperConcurrency=e.scraperConcurrency??this.concurrency,this.maxDepth=e.maxDepth??0,this.maxUrls=e.maxUrls??0,this.maxResponseSize=e.maxResponseSize??0,this.respectRobotsTxt=e.respectRobotsTxt??!1,this.followNofollow=e.followNofollow??!1,this.autoThrottle=e.autoThrottle??!0,this.autoThrottleTargetDelay=e.autoThrottleTargetDelay??1000,this.autoThrottleMinDelay=e.autoThrottleMinDelay??100,this.autoThrottleMaxDelay=e.autoThrottleMaxDelay??60000,this.maxWaitOn429=e.maxWaitOn429??1800000,this.alwaysWaitOn429=e.alwaysWaitOn429??!1,this._addHeaders(e.headers),this._addOxylabs(e.oxylabs),this._addDecodo(e.decodo),this._addProxies(e.proxy),this._addLimiters(e.limiter),this._addStealths(e.stealths),this.stealth=e.stealth}getConfiguredDomains(e){return(e==="headers"?this.requestHeaders:e==="limiters"?this.limiters:e==="oxylabs"?this.oxylabs:e==="decodo"?this.decodo:e==="stealth"?this.stealthProfiles:this.proxies).filter((s)=>s.domain).map((s)=>s.domain).filter((s,a,t)=>t.indexOf(s)===a)}removeDomain(e){this.requestHeaders=this.requestHeaders.filter((t)=>!t.domain||!this._domainsEqual(t.domain,e)),this.proxies=this.proxies.filter((t)=>!t.domain||!this._domainsEqual(t.domain,e));let r=l.splitConfigs(this.limiters,(t)=>!!t.domain&&this._domainsEqual(t.domain,e));l.destroyConfigQueues(r.removed),this.limiters=r.kept;let s=l.splitConfigs(this.oxylabs,(t)=>!!t.domain&&this._domainsEqual(t.domain,e));l.destroyConfigQueues(s.removed),this.oxylabs=s.kept;let a=l.splitConfigs(this.decodo,(t)=>!!t.domain&&this._domainsEqual(t.domain,e));return l.destroyConfigQueues(a.removed),this.decodo=a.kept,this.stealthProfiles=this.stealthProfiles.filter((t)=>!t.domain||!this._domainsEqual(t.domain,e)),this}_domainsEqual(e,r){if(Array.isArray(e)&&Array.isArray(r))return e.length===r.length&&e.every((s,a)=>s===r[a]);return e===r}getConfigurationSummary(){let e=(r)=>({total:r.length,global:r.filter((s)=>s.isGlobal).length,domainSpecific:r.filter((s)=>!s.isGlobal&&s.domain).length});return{headers:e(this.requestHeaders),proxies:e(this.proxies),limiters:e(this.limiters),oxylabs:e(this.oxylabs),decodo:e(this.decodo)}}_addHeaders(e){if(!e||!e.enable)return;for(let r of e.httpHeaders){let{domain:s,isGlobal:a,headers:t}=r;if(!s&&!a)continue;if(t instanceof Headers){let i=Object.fromEntries(t.entries());if(Object.keys(i).length<1)continue;t=i}else if(!t||Object.keys(t).length<1)continue;this.requestHeaders.push({domain:s,isGlobal:a,headers:t})}}_addProxies(e){if(!e||!e.enable)return;for(let r of e.proxies){let{domain:s,isGlobal:a,proxy:t,rotating:i}=r;if(!s&&!a)continue;if(!t||Object.keys(t).length<1)continue;this.proxies.push({domain:s,isGlobal:a,proxy:t,rotating:!!i})}}_addLimiters(e){if(!e||!e.enable)return;for(let r of e.limiters){let{domain:s,isGlobal:a,options:t,retry:i}=r;if(!s&&!a)continue;if(!t&&!i)continue;let o=t?new y(t):new y({name:"limiter"});if(this.limiters.push({domain:s,isGlobal:a,pqueue:o,randomDelay:t?.randomDelay,retry:i}),this.onLimiterAdded)this.onLimiterAdded(o)}}_addOxylabs(e){if(!e||!e.enable)return;for(let r of e.labs){let{domain:s,isGlobal:a,options:t,queueOptions:i}=r;if(!s&&!a)continue;if(!t||Object.keys(t).length<1)continue;let o=i?new y(i):void 0;if(this.oxylabs.push({domain:s,isGlobal:a,adaptar:new x(t),pqueue:o}),o&&this.onProviderAdded)this.onProviderAdded(o)}}_addDecodo(e){if(!e||!e.enable)return;for(let r of e.labs){let{domain:s,isGlobal:a,options:t,queueOptions:i}=r;if(!s&&!a)continue;if(!t||Object.keys(t).length<1)continue;let o=i?new y(i):void 0;if(this.decodo.push({domain:s,isGlobal:a,adaptar:new p(t),pqueue:o}),o&&this.onProviderAdded)this.onProviderAdded(o)}}_addStealths(e){if(!e||!e.enable)return;for(let r of e.profiles){let{domain:s,isGlobal:a,stealth:t}=r;if(!s&&!a)continue;if(!t)continue;this.stealthProfiles.push({domain:s,isGlobal:a,adaptar:new D({stealth:t})})}}addHeaders(e){return this._addHeaders({enable:!0,httpHeaders:[e]}),this}addProxy(e){return this._addProxies({enable:!0,proxies:[e]}),this}addProxies(e){return this._addProxies({enable:!0,proxies:e}),this}getProxyConfig(e,r,s){let a=this.selectConfigEntry(e,this.proxies,r,s);if(!a)return null;return{proxy:a.proxy,rotating:!!a.rotating}}addLimiter(e){return this._addLimiters({enable:!0,limiters:[e]}),this}addOxylabs(e){return this._addOxylabs({enable:!0,labs:[e]}),this}addDecodo(e){return this._addDecodo({enable:!0,labs:[e]}),this}addStealth(e){return this._addStealths({enable:!0,profiles:[e]}),this}destroyLimiters(){for(let e of this.limiters)l.destroyQueue(e.pqueue);this.limiters=[]}destroyProviderQueues(){for(let e of this.oxylabs)l.destroyQueue(e.pqueue),delete e.pqueue;for(let e of this.decodo)l.destroyQueue(e.pqueue),delete e.pqueue}getLimiters(){return this.limiters}getRandomDelay(e,r){if(!this.getDomainName(e))return;for(let a of this.limiters)if(this._hasDomain(e,a.domain)&&a.randomDelay!==void 0)return a.randomDelay;if(r){for(let a of this.limiters)if(a.isGlobal&&a.randomDelay!==void 0)return a.randomDelay}return}clearGlobalConfigs(){if(Array.isArray(this.requestHeaders))this.requestHeaders=this.requestHeaders.filter((e)=>!e.isGlobal);if(Array.isArray(this.oxylabs)){let e=l.splitConfigs(this.oxylabs,(r)=>!!r.isGlobal);l.destroyConfigQueues(e.removed),this.oxylabs=e.kept}if(Array.isArray(this.decodo)){let e=l.splitConfigs(this.decodo,(r)=>!!r.isGlobal);l.destroyConfigQueues(e.removed),this.decodo=e.kept}if(Array.isArray(this.limiters)){let e=l.splitConfigs(this.limiters,(r)=>!!r.isGlobal);l.destroyConfigQueues(e.removed),this.limiters=e.kept}if(Array.isArray(this.proxies))this.proxies=this.proxies.filter((e)=>!e.isGlobal);if(Array.isArray(this.stealthProfiles))this.stealthProfiles=this.stealthProfiles.filter((e)=>!e.isGlobal);return this}getAdapter(e,r,s,a){if(!this.getDomainName(e))return null;let i=r==="headers"?this.requestHeaders:r==="limiters"?this.limiters:r==="oxylabs"?this.oxylabs:r==="decodo"?this.decodo:r==="stealth"?this.stealthProfiles:this.proxies,o=this.selectConfigEntry(e,i,s,a);if(!o)return null;return r==="headers"?o.headers:r==="limiters"?o.pqueue:r==="oxylabs"?o.adaptar:r==="decodo"?o.adaptar:r==="stealth"?o.adaptar:o.proxy}getRetryOptions(e){if(!this.getDomainName(e))return null;for(let s=0;s<this.limiters.length;s++)if(this._hasDomain(e,this.limiters[s].domain))return this.limiters[s].retry||null;for(let s=0;s<this.limiters.length;s++)if(this.limiters[s].isGlobal&&this.limiters[s].retry)return this.limiters[s].retry;return null}rnd(e=0,r=Number.MAX_VALUE){return Math.floor(Math.random()*(r-e+1))+e}hasDomain(e,r,s){if(!this.getDomainName(e))return!1;let t=r==="headers"?this.requestHeaders:r==="limiters"?this.limiters:r==="oxylabs"?this.oxylabs:r==="decodo"?this.decodo:r==="stealth"?this.stealthProfiles:this.proxies;for(let i=0;i<t.length;i++)if(this._hasDomain(e,t[i].domain))return!0;if(s){for(let i=0;i<t.length;i++)if(t[i].isGlobal)return!0}return!1}pickHeaders(e,r,s,a){let t=this.getAdapter(e,"headers",r),i=new Headers(t??{}),o=i.count;if(s&&s instanceof Headers)for(let[d,n]of Object.entries(s.entries()))i.set(d,n);else if(s&&typeof s==="object"){for(let[d,n]of Object.entries(s))if(typeof n==="string")i.set(d,n)}if(a&&o===0&&!this.stealth)i.set("user-agent",this.getRandomUserAgent());return Object.fromEntries(i.entries())}_hasDomain(e,r){if(!r)return!1;let s=this.getDomainName(e);if(!s)return!1;let a=(i)=>{return/[\^\$\*\+\?\{\}\[\]\(\)\|\\]/.test(i)||i.startsWith("/")||i.includes(".*")||i.includes(".+")},t=(i)=>{if(i instanceof RegExp)return i.test(s)||i.test(e);let o=i.toString().trim();if(s.toLowerCase()===o.toLowerCase())return!0;if(o.includes("*")){let h=o.replace(/[.*+?^${}()|[\]\\]/g,"\\$&").replace(/\\\*/g,".*"),u=new RegExp(`^${h}$`,"i");return u.test(s)||u.test(e)}if(a(o))try{let h=o,u="i",c=o.match(/^\/(.*)\/(\w*)$/);if(c)h=c[1],u=c[2]||"i";let m=new RegExp(h,u);return m.test(s)||m.test(e)}catch(h){return s.toLowerCase().includes(o.toLowerCase())}let d=s.toLowerCase(),n=o.toLowerCase();return d===n||d.endsWith("."+n)||n.endsWith("."+d)};if(Array.isArray(r)){for(let i of r)if(t(i))return!0;return!1}return t(r)}selectConfigEntry(e,r,s,a){if(!this.getDomainName(e))return null;let i=(n)=>{if(!n.length)return null;let h=a?this.rnd(0,n.length-1):0;return n[h]},o=[];for(let n of r)if(this._hasDomain(e,n.domain))o.push(n);let d=i(o);if(d)return d;if(s){let n=[];for(let h of r)if(h.isGlobal)n.push(h);return i(n)}return null}getDomainName(e){if(this.isValidUrl(e))return new URL(e).hostname;else if(this.isHostName(e))return e;return null}isHostName(e){if(!e)return!1;if(e.length>255)return!1;let r=/^(?:(?:[a-zA-Z0-9](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?\.)+ [a-zA-Z]{2,})$/;return e=e.trim().toLowerCase(),r.test(e)&&!e.startsWith("-")&&!e.endsWith("-")}isValidUrl(e){if(!e)return!1;e=e.trim();try{let r=new URL(e);if(!r.protocol||!["http:","https:"].includes(r.protocol.toLowerCase()))return!1;if(!r.hostname)return!1;if(!/^(?:(?:[a-zA-Z0-9](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?\.)+[a-zA-Z]{2,})$/.test(r.hostname))return!1;return!0}catch{return!1}}getRandomUserAgent(){return this.userAgents[Math.floor(Math.random()*this.userAgents.length)]}}function $(){let e=[{name:"Chrome",version:"91.0.4472.124",engine:"AppleWebKit/537.36"},{name:"Firefox",version:"89.0",engine:"Gecko/20100101"},{name:"Safari",version:"14.1.1",engine:"AppleWebKit/605.1.15"},{name:"Edge",version:"91.0.864.59",engine:"AppleWebKit/537.36"},{name:"Opera",version:"77.0.4054.277",engine:"AppleWebKit/537.36"},{name:"Vivaldi",version:"3.8.2259.42",engine:"AppleWebKit/537.36"},{name:"Brave",version:"1.26.74",engine:"AppleWebKit/537.36"},{name:"Chromium",version:"91.0.4472.101",engine:"AppleWebKit/537.36"},{name:"Yandex",version:"21.5.3.742",engine:"AppleWebKit/537.36"},{name:"Maxthon",version:"5.3.8.2000",engine:"AppleWebKit/537.36"}],r=["Windows NT 10.0","Windows NT 6.1","Macintosh; Intel Mac OS X 10_15_7","Macintosh; Intel Mac OS X 11_4_0","X11; Linux x86_64","X11; Ubuntu; Linux x86_64"],s=[];for(let a=0;a<200;a++){let t=e[Math.floor(Math.random()*e.length)],i=r[Math.floor(Math.random()*r.length)],o="";switch(t.name){case"Chrome":o=`Mozilla/5.0 (${i}) ${t.engine} (KHTML, like Gecko) Chrome/${t.version} Safari/537.36`;break;case"Firefox":o=`Mozilla/5.0 (${i}; rv:${t.version}) ${t.engine} Firefox/${t.version}`;break;case"Safari":o=`Mozilla/5.0 (${i}) ${t.engine} (KHTML, like Gecko) Version/${t.version} Safari/605.1.15`;break;case"Edge":o=`Mozilla/5.0 (${i}) ${t.engine} (KHTML, like Gecko) Chrome/${t.version} Safari/537.36 Edg/${t.version}`;break;case"Opera":o=`Mozilla/5.0 (${i}) ${t.engine} (KHTML, like Gecko) Chrome/${t.version} Safari/537.36 OPR/${t.version}`;break;case"Vivaldi":o=`Mozilla/5.0 (${i}) ${t.engine} (KHTML, like Gecko) Chrome/${t.version} Safari/537.36 Vivaldi/${t.version}`;break;case"Brave":o=`Mozilla/5.0 (${i}) ${t.engine} (KHTML, like Gecko) Chrome/${t.version} Safari/537.36 Brave/${t.version}`;break;case"Chromium":o=`Mozilla/5.0 (${i}) ${t.engine} (KHTML, like Gecko) Chromium/${t.version} Chrome/${t.version} Safari/537.36`;break;case"Yandex":o=`Mozilla/5.0 (${i}) ${t.engine} (KHTML, like Gecko) Chrome/${t.version} YaBrowser/${t.version} Safari/537.36`;break;case"Maxthon":o=`Mozilla/5.0 (${i}) ${t.engine} (KHTML, like Gecko) Chrome/${t.version} Safari/537.36 Maxthon/${t.version}`;break}s.push(o)}return s}exports.CrawlerOptions=l;

package/dist/crawler/crawler-options.js

@@ -1 +1 @@
-import{RezoQueue as m}from"../queue/queue.js";import{Oxylabs as f}from"./addon/oxylabs/index.js";import b from"node:path";import x from"node:os";import{Decodo as g}from"./addon/decodo/index.js";import{Rezo as y}from"../core/rezo.js";class A{baseUrl;adapter;enableNavigationHistory;sessionId;rejectUnauthorized;userAgent;useRndUserAgent;timeout;maxRedirects;maxRetryAttempts;retryDelay;retryOnStatusCode;forceRevisit;retryWithoutProxyOnStatusCode;retryOnProxyError;maxRetryOnProxyError;allowRevisiting;enableCache;cacheTTL;cacheDir;throwFatalError;debug;enableSignalHandlers;concurrency;scraperConcurrency;maxDepth;maxUrls;maxResponseSize;respectRobotsTxt;followNofollow;autoThrottle;autoThrottleTargetDelay;autoThrottleMinDelay;autoThrottleMaxDelay;maxWaitOn429;alwaysWaitOn429;stealth;oxylabs=[];decodo=[];stealthProfiles=[];proxies=[];limiters=[];onLimiterAdded;requestHeaders=[];userAgents=D();constructor(e={}){this.baseUrl=e.baseUrl||"",this.adapter=e.adapter??"http",this.enableNavigationHistory=e.enableNavigationHistory??!1,this.sessionId=e.sessionId??`session_${Date.now()}_${Math.random().toString(36).slice(2,8)}`,this.rejectUnauthorized=e.rejectUnauthorized??!0,this.userAgent=e.userAgent,this.useRndUserAgent=e.useRndUserAgent??!1,this.timeout=e.timeout??30000,this.maxRedirects=e.maxRedirects??10,this.maxRetryAttempts=e.maxRetryAttempts??3,this.retryDelay=e.retryDelay??0,this.retryOnStatusCode=e.retryOnStatusCode??[408,429,500,502,503,504],this.forceRevisit=e.forceRevisit??!1,this.retryWithoutProxyOnStatusCode=e.retryWithoutProxyOnStatusCode??[407,403],this.retryOnProxyError=e.retryOnProxyError??!0,this.maxRetryOnProxyError=e.maxRetryOnProxyError??3,this.allowRevisiting=e.allowRevisiting??!1,this.enableCache=e.enableCache??!0,this.cacheTTL=e.cacheTTL??604800000,this.cacheDir=e.cacheDir??b.join(x.tmpdir(),"rezo_cache"),this.throwFatalError=e.throwFatalError??!1,this.debug=e.debug??!1,this.enableSignalHandlers=e.enableSignalHandlers??!1,this.concurrency=e.concurrency??100,this.scraperConcurrency=e.scraperConcurrency??this.concurrency,this.maxDepth=e.maxDepth??0,this.maxUrls=e.maxUrls??0,this.maxResponseSize=e.maxResponseSize??0,this.respectRobotsTxt=e.respectRobotsTxt??!1,this.followNofollow=e.followNofollow??!1,this.autoThrottle=e.autoThrottle??!0,this.autoThrottleTargetDelay=e.autoThrottleTargetDelay??1000,this.autoThrottleMinDelay=e.autoThrottleMinDelay??100,this.autoThrottleMaxDelay=e.autoThrottleMaxDelay??60000,this.maxWaitOn429=e.maxWaitOn429??1800000,this.alwaysWaitOn429=e.alwaysWaitOn429??!1,this._addHeaders(e.headers),this._addOxylabs(e.oxylabs),this._addDecodo(e.decodo),this._addProxies(e.proxy),this._addLimiters(e.limiter),this._addStealths(e.stealths),this.stealth=e.stealth}getConfiguredDomains(e){return(e==="headers"?this.requestHeaders:e==="limiters"?this.limiters:e==="oxylabs"?this.oxylabs:e==="stealth"?this.stealthProfiles:this.proxies).filter((i)=>i.domain).map((i)=>i.domain).filter((i,a,r)=>r.indexOf(i)===a)}removeDomain(e){return this.requestHeaders=this.requestHeaders.filter((t)=>!t.domain||!this._domainsEqual(t.domain,e)),this.proxies=this.proxies.filter((t)=>!t.domain||!this._domainsEqual(t.domain,e)),this.limiters=this.limiters.filter((t)=>!t.domain||!this._domainsEqual(t.domain,e)),this.oxylabs=this.oxylabs.filter((t)=>!t.domain||!this._domainsEqual(t.domain,e)),this.stealthProfiles=this.stealthProfiles.filter((t)=>!t.domain||!this._domainsEqual(t.domain,e)),this}_domainsEqual(e,t){if(Array.isArray(e)&&Array.isArray(t))return e.length===t.length&&e.every((i,a)=>i===t[a]);return e===t}getConfigurationSummary(){let e=(t)=>({total:t.length,global:t.filter((i)=>i.isGlobal).length,domainSpecific:t.filter((i)=>!i.isGlobal&&i.domain).length});return{headers:e(this.requestHeaders),proxies:e(this.proxies),limiters:e(this.limiters),oxylabs:e(this.oxylabs)}}_addHeaders(e){if(!e||!e.enable)return;for(let t of e.httpHeaders){let{domain:i,isGlobal:a,headers:r}=t;if(!i&&!a)continue;if(r instanceof Headers){let s=Object.fromEntries(r.entries());if(Object.keys(s).length<1)continue;r=s}else if(!r||Object.keys(r).length<1)continue;this.requestHeaders.push({domain:i,isGlobal:a,headers:r})}}_addProxies(e){if(!e||!e.enable)return;for(let t of e.proxies){let{domain:i,isGlobal:a,proxy:r}=t;if(!i&&!a)continue;if(!r||Object.keys(r).length<1)continue;this.proxies.push({domain:i,isGlobal:a,proxy:r})}}_addLimiters(e){if(!e||!e.enable)return;for(let t of e.limiters){let{domain:i,isGlobal:a,options:r,retry:s}=t;if(!i&&!a)continue;if(!r&&!s)continue;let n=r?new m(r):new m({name:"limiter"});if(this.limiters.push({domain:i,isGlobal:a,pqueue:n,randomDelay:r?.randomDelay,retry:s}),this.onLimiterAdded)this.onLimiterAdded(n)}}_addOxylabs(e){if(!e||!e.enable)return;for(let t of e.labs){let{domain:i,isGlobal:a,options:r,queueOptions:s}=t;if(!i&&!a)continue;if(!r||Object.keys(r).length<1)continue;this.oxylabs.push({domain:i,isGlobal:a,adaptar:new f(r)})}}_addDecodo(e){if(!e||!e.enable)return;for(let t of e.labs){let{domain:i,isGlobal:a,options:r,queueOptions:s}=t;if(!i&&!a)continue;if(!r||Object.keys(r).length<1)continue;this.decodo.push({domain:i,isGlobal:a,adaptar:new g(r)})}}_addStealths(e){if(!e||!e.enable)return;for(let t of e.profiles){let{domain:i,isGlobal:a,stealth:r}=t;if(!i&&!a)continue;if(!r)continue;this.stealthProfiles.push({domain:i,isGlobal:a,adaptar:new y({stealth:r})})}}addHeaders(e){return this._addHeaders({enable:!0,httpHeaders:[e]}),this}addProxy(e){return this._addProxies({enable:!0,proxies:[e]}),this}addLimiter(e){return this._addLimiters({enable:!0,limiters:[e]}),this}addOxylabs(e){return this._addOxylabs({enable:!0,labs:[e]}),this}addDecodo(e){return this._addDecodo({enable:!0,labs:[e]}),this}addStealth(e){return this._addStealths({enable:!0,profiles:[e]}),this}destroyLimiters(){for(let e of this.limiters)if(e.pqueue&&typeof e.pqueue.destroy==="function")e.pqueue.destroy();this.limiters=[]}getLimiters(){return this.limiters}getRandomDelay(e,t){if(!this.getDomainName(e))return;for(let a of this.limiters)if(this._hasDomain(e,a.domain)&&a.randomDelay!==void 0)return a.randomDelay;if(t){for(let a of this.limiters)if(a.isGlobal&&a.randomDelay!==void 0)return a.randomDelay}return}clearGlobalConfigs(){if(Array.isArray(this.requestHeaders))this.requestHeaders=this.requestHeaders.filter((e)=>!e.isGlobal);if(Array.isArray(this.oxylabs))this.oxylabs=this.oxylabs.filter((e)=>!e.isGlobal);if(Array.isArray(this.limiters))this.limiters=this.limiters.filter((e)=>!e.isGlobal);if(Array.isArray(this.proxies))this.proxies=this.proxies.filter((e)=>!e.isGlobal);if(Array.isArray(this.stealthProfiles))this.stealthProfiles=this.stealthProfiles.filter((e)=>!e.isGlobal);return this}getAdapter(e,t,i,a){if(!this.getDomainName(e))return null;let s=[],n=t==="headers"?this.requestHeaders:t==="limiters"?this.limiters:t==="oxylabs"?this.oxylabs:t==="decodo"?this.decodo:t==="stealth"?this.stealthProfiles:this.proxies;for(let o=0;o<n.length;o++)if(this._hasDomain(e,n[o].domain))s.push(o);if(s.length){let o=a?s[this.rnd(0,s.length-1)]:s[0];return t==="headers"?this.requestHeaders[o].headers:t==="limiters"?this.limiters[o].pqueue:t==="oxylabs"?this.oxylabs[o].adaptar:t==="decodo"?this.decodo[o].adaptar:t==="stealth"?this.stealthProfiles[o].adaptar:this.proxies[o].proxy}s.length=0;for(let o=0;o<n.length;o++)s.push(o);if(s.length){let o=a?s[this.rnd(0,s.length-1)]:s[0];if(n[o].isGlobal&&i)return t==="headers"?this.requestHeaders[o].headers:t==="limiters"?this.limiters[o].pqueue:t==="oxylabs"?this.oxylabs[o].adaptar:t==="decodo"?this.decodo[o].adaptar:t==="stealth"?this.stealthProfiles[o].adaptar:this.proxies[o].proxy}return null}getRetryOptions(e){if(!this.getDomainName(e))return null;for(let i=0;i<this.limiters.length;i++)if(this._hasDomain(e,this.limiters[i].domain))return this.limiters[i].retry||null;for(let i=0;i<this.limiters.length;i++)if(this.limiters[i].isGlobal&&this.limiters[i].retry)return this.limiters[i].retry;return null}rnd(e=0,t=Number.MAX_VALUE){return Math.floor(Math.random()*(t-e+1))+e}hasDomain(e,t,i){if(!this.getDomainName(e))return!1;let r=t==="headers"?this.requestHeaders:t==="limiters"?this.limiters:t==="oxylabs"?this.oxylabs:t==="decodo"?this.decodo:t==="stealth"?this.stealthProfiles:this.proxies;for(let s=0;s<r.length;s++)if(this._hasDomain(e,r[s].domain))return!0;if(i){for(let s=0;s<r.length;s++)if(r[s].isGlobal)return!0}return!1}pickHeaders(e,t,i,a){let r=this.getAdapter(e,"headers",t),s=new Headers(r??{}),n=s.count;if(i&&i instanceof Headers)for(let[o,l]of Object.entries(i.entries()))s.set(o,l);else if(i&&typeof i==="object"){for(let[o,l]of Object.entries(i))if(typeof l==="string")s.set(o,l)}if(a&&n===0&&!this.stealth)s.set("user-agent",this.getRandomUserAgent());return Object.fromEntries(s.entries())}_hasDomain(e,t){if(!t)return!1;let i=this.getDomainName(e);if(!i)return!1;let a=(s)=>{return/[\^\$\*\+\?\{\}\[\]\(\)\|\\]/.test(s)||s.startsWith("/")||s.includes(".*")||s.includes(".+")},r=(s)=>{if(s instanceof RegExp)return s.test(i)||s.test(e);let n=s.toString().trim();if(i.toLowerCase()===n.toLowerCase())return!0;if(n.includes("*")){let h=n.replace(/[.*+?^${}()|[\]\\]/g,"\\$&").replace(/\\\*/g,".*"),d=new RegExp(`^${h}$`,"i");return d.test(i)||d.test(e)}if(a(n))try{let h=n,d="i",u=n.match(/^\/(.*)\/(\w*)$/);if(u)h=u[1],d=u[2]||"i";let c=new RegExp(h,d);return c.test(i)||c.test(e)}catch(h){return i.toLowerCase().includes(n.toLowerCase())}let o=i.toLowerCase(),l=n.toLowerCase();return o===l||o.endsWith("."+l)||l.endsWith("."+o)};if(Array.isArray(t)){for(let s of t)if(r(s))return!0;return!1}return r(t)}getDomainName(e){if(this.isValidUrl(e))return new URL(e).hostname;else if(this.isHostName(e))return e;return null}isHostName(e){if(!e)return!1;if(e.length>255)return!1;let t=/^(?:(?:[a-zA-Z0-9](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?\.)+ [a-zA-Z]{2,})$/;return e=e.trim().toLowerCase(),t.test(e)&&!e.startsWith("-")&&!e.endsWith("-")}isValidUrl(e){if(!e)return!1;e=e.trim();try{let t=new URL(e);if(!t.protocol||!["http:","https:"].includes(t.protocol.toLowerCase()))return!1;if(!t.hostname)return!1;if(!/^(?:(?:[a-zA-Z0-9](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?\.)+[a-zA-Z]{2,})$/.test(t.hostname))return!1;return!0}catch{return!1}}getRandomUserAgent(){return this.userAgents[Math.floor(Math.random()*this.userAgents.length)]}}function D(){let e=[{name:"Chrome",version:"91.0.4472.124",engine:"AppleWebKit/537.36"},{name:"Firefox",version:"89.0",engine:"Gecko/20100101"},{name:"Safari",version:"14.1.1",engine:"AppleWebKit/605.1.15"},{name:"Edge",version:"91.0.864.59",engine:"AppleWebKit/537.36"},{name:"Opera",version:"77.0.4054.277",engine:"AppleWebKit/537.36"},{name:"Vivaldi",version:"3.8.2259.42",engine:"AppleWebKit/537.36"},{name:"Brave",version:"1.26.74",engine:"AppleWebKit/537.36"},{name:"Chromium",version:"91.0.4472.101",engine:"AppleWebKit/537.36"},{name:"Yandex",version:"21.5.3.742",engine:"AppleWebKit/537.36"},{name:"Maxthon",version:"5.3.8.2000",engine:"AppleWebKit/537.36"}],t=["Windows NT 10.0","Windows NT 6.1","Macintosh; Intel Mac OS X 10_15_7","Macintosh; Intel Mac OS X 11_4_0","X11; Linux x86_64","X11; Ubuntu; Linux x86_64"],i=[];for(let a=0;a<200;a++){let r=e[Math.floor(Math.random()*e.length)],s=t[Math.floor(Math.random()*t.length)],n="";switch(r.name){case"Chrome":n=`Mozilla/5.0 (${s}) ${r.engine} (KHTML, like Gecko) Chrome/${r.version} Safari/537.36`;break;case"Firefox":n=`Mozilla/5.0 (${s}; rv:${r.version}) ${r.engine} Firefox/${r.version}`;break;case"Safari":n=`Mozilla/5.0 (${s}) ${r.engine} (KHTML, like Gecko) Version/${r.version} Safari/605.1.15`;break;case"Edge":n=`Mozilla/5.0 (${s}) ${r.engine} (KHTML, like Gecko) Chrome/${r.version} Safari/537.36 Edg/${r.version}`;break;case"Opera":n=`Mozilla/5.0 (${s}) ${r.engine} (KHTML, like Gecko) Chrome/${r.version} Safari/537.36 OPR/${r.version}`;break;case"Vivaldi":n=`Mozilla/5.0 (${s}) ${r.engine} (KHTML, like Gecko) Chrome/${r.version} Safari/537.36 Vivaldi/${r.version}`;break;case"Brave":n=`Mozilla/5.0 (${s}) ${r.engine} (KHTML, like Gecko) Chrome/${r.version} Safari/537.36 Brave/${r.version}`;break;case"Chromium":n=`Mozilla/5.0 (${s}) ${r.engine} (KHTML, like Gecko) Chromium/${r.version} Chrome/${r.version} Safari/537.36`;break;case"Yandex":n=`Mozilla/5.0 (${s}) ${r.engine} (KHTML, like Gecko) Chrome/${r.version} YaBrowser/${r.version} Safari/537.36`;break;case"Maxthon":n=`Mozilla/5.0 (${s}) ${r.engine} (KHTML, like Gecko) Chrome/${r.version} Safari/537.36 Maxthon/${r.version}`;break}i.push(n)}return i}export{A as CrawlerOptions};
+import{RezoQueue as y}from"../queue/queue.js";import{Oxylabs as x}from"./addon/oxylabs/index.js";import p from"node:path";import v from"node:os";import{Decodo as A}from"./addon/decodo/index.js";import{Rezo as D}from"../core/rezo.js";class l{static destroyQueue(e){if(e&&typeof e.destroy==="function")e.destroy()}static destroyConfigQueues(e){for(let r of e)l.destroyQueue(r.pqueue)}static splitConfigs(e,r){let i=[],a=[];for(let t of e)if(r(t))a.push(t);else i.push(t);return{kept:i,removed:a}}static getHostname(e){try{return new URL(e).hostname||void 0}catch{return}}static createStableThroughputOptions(e){let{baseUrl:r,concurrency:i=40,scraperConcurrency:a=10,retryDelay:t=1000,maxRetryAttempts:s=2,retryOnStatusCode:o=[408,500,502,503,504],maxWaitOn429:d=15000,alwaysWaitOn429:n=!1,globalLimiter:h={concurrency:8},domain:u=l.getHostname(r),domainLimiter:c={concurrency:2,interval:1000,intervalCap:2,randomDelay:150},domainRetry:m={enable:!0,max429Retries:2,retryDelay:1000,maxRetryAttempts:2,backoff:!0},extraLimiters:g=[],overrides:b={}}=e,f=[];if(h)f.push({isGlobal:!0,options:h});if(u&&c)f.push({domain:u,options:c,retry:m||void 0});return f.push(...g),{...{baseUrl:r,autoThrottle:!1,concurrency:i,scraperConcurrency:a,retryDelay:t,maxRetryAttempts:s,retryOnStatusCode:o,maxWaitOn429:d,alwaysWaitOn429:n,limiter:f.length>0?{enable:!0,limiters:f}:!1},...b,baseUrl:r}}baseUrl;adapter;enableNavigationHistory;sessionId;rejectUnauthorized;userAgent;useRndUserAgent;timeout;maxRedirects;maxRetryAttempts;retryDelay;retryOnStatusCode;forceRevisit;retryWithoutProxyOnStatusCode;retryOnProxyError;maxRetryOnProxyError;allowRevisiting;enableCache;cacheTTL;cacheDir;throwFatalError;debug;enableSignalHandlers;concurrency;scraperConcurrency;maxDepth;maxUrls;maxResponseSize;respectRobotsTxt;followNofollow;autoThrottle;autoThrottleTargetDelay;autoThrottleMinDelay;autoThrottleMaxDelay;maxWaitOn429;alwaysWaitOn429;stealth;oxylabs=[];decodo=[];stealthProfiles=[];proxies=[];limiters=[];onLimiterAdded;onProviderAdded;requestHeaders=[];userAgents=$();constructor(e={}){this.baseUrl=e.baseUrl||"",this.adapter=e.adapter??"http",this.enableNavigationHistory=e.enableNavigationHistory??!1,this.sessionId=e.sessionId??`session_${Date.now()}_${Math.random().toString(36).slice(2,8)}`,this.rejectUnauthorized=e.rejectUnauthorized??!0,this.userAgent=e.userAgent,this.useRndUserAgent=e.useRndUserAgent??!1,this.timeout=e.timeout??30000,this.maxRedirects=e.maxRedirects??10,this.maxRetryAttempts=e.maxRetryAttempts??3,this.retryDelay=e.retryDelay??0,this.retryOnStatusCode=e.retryOnStatusCode??[408,429,500,502,503,504],this.forceRevisit=e.forceRevisit??!1,this.retryWithoutProxyOnStatusCode=e.retryWithoutProxyOnStatusCode??[407,403],this.retryOnProxyError=e.retryOnProxyError??!0,this.maxRetryOnProxyError=e.maxRetryOnProxyError??3,this.allowRevisiting=e.allowRevisiting??!1,this.enableCache=e.enableCache??!0,this.cacheTTL=e.cacheTTL??604800000,this.cacheDir=e.cacheDir??p.join(v.tmpdir(),"rezo_cache"),this.throwFatalError=e.throwFatalError??!1,this.debug=e.debug??!1,this.enableSignalHandlers=e.enableSignalHandlers??!1,this.concurrency=e.concurrency??100,this.scraperConcurrency=e.scraperConcurrency??this.concurrency,this.maxDepth=e.maxDepth??0,this.maxUrls=e.maxUrls??0,this.maxResponseSize=e.maxResponseSize??0,this.respectRobotsTxt=e.respectRobotsTxt??!1,this.followNofollow=e.followNofollow??!1,this.autoThrottle=e.autoThrottle??!0,this.autoThrottleTargetDelay=e.autoThrottleTargetDelay??1000,this.autoThrottleMinDelay=e.autoThrottleMinDelay??100,this.autoThrottleMaxDelay=e.autoThrottleMaxDelay??60000,this.maxWaitOn429=e.maxWaitOn429??1800000,this.alwaysWaitOn429=e.alwaysWaitOn429??!1,this._addHeaders(e.headers),this._addOxylabs(e.oxylabs),this._addDecodo(e.decodo),this._addProxies(e.proxy),this._addLimiters(e.limiter),this._addStealths(e.stealths),this.stealth=e.stealth}getConfiguredDomains(e){return(e==="headers"?this.requestHeaders:e==="limiters"?this.limiters:e==="oxylabs"?this.oxylabs:e==="decodo"?this.decodo:e==="stealth"?this.stealthProfiles:this.proxies).filter((i)=>i.domain).map((i)=>i.domain).filter((i,a,t)=>t.indexOf(i)===a)}removeDomain(e){this.requestHeaders=this.requestHeaders.filter((t)=>!t.domain||!this._domainsEqual(t.domain,e)),this.proxies=this.proxies.filter((t)=>!t.domain||!this._domainsEqual(t.domain,e));let r=l.splitConfigs(this.limiters,(t)=>!!t.domain&&this._domainsEqual(t.domain,e));l.destroyConfigQueues(r.removed),this.limiters=r.kept;let i=l.splitConfigs(this.oxylabs,(t)=>!!t.domain&&this._domainsEqual(t.domain,e));l.destroyConfigQueues(i.removed),this.oxylabs=i.kept;let a=l.splitConfigs(this.decodo,(t)=>!!t.domain&&this._domainsEqual(t.domain,e));return l.destroyConfigQueues(a.removed),this.decodo=a.kept,this.stealthProfiles=this.stealthProfiles.filter((t)=>!t.domain||!this._domainsEqual(t.domain,e)),this}_domainsEqual(e,r){if(Array.isArray(e)&&Array.isArray(r))return e.length===r.length&&e.every((i,a)=>i===r[a]);return e===r}getConfigurationSummary(){let e=(r)=>({total:r.length,global:r.filter((i)=>i.isGlobal).length,domainSpecific:r.filter((i)=>!i.isGlobal&&i.domain).length});return{headers:e(this.requestHeaders),proxies:e(this.proxies),limiters:e(this.limiters),oxylabs:e(this.oxylabs),decodo:e(this.decodo)}}_addHeaders(e){if(!e||!e.enable)return;for(let r of e.httpHeaders){let{domain:i,isGlobal:a,headers:t}=r;if(!i&&!a)continue;if(t instanceof Headers){let s=Object.fromEntries(t.entries());if(Object.keys(s).length<1)continue;t=s}else if(!t||Object.keys(t).length<1)continue;this.requestHeaders.push({domain:i,isGlobal:a,headers:t})}}_addProxies(e){if(!e||!e.enable)return;for(let r of e.proxies){let{domain:i,isGlobal:a,proxy:t,rotating:s}=r;if(!i&&!a)continue;if(!t||Object.keys(t).length<1)continue;this.proxies.push({domain:i,isGlobal:a,proxy:t,rotating:!!s})}}_addLimiters(e){if(!e||!e.enable)return;for(let r of e.limiters){let{domain:i,isGlobal:a,options:t,retry:s}=r;if(!i&&!a)continue;if(!t&&!s)continue;let o=t?new y(t):new y({name:"limiter"});if(this.limiters.push({domain:i,isGlobal:a,pqueue:o,randomDelay:t?.randomDelay,retry:s}),this.onLimiterAdded)this.onLimiterAdded(o)}}_addOxylabs(e){if(!e||!e.enable)return;for(let r of e.labs){let{domain:i,isGlobal:a,options:t,queueOptions:s}=r;if(!i&&!a)continue;if(!t||Object.keys(t).length<1)continue;let o=s?new y(s):void 0;if(this.oxylabs.push({domain:i,isGlobal:a,adaptar:new x(t),pqueue:o}),o&&this.onProviderAdded)this.onProviderAdded(o)}}_addDecodo(e){if(!e||!e.enable)return;for(let r of e.labs){let{domain:i,isGlobal:a,options:t,queueOptions:s}=r;if(!i&&!a)continue;if(!t||Object.keys(t).length<1)continue;let o=s?new y(s):void 0;if(this.decodo.push({domain:i,isGlobal:a,adaptar:new A(t),pqueue:o}),o&&this.onProviderAdded)this.onProviderAdded(o)}}_addStealths(e){if(!e||!e.enable)return;for(let r of e.profiles){let{domain:i,isGlobal:a,stealth:t}=r;if(!i&&!a)continue;if(!t)continue;this.stealthProfiles.push({domain:i,isGlobal:a,adaptar:new D({stealth:t})})}}addHeaders(e){return this._addHeaders({enable:!0,httpHeaders:[e]}),this}addProxy(e){return this._addProxies({enable:!0,proxies:[e]}),this}addProxies(e){return this._addProxies({enable:!0,proxies:e}),this}getProxyConfig(e,r,i){let a=this.selectConfigEntry(e,this.proxies,r,i);if(!a)return null;return{proxy:a.proxy,rotating:!!a.rotating}}addLimiter(e){return this._addLimiters({enable:!0,limiters:[e]}),this}addOxylabs(e){return this._addOxylabs({enable:!0,labs:[e]}),this}addDecodo(e){return this._addDecodo({enable:!0,labs:[e]}),this}addStealth(e){return this._addStealths({enable:!0,profiles:[e]}),this}destroyLimiters(){for(let e of this.limiters)l.destroyQueue(e.pqueue);this.limiters=[]}destroyProviderQueues(){for(let e of this.oxylabs)l.destroyQueue(e.pqueue),delete e.pqueue;for(let e of this.decodo)l.destroyQueue(e.pqueue),delete e.pqueue}getLimiters(){return this.limiters}getRandomDelay(e,r){if(!this.getDomainName(e))return;for(let a of this.limiters)if(this._hasDomain(e,a.domain)&&a.randomDelay!==void 0)return a.randomDelay;if(r){for(let a of this.limiters)if(a.isGlobal&&a.randomDelay!==void 0)return a.randomDelay}return}clearGlobalConfigs(){if(Array.isArray(this.requestHeaders))this.requestHeaders=this.requestHeaders.filter((e)=>!e.isGlobal);if(Array.isArray(this.oxylabs)){let e=l.splitConfigs(this.oxylabs,(r)=>!!r.isGlobal);l.destroyConfigQueues(e.removed),this.oxylabs=e.kept}if(Array.isArray(this.decodo)){let e=l.splitConfigs(this.decodo,(r)=>!!r.isGlobal);l.destroyConfigQueues(e.removed),this.decodo=e.kept}if(Array.isArray(this.limiters)){let e=l.splitConfigs(this.limiters,(r)=>!!r.isGlobal);l.destroyConfigQueues(e.removed),this.limiters=e.kept}if(Array.isArray(this.proxies))this.proxies=this.proxies.filter((e)=>!e.isGlobal);if(Array.isArray(this.stealthProfiles))this.stealthProfiles=this.stealthProfiles.filter((e)=>!e.isGlobal);return this}getAdapter(e,r,i,a){if(!this.getDomainName(e))return null;let s=r==="headers"?this.requestHeaders:r==="limiters"?this.limiters:r==="oxylabs"?this.oxylabs:r==="decodo"?this.decodo:r==="stealth"?this.stealthProfiles:this.proxies,o=this.selectConfigEntry(e,s,i,a);if(!o)return null;return r==="headers"?o.headers:r==="limiters"?o.pqueue:r==="oxylabs"?o.adaptar:r==="decodo"?o.adaptar:r==="stealth"?o.adaptar:o.proxy}getRetryOptions(e){if(!this.getDomainName(e))return null;for(let i=0;i<this.limiters.length;i++)if(this._hasDomain(e,this.limiters[i].domain))return this.limiters[i].retry||null;for(let i=0;i<this.limiters.length;i++)if(this.limiters[i].isGlobal&&this.limiters[i].retry)return this.limiters[i].retry;return null}rnd(e=0,r=Number.MAX_VALUE){return Math.floor(Math.random()*(r-e+1))+e}hasDomain(e,r,i){if(!this.getDomainName(e))return!1;let t=r==="headers"?this.requestHeaders:r==="limiters"?this.limiters:r==="oxylabs"?this.oxylabs:r==="decodo"?this.decodo:r==="stealth"?this.stealthProfiles:this.proxies;for(let s=0;s<t.length;s++)if(this._hasDomain(e,t[s].domain))return!0;if(i){for(let s=0;s<t.length;s++)if(t[s].isGlobal)return!0}return!1}pickHeaders(e,r,i,a){let t=this.getAdapter(e,"headers",r),s=new Headers(t??{}),o=s.count;if(i&&i instanceof Headers)for(let[d,n]of Object.entries(i.entries()))s.set(d,n);else if(i&&typeof i==="object"){for(let[d,n]of Object.entries(i))if(typeof n==="string")s.set(d,n)}if(a&&o===0&&!this.stealth)s.set("user-agent",this.getRandomUserAgent());return Object.fromEntries(s.entries())}_hasDomain(e,r){if(!r)return!1;let i=this.getDomainName(e);if(!i)return!1;let a=(s)=>{return/[\^\$\*\+\?\{\}\[\]\(\)\|\\]/.test(s)||s.startsWith("/")||s.includes(".*")||s.includes(".+")},t=(s)=>{if(s instanceof RegExp)return s.test(i)||s.test(e);let o=s.toString().trim();if(i.toLowerCase()===o.toLowerCase())return!0;if(o.includes("*")){let h=o.replace(/[.*+?^${}()|[\]\\]/g,"\\$&").replace(/\\\*/g,".*"),u=new RegExp(`^${h}$`,"i");return u.test(i)||u.test(e)}if(a(o))try{let h=o,u="i",c=o.match(/^\/(.*)\/(\w*)$/);if(c)h=c[1],u=c[2]||"i";let m=new RegExp(h,u);return m.test(i)||m.test(e)}catch(h){return i.toLowerCase().includes(o.toLowerCase())}let d=i.toLowerCase(),n=o.toLowerCase();return d===n||d.endsWith("."+n)||n.endsWith("."+d)};if(Array.isArray(r)){for(let s of r)if(t(s))return!0;return!1}return t(r)}selectConfigEntry(e,r,i,a){if(!this.getDomainName(e))return null;let s=(n)=>{if(!n.length)return null;let h=a?this.rnd(0,n.length-1):0;return n[h]},o=[];for(let n of r)if(this._hasDomain(e,n.domain))o.push(n);let d=s(o);if(d)return d;if(i){let n=[];for(let h of r)if(h.isGlobal)n.push(h);return s(n)}return null}getDomainName(e){if(this.isValidUrl(e))return new URL(e).hostname;else if(this.isHostName(e))return e;return null}isHostName(e){if(!e)return!1;if(e.length>255)return!1;let r=/^(?:(?:[a-zA-Z0-9](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?\.)+ [a-zA-Z]{2,})$/;return e=e.trim().toLowerCase(),r.test(e)&&!e.startsWith("-")&&!e.endsWith("-")}isValidUrl(e){if(!e)return!1;e=e.trim();try{let r=new URL(e);if(!r.protocol||!["http:","https:"].includes(r.protocol.toLowerCase()))return!1;if(!r.hostname)return!1;if(!/^(?:(?:[a-zA-Z0-9](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?\.)+[a-zA-Z]{2,})$/.test(r.hostname))return!1;return!0}catch{return!1}}getRandomUserAgent(){return this.userAgents[Math.floor(Math.random()*this.userAgents.length)]}}function $(){let e=[{name:"Chrome",version:"91.0.4472.124",engine:"AppleWebKit/537.36"},{name:"Firefox",version:"89.0",engine:"Gecko/20100101"},{name:"Safari",version:"14.1.1",engine:"AppleWebKit/605.1.15"},{name:"Edge",version:"91.0.864.59",engine:"AppleWebKit/537.36"},{name:"Opera",version:"77.0.4054.277",engine:"AppleWebKit/537.36"},{name:"Vivaldi",version:"3.8.2259.42",engine:"AppleWebKit/537.36"},{name:"Brave",version:"1.26.74",engine:"AppleWebKit/537.36"},{name:"Chromium",version:"91.0.4472.101",engine:"AppleWebKit/537.36"},{name:"Yandex",version:"21.5.3.742",engine:"AppleWebKit/537.36"},{name:"Maxthon",version:"5.3.8.2000",engine:"AppleWebKit/537.36"}],r=["Windows NT 10.0","Windows NT 6.1","Macintosh; Intel Mac OS X 10_15_7","Macintosh; Intel Mac OS X 11_4_0","X11; Linux x86_64","X11; Ubuntu; Linux x86_64"],i=[];for(let a=0;a<200;a++){let t=e[Math.floor(Math.random()*e.length)],s=r[Math.floor(Math.random()*r.length)],o="";switch(t.name){case"Chrome":o=`Mozilla/5.0 (${s}) ${t.engine} (KHTML, like Gecko) Chrome/${t.version} Safari/537.36`;break;case"Firefox":o=`Mozilla/5.0 (${s}; rv:${t.version}) ${t.engine} Firefox/${t.version}`;break;case"Safari":o=`Mozilla/5.0 (${s}) ${t.engine} (KHTML, like Gecko) Version/${t.version} Safari/605.1.15`;break;case"Edge":o=`Mozilla/5.0 (${s}) ${t.engine} (KHTML, like Gecko) Chrome/${t.version} Safari/537.36 Edg/${t.version}`;break;case"Opera":o=`Mozilla/5.0 (${s}) ${t.engine} (KHTML, like Gecko) Chrome/${t.version} Safari/537.36 OPR/${t.version}`;break;case"Vivaldi":o=`Mozilla/5.0 (${s}) ${t.engine} (KHTML, like Gecko) Chrome/${t.version} Safari/537.36 Vivaldi/${t.version}`;break;case"Brave":o=`Mozilla/5.0 (${s}) ${t.engine} (KHTML, like Gecko) Chrome/${t.version} Safari/537.36 Brave/${t.version}`;break;case"Chromium":o=`Mozilla/5.0 (${s}) ${t.engine} (KHTML, like Gecko) Chromium/${t.version} Chrome/${t.version} Safari/537.36`;break;case"Yandex":o=`Mozilla/5.0 (${s}) ${t.engine} (KHTML, like Gecko) Chrome/${t.version} YaBrowser/${t.version} Safari/537.36`;break;case"Maxthon":o=`Mozilla/5.0 (${s}) ${t.engine} (KHTML, like Gecko) Chrome/${t.version} Safari/537.36 Maxthon/${t.version}`;break}i.push(o)}return i}export{l as CrawlerOptions};