rezo 1.0.102 → 1.0.103

package/dist/adapters/entries/curl.d.ts

@@ -5514,7 +5514,7 @@ export interface RezoInstance extends Rezo, RezoCallable {
  *
  * IMPORTANT: Update these values when bumping package version.
  */
-export declare const VERSION = "1.0.102";
+export declare const VERSION = "1.0.103";
 /**
  * cURL Options Configuration
  *

package/dist/adapters/entries/fetch.d.ts

@@ -5514,7 +5514,7 @@ export interface RezoInstance extends Rezo, RezoCallable {
  *
  * IMPORTANT: Update these values when bumping package version.
  */
-export declare const VERSION = "1.0.102";
+export declare const VERSION = "1.0.103";
 export declare const isRezoError: typeof RezoError.isRezoError;
 export declare const Cancel: typeof RezoError;
 export declare const CancelToken: {

package/dist/adapters/entries/http.d.ts

@@ -2664,7 +2664,7 @@ export interface ResolvedStealthProfile {
 	/** Navigator properties for JS environment emulation */
 	navigator: BrowserProfile["navigator"];
 }
-declare class RezoStealth {
+export declare class RezoStealth {
 	private readonly _input;
 	private _resolved;
 	/** True when constructed with no args — profile will be detected from request headers */
@@ -5514,7 +5514,7 @@ export interface RezoInstance extends Rezo, RezoCallable {
  *
  * IMPORTANT: Update these values when bumping package version.
  */
-export declare const VERSION = "1.0.102";
+export declare const VERSION = "1.0.103";
 /**
  * Type guard to check if an error is a RezoError instance.
  */

package/dist/adapters/entries/http2.d.ts

@@ -2664,7 +2664,7 @@ export interface ResolvedStealthProfile {
 	/** Navigator properties for JS environment emulation */
 	navigator: BrowserProfile["navigator"];
 }
-declare class RezoStealth {
+export declare class RezoStealth {
 	private readonly _input;
 	private _resolved;
 	/** True when constructed with no args — profile will be detected from request headers */
@@ -5514,7 +5514,7 @@ export interface RezoInstance extends Rezo, RezoCallable {
  *
  * IMPORTANT: Update these values when bumping package version.
  */
-export declare const VERSION = "1.0.102";
+export declare const VERSION = "1.0.103";
 export declare const isRezoError: typeof RezoError.isRezoError;
 export declare const Cancel: typeof RezoError;
 export declare const CancelToken: {

package/dist/adapters/entries/react-native.d.ts

@@ -5514,7 +5514,7 @@ export interface RezoInstance extends Rezo, RezoCallable {
  *
  * IMPORTANT: Update these values when bumping package version.
  */
-export declare const VERSION = "1.0.102";
+export declare const VERSION = "1.0.103";
 export declare const isRezoError: typeof RezoError.isRezoError;
 export declare const Cancel: typeof RezoError;
 export declare const CancelToken: {

package/dist/adapters/entries/xhr.d.ts

@@ -5514,7 +5514,7 @@ export interface RezoInstance extends Rezo, RezoCallable {
  *
  * IMPORTANT: Update these values when bumping package version.
  */
-export declare const VERSION = "1.0.102";
+export declare const VERSION = "1.0.103";
 export declare const isRezoError: typeof RezoError.isRezoError;
 export declare const Cancel: typeof RezoError;
 export declare const CancelToken: {

package/dist/adapters/index.cjs

@@ -1,6 +1,6 @@
-const _mod_60sg3x = require('./picker.cjs');
-exports.detectRuntime = _mod_60sg3x.detectRuntime;
-exports.getAdapterCapabilities = _mod_60sg3x.getAdapterCapabilities;
-exports.buildAdapterContext = _mod_60sg3x.buildAdapterContext;
-exports.getAvailableAdapters = _mod_60sg3x.getAvailableAdapters;
-exports.selectAdapter = _mod_60sg3x.selectAdapter;;
+const _mod_1q3b5r = require('./picker.cjs');
+exports.detectRuntime = _mod_1q3b5r.detectRuntime;
+exports.getAdapterCapabilities = _mod_1q3b5r.getAdapterCapabilities;
+exports.buildAdapterContext = _mod_1q3b5r.buildAdapterContext;
+exports.getAvailableAdapters = _mod_1q3b5r.getAvailableAdapters;
+exports.selectAdapter = _mod_1q3b5r.selectAdapter;;

package/dist/cache/index.cjs

@@ -1,9 +1,9 @@
-const _mod_0ts4qf = require('./lru-cache.cjs');
-exports.LRUCache = _mod_0ts4qf.LRUCache;;
-const _mod_q9app0 = require('./dns-cache.cjs');
-exports.DNSCache = _mod_q9app0.DNSCache;
-exports.getGlobalDNSCache = _mod_q9app0.getGlobalDNSCache;
-exports.resetGlobalDNSCache = _mod_q9app0.resetGlobalDNSCache;;
-const _mod_pntq5r = require('./response-cache.cjs');
-exports.ResponseCache = _mod_pntq5r.ResponseCache;
-exports.normalizeResponseCacheConfig = _mod_pntq5r.normalizeResponseCacheConfig;;
+const _mod_qju1ca = require('./lru-cache.cjs');
+exports.LRUCache = _mod_qju1ca.LRUCache;;
+const _mod_w5a8hm = require('./dns-cache.cjs');
+exports.DNSCache = _mod_w5a8hm.DNSCache;
+exports.getGlobalDNSCache = _mod_w5a8hm.getGlobalDNSCache;
+exports.resetGlobalDNSCache = _mod_w5a8hm.resetGlobalDNSCache;;
+const _mod_jl8e34 = require('./response-cache.cjs');
+exports.ResponseCache = _mod_jl8e34.ResponseCache;
+exports.normalizeResponseCacheConfig = _mod_jl8e34.normalizeResponseCacheConfig;;

package/dist/cookies/cookie-jar.cjs

@@ -1,7 +1,7 @@
 const { CookieJar: TouchCookieJar } = require("tough-cookie");
 const { Cookie } = require('./cookie.cjs');
-const _mod_ocaxe6 = require('./cookie.cjs');
-exports.Cookie = _mod_ocaxe6.Cookie;;
+const _mod_e2fve9 = require('./cookie.cjs');
+exports.Cookie = _mod_e2fve9.Cookie;;
 
 class RezoCookieJar extends TouchCookieJar {
   constructor(store, options) {
@@ -469,7 +469,7 @@ class RezoCookieJar extends TouchCookieJar {
   }
 }
 const CookieJar = exports.CookieJar = RezoCookieJar;
-const _mod_4k9e34 = require("tough-cookie");
-exports.Store = _mod_4k9e34.Store;;
+const _mod_fmpdbk = require("tough-cookie");
+exports.Store = _mod_fmpdbk.Store;;
 
 exports.RezoCookieJar = RezoCookieJar;

package/dist/cookies/index.cjs

@@ -1,12 +1,12 @@
-const _mod_tomuvf = require('./cookie.cjs');
-exports.Cookie = _mod_tomuvf.Cookie;
-exports.RezoCookie = _mod_tomuvf.RezoCookie;;
-const _mod_o28nqg = require('./cookie-store.cjs');
-exports.RezoCookieStore = _mod_o28nqg.RezoCookieStore;;
-const _mod_xy2kyc = require('./file-store.cjs');
-exports.RezoFileCookieStore = _mod_xy2kyc.RezoFileCookieStore;;
-const _mod_ig4rxs = require('./cookie-jar.cjs');
-exports.RezoCookieJar = _mod_ig4rxs.RezoCookieJar;
-exports.CookieJar = _mod_ig4rxs.CookieJar;;
-const _mod_gvq5ik = require("tough-cookie");
-exports.Store = _mod_gvq5ik.Store;;
+const _mod_pbjc3l = require('./cookie.cjs');
+exports.Cookie = _mod_pbjc3l.Cookie;
+exports.RezoCookie = _mod_pbjc3l.RezoCookie;;
+const _mod_9t7p5v = require('./cookie-store.cjs');
+exports.RezoCookieStore = _mod_9t7p5v.RezoCookieStore;;
+const _mod_7nhc7y = require('./file-store.cjs');
+exports.RezoFileCookieStore = _mod_7nhc7y.RezoFileCookieStore;;
+const _mod_atrck9 = require('./cookie-jar.cjs');
+exports.RezoCookieJar = _mod_atrck9.RezoCookieJar;
+exports.CookieJar = _mod_atrck9.CookieJar;;
+const _mod_fbhb47 = require("tough-cookie");
+exports.Store = _mod_fbhb47.Store;;

package/dist/crawler/crawler-options.cjs

@@ -1 +1 @@
-var{RezoQueue:m}=require("../queue/queue.cjs"),{Oxylabs:b}=require("./addon/oxylabs/index.cjs"),x=require("node:path"),g=require("node:os"),{Decodo:y}=require("./addon/decodo/index.cjs"),{Rezo:A}=require("../core/rezo.cjs");class f{baseUrl;adapter;enableNavigationHistory;sessionId;rejectUnauthorized;userAgent;useRndUserAgent;timeout;maxRedirects;maxRetryAttempts;retryDelay;retryOnStatusCode;forceRevisit;retryWithoutProxyOnStatusCode;retryOnProxyError;maxRetryOnProxyError;allowRevisiting;enableCache;cacheTTL;cacheDir;throwFatalError;debug;enableSignalHandlers;concurrency;scraperConcurrency;maxDepth;maxUrls;maxResponseSize;respectRobotsTxt;followNofollow;autoThrottle;autoThrottleTargetDelay;autoThrottleMinDelay;autoThrottleMaxDelay;maxWaitOn429;alwaysWaitOn429;stealth;oxylabs=[];decodo=[];stealthProfiles=[];proxies=[];limiters=[];onLimiterAdded;requestHeaders=[];userAgents=v();constructor(e={}){this.baseUrl=e.baseUrl||"",this.adapter=e.adapter??"http",this.enableNavigationHistory=e.enableNavigationHistory??!1,this.sessionId=e.sessionId??`session_${Date.now()}_${Math.random().toString(36).slice(2,8)}`,this.rejectUnauthorized=e.rejectUnauthorized??!0,this.userAgent=e.userAgent,this.useRndUserAgent=e.useRndUserAgent??!1,this.timeout=e.timeout??30000,this.maxRedirects=e.maxRedirects??10,this.maxRetryAttempts=e.maxRetryAttempts??3,this.retryDelay=e.retryDelay??0,this.retryOnStatusCode=e.retryOnStatusCode??[408,429,500,502,503,504],this.forceRevisit=e.forceRevisit??!1,this.retryWithoutProxyOnStatusCode=e.retryWithoutProxyOnStatusCode??[407,403],this.retryOnProxyError=e.retryOnProxyError??!0,this.maxRetryOnProxyError=e.maxRetryOnProxyError??3,this.allowRevisiting=e.allowRevisiting??!1,this.enableCache=e.enableCache??!0,this.cacheTTL=e.cacheTTL??604800000,this.cacheDir=e.cacheDir??x.join(g.tmpdir(),"uiniqhtt_cache"),this.throwFatalError=e.throwFatalError??!1,this.debug=e.debug??!1,this.enableSignalHandlers=e.enableSignalHandlers??!1,this.concurrency=e.concurrency??100,this.scraperConcurrency=e.scraperConcurrency??this.concurrency,this.maxDepth=e.maxDepth??0,this.maxUrls=e.maxUrls??0,this.maxResponseSize=e.maxResponseSize??0,this.respectRobotsTxt=e.respectRobotsTxt??!1,this.followNofollow=e.followNofollow??!1,this.autoThrottle=e.autoThrottle??!0,this.autoThrottleTargetDelay=e.autoThrottleTargetDelay??1000,this.autoThrottleMinDelay=e.autoThrottleMinDelay??100,this.autoThrottleMaxDelay=e.autoThrottleMaxDelay??60000,this.maxWaitOn429=e.maxWaitOn429??1800000,this.alwaysWaitOn429=e.alwaysWaitOn429??!1,this._addHeaders(e.headers),this._addOxylabs(e.oxylabs),this._addDecodo(e.decodo),this._addProxies(e.proxy),this._addLimiters(e.limiter),this._addStealths(e.stealths),this.stealth=e.stealth}getConfiguredDomains(e){return(e==="headers"?this.requestHeaders:e==="limiters"?this.limiters:e==="oxylabs"?this.oxylabs:e==="stealth"?this.stealthProfiles:this.proxies).filter((i)=>i.domain).map((i)=>i.domain).filter((i,a,r)=>r.indexOf(i)===a)}removeDomain(e){return this.requestHeaders=this.requestHeaders.filter((t)=>!t.domain||!this._domainsEqual(t.domain,e)),this.proxies=this.proxies.filter((t)=>!t.domain||!this._domainsEqual(t.domain,e)),this.limiters=this.limiters.filter((t)=>!t.domain||!this._domainsEqual(t.domain,e)),this.oxylabs=this.oxylabs.filter((t)=>!t.domain||!this._domainsEqual(t.domain,e)),this.stealthProfiles=this.stealthProfiles.filter((t)=>!t.domain||!this._domainsEqual(t.domain,e)),this}_domainsEqual(e,t){if(Array.isArray(e)&&Array.isArray(t))return e.length===t.length&&e.every((i,a)=>i===t[a]);return e===t}getConfigurationSummary(){let e=(t)=>({total:t.length,global:t.filter((i)=>i.isGlobal).length,domainSpecific:t.filter((i)=>!i.isGlobal&&i.domain).length});return{headers:e(this.requestHeaders),proxies:e(this.proxies),limiters:e(this.limiters),oxylabs:e(this.oxylabs)}}_addHeaders(e){if(!e||!e.enable)return;for(let t of e.httpHeaders){let{domain:i,isGlobal:a,headers:r}=t;if(!i&&!a)continue;if(r instanceof Headers){let s=Object.fromEntries(r.entries());if(Object.keys(s).length<1)continue;r=s}else if(!r||Object.keys(r).length<1)continue;this.requestHeaders.push({domain:i,isGlobal:a,headers:r})}}_addProxies(e){if(!e||!e.enable)return;for(let t of e.proxies){let{domain:i,isGlobal:a,proxy:r}=t;if(!i&&!a)continue;if(!r||Object.keys(r).length<1)continue;this.proxies.push({domain:i,isGlobal:a,proxy:r})}}_addLimiters(e){if(!e||!e.enable)return;for(let t of e.limiters){let{domain:i,isGlobal:a,options:r}=t;if(!i&&!a)continue;if(!r||Object.keys(r).length<1)continue;let s=new m(r);if(this.limiters.push({domain:i,isGlobal:a,pqueue:s,randomDelay:r.randomDelay}),this.onLimiterAdded)this.onLimiterAdded(s)}}_addOxylabs(e){if(!e||!e.enable)return;for(let t of e.labs){let{domain:i,isGlobal:a,options:r,queueOptions:s}=t;if(!i&&!a)continue;if(!r||Object.keys(r).length<1)continue;this.oxylabs.push({domain:i,isGlobal:a,adaptar:new b(r)})}}_addDecodo(e){if(!e||!e.enable)return;for(let t of e.labs){let{domain:i,isGlobal:a,options:r,queueOptions:s}=t;if(!i&&!a)continue;if(!r||Object.keys(r).length<1)continue;this.decodo.push({domain:i,isGlobal:a,adaptar:new y(r)})}}_addStealths(e){if(!e||!e.enable)return;for(let t of e.profiles){let{domain:i,isGlobal:a,stealth:r}=t;if(!i&&!a)continue;if(!r)continue;this.stealthProfiles.push({domain:i,isGlobal:a,adaptar:new A({stealth:r})})}}addHeaders(e){return this._addHeaders({enable:!0,httpHeaders:[e]}),this}addProxy(e){return this._addProxies({enable:!0,proxies:[e]}),this}addLimiter(e){return this._addLimiters({enable:!0,limiters:[e]}),this}addOxylabs(e){return this._addOxylabs({enable:!0,labs:[e]}),this}addDecodo(e){return this._addDecodo({enable:!0,labs:[e]}),this}addStealth(e){return this._addStealths({enable:!0,profiles:[e]}),this}destroyLimiters(){for(let e of this.limiters)if(e.pqueue&&typeof e.pqueue.destroy==="function")e.pqueue.destroy();this.limiters=[]}getLimiters(){return this.limiters}getRandomDelay(e,t){if(!this.getDomainName(e))return;for(let a of this.limiters)if(this._hasDomain(e,a.domain)&&a.randomDelay!==void 0)return a.randomDelay;if(t){for(let a of this.limiters)if(a.isGlobal&&a.randomDelay!==void 0)return a.randomDelay}return}clearGlobalConfigs(){if(Array.isArray(this.requestHeaders))this.requestHeaders=this.requestHeaders.filter((e)=>!e.isGlobal);if(Array.isArray(this.oxylabs))this.oxylabs=this.oxylabs.filter((e)=>!e.isGlobal);if(Array.isArray(this.limiters))this.limiters=this.limiters.filter((e)=>!e.isGlobal);if(Array.isArray(this.proxies))this.proxies=this.proxies.filter((e)=>!e.isGlobal);if(Array.isArray(this.stealthProfiles))this.stealthProfiles=this.stealthProfiles.filter((e)=>!e.isGlobal);return this}getAdapter(e,t,i,a){if(!this.getDomainName(e))return null;let s=[],o=t==="headers"?this.requestHeaders:t==="limiters"?this.limiters:t==="oxylabs"?this.oxylabs:t==="decodo"?this.decodo:t==="stealth"?this.stealthProfiles:this.proxies;for(let n=0;n<o.length;n++)if(this._hasDomain(e,o[n].domain))s.push(n);if(s.length){let n=a?s[this.rnd(0,s.length-1)]:s[0];return t==="headers"?this.requestHeaders[n].headers:t==="limiters"?this.limiters[n].pqueue:t==="oxylabs"?this.oxylabs[n].adaptar:t==="decodo"?this.decodo[n].adaptar:t==="stealth"?this.stealthProfiles[n].adaptar:this.proxies[n].proxy}s.length=0;for(let n=0;n<o.length;n++)s.push(n);if(s.length){let n=a?s[this.rnd(0,s.length-1)]:s[0];if(o[n].isGlobal&&i)return t==="headers"?this.requestHeaders[n].headers:t==="limiters"?this.limiters[n].pqueue:t==="oxylabs"?this.oxylabs[n].adaptar:t==="decodo"?this.decodo[n].adaptar:t==="stealth"?this.stealthProfiles[n].adaptar:this.proxies[n].proxy}return null}rnd(e=0,t=Number.MAX_VALUE){return Math.floor(Math.random()*(t-e+1))+e}hasDomain(e,t,i){if(!this.getDomainName(e))return!1;let r=t==="headers"?this.requestHeaders:t==="limiters"?this.limiters:t==="oxylabs"?this.oxylabs:t==="decodo"?this.decodo:t==="stealth"?this.stealthProfiles:this.proxies;for(let s=0;s<r.length;s++)if(this._hasDomain(e,r[s].domain))return!0;if(i){for(let s=0;s<r.length;s++)if(r[s].isGlobal)return!0}return!1}pickHeaders(e,t,i,a){let r=this.getAdapter(e,"headers",t),s=new Headers(r??{}),o=s.count;if(i&&i instanceof Headers)for(let[n,l]of Object.entries(i.entries()))s.set(n,l);else if(i&&typeof i==="object"){for(let[n,l]of Object.entries(i))if(typeof l==="string")s.set(n,l)}if(a&&o===0&&!this.stealth)s.set("user-agent",this.getRandomUserAgent());return Object.fromEntries(s.entries())}_hasDomain(e,t){if(!t)return!1;let i=this.getDomainName(e);if(!i)return!1;let a=(s)=>{return/[\^\$\*\+\?\{\}\[\]\(\)\|\\]/.test(s)||s.startsWith("/")||s.includes(".*")||s.includes(".+")},r=(s)=>{if(s instanceof RegExp)return s.test(i)||s.test(e);let o=s.toString().trim();if(i.toLowerCase()===o.toLowerCase())return!0;if(o.includes("*")){let h=o.replace(/[.*+?^${}()|[\]\\]/g,"\\$&").replace(/\\\*/g,".*"),d=new RegExp(`^${h}$`,"i");return d.test(i)||d.test(e)}if(a(o))try{let h=o,d="i",u=o.match(/^\/(.*)\/(\w*)$/);if(u)h=u[1],d=u[2]||"i";let c=new RegExp(h,d);return c.test(i)||c.test(e)}catch(h){return i.toLowerCase().includes(o.toLowerCase())}let n=i.toLowerCase(),l=o.toLowerCase();return n===l||n.endsWith("."+l)||l.endsWith("."+n)};if(Array.isArray(t)){for(let s of t)if(r(s))return!0;return!1}return r(t)}getDomainName(e){if(this.isValidUrl(e))return new URL(e).hostname;else if(this.isHostName(e))return e;return null}isHostName(e){if(!e)return!1;if(e.length>255)return!1;let t=/^(?:(?:[a-zA-Z0-9](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?\.)+ [a-zA-Z]{2,})$/;return e=e.trim().toLowerCase(),t.test(e)&&!e.startsWith("-")&&!e.endsWith("-")}isValidUrl(e){if(!e)return!1;e=e.trim();try{let t=new URL(e);if(!t.protocol||!["http:","https:"].includes(t.protocol.toLowerCase()))return!1;if(!t.hostname)return!1;if(!/^(?:(?:[a-zA-Z0-9](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?\.)+[a-zA-Z]{2,})$/.test(t.hostname))return!1;return!0}catch{return!1}}getRandomUserAgent(){return this.userAgents[Math.floor(Math.random()*this.userAgents.length)]}}function v(){let e=[{name:"Chrome",version:"91.0.4472.124",engine:"AppleWebKit/537.36"},{name:"Firefox",version:"89.0",engine:"Gecko/20100101"},{name:"Safari",version:"14.1.1",engine:"AppleWebKit/605.1.15"},{name:"Edge",version:"91.0.864.59",engine:"AppleWebKit/537.36"},{name:"Opera",version:"77.0.4054.277",engine:"AppleWebKit/537.36"},{name:"Vivaldi",version:"3.8.2259.42",engine:"AppleWebKit/537.36"},{name:"Brave",version:"1.26.74",engine:"AppleWebKit/537.36"},{name:"Chromium",version:"91.0.4472.101",engine:"AppleWebKit/537.36"},{name:"Yandex",version:"21.5.3.742",engine:"AppleWebKit/537.36"},{name:"Maxthon",version:"5.3.8.2000",engine:"AppleWebKit/537.36"}],t=["Windows NT 10.0","Windows NT 6.1","Macintosh; Intel Mac OS X 10_15_7","Macintosh; Intel Mac OS X 11_4_0","X11; Linux x86_64","X11; Ubuntu; Linux x86_64"],i=[];for(let a=0;a<200;a++){let r=e[Math.floor(Math.random()*e.length)],s=t[Math.floor(Math.random()*t.length)],o="";switch(r.name){case"Chrome":o=`Mozilla/5.0 (${s}) ${r.engine} (KHTML, like Gecko) Chrome/${r.version} Safari/537.36`;break;case"Firefox":o=`Mozilla/5.0 (${s}; rv:${r.version}) ${r.engine} Firefox/${r.version}`;break;case"Safari":o=`Mozilla/5.0 (${s}) ${r.engine} (KHTML, like Gecko) Version/${r.version} Safari/605.1.15`;break;case"Edge":o=`Mozilla/5.0 (${s}) ${r.engine} (KHTML, like Gecko) Chrome/${r.version} Safari/537.36 Edg/${r.version}`;break;case"Opera":o=`Mozilla/5.0 (${s}) ${r.engine} (KHTML, like Gecko) Chrome/${r.version} Safari/537.36 OPR/${r.version}`;break;case"Vivaldi":o=`Mozilla/5.0 (${s}) ${r.engine} (KHTML, like Gecko) Chrome/${r.version} Safari/537.36 Vivaldi/${r.version}`;break;case"Brave":o=`Mozilla/5.0 (${s}) ${r.engine} (KHTML, like Gecko) Chrome/${r.version} Safari/537.36 Brave/${r.version}`;break;case"Chromium":o=`Mozilla/5.0 (${s}) ${r.engine} (KHTML, like Gecko) Chromium/${r.version} Chrome/${r.version} Safari/537.36`;break;case"Yandex":o=`Mozilla/5.0 (${s}) ${r.engine} (KHTML, like Gecko) Chrome/${r.version} YaBrowser/${r.version} Safari/537.36`;break;case"Maxthon":o=`Mozilla/5.0 (${s}) ${r.engine} (KHTML, like Gecko) Chrome/${r.version} Safari/537.36 Maxthon/${r.version}`;break}i.push(o)}return i}exports.CrawlerOptions=f;
+var{RezoQueue:f}=require("../queue/queue.cjs"),{Oxylabs:b}=require("./addon/oxylabs/index.cjs"),g=require("node:path"),x=require("node:os"),{Decodo:y}=require("./addon/decodo/index.cjs"),{Rezo:A}=require("../core/rezo.cjs");class m{baseUrl;adapter;enableNavigationHistory;sessionId;rejectUnauthorized;userAgent;useRndUserAgent;timeout;maxRedirects;maxRetryAttempts;retryDelay;retryOnStatusCode;forceRevisit;retryWithoutProxyOnStatusCode;retryOnProxyError;maxRetryOnProxyError;allowRevisiting;enableCache;cacheTTL;cacheDir;throwFatalError;debug;enableSignalHandlers;concurrency;scraperConcurrency;maxDepth;maxUrls;maxResponseSize;respectRobotsTxt;followNofollow;autoThrottle;autoThrottleTargetDelay;autoThrottleMinDelay;autoThrottleMaxDelay;maxWaitOn429;alwaysWaitOn429;stealth;oxylabs=[];decodo=[];stealthProfiles=[];proxies=[];limiters=[];onLimiterAdded;requestHeaders=[];userAgents=D();constructor(e={}){this.baseUrl=e.baseUrl||"",this.adapter=e.adapter??"http",this.enableNavigationHistory=e.enableNavigationHistory??!1,this.sessionId=e.sessionId??`session_${Date.now()}_${Math.random().toString(36).slice(2,8)}`,this.rejectUnauthorized=e.rejectUnauthorized??!0,this.userAgent=e.userAgent,this.useRndUserAgent=e.useRndUserAgent??!1,this.timeout=e.timeout??30000,this.maxRedirects=e.maxRedirects??10,this.maxRetryAttempts=e.maxRetryAttempts??3,this.retryDelay=e.retryDelay??0,this.retryOnStatusCode=e.retryOnStatusCode??[408,429,500,502,503,504],this.forceRevisit=e.forceRevisit??!1,this.retryWithoutProxyOnStatusCode=e.retryWithoutProxyOnStatusCode??[407,403],this.retryOnProxyError=e.retryOnProxyError??!0,this.maxRetryOnProxyError=e.maxRetryOnProxyError??3,this.allowRevisiting=e.allowRevisiting??!1,this.enableCache=e.enableCache??!0,this.cacheTTL=e.cacheTTL??604800000,this.cacheDir=e.cacheDir??g.join(x.tmpdir(),"rezo_cache"),this.throwFatalError=e.throwFatalError??!1,this.debug=e.debug??!1,this.enableSignalHandlers=e.enableSignalHandlers??!1,this.concurrency=e.concurrency??100,this.scraperConcurrency=e.scraperConcurrency??this.concurrency,this.maxDepth=e.maxDepth??0,this.maxUrls=e.maxUrls??0,this.maxResponseSize=e.maxResponseSize??0,this.respectRobotsTxt=e.respectRobotsTxt??!1,this.followNofollow=e.followNofollow??!1,this.autoThrottle=e.autoThrottle??!0,this.autoThrottleTargetDelay=e.autoThrottleTargetDelay??1000,this.autoThrottleMinDelay=e.autoThrottleMinDelay??100,this.autoThrottleMaxDelay=e.autoThrottleMaxDelay??60000,this.maxWaitOn429=e.maxWaitOn429??1800000,this.alwaysWaitOn429=e.alwaysWaitOn429??!1,this._addHeaders(e.headers),this._addOxylabs(e.oxylabs),this._addDecodo(e.decodo),this._addProxies(e.proxy),this._addLimiters(e.limiter),this._addStealths(e.stealths),this.stealth=e.stealth}getConfiguredDomains(e){return(e==="headers"?this.requestHeaders:e==="limiters"?this.limiters:e==="oxylabs"?this.oxylabs:e==="stealth"?this.stealthProfiles:this.proxies).filter((i)=>i.domain).map((i)=>i.domain).filter((i,a,r)=>r.indexOf(i)===a)}removeDomain(e){return this.requestHeaders=this.requestHeaders.filter((t)=>!t.domain||!this._domainsEqual(t.domain,e)),this.proxies=this.proxies.filter((t)=>!t.domain||!this._domainsEqual(t.domain,e)),this.limiters=this.limiters.filter((t)=>!t.domain||!this._domainsEqual(t.domain,e)),this.oxylabs=this.oxylabs.filter((t)=>!t.domain||!this._domainsEqual(t.domain,e)),this.stealthProfiles=this.stealthProfiles.filter((t)=>!t.domain||!this._domainsEqual(t.domain,e)),this}_domainsEqual(e,t){if(Array.isArray(e)&&Array.isArray(t))return e.length===t.length&&e.every((i,a)=>i===t[a]);return e===t}getConfigurationSummary(){let e=(t)=>({total:t.length,global:t.filter((i)=>i.isGlobal).length,domainSpecific:t.filter((i)=>!i.isGlobal&&i.domain).length});return{headers:e(this.requestHeaders),proxies:e(this.proxies),limiters:e(this.limiters),oxylabs:e(this.oxylabs)}}_addHeaders(e){if(!e||!e.enable)return;for(let t of e.httpHeaders){let{domain:i,isGlobal:a,headers:r}=t;if(!i&&!a)continue;if(r instanceof Headers){let s=Object.fromEntries(r.entries());if(Object.keys(s).length<1)continue;r=s}else if(!r||Object.keys(r).length<1)continue;this.requestHeaders.push({domain:i,isGlobal:a,headers:r})}}_addProxies(e){if(!e||!e.enable)return;for(let t of e.proxies){let{domain:i,isGlobal:a,proxy:r}=t;if(!i&&!a)continue;if(!r||Object.keys(r).length<1)continue;this.proxies.push({domain:i,isGlobal:a,proxy:r})}}_addLimiters(e){if(!e||!e.enable)return;for(let t of e.limiters){let{domain:i,isGlobal:a,options:r,retry:s}=t;if(!i&&!a)continue;if(!r&&!s)continue;let n=r?new f(r):new f({name:"limiter"});if(this.limiters.push({domain:i,isGlobal:a,pqueue:n,randomDelay:r?.randomDelay,retry:s}),this.onLimiterAdded)this.onLimiterAdded(n)}}_addOxylabs(e){if(!e||!e.enable)return;for(let t of e.labs){let{domain:i,isGlobal:a,options:r,queueOptions:s}=t;if(!i&&!a)continue;if(!r||Object.keys(r).length<1)continue;this.oxylabs.push({domain:i,isGlobal:a,adaptar:new b(r)})}}_addDecodo(e){if(!e||!e.enable)return;for(let t of e.labs){let{domain:i,isGlobal:a,options:r,queueOptions:s}=t;if(!i&&!a)continue;if(!r||Object.keys(r).length<1)continue;this.decodo.push({domain:i,isGlobal:a,adaptar:new y(r)})}}_addStealths(e){if(!e||!e.enable)return;for(let t of e.profiles){let{domain:i,isGlobal:a,stealth:r}=t;if(!i&&!a)continue;if(!r)continue;this.stealthProfiles.push({domain:i,isGlobal:a,adaptar:new A({stealth:r})})}}addHeaders(e){return this._addHeaders({enable:!0,httpHeaders:[e]}),this}addProxy(e){return this._addProxies({enable:!0,proxies:[e]}),this}addLimiter(e){return this._addLimiters({enable:!0,limiters:[e]}),this}addOxylabs(e){return this._addOxylabs({enable:!0,labs:[e]}),this}addDecodo(e){return this._addDecodo({enable:!0,labs:[e]}),this}addStealth(e){return this._addStealths({enable:!0,profiles:[e]}),this}destroyLimiters(){for(let e of this.limiters)if(e.pqueue&&typeof e.pqueue.destroy==="function")e.pqueue.destroy();this.limiters=[]}getLimiters(){return this.limiters}getRandomDelay(e,t){if(!this.getDomainName(e))return;for(let a of this.limiters)if(this._hasDomain(e,a.domain)&&a.randomDelay!==void 0)return a.randomDelay;if(t){for(let a of this.limiters)if(a.isGlobal&&a.randomDelay!==void 0)return a.randomDelay}return}clearGlobalConfigs(){if(Array.isArray(this.requestHeaders))this.requestHeaders=this.requestHeaders.filter((e)=>!e.isGlobal);if(Array.isArray(this.oxylabs))this.oxylabs=this.oxylabs.filter((e)=>!e.isGlobal);if(Array.isArray(this.limiters))this.limiters=this.limiters.filter((e)=>!e.isGlobal);if(Array.isArray(this.proxies))this.proxies=this.proxies.filter((e)=>!e.isGlobal);if(Array.isArray(this.stealthProfiles))this.stealthProfiles=this.stealthProfiles.filter((e)=>!e.isGlobal);return this}getAdapter(e,t,i,a){if(!this.getDomainName(e))return null;let s=[],n=t==="headers"?this.requestHeaders:t==="limiters"?this.limiters:t==="oxylabs"?this.oxylabs:t==="decodo"?this.decodo:t==="stealth"?this.stealthProfiles:this.proxies;for(let o=0;o<n.length;o++)if(this._hasDomain(e,n[o].domain))s.push(o);if(s.length){let o=a?s[this.rnd(0,s.length-1)]:s[0];return t==="headers"?this.requestHeaders[o].headers:t==="limiters"?this.limiters[o].pqueue:t==="oxylabs"?this.oxylabs[o].adaptar:t==="decodo"?this.decodo[o].adaptar:t==="stealth"?this.stealthProfiles[o].adaptar:this.proxies[o].proxy}s.length=0;for(let o=0;o<n.length;o++)s.push(o);if(s.length){let o=a?s[this.rnd(0,s.length-1)]:s[0];if(n[o].isGlobal&&i)return t==="headers"?this.requestHeaders[o].headers:t==="limiters"?this.limiters[o].pqueue:t==="oxylabs"?this.oxylabs[o].adaptar:t==="decodo"?this.decodo[o].adaptar:t==="stealth"?this.stealthProfiles[o].adaptar:this.proxies[o].proxy}return null}getRetryOptions(e){if(!this.getDomainName(e))return null;for(let i=0;i<this.limiters.length;i++)if(this._hasDomain(e,this.limiters[i].domain))return this.limiters[i].retry||null;for(let i=0;i<this.limiters.length;i++)if(this.limiters[i].isGlobal&&this.limiters[i].retry)return this.limiters[i].retry;return null}rnd(e=0,t=Number.MAX_VALUE){return Math.floor(Math.random()*(t-e+1))+e}hasDomain(e,t,i){if(!this.getDomainName(e))return!1;let r=t==="headers"?this.requestHeaders:t==="limiters"?this.limiters:t==="oxylabs"?this.oxylabs:t==="decodo"?this.decodo:t==="stealth"?this.stealthProfiles:this.proxies;for(let s=0;s<r.length;s++)if(this._hasDomain(e,r[s].domain))return!0;if(i){for(let s=0;s<r.length;s++)if(r[s].isGlobal)return!0}return!1}pickHeaders(e,t,i,a){let r=this.getAdapter(e,"headers",t),s=new Headers(r??{}),n=s.count;if(i&&i instanceof Headers)for(let[o,l]of Object.entries(i.entries()))s.set(o,l);else if(i&&typeof i==="object"){for(let[o,l]of Object.entries(i))if(typeof l==="string")s.set(o,l)}if(a&&n===0&&!this.stealth)s.set("user-agent",this.getRandomUserAgent());return Object.fromEntries(s.entries())}_hasDomain(e,t){if(!t)return!1;let i=this.getDomainName(e);if(!i)return!1;let a=(s)=>{return/[\^\$\*\+\?\{\}\[\]\(\)\|\\]/.test(s)||s.startsWith("/")||s.includes(".*")||s.includes(".+")},r=(s)=>{if(s instanceof RegExp)return s.test(i)||s.test(e);let n=s.toString().trim();if(i.toLowerCase()===n.toLowerCase())return!0;if(n.includes("*")){let h=n.replace(/[.*+?^${}()|[\]\\]/g,"\\$&").replace(/\\\*/g,".*"),d=new RegExp(`^${h}$`,"i");return d.test(i)||d.test(e)}if(a(n))try{let h=n,d="i",u=n.match(/^\/(.*)\/(\w*)$/);if(u)h=u[1],d=u[2]||"i";let c=new RegExp(h,d);return c.test(i)||c.test(e)}catch(h){return i.toLowerCase().includes(n.toLowerCase())}let o=i.toLowerCase(),l=n.toLowerCase();return o===l||o.endsWith("."+l)||l.endsWith("."+o)};if(Array.isArray(t)){for(let s of t)if(r(s))return!0;return!1}return r(t)}getDomainName(e){if(this.isValidUrl(e))return new URL(e).hostname;else if(this.isHostName(e))return e;return null}isHostName(e){if(!e)return!1;if(e.length>255)return!1;let t=/^(?:(?:[a-zA-Z0-9](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?\.)+ [a-zA-Z]{2,})$/;return e=e.trim().toLowerCase(),t.test(e)&&!e.startsWith("-")&&!e.endsWith("-")}isValidUrl(e){if(!e)return!1;e=e.trim();try{let t=new URL(e);if(!t.protocol||!["http:","https:"].includes(t.protocol.toLowerCase()))return!1;if(!t.hostname)return!1;if(!/^(?:(?:[a-zA-Z0-9](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?\.)+[a-zA-Z]{2,})$/.test(t.hostname))return!1;return!0}catch{return!1}}getRandomUserAgent(){return this.userAgents[Math.floor(Math.random()*this.userAgents.length)]}}function D(){let e=[{name:"Chrome",version:"91.0.4472.124",engine:"AppleWebKit/537.36"},{name:"Firefox",version:"89.0",engine:"Gecko/20100101"},{name:"Safari",version:"14.1.1",engine:"AppleWebKit/605.1.15"},{name:"Edge",version:"91.0.864.59",engine:"AppleWebKit/537.36"},{name:"Opera",version:"77.0.4054.277",engine:"AppleWebKit/537.36"},{name:"Vivaldi",version:"3.8.2259.42",engine:"AppleWebKit/537.36"},{name:"Brave",version:"1.26.74",engine:"AppleWebKit/537.36"},{name:"Chromium",version:"91.0.4472.101",engine:"AppleWebKit/537.36"},{name:"Yandex",version:"21.5.3.742",engine:"AppleWebKit/537.36"},{name:"Maxthon",version:"5.3.8.2000",engine:"AppleWebKit/537.36"}],t=["Windows NT 10.0","Windows NT 6.1","Macintosh; Intel Mac OS X 10_15_7","Macintosh; Intel Mac OS X 11_4_0","X11; Linux x86_64","X11; Ubuntu; Linux x86_64"],i=[];for(let a=0;a<200;a++){let r=e[Math.floor(Math.random()*e.length)],s=t[Math.floor(Math.random()*t.length)],n="";switch(r.name){case"Chrome":n=`Mozilla/5.0 (${s}) ${r.engine} (KHTML, like Gecko) Chrome/${r.version} Safari/537.36`;break;case"Firefox":n=`Mozilla/5.0 (${s}; rv:${r.version}) ${r.engine} Firefox/${r.version}`;break;case"Safari":n=`Mozilla/5.0 (${s}) ${r.engine} (KHTML, like Gecko) Version/${r.version} Safari/605.1.15`;break;case"Edge":n=`Mozilla/5.0 (${s}) ${r.engine} (KHTML, like Gecko) Chrome/${r.version} Safari/537.36 Edg/${r.version}`;break;case"Opera":n=`Mozilla/5.0 (${s}) ${r.engine} (KHTML, like Gecko) Chrome/${r.version} Safari/537.36 OPR/${r.version}`;break;case"Vivaldi":n=`Mozilla/5.0 (${s}) ${r.engine} (KHTML, like Gecko) Chrome/${r.version} Safari/537.36 Vivaldi/${r.version}`;break;case"Brave":n=`Mozilla/5.0 (${s}) ${r.engine} (KHTML, like Gecko) Chrome/${r.version} Safari/537.36 Brave/${r.version}`;break;case"Chromium":n=`Mozilla/5.0 (${s}) ${r.engine} (KHTML, like Gecko) Chromium/${r.version} Chrome/${r.version} Safari/537.36`;break;case"Yandex":n=`Mozilla/5.0 (${s}) ${r.engine} (KHTML, like Gecko) Chrome/${r.version} YaBrowser/${r.version} Safari/537.36`;break;case"Maxthon":n=`Mozilla/5.0 (${s}) ${r.engine} (KHTML, like Gecko) Chrome/${r.version} Safari/537.36 Maxthon/${r.version}`;break}i.push(n)}return i}exports.CrawlerOptions=m;

package/dist/crawler/crawler-options.js

@@ -1 +1 @@
-import{RezoQueue as f}from"../queue/queue.js";import{Oxylabs as m}from"./addon/oxylabs/index.js";import b from"node:path";import x from"node:os";import{Decodo as g}from"./addon/decodo/index.js";import{Rezo as y}from"../core/rezo.js";class A{baseUrl;adapter;enableNavigationHistory;sessionId;rejectUnauthorized;userAgent;useRndUserAgent;timeout;maxRedirects;maxRetryAttempts;retryDelay;retryOnStatusCode;forceRevisit;retryWithoutProxyOnStatusCode;retryOnProxyError;maxRetryOnProxyError;allowRevisiting;enableCache;cacheTTL;cacheDir;throwFatalError;debug;enableSignalHandlers;concurrency;scraperConcurrency;maxDepth;maxUrls;maxResponseSize;respectRobotsTxt;followNofollow;autoThrottle;autoThrottleTargetDelay;autoThrottleMinDelay;autoThrottleMaxDelay;maxWaitOn429;alwaysWaitOn429;stealth;oxylabs=[];decodo=[];stealthProfiles=[];proxies=[];limiters=[];onLimiterAdded;requestHeaders=[];userAgents=v();constructor(e={}){this.baseUrl=e.baseUrl||"",this.adapter=e.adapter??"http",this.enableNavigationHistory=e.enableNavigationHistory??!1,this.sessionId=e.sessionId??`session_${Date.now()}_${Math.random().toString(36).slice(2,8)}`,this.rejectUnauthorized=e.rejectUnauthorized??!0,this.userAgent=e.userAgent,this.useRndUserAgent=e.useRndUserAgent??!1,this.timeout=e.timeout??30000,this.maxRedirects=e.maxRedirects??10,this.maxRetryAttempts=e.maxRetryAttempts??3,this.retryDelay=e.retryDelay??0,this.retryOnStatusCode=e.retryOnStatusCode??[408,429,500,502,503,504],this.forceRevisit=e.forceRevisit??!1,this.retryWithoutProxyOnStatusCode=e.retryWithoutProxyOnStatusCode??[407,403],this.retryOnProxyError=e.retryOnProxyError??!0,this.maxRetryOnProxyError=e.maxRetryOnProxyError??3,this.allowRevisiting=e.allowRevisiting??!1,this.enableCache=e.enableCache??!0,this.cacheTTL=e.cacheTTL??604800000,this.cacheDir=e.cacheDir??b.join(x.tmpdir(),"uiniqhtt_cache"),this.throwFatalError=e.throwFatalError??!1,this.debug=e.debug??!1,this.enableSignalHandlers=e.enableSignalHandlers??!1,this.concurrency=e.concurrency??100,this.scraperConcurrency=e.scraperConcurrency??this.concurrency,this.maxDepth=e.maxDepth??0,this.maxUrls=e.maxUrls??0,this.maxResponseSize=e.maxResponseSize??0,this.respectRobotsTxt=e.respectRobotsTxt??!1,this.followNofollow=e.followNofollow??!1,this.autoThrottle=e.autoThrottle??!0,this.autoThrottleTargetDelay=e.autoThrottleTargetDelay??1000,this.autoThrottleMinDelay=e.autoThrottleMinDelay??100,this.autoThrottleMaxDelay=e.autoThrottleMaxDelay??60000,this.maxWaitOn429=e.maxWaitOn429??1800000,this.alwaysWaitOn429=e.alwaysWaitOn429??!1,this._addHeaders(e.headers),this._addOxylabs(e.oxylabs),this._addDecodo(e.decodo),this._addProxies(e.proxy),this._addLimiters(e.limiter),this._addStealths(e.stealths),this.stealth=e.stealth}getConfiguredDomains(e){return(e==="headers"?this.requestHeaders:e==="limiters"?this.limiters:e==="oxylabs"?this.oxylabs:e==="stealth"?this.stealthProfiles:this.proxies).filter((s)=>s.domain).map((s)=>s.domain).filter((s,a,r)=>r.indexOf(s)===a)}removeDomain(e){return this.requestHeaders=this.requestHeaders.filter((t)=>!t.domain||!this._domainsEqual(t.domain,e)),this.proxies=this.proxies.filter((t)=>!t.domain||!this._domainsEqual(t.domain,e)),this.limiters=this.limiters.filter((t)=>!t.domain||!this._domainsEqual(t.domain,e)),this.oxylabs=this.oxylabs.filter((t)=>!t.domain||!this._domainsEqual(t.domain,e)),this.stealthProfiles=this.stealthProfiles.filter((t)=>!t.domain||!this._domainsEqual(t.domain,e)),this}_domainsEqual(e,t){if(Array.isArray(e)&&Array.isArray(t))return e.length===t.length&&e.every((s,a)=>s===t[a]);return e===t}getConfigurationSummary(){let e=(t)=>({total:t.length,global:t.filter((s)=>s.isGlobal).length,domainSpecific:t.filter((s)=>!s.isGlobal&&s.domain).length});return{headers:e(this.requestHeaders),proxies:e(this.proxies),limiters:e(this.limiters),oxylabs:e(this.oxylabs)}}_addHeaders(e){if(!e||!e.enable)return;for(let t of e.httpHeaders){let{domain:s,isGlobal:a,headers:r}=t;if(!s&&!a)continue;if(r instanceof Headers){let i=Object.fromEntries(r.entries());if(Object.keys(i).length<1)continue;r=i}else if(!r||Object.keys(r).length<1)continue;this.requestHeaders.push({domain:s,isGlobal:a,headers:r})}}_addProxies(e){if(!e||!e.enable)return;for(let t of e.proxies){let{domain:s,isGlobal:a,proxy:r}=t;if(!s&&!a)continue;if(!r||Object.keys(r).length<1)continue;this.proxies.push({domain:s,isGlobal:a,proxy:r})}}_addLimiters(e){if(!e||!e.enable)return;for(let t of e.limiters){let{domain:s,isGlobal:a,options:r}=t;if(!s&&!a)continue;if(!r||Object.keys(r).length<1)continue;let i=new f(r);if(this.limiters.push({domain:s,isGlobal:a,pqueue:i,randomDelay:r.randomDelay}),this.onLimiterAdded)this.onLimiterAdded(i)}}_addOxylabs(e){if(!e||!e.enable)return;for(let t of e.labs){let{domain:s,isGlobal:a,options:r,queueOptions:i}=t;if(!s&&!a)continue;if(!r||Object.keys(r).length<1)continue;this.oxylabs.push({domain:s,isGlobal:a,adaptar:new m(r)})}}_addDecodo(e){if(!e||!e.enable)return;for(let t of e.labs){let{domain:s,isGlobal:a,options:r,queueOptions:i}=t;if(!s&&!a)continue;if(!r||Object.keys(r).length<1)continue;this.decodo.push({domain:s,isGlobal:a,adaptar:new g(r)})}}_addStealths(e){if(!e||!e.enable)return;for(let t of e.profiles){let{domain:s,isGlobal:a,stealth:r}=t;if(!s&&!a)continue;if(!r)continue;this.stealthProfiles.push({domain:s,isGlobal:a,adaptar:new y({stealth:r})})}}addHeaders(e){return this._addHeaders({enable:!0,httpHeaders:[e]}),this}addProxy(e){return this._addProxies({enable:!0,proxies:[e]}),this}addLimiter(e){return this._addLimiters({enable:!0,limiters:[e]}),this}addOxylabs(e){return this._addOxylabs({enable:!0,labs:[e]}),this}addDecodo(e){return this._addDecodo({enable:!0,labs:[e]}),this}addStealth(e){return this._addStealths({enable:!0,profiles:[e]}),this}destroyLimiters(){for(let e of this.limiters)if(e.pqueue&&typeof e.pqueue.destroy==="function")e.pqueue.destroy();this.limiters=[]}getLimiters(){return this.limiters}getRandomDelay(e,t){if(!this.getDomainName(e))return;for(let a of this.limiters)if(this._hasDomain(e,a.domain)&&a.randomDelay!==void 0)return a.randomDelay;if(t){for(let a of this.limiters)if(a.isGlobal&&a.randomDelay!==void 0)return a.randomDelay}return}clearGlobalConfigs(){if(Array.isArray(this.requestHeaders))this.requestHeaders=this.requestHeaders.filter((e)=>!e.isGlobal);if(Array.isArray(this.oxylabs))this.oxylabs=this.oxylabs.filter((e)=>!e.isGlobal);if(Array.isArray(this.limiters))this.limiters=this.limiters.filter((e)=>!e.isGlobal);if(Array.isArray(this.proxies))this.proxies=this.proxies.filter((e)=>!e.isGlobal);if(Array.isArray(this.stealthProfiles))this.stealthProfiles=this.stealthProfiles.filter((e)=>!e.isGlobal);return this}getAdapter(e,t,s,a){if(!this.getDomainName(e))return null;let i=[],n=t==="headers"?this.requestHeaders:t==="limiters"?this.limiters:t==="oxylabs"?this.oxylabs:t==="decodo"?this.decodo:t==="stealth"?this.stealthProfiles:this.proxies;for(let o=0;o<n.length;o++)if(this._hasDomain(e,n[o].domain))i.push(o);if(i.length){let o=a?i[this.rnd(0,i.length-1)]:i[0];return t==="headers"?this.requestHeaders[o].headers:t==="limiters"?this.limiters[o].pqueue:t==="oxylabs"?this.oxylabs[o].adaptar:t==="decodo"?this.decodo[o].adaptar:t==="stealth"?this.stealthProfiles[o].adaptar:this.proxies[o].proxy}i.length=0;for(let o=0;o<n.length;o++)i.push(o);if(i.length){let o=a?i[this.rnd(0,i.length-1)]:i[0];if(n[o].isGlobal&&s)return t==="headers"?this.requestHeaders[o].headers:t==="limiters"?this.limiters[o].pqueue:t==="oxylabs"?this.oxylabs[o].adaptar:t==="decodo"?this.decodo[o].adaptar:t==="stealth"?this.stealthProfiles[o].adaptar:this.proxies[o].proxy}return null}rnd(e=0,t=Number.MAX_VALUE){return Math.floor(Math.random()*(t-e+1))+e}hasDomain(e,t,s){if(!this.getDomainName(e))return!1;let r=t==="headers"?this.requestHeaders:t==="limiters"?this.limiters:t==="oxylabs"?this.oxylabs:t==="decodo"?this.decodo:t==="stealth"?this.stealthProfiles:this.proxies;for(let i=0;i<r.length;i++)if(this._hasDomain(e,r[i].domain))return!0;if(s){for(let i=0;i<r.length;i++)if(r[i].isGlobal)return!0}return!1}pickHeaders(e,t,s,a){let r=this.getAdapter(e,"headers",t),i=new Headers(r??{}),n=i.count;if(s&&s instanceof Headers)for(let[o,l]of Object.entries(s.entries()))i.set(o,l);else if(s&&typeof s==="object"){for(let[o,l]of Object.entries(s))if(typeof l==="string")i.set(o,l)}if(a&&n===0&&!this.stealth)i.set("user-agent",this.getRandomUserAgent());return Object.fromEntries(i.entries())}_hasDomain(e,t){if(!t)return!1;let s=this.getDomainName(e);if(!s)return!1;let a=(i)=>{return/[\^\$\*\+\?\{\}\[\]\(\)\|\\]/.test(i)||i.startsWith("/")||i.includes(".*")||i.includes(".+")},r=(i)=>{if(i instanceof RegExp)return i.test(s)||i.test(e);let n=i.toString().trim();if(s.toLowerCase()===n.toLowerCase())return!0;if(n.includes("*")){let h=n.replace(/[.*+?^${}()|[\]\\]/g,"\\$&").replace(/\\\*/g,".*"),d=new RegExp(`^${h}$`,"i");return d.test(s)||d.test(e)}if(a(n))try{let h=n,d="i",u=n.match(/^\/(.*)\/(\w*)$/);if(u)h=u[1],d=u[2]||"i";let c=new RegExp(h,d);return c.test(s)||c.test(e)}catch(h){return s.toLowerCase().includes(n.toLowerCase())}let o=s.toLowerCase(),l=n.toLowerCase();return o===l||o.endsWith("."+l)||l.endsWith("."+o)};if(Array.isArray(t)){for(let i of t)if(r(i))return!0;return!1}return r(t)}getDomainName(e){if(this.isValidUrl(e))return new URL(e).hostname;else if(this.isHostName(e))return e;return null}isHostName(e){if(!e)return!1;if(e.length>255)return!1;let t=/^(?:(?:[a-zA-Z0-9](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?\.)+ [a-zA-Z]{2,})$/;return e=e.trim().toLowerCase(),t.test(e)&&!e.startsWith("-")&&!e.endsWith("-")}isValidUrl(e){if(!e)return!1;e=e.trim();try{let t=new URL(e);if(!t.protocol||!["http:","https:"].includes(t.protocol.toLowerCase()))return!1;if(!t.hostname)return!1;if(!/^(?:(?:[a-zA-Z0-9](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?\.)+[a-zA-Z]{2,})$/.test(t.hostname))return!1;return!0}catch{return!1}}getRandomUserAgent(){return this.userAgents[Math.floor(Math.random()*this.userAgents.length)]}}function v(){let e=[{name:"Chrome",version:"91.0.4472.124",engine:"AppleWebKit/537.36"},{name:"Firefox",version:"89.0",engine:"Gecko/20100101"},{name:"Safari",version:"14.1.1",engine:"AppleWebKit/605.1.15"},{name:"Edge",version:"91.0.864.59",engine:"AppleWebKit/537.36"},{name:"Opera",version:"77.0.4054.277",engine:"AppleWebKit/537.36"},{name:"Vivaldi",version:"3.8.2259.42",engine:"AppleWebKit/537.36"},{name:"Brave",version:"1.26.74",engine:"AppleWebKit/537.36"},{name:"Chromium",version:"91.0.4472.101",engine:"AppleWebKit/537.36"},{name:"Yandex",version:"21.5.3.742",engine:"AppleWebKit/537.36"},{name:"Maxthon",version:"5.3.8.2000",engine:"AppleWebKit/537.36"}],t=["Windows NT 10.0","Windows NT 6.1","Macintosh; Intel Mac OS X 10_15_7","Macintosh; Intel Mac OS X 11_4_0","X11; Linux x86_64","X11; Ubuntu; Linux x86_64"],s=[];for(let a=0;a<200;a++){let r=e[Math.floor(Math.random()*e.length)],i=t[Math.floor(Math.random()*t.length)],n="";switch(r.name){case"Chrome":n=`Mozilla/5.0 (${i}) ${r.engine} (KHTML, like Gecko) Chrome/${r.version} Safari/537.36`;break;case"Firefox":n=`Mozilla/5.0 (${i}; rv:${r.version}) ${r.engine} Firefox/${r.version}`;break;case"Safari":n=`Mozilla/5.0 (${i}) ${r.engine} (KHTML, like Gecko) Version/${r.version} Safari/605.1.15`;break;case"Edge":n=`Mozilla/5.0 (${i}) ${r.engine} (KHTML, like Gecko) Chrome/${r.version} Safari/537.36 Edg/${r.version}`;break;case"Opera":n=`Mozilla/5.0 (${i}) ${r.engine} (KHTML, like Gecko) Chrome/${r.version} Safari/537.36 OPR/${r.version}`;break;case"Vivaldi":n=`Mozilla/5.0 (${i}) ${r.engine} (KHTML, like Gecko) Chrome/${r.version} Safari/537.36 Vivaldi/${r.version}`;break;case"Brave":n=`Mozilla/5.0 (${i}) ${r.engine} (KHTML, like Gecko) Chrome/${r.version} Safari/537.36 Brave/${r.version}`;break;case"Chromium":n=`Mozilla/5.0 (${i}) ${r.engine} (KHTML, like Gecko) Chromium/${r.version} Chrome/${r.version} Safari/537.36`;break;case"Yandex":n=`Mozilla/5.0 (${i}) ${r.engine} (KHTML, like Gecko) Chrome/${r.version} YaBrowser/${r.version} Safari/537.36`;break;case"Maxthon":n=`Mozilla/5.0 (${i}) ${r.engine} (KHTML, like Gecko) Chrome/${r.version} Safari/537.36 Maxthon/${r.version}`;break}s.push(n)}return s}export{A as CrawlerOptions};
+import{RezoQueue as m}from"../queue/queue.js";import{Oxylabs as f}from"./addon/oxylabs/index.js";import b from"node:path";import x from"node:os";import{Decodo as g}from"./addon/decodo/index.js";import{Rezo as y}from"../core/rezo.js";class A{baseUrl;adapter;enableNavigationHistory;sessionId;rejectUnauthorized;userAgent;useRndUserAgent;timeout;maxRedirects;maxRetryAttempts;retryDelay;retryOnStatusCode;forceRevisit;retryWithoutProxyOnStatusCode;retryOnProxyError;maxRetryOnProxyError;allowRevisiting;enableCache;cacheTTL;cacheDir;throwFatalError;debug;enableSignalHandlers;concurrency;scraperConcurrency;maxDepth;maxUrls;maxResponseSize;respectRobotsTxt;followNofollow;autoThrottle;autoThrottleTargetDelay;autoThrottleMinDelay;autoThrottleMaxDelay;maxWaitOn429;alwaysWaitOn429;stealth;oxylabs=[];decodo=[];stealthProfiles=[];proxies=[];limiters=[];onLimiterAdded;requestHeaders=[];userAgents=D();constructor(e={}){this.baseUrl=e.baseUrl||"",this.adapter=e.adapter??"http",this.enableNavigationHistory=e.enableNavigationHistory??!1,this.sessionId=e.sessionId??`session_${Date.now()}_${Math.random().toString(36).slice(2,8)}`,this.rejectUnauthorized=e.rejectUnauthorized??!0,this.userAgent=e.userAgent,this.useRndUserAgent=e.useRndUserAgent??!1,this.timeout=e.timeout??30000,this.maxRedirects=e.maxRedirects??10,this.maxRetryAttempts=e.maxRetryAttempts??3,this.retryDelay=e.retryDelay??0,this.retryOnStatusCode=e.retryOnStatusCode??[408,429,500,502,503,504],this.forceRevisit=e.forceRevisit??!1,this.retryWithoutProxyOnStatusCode=e.retryWithoutProxyOnStatusCode??[407,403],this.retryOnProxyError=e.retryOnProxyError??!0,this.maxRetryOnProxyError=e.maxRetryOnProxyError??3,this.allowRevisiting=e.allowRevisiting??!1,this.enableCache=e.enableCache??!0,this.cacheTTL=e.cacheTTL??604800000,this.cacheDir=e.cacheDir??b.join(x.tmpdir(),"rezo_cache"),this.throwFatalError=e.throwFatalError??!1,this.debug=e.debug??!1,this.enableSignalHandlers=e.enableSignalHandlers??!1,this.concurrency=e.concurrency??100,this.scraperConcurrency=e.scraperConcurrency??this.concurrency,this.maxDepth=e.maxDepth??0,this.maxUrls=e.maxUrls??0,this.maxResponseSize=e.maxResponseSize??0,this.respectRobotsTxt=e.respectRobotsTxt??!1,this.followNofollow=e.followNofollow??!1,this.autoThrottle=e.autoThrottle??!0,this.autoThrottleTargetDelay=e.autoThrottleTargetDelay??1000,this.autoThrottleMinDelay=e.autoThrottleMinDelay??100,this.autoThrottleMaxDelay=e.autoThrottleMaxDelay??60000,this.maxWaitOn429=e.maxWaitOn429??1800000,this.alwaysWaitOn429=e.alwaysWaitOn429??!1,this._addHeaders(e.headers),this._addOxylabs(e.oxylabs),this._addDecodo(e.decodo),this._addProxies(e.proxy),this._addLimiters(e.limiter),this._addStealths(e.stealths),this.stealth=e.stealth}getConfiguredDomains(e){return(e==="headers"?this.requestHeaders:e==="limiters"?this.limiters:e==="oxylabs"?this.oxylabs:e==="stealth"?this.stealthProfiles:this.proxies).filter((i)=>i.domain).map((i)=>i.domain).filter((i,a,r)=>r.indexOf(i)===a)}removeDomain(e){return this.requestHeaders=this.requestHeaders.filter((t)=>!t.domain||!this._domainsEqual(t.domain,e)),this.proxies=this.proxies.filter((t)=>!t.domain||!this._domainsEqual(t.domain,e)),this.limiters=this.limiters.filter((t)=>!t.domain||!this._domainsEqual(t.domain,e)),this.oxylabs=this.oxylabs.filter((t)=>!t.domain||!this._domainsEqual(t.domain,e)),this.stealthProfiles=this.stealthProfiles.filter((t)=>!t.domain||!this._domainsEqual(t.domain,e)),this}_domainsEqual(e,t){if(Array.isArray(e)&&Array.isArray(t))return e.length===t.length&&e.every((i,a)=>i===t[a]);return e===t}getConfigurationSummary(){let e=(t)=>({total:t.length,global:t.filter((i)=>i.isGlobal).length,domainSpecific:t.filter((i)=>!i.isGlobal&&i.domain).length});return{headers:e(this.requestHeaders),proxies:e(this.proxies),limiters:e(this.limiters),oxylabs:e(this.oxylabs)}}_addHeaders(e){if(!e||!e.enable)return;for(let t of e.httpHeaders){let{domain:i,isGlobal:a,headers:r}=t;if(!i&&!a)continue;if(r instanceof Headers){let s=Object.fromEntries(r.entries());if(Object.keys(s).length<1)continue;r=s}else if(!r||Object.keys(r).length<1)continue;this.requestHeaders.push({domain:i,isGlobal:a,headers:r})}}_addProxies(e){if(!e||!e.enable)return;for(let t of e.proxies){let{domain:i,isGlobal:a,proxy:r}=t;if(!i&&!a)continue;if(!r||Object.keys(r).length<1)continue;this.proxies.push({domain:i,isGlobal:a,proxy:r})}}_addLimiters(e){if(!e||!e.enable)return;for(let t of e.limiters){let{domain:i,isGlobal:a,options:r,retry:s}=t;if(!i&&!a)continue;if(!r&&!s)continue;let n=r?new m(r):new m({name:"limiter"});if(this.limiters.push({domain:i,isGlobal:a,pqueue:n,randomDelay:r?.randomDelay,retry:s}),this.onLimiterAdded)this.onLimiterAdded(n)}}_addOxylabs(e){if(!e||!e.enable)return;for(let t of e.labs){let{domain:i,isGlobal:a,options:r,queueOptions:s}=t;if(!i&&!a)continue;if(!r||Object.keys(r).length<1)continue;this.oxylabs.push({domain:i,isGlobal:a,adaptar:new f(r)})}}_addDecodo(e){if(!e||!e.enable)return;for(let t of e.labs){let{domain:i,isGlobal:a,options:r,queueOptions:s}=t;if(!i&&!a)continue;if(!r||Object.keys(r).length<1)continue;this.decodo.push({domain:i,isGlobal:a,adaptar:new g(r)})}}_addStealths(e){if(!e||!e.enable)return;for(let t of e.profiles){let{domain:i,isGlobal:a,stealth:r}=t;if(!i&&!a)continue;if(!r)continue;this.stealthProfiles.push({domain:i,isGlobal:a,adaptar:new y({stealth:r})})}}addHeaders(e){return this._addHeaders({enable:!0,httpHeaders:[e]}),this}addProxy(e){return this._addProxies({enable:!0,proxies:[e]}),this}addLimiter(e){return this._addLimiters({enable:!0,limiters:[e]}),this}addOxylabs(e){return this._addOxylabs({enable:!0,labs:[e]}),this}addDecodo(e){return this._addDecodo({enable:!0,labs:[e]}),this}addStealth(e){return this._addStealths({enable:!0,profiles:[e]}),this}destroyLimiters(){for(let e of this.limiters)if(e.pqueue&&typeof e.pqueue.destroy==="function")e.pqueue.destroy();this.limiters=[]}getLimiters(){return this.limiters}getRandomDelay(e,t){if(!this.getDomainName(e))return;for(let a of this.limiters)if(this._hasDomain(e,a.domain)&&a.randomDelay!==void 0)return a.randomDelay;if(t){for(let a of this.limiters)if(a.isGlobal&&a.randomDelay!==void 0)return a.randomDelay}return}clearGlobalConfigs(){if(Array.isArray(this.requestHeaders))this.requestHeaders=this.requestHeaders.filter((e)=>!e.isGlobal);if(Array.isArray(this.oxylabs))this.oxylabs=this.oxylabs.filter((e)=>!e.isGlobal);if(Array.isArray(this.limiters))this.limiters=this.limiters.filter((e)=>!e.isGlobal);if(Array.isArray(this.proxies))this.proxies=this.proxies.filter((e)=>!e.isGlobal);if(Array.isArray(this.stealthProfiles))this.stealthProfiles=this.stealthProfiles.filter((e)=>!e.isGlobal);return this}getAdapter(e,t,i,a){if(!this.getDomainName(e))return null;let s=[],n=t==="headers"?this.requestHeaders:t==="limiters"?this.limiters:t==="oxylabs"?this.oxylabs:t==="decodo"?this.decodo:t==="stealth"?this.stealthProfiles:this.proxies;for(let o=0;o<n.length;o++)if(this._hasDomain(e,n[o].domain))s.push(o);if(s.length){let o=a?s[this.rnd(0,s.length-1)]:s[0];return t==="headers"?this.requestHeaders[o].headers:t==="limiters"?this.limiters[o].pqueue:t==="oxylabs"?this.oxylabs[o].adaptar:t==="decodo"?this.decodo[o].adaptar:t==="stealth"?this.stealthProfiles[o].adaptar:this.proxies[o].proxy}s.length=0;for(let o=0;o<n.length;o++)s.push(o);if(s.length){let o=a?s[this.rnd(0,s.length-1)]:s[0];if(n[o].isGlobal&&i)return t==="headers"?this.requestHeaders[o].headers:t==="limiters"?this.limiters[o].pqueue:t==="oxylabs"?this.oxylabs[o].adaptar:t==="decodo"?this.decodo[o].adaptar:t==="stealth"?this.stealthProfiles[o].adaptar:this.proxies[o].proxy}return null}getRetryOptions(e){if(!this.getDomainName(e))return null;for(let i=0;i<this.limiters.length;i++)if(this._hasDomain(e,this.limiters[i].domain))return this.limiters[i].retry||null;for(let i=0;i<this.limiters.length;i++)if(this.limiters[i].isGlobal&&this.limiters[i].retry)return this.limiters[i].retry;return null}rnd(e=0,t=Number.MAX_VALUE){return Math.floor(Math.random()*(t-e+1))+e}hasDomain(e,t,i){if(!this.getDomainName(e))return!1;let r=t==="headers"?this.requestHeaders:t==="limiters"?this.limiters:t==="oxylabs"?this.oxylabs:t==="decodo"?this.decodo:t==="stealth"?this.stealthProfiles:this.proxies;for(let s=0;s<r.length;s++)if(this._hasDomain(e,r[s].domain))return!0;if(i){for(let s=0;s<r.length;s++)if(r[s].isGlobal)return!0}return!1}pickHeaders(e,t,i,a){let r=this.getAdapter(e,"headers",t),s=new Headers(r??{}),n=s.count;if(i&&i instanceof Headers)for(let[o,l]of Object.entries(i.entries()))s.set(o,l);else if(i&&typeof i==="object"){for(let[o,l]of Object.entries(i))if(typeof l==="string")s.set(o,l)}if(a&&n===0&&!this.stealth)s.set("user-agent",this.getRandomUserAgent());return Object.fromEntries(s.entries())}_hasDomain(e,t){if(!t)return!1;let i=this.getDomainName(e);if(!i)return!1;let a=(s)=>{return/[\^\$\*\+\?\{\}\[\]\(\)\|\\]/.test(s)||s.startsWith("/")||s.includes(".*")||s.includes(".+")},r=(s)=>{if(s instanceof RegExp)return s.test(i)||s.test(e);let n=s.toString().trim();if(i.toLowerCase()===n.toLowerCase())return!0;if(n.includes("*")){let h=n.replace(/[.*+?^${}()|[\]\\]/g,"\\$&").replace(/\\\*/g,".*"),d=new RegExp(`^${h}$`,"i");return d.test(i)||d.test(e)}if(a(n))try{let h=n,d="i",u=n.match(/^\/(.*)\/(\w*)$/);if(u)h=u[1],d=u[2]||"i";let c=new RegExp(h,d);return c.test(i)||c.test(e)}catch(h){return i.toLowerCase().includes(n.toLowerCase())}let o=i.toLowerCase(),l=n.toLowerCase();return o===l||o.endsWith("."+l)||l.endsWith("."+o)};if(Array.isArray(t)){for(let s of t)if(r(s))return!0;return!1}return r(t)}getDomainName(e){if(this.isValidUrl(e))return new URL(e).hostname;else if(this.isHostName(e))return e;return null}isHostName(e){if(!e)return!1;if(e.length>255)return!1;let t=/^(?:(?:[a-zA-Z0-9](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?\.)+ [a-zA-Z]{2,})$/;return e=e.trim().toLowerCase(),t.test(e)&&!e.startsWith("-")&&!e.endsWith("-")}isValidUrl(e){if(!e)return!1;e=e.trim();try{let t=new URL(e);if(!t.protocol||!["http:","https:"].includes(t.protocol.toLowerCase()))return!1;if(!t.hostname)return!1;if(!/^(?:(?:[a-zA-Z0-9](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?\.)+[a-zA-Z]{2,})$/.test(t.hostname))return!1;return!0}catch{return!1}}getRandomUserAgent(){return this.userAgents[Math.floor(Math.random()*this.userAgents.length)]}}function D(){let e=[{name:"Chrome",version:"91.0.4472.124",engine:"AppleWebKit/537.36"},{name:"Firefox",version:"89.0",engine:"Gecko/20100101"},{name:"Safari",version:"14.1.1",engine:"AppleWebKit/605.1.15"},{name:"Edge",version:"91.0.864.59",engine:"AppleWebKit/537.36"},{name:"Opera",version:"77.0.4054.277",engine:"AppleWebKit/537.36"},{name:"Vivaldi",version:"3.8.2259.42",engine:"AppleWebKit/537.36"},{name:"Brave",version:"1.26.74",engine:"AppleWebKit/537.36"},{name:"Chromium",version:"91.0.4472.101",engine:"AppleWebKit/537.36"},{name:"Yandex",version:"21.5.3.742",engine:"AppleWebKit/537.36"},{name:"Maxthon",version:"5.3.8.2000",engine:"AppleWebKit/537.36"}],t=["Windows NT 10.0","Windows NT 6.1","Macintosh; Intel Mac OS X 10_15_7","Macintosh; Intel Mac OS X 11_4_0","X11; Linux x86_64","X11; Ubuntu; Linux x86_64"],i=[];for(let a=0;a<200;a++){let r=e[Math.floor(Math.random()*e.length)],s=t[Math.floor(Math.random()*t.length)],n="";switch(r.name){case"Chrome":n=`Mozilla/5.0 (${s}) ${r.engine} (KHTML, like Gecko) Chrome/${r.version} Safari/537.36`;break;case"Firefox":n=`Mozilla/5.0 (${s}; rv:${r.version}) ${r.engine} Firefox/${r.version}`;break;case"Safari":n=`Mozilla/5.0 (${s}) ${r.engine} (KHTML, like Gecko) Version/${r.version} Safari/605.1.15`;break;case"Edge":n=`Mozilla/5.0 (${s}) ${r.engine} (KHTML, like Gecko) Chrome/${r.version} Safari/537.36 Edg/${r.version}`;break;case"Opera":n=`Mozilla/5.0 (${s}) ${r.engine} (KHTML, like Gecko) Chrome/${r.version} Safari/537.36 OPR/${r.version}`;break;case"Vivaldi":n=`Mozilla/5.0 (${s}) ${r.engine} (KHTML, like Gecko) Chrome/${r.version} Safari/537.36 Vivaldi/${r.version}`;break;case"Brave":n=`Mozilla/5.0 (${s}) ${r.engine} (KHTML, like Gecko) Chrome/${r.version} Safari/537.36 Brave/${r.version}`;break;case"Chromium":n=`Mozilla/5.0 (${s}) ${r.engine} (KHTML, like Gecko) Chromium/${r.version} Chrome/${r.version} Safari/537.36`;break;case"Yandex":n=`Mozilla/5.0 (${s}) ${r.engine} (KHTML, like Gecko) Chrome/${r.version} YaBrowser/${r.version} Safari/537.36`;break;case"Maxthon":n=`Mozilla/5.0 (${s}) ${r.engine} (KHTML, like Gecko) Chrome/${r.version} Safari/537.36 Maxthon/${r.version}`;break}i.push(n)}return i}export{A as CrawlerOptions};

package/dist/crawler/crawler.cjs

@@ -9,7 +9,7 @@ const { CappedMap } = require('./plugin/capped-map.cjs');
 const { CappedArray } = require('./plugin/capped-array.cjs');
 const { parseHTML } = require("linkedom");
 const path = require("node:path");
-const rezo = require('../adapters/entries/http.cjs');
+import rezo, { RezoHeaders } from '../index.cjs';
 const { RezoQueue } = require('../queue/queue.cjs');
 const { Scraper } = require('./scraper.cjs');
 const { CrawlerOptions } = require('./crawler-options.cjs');
@@ -83,6 +83,7 @@ class Crawler {
   healthMetrics;
   originalConcurrency = 100;
   shutdownHandler = null;
+  rateLimitedDomains = new Map;
   startHandlers = [];
   finishHandlers = [];
   redirectHandlers = [];
@@ -101,7 +102,9 @@ class Crawler {
     this.config = new CrawlerOptions(crawlerOptions);
     this.adapterType = this.config.adapter;
     if (this.config.stealth) {
-      this.http.defaults.stealth = this.config.stealth;
+      this.http = rezo.create({
+        stealth: this.config.stealth
+      });
     }
     const concurrency = this.config.concurrency;
     this.queue = new RezoQueue({
@@ -863,7 +866,17 @@ class Crawler {
       return 0;
     return this.domainCurrentDelay.get(domain) || this.config.autoThrottleMinDelay;
   }
-  async handle429Response(url, response) {
+  async handle429Response(url, response, retryCount) {
+    const retryOpts = this.config.getRetryOptions(url);
+    const max429Retries = retryOpts?.max429Retries ?? 3;
+    const baseDelay = retryOpts?.retryDelay ?? 1000;
+    const useBackoff = retryOpts?.backoff ?? true;
+    if (retryCount >= max429Retries) {
+      if (this.config.debug) {
+        console.log(`[Crawler] 429 retry cap reached (${max429Retries}) for ${url}, giving up`);
+      }
+      return { shouldRetry: false, waitTime: 0 };
+    }
     let retryAfter = 0;
     const retryAfterHeader = response?.headers?.["retry-after"] || response?.headers?.get?.("retry-after");
     if (retryAfterHeader) {
@@ -878,7 +891,7 @@ class Crawler {
       }
     }
     if (retryAfter <= 0) {
-      retryAfter = 60000;
+      retryAfter = useBackoff ? baseDelay * Math.pow(2, retryCount) : baseDelay;
     }
     const maxWait = this.config.maxWaitOn429;
     const alwaysWait = this.config.alwaysWaitOn429;
@@ -894,8 +907,12 @@ class Crawler {
       const waitMinutes = Math.round(retryAfter / 60000);
       console.warn(`[Crawler] WARNING: Rate limited on ${url}. Server requested ${waitMinutes} minute wait. Waiting because alwaysWaitOn429 is enabled.`);
     }
+    try {
+      const domain = new URL(url).hostname;
+      this.rateLimitedDomains.set(domain, Date.now() + retryAfter);
+    } catch {}
     if (this.config.debug) {
-      console.log(`[Crawler] 429 Rate Limited: waiting ${Math.round(retryAfter / 1000)}s before retry`);
+      console.log(`[Crawler] 429 Rate Limited: waiting ${Math.round(retryAfter / 1000)}s before retry (attempt ${retryCount + 1}/${max429Retries})`);
     }
     return { shouldRetry: true, waitTime: retryAfter };
   }
@@ -1208,6 +1225,17 @@ class Crawler {
       if (delay > 0) {
         await new Promise((resolve) => setTimeout(resolve, delay));
       }
+      const rateLimitUntil = this.rateLimitedDomains.get(domain);
+      if (rateLimitUntil) {
+        const remaining = rateLimitUntil - Date.now();
+        if (remaining > 0) {
+          if (this.config.debug) {
+            console.log(`[Crawler] Domain ${domain} rate-limited, waiting ${Math.round(remaining / 1000)}s`);
+          }
+          await new Promise((resolve) => setTimeout(resolve, remaining));
+        }
+        this.rateLimitedDomains.delete(domain);
+      }
       const isVisited = forceRevisit ? false : await this.hasUrlInCache(url);
       const cache = method.toLowerCase() === "get" ? await this.getCache(url) : undefined;
       if (method.toLowerCase() === "get") {
@@ -1218,6 +1246,8 @@ class Crawler {
       }
       const requestStartTime = Date.now();
       const http = stealthInstanse || this.http;
+      console.log(new RezoHeaders(options.headers));
+      console.log(http.defaults);
       const response = cache && method === "GET" && !skipCache ? cache : oxylabsInstanse && oxylabsOptions ? await oxylabsInstanse.scrape(url) : decodoInstanse && decodoOptions ? await decodoInstanse.scrape(url) : method === "GET" ? await http.get(url, options) : method === "PATCH" ? await http.patch(url, body, options) : method === "POST" ? await http.post(url, body, options) : await http.put(url, body, options);
       if (!response) {
         this.crawlStats.urlsFailed++;
@@ -1298,12 +1328,20 @@ class Crawler {
       }
     } catch (e) {
       const error = e;
+      console.log(error);
       if (error?.response?.status === 429 || error?.status === 429) {
         try {
-          const { shouldRetry, waitTime } = await this.handle429Response(url, error.response || error);
+          const { shouldRetry, waitTime } = await this.handle429Response(url, error.response || error, retryCount);
           if (shouldRetry) {
             await this.sleep(waitTime);
             return await this.executeHttp(method, url, body, options, isEmail, forceRevisit, oxylabsOptions, oxylabsInstanse, decodoInstanse, decodoOptions, retryCount + 1, parentUrl, skipCache, emailMetadata, stealthInstanse);
+          } else {
+            this.crawlStats.urlsFailed++;
+            for (let i = 0;i < this.errorEvents.length; i++) {
+              const event = this.errorEvents[i];
+              this[event.handler](...event.attr, error);
+            }
+            return;
           }
         } catch (rateLimitError) {
           this.crawlStats.urlsFailed++;

package/dist/crawler/crawler.js

@@ -9,7 +9,7 @@ import { CappedMap } from './plugin/capped-map.js';
 import { CappedArray } from './plugin/capped-array.js';
 import { parseHTML } from "linkedom";
 import path from "node:path";
-import rezo from '../adapters/entries/http.js';
+import rezo, { RezoHeaders } from '../index.js';
 import { RezoQueue } from '../queue/queue.js';
 import { Scraper } from './scraper.js';
 import { CrawlerOptions } from './crawler-options.js';
@@ -83,6 +83,7 @@ export class Crawler {
   healthMetrics;
   originalConcurrency = 100;
   shutdownHandler = null;
+  rateLimitedDomains = new Map;
   startHandlers = [];
   finishHandlers = [];
   redirectHandlers = [];
@@ -101,7 +102,9 @@ export class Crawler {
     this.config = new CrawlerOptions(crawlerOptions);
     this.adapterType = this.config.adapter;
     if (this.config.stealth) {
-      this.http.defaults.stealth = this.config.stealth;
+      this.http = rezo.create({
+        stealth: this.config.stealth
+      });
     }
     const concurrency = this.config.concurrency;
     this.queue = new RezoQueue({
@@ -863,7 +866,17 @@ export class Crawler {
       return 0;
     return this.domainCurrentDelay.get(domain) || this.config.autoThrottleMinDelay;
   }
-  async handle429Response(url, response) {
+  async handle429Response(url, response, retryCount) {
+    const retryOpts = this.config.getRetryOptions(url);
+    const max429Retries = retryOpts?.max429Retries ?? 3;
+    const baseDelay = retryOpts?.retryDelay ?? 1000;
+    const useBackoff = retryOpts?.backoff ?? true;
+    if (retryCount >= max429Retries) {
+      if (this.config.debug) {
+        console.log(`[Crawler] 429 retry cap reached (${max429Retries}) for ${url}, giving up`);
+      }
+      return { shouldRetry: false, waitTime: 0 };
+    }
     let retryAfter = 0;
     const retryAfterHeader = response?.headers?.["retry-after"] || response?.headers?.get?.("retry-after");
     if (retryAfterHeader) {
@@ -878,7 +891,7 @@ export class Crawler {
       }
     }
     if (retryAfter <= 0) {
-      retryAfter = 60000;
+      retryAfter = useBackoff ? baseDelay * Math.pow(2, retryCount) : baseDelay;
     }
     const maxWait = this.config.maxWaitOn429;
     const alwaysWait = this.config.alwaysWaitOn429;
@@ -894,8 +907,12 @@ export class Crawler {
       const waitMinutes = Math.round(retryAfter / 60000);
       console.warn(`[Crawler] WARNING: Rate limited on ${url}. Server requested ${waitMinutes} minute wait. Waiting because alwaysWaitOn429 is enabled.`);
     }
+    try {
+      const domain = new URL(url).hostname;
+      this.rateLimitedDomains.set(domain, Date.now() + retryAfter);
+    } catch {}
     if (this.config.debug) {
-      console.log(`[Crawler] 429 Rate Limited: waiting ${Math.round(retryAfter / 1000)}s before retry`);
+      console.log(`[Crawler] 429 Rate Limited: waiting ${Math.round(retryAfter / 1000)}s before retry (attempt ${retryCount + 1}/${max429Retries})`);
     }
     return { shouldRetry: true, waitTime: retryAfter };
   }
@@ -1208,6 +1225,17 @@ export class Crawler {
       if (delay > 0) {
         await new Promise((resolve) => setTimeout(resolve, delay));
       }
+      const rateLimitUntil = this.rateLimitedDomains.get(domain);
+      if (rateLimitUntil) {
+        const remaining = rateLimitUntil - Date.now();
+        if (remaining > 0) {
+          if (this.config.debug) {
+            console.log(`[Crawler] Domain ${domain} rate-limited, waiting ${Math.round(remaining / 1000)}s`);
+          }
+          await new Promise((resolve) => setTimeout(resolve, remaining));
+        }
+        this.rateLimitedDomains.delete(domain);
+      }
       const isVisited = forceRevisit ? false : await this.hasUrlInCache(url);
       const cache = method.toLowerCase() === "get" ? await this.getCache(url) : undefined;
       if (method.toLowerCase() === "get") {
@@ -1218,6 +1246,8 @@ export class Crawler {
       }
       const requestStartTime = Date.now();
       const http = stealthInstanse || this.http;
+      console.log(new RezoHeaders(options.headers));
+      console.log(http.defaults);
       const response = cache && method === "GET" && !skipCache ? cache : oxylabsInstanse && oxylabsOptions ? await oxylabsInstanse.scrape(url) : decodoInstanse && decodoOptions ? await decodoInstanse.scrape(url) : method === "GET" ? await http.get(url, options) : method === "PATCH" ? await http.patch(url, body, options) : method === "POST" ? await http.post(url, body, options) : await http.put(url, body, options);
       if (!response) {
         this.crawlStats.urlsFailed++;
@@ -1298,12 +1328,20 @@ export class Crawler {
       }
     } catch (e) {
       const error = e;
+      console.log(error);
       if (error?.response?.status === 429 || error?.status === 429) {
         try {
-          const { shouldRetry, waitTime } = await this.handle429Response(url, error.response || error);
+          const { shouldRetry, waitTime } = await this.handle429Response(url, error.response || error, retryCount);
           if (shouldRetry) {
             await this.sleep(waitTime);
             return await this.executeHttp(method, url, body, options, isEmail, forceRevisit, oxylabsOptions, oxylabsInstanse, decodoInstanse, decodoOptions, retryCount + 1, parentUrl, skipCache, emailMetadata, stealthInstanse);
+          } else {
+            this.crawlStats.urlsFailed++;
+            for (let i = 0;i < this.errorEvents.length; i++) {
+              const event = this.errorEvents[i];
+              this[event.handler](...event.attr, error);
+            }
+            return;
           }
         } catch (rateLimitError) {
           this.crawlStats.urlsFailed++;

package/dist/crawler/index.cjs

@@ -1,42 +1,42 @@
-const _mod_dbafqs = require('./crawler.cjs');
-exports.Crawler = _mod_dbafqs.Crawler;;
-const _mod_zojdm8 = require('./crawler-options.cjs');
-exports.CrawlerOptions = _mod_zojdm8.CrawlerOptions;;
-const _mod_uo1px3 = require('./plugin/robots-txt.cjs');
-exports.RobotsTxt = _mod_uo1px3.RobotsTxt;;
-const _mod_pmgd68 = require('./plugin/file-cacher.cjs');
-exports.FileCacher = _mod_pmgd68.FileCacher;;
-const _mod_el97fa = require('./plugin/url-store.cjs');
-exports.UrlStore = _mod_el97fa.UrlStore;;
-const _mod_1ka0w5 = require('./plugin/navigation-history.cjs');
-exports.NavigationHistory = _mod_1ka0w5.NavigationHistory;;
-const _mod_fadxm4 = require('./addon/oxylabs/index.cjs');
-exports.Oxylabs = _mod_fadxm4.Oxylabs;;
-const _mod_a0vzxm = require('./addon/oxylabs/options.cjs');
-exports.OXYLABS_BROWSER_TYPES = _mod_a0vzxm.OXYLABS_BROWSER_TYPES;
-exports.OXYLABS_COMMON_LOCALES = _mod_a0vzxm.OXYLABS_COMMON_LOCALES;
-exports.OXYLABS_COMMON_GEO_LOCATIONS = _mod_a0vzxm.OXYLABS_COMMON_GEO_LOCATIONS;
-exports.OXYLABS_US_STATES = _mod_a0vzxm.OXYLABS_US_STATES;
-exports.OXYLABS_EUROPEAN_COUNTRIES = _mod_a0vzxm.OXYLABS_EUROPEAN_COUNTRIES;
-exports.OXYLABS_ASIAN_COUNTRIES = _mod_a0vzxm.OXYLABS_ASIAN_COUNTRIES;
-exports.getRandomOxylabsBrowserType = _mod_a0vzxm.getRandomBrowserType;
-exports.getRandomOxylabsLocale = _mod_a0vzxm.getRandomLocale;
-exports.getRandomOxylabsGeoLocation = _mod_a0vzxm.getRandomGeoLocation;;
-const _mod_cz2alp = require('./scraper.cjs');
-exports.isRestrictedDomain = _mod_cz2alp.isRestrictedDomain;;
-const _mod_0qw67y = require('./addon/decodo/index.cjs');
-exports.Decodo = _mod_0qw67y.Decodo;;
-const _mod_9apcw0 = require('./addon/decodo/options.cjs');
-exports.DECODO_DEVICE_TYPES = _mod_9apcw0.DECODO_DEVICE_TYPES;
-exports.DECODO_HEADLESS_MODES = _mod_9apcw0.DECODO_HEADLESS_MODES;
-exports.DECODO_COMMON_LOCALES = _mod_9apcw0.DECODO_COMMON_LOCALES;
-exports.DECODO_COMMON_COUNTRIES = _mod_9apcw0.DECODO_COMMON_COUNTRIES;
-exports.DECODO_EUROPEAN_COUNTRIES = _mod_9apcw0.DECODO_EUROPEAN_COUNTRIES;
-exports.DECODO_ASIAN_COUNTRIES = _mod_9apcw0.DECODO_ASIAN_COUNTRIES;
-exports.DECODO_US_STATES = _mod_9apcw0.DECODO_US_STATES;
-exports.DECODO_COMMON_CITIES = _mod_9apcw0.DECODO_COMMON_CITIES;
-exports.getRandomDecodoDeviceType = _mod_9apcw0.getRandomDeviceType;
-exports.getRandomDecodoLocale = _mod_9apcw0.getRandomLocale;
-exports.getRandomDecodoCountry = _mod_9apcw0.getRandomCountry;
-exports.getRandomDecodoCity = _mod_9apcw0.getRandomCity;
-exports.generateDecodoSessionId = _mod_9apcw0.generateSessionId;;
+const _mod_8bpcyt = require('./crawler.cjs');
+exports.Crawler = _mod_8bpcyt.Crawler;;
+const _mod_988v6z = require('./crawler-options.cjs');
+exports.CrawlerOptions = _mod_988v6z.CrawlerOptions;;
+const _mod_4plonu = require('./plugin/robots-txt.cjs');
+exports.RobotsTxt = _mod_4plonu.RobotsTxt;;
+const _mod_ev7lys = require('./plugin/file-cacher.cjs');
+exports.FileCacher = _mod_ev7lys.FileCacher;;
+const _mod_jcjmld = require('./plugin/url-store.cjs');
+exports.UrlStore = _mod_jcjmld.UrlStore;;
+const _mod_s1b6ok = require('./plugin/navigation-history.cjs');
+exports.NavigationHistory = _mod_s1b6ok.NavigationHistory;;
+const _mod_ekix3v = require('./addon/oxylabs/index.cjs');
+exports.Oxylabs = _mod_ekix3v.Oxylabs;;
+const _mod_unv0ew = require('./addon/oxylabs/options.cjs');
+exports.OXYLABS_BROWSER_TYPES = _mod_unv0ew.OXYLABS_BROWSER_TYPES;
+exports.OXYLABS_COMMON_LOCALES = _mod_unv0ew.OXYLABS_COMMON_LOCALES;
+exports.OXYLABS_COMMON_GEO_LOCATIONS = _mod_unv0ew.OXYLABS_COMMON_GEO_LOCATIONS;
+exports.OXYLABS_US_STATES = _mod_unv0ew.OXYLABS_US_STATES;
+exports.OXYLABS_EUROPEAN_COUNTRIES = _mod_unv0ew.OXYLABS_EUROPEAN_COUNTRIES;
+exports.OXYLABS_ASIAN_COUNTRIES = _mod_unv0ew.OXYLABS_ASIAN_COUNTRIES;
+exports.getRandomOxylabsBrowserType = _mod_unv0ew.getRandomBrowserType;
+exports.getRandomOxylabsLocale = _mod_unv0ew.getRandomLocale;
+exports.getRandomOxylabsGeoLocation = _mod_unv0ew.getRandomGeoLocation;;
+const _mod_fcrp4b = require('./scraper.cjs');
+exports.isRestrictedDomain = _mod_fcrp4b.isRestrictedDomain;;
+const _mod_wte3po = require('./addon/decodo/index.cjs');
+exports.Decodo = _mod_wte3po.Decodo;;
+const _mod_mllxcw = require('./addon/decodo/options.cjs');
+exports.DECODO_DEVICE_TYPES = _mod_mllxcw.DECODO_DEVICE_TYPES;
+exports.DECODO_HEADLESS_MODES = _mod_mllxcw.DECODO_HEADLESS_MODES;
+exports.DECODO_COMMON_LOCALES = _mod_mllxcw.DECODO_COMMON_LOCALES;
+exports.DECODO_COMMON_COUNTRIES = _mod_mllxcw.DECODO_COMMON_COUNTRIES;
+exports.DECODO_EUROPEAN_COUNTRIES = _mod_mllxcw.DECODO_EUROPEAN_COUNTRIES;
+exports.DECODO_ASIAN_COUNTRIES = _mod_mllxcw.DECODO_ASIAN_COUNTRIES;
+exports.DECODO_US_STATES = _mod_mllxcw.DECODO_US_STATES;
+exports.DECODO_COMMON_CITIES = _mod_mllxcw.DECODO_COMMON_CITIES;
+exports.getRandomDecodoDeviceType = _mod_mllxcw.getRandomDeviceType;
+exports.getRandomDecodoLocale = _mod_mllxcw.getRandomLocale;
+exports.getRandomDecodoCountry = _mod_mllxcw.getRandomCountry;
+exports.getRandomDecodoCity = _mod_mllxcw.getRandomCity;
+exports.generateDecodoSessionId = _mod_mllxcw.generateSessionId;;

package/dist/crawler/plugin/index.cjs

@@ -1 +1 @@
-var e=require("./file-cacher.cjs");exports.FileCacher=e.FileCacher;var r=require("./url-store.cjs");exports.UrlStore=r.UrlStore;var o=require("./result-stream.cjs");exports.ResultStream=o.ResultStream;var t=require("./memory-monitor.cjs");exports.MemoryMonitor=t.MemoryMonitor;var a=require("./health-metrics.cjs");exports.HealthMetrics=a.HealthMetrics;var i=require("./capped-map.cjs");exports.CappedMap=i.CappedMap;var _=require("./capped-array.cjs");exports.CappedArray=_.CappedArray;
+var e=require("./file-cacher.cjs");exports.FileCacher=e.FileCacher;var r=require("./url-store.cjs");exports.UrlStore=r.UrlStore;var o=require("./result-stream.cjs");exports.ResultStream=o.ResultStream;var t=require("./memory-monitor.cjs");exports.MemoryMonitor=t.MemoryMonitor;var a=require("./health-metrics.cjs");exports.HealthMetrics=a.HealthMetrics;var i=require("./capped-map.cjs");exports.CappedMap=i.CappedMap;var m=require("./capped-array.cjs");exports.CappedArray=m.CappedArray;

package/dist/crawler.d.ts

@@ -7334,6 +7334,25 @@ export type Domain = string[] | string | RegExp;
  * - 'fetch': Browser-compatible Fetch API adapter
  */
 export type CrawlerAdapterType = "http" | "http2" | "curl" | "fetch";
+/**
+ * Per-domain retry/rate-limit options, attachable via addLimiter()
+ */
+export interface LimiterRetryOptions {
+	/** Enable retry handling for this domain (default: true) */
+	enable: boolean;
+	/** Max retries on 429 before giving up on that URL (default: 3) */
+	max429Retries?: number;
+	/** Base delay in ms for retry (default: 1000) */
+	retryDelay?: number;
+	/** Status codes to retry on (e.g. [500, 502, 503]) */
+	retryOnStatusCode?: number[];
+	/** Status codes that should retry without proxy */
+	retryWithoutProxyOnStatusCode?: number[];
+	/** Max retry attempts for non-429 errors (default: 3) */
+	maxRetryAttempts?: number;
+	/** Use exponential backoff — doubles delay on each retry (default: true) */
+	backoff?: boolean;
+}
 /**
  * Configuration interface for the CrawlerOptions class
  * @description Defines all available options for configuring web crawler behavior,
@@ -7823,8 +7842,13 @@ export declare class CrawlerOptions {
 	addLimiter(options: {
 		domain: Domain;
 		isGlobal?: boolean;
+	} & ({
 		options: queueOptions$1;
-	}): this;
+		retry?: LimiterRetryOptions;
+	} | {
+		retry: LimiterRetryOptions;
+		options?: queueOptions$1;
+	})): this;
 	/**
 	 * Add Oxylabs proxy service configuration for specific domains or globally
 	 * @param options - Configuration object containing domain pattern, Oxylabs settings, and global flag
@@ -7964,6 +7988,12 @@ export declare class CrawlerOptions {
 	getAdapter(url: string, type: "decodo", useGlobal?: boolean, random?: boolean): Decodo | null;
 	getAdapter(url: string, type: "stealth", useGlobal?: boolean, random?: boolean): Rezo | null;
 	getAdapter(url: string, type: "headers", useGlobal?: boolean, random?: boolean): OutgoingHttpHeaders | null;
+	/**
+	 * Get retry options for a specific URL from its domain limiter config
+	 * @param url - The URL to look up retry options for
+	 * @returns LimiterRetryOptions if configured for this domain, null otherwise
+	 */
+	getRetryOptions(url: string): LimiterRetryOptions | null;
 	/**
 	 * Generate a random integer between min and max values (inclusive)
 	 * @param min - Minimum value (default: 0)
@@ -8239,6 +8269,8 @@ export declare class Crawler {
 	private originalConcurrency;
 	/** Shutdown handler reference for cleanup */
 	private shutdownHandler;
+	/** Per-domain rate limit tracking: domain → backoff-until timestamp */
+	private rateLimitedDomains;
 	/** Lifecycle event handlers */
 	private startHandlers;
 	private finishHandlers;
@@ -8772,7 +8804,8 @@ export declare class Crawler {
 	 */
 	private getAutoThrottleDelay;
 	/**
-	 * Handle 429 Too Many Requests response with Retry-After header parsing
+	 * Handle 429 Too Many Requests response with retry cap, exponential backoff, and per-domain tracking.
+	 * Uses domain-specific LimiterRetryOptions when configured via addLimiter(), otherwise falls back to global config.
 	 */
 	private handle429Response;
 	/**

package/dist/entries/crawler.cjs

@@ -1,6 +1,6 @@
-const _mod_sn5fgl = require('../crawler/crawler.cjs');
-exports.Crawler = _mod_sn5fgl.Crawler;;
-const _mod_hkoc2f = require('../crawler/crawler-options.cjs');
-exports.CrawlerOptions = _mod_hkoc2f.CrawlerOptions;;
-const _mod_oxo0wk = require('../crawler/scraper.cjs');
-exports.isRestrictedDomain = _mod_oxo0wk.isRestrictedDomain;;
+const _mod_u5d3zp = require('../crawler/crawler.cjs');
+exports.Crawler = _mod_u5d3zp.Crawler;;
+const _mod_w6q6rd = require('../crawler/crawler-options.cjs');
+exports.CrawlerOptions = _mod_w6q6rd.CrawlerOptions;;
+const _mod_rcnka5 = require('../crawler/scraper.cjs');
+exports.isRestrictedDomain = _mod_rcnka5.isRestrictedDomain;;

package/dist/index.cjs

@@ -1,41 +1,41 @@
-const _mod_mg1w6g = require('./core/rezo.cjs');
-exports.Rezo = _mod_mg1w6g.Rezo;
-exports.createRezoInstance = _mod_mg1w6g.createRezoInstance;
-exports.createDefaultInstance = _mod_mg1w6g.createDefaultInstance;;
-const _mod_paeic8 = require('./errors/rezo-error.cjs');
-exports.RezoError = _mod_paeic8.RezoError;
-exports.RezoErrorCode = _mod_paeic8.RezoErrorCode;;
-const _mod_ybqi5i = require('./utils/headers.cjs');
-exports.RezoHeaders = _mod_ybqi5i.RezoHeaders;;
-const _mod_e2ge7f = require('./utils/form-data.cjs');
-exports.RezoFormData = _mod_e2ge7f.RezoFormData;;
-const _mod_id26gk = require('./cookies/cookie-jar.cjs');
-exports.RezoCookieJar = _mod_id26gk.RezoCookieJar;
-exports.CookieJar = _mod_id26gk.CookieJar;
-exports.Cookie = _mod_id26gk.Cookie;
-exports.Store = _mod_id26gk.Store;;
-const _mod_l9qefp = require('./cookies/cookie-store.cjs');
-exports.RezoCookieStore = _mod_l9qefp.RezoCookieStore;;
-const _mod_1fgru3 = require('./utils/curl.cjs');
-exports.toCurl = _mod_1fgru3.toCurl;
-exports.fromCurl = _mod_1fgru3.fromCurl;;
-const _mod_n72bhw = require('./core/hooks.cjs');
-exports.createDefaultHooks = _mod_n72bhw.createDefaultHooks;
-exports.mergeHooks = _mod_n72bhw.mergeHooks;;
-const _mod_rhb7bv = require('./proxy/manager.cjs');
-exports.ProxyManager = _mod_rhb7bv.ProxyManager;;
-const _mod_aibkmp = require('./stealth/stealth.cjs');
-exports.RezoStealth = _mod_aibkmp.RezoStealth;;
-const _mod_eekvw7 = require('./stealth/profiles/index.cjs');
-exports.listProfiles = _mod_eekvw7.listProfiles;
-exports.getProfile = _mod_eekvw7.getProfile;
-exports.getProfilesByFamily = _mod_eekvw7.getProfilesByFamily;;
-const _mod_92fr6i = require('./queue/index.cjs');
-exports.RezoQueue = _mod_92fr6i.RezoQueue;
-exports.HttpQueue = _mod_92fr6i.HttpQueue;
-exports.RezoHttpQueue = _mod_92fr6i.RezoHttpQueue;
-exports.Priority = _mod_92fr6i.Priority;
-exports.HttpMethodPriority = _mod_92fr6i.HttpMethodPriority;;
+const _mod_4zcq5u = require('./core/rezo.cjs');
+exports.Rezo = _mod_4zcq5u.Rezo;
+exports.createRezoInstance = _mod_4zcq5u.createRezoInstance;
+exports.createDefaultInstance = _mod_4zcq5u.createDefaultInstance;;
+const _mod_370086 = require('./errors/rezo-error.cjs');
+exports.RezoError = _mod_370086.RezoError;
+exports.RezoErrorCode = _mod_370086.RezoErrorCode;;
+const _mod_wc6ukr = require('./utils/headers.cjs');
+exports.RezoHeaders = _mod_wc6ukr.RezoHeaders;;
+const _mod_6492nx = require('./utils/form-data.cjs');
+exports.RezoFormData = _mod_6492nx.RezoFormData;;
+const _mod_fyvxa9 = require('./cookies/cookie-jar.cjs');
+exports.RezoCookieJar = _mod_fyvxa9.RezoCookieJar;
+exports.CookieJar = _mod_fyvxa9.CookieJar;
+exports.Cookie = _mod_fyvxa9.Cookie;
+exports.Store = _mod_fyvxa9.Store;;
+const _mod_kce1ch = require('./cookies/cookie-store.cjs');
+exports.RezoCookieStore = _mod_kce1ch.RezoCookieStore;;
+const _mod_ysqe1g = require('./utils/curl.cjs');
+exports.toCurl = _mod_ysqe1g.toCurl;
+exports.fromCurl = _mod_ysqe1g.fromCurl;;
+const _mod_e1uka3 = require('./core/hooks.cjs');
+exports.createDefaultHooks = _mod_e1uka3.createDefaultHooks;
+exports.mergeHooks = _mod_e1uka3.mergeHooks;;
+const _mod_prmnn3 = require('./proxy/manager.cjs');
+exports.ProxyManager = _mod_prmnn3.ProxyManager;;
+const _mod_jsw5rc = require('./stealth/stealth.cjs');
+exports.RezoStealth = _mod_jsw5rc.RezoStealth;;
+const _mod_brkcii = require('./stealth/profiles/index.cjs');
+exports.listProfiles = _mod_brkcii.listProfiles;
+exports.getProfile = _mod_brkcii.getProfile;
+exports.getProfilesByFamily = _mod_brkcii.getProfilesByFamily;;
+const _mod_sna33b = require('./queue/index.cjs');
+exports.RezoQueue = _mod_sna33b.RezoQueue;
+exports.HttpQueue = _mod_sna33b.HttpQueue;
+exports.RezoHttpQueue = _mod_sna33b.RezoHttpQueue;
+exports.Priority = _mod_sna33b.Priority;
+exports.HttpMethodPriority = _mod_sna33b.HttpMethodPriority;;
 const { RezoError } = require('./errors/rezo-error.cjs');
 const isRezoError = exports.isRezoError = RezoError.isRezoError;
 const Cancel = exports.Cancel = RezoError;
@@ -45,9 +45,9 @@ const isCancel = exports.isCancel = (error) => {
 };
 const all = exports.all = Promise.all.bind(Promise);
 const spread = exports.spread = (callback) => (array) => callback(...array);
-const _mod_zy853k = require('./version.cjs');
-exports.VERSION = _mod_zy853k.VERSION;
-exports.PACKAGE_NAME = _mod_zy853k.PACKAGE_NAME;;
+const _mod_fi788b = require('./version.cjs');
+exports.VERSION = _mod_fi788b.VERSION;
+exports.PACKAGE_NAME = _mod_fi788b.PACKAGE_NAME;;
 const { executeRequest } = require('./adapters/http.cjs');
 const { setGlobalAdapter, createRezoInstance } = require('./core/rezo.cjs');
 setGlobalAdapter(executeRequest);

package/dist/index.d.ts

@@ -5736,7 +5736,7 @@ export declare function listProfiles(): string[];
  *
  * IMPORTANT: Update these values when bumping package version.
  */
-export declare const VERSION = "1.0.102";
+export declare const VERSION = "1.0.103";
 export declare const PACKAGE_NAME = "rezo";
 export declare const isRezoError: typeof RezoError.isRezoError;
 export declare const Cancel: typeof RezoError;

package/dist/internal/agents/index.cjs

@@ -1,14 +1,14 @@
-const _mod_ovgh5z = require('./base.cjs');
-exports.Agent = _mod_ovgh5z.Agent;;
-const _mod_xc5kwx = require('./http-proxy.cjs');
-exports.HttpProxyAgent = _mod_xc5kwx.HttpProxyAgent;;
-const _mod_z7s46e = require('./https-proxy.cjs');
-exports.HttpsProxyAgent = _mod_z7s46e.HttpsProxyAgent;;
-const _mod_zkeo38 = require('./socks-proxy.cjs');
-exports.SocksProxyAgent = _mod_zkeo38.SocksProxyAgent;;
-const _mod_vttg6w = require('./socks-client.cjs');
-exports.SocksClient = _mod_vttg6w.SocksClient;;
-const _mod_o1hm5e = require('./bun-socks-http.cjs');
-exports.bunHttp = _mod_o1hm5e.bunHttp;
-exports.isBunRuntime = _mod_o1hm5e.isBunRuntime;
-exports.isBunSocksRequest = _mod_o1hm5e.isBunSocksRequest;;
+const _mod_y8ufew = require('./base.cjs');
+exports.Agent = _mod_y8ufew.Agent;;
+const _mod_5j5k29 = require('./http-proxy.cjs');
+exports.HttpProxyAgent = _mod_5j5k29.HttpProxyAgent;;
+const _mod_b781z2 = require('./https-proxy.cjs');
+exports.HttpsProxyAgent = _mod_b781z2.HttpsProxyAgent;;
+const _mod_w4vo7h = require('./socks-proxy.cjs');
+exports.SocksProxyAgent = _mod_w4vo7h.SocksProxyAgent;;
+const _mod_xo6q9q = require('./socks-client.cjs');
+exports.SocksClient = _mod_xo6q9q.SocksClient;;
+const _mod_cbcssg = require('./bun-socks-http.cjs');
+exports.bunHttp = _mod_cbcssg.bunHttp;
+exports.isBunRuntime = _mod_cbcssg.isBunRuntime;
+exports.isBunSocksRequest = _mod_cbcssg.isBunSocksRequest;;

package/dist/platform/browser.d.ts

@@ -5514,7 +5514,7 @@ export interface RezoInstance extends Rezo, RezoCallable {
  *
  * IMPORTANT: Update these values when bumping package version.
  */
-export declare const VERSION = "1.0.102";
+export declare const VERSION = "1.0.103";
 export declare const isRezoError: typeof RezoError.isRezoError;
 export declare const Cancel: typeof RezoError;
 export declare const CancelToken: {

package/dist/platform/bun.d.ts

@@ -5514,7 +5514,7 @@ export interface RezoInstance extends Rezo, RezoCallable {
  *
  * IMPORTANT: Update these values when bumping package version.
  */
-export declare const VERSION = "1.0.102";
+export declare const VERSION = "1.0.103";
 export declare const isRezoError: typeof RezoError.isRezoError;
 export declare const Cancel: typeof RezoError;
 export declare const CancelToken: {

package/dist/platform/deno.d.ts

@@ -5514,7 +5514,7 @@ export interface RezoInstance extends Rezo, RezoCallable {
  *
  * IMPORTANT: Update these values when bumping package version.
  */
-export declare const VERSION = "1.0.102";
+export declare const VERSION = "1.0.103";
 export declare const isRezoError: typeof RezoError.isRezoError;
 export declare const Cancel: typeof RezoError;
 export declare const CancelToken: {

package/dist/platform/node.d.ts

@@ -5514,7 +5514,7 @@ export interface RezoInstance extends Rezo, RezoCallable {
  *
  * IMPORTANT: Update these values when bumping package version.
  */
-export declare const VERSION = "1.0.102";
+export declare const VERSION = "1.0.103";
 export declare const isRezoError: typeof RezoError.isRezoError;
 export declare const Cancel: typeof RezoError;
 export declare const CancelToken: {

package/dist/platform/react-native.d.ts

@@ -5514,7 +5514,7 @@ export interface RezoInstance extends Rezo, RezoCallable {
  *
  * IMPORTANT: Update these values when bumping package version.
  */
-export declare const VERSION = "1.0.102";
+export declare const VERSION = "1.0.103";
 export declare const isRezoError: typeof RezoError.isRezoError;
 export declare const Cancel: typeof RezoError;
 export declare const CancelToken: {

package/dist/platform/worker.d.ts

@@ -5514,7 +5514,7 @@ export interface RezoInstance extends Rezo, RezoCallable {
  *
  * IMPORTANT: Update these values when bumping package version.
  */
-export declare const VERSION = "1.0.102";
+export declare const VERSION = "1.0.103";
 export declare const isRezoError: typeof RezoError.isRezoError;
 export declare const Cancel: typeof RezoError;
 export declare const CancelToken: {

package/dist/proxy/index.cjs

@@ -1,9 +1,9 @@
 const { Agent, HttpProxyAgent, HttpsProxyAgent, SocksProxyAgent } = require('../internal/agents/index.cjs');
 const { parseProxyString } = require('./parse.cjs');
-const _mod_m6jdw2 = require('./manager.cjs');
-exports.ProxyManager = _mod_m6jdw2.ProxyManager;;
-const _mod_wqm2on = require('./parse.cjs');
-exports.parseProxyString = _mod_wqm2on.parseProxyString;;
+const _mod_r8nh4v = require('./manager.cjs');
+exports.ProxyManager = _mod_r8nh4v.ProxyManager;;
+const _mod_28d64o = require('./parse.cjs');
+exports.parseProxyString = _mod_28d64o.parseProxyString;;
 function createOptions(uri, opts) {
   if (uri instanceof URL || typeof uri === "string") {
     return {

package/dist/queue/index.cjs

@@ -1,9 +1,9 @@
-const _mod_rbr7tk = require('./queue.cjs');
-exports.RezoQueue = _mod_rbr7tk.RezoQueue;;
-const _mod_xxemqi = require('./http-queue.cjs');
-exports.HttpQueue = _mod_xxemqi.HttpQueue;
-exports.RezoHttpQueue = _mod_xxemqi.HttpQueue;
-exports.extractDomain = _mod_xxemqi.extractDomain;;
-const _mod_dahb8u = require('./types.cjs');
-exports.Priority = _mod_dahb8u.Priority;
-exports.HttpMethodPriority = _mod_dahb8u.HttpMethodPriority;;
+const _mod_2iykbp = require('./queue.cjs');
+exports.RezoQueue = _mod_2iykbp.RezoQueue;;
+const _mod_0gzglg = require('./http-queue.cjs');
+exports.HttpQueue = _mod_0gzglg.HttpQueue;
+exports.RezoHttpQueue = _mod_0gzglg.HttpQueue;
+exports.extractDomain = _mod_0gzglg.extractDomain;;
+const _mod_rst962 = require('./types.cjs');
+exports.Priority = _mod_rst962.Priority;
+exports.HttpMethodPriority = _mod_rst962.HttpMethodPriority;;

package/dist/responses/universal/index.cjs

@@ -1,11 +1,11 @@
-const _mod_osfa7y = require('./event-emitter.cjs');
-exports.UniversalEventEmitter = _mod_osfa7y.UniversalEventEmitter;;
-const _mod_q3ksg9 = require('./stream.cjs');
-exports.UniversalStreamResponse = _mod_q3ksg9.UniversalStreamResponse;
-exports.StreamResponse = _mod_q3ksg9.StreamResponse;;
-const _mod_pe1yjf = require('./download.cjs');
-exports.UniversalDownloadResponse = _mod_pe1yjf.UniversalDownloadResponse;
-exports.DownloadResponse = _mod_pe1yjf.DownloadResponse;;
-const _mod_73ij6j = require('./upload.cjs');
-exports.UniversalUploadResponse = _mod_73ij6j.UniversalUploadResponse;
-exports.UploadResponse = _mod_73ij6j.UploadResponse;;
+const _mod_0qnk0d = require('./event-emitter.cjs');
+exports.UniversalEventEmitter = _mod_0qnk0d.UniversalEventEmitter;;
+const _mod_y2ec7n = require('./stream.cjs');
+exports.UniversalStreamResponse = _mod_y2ec7n.UniversalStreamResponse;
+exports.StreamResponse = _mod_y2ec7n.StreamResponse;;
+const _mod_6hn0du = require('./download.cjs');
+exports.UniversalDownloadResponse = _mod_6hn0du.UniversalDownloadResponse;
+exports.DownloadResponse = _mod_6hn0du.DownloadResponse;;
+const _mod_hi9myu = require('./upload.cjs');
+exports.UniversalUploadResponse = _mod_hi9myu.UniversalUploadResponse;
+exports.UploadResponse = _mod_hi9myu.UploadResponse;;

package/dist/stealth/index.cjs

@@ -1,17 +1,17 @@
-const _mod_azxry1 = require('./stealth.cjs');
-exports.RezoStealth = _mod_azxry1.RezoStealth;;
-const _mod_4enzhh = require('./tls-fingerprint.cjs');
-exports.createSecureContext = _mod_4enzhh.createSecureContext;
-exports.buildTlsOptions = _mod_4enzhh.buildTlsOptions;;
-const _mod_3fit8q = require('./resolver.cjs');
-exports.resolveProfile = _mod_3fit8q.resolveProfile;
-exports.detectProfileFromUserAgent = _mod_3fit8q.detectProfileFromUserAgent;;
-const _mod_86x53k = require('./profiles/index.cjs');
-exports.getProfile = _mod_86x53k.getProfile;
-exports.getProfilesByFamily = _mod_86x53k.getProfilesByFamily;
-exports.getProfilesByDevice = _mod_86x53k.getProfilesByDevice;
-exports.getRandomProfile = _mod_86x53k.getRandomProfile;
-exports.getRandomProfileByFamily = _mod_86x53k.getRandomProfileByFamily;
-exports.listProfiles = _mod_86x53k.listProfiles;
-exports.listProfilesByFamily = _mod_86x53k.listProfilesByFamily;
-exports.PROFILE_REGISTRY = _mod_86x53k.PROFILE_REGISTRY;;
+const _mod_lb9c38 = require('./stealth.cjs');
+exports.RezoStealth = _mod_lb9c38.RezoStealth;;
+const _mod_25yxla = require('./tls-fingerprint.cjs');
+exports.createSecureContext = _mod_25yxla.createSecureContext;
+exports.buildTlsOptions = _mod_25yxla.buildTlsOptions;;
+const _mod_iuoyzn = require('./resolver.cjs');
+exports.resolveProfile = _mod_iuoyzn.resolveProfile;
+exports.detectProfileFromUserAgent = _mod_iuoyzn.detectProfileFromUserAgent;;
+const _mod_asoox9 = require('./profiles/index.cjs');
+exports.getProfile = _mod_asoox9.getProfile;
+exports.getProfilesByFamily = _mod_asoox9.getProfilesByFamily;
+exports.getProfilesByDevice = _mod_asoox9.getProfilesByDevice;
+exports.getRandomProfile = _mod_asoox9.getRandomProfile;
+exports.getRandomProfileByFamily = _mod_asoox9.getRandomProfileByFamily;
+exports.listProfiles = _mod_asoox9.listProfiles;
+exports.listProfilesByFamily = _mod_asoox9.listProfilesByFamily;
+exports.PROFILE_REGISTRY = _mod_asoox9.PROFILE_REGISTRY;;

package/dist/stealth/profiles/index.cjs

@@ -43,16 +43,16 @@ function getRandomProfileByFamily(family) {
     throw new Error(`No profiles found for family: ${family}`);
   return profiles[Math.floor(Math.random() * profiles.length)];
 }
-const _mod_qmgwzp = require('./constants.cjs');
-exports.expandPseudoOrder = _mod_qmgwzp.expandPseudoOrder;;
-const _mod_hzim6p = require('./chrome-profiles.cjs');
-exports.CHROME_PROFILES = _mod_hzim6p.CHROME_PROFILES;;
-const _mod_x0q2wa = require('./firefox-profiles.cjs');
-exports.FIREFOX_PROFILES = _mod_x0q2wa.FIREFOX_PROFILES;;
-const _mod_wdfi04 = require('./safari-profiles.cjs');
-exports.SAFARI_PROFILES = _mod_wdfi04.SAFARI_PROFILES;;
-const _mod_6nah86 = require('./edge-profiles.cjs');
-exports.EDGE_PROFILES = _mod_6nah86.EDGE_PROFILES;;
+const _mod_eo8qle = require('./constants.cjs');
+exports.expandPseudoOrder = _mod_eo8qle.expandPseudoOrder;;
+const _mod_6wzedx = require('./chrome-profiles.cjs');
+exports.CHROME_PROFILES = _mod_6wzedx.CHROME_PROFILES;;
+const _mod_x85qd3 = require('./firefox-profiles.cjs');
+exports.FIREFOX_PROFILES = _mod_x85qd3.FIREFOX_PROFILES;;
+const _mod_fqnol1 = require('./safari-profiles.cjs');
+exports.SAFARI_PROFILES = _mod_fqnol1.SAFARI_PROFILES;;
+const _mod_zu0qne = require('./edge-profiles.cjs');
+exports.EDGE_PROFILES = _mod_zu0qne.EDGE_PROFILES;;
 
 exports.getProfile = getProfile;
 exports.getProfilesByFamily = getProfilesByFamily;

package/dist/version.cjs

@@ -1,2 +1,2 @@
-const VERSION = exports.VERSION = "1.0.102";
+const VERSION = exports.VERSION = "1.0.103";
 const PACKAGE_NAME = exports.PACKAGE_NAME = "rezo";

package/dist/version.js

@@ -1,2 +1,2 @@
-export const VERSION = "1.0.102";
+export const VERSION = "1.0.103";
 export const PACKAGE_NAME = "rezo";

package/dist/wget/index.cjs

@@ -1,54 +1,54 @@
-const _mod_esmcd0 = require('./types.cjs');
-exports.WgetError = _mod_esmcd0.WgetError;;
-const _mod_j1tggk = require('./asset-extractor.cjs');
-exports.AssetExtractor = _mod_j1tggk.AssetExtractor;;
-const _mod_614ddi = require('./url-filter.cjs');
-exports.UrlFilter = _mod_614ddi.UrlFilter;;
-const _mod_jb05do = require('./file-writer.cjs');
-exports.FileWriter = _mod_jb05do.FileWriter;;
-const _mod_x5na3i = require('./robots.cjs');
-exports.RobotsHandler = _mod_x5na3i.RobotsHandler;;
-const _mod_02bgnx = require('./resume.cjs');
-exports.ResumeHandler = _mod_02bgnx.ResumeHandler;;
-const _mod_j5gr9z = require('./progress.cjs');
-exports.ProgressReporter = _mod_j5gr9z.ProgressReporter;
-exports.ProgressTracker = _mod_j5gr9z.ProgressTracker;
-exports.parseSize = _mod_j5gr9z.parseSize;;
-const _mod_5ku3xo = require('./link-converter.cjs');
-exports.LinkConverter = _mod_5ku3xo.LinkConverter;;
-const _mod_1mm28a = require('./style-extractor.cjs');
-exports.StyleExtractor = _mod_1mm28a.StyleExtractor;;
-const _mod_1n8zst = require('./downloader.cjs');
-exports.Downloader = _mod_1n8zst.Downloader;;
-const _mod_bkj9pe = require('./asset-organizer.cjs');
-exports.AssetOrganizer = _mod_bkj9pe.AssetOrganizer;
-exports.DEFAULT_ASSET_FOLDERS = _mod_bkj9pe.DEFAULT_ASSET_FOLDERS;;
-const _mod_8e2ppi = require('./download-cache.cjs');
-exports.DownloadCache = _mod_8e2ppi.DownloadCache;;
-const _mod_6rld3b = require('./filter-lists.cjs');
-exports.EXECUTABLE_EXTENSIONS = _mod_6rld3b.EXECUTABLE_EXTENSIONS;
-exports.ARCHIVE_EXTENSIONS = _mod_6rld3b.ARCHIVE_EXTENSIONS;
-exports.DOCUMENT_EXTENSIONS = _mod_6rld3b.DOCUMENT_EXTENSIONS;
-exports.IMAGE_EXTENSIONS = _mod_6rld3b.IMAGE_EXTENSIONS;
-exports.VIDEO_EXTENSIONS = _mod_6rld3b.VIDEO_EXTENSIONS;
-exports.AUDIO_EXTENSIONS = _mod_6rld3b.AUDIO_EXTENSIONS;
-exports.FONT_EXTENSIONS = _mod_6rld3b.FONT_EXTENSIONS;
-exports.WEB_ASSET_EXTENSIONS = _mod_6rld3b.WEB_ASSET_EXTENSIONS;
-exports.DATA_EXTENSIONS = _mod_6rld3b.DATA_EXTENSIONS;
-exports.EXECUTABLE_MIME_TYPES = _mod_6rld3b.EXECUTABLE_MIME_TYPES;
-exports.ARCHIVE_MIME_TYPES = _mod_6rld3b.ARCHIVE_MIME_TYPES;
-exports.DOCUMENT_MIME_TYPES = _mod_6rld3b.DOCUMENT_MIME_TYPES;
-exports.IMAGE_MIME_TYPES = _mod_6rld3b.IMAGE_MIME_TYPES;
-exports.VIDEO_MIME_TYPES = _mod_6rld3b.VIDEO_MIME_TYPES;
-exports.AUDIO_MIME_TYPES = _mod_6rld3b.AUDIO_MIME_TYPES;
-exports.FONT_MIME_TYPES = _mod_6rld3b.FONT_MIME_TYPES;
-exports.WEB_ASSET_MIME_TYPES = _mod_6rld3b.WEB_ASSET_MIME_TYPES;
-exports.DATA_MIME_TYPES = _mod_6rld3b.DATA_MIME_TYPES;
-exports.SAFE_WEB_PRESET = _mod_6rld3b.SAFE_WEB_PRESET;
-exports.DOCUMENTS_ONLY_PRESET = _mod_6rld3b.DOCUMENTS_ONLY_PRESET;
-exports.NO_MEDIA_PRESET = _mod_6rld3b.NO_MEDIA_PRESET;
-exports.MINIMAL_MIRROR_PRESET = _mod_6rld3b.MINIMAL_MIRROR_PRESET;
-exports.TEXT_ONLY_PRESET = _mod_6rld3b.TEXT_ONLY_PRESET;;
+const _mod_4cn6wv = require('./types.cjs');
+exports.WgetError = _mod_4cn6wv.WgetError;;
+const _mod_egjoud = require('./asset-extractor.cjs');
+exports.AssetExtractor = _mod_egjoud.AssetExtractor;;
+const _mod_re94hs = require('./url-filter.cjs');
+exports.UrlFilter = _mod_re94hs.UrlFilter;;
+const _mod_m0jacj = require('./file-writer.cjs');
+exports.FileWriter = _mod_m0jacj.FileWriter;;
+const _mod_3bg6d3 = require('./robots.cjs');
+exports.RobotsHandler = _mod_3bg6d3.RobotsHandler;;
+const _mod_v6zi0p = require('./resume.cjs');
+exports.ResumeHandler = _mod_v6zi0p.ResumeHandler;;
+const _mod_y7oler = require('./progress.cjs');
+exports.ProgressReporter = _mod_y7oler.ProgressReporter;
+exports.ProgressTracker = _mod_y7oler.ProgressTracker;
+exports.parseSize = _mod_y7oler.parseSize;;
+const _mod_mh0crx = require('./link-converter.cjs');
+exports.LinkConverter = _mod_mh0crx.LinkConverter;;
+const _mod_be1ijk = require('./style-extractor.cjs');
+exports.StyleExtractor = _mod_be1ijk.StyleExtractor;;
+const _mod_2jcf4q = require('./downloader.cjs');
+exports.Downloader = _mod_2jcf4q.Downloader;;
+const _mod_v1l9cr = require('./asset-organizer.cjs');
+exports.AssetOrganizer = _mod_v1l9cr.AssetOrganizer;
+exports.DEFAULT_ASSET_FOLDERS = _mod_v1l9cr.DEFAULT_ASSET_FOLDERS;;
+const _mod_zo6uf5 = require('./download-cache.cjs');
+exports.DownloadCache = _mod_zo6uf5.DownloadCache;;
+const _mod_f0yq16 = require('./filter-lists.cjs');
+exports.EXECUTABLE_EXTENSIONS = _mod_f0yq16.EXECUTABLE_EXTENSIONS;
+exports.ARCHIVE_EXTENSIONS = _mod_f0yq16.ARCHIVE_EXTENSIONS;
+exports.DOCUMENT_EXTENSIONS = _mod_f0yq16.DOCUMENT_EXTENSIONS;
+exports.IMAGE_EXTENSIONS = _mod_f0yq16.IMAGE_EXTENSIONS;
+exports.VIDEO_EXTENSIONS = _mod_f0yq16.VIDEO_EXTENSIONS;
+exports.AUDIO_EXTENSIONS = _mod_f0yq16.AUDIO_EXTENSIONS;
+exports.FONT_EXTENSIONS = _mod_f0yq16.FONT_EXTENSIONS;
+exports.WEB_ASSET_EXTENSIONS = _mod_f0yq16.WEB_ASSET_EXTENSIONS;
+exports.DATA_EXTENSIONS = _mod_f0yq16.DATA_EXTENSIONS;
+exports.EXECUTABLE_MIME_TYPES = _mod_f0yq16.EXECUTABLE_MIME_TYPES;
+exports.ARCHIVE_MIME_TYPES = _mod_f0yq16.ARCHIVE_MIME_TYPES;
+exports.DOCUMENT_MIME_TYPES = _mod_f0yq16.DOCUMENT_MIME_TYPES;
+exports.IMAGE_MIME_TYPES = _mod_f0yq16.IMAGE_MIME_TYPES;
+exports.VIDEO_MIME_TYPES = _mod_f0yq16.VIDEO_MIME_TYPES;
+exports.AUDIO_MIME_TYPES = _mod_f0yq16.AUDIO_MIME_TYPES;
+exports.FONT_MIME_TYPES = _mod_f0yq16.FONT_MIME_TYPES;
+exports.WEB_ASSET_MIME_TYPES = _mod_f0yq16.WEB_ASSET_MIME_TYPES;
+exports.DATA_MIME_TYPES = _mod_f0yq16.DATA_MIME_TYPES;
+exports.SAFE_WEB_PRESET = _mod_f0yq16.SAFE_WEB_PRESET;
+exports.DOCUMENTS_ONLY_PRESET = _mod_f0yq16.DOCUMENTS_ONLY_PRESET;
+exports.NO_MEDIA_PRESET = _mod_f0yq16.NO_MEDIA_PRESET;
+exports.MINIMAL_MIRROR_PRESET = _mod_f0yq16.MINIMAL_MIRROR_PRESET;
+exports.TEXT_ONLY_PRESET = _mod_f0yq16.TEXT_ONLY_PRESET;;
 const { Downloader } = require('./downloader.cjs');
 const rezo = require('../index.cjs');
 const { promises: fs } = require("node:fs");

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "rezo",
-  "version": "1.0.102",
+  "version": "1.0.103",
   "description": "Lightning-fast, enterprise-grade HTTP client for modern JavaScript. Full HTTP/2 support, intelligent cookie management, multiple adapters (HTTP, Fetch, cURL, XHR), streaming, proxy support (HTTP/HTTPS/SOCKS), and cross-environment compatibility.",
   "main": "dist/index.cjs",
   "module": "dist/index.js",