rezo 1.0.55 → 1.0.57

package/dist/adapters/index.cjs

@@ -1,6 +1,6 @@
-const _mod_eqitn2 = require('./picker.cjs');
-exports.detectRuntime = _mod_eqitn2.detectRuntime;
-exports.getAdapterCapabilities = _mod_eqitn2.getAdapterCapabilities;
-exports.buildAdapterContext = _mod_eqitn2.buildAdapterContext;
-exports.getAvailableAdapters = _mod_eqitn2.getAvailableAdapters;
-exports.selectAdapter = _mod_eqitn2.selectAdapter;;
+const _mod_w274ym = require('./picker.cjs');
+exports.detectRuntime = _mod_w274ym.detectRuntime;
+exports.getAdapterCapabilities = _mod_w274ym.getAdapterCapabilities;
+exports.buildAdapterContext = _mod_w274ym.buildAdapterContext;
+exports.getAvailableAdapters = _mod_w274ym.getAvailableAdapters;
+exports.selectAdapter = _mod_w274ym.selectAdapter;;

package/dist/cache/index.cjs

@@ -1,9 +1,9 @@
-const _mod_6631rj = require('./lru-cache.cjs');
-exports.LRUCache = _mod_6631rj.LRUCache;;
-const _mod_sbmm9e = require('./dns-cache.cjs');
-exports.DNSCache = _mod_sbmm9e.DNSCache;
-exports.getGlobalDNSCache = _mod_sbmm9e.getGlobalDNSCache;
-exports.resetGlobalDNSCache = _mod_sbmm9e.resetGlobalDNSCache;;
-const _mod_1hzlec = require('./response-cache.cjs');
-exports.ResponseCache = _mod_1hzlec.ResponseCache;
-exports.normalizeResponseCacheConfig = _mod_1hzlec.normalizeResponseCacheConfig;;
+const _mod_n2z5g1 = require('./lru-cache.cjs');
+exports.LRUCache = _mod_n2z5g1.LRUCache;;
+const _mod_l1roxo = require('./dns-cache.cjs');
+exports.DNSCache = _mod_l1roxo.DNSCache;
+exports.getGlobalDNSCache = _mod_l1roxo.getGlobalDNSCache;
+exports.resetGlobalDNSCache = _mod_l1roxo.resetGlobalDNSCache;;
+const _mod_fbczja = require('./response-cache.cjs');
+exports.ResponseCache = _mod_fbczja.ResponseCache;
+exports.normalizeResponseCacheConfig = _mod_fbczja.normalizeResponseCacheConfig;;

package/dist/crawler/crawler-options.cjs

@@ -1 +1 @@
-var{RezoQueue:m}=require("../queue/queue.cjs"),{Oxylabs:x}=require("./addon/oxylabs/index.cjs"),b=require("node:path"),g=require("node:os"),{Decodo:y}=require("./addon/decodo/index.cjs");class f{baseUrl;adapter;enableNavigationHistory;sessionId;rejectUnauthorized;userAgent;useRndUserAgent;timeout;maxRedirects;maxRetryAttempts;retryDelay;retryOnStatusCode;forceRevisit;retryWithoutProxyOnStatusCode;retryOnProxyError;maxRetryOnProxyError;allowRevisiting;enableCache;cacheTTL;cacheDir;throwFatalError;debug;maxDepth;maxUrls;maxResponseSize;respectRobotsTxt;followNofollow;autoThrottle;autoThrottleTargetDelay;autoThrottleMinDelay;autoThrottleMaxDelay;maxWaitOn429;alwaysWaitOn429;oxylabs=[];decodo=[];proxies=[];limiters=[];requestHeaders=[];userAgents=A();constructor(e={}){this.baseUrl=e.baseUrl||"",this.adapter=e.adapter??"http",this.enableNavigationHistory=e.enableNavigationHistory??!1,this.sessionId=e.sessionId??`session_${Date.now()}_${Math.random().toString(36).slice(2,8)}`,this.rejectUnauthorized=e.rejectUnauthorized??!0,this.userAgent=e.userAgent,this.useRndUserAgent=e.useRndUserAgent??!1,this.timeout=e.timeout??30000,this.maxRedirects=e.maxRedirects??10,this.maxRetryAttempts=e.maxRetryAttempts??3,this.retryDelay=e.retryDelay??0,this.retryOnStatusCode=e.retryOnStatusCode??[408,429,500,502,503,504],this.forceRevisit=e.forceRevisit??!1,this.retryWithoutProxyOnStatusCode=e.retryWithoutProxyOnStatusCode??[407,403],this.retryOnProxyError=e.retryOnProxyError??!0,this.maxRetryOnProxyError=e.maxRetryOnProxyError??3,this.allowRevisiting=e.allowRevisiting??!1,this.enableCache=e.enableCache??!0,this.cacheTTL=e.cacheTTL??604800000,this.cacheDir=e.cacheDir??b.join(g.tmpdir(),"uiniqhtt_cache"),this.throwFatalError=e.throwFatalError??!1,this.debug=e.debug??!1,this.maxDepth=e.maxDepth??0,this.maxUrls=e.maxUrls??0,this.maxResponseSize=e.maxResponseSize??0,this.respectRobotsTxt=e.respectRobotsTxt??!1,this.followNofollow=e.followNofollow??!1,this.autoThrottle=e.autoThrottle??!0,this.autoThrottleTargetDelay=e.autoThrottleTargetDelay??1000,this.autoThrottleMinDelay=e.autoThrottleMinDelay??100,this.autoThrottleMaxDelay=e.autoThrottleMaxDelay??60000,this.maxWaitOn429=e.maxWaitOn429??1800000,this.alwaysWaitOn429=e.alwaysWaitOn429??!1,this._addHeaders(e.headers),this._addOxylabs(e.oxylabs),this._addDecodo(e.decodo),this._addProxies(e.proxy),this._addLimiters(e.limiter)}getConfiguredDomains(e){return(e==="headers"?this.requestHeaders:e==="limiters"?this.limiters:e==="oxylabs"?this.oxylabs:this.proxies).filter((i)=>i.domain).map((i)=>i.domain).filter((i,o,r)=>r.indexOf(i)===o)}removeDomain(e){return this.requestHeaders=this.requestHeaders.filter((t)=>!t.domain||!this._domainsEqual(t.domain,e)),this.proxies=this.proxies.filter((t)=>!t.domain||!this._domainsEqual(t.domain,e)),this.limiters=this.limiters.filter((t)=>!t.domain||!this._domainsEqual(t.domain,e)),this.oxylabs=this.oxylabs.filter((t)=>!t.domain||!this._domainsEqual(t.domain,e)),this}_domainsEqual(e,t){if(Array.isArray(e)&&Array.isArray(t))return e.length===t.length&&e.every((i,o)=>i===t[o]);return e===t}getConfigurationSummary(){let e=(t)=>({total:t.length,global:t.filter((i)=>i.isGlobal).length,domainSpecific:t.filter((i)=>!i.isGlobal&&i.domain).length});return{headers:e(this.requestHeaders),proxies:e(this.proxies),limiters:e(this.limiters),oxylabs:e(this.oxylabs)}}_addHeaders(e){if(!e||!e.enable)return;for(let t of e.httpHeaders){let{domain:i,isGlobal:o,headers:r}=t;if(!i&&!o)continue;if(r instanceof Headers){let s=Object.fromEntries(r.entries());if(Object.keys(s).length<1)continue;r=s}else if(!r||Object.keys(r).length<1)continue;this.requestHeaders.push({domain:i,isGlobal:o,headers:r})}}_addProxies(e){if(!e||!e.enable)return;for(let t of e.proxies){let{domain:i,isGlobal:o,proxy:r}=t;if(!i&&!o)continue;if(!r||Object.keys(r).length<1)continue;this.proxies.push({domain:i,isGlobal:o,proxy:r})}}_addLimiters(e){if(!e||!e.enable)return;for(let t of e.limiters){let{domain:i,isGlobal:o,options:r}=t;if(!i&&!o)continue;if(!r||Object.keys(r).length<1)continue;this.limiters.push({domain:i,isGlobal:o,pqueue:new m(r)})}}_addOxylabs(e){if(!e||!e.enable)return;for(let t of e.labs){let{domain:i,isGlobal:o,options:r,queueOptions:s}=t;if(!i&&!o)continue;if(!r||Object.keys(r).length<1)continue;this.oxylabs.push({domain:i,isGlobal:o,adaptar:new x(r)})}}_addDecodo(e){if(!e||!e.enable)return;for(let t of e.labs){let{domain:i,isGlobal:o,options:r,queueOptions:s}=t;if(!i&&!o)continue;if(!r||Object.keys(r).length<1)continue;this.decodo.push({domain:i,isGlobal:o,adaptar:new y(r)})}}addHeaders(e){return this._addHeaders({enable:!0,httpHeaders:[e]}),this}addProxy(e){return this._addProxies({enable:!0,proxies:[e]}),this}addLimiter(e){return this._addLimiters({enable:!0,limiters:[e]}),this}addOxylabs(e){return this._addOxylabs({enable:!0,labs:[e]}),this}addDecodo(e){return this._addDecodo({enable:!0,labs:[e]}),this}clearGlobalConfigs(){if(Array.isArray(this.requestHeaders))this.requestHeaders=this.requestHeaders.filter((e)=>!e.isGlobal);if(Array.isArray(this.oxylabs))this.oxylabs=this.oxylabs.filter((e)=>!e.isGlobal);if(Array.isArray(this.limiters))this.limiters=this.limiters.filter((e)=>!e.isGlobal);if(Array.isArray(this.proxies))this.proxies=this.proxies.filter((e)=>!e.isGlobal);return this}getAdapter(e,t,i,o){if(!this.getDomainName(e))return null;let s=[],a=t==="headers"?this.requestHeaders:t==="limiters"?this.limiters:t==="oxylabs"?this.oxylabs:t==="decodo"?this.decodo:this.proxies;for(let n=0;n<a.length;n++)if(this._hasDomain(e,a[n].domain))s.push(n);if(s.length){let n=o?s[this.rnd(0,s.length-1)]:s[0];return t==="headers"?this.requestHeaders[n].headers:t==="limiters"?this.limiters[n].pqueue:t==="oxylabs"?this.oxylabs[n].adaptar:t==="decodo"?this.decodo[n].adaptar:this.proxies[n].proxy}s.length=0;for(let n=0;n<a.length;n++)s.push(n);if(s.length){let n=o?s[this.rnd(0,s.length-1)]:s[0];if(a[n].isGlobal&&i)return t==="headers"?this.requestHeaders[n].headers:t==="limiters"?this.limiters[n].pqueue:t==="oxylabs"?this.oxylabs[n].adaptar:t==="decodo"?this.decodo[n].adaptar:this.proxies[n].proxy}return null}rnd(e=0,t=Number.MAX_VALUE){return Math.floor(Math.random()*(t-e+1))+e}hasDomain(e,t,i){if(!this.getDomainName(e))return!1;let r=t==="headers"?this.requestHeaders:t==="limiters"?this.limiters:t==="oxylabs"?this.oxylabs:t==="decodo"?this.decodo:this.proxies;for(let s=0;s<r.length;s++)if(this._hasDomain(e,r[s].domain))return!0;if(i){for(let s=0;s<r.length;s++)if(r[s].isGlobal)return!0}return!1}pickHeaders(e,t,i,o){let r=this.getAdapter(e,"headers",t),s=new Headers(r??{});if(i&&i instanceof Headers)for(let[a,n]of Object.entries(i.entries()))s.set(a,n);else if(i&&typeof i==="object"){for(let[a,n]of Object.entries(i))if(typeof n==="string")s.set(a,n)}if(o)s.set("user-agent",this.getRandomUserAgent());return Object.fromEntries(s.entries())}_hasDomain(e,t){if(!t)return!1;let i=this.getDomainName(e);if(!i)return!1;let o=(s)=>{return/[\^\$\*\+\?\{\}\[\]\(\)\|\\]/.test(s)||s.startsWith("/")||s.includes(".*")||s.includes(".+")},r=(s)=>{if(s instanceof RegExp)return s.test(i)||s.test(e);let a=s.toString().trim();if(i.toLowerCase()===a.toLowerCase())return!0;if(a.includes("*")){let l=a.replace(/[.*+?^${}()|[\]\\]/g,"\\$&").replace(/\\\*/g,".*"),h=new RegExp(`^${l}$`,"i");return h.test(i)||h.test(e)}if(o(a))try{let l=a,h="i",u=a.match(/^\/(.*)\/(\w*)$/);if(u)l=u[1],h=u[2]||"i";let c=new RegExp(l,h);return c.test(i)||c.test(e)}catch(l){return i.toLowerCase().includes(a.toLowerCase())}let n=i.toLowerCase(),d=a.toLowerCase();return n===d||n.endsWith("."+d)||d.endsWith("."+n)};if(Array.isArray(t)){for(let s of t)if(r(s))return!0;return!1}return r(t)}getDomainName(e){if(this.isValidUrl(e))return new URL(e).hostname;else if(this.isHostName(e))return e;return null}isHostName(e){if(!e)return!1;if(e.length>255)return!1;let t=/^(?:(?:[a-zA-Z0-9](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?\.)+ [a-zA-Z]{2,})$/;return e=e.trim().toLowerCase(),t.test(e)&&!e.startsWith("-")&&!e.endsWith("-")}isValidUrl(e){if(!e)return!1;e=e.trim();try{let t=new URL(e);if(!t.protocol||!["http:","https:"].includes(t.protocol.toLowerCase()))return!1;if(!t.hostname)return!1;if(!/^(?:(?:[a-zA-Z0-9](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?\.)+[a-zA-Z]{2,})$/.test(t.hostname))return!1;return!0}catch{return!1}}getRandomUserAgent(){return this.userAgents[Math.floor(Math.random()*this.userAgents.length)]}}function A(){let e=[{name:"Chrome",version:"91.0.4472.124",engine:"AppleWebKit/537.36"},{name:"Firefox",version:"89.0",engine:"Gecko/20100101"},{name:"Safari",version:"14.1.1",engine:"AppleWebKit/605.1.15"},{name:"Edge",version:"91.0.864.59",engine:"AppleWebKit/537.36"},{name:"Opera",version:"77.0.4054.277",engine:"AppleWebKit/537.36"},{name:"Vivaldi",version:"3.8.2259.42",engine:"AppleWebKit/537.36"},{name:"Brave",version:"1.26.74",engine:"AppleWebKit/537.36"},{name:"Chromium",version:"91.0.4472.101",engine:"AppleWebKit/537.36"},{name:"Yandex",version:"21.5.3.742",engine:"AppleWebKit/537.36"},{name:"Maxthon",version:"5.3.8.2000",engine:"AppleWebKit/537.36"}],t=["Windows NT 10.0","Windows NT 6.1","Macintosh; Intel Mac OS X 10_15_7","Macintosh; Intel Mac OS X 11_4_0","X11; Linux x86_64","X11; Ubuntu; Linux x86_64"],i=[];for(let o=0;o<200;o++){let r=e[Math.floor(Math.random()*e.length)],s=t[Math.floor(Math.random()*t.length)],a="";switch(r.name){case"Chrome":a=`Mozilla/5.0 (${s}) ${r.engine} (KHTML, like Gecko) Chrome/${r.version} Safari/537.36`;break;case"Firefox":a=`Mozilla/5.0 (${s}; rv:${r.version}) ${r.engine} Firefox/${r.version}`;break;case"Safari":a=`Mozilla/5.0 (${s}) ${r.engine} (KHTML, like Gecko) Version/${r.version} Safari/605.1.15`;break;case"Edge":a=`Mozilla/5.0 (${s}) ${r.engine} (KHTML, like Gecko) Chrome/${r.version} Safari/537.36 Edg/${r.version}`;break;case"Opera":a=`Mozilla/5.0 (${s}) ${r.engine} (KHTML, like Gecko) Chrome/${r.version} Safari/537.36 OPR/${r.version}`;break;case"Vivaldi":a=`Mozilla/5.0 (${s}) ${r.engine} (KHTML, like Gecko) Chrome/${r.version} Safari/537.36 Vivaldi/${r.version}`;break;case"Brave":a=`Mozilla/5.0 (${s}) ${r.engine} (KHTML, like Gecko) Chrome/${r.version} Safari/537.36 Brave/${r.version}`;break;case"Chromium":a=`Mozilla/5.0 (${s}) ${r.engine} (KHTML, like Gecko) Chromium/${r.version} Chrome/${r.version} Safari/537.36`;break;case"Yandex":a=`Mozilla/5.0 (${s}) ${r.engine} (KHTML, like Gecko) Chrome/${r.version} YaBrowser/${r.version} Safari/537.36`;break;case"Maxthon":a=`Mozilla/5.0 (${s}) ${r.engine} (KHTML, like Gecko) Chrome/${r.version} Safari/537.36 Maxthon/${r.version}`;break}i.push(a)}return i}exports.CrawlerOptions=f;
+var{RezoQueue:m}=require("../queue/queue.cjs"),{Oxylabs:x}=require("./addon/oxylabs/index.cjs"),b=require("node:path"),g=require("node:os"),{Decodo:y}=require("./addon/decodo/index.cjs");class f{baseUrl;adapter;enableNavigationHistory;sessionId;rejectUnauthorized;userAgent;useRndUserAgent;timeout;maxRedirects;maxRetryAttempts;retryDelay;retryOnStatusCode;forceRevisit;retryWithoutProxyOnStatusCode;retryOnProxyError;maxRetryOnProxyError;allowRevisiting;enableCache;cacheTTL;cacheDir;throwFatalError;debug;maxDepth;maxUrls;maxResponseSize;respectRobotsTxt;followNofollow;autoThrottle;autoThrottleTargetDelay;autoThrottleMinDelay;autoThrottleMaxDelay;maxWaitOn429;alwaysWaitOn429;oxylabs=[];decodo=[];proxies=[];limiters=[];requestHeaders=[];userAgents=A();constructor(e={}){this.baseUrl=e.baseUrl||"",this.adapter=e.adapter??"http",this.enableNavigationHistory=e.enableNavigationHistory??!1,this.sessionId=e.sessionId??`session_${Date.now()}_${Math.random().toString(36).slice(2,8)}`,this.rejectUnauthorized=e.rejectUnauthorized??!0,this.userAgent=e.userAgent,this.useRndUserAgent=e.useRndUserAgent??!1,this.timeout=e.timeout??30000,this.maxRedirects=e.maxRedirects??10,this.maxRetryAttempts=e.maxRetryAttempts??3,this.retryDelay=e.retryDelay??0,this.retryOnStatusCode=e.retryOnStatusCode??[408,429,500,502,503,504],this.forceRevisit=e.forceRevisit??!1,this.retryWithoutProxyOnStatusCode=e.retryWithoutProxyOnStatusCode??[407,403],this.retryOnProxyError=e.retryOnProxyError??!0,this.maxRetryOnProxyError=e.maxRetryOnProxyError??3,this.allowRevisiting=e.allowRevisiting??!1,this.enableCache=e.enableCache??!0,this.cacheTTL=e.cacheTTL??604800000,this.cacheDir=e.cacheDir??b.join(g.tmpdir(),"uiniqhtt_cache"),this.throwFatalError=e.throwFatalError??!1,this.debug=e.debug??!1,this.maxDepth=e.maxDepth??0,this.maxUrls=e.maxUrls??0,this.maxResponseSize=e.maxResponseSize??0,this.respectRobotsTxt=e.respectRobotsTxt??!1,this.followNofollow=e.followNofollow??!1,this.autoThrottle=e.autoThrottle??!0,this.autoThrottleTargetDelay=e.autoThrottleTargetDelay??1000,this.autoThrottleMinDelay=e.autoThrottleMinDelay??100,this.autoThrottleMaxDelay=e.autoThrottleMaxDelay??60000,this.maxWaitOn429=e.maxWaitOn429??1800000,this.alwaysWaitOn429=e.alwaysWaitOn429??!1,this._addHeaders(e.headers),this._addOxylabs(e.oxylabs),this._addDecodo(e.decodo),this._addProxies(e.proxy),this._addLimiters(e.limiter)}getConfiguredDomains(e){return(e==="headers"?this.requestHeaders:e==="limiters"?this.limiters:e==="oxylabs"?this.oxylabs:this.proxies).filter((i)=>i.domain).map((i)=>i.domain).filter((i,n,r)=>r.indexOf(i)===n)}removeDomain(e){return this.requestHeaders=this.requestHeaders.filter((t)=>!t.domain||!this._domainsEqual(t.domain,e)),this.proxies=this.proxies.filter((t)=>!t.domain||!this._domainsEqual(t.domain,e)),this.limiters=this.limiters.filter((t)=>!t.domain||!this._domainsEqual(t.domain,e)),this.oxylabs=this.oxylabs.filter((t)=>!t.domain||!this._domainsEqual(t.domain,e)),this}_domainsEqual(e,t){if(Array.isArray(e)&&Array.isArray(t))return e.length===t.length&&e.every((i,n)=>i===t[n]);return e===t}getConfigurationSummary(){let e=(t)=>({total:t.length,global:t.filter((i)=>i.isGlobal).length,domainSpecific:t.filter((i)=>!i.isGlobal&&i.domain).length});return{headers:e(this.requestHeaders),proxies:e(this.proxies),limiters:e(this.limiters),oxylabs:e(this.oxylabs)}}_addHeaders(e){if(!e||!e.enable)return;for(let t of e.httpHeaders){let{domain:i,isGlobal:n,headers:r}=t;if(!i&&!n)continue;if(r instanceof Headers){let s=Object.fromEntries(r.entries());if(Object.keys(s).length<1)continue;r=s}else if(!r||Object.keys(r).length<1)continue;this.requestHeaders.push({domain:i,isGlobal:n,headers:r})}}_addProxies(e){if(!e||!e.enable)return;for(let t of e.proxies){let{domain:i,isGlobal:n,proxy:r}=t;if(!i&&!n)continue;if(!r||Object.keys(r).length<1)continue;this.proxies.push({domain:i,isGlobal:n,proxy:r})}}_addLimiters(e){if(!e||!e.enable)return;for(let t of e.limiters){let{domain:i,isGlobal:n,options:r}=t;if(!i&&!n)continue;if(!r||Object.keys(r).length<1)continue;this.limiters.push({domain:i,isGlobal:n,pqueue:new m(r)})}}_addOxylabs(e){if(!e||!e.enable)return;for(let t of e.labs){let{domain:i,isGlobal:n,options:r,queueOptions:s}=t;if(!i&&!n)continue;if(!r||Object.keys(r).length<1)continue;this.oxylabs.push({domain:i,isGlobal:n,adaptar:new x(r)})}}_addDecodo(e){if(!e||!e.enable)return;for(let t of e.labs){let{domain:i,isGlobal:n,options:r,queueOptions:s}=t;if(!i&&!n)continue;if(!r||Object.keys(r).length<1)continue;this.decodo.push({domain:i,isGlobal:n,adaptar:new y(r)})}}addHeaders(e){return this._addHeaders({enable:!0,httpHeaders:[e]}),this}addProxy(e){return this._addProxies({enable:!0,proxies:[e]}),this}addLimiter(e){return this._addLimiters({enable:!0,limiters:[e]}),this}addOxylabs(e){return this._addOxylabs({enable:!0,labs:[e]}),this}addDecodo(e){return this._addDecodo({enable:!0,labs:[e]}),this}destroyLimiters(){for(let e of this.limiters)if(e.pqueue&&typeof e.pqueue.destroy==="function")e.pqueue.destroy();this.limiters=[]}clearGlobalConfigs(){if(Array.isArray(this.requestHeaders))this.requestHeaders=this.requestHeaders.filter((e)=>!e.isGlobal);if(Array.isArray(this.oxylabs))this.oxylabs=this.oxylabs.filter((e)=>!e.isGlobal);if(Array.isArray(this.limiters))this.limiters=this.limiters.filter((e)=>!e.isGlobal);if(Array.isArray(this.proxies))this.proxies=this.proxies.filter((e)=>!e.isGlobal);return this}getAdapter(e,t,i,n){if(!this.getDomainName(e))return null;let s=[],a=t==="headers"?this.requestHeaders:t==="limiters"?this.limiters:t==="oxylabs"?this.oxylabs:t==="decodo"?this.decodo:this.proxies;for(let o=0;o<a.length;o++)if(this._hasDomain(e,a[o].domain))s.push(o);if(s.length){let o=n?s[this.rnd(0,s.length-1)]:s[0];return t==="headers"?this.requestHeaders[o].headers:t==="limiters"?this.limiters[o].pqueue:t==="oxylabs"?this.oxylabs[o].adaptar:t==="decodo"?this.decodo[o].adaptar:this.proxies[o].proxy}s.length=0;for(let o=0;o<a.length;o++)s.push(o);if(s.length){let o=n?s[this.rnd(0,s.length-1)]:s[0];if(a[o].isGlobal&&i)return t==="headers"?this.requestHeaders[o].headers:t==="limiters"?this.limiters[o].pqueue:t==="oxylabs"?this.oxylabs[o].adaptar:t==="decodo"?this.decodo[o].adaptar:this.proxies[o].proxy}return null}rnd(e=0,t=Number.MAX_VALUE){return Math.floor(Math.random()*(t-e+1))+e}hasDomain(e,t,i){if(!this.getDomainName(e))return!1;let r=t==="headers"?this.requestHeaders:t==="limiters"?this.limiters:t==="oxylabs"?this.oxylabs:t==="decodo"?this.decodo:this.proxies;for(let s=0;s<r.length;s++)if(this._hasDomain(e,r[s].domain))return!0;if(i){for(let s=0;s<r.length;s++)if(r[s].isGlobal)return!0}return!1}pickHeaders(e,t,i,n){let r=this.getAdapter(e,"headers",t),s=new Headers(r??{});if(i&&i instanceof Headers)for(let[a,o]of Object.entries(i.entries()))s.set(a,o);else if(i&&typeof i==="object"){for(let[a,o]of Object.entries(i))if(typeof o==="string")s.set(a,o)}if(n)s.set("user-agent",this.getRandomUserAgent());return Object.fromEntries(s.entries())}_hasDomain(e,t){if(!t)return!1;let i=this.getDomainName(e);if(!i)return!1;let n=(s)=>{return/[\^\$\*\+\?\{\}\[\]\(\)\|\\]/.test(s)||s.startsWith("/")||s.includes(".*")||s.includes(".+")},r=(s)=>{if(s instanceof RegExp)return s.test(i)||s.test(e);let a=s.toString().trim();if(i.toLowerCase()===a.toLowerCase())return!0;if(a.includes("*")){let l=a.replace(/[.*+?^${}()|[\]\\]/g,"\\$&").replace(/\\\*/g,".*"),h=new RegExp(`^${l}$`,"i");return h.test(i)||h.test(e)}if(n(a))try{let l=a,h="i",u=a.match(/^\/(.*)\/(\w*)$/);if(u)l=u[1],h=u[2]||"i";let c=new RegExp(l,h);return c.test(i)||c.test(e)}catch(l){return i.toLowerCase().includes(a.toLowerCase())}let o=i.toLowerCase(),d=a.toLowerCase();return o===d||o.endsWith("."+d)||d.endsWith("."+o)};if(Array.isArray(t)){for(let s of t)if(r(s))return!0;return!1}return r(t)}getDomainName(e){if(this.isValidUrl(e))return new URL(e).hostname;else if(this.isHostName(e))return e;return null}isHostName(e){if(!e)return!1;if(e.length>255)return!1;let t=/^(?:(?:[a-zA-Z0-9](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?\.)+ [a-zA-Z]{2,})$/;return e=e.trim().toLowerCase(),t.test(e)&&!e.startsWith("-")&&!e.endsWith("-")}isValidUrl(e){if(!e)return!1;e=e.trim();try{let t=new URL(e);if(!t.protocol||!["http:","https:"].includes(t.protocol.toLowerCase()))return!1;if(!t.hostname)return!1;if(!/^(?:(?:[a-zA-Z0-9](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?\.)+[a-zA-Z]{2,})$/.test(t.hostname))return!1;return!0}catch{return!1}}getRandomUserAgent(){return this.userAgents[Math.floor(Math.random()*this.userAgents.length)]}}function A(){let e=[{name:"Chrome",version:"91.0.4472.124",engine:"AppleWebKit/537.36"},{name:"Firefox",version:"89.0",engine:"Gecko/20100101"},{name:"Safari",version:"14.1.1",engine:"AppleWebKit/605.1.15"},{name:"Edge",version:"91.0.864.59",engine:"AppleWebKit/537.36"},{name:"Opera",version:"77.0.4054.277",engine:"AppleWebKit/537.36"},{name:"Vivaldi",version:"3.8.2259.42",engine:"AppleWebKit/537.36"},{name:"Brave",version:"1.26.74",engine:"AppleWebKit/537.36"},{name:"Chromium",version:"91.0.4472.101",engine:"AppleWebKit/537.36"},{name:"Yandex",version:"21.5.3.742",engine:"AppleWebKit/537.36"},{name:"Maxthon",version:"5.3.8.2000",engine:"AppleWebKit/537.36"}],t=["Windows NT 10.0","Windows NT 6.1","Macintosh; Intel Mac OS X 10_15_7","Macintosh; Intel Mac OS X 11_4_0","X11; Linux x86_64","X11; Ubuntu; Linux x86_64"],i=[];for(let n=0;n<200;n++){let r=e[Math.floor(Math.random()*e.length)],s=t[Math.floor(Math.random()*t.length)],a="";switch(r.name){case"Chrome":a=`Mozilla/5.0 (${s}) ${r.engine} (KHTML, like Gecko) Chrome/${r.version} Safari/537.36`;break;case"Firefox":a=`Mozilla/5.0 (${s}; rv:${r.version}) ${r.engine} Firefox/${r.version}`;break;case"Safari":a=`Mozilla/5.0 (${s}) ${r.engine} (KHTML, like Gecko) Version/${r.version} Safari/605.1.15`;break;case"Edge":a=`Mozilla/5.0 (${s}) ${r.engine} (KHTML, like Gecko) Chrome/${r.version} Safari/537.36 Edg/${r.version}`;break;case"Opera":a=`Mozilla/5.0 (${s}) ${r.engine} (KHTML, like Gecko) Chrome/${r.version} Safari/537.36 OPR/${r.version}`;break;case"Vivaldi":a=`Mozilla/5.0 (${s}) ${r.engine} (KHTML, like Gecko) Chrome/${r.version} Safari/537.36 Vivaldi/${r.version}`;break;case"Brave":a=`Mozilla/5.0 (${s}) ${r.engine} (KHTML, like Gecko) Chrome/${r.version} Safari/537.36 Brave/${r.version}`;break;case"Chromium":a=`Mozilla/5.0 (${s}) ${r.engine} (KHTML, like Gecko) Chromium/${r.version} Chrome/${r.version} Safari/537.36`;break;case"Yandex":a=`Mozilla/5.0 (${s}) ${r.engine} (KHTML, like Gecko) Chrome/${r.version} YaBrowser/${r.version} Safari/537.36`;break;case"Maxthon":a=`Mozilla/5.0 (${s}) ${r.engine} (KHTML, like Gecko) Chrome/${r.version} Safari/537.36 Maxthon/${r.version}`;break}i.push(a)}return i}exports.CrawlerOptions=f;

package/dist/crawler/crawler-options.js

@@ -1 +1 @@
-import{RezoQueue as m}from"../queue/queue.js";import{Oxylabs as f}from"./addon/oxylabs/index.js";import x from"node:path";import b from"node:os";import{Decodo as g}from"./addon/decodo/index.js";class y{baseUrl;adapter;enableNavigationHistory;sessionId;rejectUnauthorized;userAgent;useRndUserAgent;timeout;maxRedirects;maxRetryAttempts;retryDelay;retryOnStatusCode;forceRevisit;retryWithoutProxyOnStatusCode;retryOnProxyError;maxRetryOnProxyError;allowRevisiting;enableCache;cacheTTL;cacheDir;throwFatalError;debug;maxDepth;maxUrls;maxResponseSize;respectRobotsTxt;followNofollow;autoThrottle;autoThrottleTargetDelay;autoThrottleMinDelay;autoThrottleMaxDelay;maxWaitOn429;alwaysWaitOn429;oxylabs=[];decodo=[];proxies=[];limiters=[];requestHeaders=[];userAgents=A();constructor(e={}){this.baseUrl=e.baseUrl||"",this.adapter=e.adapter??"http",this.enableNavigationHistory=e.enableNavigationHistory??!1,this.sessionId=e.sessionId??`session_${Date.now()}_${Math.random().toString(36).slice(2,8)}`,this.rejectUnauthorized=e.rejectUnauthorized??!0,this.userAgent=e.userAgent,this.useRndUserAgent=e.useRndUserAgent??!1,this.timeout=e.timeout??30000,this.maxRedirects=e.maxRedirects??10,this.maxRetryAttempts=e.maxRetryAttempts??3,this.retryDelay=e.retryDelay??0,this.retryOnStatusCode=e.retryOnStatusCode??[408,429,500,502,503,504],this.forceRevisit=e.forceRevisit??!1,this.retryWithoutProxyOnStatusCode=e.retryWithoutProxyOnStatusCode??[407,403],this.retryOnProxyError=e.retryOnProxyError??!0,this.maxRetryOnProxyError=e.maxRetryOnProxyError??3,this.allowRevisiting=e.allowRevisiting??!1,this.enableCache=e.enableCache??!0,this.cacheTTL=e.cacheTTL??604800000,this.cacheDir=e.cacheDir??x.join(b.tmpdir(),"uiniqhtt_cache"),this.throwFatalError=e.throwFatalError??!1,this.debug=e.debug??!1,this.maxDepth=e.maxDepth??0,this.maxUrls=e.maxUrls??0,this.maxResponseSize=e.maxResponseSize??0,this.respectRobotsTxt=e.respectRobotsTxt??!1,this.followNofollow=e.followNofollow??!1,this.autoThrottle=e.autoThrottle??!0,this.autoThrottleTargetDelay=e.autoThrottleTargetDelay??1000,this.autoThrottleMinDelay=e.autoThrottleMinDelay??100,this.autoThrottleMaxDelay=e.autoThrottleMaxDelay??60000,this.maxWaitOn429=e.maxWaitOn429??1800000,this.alwaysWaitOn429=e.alwaysWaitOn429??!1,this._addHeaders(e.headers),this._addOxylabs(e.oxylabs),this._addDecodo(e.decodo),this._addProxies(e.proxy),this._addLimiters(e.limiter)}getConfiguredDomains(e){return(e==="headers"?this.requestHeaders:e==="limiters"?this.limiters:e==="oxylabs"?this.oxylabs:this.proxies).filter((s)=>s.domain).map((s)=>s.domain).filter((s,n,r)=>r.indexOf(s)===n)}removeDomain(e){return this.requestHeaders=this.requestHeaders.filter((t)=>!t.domain||!this._domainsEqual(t.domain,e)),this.proxies=this.proxies.filter((t)=>!t.domain||!this._domainsEqual(t.domain,e)),this.limiters=this.limiters.filter((t)=>!t.domain||!this._domainsEqual(t.domain,e)),this.oxylabs=this.oxylabs.filter((t)=>!t.domain||!this._domainsEqual(t.domain,e)),this}_domainsEqual(e,t){if(Array.isArray(e)&&Array.isArray(t))return e.length===t.length&&e.every((s,n)=>s===t[n]);return e===t}getConfigurationSummary(){let e=(t)=>({total:t.length,global:t.filter((s)=>s.isGlobal).length,domainSpecific:t.filter((s)=>!s.isGlobal&&s.domain).length});return{headers:e(this.requestHeaders),proxies:e(this.proxies),limiters:e(this.limiters),oxylabs:e(this.oxylabs)}}_addHeaders(e){if(!e||!e.enable)return;for(let t of e.httpHeaders){let{domain:s,isGlobal:n,headers:r}=t;if(!s&&!n)continue;if(r instanceof Headers){let i=Object.fromEntries(r.entries());if(Object.keys(i).length<1)continue;r=i}else if(!r||Object.keys(r).length<1)continue;this.requestHeaders.push({domain:s,isGlobal:n,headers:r})}}_addProxies(e){if(!e||!e.enable)return;for(let t of e.proxies){let{domain:s,isGlobal:n,proxy:r}=t;if(!s&&!n)continue;if(!r||Object.keys(r).length<1)continue;this.proxies.push({domain:s,isGlobal:n,proxy:r})}}_addLimiters(e){if(!e||!e.enable)return;for(let t of e.limiters){let{domain:s,isGlobal:n,options:r}=t;if(!s&&!n)continue;if(!r||Object.keys(r).length<1)continue;this.limiters.push({domain:s,isGlobal:n,pqueue:new m(r)})}}_addOxylabs(e){if(!e||!e.enable)return;for(let t of e.labs){let{domain:s,isGlobal:n,options:r,queueOptions:i}=t;if(!s&&!n)continue;if(!r||Object.keys(r).length<1)continue;this.oxylabs.push({domain:s,isGlobal:n,adaptar:new f(r)})}}_addDecodo(e){if(!e||!e.enable)return;for(let t of e.labs){let{domain:s,isGlobal:n,options:r,queueOptions:i}=t;if(!s&&!n)continue;if(!r||Object.keys(r).length<1)continue;this.decodo.push({domain:s,isGlobal:n,adaptar:new g(r)})}}addHeaders(e){return this._addHeaders({enable:!0,httpHeaders:[e]}),this}addProxy(e){return this._addProxies({enable:!0,proxies:[e]}),this}addLimiter(e){return this._addLimiters({enable:!0,limiters:[e]}),this}addOxylabs(e){return this._addOxylabs({enable:!0,labs:[e]}),this}addDecodo(e){return this._addDecodo({enable:!0,labs:[e]}),this}clearGlobalConfigs(){if(Array.isArray(this.requestHeaders))this.requestHeaders=this.requestHeaders.filter((e)=>!e.isGlobal);if(Array.isArray(this.oxylabs))this.oxylabs=this.oxylabs.filter((e)=>!e.isGlobal);if(Array.isArray(this.limiters))this.limiters=this.limiters.filter((e)=>!e.isGlobal);if(Array.isArray(this.proxies))this.proxies=this.proxies.filter((e)=>!e.isGlobal);return this}getAdapter(e,t,s,n){if(!this.getDomainName(e))return null;let i=[],a=t==="headers"?this.requestHeaders:t==="limiters"?this.limiters:t==="oxylabs"?this.oxylabs:t==="decodo"?this.decodo:this.proxies;for(let o=0;o<a.length;o++)if(this._hasDomain(e,a[o].domain))i.push(o);if(i.length){let o=n?i[this.rnd(0,i.length-1)]:i[0];return t==="headers"?this.requestHeaders[o].headers:t==="limiters"?this.limiters[o].pqueue:t==="oxylabs"?this.oxylabs[o].adaptar:t==="decodo"?this.decodo[o].adaptar:this.proxies[o].proxy}i.length=0;for(let o=0;o<a.length;o++)i.push(o);if(i.length){let o=n?i[this.rnd(0,i.length-1)]:i[0];if(a[o].isGlobal&&s)return t==="headers"?this.requestHeaders[o].headers:t==="limiters"?this.limiters[o].pqueue:t==="oxylabs"?this.oxylabs[o].adaptar:t==="decodo"?this.decodo[o].adaptar:this.proxies[o].proxy}return null}rnd(e=0,t=Number.MAX_VALUE){return Math.floor(Math.random()*(t-e+1))+e}hasDomain(e,t,s){if(!this.getDomainName(e))return!1;let r=t==="headers"?this.requestHeaders:t==="limiters"?this.limiters:t==="oxylabs"?this.oxylabs:t==="decodo"?this.decodo:this.proxies;for(let i=0;i<r.length;i++)if(this._hasDomain(e,r[i].domain))return!0;if(s){for(let i=0;i<r.length;i++)if(r[i].isGlobal)return!0}return!1}pickHeaders(e,t,s,n){let r=this.getAdapter(e,"headers",t),i=new Headers(r??{});if(s&&s instanceof Headers)for(let[a,o]of Object.entries(s.entries()))i.set(a,o);else if(s&&typeof s==="object"){for(let[a,o]of Object.entries(s))if(typeof o==="string")i.set(a,o)}if(n)i.set("user-agent",this.getRandomUserAgent());return Object.fromEntries(i.entries())}_hasDomain(e,t){if(!t)return!1;let s=this.getDomainName(e);if(!s)return!1;let n=(i)=>{return/[\^\$\*\+\?\{\}\[\]\(\)\|\\]/.test(i)||i.startsWith("/")||i.includes(".*")||i.includes(".+")},r=(i)=>{if(i instanceof RegExp)return i.test(s)||i.test(e);let a=i.toString().trim();if(s.toLowerCase()===a.toLowerCase())return!0;if(a.includes("*")){let l=a.replace(/[.*+?^${}()|[\]\\]/g,"\\$&").replace(/\\\*/g,".*"),h=new RegExp(`^${l}$`,"i");return h.test(s)||h.test(e)}if(n(a))try{let l=a,h="i",u=a.match(/^\/(.*)\/(\w*)$/);if(u)l=u[1],h=u[2]||"i";let c=new RegExp(l,h);return c.test(s)||c.test(e)}catch(l){return s.toLowerCase().includes(a.toLowerCase())}let o=s.toLowerCase(),d=a.toLowerCase();return o===d||o.endsWith("."+d)||d.endsWith("."+o)};if(Array.isArray(t)){for(let i of t)if(r(i))return!0;return!1}return r(t)}getDomainName(e){if(this.isValidUrl(e))return new URL(e).hostname;else if(this.isHostName(e))return e;return null}isHostName(e){if(!e)return!1;if(e.length>255)return!1;let t=/^(?:(?:[a-zA-Z0-9](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?\.)+ [a-zA-Z]{2,})$/;return e=e.trim().toLowerCase(),t.test(e)&&!e.startsWith("-")&&!e.endsWith("-")}isValidUrl(e){if(!e)return!1;e=e.trim();try{let t=new URL(e);if(!t.protocol||!["http:","https:"].includes(t.protocol.toLowerCase()))return!1;if(!t.hostname)return!1;if(!/^(?:(?:[a-zA-Z0-9](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?\.)+[a-zA-Z]{2,})$/.test(t.hostname))return!1;return!0}catch{return!1}}getRandomUserAgent(){return this.userAgents[Math.floor(Math.random()*this.userAgents.length)]}}function A(){let e=[{name:"Chrome",version:"91.0.4472.124",engine:"AppleWebKit/537.36"},{name:"Firefox",version:"89.0",engine:"Gecko/20100101"},{name:"Safari",version:"14.1.1",engine:"AppleWebKit/605.1.15"},{name:"Edge",version:"91.0.864.59",engine:"AppleWebKit/537.36"},{name:"Opera",version:"77.0.4054.277",engine:"AppleWebKit/537.36"},{name:"Vivaldi",version:"3.8.2259.42",engine:"AppleWebKit/537.36"},{name:"Brave",version:"1.26.74",engine:"AppleWebKit/537.36"},{name:"Chromium",version:"91.0.4472.101",engine:"AppleWebKit/537.36"},{name:"Yandex",version:"21.5.3.742",engine:"AppleWebKit/537.36"},{name:"Maxthon",version:"5.3.8.2000",engine:"AppleWebKit/537.36"}],t=["Windows NT 10.0","Windows NT 6.1","Macintosh; Intel Mac OS X 10_15_7","Macintosh; Intel Mac OS X 11_4_0","X11; Linux x86_64","X11; Ubuntu; Linux x86_64"],s=[];for(let n=0;n<200;n++){let r=e[Math.floor(Math.random()*e.length)],i=t[Math.floor(Math.random()*t.length)],a="";switch(r.name){case"Chrome":a=`Mozilla/5.0 (${i}) ${r.engine} (KHTML, like Gecko) Chrome/${r.version} Safari/537.36`;break;case"Firefox":a=`Mozilla/5.0 (${i}; rv:${r.version}) ${r.engine} Firefox/${r.version}`;break;case"Safari":a=`Mozilla/5.0 (${i}) ${r.engine} (KHTML, like Gecko) Version/${r.version} Safari/605.1.15`;break;case"Edge":a=`Mozilla/5.0 (${i}) ${r.engine} (KHTML, like Gecko) Chrome/${r.version} Safari/537.36 Edg/${r.version}`;break;case"Opera":a=`Mozilla/5.0 (${i}) ${r.engine} (KHTML, like Gecko) Chrome/${r.version} Safari/537.36 OPR/${r.version}`;break;case"Vivaldi":a=`Mozilla/5.0 (${i}) ${r.engine} (KHTML, like Gecko) Chrome/${r.version} Safari/537.36 Vivaldi/${r.version}`;break;case"Brave":a=`Mozilla/5.0 (${i}) ${r.engine} (KHTML, like Gecko) Chrome/${r.version} Safari/537.36 Brave/${r.version}`;break;case"Chromium":a=`Mozilla/5.0 (${i}) ${r.engine} (KHTML, like Gecko) Chromium/${r.version} Chrome/${r.version} Safari/537.36`;break;case"Yandex":a=`Mozilla/5.0 (${i}) ${r.engine} (KHTML, like Gecko) Chrome/${r.version} YaBrowser/${r.version} Safari/537.36`;break;case"Maxthon":a=`Mozilla/5.0 (${i}) ${r.engine} (KHTML, like Gecko) Chrome/${r.version} Safari/537.36 Maxthon/${r.version}`;break}s.push(a)}return s}export{y as CrawlerOptions};
+import{RezoQueue as m}from"../queue/queue.js";import{Oxylabs as f}from"./addon/oxylabs/index.js";import x from"node:path";import b from"node:os";import{Decodo as g}from"./addon/decodo/index.js";class y{baseUrl;adapter;enableNavigationHistory;sessionId;rejectUnauthorized;userAgent;useRndUserAgent;timeout;maxRedirects;maxRetryAttempts;retryDelay;retryOnStatusCode;forceRevisit;retryWithoutProxyOnStatusCode;retryOnProxyError;maxRetryOnProxyError;allowRevisiting;enableCache;cacheTTL;cacheDir;throwFatalError;debug;maxDepth;maxUrls;maxResponseSize;respectRobotsTxt;followNofollow;autoThrottle;autoThrottleTargetDelay;autoThrottleMinDelay;autoThrottleMaxDelay;maxWaitOn429;alwaysWaitOn429;oxylabs=[];decodo=[];proxies=[];limiters=[];requestHeaders=[];userAgents=A();constructor(e={}){this.baseUrl=e.baseUrl||"",this.adapter=e.adapter??"http",this.enableNavigationHistory=e.enableNavigationHistory??!1,this.sessionId=e.sessionId??`session_${Date.now()}_${Math.random().toString(36).slice(2,8)}`,this.rejectUnauthorized=e.rejectUnauthorized??!0,this.userAgent=e.userAgent,this.useRndUserAgent=e.useRndUserAgent??!1,this.timeout=e.timeout??30000,this.maxRedirects=e.maxRedirects??10,this.maxRetryAttempts=e.maxRetryAttempts??3,this.retryDelay=e.retryDelay??0,this.retryOnStatusCode=e.retryOnStatusCode??[408,429,500,502,503,504],this.forceRevisit=e.forceRevisit??!1,this.retryWithoutProxyOnStatusCode=e.retryWithoutProxyOnStatusCode??[407,403],this.retryOnProxyError=e.retryOnProxyError??!0,this.maxRetryOnProxyError=e.maxRetryOnProxyError??3,this.allowRevisiting=e.allowRevisiting??!1,this.enableCache=e.enableCache??!0,this.cacheTTL=e.cacheTTL??604800000,this.cacheDir=e.cacheDir??x.join(b.tmpdir(),"uiniqhtt_cache"),this.throwFatalError=e.throwFatalError??!1,this.debug=e.debug??!1,this.maxDepth=e.maxDepth??0,this.maxUrls=e.maxUrls??0,this.maxResponseSize=e.maxResponseSize??0,this.respectRobotsTxt=e.respectRobotsTxt??!1,this.followNofollow=e.followNofollow??!1,this.autoThrottle=e.autoThrottle??!0,this.autoThrottleTargetDelay=e.autoThrottleTargetDelay??1000,this.autoThrottleMinDelay=e.autoThrottleMinDelay??100,this.autoThrottleMaxDelay=e.autoThrottleMaxDelay??60000,this.maxWaitOn429=e.maxWaitOn429??1800000,this.alwaysWaitOn429=e.alwaysWaitOn429??!1,this._addHeaders(e.headers),this._addOxylabs(e.oxylabs),this._addDecodo(e.decodo),this._addProxies(e.proxy),this._addLimiters(e.limiter)}getConfiguredDomains(e){return(e==="headers"?this.requestHeaders:e==="limiters"?this.limiters:e==="oxylabs"?this.oxylabs:this.proxies).filter((s)=>s.domain).map((s)=>s.domain).filter((s,n,r)=>r.indexOf(s)===n)}removeDomain(e){return this.requestHeaders=this.requestHeaders.filter((t)=>!t.domain||!this._domainsEqual(t.domain,e)),this.proxies=this.proxies.filter((t)=>!t.domain||!this._domainsEqual(t.domain,e)),this.limiters=this.limiters.filter((t)=>!t.domain||!this._domainsEqual(t.domain,e)),this.oxylabs=this.oxylabs.filter((t)=>!t.domain||!this._domainsEqual(t.domain,e)),this}_domainsEqual(e,t){if(Array.isArray(e)&&Array.isArray(t))return e.length===t.length&&e.every((s,n)=>s===t[n]);return e===t}getConfigurationSummary(){let e=(t)=>({total:t.length,global:t.filter((s)=>s.isGlobal).length,domainSpecific:t.filter((s)=>!s.isGlobal&&s.domain).length});return{headers:e(this.requestHeaders),proxies:e(this.proxies),limiters:e(this.limiters),oxylabs:e(this.oxylabs)}}_addHeaders(e){if(!e||!e.enable)return;for(let t of e.httpHeaders){let{domain:s,isGlobal:n,headers:r}=t;if(!s&&!n)continue;if(r instanceof Headers){let i=Object.fromEntries(r.entries());if(Object.keys(i).length<1)continue;r=i}else if(!r||Object.keys(r).length<1)continue;this.requestHeaders.push({domain:s,isGlobal:n,headers:r})}}_addProxies(e){if(!e||!e.enable)return;for(let t of e.proxies){let{domain:s,isGlobal:n,proxy:r}=t;if(!s&&!n)continue;if(!r||Object.keys(r).length<1)continue;this.proxies.push({domain:s,isGlobal:n,proxy:r})}}_addLimiters(e){if(!e||!e.enable)return;for(let t of e.limiters){let{domain:s,isGlobal:n,options:r}=t;if(!s&&!n)continue;if(!r||Object.keys(r).length<1)continue;this.limiters.push({domain:s,isGlobal:n,pqueue:new m(r)})}}_addOxylabs(e){if(!e||!e.enable)return;for(let t of e.labs){let{domain:s,isGlobal:n,options:r,queueOptions:i}=t;if(!s&&!n)continue;if(!r||Object.keys(r).length<1)continue;this.oxylabs.push({domain:s,isGlobal:n,adaptar:new f(r)})}}_addDecodo(e){if(!e||!e.enable)return;for(let t of e.labs){let{domain:s,isGlobal:n,options:r,queueOptions:i}=t;if(!s&&!n)continue;if(!r||Object.keys(r).length<1)continue;this.decodo.push({domain:s,isGlobal:n,adaptar:new g(r)})}}addHeaders(e){return this._addHeaders({enable:!0,httpHeaders:[e]}),this}addProxy(e){return this._addProxies({enable:!0,proxies:[e]}),this}addLimiter(e){return this._addLimiters({enable:!0,limiters:[e]}),this}addOxylabs(e){return this._addOxylabs({enable:!0,labs:[e]}),this}addDecodo(e){return this._addDecodo({enable:!0,labs:[e]}),this}destroyLimiters(){for(let e of this.limiters)if(e.pqueue&&typeof e.pqueue.destroy==="function")e.pqueue.destroy();this.limiters=[]}clearGlobalConfigs(){if(Array.isArray(this.requestHeaders))this.requestHeaders=this.requestHeaders.filter((e)=>!e.isGlobal);if(Array.isArray(this.oxylabs))this.oxylabs=this.oxylabs.filter((e)=>!e.isGlobal);if(Array.isArray(this.limiters))this.limiters=this.limiters.filter((e)=>!e.isGlobal);if(Array.isArray(this.proxies))this.proxies=this.proxies.filter((e)=>!e.isGlobal);return this}getAdapter(e,t,s,n){if(!this.getDomainName(e))return null;let i=[],a=t==="headers"?this.requestHeaders:t==="limiters"?this.limiters:t==="oxylabs"?this.oxylabs:t==="decodo"?this.decodo:this.proxies;for(let o=0;o<a.length;o++)if(this._hasDomain(e,a[o].domain))i.push(o);if(i.length){let o=n?i[this.rnd(0,i.length-1)]:i[0];return t==="headers"?this.requestHeaders[o].headers:t==="limiters"?this.limiters[o].pqueue:t==="oxylabs"?this.oxylabs[o].adaptar:t==="decodo"?this.decodo[o].adaptar:this.proxies[o].proxy}i.length=0;for(let o=0;o<a.length;o++)i.push(o);if(i.length){let o=n?i[this.rnd(0,i.length-1)]:i[0];if(a[o].isGlobal&&s)return t==="headers"?this.requestHeaders[o].headers:t==="limiters"?this.limiters[o].pqueue:t==="oxylabs"?this.oxylabs[o].adaptar:t==="decodo"?this.decodo[o].adaptar:this.proxies[o].proxy}return null}rnd(e=0,t=Number.MAX_VALUE){return Math.floor(Math.random()*(t-e+1))+e}hasDomain(e,t,s){if(!this.getDomainName(e))return!1;let r=t==="headers"?this.requestHeaders:t==="limiters"?this.limiters:t==="oxylabs"?this.oxylabs:t==="decodo"?this.decodo:this.proxies;for(let i=0;i<r.length;i++)if(this._hasDomain(e,r[i].domain))return!0;if(s){for(let i=0;i<r.length;i++)if(r[i].isGlobal)return!0}return!1}pickHeaders(e,t,s,n){let r=this.getAdapter(e,"headers",t),i=new Headers(r??{});if(s&&s instanceof Headers)for(let[a,o]of Object.entries(s.entries()))i.set(a,o);else if(s&&typeof s==="object"){for(let[a,o]of Object.entries(s))if(typeof o==="string")i.set(a,o)}if(n)i.set("user-agent",this.getRandomUserAgent());return Object.fromEntries(i.entries())}_hasDomain(e,t){if(!t)return!1;let s=this.getDomainName(e);if(!s)return!1;let n=(i)=>{return/[\^\$\*\+\?\{\}\[\]\(\)\|\\]/.test(i)||i.startsWith("/")||i.includes(".*")||i.includes(".+")},r=(i)=>{if(i instanceof RegExp)return i.test(s)||i.test(e);let a=i.toString().trim();if(s.toLowerCase()===a.toLowerCase())return!0;if(a.includes("*")){let l=a.replace(/[.*+?^${}()|[\]\\]/g,"\\$&").replace(/\\\*/g,".*"),h=new RegExp(`^${l}$`,"i");return h.test(s)||h.test(e)}if(n(a))try{let l=a,h="i",u=a.match(/^\/(.*)\/(\w*)$/);if(u)l=u[1],h=u[2]||"i";let c=new RegExp(l,h);return c.test(s)||c.test(e)}catch(l){return s.toLowerCase().includes(a.toLowerCase())}let o=s.toLowerCase(),d=a.toLowerCase();return o===d||o.endsWith("."+d)||d.endsWith("."+o)};if(Array.isArray(t)){for(let i of t)if(r(i))return!0;return!1}return r(t)}getDomainName(e){if(this.isValidUrl(e))return new URL(e).hostname;else if(this.isHostName(e))return e;return null}isHostName(e){if(!e)return!1;if(e.length>255)return!1;let t=/^(?:(?:[a-zA-Z0-9](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?\.)+ [a-zA-Z]{2,})$/;return e=e.trim().toLowerCase(),t.test(e)&&!e.startsWith("-")&&!e.endsWith("-")}isValidUrl(e){if(!e)return!1;e=e.trim();try{let t=new URL(e);if(!t.protocol||!["http:","https:"].includes(t.protocol.toLowerCase()))return!1;if(!t.hostname)return!1;if(!/^(?:(?:[a-zA-Z0-9](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?\.)+[a-zA-Z]{2,})$/.test(t.hostname))return!1;return!0}catch{return!1}}getRandomUserAgent(){return this.userAgents[Math.floor(Math.random()*this.userAgents.length)]}}function A(){let e=[{name:"Chrome",version:"91.0.4472.124",engine:"AppleWebKit/537.36"},{name:"Firefox",version:"89.0",engine:"Gecko/20100101"},{name:"Safari",version:"14.1.1",engine:"AppleWebKit/605.1.15"},{name:"Edge",version:"91.0.864.59",engine:"AppleWebKit/537.36"},{name:"Opera",version:"77.0.4054.277",engine:"AppleWebKit/537.36"},{name:"Vivaldi",version:"3.8.2259.42",engine:"AppleWebKit/537.36"},{name:"Brave",version:"1.26.74",engine:"AppleWebKit/537.36"},{name:"Chromium",version:"91.0.4472.101",engine:"AppleWebKit/537.36"},{name:"Yandex",version:"21.5.3.742",engine:"AppleWebKit/537.36"},{name:"Maxthon",version:"5.3.8.2000",engine:"AppleWebKit/537.36"}],t=["Windows NT 10.0","Windows NT 6.1","Macintosh; Intel Mac OS X 10_15_7","Macintosh; Intel Mac OS X 11_4_0","X11; Linux x86_64","X11; Ubuntu; Linux x86_64"],s=[];for(let n=0;n<200;n++){let r=e[Math.floor(Math.random()*e.length)],i=t[Math.floor(Math.random()*t.length)],a="";switch(r.name){case"Chrome":a=`Mozilla/5.0 (${i}) ${r.engine} (KHTML, like Gecko) Chrome/${r.version} Safari/537.36`;break;case"Firefox":a=`Mozilla/5.0 (${i}; rv:${r.version}) ${r.engine} Firefox/${r.version}`;break;case"Safari":a=`Mozilla/5.0 (${i}) ${r.engine} (KHTML, like Gecko) Version/${r.version} Safari/605.1.15`;break;case"Edge":a=`Mozilla/5.0 (${i}) ${r.engine} (KHTML, like Gecko) Chrome/${r.version} Safari/537.36 Edg/${r.version}`;break;case"Opera":a=`Mozilla/5.0 (${i}) ${r.engine} (KHTML, like Gecko) Chrome/${r.version} Safari/537.36 OPR/${r.version}`;break;case"Vivaldi":a=`Mozilla/5.0 (${i}) ${r.engine} (KHTML, like Gecko) Chrome/${r.version} Safari/537.36 Vivaldi/${r.version}`;break;case"Brave":a=`Mozilla/5.0 (${i}) ${r.engine} (KHTML, like Gecko) Chrome/${r.version} Safari/537.36 Brave/${r.version}`;break;case"Chromium":a=`Mozilla/5.0 (${i}) ${r.engine} (KHTML, like Gecko) Chromium/${r.version} Chrome/${r.version} Safari/537.36`;break;case"Yandex":a=`Mozilla/5.0 (${i}) ${r.engine} (KHTML, like Gecko) Chrome/${r.version} YaBrowser/${r.version} Safari/537.36`;break;case"Maxthon":a=`Mozilla/5.0 (${i}) ${r.engine} (KHTML, like Gecko) Chrome/${r.version} Safari/537.36 Maxthon/${r.version}`;break}s.push(a)}return s}export{y as CrawlerOptions};

package/dist/crawler/crawler.cjs

@@ -57,6 +57,8 @@ class Crawler {
   adapterExecutor = null;
   adapterType;
   pendingExecutions = new Set;
+  isDestroyed = false;
+  queueOptions = { concurrency: 1000 };
   robotsTxt;
   domainResponseTimes = new Map;
   domainCurrentDelay = new Map;
@@ -159,6 +161,16 @@ class Crawler {
       }
     }
   }
+  ensureActive() {
+    if (!this.isDestroyed)
+      return;
+    this.queue = new RezoQueue(this.queueOptions);
+    this.pendingExecutions.clear();
+    this.isDestroyed = false;
+    if (this.config.debug) {
+      console.log("[Crawler] Restored from destroyed state");
+    }
+  }
   async initializeNavigationHistory(navHistoryDir) {
     try {
       const history = await NavigationHistory.create({
@@ -233,6 +245,7 @@ class Crawler {
     return this.config.sessionId;
   }
   async resume(sessionId) {
+    this.ensureActive();
     if (!this.config.enableNavigationHistory) {
       throw new Error("Navigation history is not enabled. Set enableNavigationHistory: true in options.");
     }
@@ -799,6 +812,7 @@ class Crawler {
     return url;
   }
   visit(url, options) {
+    this.ensureActive();
     if (this.config.baseUrl)
       url = new URL(url, this.config.baseUrl).href;
     if (options?.params && (options.useOxylabsScraperAi || this.config.hasDomain(url, "oxylabs"))) {
@@ -1077,7 +1091,11 @@ class Crawler {
     } catch {}
   }
   async destroy() {
+    if (this.isDestroyed)
+      return;
+    this.isDestroyed = true;
     this.queue.destroy();
+    this.config.destroyLimiters();
     this.events.length = 0;
     this.jsonEvents.length = 0;
     this.errorEvents.length = 0;

package/dist/crawler/crawler.js

@@ -57,6 +57,8 @@ export class Crawler {
   adapterExecutor = null;
   adapterType;
   pendingExecutions = new Set;
+  isDestroyed = false;
+  queueOptions = { concurrency: 1000 };
   robotsTxt;
   domainResponseTimes = new Map;
   domainCurrentDelay = new Map;
@@ -159,6 +161,16 @@ export class Crawler {
       }
     }
   }
+  ensureActive() {
+    if (!this.isDestroyed)
+      return;
+    this.queue = new RezoQueue(this.queueOptions);
+    this.pendingExecutions.clear();
+    this.isDestroyed = false;
+    if (this.config.debug) {
+      console.log("[Crawler] Restored from destroyed state");
+    }
+  }
   async initializeNavigationHistory(navHistoryDir) {
     try {
       const history = await NavigationHistory.create({
@@ -233,6 +245,7 @@ export class Crawler {
     return this.config.sessionId;
   }
   async resume(sessionId) {
+    this.ensureActive();
     if (!this.config.enableNavigationHistory) {
       throw new Error("Navigation history is not enabled. Set enableNavigationHistory: true in options.");
     }
@@ -799,6 +812,7 @@ export class Crawler {
     return url;
   }
   visit(url, options) {
+    this.ensureActive();
     if (this.config.baseUrl)
       url = new URL(url, this.config.baseUrl).href;
     if (options?.params && (options.useOxylabsScraperAi || this.config.hasDomain(url, "oxylabs"))) {
@@ -1077,7 +1091,11 @@ export class Crawler {
     } catch {}
   }
   async destroy() {
+    if (this.isDestroyed)
+      return;
+    this.isDestroyed = true;
     this.queue.destroy();
+    this.config.destroyLimiters();
     this.events.length = 0;
     this.jsonEvents.length = 0;
     this.errorEvents.length = 0;

package/dist/crawler/index.cjs

@@ -1,40 +1,40 @@
-const _mod_iiz8t5 = require('./crawler.cjs');
-exports.Crawler = _mod_iiz8t5.Crawler;;
-const _mod_o5xr59 = require('./crawler-options.cjs');
-exports.CrawlerOptions = _mod_o5xr59.CrawlerOptions;;
-const _mod_nle9x4 = require('./plugin/robots-txt.cjs');
-exports.RobotsTxt = _mod_nle9x4.RobotsTxt;;
-const _mod_nja8nh = require('./plugin/file-cacher.cjs');
-exports.FileCacher = _mod_nja8nh.FileCacher;;
-const _mod_o78tts = require('./plugin/url-store.cjs');
-exports.UrlStore = _mod_o78tts.UrlStore;;
-const _mod_7oju9n = require('./plugin/navigation-history.cjs');
-exports.NavigationHistory = _mod_7oju9n.NavigationHistory;;
-const _mod_227go8 = require('./addon/oxylabs/index.cjs');
-exports.Oxylabs = _mod_227go8.Oxylabs;;
-const _mod_hwujm9 = require('./addon/oxylabs/options.cjs');
-exports.OXYLABS_BROWSER_TYPES = _mod_hwujm9.OXYLABS_BROWSER_TYPES;
-exports.OXYLABS_COMMON_LOCALES = _mod_hwujm9.OXYLABS_COMMON_LOCALES;
-exports.OXYLABS_COMMON_GEO_LOCATIONS = _mod_hwujm9.OXYLABS_COMMON_GEO_LOCATIONS;
-exports.OXYLABS_US_STATES = _mod_hwujm9.OXYLABS_US_STATES;
-exports.OXYLABS_EUROPEAN_COUNTRIES = _mod_hwujm9.OXYLABS_EUROPEAN_COUNTRIES;
-exports.OXYLABS_ASIAN_COUNTRIES = _mod_hwujm9.OXYLABS_ASIAN_COUNTRIES;
-exports.getRandomOxylabsBrowserType = _mod_hwujm9.getRandomBrowserType;
-exports.getRandomOxylabsLocale = _mod_hwujm9.getRandomLocale;
-exports.getRandomOxylabsGeoLocation = _mod_hwujm9.getRandomGeoLocation;;
-const _mod_fcwzzd = require('./addon/decodo/index.cjs');
-exports.Decodo = _mod_fcwzzd.Decodo;;
-const _mod_0ljocp = require('./addon/decodo/options.cjs');
-exports.DECODO_DEVICE_TYPES = _mod_0ljocp.DECODO_DEVICE_TYPES;
-exports.DECODO_HEADLESS_MODES = _mod_0ljocp.DECODO_HEADLESS_MODES;
-exports.DECODO_COMMON_LOCALES = _mod_0ljocp.DECODO_COMMON_LOCALES;
-exports.DECODO_COMMON_COUNTRIES = _mod_0ljocp.DECODO_COMMON_COUNTRIES;
-exports.DECODO_EUROPEAN_COUNTRIES = _mod_0ljocp.DECODO_EUROPEAN_COUNTRIES;
-exports.DECODO_ASIAN_COUNTRIES = _mod_0ljocp.DECODO_ASIAN_COUNTRIES;
-exports.DECODO_US_STATES = _mod_0ljocp.DECODO_US_STATES;
-exports.DECODO_COMMON_CITIES = _mod_0ljocp.DECODO_COMMON_CITIES;
-exports.getRandomDecodoDeviceType = _mod_0ljocp.getRandomDeviceType;
-exports.getRandomDecodoLocale = _mod_0ljocp.getRandomLocale;
-exports.getRandomDecodoCountry = _mod_0ljocp.getRandomCountry;
-exports.getRandomDecodoCity = _mod_0ljocp.getRandomCity;
-exports.generateDecodoSessionId = _mod_0ljocp.generateSessionId;;
+const _mod_ooxmqc = require('./crawler.cjs');
+exports.Crawler = _mod_ooxmqc.Crawler;;
+const _mod_vzzk69 = require('./crawler-options.cjs');
+exports.CrawlerOptions = _mod_vzzk69.CrawlerOptions;;
+const _mod_gsic6h = require('./plugin/robots-txt.cjs');
+exports.RobotsTxt = _mod_gsic6h.RobotsTxt;;
+const _mod_8jo54n = require('./plugin/file-cacher.cjs');
+exports.FileCacher = _mod_8jo54n.FileCacher;;
+const _mod_hkwhmy = require('./plugin/url-store.cjs');
+exports.UrlStore = _mod_hkwhmy.UrlStore;;
+const _mod_9bidrg = require('./plugin/navigation-history.cjs');
+exports.NavigationHistory = _mod_9bidrg.NavigationHistory;;
+const _mod_8ldzfr = require('./addon/oxylabs/index.cjs');
+exports.Oxylabs = _mod_8ldzfr.Oxylabs;;
+const _mod_x8yxp9 = require('./addon/oxylabs/options.cjs');
+exports.OXYLABS_BROWSER_TYPES = _mod_x8yxp9.OXYLABS_BROWSER_TYPES;
+exports.OXYLABS_COMMON_LOCALES = _mod_x8yxp9.OXYLABS_COMMON_LOCALES;
+exports.OXYLABS_COMMON_GEO_LOCATIONS = _mod_x8yxp9.OXYLABS_COMMON_GEO_LOCATIONS;
+exports.OXYLABS_US_STATES = _mod_x8yxp9.OXYLABS_US_STATES;
+exports.OXYLABS_EUROPEAN_COUNTRIES = _mod_x8yxp9.OXYLABS_EUROPEAN_COUNTRIES;
+exports.OXYLABS_ASIAN_COUNTRIES = _mod_x8yxp9.OXYLABS_ASIAN_COUNTRIES;
+exports.getRandomOxylabsBrowserType = _mod_x8yxp9.getRandomBrowserType;
+exports.getRandomOxylabsLocale = _mod_x8yxp9.getRandomLocale;
+exports.getRandomOxylabsGeoLocation = _mod_x8yxp9.getRandomGeoLocation;;
+const _mod_jeqdt1 = require('./addon/decodo/index.cjs');
+exports.Decodo = _mod_jeqdt1.Decodo;;
+const _mod_p59evs = require('./addon/decodo/options.cjs');
+exports.DECODO_DEVICE_TYPES = _mod_p59evs.DECODO_DEVICE_TYPES;
+exports.DECODO_HEADLESS_MODES = _mod_p59evs.DECODO_HEADLESS_MODES;
+exports.DECODO_COMMON_LOCALES = _mod_p59evs.DECODO_COMMON_LOCALES;
+exports.DECODO_COMMON_COUNTRIES = _mod_p59evs.DECODO_COMMON_COUNTRIES;
+exports.DECODO_EUROPEAN_COUNTRIES = _mod_p59evs.DECODO_EUROPEAN_COUNTRIES;
+exports.DECODO_ASIAN_COUNTRIES = _mod_p59evs.DECODO_ASIAN_COUNTRIES;
+exports.DECODO_US_STATES = _mod_p59evs.DECODO_US_STATES;
+exports.DECODO_COMMON_CITIES = _mod_p59evs.DECODO_COMMON_CITIES;
+exports.getRandomDecodoDeviceType = _mod_p59evs.getRandomDeviceType;
+exports.getRandomDecodoLocale = _mod_p59evs.getRandomLocale;
+exports.getRandomDecodoCountry = _mod_p59evs.getRandomCountry;
+exports.getRandomDecodoCity = _mod_p59evs.getRandomCity;
+exports.generateDecodoSessionId = _mod_p59evs.generateSessionId;;

package/dist/crawler.d.ts

@@ -6871,6 +6871,18 @@ export declare class CrawlerOptions {
 		options: DecodoOptions;
 		queueOptions: queueOptions$1;
 	}): CrawlerOptions;
+	/**
+	 * Destroy all limiter queues to release resources and stop intervals
+	 * @description Properly destroys all RezoQueue instances created by addLimiter().
+	 * This stops any setInterval timers that would otherwise keep the process alive.
+	 * Called automatically by Crawler.destroy().
+	 * @example
+	 * ```typescript
+	 * // Clean up all limiters
+	 * options.destroyLimiters();
+	 * ```
+	 */
+	destroyLimiters(): void;
 	/**
 	 * Clear all global configurations from headers, proxies, limiters, Decodo, and Oxylabs
 	 * @returns The CrawlerOptions instance for method chaining
@@ -7072,7 +7084,7 @@ export declare class Crawler {
 	 * Uses SQLite as the underlying storage mechanism.
 	 */
 	cacher: FileCacher;
-	private readonly queue;
+	private queue;
 	private readonly isCacheEnabled;
 	readonly config: CrawlerOptions;
 	private urlStorage;
@@ -7090,6 +7102,10 @@ export declare class Crawler {
 	private adapterType;
 	/** Track pending execute() calls for proper done() behavior */
 	private pendingExecutions;
+	/** Track if the crawler has been destroyed */
+	private isDestroyed;
+	/** Original queue options for restoration */
+	private readonly queueOptions;
 	/** robots.txt parser and validator */
 	private robotsTxt;
 	/** AutoThrottle: track response times per domain for adaptive rate limiting */
@@ -7142,6 +7158,12 @@ export declare class Crawler {
 	 * Initialize the HTTP adapter based on configuration
 	 */
 	private initializeAdapter;
+	/**
+	 * Ensures the crawler is active and ready for use.
+	 * If the crawler was previously destroyed, this method will restore it.
+	 * Called internally before any public operation.
+	 */
+	private ensureActive;
 	/**
 	 * Initialize navigation history and session
 	 */

package/dist/entries/crawler.cjs

@@ -1,5 +1,4 @@
-const _mod_vioeet = require('../crawler/crawler.cjs');
-exports.Crawler = _mod_vioeet.Crawler;;
-const _mod_qsdeb2 = require('../crawler/crawler-options.cjs');
-exports.CrawlerOptions = _mod_qsdeb2.CrawlerOptions;
-exports.Domain = _mod_qsdeb2.Domain;;
+const _mod_7u46bc = require('../crawler/crawler.cjs');
+exports.Crawler = _mod_7u46bc.Crawler;;
+const _mod_nvz29w = require('../crawler/crawler-options.cjs');
+exports.CrawlerOptions = _mod_nvz29w.CrawlerOptions;;

package/dist/entries/crawler.js

@@ -1,2 +1,2 @@
 export { Crawler } from '../crawler/crawler.js';
-export { CrawlerOptions, Domain } from '../crawler/crawler-options.js';
+export { CrawlerOptions } from '../crawler/crawler-options.js';

package/dist/index.cjs

@@ -1,30 +1,30 @@
-const _mod_o4ckug = require('./core/rezo.cjs');
-exports.Rezo = _mod_o4ckug.Rezo;
-exports.createRezoInstance = _mod_o4ckug.createRezoInstance;
-exports.createDefaultInstance = _mod_o4ckug.createDefaultInstance;;
-const _mod_ngg7ca = require('./errors/rezo-error.cjs');
-exports.RezoError = _mod_ngg7ca.RezoError;
-exports.RezoErrorCode = _mod_ngg7ca.RezoErrorCode;;
-const _mod_7hyvc0 = require('./utils/headers.cjs');
-exports.RezoHeaders = _mod_7hyvc0.RezoHeaders;;
-const _mod_n4qpbg = require('./utils/form-data.cjs');
-exports.RezoFormData = _mod_n4qpbg.RezoFormData;;
-const _mod_02e54t = require('./utils/cookies.cjs');
-exports.RezoCookieJar = _mod_02e54t.RezoCookieJar;
-exports.Cookie = _mod_02e54t.Cookie;;
-const _mod_vtlv3s = require('./utils/curl.cjs');
-exports.toCurl = _mod_vtlv3s.toCurl;
-exports.fromCurl = _mod_vtlv3s.fromCurl;;
-const _mod_sl47g0 = require('./core/hooks.cjs');
-exports.createDefaultHooks = _mod_sl47g0.createDefaultHooks;
-exports.mergeHooks = _mod_sl47g0.mergeHooks;;
-const _mod_hbgvml = require('./proxy/manager.cjs');
-exports.ProxyManager = _mod_hbgvml.ProxyManager;;
-const _mod_gt9it7 = require('./queue/index.cjs');
-exports.RezoQueue = _mod_gt9it7.RezoQueue;
-exports.HttpQueue = _mod_gt9it7.HttpQueue;
-exports.Priority = _mod_gt9it7.Priority;
-exports.HttpMethodPriority = _mod_gt9it7.HttpMethodPriority;;
+const _mod_s9lieh = require('./core/rezo.cjs');
+exports.Rezo = _mod_s9lieh.Rezo;
+exports.createRezoInstance = _mod_s9lieh.createRezoInstance;
+exports.createDefaultInstance = _mod_s9lieh.createDefaultInstance;;
+const _mod_5rlfq9 = require('./errors/rezo-error.cjs');
+exports.RezoError = _mod_5rlfq9.RezoError;
+exports.RezoErrorCode = _mod_5rlfq9.RezoErrorCode;;
+const _mod_6xv93u = require('./utils/headers.cjs');
+exports.RezoHeaders = _mod_6xv93u.RezoHeaders;;
+const _mod_fqgdip = require('./utils/form-data.cjs');
+exports.RezoFormData = _mod_fqgdip.RezoFormData;;
+const _mod_56ytl2 = require('./utils/cookies.cjs');
+exports.RezoCookieJar = _mod_56ytl2.RezoCookieJar;
+exports.Cookie = _mod_56ytl2.Cookie;;
+const _mod_99elmq = require('./utils/curl.cjs');
+exports.toCurl = _mod_99elmq.toCurl;
+exports.fromCurl = _mod_99elmq.fromCurl;;
+const _mod_tvrmqp = require('./core/hooks.cjs');
+exports.createDefaultHooks = _mod_tvrmqp.createDefaultHooks;
+exports.mergeHooks = _mod_tvrmqp.mergeHooks;;
+const _mod_04et4j = require('./proxy/manager.cjs');
+exports.ProxyManager = _mod_04et4j.ProxyManager;;
+const _mod_vxzvia = require('./queue/index.cjs');
+exports.RezoQueue = _mod_vxzvia.RezoQueue;
+exports.HttpQueue = _mod_vxzvia.HttpQueue;
+exports.Priority = _mod_vxzvia.Priority;
+exports.HttpMethodPriority = _mod_vxzvia.HttpMethodPriority;;
 const { RezoError } = require('./errors/rezo-error.cjs');
 const isRezoError = exports.isRezoError = RezoError.isRezoError;
 const Cancel = exports.Cancel = RezoError;

package/dist/internal/agents/index.cjs

@@ -1,10 +1,10 @@
-const _mod_mpah7f = require('./base.cjs');
-exports.Agent = _mod_mpah7f.Agent;;
-const _mod_v61kol = require('./http-proxy.cjs');
-exports.HttpProxyAgent = _mod_v61kol.HttpProxyAgent;;
-const _mod_tu5489 = require('./https-proxy.cjs');
-exports.HttpsProxyAgent = _mod_tu5489.HttpsProxyAgent;;
-const _mod_qzfr9v = require('./socks-proxy.cjs');
-exports.SocksProxyAgent = _mod_qzfr9v.SocksProxyAgent;;
-const _mod_c6r2y0 = require('./socks-client.cjs');
-exports.SocksClient = _mod_c6r2y0.SocksClient;;
+const _mod_e7qx8k = require('./base.cjs');
+exports.Agent = _mod_e7qx8k.Agent;;
+const _mod_q2hbye = require('./http-proxy.cjs');
+exports.HttpProxyAgent = _mod_q2hbye.HttpProxyAgent;;
+const _mod_bl101b = require('./https-proxy.cjs');
+exports.HttpsProxyAgent = _mod_bl101b.HttpsProxyAgent;;
+const _mod_ad41no = require('./socks-proxy.cjs');
+exports.SocksProxyAgent = _mod_ad41no.SocksProxyAgent;;
+const _mod_5q8p36 = require('./socks-client.cjs');
+exports.SocksClient = _mod_5q8p36.SocksClient;;

package/dist/proxy/index.cjs

@@ -1,9 +1,9 @@
 const { Agent, HttpProxyAgent, HttpsProxyAgent, SocksProxyAgent } = require('../internal/agents/index.cjs');
 const { parseProxyString } = require('./parse.cjs');
-const _mod_r48ne8 = require('./manager.cjs');
-exports.ProxyManager = _mod_r48ne8.ProxyManager;;
-const _mod_taz7l0 = require('./parse.cjs');
-exports.parseProxyString = _mod_taz7l0.parseProxyString;;
+const _mod_mucfqi = require('./manager.cjs');
+exports.ProxyManager = _mod_mucfqi.ProxyManager;;
+const _mod_gp9w9o = require('./parse.cjs');
+exports.parseProxyString = _mod_gp9w9o.parseProxyString;;
 function createOptions(uri, opts) {
   if (uri instanceof URL || typeof uri === "string") {
     return {

package/dist/queue/index.cjs

@@ -1,8 +1,8 @@
-const _mod_2mav18 = require('./queue.cjs');
-exports.RezoQueue = _mod_2mav18.RezoQueue;;
-const _mod_0t9tzq = require('./http-queue.cjs');
-exports.HttpQueue = _mod_0t9tzq.HttpQueue;
-exports.extractDomain = _mod_0t9tzq.extractDomain;;
-const _mod_qedong = require('./types.cjs');
-exports.Priority = _mod_qedong.Priority;
-exports.HttpMethodPriority = _mod_qedong.HttpMethodPriority;;
+const _mod_w5fj4r = require('./queue.cjs');
+exports.RezoQueue = _mod_w5fj4r.RezoQueue;;
+const _mod_td8atq = require('./http-queue.cjs');
+exports.HttpQueue = _mod_td8atq.HttpQueue;
+exports.extractDomain = _mod_td8atq.extractDomain;;
+const _mod_togvwr = require('./types.cjs');
+exports.Priority = _mod_togvwr.Priority;
+exports.HttpMethodPriority = _mod_togvwr.HttpMethodPriority;;

package/dist/responses/universal/index.cjs

@@ -1,11 +1,11 @@
-const _mod_ourzzn = require('./event-emitter.cjs');
-exports.UniversalEventEmitter = _mod_ourzzn.UniversalEventEmitter;;
-const _mod_epyk0f = require('./stream.cjs');
-exports.UniversalStreamResponse = _mod_epyk0f.UniversalStreamResponse;
-exports.StreamResponse = _mod_epyk0f.StreamResponse;;
-const _mod_124wmp = require('./download.cjs');
-exports.UniversalDownloadResponse = _mod_124wmp.UniversalDownloadResponse;
-exports.DownloadResponse = _mod_124wmp.DownloadResponse;;
-const _mod_qjrj9w = require('./upload.cjs');
-exports.UniversalUploadResponse = _mod_qjrj9w.UniversalUploadResponse;
-exports.UploadResponse = _mod_qjrj9w.UploadResponse;;
+const _mod_wh90je = require('./event-emitter.cjs');
+exports.UniversalEventEmitter = _mod_wh90je.UniversalEventEmitter;;
+const _mod_8xyb5w = require('./stream.cjs');
+exports.UniversalStreamResponse = _mod_8xyb5w.UniversalStreamResponse;
+exports.StreamResponse = _mod_8xyb5w.StreamResponse;;
+const _mod_a20qdf = require('./download.cjs');
+exports.UniversalDownloadResponse = _mod_a20qdf.UniversalDownloadResponse;
+exports.DownloadResponse = _mod_a20qdf.DownloadResponse;;
+const _mod_orr5i8 = require('./upload.cjs');
+exports.UniversalUploadResponse = _mod_orr5i8.UniversalUploadResponse;
+exports.UploadResponse = _mod_orr5i8.UploadResponse;;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "rezo",
-  "version": "1.0.55",
+  "version": "1.0.57",
   "description": "Lightning-fast, enterprise-grade HTTP client for modern JavaScript. Full HTTP/2 support, intelligent cookie management, multiple adapters (HTTP, Fetch, cURL, XHR), streaming, proxy support (HTTP/HTTPS/SOCKS), and cross-environment compatibility.",
   "main": "dist/index.cjs",
   "module": "dist/index.js",