rezo 1.0.73 → 1.0.74

package/dist/adapters/entries/curl.d.ts

@@ -4579,7 +4579,7 @@ export interface RezoInstance extends Rezo {
  *
  * IMPORTANT: Update these values when bumping package version.
  */
-export declare const VERSION = "1.0.73";
+export declare const VERSION = "1.0.74";
 /**
  * cURL Options Configuration
  *

package/dist/adapters/entries/fetch.d.ts

@@ -4579,7 +4579,7 @@ export interface RezoInstance extends Rezo {
  *
  * IMPORTANT: Update these values when bumping package version.
  */
-export declare const VERSION = "1.0.73";
+export declare const VERSION = "1.0.74";
 export declare const isRezoError: typeof RezoError.isRezoError;
 export declare const Cancel: typeof RezoError;
 export declare const CancelToken: {

package/dist/adapters/entries/http.d.ts

@@ -4579,7 +4579,7 @@ export interface RezoInstance extends Rezo {
  *
  * IMPORTANT: Update these values when bumping package version.
  */
-export declare const VERSION = "1.0.73";
+export declare const VERSION = "1.0.74";
 /**
  * Type guard to check if an error is a RezoError instance.
  */

package/dist/adapters/entries/http2.d.ts

@@ -4579,7 +4579,7 @@ export interface RezoInstance extends Rezo {
  *
  * IMPORTANT: Update these values when bumping package version.
  */
-export declare const VERSION = "1.0.73";
+export declare const VERSION = "1.0.74";
 export declare const isRezoError: typeof RezoError.isRezoError;
 export declare const Cancel: typeof RezoError;
 export declare const CancelToken: {

package/dist/adapters/entries/react-native.d.ts

@@ -4579,7 +4579,7 @@ export interface RezoInstance extends Rezo {
  *
  * IMPORTANT: Update these values when bumping package version.
  */
-export declare const VERSION = "1.0.73";
+export declare const VERSION = "1.0.74";
 export declare const isRezoError: typeof RezoError.isRezoError;
 export declare const Cancel: typeof RezoError;
 export declare const CancelToken: {

package/dist/adapters/entries/xhr.d.ts

@@ -4579,7 +4579,7 @@ export interface RezoInstance extends Rezo {
  *
  * IMPORTANT: Update these values when bumping package version.
  */
-export declare const VERSION = "1.0.73";
+export declare const VERSION = "1.0.74";
 export declare const isRezoError: typeof RezoError.isRezoError;
 export declare const Cancel: typeof RezoError;
 export declare const CancelToken: {

package/dist/adapters/index.cjs

@@ -1,6 +1,6 @@
-const _mod_vstjof = require('./picker.cjs');
-exports.detectRuntime = _mod_vstjof.detectRuntime;
-exports.getAdapterCapabilities = _mod_vstjof.getAdapterCapabilities;
-exports.buildAdapterContext = _mod_vstjof.buildAdapterContext;
-exports.getAvailableAdapters = _mod_vstjof.getAvailableAdapters;
-exports.selectAdapter = _mod_vstjof.selectAdapter;;
+const _mod_pls1zs = require('./picker.cjs');
+exports.detectRuntime = _mod_pls1zs.detectRuntime;
+exports.getAdapterCapabilities = _mod_pls1zs.getAdapterCapabilities;
+exports.buildAdapterContext = _mod_pls1zs.buildAdapterContext;
+exports.getAvailableAdapters = _mod_pls1zs.getAvailableAdapters;
+exports.selectAdapter = _mod_pls1zs.selectAdapter;;

package/dist/cache/index.cjs

@@ -1,9 +1,9 @@
-const _mod_4yb0ry = require('./lru-cache.cjs');
-exports.LRUCache = _mod_4yb0ry.LRUCache;;
-const _mod_lb3zhr = require('./dns-cache.cjs');
-exports.DNSCache = _mod_lb3zhr.DNSCache;
-exports.getGlobalDNSCache = _mod_lb3zhr.getGlobalDNSCache;
-exports.resetGlobalDNSCache = _mod_lb3zhr.resetGlobalDNSCache;;
-const _mod_ogn45d = require('./response-cache.cjs');
-exports.ResponseCache = _mod_ogn45d.ResponseCache;
-exports.normalizeResponseCacheConfig = _mod_ogn45d.normalizeResponseCacheConfig;;
+const _mod_121cas = require('./lru-cache.cjs');
+exports.LRUCache = _mod_121cas.LRUCache;;
+const _mod_9vbe4h = require('./dns-cache.cjs');
+exports.DNSCache = _mod_9vbe4h.DNSCache;
+exports.getGlobalDNSCache = _mod_9vbe4h.getGlobalDNSCache;
+exports.resetGlobalDNSCache = _mod_9vbe4h.resetGlobalDNSCache;;
+const _mod_to71v9 = require('./response-cache.cjs');
+exports.ResponseCache = _mod_to71v9.ResponseCache;
+exports.normalizeResponseCacheConfig = _mod_to71v9.normalizeResponseCacheConfig;;

package/dist/crawler/crawler-options.cjs

@@ -1 +1 @@
-var{RezoQueue:m}=require("../queue/queue.cjs"),{Oxylabs:x}=require("./addon/oxylabs/index.cjs"),b=require("node:path"),g=require("node:os"),{Decodo:y}=require("./addon/decodo/index.cjs");class f{baseUrl;adapter;enableNavigationHistory;sessionId;rejectUnauthorized;userAgent;useRndUserAgent;timeout;maxRedirects;maxRetryAttempts;retryDelay;retryOnStatusCode;forceRevisit;retryWithoutProxyOnStatusCode;retryOnProxyError;maxRetryOnProxyError;allowRevisiting;enableCache;cacheTTL;cacheDir;throwFatalError;debug;concurrency;scraperConcurrency;maxDepth;maxUrls;maxResponseSize;respectRobotsTxt;followNofollow;autoThrottle;autoThrottleTargetDelay;autoThrottleMinDelay;autoThrottleMaxDelay;maxWaitOn429;alwaysWaitOn429;oxylabs=[];decodo=[];proxies=[];limiters=[];requestHeaders=[];userAgents=A();constructor(e={}){this.baseUrl=e.baseUrl||"",this.adapter=e.adapter??"http",this.enableNavigationHistory=e.enableNavigationHistory??!1,this.sessionId=e.sessionId??`session_${Date.now()}_${Math.random().toString(36).slice(2,8)}`,this.rejectUnauthorized=e.rejectUnauthorized??!0,this.userAgent=e.userAgent,this.useRndUserAgent=e.useRndUserAgent??!1,this.timeout=e.timeout??30000,this.maxRedirects=e.maxRedirects??10,this.maxRetryAttempts=e.maxRetryAttempts??3,this.retryDelay=e.retryDelay??0,this.retryOnStatusCode=e.retryOnStatusCode??[408,429,500,502,503,504],this.forceRevisit=e.forceRevisit??!1,this.retryWithoutProxyOnStatusCode=e.retryWithoutProxyOnStatusCode??[407,403],this.retryOnProxyError=e.retryOnProxyError??!0,this.maxRetryOnProxyError=e.maxRetryOnProxyError??3,this.allowRevisiting=e.allowRevisiting??!1,this.enableCache=e.enableCache??!0,this.cacheTTL=e.cacheTTL??604800000,this.cacheDir=e.cacheDir??b.join(g.tmpdir(),"uiniqhtt_cache"),this.throwFatalError=e.throwFatalError??!1,this.debug=e.debug??!1,this.concurrency=e.concurrency??100,this.scraperConcurrency=e.scraperConcurrency??this.concurrency,this.maxDepth=e.maxDepth??0,this.maxUrls=e.maxUrls??0,this.maxResponseSize=e.maxResponseSize??0,this.respectRobotsTxt=e.respectRobotsTxt??!1,this.followNofollow=e.followNofollow??!1,this.autoThrottle=e.autoThrottle??!0,this.autoThrottleTargetDelay=e.autoThrottleTargetDelay??1000,this.autoThrottleMinDelay=e.autoThrottleMinDelay??100,this.autoThrottleMaxDelay=e.autoThrottleMaxDelay??60000,this.maxWaitOn429=e.maxWaitOn429??1800000,this.alwaysWaitOn429=e.alwaysWaitOn429??!1,this._addHeaders(e.headers),this._addOxylabs(e.oxylabs),this._addDecodo(e.decodo),this._addProxies(e.proxy),this._addLimiters(e.limiter)}getConfiguredDomains(e){return(e==="headers"?this.requestHeaders:e==="limiters"?this.limiters:e==="oxylabs"?this.oxylabs:this.proxies).filter((i)=>i.domain).map((i)=>i.domain).filter((i,a,r)=>r.indexOf(i)===a)}removeDomain(e){return this.requestHeaders=this.requestHeaders.filter((t)=>!t.domain||!this._domainsEqual(t.domain,e)),this.proxies=this.proxies.filter((t)=>!t.domain||!this._domainsEqual(t.domain,e)),this.limiters=this.limiters.filter((t)=>!t.domain||!this._domainsEqual(t.domain,e)),this.oxylabs=this.oxylabs.filter((t)=>!t.domain||!this._domainsEqual(t.domain,e)),this}_domainsEqual(e,t){if(Array.isArray(e)&&Array.isArray(t))return e.length===t.length&&e.every((i,a)=>i===t[a]);return e===t}getConfigurationSummary(){let e=(t)=>({total:t.length,global:t.filter((i)=>i.isGlobal).length,domainSpecific:t.filter((i)=>!i.isGlobal&&i.domain).length});return{headers:e(this.requestHeaders),proxies:e(this.proxies),limiters:e(this.limiters),oxylabs:e(this.oxylabs)}}_addHeaders(e){if(!e||!e.enable)return;for(let t of e.httpHeaders){let{domain:i,isGlobal:a,headers:r}=t;if(!i&&!a)continue;if(r instanceof Headers){let s=Object.fromEntries(r.entries());if(Object.keys(s).length<1)continue;r=s}else if(!r||Object.keys(r).length<1)continue;this.requestHeaders.push({domain:i,isGlobal:a,headers:r})}}_addProxies(e){if(!e||!e.enable)return;for(let t of e.proxies){let{domain:i,isGlobal:a,proxy:r}=t;if(!i&&!a)continue;if(!r||Object.keys(r).length<1)continue;this.proxies.push({domain:i,isGlobal:a,proxy:r})}}_addLimiters(e){if(!e||!e.enable)return;for(let t of e.limiters){let{domain:i,isGlobal:a,options:r}=t;if(!i&&!a)continue;if(!r||Object.keys(r).length<1)continue;this.limiters.push({domain:i,isGlobal:a,pqueue:new m(r)})}}_addOxylabs(e){if(!e||!e.enable)return;for(let t of e.labs){let{domain:i,isGlobal:a,options:r,queueOptions:s}=t;if(!i&&!a)continue;if(!r||Object.keys(r).length<1)continue;this.oxylabs.push({domain:i,isGlobal:a,adaptar:new x(r)})}}_addDecodo(e){if(!e||!e.enable)return;for(let t of e.labs){let{domain:i,isGlobal:a,options:r,queueOptions:s}=t;if(!i&&!a)continue;if(!r||Object.keys(r).length<1)continue;this.decodo.push({domain:i,isGlobal:a,adaptar:new y(r)})}}addHeaders(e){return this._addHeaders({enable:!0,httpHeaders:[e]}),this}addProxy(e){return this._addProxies({enable:!0,proxies:[e]}),this}addLimiter(e){return this._addLimiters({enable:!0,limiters:[e]}),this}addOxylabs(e){return this._addOxylabs({enable:!0,labs:[e]}),this}addDecodo(e){return this._addDecodo({enable:!0,labs:[e]}),this}destroyLimiters(){for(let e of this.limiters)if(e.pqueue&&typeof e.pqueue.destroy==="function")e.pqueue.destroy();this.limiters=[]}clearGlobalConfigs(){if(Array.isArray(this.requestHeaders))this.requestHeaders=this.requestHeaders.filter((e)=>!e.isGlobal);if(Array.isArray(this.oxylabs))this.oxylabs=this.oxylabs.filter((e)=>!e.isGlobal);if(Array.isArray(this.limiters))this.limiters=this.limiters.filter((e)=>!e.isGlobal);if(Array.isArray(this.proxies))this.proxies=this.proxies.filter((e)=>!e.isGlobal);return this}getAdapter(e,t,i,a){if(!this.getDomainName(e))return null;let s=[],o=t==="headers"?this.requestHeaders:t==="limiters"?this.limiters:t==="oxylabs"?this.oxylabs:t==="decodo"?this.decodo:this.proxies;for(let n=0;n<o.length;n++)if(this._hasDomain(e,o[n].domain))s.push(n);if(s.length){let n=a?s[this.rnd(0,s.length-1)]:s[0];return t==="headers"?this.requestHeaders[n].headers:t==="limiters"?this.limiters[n].pqueue:t==="oxylabs"?this.oxylabs[n].adaptar:t==="decodo"?this.decodo[n].adaptar:this.proxies[n].proxy}s.length=0;for(let n=0;n<o.length;n++)s.push(n);if(s.length){let n=a?s[this.rnd(0,s.length-1)]:s[0];if(o[n].isGlobal&&i)return t==="headers"?this.requestHeaders[n].headers:t==="limiters"?this.limiters[n].pqueue:t==="oxylabs"?this.oxylabs[n].adaptar:t==="decodo"?this.decodo[n].adaptar:this.proxies[n].proxy}return null}rnd(e=0,t=Number.MAX_VALUE){return Math.floor(Math.random()*(t-e+1))+e}hasDomain(e,t,i){if(!this.getDomainName(e))return!1;let r=t==="headers"?this.requestHeaders:t==="limiters"?this.limiters:t==="oxylabs"?this.oxylabs:t==="decodo"?this.decodo:this.proxies;for(let s=0;s<r.length;s++)if(this._hasDomain(e,r[s].domain))return!0;if(i){for(let s=0;s<r.length;s++)if(r[s].isGlobal)return!0}return!1}pickHeaders(e,t,i,a){let r=this.getAdapter(e,"headers",t),s=new Headers(r??{}),o=s.count;if(i&&i instanceof Headers)for(let[n,l]of Object.entries(i.entries()))s.set(n,l);else if(i&&typeof i==="object"){for(let[n,l]of Object.entries(i))if(typeof l==="string")s.set(n,l)}if(a&&o===0)s.set("user-agent",this.getRandomUserAgent());return Object.fromEntries(s.entries())}_hasDomain(e,t){if(!t)return!1;let i=this.getDomainName(e);if(!i)return!1;let a=(s)=>{return/[\^\$\*\+\?\{\}\[\]\(\)\|\\]/.test(s)||s.startsWith("/")||s.includes(".*")||s.includes(".+")},r=(s)=>{if(s instanceof RegExp)return s.test(i)||s.test(e);let o=s.toString().trim();if(i.toLowerCase()===o.toLowerCase())return!0;if(o.includes("*")){let h=o.replace(/[.*+?^${}()|[\]\\]/g,"\\$&").replace(/\\\*/g,".*"),c=new RegExp(`^${h}$`,"i");return c.test(i)||c.test(e)}if(a(o))try{let h=o,c="i",u=o.match(/^\/(.*)\/(\w*)$/);if(u)h=u[1],c=u[2]||"i";let d=new RegExp(h,c);return d.test(i)||d.test(e)}catch(h){return i.toLowerCase().includes(o.toLowerCase())}let n=i.toLowerCase(),l=o.toLowerCase();return n===l||n.endsWith("."+l)||l.endsWith("."+n)};if(Array.isArray(t)){for(let s of t)if(r(s))return!0;return!1}return r(t)}getDomainName(e){if(this.isValidUrl(e))return new URL(e).hostname;else if(this.isHostName(e))return e;return null}isHostName(e){if(!e)return!1;if(e.length>255)return!1;let t=/^(?:(?:[a-zA-Z0-9](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?\.)+ [a-zA-Z]{2,})$/;return e=e.trim().toLowerCase(),t.test(e)&&!e.startsWith("-")&&!e.endsWith("-")}isValidUrl(e){if(!e)return!1;e=e.trim();try{let t=new URL(e);if(!t.protocol||!["http:","https:"].includes(t.protocol.toLowerCase()))return!1;if(!t.hostname)return!1;if(!/^(?:(?:[a-zA-Z0-9](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?\.)+[a-zA-Z]{2,})$/.test(t.hostname))return!1;return!0}catch{return!1}}getRandomUserAgent(){return this.userAgents[Math.floor(Math.random()*this.userAgents.length)]}}function A(){let e=[{name:"Chrome",version:"91.0.4472.124",engine:"AppleWebKit/537.36"},{name:"Firefox",version:"89.0",engine:"Gecko/20100101"},{name:"Safari",version:"14.1.1",engine:"AppleWebKit/605.1.15"},{name:"Edge",version:"91.0.864.59",engine:"AppleWebKit/537.36"},{name:"Opera",version:"77.0.4054.277",engine:"AppleWebKit/537.36"},{name:"Vivaldi",version:"3.8.2259.42",engine:"AppleWebKit/537.36"},{name:"Brave",version:"1.26.74",engine:"AppleWebKit/537.36"},{name:"Chromium",version:"91.0.4472.101",engine:"AppleWebKit/537.36"},{name:"Yandex",version:"21.5.3.742",engine:"AppleWebKit/537.36"},{name:"Maxthon",version:"5.3.8.2000",engine:"AppleWebKit/537.36"}],t=["Windows NT 10.0","Windows NT 6.1","Macintosh; Intel Mac OS X 10_15_7","Macintosh; Intel Mac OS X 11_4_0","X11; Linux x86_64","X11; Ubuntu; Linux x86_64"],i=[];for(let a=0;a<200;a++){let r=e[Math.floor(Math.random()*e.length)],s=t[Math.floor(Math.random()*t.length)],o="";switch(r.name){case"Chrome":o=`Mozilla/5.0 (${s}) ${r.engine} (KHTML, like Gecko) Chrome/${r.version} Safari/537.36`;break;case"Firefox":o=`Mozilla/5.0 (${s}; rv:${r.version}) ${r.engine} Firefox/${r.version}`;break;case"Safari":o=`Mozilla/5.0 (${s}) ${r.engine} (KHTML, like Gecko) Version/${r.version} Safari/605.1.15`;break;case"Edge":o=`Mozilla/5.0 (${s}) ${r.engine} (KHTML, like Gecko) Chrome/${r.version} Safari/537.36 Edg/${r.version}`;break;case"Opera":o=`Mozilla/5.0 (${s}) ${r.engine} (KHTML, like Gecko) Chrome/${r.version} Safari/537.36 OPR/${r.version}`;break;case"Vivaldi":o=`Mozilla/5.0 (${s}) ${r.engine} (KHTML, like Gecko) Chrome/${r.version} Safari/537.36 Vivaldi/${r.version}`;break;case"Brave":o=`Mozilla/5.0 (${s}) ${r.engine} (KHTML, like Gecko) Chrome/${r.version} Safari/537.36 Brave/${r.version}`;break;case"Chromium":o=`Mozilla/5.0 (${s}) ${r.engine} (KHTML, like Gecko) Chromium/${r.version} Chrome/${r.version} Safari/537.36`;break;case"Yandex":o=`Mozilla/5.0 (${s}) ${r.engine} (KHTML, like Gecko) Chrome/${r.version} YaBrowser/${r.version} Safari/537.36`;break;case"Maxthon":o=`Mozilla/5.0 (${s}) ${r.engine} (KHTML, like Gecko) Chrome/${r.version} Safari/537.36 Maxthon/${r.version}`;break}i.push(o)}return i}exports.CrawlerOptions=f;
+var{RezoQueue:m}=require("../queue/queue.cjs"),{Oxylabs:x}=require("./addon/oxylabs/index.cjs"),b=require("node:path"),g=require("node:os"),{Decodo:y}=require("./addon/decodo/index.cjs");class f{baseUrl;adapter;enableNavigationHistory;sessionId;rejectUnauthorized;userAgent;useRndUserAgent;timeout;maxRedirects;maxRetryAttempts;retryDelay;retryOnStatusCode;forceRevisit;retryWithoutProxyOnStatusCode;retryOnProxyError;maxRetryOnProxyError;allowRevisiting;enableCache;cacheTTL;cacheDir;throwFatalError;debug;enableSignalHandlers;concurrency;scraperConcurrency;maxDepth;maxUrls;maxResponseSize;respectRobotsTxt;followNofollow;autoThrottle;autoThrottleTargetDelay;autoThrottleMinDelay;autoThrottleMaxDelay;maxWaitOn429;alwaysWaitOn429;oxylabs=[];decodo=[];proxies=[];limiters=[];requestHeaders=[];userAgents=A();constructor(e={}){this.baseUrl=e.baseUrl||"",this.adapter=e.adapter??"http",this.enableNavigationHistory=e.enableNavigationHistory??!1,this.sessionId=e.sessionId??`session_${Date.now()}_${Math.random().toString(36).slice(2,8)}`,this.rejectUnauthorized=e.rejectUnauthorized??!0,this.userAgent=e.userAgent,this.useRndUserAgent=e.useRndUserAgent??!1,this.timeout=e.timeout??30000,this.maxRedirects=e.maxRedirects??10,this.maxRetryAttempts=e.maxRetryAttempts??3,this.retryDelay=e.retryDelay??0,this.retryOnStatusCode=e.retryOnStatusCode??[408,429,500,502,503,504],this.forceRevisit=e.forceRevisit??!1,this.retryWithoutProxyOnStatusCode=e.retryWithoutProxyOnStatusCode??[407,403],this.retryOnProxyError=e.retryOnProxyError??!0,this.maxRetryOnProxyError=e.maxRetryOnProxyError??3,this.allowRevisiting=e.allowRevisiting??!1,this.enableCache=e.enableCache??!0,this.cacheTTL=e.cacheTTL??604800000,this.cacheDir=e.cacheDir??b.join(g.tmpdir(),"uiniqhtt_cache"),this.throwFatalError=e.throwFatalError??!1,this.debug=e.debug??!1,this.enableSignalHandlers=e.enableSignalHandlers??!1,this.concurrency=e.concurrency??100,this.scraperConcurrency=e.scraperConcurrency??this.concurrency,this.maxDepth=e.maxDepth??0,this.maxUrls=e.maxUrls??0,this.maxResponseSize=e.maxResponseSize??0,this.respectRobotsTxt=e.respectRobotsTxt??!1,this.followNofollow=e.followNofollow??!1,this.autoThrottle=e.autoThrottle??!0,this.autoThrottleTargetDelay=e.autoThrottleTargetDelay??1000,this.autoThrottleMinDelay=e.autoThrottleMinDelay??100,this.autoThrottleMaxDelay=e.autoThrottleMaxDelay??60000,this.maxWaitOn429=e.maxWaitOn429??1800000,this.alwaysWaitOn429=e.alwaysWaitOn429??!1,this._addHeaders(e.headers),this._addOxylabs(e.oxylabs),this._addDecodo(e.decodo),this._addProxies(e.proxy),this._addLimiters(e.limiter)}getConfiguredDomains(e){return(e==="headers"?this.requestHeaders:e==="limiters"?this.limiters:e==="oxylabs"?this.oxylabs:this.proxies).filter((i)=>i.domain).map((i)=>i.domain).filter((i,a,r)=>r.indexOf(i)===a)}removeDomain(e){return this.requestHeaders=this.requestHeaders.filter((t)=>!t.domain||!this._domainsEqual(t.domain,e)),this.proxies=this.proxies.filter((t)=>!t.domain||!this._domainsEqual(t.domain,e)),this.limiters=this.limiters.filter((t)=>!t.domain||!this._domainsEqual(t.domain,e)),this.oxylabs=this.oxylabs.filter((t)=>!t.domain||!this._domainsEqual(t.domain,e)),this}_domainsEqual(e,t){if(Array.isArray(e)&&Array.isArray(t))return e.length===t.length&&e.every((i,a)=>i===t[a]);return e===t}getConfigurationSummary(){let e=(t)=>({total:t.length,global:t.filter((i)=>i.isGlobal).length,domainSpecific:t.filter((i)=>!i.isGlobal&&i.domain).length});return{headers:e(this.requestHeaders),proxies:e(this.proxies),limiters:e(this.limiters),oxylabs:e(this.oxylabs)}}_addHeaders(e){if(!e||!e.enable)return;for(let t of e.httpHeaders){let{domain:i,isGlobal:a,headers:r}=t;if(!i&&!a)continue;if(r instanceof Headers){let s=Object.fromEntries(r.entries());if(Object.keys(s).length<1)continue;r=s}else if(!r||Object.keys(r).length<1)continue;this.requestHeaders.push({domain:i,isGlobal:a,headers:r})}}_addProxies(e){if(!e||!e.enable)return;for(let t of e.proxies){let{domain:i,isGlobal:a,proxy:r}=t;if(!i&&!a)continue;if(!r||Object.keys(r).length<1)continue;this.proxies.push({domain:i,isGlobal:a,proxy:r})}}_addLimiters(e){if(!e||!e.enable)return;for(let t of e.limiters){let{domain:i,isGlobal:a,options:r}=t;if(!i&&!a)continue;if(!r||Object.keys(r).length<1)continue;this.limiters.push({domain:i,isGlobal:a,pqueue:new m(r)})}}_addOxylabs(e){if(!e||!e.enable)return;for(let t of e.labs){let{domain:i,isGlobal:a,options:r,queueOptions:s}=t;if(!i&&!a)continue;if(!r||Object.keys(r).length<1)continue;this.oxylabs.push({domain:i,isGlobal:a,adaptar:new x(r)})}}_addDecodo(e){if(!e||!e.enable)return;for(let t of e.labs){let{domain:i,isGlobal:a,options:r,queueOptions:s}=t;if(!i&&!a)continue;if(!r||Object.keys(r).length<1)continue;this.decodo.push({domain:i,isGlobal:a,adaptar:new y(r)})}}addHeaders(e){return this._addHeaders({enable:!0,httpHeaders:[e]}),this}addProxy(e){return this._addProxies({enable:!0,proxies:[e]}),this}addLimiter(e){return this._addLimiters({enable:!0,limiters:[e]}),this}addOxylabs(e){return this._addOxylabs({enable:!0,labs:[e]}),this}addDecodo(e){return this._addDecodo({enable:!0,labs:[e]}),this}destroyLimiters(){for(let e of this.limiters)if(e.pqueue&&typeof e.pqueue.destroy==="function")e.pqueue.destroy();this.limiters=[]}clearGlobalConfigs(){if(Array.isArray(this.requestHeaders))this.requestHeaders=this.requestHeaders.filter((e)=>!e.isGlobal);if(Array.isArray(this.oxylabs))this.oxylabs=this.oxylabs.filter((e)=>!e.isGlobal);if(Array.isArray(this.limiters))this.limiters=this.limiters.filter((e)=>!e.isGlobal);if(Array.isArray(this.proxies))this.proxies=this.proxies.filter((e)=>!e.isGlobal);return this}getAdapter(e,t,i,a){if(!this.getDomainName(e))return null;let s=[],o=t==="headers"?this.requestHeaders:t==="limiters"?this.limiters:t==="oxylabs"?this.oxylabs:t==="decodo"?this.decodo:this.proxies;for(let n=0;n<o.length;n++)if(this._hasDomain(e,o[n].domain))s.push(n);if(s.length){let n=a?s[this.rnd(0,s.length-1)]:s[0];return t==="headers"?this.requestHeaders[n].headers:t==="limiters"?this.limiters[n].pqueue:t==="oxylabs"?this.oxylabs[n].adaptar:t==="decodo"?this.decodo[n].adaptar:this.proxies[n].proxy}s.length=0;for(let n=0;n<o.length;n++)s.push(n);if(s.length){let n=a?s[this.rnd(0,s.length-1)]:s[0];if(o[n].isGlobal&&i)return t==="headers"?this.requestHeaders[n].headers:t==="limiters"?this.limiters[n].pqueue:t==="oxylabs"?this.oxylabs[n].adaptar:t==="decodo"?this.decodo[n].adaptar:this.proxies[n].proxy}return null}rnd(e=0,t=Number.MAX_VALUE){return Math.floor(Math.random()*(t-e+1))+e}hasDomain(e,t,i){if(!this.getDomainName(e))return!1;let r=t==="headers"?this.requestHeaders:t==="limiters"?this.limiters:t==="oxylabs"?this.oxylabs:t==="decodo"?this.decodo:this.proxies;for(let s=0;s<r.length;s++)if(this._hasDomain(e,r[s].domain))return!0;if(i){for(let s=0;s<r.length;s++)if(r[s].isGlobal)return!0}return!1}pickHeaders(e,t,i,a){let r=this.getAdapter(e,"headers",t),s=new Headers(r??{}),o=s.count;if(i&&i instanceof Headers)for(let[n,l]of Object.entries(i.entries()))s.set(n,l);else if(i&&typeof i==="object"){for(let[n,l]of Object.entries(i))if(typeof l==="string")s.set(n,l)}if(a&&o===0)s.set("user-agent",this.getRandomUserAgent());return Object.fromEntries(s.entries())}_hasDomain(e,t){if(!t)return!1;let i=this.getDomainName(e);if(!i)return!1;let a=(s)=>{return/[\^\$\*\+\?\{\}\[\]\(\)\|\\]/.test(s)||s.startsWith("/")||s.includes(".*")||s.includes(".+")},r=(s)=>{if(s instanceof RegExp)return s.test(i)||s.test(e);let o=s.toString().trim();if(i.toLowerCase()===o.toLowerCase())return!0;if(o.includes("*")){let h=o.replace(/[.*+?^${}()|[\]\\]/g,"\\$&").replace(/\\\*/g,".*"),c=new RegExp(`^${h}$`,"i");return c.test(i)||c.test(e)}if(a(o))try{let h=o,c="i",u=o.match(/^\/(.*)\/(\w*)$/);if(u)h=u[1],c=u[2]||"i";let d=new RegExp(h,c);return d.test(i)||d.test(e)}catch(h){return i.toLowerCase().includes(o.toLowerCase())}let n=i.toLowerCase(),l=o.toLowerCase();return n===l||n.endsWith("."+l)||l.endsWith("."+n)};if(Array.isArray(t)){for(let s of t)if(r(s))return!0;return!1}return r(t)}getDomainName(e){if(this.isValidUrl(e))return new URL(e).hostname;else if(this.isHostName(e))return e;return null}isHostName(e){if(!e)return!1;if(e.length>255)return!1;let t=/^(?:(?:[a-zA-Z0-9](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?\.)+ [a-zA-Z]{2,})$/;return e=e.trim().toLowerCase(),t.test(e)&&!e.startsWith("-")&&!e.endsWith("-")}isValidUrl(e){if(!e)return!1;e=e.trim();try{let t=new URL(e);if(!t.protocol||!["http:","https:"].includes(t.protocol.toLowerCase()))return!1;if(!t.hostname)return!1;if(!/^(?:(?:[a-zA-Z0-9](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?\.)+[a-zA-Z]{2,})$/.test(t.hostname))return!1;return!0}catch{return!1}}getRandomUserAgent(){return this.userAgents[Math.floor(Math.random()*this.userAgents.length)]}}function A(){let e=[{name:"Chrome",version:"91.0.4472.124",engine:"AppleWebKit/537.36"},{name:"Firefox",version:"89.0",engine:"Gecko/20100101"},{name:"Safari",version:"14.1.1",engine:"AppleWebKit/605.1.15"},{name:"Edge",version:"91.0.864.59",engine:"AppleWebKit/537.36"},{name:"Opera",version:"77.0.4054.277",engine:"AppleWebKit/537.36"},{name:"Vivaldi",version:"3.8.2259.42",engine:"AppleWebKit/537.36"},{name:"Brave",version:"1.26.74",engine:"AppleWebKit/537.36"},{name:"Chromium",version:"91.0.4472.101",engine:"AppleWebKit/537.36"},{name:"Yandex",version:"21.5.3.742",engine:"AppleWebKit/537.36"},{name:"Maxthon",version:"5.3.8.2000",engine:"AppleWebKit/537.36"}],t=["Windows NT 10.0","Windows NT 6.1","Macintosh; Intel Mac OS X 10_15_7","Macintosh; Intel Mac OS X 11_4_0","X11; Linux x86_64","X11; Ubuntu; Linux x86_64"],i=[];for(let a=0;a<200;a++){let r=e[Math.floor(Math.random()*e.length)],s=t[Math.floor(Math.random()*t.length)],o="";switch(r.name){case"Chrome":o=`Mozilla/5.0 (${s}) ${r.engine} (KHTML, like Gecko) Chrome/${r.version} Safari/537.36`;break;case"Firefox":o=`Mozilla/5.0 (${s}; rv:${r.version}) ${r.engine} Firefox/${r.version}`;break;case"Safari":o=`Mozilla/5.0 (${s}) ${r.engine} (KHTML, like Gecko) Version/${r.version} Safari/605.1.15`;break;case"Edge":o=`Mozilla/5.0 (${s}) ${r.engine} (KHTML, like Gecko) Chrome/${r.version} Safari/537.36 Edg/${r.version}`;break;case"Opera":o=`Mozilla/5.0 (${s}) ${r.engine} (KHTML, like Gecko) Chrome/${r.version} Safari/537.36 OPR/${r.version}`;break;case"Vivaldi":o=`Mozilla/5.0 (${s}) ${r.engine} (KHTML, like Gecko) Chrome/${r.version} Safari/537.36 Vivaldi/${r.version}`;break;case"Brave":o=`Mozilla/5.0 (${s}) ${r.engine} (KHTML, like Gecko) Chrome/${r.version} Safari/537.36 Brave/${r.version}`;break;case"Chromium":o=`Mozilla/5.0 (${s}) ${r.engine} (KHTML, like Gecko) Chromium/${r.version} Chrome/${r.version} Safari/537.36`;break;case"Yandex":o=`Mozilla/5.0 (${s}) ${r.engine} (KHTML, like Gecko) Chrome/${r.version} YaBrowser/${r.version} Safari/537.36`;break;case"Maxthon":o=`Mozilla/5.0 (${s}) ${r.engine} (KHTML, like Gecko) Chrome/${r.version} Safari/537.36 Maxthon/${r.version}`;break}i.push(o)}return i}exports.CrawlerOptions=f;

package/dist/crawler/crawler-options.js

@@ -1 +1 @@
-import{RezoQueue as m}from"../queue/queue.js";import{Oxylabs as f}from"./addon/oxylabs/index.js";import x from"node:path";import b from"node:os";import{Decodo as g}from"./addon/decodo/index.js";class y{baseUrl;adapter;enableNavigationHistory;sessionId;rejectUnauthorized;userAgent;useRndUserAgent;timeout;maxRedirects;maxRetryAttempts;retryDelay;retryOnStatusCode;forceRevisit;retryWithoutProxyOnStatusCode;retryOnProxyError;maxRetryOnProxyError;allowRevisiting;enableCache;cacheTTL;cacheDir;throwFatalError;debug;concurrency;scraperConcurrency;maxDepth;maxUrls;maxResponseSize;respectRobotsTxt;followNofollow;autoThrottle;autoThrottleTargetDelay;autoThrottleMinDelay;autoThrottleMaxDelay;maxWaitOn429;alwaysWaitOn429;oxylabs=[];decodo=[];proxies=[];limiters=[];requestHeaders=[];userAgents=A();constructor(e={}){this.baseUrl=e.baseUrl||"",this.adapter=e.adapter??"http",this.enableNavigationHistory=e.enableNavigationHistory??!1,this.sessionId=e.sessionId??`session_${Date.now()}_${Math.random().toString(36).slice(2,8)}`,this.rejectUnauthorized=e.rejectUnauthorized??!0,this.userAgent=e.userAgent,this.useRndUserAgent=e.useRndUserAgent??!1,this.timeout=e.timeout??30000,this.maxRedirects=e.maxRedirects??10,this.maxRetryAttempts=e.maxRetryAttempts??3,this.retryDelay=e.retryDelay??0,this.retryOnStatusCode=e.retryOnStatusCode??[408,429,500,502,503,504],this.forceRevisit=e.forceRevisit??!1,this.retryWithoutProxyOnStatusCode=e.retryWithoutProxyOnStatusCode??[407,403],this.retryOnProxyError=e.retryOnProxyError??!0,this.maxRetryOnProxyError=e.maxRetryOnProxyError??3,this.allowRevisiting=e.allowRevisiting??!1,this.enableCache=e.enableCache??!0,this.cacheTTL=e.cacheTTL??604800000,this.cacheDir=e.cacheDir??x.join(b.tmpdir(),"uiniqhtt_cache"),this.throwFatalError=e.throwFatalError??!1,this.debug=e.debug??!1,this.concurrency=e.concurrency??100,this.scraperConcurrency=e.scraperConcurrency??this.concurrency,this.maxDepth=e.maxDepth??0,this.maxUrls=e.maxUrls??0,this.maxResponseSize=e.maxResponseSize??0,this.respectRobotsTxt=e.respectRobotsTxt??!1,this.followNofollow=e.followNofollow??!1,this.autoThrottle=e.autoThrottle??!0,this.autoThrottleTargetDelay=e.autoThrottleTargetDelay??1000,this.autoThrottleMinDelay=e.autoThrottleMinDelay??100,this.autoThrottleMaxDelay=e.autoThrottleMaxDelay??60000,this.maxWaitOn429=e.maxWaitOn429??1800000,this.alwaysWaitOn429=e.alwaysWaitOn429??!1,this._addHeaders(e.headers),this._addOxylabs(e.oxylabs),this._addDecodo(e.decodo),this._addProxies(e.proxy),this._addLimiters(e.limiter)}getConfiguredDomains(e){return(e==="headers"?this.requestHeaders:e==="limiters"?this.limiters:e==="oxylabs"?this.oxylabs:this.proxies).filter((i)=>i.domain).map((i)=>i.domain).filter((i,a,r)=>r.indexOf(i)===a)}removeDomain(e){return this.requestHeaders=this.requestHeaders.filter((t)=>!t.domain||!this._domainsEqual(t.domain,e)),this.proxies=this.proxies.filter((t)=>!t.domain||!this._domainsEqual(t.domain,e)),this.limiters=this.limiters.filter((t)=>!t.domain||!this._domainsEqual(t.domain,e)),this.oxylabs=this.oxylabs.filter((t)=>!t.domain||!this._domainsEqual(t.domain,e)),this}_domainsEqual(e,t){if(Array.isArray(e)&&Array.isArray(t))return e.length===t.length&&e.every((i,a)=>i===t[a]);return e===t}getConfigurationSummary(){let e=(t)=>({total:t.length,global:t.filter((i)=>i.isGlobal).length,domainSpecific:t.filter((i)=>!i.isGlobal&&i.domain).length});return{headers:e(this.requestHeaders),proxies:e(this.proxies),limiters:e(this.limiters),oxylabs:e(this.oxylabs)}}_addHeaders(e){if(!e||!e.enable)return;for(let t of e.httpHeaders){let{domain:i,isGlobal:a,headers:r}=t;if(!i&&!a)continue;if(r instanceof Headers){let s=Object.fromEntries(r.entries());if(Object.keys(s).length<1)continue;r=s}else if(!r||Object.keys(r).length<1)continue;this.requestHeaders.push({domain:i,isGlobal:a,headers:r})}}_addProxies(e){if(!e||!e.enable)return;for(let t of e.proxies){let{domain:i,isGlobal:a,proxy:r}=t;if(!i&&!a)continue;if(!r||Object.keys(r).length<1)continue;this.proxies.push({domain:i,isGlobal:a,proxy:r})}}_addLimiters(e){if(!e||!e.enable)return;for(let t of e.limiters){let{domain:i,isGlobal:a,options:r}=t;if(!i&&!a)continue;if(!r||Object.keys(r).length<1)continue;this.limiters.push({domain:i,isGlobal:a,pqueue:new m(r)})}}_addOxylabs(e){if(!e||!e.enable)return;for(let t of e.labs){let{domain:i,isGlobal:a,options:r,queueOptions:s}=t;if(!i&&!a)continue;if(!r||Object.keys(r).length<1)continue;this.oxylabs.push({domain:i,isGlobal:a,adaptar:new f(r)})}}_addDecodo(e){if(!e||!e.enable)return;for(let t of e.labs){let{domain:i,isGlobal:a,options:r,queueOptions:s}=t;if(!i&&!a)continue;if(!r||Object.keys(r).length<1)continue;this.decodo.push({domain:i,isGlobal:a,adaptar:new g(r)})}}addHeaders(e){return this._addHeaders({enable:!0,httpHeaders:[e]}),this}addProxy(e){return this._addProxies({enable:!0,proxies:[e]}),this}addLimiter(e){return this._addLimiters({enable:!0,limiters:[e]}),this}addOxylabs(e){return this._addOxylabs({enable:!0,labs:[e]}),this}addDecodo(e){return this._addDecodo({enable:!0,labs:[e]}),this}destroyLimiters(){for(let e of this.limiters)if(e.pqueue&&typeof e.pqueue.destroy==="function")e.pqueue.destroy();this.limiters=[]}clearGlobalConfigs(){if(Array.isArray(this.requestHeaders))this.requestHeaders=this.requestHeaders.filter((e)=>!e.isGlobal);if(Array.isArray(this.oxylabs))this.oxylabs=this.oxylabs.filter((e)=>!e.isGlobal);if(Array.isArray(this.limiters))this.limiters=this.limiters.filter((e)=>!e.isGlobal);if(Array.isArray(this.proxies))this.proxies=this.proxies.filter((e)=>!e.isGlobal);return this}getAdapter(e,t,i,a){if(!this.getDomainName(e))return null;let s=[],n=t==="headers"?this.requestHeaders:t==="limiters"?this.limiters:t==="oxylabs"?this.oxylabs:t==="decodo"?this.decodo:this.proxies;for(let o=0;o<n.length;o++)if(this._hasDomain(e,n[o].domain))s.push(o);if(s.length){let o=a?s[this.rnd(0,s.length-1)]:s[0];return t==="headers"?this.requestHeaders[o].headers:t==="limiters"?this.limiters[o].pqueue:t==="oxylabs"?this.oxylabs[o].adaptar:t==="decodo"?this.decodo[o].adaptar:this.proxies[o].proxy}s.length=0;for(let o=0;o<n.length;o++)s.push(o);if(s.length){let o=a?s[this.rnd(0,s.length-1)]:s[0];if(n[o].isGlobal&&i)return t==="headers"?this.requestHeaders[o].headers:t==="limiters"?this.limiters[o].pqueue:t==="oxylabs"?this.oxylabs[o].adaptar:t==="decodo"?this.decodo[o].adaptar:this.proxies[o].proxy}return null}rnd(e=0,t=Number.MAX_VALUE){return Math.floor(Math.random()*(t-e+1))+e}hasDomain(e,t,i){if(!this.getDomainName(e))return!1;let r=t==="headers"?this.requestHeaders:t==="limiters"?this.limiters:t==="oxylabs"?this.oxylabs:t==="decodo"?this.decodo:this.proxies;for(let s=0;s<r.length;s++)if(this._hasDomain(e,r[s].domain))return!0;if(i){for(let s=0;s<r.length;s++)if(r[s].isGlobal)return!0}return!1}pickHeaders(e,t,i,a){let r=this.getAdapter(e,"headers",t),s=new Headers(r??{}),n=s.count;if(i&&i instanceof Headers)for(let[o,l]of Object.entries(i.entries()))s.set(o,l);else if(i&&typeof i==="object"){for(let[o,l]of Object.entries(i))if(typeof l==="string")s.set(o,l)}if(a&&n===0)s.set("user-agent",this.getRandomUserAgent());return Object.fromEntries(s.entries())}_hasDomain(e,t){if(!t)return!1;let i=this.getDomainName(e);if(!i)return!1;let a=(s)=>{return/[\^\$\*\+\?\{\}\[\]\(\)\|\\]/.test(s)||s.startsWith("/")||s.includes(".*")||s.includes(".+")},r=(s)=>{if(s instanceof RegExp)return s.test(i)||s.test(e);let n=s.toString().trim();if(i.toLowerCase()===n.toLowerCase())return!0;if(n.includes("*")){let h=n.replace(/[.*+?^${}()|[\]\\]/g,"\\$&").replace(/\\\*/g,".*"),c=new RegExp(`^${h}$`,"i");return c.test(i)||c.test(e)}if(a(n))try{let h=n,c="i",u=n.match(/^\/(.*)\/(\w*)$/);if(u)h=u[1],c=u[2]||"i";let d=new RegExp(h,c);return d.test(i)||d.test(e)}catch(h){return i.toLowerCase().includes(n.toLowerCase())}let o=i.toLowerCase(),l=n.toLowerCase();return o===l||o.endsWith("."+l)||l.endsWith("."+o)};if(Array.isArray(t)){for(let s of t)if(r(s))return!0;return!1}return r(t)}getDomainName(e){if(this.isValidUrl(e))return new URL(e).hostname;else if(this.isHostName(e))return e;return null}isHostName(e){if(!e)return!1;if(e.length>255)return!1;let t=/^(?:(?:[a-zA-Z0-9](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?\.)+ [a-zA-Z]{2,})$/;return e=e.trim().toLowerCase(),t.test(e)&&!e.startsWith("-")&&!e.endsWith("-")}isValidUrl(e){if(!e)return!1;e=e.trim();try{let t=new URL(e);if(!t.protocol||!["http:","https:"].includes(t.protocol.toLowerCase()))return!1;if(!t.hostname)return!1;if(!/^(?:(?:[a-zA-Z0-9](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?\.)+[a-zA-Z]{2,})$/.test(t.hostname))return!1;return!0}catch{return!1}}getRandomUserAgent(){return this.userAgents[Math.floor(Math.random()*this.userAgents.length)]}}function A(){let e=[{name:"Chrome",version:"91.0.4472.124",engine:"AppleWebKit/537.36"},{name:"Firefox",version:"89.0",engine:"Gecko/20100101"},{name:"Safari",version:"14.1.1",engine:"AppleWebKit/605.1.15"},{name:"Edge",version:"91.0.864.59",engine:"AppleWebKit/537.36"},{name:"Opera",version:"77.0.4054.277",engine:"AppleWebKit/537.36"},{name:"Vivaldi",version:"3.8.2259.42",engine:"AppleWebKit/537.36"},{name:"Brave",version:"1.26.74",engine:"AppleWebKit/537.36"},{name:"Chromium",version:"91.0.4472.101",engine:"AppleWebKit/537.36"},{name:"Yandex",version:"21.5.3.742",engine:"AppleWebKit/537.36"},{name:"Maxthon",version:"5.3.8.2000",engine:"AppleWebKit/537.36"}],t=["Windows NT 10.0","Windows NT 6.1","Macintosh; Intel Mac OS X 10_15_7","Macintosh; Intel Mac OS X 11_4_0","X11; Linux x86_64","X11; Ubuntu; Linux x86_64"],i=[];for(let a=0;a<200;a++){let r=e[Math.floor(Math.random()*e.length)],s=t[Math.floor(Math.random()*t.length)],n="";switch(r.name){case"Chrome":n=`Mozilla/5.0 (${s}) ${r.engine} (KHTML, like Gecko) Chrome/${r.version} Safari/537.36`;break;case"Firefox":n=`Mozilla/5.0 (${s}; rv:${r.version}) ${r.engine} Firefox/${r.version}`;break;case"Safari":n=`Mozilla/5.0 (${s}) ${r.engine} (KHTML, like Gecko) Version/${r.version} Safari/605.1.15`;break;case"Edge":n=`Mozilla/5.0 (${s}) ${r.engine} (KHTML, like Gecko) Chrome/${r.version} Safari/537.36 Edg/${r.version}`;break;case"Opera":n=`Mozilla/5.0 (${s}) ${r.engine} (KHTML, like Gecko) Chrome/${r.version} Safari/537.36 OPR/${r.version}`;break;case"Vivaldi":n=`Mozilla/5.0 (${s}) ${r.engine} (KHTML, like Gecko) Chrome/${r.version} Safari/537.36 Vivaldi/${r.version}`;break;case"Brave":n=`Mozilla/5.0 (${s}) ${r.engine} (KHTML, like Gecko) Chrome/${r.version} Safari/537.36 Brave/${r.version}`;break;case"Chromium":n=`Mozilla/5.0 (${s}) ${r.engine} (KHTML, like Gecko) Chromium/${r.version} Chrome/${r.version} Safari/537.36`;break;case"Yandex":n=`Mozilla/5.0 (${s}) ${r.engine} (KHTML, like Gecko) Chrome/${r.version} YaBrowser/${r.version} Safari/537.36`;break;case"Maxthon":n=`Mozilla/5.0 (${s}) ${r.engine} (KHTML, like Gecko) Chrome/${r.version} Safari/537.36 Maxthon/${r.version}`;break}i.push(n)}return i}export{y as CrawlerOptions};
+import{RezoQueue as f}from"../queue/queue.js";import{Oxylabs as m}from"./addon/oxylabs/index.js";import x from"node:path";import b from"node:os";import{Decodo as g}from"./addon/decodo/index.js";class y{baseUrl;adapter;enableNavigationHistory;sessionId;rejectUnauthorized;userAgent;useRndUserAgent;timeout;maxRedirects;maxRetryAttempts;retryDelay;retryOnStatusCode;forceRevisit;retryWithoutProxyOnStatusCode;retryOnProxyError;maxRetryOnProxyError;allowRevisiting;enableCache;cacheTTL;cacheDir;throwFatalError;debug;enableSignalHandlers;concurrency;scraperConcurrency;maxDepth;maxUrls;maxResponseSize;respectRobotsTxt;followNofollow;autoThrottle;autoThrottleTargetDelay;autoThrottleMinDelay;autoThrottleMaxDelay;maxWaitOn429;alwaysWaitOn429;oxylabs=[];decodo=[];proxies=[];limiters=[];requestHeaders=[];userAgents=A();constructor(e={}){this.baseUrl=e.baseUrl||"",this.adapter=e.adapter??"http",this.enableNavigationHistory=e.enableNavigationHistory??!1,this.sessionId=e.sessionId??`session_${Date.now()}_${Math.random().toString(36).slice(2,8)}`,this.rejectUnauthorized=e.rejectUnauthorized??!0,this.userAgent=e.userAgent,this.useRndUserAgent=e.useRndUserAgent??!1,this.timeout=e.timeout??30000,this.maxRedirects=e.maxRedirects??10,this.maxRetryAttempts=e.maxRetryAttempts??3,this.retryDelay=e.retryDelay??0,this.retryOnStatusCode=e.retryOnStatusCode??[408,429,500,502,503,504],this.forceRevisit=e.forceRevisit??!1,this.retryWithoutProxyOnStatusCode=e.retryWithoutProxyOnStatusCode??[407,403],this.retryOnProxyError=e.retryOnProxyError??!0,this.maxRetryOnProxyError=e.maxRetryOnProxyError??3,this.allowRevisiting=e.allowRevisiting??!1,this.enableCache=e.enableCache??!0,this.cacheTTL=e.cacheTTL??604800000,this.cacheDir=e.cacheDir??x.join(b.tmpdir(),"uiniqhtt_cache"),this.throwFatalError=e.throwFatalError??!1,this.debug=e.debug??!1,this.enableSignalHandlers=e.enableSignalHandlers??!1,this.concurrency=e.concurrency??100,this.scraperConcurrency=e.scraperConcurrency??this.concurrency,this.maxDepth=e.maxDepth??0,this.maxUrls=e.maxUrls??0,this.maxResponseSize=e.maxResponseSize??0,this.respectRobotsTxt=e.respectRobotsTxt??!1,this.followNofollow=e.followNofollow??!1,this.autoThrottle=e.autoThrottle??!0,this.autoThrottleTargetDelay=e.autoThrottleTargetDelay??1000,this.autoThrottleMinDelay=e.autoThrottleMinDelay??100,this.autoThrottleMaxDelay=e.autoThrottleMaxDelay??60000,this.maxWaitOn429=e.maxWaitOn429??1800000,this.alwaysWaitOn429=e.alwaysWaitOn429??!1,this._addHeaders(e.headers),this._addOxylabs(e.oxylabs),this._addDecodo(e.decodo),this._addProxies(e.proxy),this._addLimiters(e.limiter)}getConfiguredDomains(e){return(e==="headers"?this.requestHeaders:e==="limiters"?this.limiters:e==="oxylabs"?this.oxylabs:this.proxies).filter((i)=>i.domain).map((i)=>i.domain).filter((i,a,r)=>r.indexOf(i)===a)}removeDomain(e){return this.requestHeaders=this.requestHeaders.filter((t)=>!t.domain||!this._domainsEqual(t.domain,e)),this.proxies=this.proxies.filter((t)=>!t.domain||!this._domainsEqual(t.domain,e)),this.limiters=this.limiters.filter((t)=>!t.domain||!this._domainsEqual(t.domain,e)),this.oxylabs=this.oxylabs.filter((t)=>!t.domain||!this._domainsEqual(t.domain,e)),this}_domainsEqual(e,t){if(Array.isArray(e)&&Array.isArray(t))return e.length===t.length&&e.every((i,a)=>i===t[a]);return e===t}getConfigurationSummary(){let e=(t)=>({total:t.length,global:t.filter((i)=>i.isGlobal).length,domainSpecific:t.filter((i)=>!i.isGlobal&&i.domain).length});return{headers:e(this.requestHeaders),proxies:e(this.proxies),limiters:e(this.limiters),oxylabs:e(this.oxylabs)}}_addHeaders(e){if(!e||!e.enable)return;for(let t of e.httpHeaders){let{domain:i,isGlobal:a,headers:r}=t;if(!i&&!a)continue;if(r instanceof Headers){let s=Object.fromEntries(r.entries());if(Object.keys(s).length<1)continue;r=s}else if(!r||Object.keys(r).length<1)continue;this.requestHeaders.push({domain:i,isGlobal:a,headers:r})}}_addProxies(e){if(!e||!e.enable)return;for(let t of e.proxies){let{domain:i,isGlobal:a,proxy:r}=t;if(!i&&!a)continue;if(!r||Object.keys(r).length<1)continue;this.proxies.push({domain:i,isGlobal:a,proxy:r})}}_addLimiters(e){if(!e||!e.enable)return;for(let t of e.limiters){let{domain:i,isGlobal:a,options:r}=t;if(!i&&!a)continue;if(!r||Object.keys(r).length<1)continue;this.limiters.push({domain:i,isGlobal:a,pqueue:new f(r)})}}_addOxylabs(e){if(!e||!e.enable)return;for(let t of e.labs){let{domain:i,isGlobal:a,options:r,queueOptions:s}=t;if(!i&&!a)continue;if(!r||Object.keys(r).length<1)continue;this.oxylabs.push({domain:i,isGlobal:a,adaptar:new m(r)})}}_addDecodo(e){if(!e||!e.enable)return;for(let t of e.labs){let{domain:i,isGlobal:a,options:r,queueOptions:s}=t;if(!i&&!a)continue;if(!r||Object.keys(r).length<1)continue;this.decodo.push({domain:i,isGlobal:a,adaptar:new g(r)})}}addHeaders(e){return this._addHeaders({enable:!0,httpHeaders:[e]}),this}addProxy(e){return this._addProxies({enable:!0,proxies:[e]}),this}addLimiter(e){return this._addLimiters({enable:!0,limiters:[e]}),this}addOxylabs(e){return this._addOxylabs({enable:!0,labs:[e]}),this}addDecodo(e){return this._addDecodo({enable:!0,labs:[e]}),this}destroyLimiters(){for(let e of this.limiters)if(e.pqueue&&typeof e.pqueue.destroy==="function")e.pqueue.destroy();this.limiters=[]}clearGlobalConfigs(){if(Array.isArray(this.requestHeaders))this.requestHeaders=this.requestHeaders.filter((e)=>!e.isGlobal);if(Array.isArray(this.oxylabs))this.oxylabs=this.oxylabs.filter((e)=>!e.isGlobal);if(Array.isArray(this.limiters))this.limiters=this.limiters.filter((e)=>!e.isGlobal);if(Array.isArray(this.proxies))this.proxies=this.proxies.filter((e)=>!e.isGlobal);return this}getAdapter(e,t,i,a){if(!this.getDomainName(e))return null;let s=[],o=t==="headers"?this.requestHeaders:t==="limiters"?this.limiters:t==="oxylabs"?this.oxylabs:t==="decodo"?this.decodo:this.proxies;for(let n=0;n<o.length;n++)if(this._hasDomain(e,o[n].domain))s.push(n);if(s.length){let n=a?s[this.rnd(0,s.length-1)]:s[0];return t==="headers"?this.requestHeaders[n].headers:t==="limiters"?this.limiters[n].pqueue:t==="oxylabs"?this.oxylabs[n].adaptar:t==="decodo"?this.decodo[n].adaptar:this.proxies[n].proxy}s.length=0;for(let n=0;n<o.length;n++)s.push(n);if(s.length){let n=a?s[this.rnd(0,s.length-1)]:s[0];if(o[n].isGlobal&&i)return t==="headers"?this.requestHeaders[n].headers:t==="limiters"?this.limiters[n].pqueue:t==="oxylabs"?this.oxylabs[n].adaptar:t==="decodo"?this.decodo[n].adaptar:this.proxies[n].proxy}return null}rnd(e=0,t=Number.MAX_VALUE){return Math.floor(Math.random()*(t-e+1))+e}hasDomain(e,t,i){if(!this.getDomainName(e))return!1;let r=t==="headers"?this.requestHeaders:t==="limiters"?this.limiters:t==="oxylabs"?this.oxylabs:t==="decodo"?this.decodo:this.proxies;for(let s=0;s<r.length;s++)if(this._hasDomain(e,r[s].domain))return!0;if(i){for(let s=0;s<r.length;s++)if(r[s].isGlobal)return!0}return!1}pickHeaders(e,t,i,a){let r=this.getAdapter(e,"headers",t),s=new Headers(r??{}),o=s.count;if(i&&i instanceof Headers)for(let[n,l]of Object.entries(i.entries()))s.set(n,l);else if(i&&typeof i==="object"){for(let[n,l]of Object.entries(i))if(typeof l==="string")s.set(n,l)}if(a&&o===0)s.set("user-agent",this.getRandomUserAgent());return Object.fromEntries(s.entries())}_hasDomain(e,t){if(!t)return!1;let i=this.getDomainName(e);if(!i)return!1;let a=(s)=>{return/[\^\$\*\+\?\{\}\[\]\(\)\|\\]/.test(s)||s.startsWith("/")||s.includes(".*")||s.includes(".+")},r=(s)=>{if(s instanceof RegExp)return s.test(i)||s.test(e);let o=s.toString().trim();if(i.toLowerCase()===o.toLowerCase())return!0;if(o.includes("*")){let h=o.replace(/[.*+?^${}()|[\]\\]/g,"\\$&").replace(/\\\*/g,".*"),c=new RegExp(`^${h}$`,"i");return c.test(i)||c.test(e)}if(a(o))try{let h=o,c="i",d=o.match(/^\/(.*)\/(\w*)$/);if(d)h=d[1],c=d[2]||"i";let u=new RegExp(h,c);return u.test(i)||u.test(e)}catch(h){return i.toLowerCase().includes(o.toLowerCase())}let n=i.toLowerCase(),l=o.toLowerCase();return n===l||n.endsWith("."+l)||l.endsWith("."+n)};if(Array.isArray(t)){for(let s of t)if(r(s))return!0;return!1}return r(t)}getDomainName(e){if(this.isValidUrl(e))return new URL(e).hostname;else if(this.isHostName(e))return e;return null}isHostName(e){if(!e)return!1;if(e.length>255)return!1;let t=/^(?:(?:[a-zA-Z0-9](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?\.)+ [a-zA-Z]{2,})$/;return e=e.trim().toLowerCase(),t.test(e)&&!e.startsWith("-")&&!e.endsWith("-")}isValidUrl(e){if(!e)return!1;e=e.trim();try{let t=new URL(e);if(!t.protocol||!["http:","https:"].includes(t.protocol.toLowerCase()))return!1;if(!t.hostname)return!1;if(!/^(?:(?:[a-zA-Z0-9](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?\.)+[a-zA-Z]{2,})$/.test(t.hostname))return!1;return!0}catch{return!1}}getRandomUserAgent(){return this.userAgents[Math.floor(Math.random()*this.userAgents.length)]}}function A(){let e=[{name:"Chrome",version:"91.0.4472.124",engine:"AppleWebKit/537.36"},{name:"Firefox",version:"89.0",engine:"Gecko/20100101"},{name:"Safari",version:"14.1.1",engine:"AppleWebKit/605.1.15"},{name:"Edge",version:"91.0.864.59",engine:"AppleWebKit/537.36"},{name:"Opera",version:"77.0.4054.277",engine:"AppleWebKit/537.36"},{name:"Vivaldi",version:"3.8.2259.42",engine:"AppleWebKit/537.36"},{name:"Brave",version:"1.26.74",engine:"AppleWebKit/537.36"},{name:"Chromium",version:"91.0.4472.101",engine:"AppleWebKit/537.36"},{name:"Yandex",version:"21.5.3.742",engine:"AppleWebKit/537.36"},{name:"Maxthon",version:"5.3.8.2000",engine:"AppleWebKit/537.36"}],t=["Windows NT 10.0","Windows NT 6.1","Macintosh; Intel Mac OS X 10_15_7","Macintosh; Intel Mac OS X 11_4_0","X11; Linux x86_64","X11; Ubuntu; Linux x86_64"],i=[];for(let a=0;a<200;a++){let r=e[Math.floor(Math.random()*e.length)],s=t[Math.floor(Math.random()*t.length)],o="";switch(r.name){case"Chrome":o=`Mozilla/5.0 (${s}) ${r.engine} (KHTML, like Gecko) Chrome/${r.version} Safari/537.36`;break;case"Firefox":o=`Mozilla/5.0 (${s}; rv:${r.version}) ${r.engine} Firefox/${r.version}`;break;case"Safari":o=`Mozilla/5.0 (${s}) ${r.engine} (KHTML, like Gecko) Version/${r.version} Safari/605.1.15`;break;case"Edge":o=`Mozilla/5.0 (${s}) ${r.engine} (KHTML, like Gecko) Chrome/${r.version} Safari/537.36 Edg/${r.version}`;break;case"Opera":o=`Mozilla/5.0 (${s}) ${r.engine} (KHTML, like Gecko) Chrome/${r.version} Safari/537.36 OPR/${r.version}`;break;case"Vivaldi":o=`Mozilla/5.0 (${s}) ${r.engine} (KHTML, like Gecko) Chrome/${r.version} Safari/537.36 Vivaldi/${r.version}`;break;case"Brave":o=`Mozilla/5.0 (${s}) ${r.engine} (KHTML, like Gecko) Chrome/${r.version} Safari/537.36 Brave/${r.version}`;break;case"Chromium":o=`Mozilla/5.0 (${s}) ${r.engine} (KHTML, like Gecko) Chromium/${r.version} Chrome/${r.version} Safari/537.36`;break;case"Yandex":o=`Mozilla/5.0 (${s}) ${r.engine} (KHTML, like Gecko) Chrome/${r.version} YaBrowser/${r.version} Safari/537.36`;break;case"Maxthon":o=`Mozilla/5.0 (${s}) ${r.engine} (KHTML, like Gecko) Chrome/${r.version} Safari/537.36 Maxthon/${r.version}`;break}i.push(o)}return i}export{y as CrawlerOptions};

package/dist/crawler/crawler.cjs

@@ -9,7 +9,7 @@ const { CappedMap } = require('./plugin/capped-map.cjs');
 const { CappedArray } = require('./plugin/capped-array.cjs');
 const { parseHTML } = require("linkedom");
 const path = require("node:path");
-const { Rezo } = require('../adapters/entries/http.cjs');
+const rezo = require('../adapters/entries/http.cjs');
 const { RezoQueue } = require('../queue/queue.cjs');
 const { Scraper } = require('./scraper.cjs');
 const { CrawlerOptions } = require('./crawler-options.cjs');
@@ -94,19 +94,17 @@ class Crawler {
   });
   crawlStarted = false;
   startHandlersPromise = null;
-  constructor(crawlerOptions, http = new Rezo) {
+  constructor(crawlerOptions, http = rezo.create()) {
     this.http = http;
     this.config = new CrawlerOptions(crawlerOptions);
     this.adapterType = this.config.adapter;
     const concurrency = this.config.concurrency;
     this.queue = new RezoQueue({
-      concurrency,
-      timeout: 60000
+      concurrency
     });
     this.originalConcurrency = concurrency;
     this.scraperQueue = new RezoQueue({
-      concurrency: this.config.scraperConcurrency,
-      timeout: 60000
+      concurrency: this.config.scraperConcurrency
     });
     this.memoryMonitor = new MemoryMonitor({ warningRatio: 0.7, criticalRatio: 0.85 });
     this.healthMetrics = new HealthMetrics({ windowSize: 60000 });
@@ -178,7 +176,9 @@ class Crawler {
     if (this.config.baseUrl) {
       this.urlDepthMap.set(this.config.baseUrl, 0);
     }
-    this.registerShutdownHandlers();
+    if (this.config.enableSignalHandlers) {
+      this.registerShutdownHandlers();
+    }
   }
   registerShutdownHandlers() {
     if (this.shutdownHandler)
@@ -227,6 +227,7 @@ class Crawler {
     }
     await this.destroy();
     console.log("[Crawler] Graceful shutdown complete");
+    process.exit(0);
   }
   async initializeAdapter() {
     try {
@@ -1056,8 +1057,6 @@ class Crawler {
       const headersObj = headers instanceof Headers ? Object.fromEntries(headers.entries()) : headers;
       this.addToNavigationQueue(url, method, body, headersObj);
     }
-    if (url.includes(`/www.yellowpages.com/search?`))
-      console.log("Visiting: ", url);
     this.crawlStarted = true;
     if (deepEmailFinder) {
       this.execute2(method, url, body, _options, forceRevisit, emailMetadata);
@@ -1133,7 +1132,7 @@ class Crawler {
           return;
       }
       const requestStartTime = Date.now();
-      const response = cache && method === "GET" && !skipCache ? cache : oxylabsInstanse && oxylabsOptions ? await oxylabsInstanse.scrape(url) : decodoInstanse && decodoOptions ? await decodoInstanse.scrape(url) : await (method === "GET" ? this.http.get(url, options) : method === "PATCH" ? this.http.patch(url, body, options) : method === "POST" ? this.http.post(url, body, options) : this.http.put(url, body, options));
+      const response = cache && method === "GET" && !skipCache ? cache : oxylabsInstanse && oxylabsOptions ? await oxylabsInstanse.scrape(url) : decodoInstanse && decodoOptions ? await decodoInstanse.scrape(url) : method === "GET" ? await this.http.get(url, options) : method === "PATCH" ? await this.http.patch(url, body, options) : method === "POST" ? await this.http.post(url, body, options) : await this.http.put(url, body, options);
       if (!response) {
         this.crawlStats.urlsFailed++;
         this.healthMetrics.recordRequest(Date.now() - requestStartTime, false);
@@ -1268,6 +1267,17 @@ class Crawler {
     }
   }
   async waitForAll() {
+    if (!this.crawlStarted) {
+      await new Promise((resolve) => setImmediate(resolve));
+      const maxWaitForStart = 1000;
+      const startWait = Date.now();
+      while (!this.crawlStarted && Date.now() - startWait < maxWaitForStart) {
+        await new Promise((resolve) => setTimeout(resolve, 10));
+      }
+      if (!this.crawlStarted) {
+        return;
+      }
+    }
     const MIN_DELAY = 50;
     const MAX_DELAY = 500;
     let currentDelay = MIN_DELAY;

package/dist/crawler/crawler.js

@@ -9,7 +9,7 @@ import { CappedMap } from './plugin/capped-map.js';
 import { CappedArray } from './plugin/capped-array.js';
 import { parseHTML } from "linkedom";
 import path from "node:path";
-import { Rezo } from '../adapters/entries/http.js';
+import rezo from '../adapters/entries/http.js';
 import { RezoQueue } from '../queue/queue.js';
 import { Scraper } from './scraper.js';
 import { CrawlerOptions } from './crawler-options.js';
@@ -94,19 +94,17 @@ export class Crawler {
   });
   crawlStarted = false;
   startHandlersPromise = null;
-  constructor(crawlerOptions, http = new Rezo) {
+  constructor(crawlerOptions, http = rezo.create()) {
     this.http = http;
     this.config = new CrawlerOptions(crawlerOptions);
     this.adapterType = this.config.adapter;
     const concurrency = this.config.concurrency;
     this.queue = new RezoQueue({
-      concurrency,
-      timeout: 60000
+      concurrency
     });
     this.originalConcurrency = concurrency;
     this.scraperQueue = new RezoQueue({
-      concurrency: this.config.scraperConcurrency,
-      timeout: 60000
+      concurrency: this.config.scraperConcurrency
     });
     this.memoryMonitor = new MemoryMonitor({ warningRatio: 0.7, criticalRatio: 0.85 });
     this.healthMetrics = new HealthMetrics({ windowSize: 60000 });
@@ -178,7 +176,9 @@ export class Crawler {
     if (this.config.baseUrl) {
       this.urlDepthMap.set(this.config.baseUrl, 0);
     }
-    this.registerShutdownHandlers();
+    if (this.config.enableSignalHandlers) {
+      this.registerShutdownHandlers();
+    }
   }
   registerShutdownHandlers() {
     if (this.shutdownHandler)
@@ -227,6 +227,7 @@ export class Crawler {
     }
     await this.destroy();
     console.log("[Crawler] Graceful shutdown complete");
+    process.exit(0);
   }
   async initializeAdapter() {
     try {
@@ -1056,8 +1057,6 @@ export class Crawler {
       const headersObj = headers instanceof Headers ? Object.fromEntries(headers.entries()) : headers;
       this.addToNavigationQueue(url, method, body, headersObj);
     }
-    if (url.includes(`/www.yellowpages.com/search?`))
-      console.log("Visiting: ", url);
     this.crawlStarted = true;
     if (deepEmailFinder) {
       this.execute2(method, url, body, _options, forceRevisit, emailMetadata);
@@ -1133,7 +1132,7 @@ export class Crawler {
           return;
       }
       const requestStartTime = Date.now();
-      const response = cache && method === "GET" && !skipCache ? cache : oxylabsInstanse && oxylabsOptions ? await oxylabsInstanse.scrape(url) : decodoInstanse && decodoOptions ? await decodoInstanse.scrape(url) : await (method === "GET" ? this.http.get(url, options) : method === "PATCH" ? this.http.patch(url, body, options) : method === "POST" ? this.http.post(url, body, options) : this.http.put(url, body, options));
+      const response = cache && method === "GET" && !skipCache ? cache : oxylabsInstanse && oxylabsOptions ? await oxylabsInstanse.scrape(url) : decodoInstanse && decodoOptions ? await decodoInstanse.scrape(url) : method === "GET" ? await this.http.get(url, options) : method === "PATCH" ? await this.http.patch(url, body, options) : method === "POST" ? await this.http.post(url, body, options) : await this.http.put(url, body, options);
       if (!response) {
         this.crawlStats.urlsFailed++;
         this.healthMetrics.recordRequest(Date.now() - requestStartTime, false);
@@ -1268,6 +1267,17 @@ export class Crawler {
     }
   }
   async waitForAll() {
+    if (!this.crawlStarted) {
+      await new Promise((resolve) => setImmediate(resolve));
+      const maxWaitForStart = 1000;
+      const startWait = Date.now();
+      while (!this.crawlStarted && Date.now() - startWait < maxWaitForStart) {
+        await new Promise((resolve) => setTimeout(resolve, 10));
+      }
+      if (!this.crawlStarted) {
+        return;
+      }
+    }
     const MIN_DELAY = 50;
     const MAX_DELAY = 500;
     let currentDelay = MIN_DELAY;

package/dist/crawler/index.cjs

@@ -1,40 +1,40 @@
-const _mod_7qsaj1 = require('./crawler.cjs');
-exports.Crawler = _mod_7qsaj1.Crawler;;
-const _mod_5rlu0e = require('./crawler-options.cjs');
-exports.CrawlerOptions = _mod_5rlu0e.CrawlerOptions;;
-const _mod_skia2o = require('./plugin/robots-txt.cjs');
-exports.RobotsTxt = _mod_skia2o.RobotsTxt;;
-const _mod_w2aaxb = require('./plugin/file-cacher.cjs');
-exports.FileCacher = _mod_w2aaxb.FileCacher;;
-const _mod_f9de8f = require('./plugin/url-store.cjs');
-exports.UrlStore = _mod_f9de8f.UrlStore;;
-const _mod_wqtwlm = require('./plugin/navigation-history.cjs');
-exports.NavigationHistory = _mod_wqtwlm.NavigationHistory;;
-const _mod_hj9vi3 = require('./addon/oxylabs/index.cjs');
-exports.Oxylabs = _mod_hj9vi3.Oxylabs;;
-const _mod_7kzws5 = require('./addon/oxylabs/options.cjs');
-exports.OXYLABS_BROWSER_TYPES = _mod_7kzws5.OXYLABS_BROWSER_TYPES;
-exports.OXYLABS_COMMON_LOCALES = _mod_7kzws5.OXYLABS_COMMON_LOCALES;
-exports.OXYLABS_COMMON_GEO_LOCATIONS = _mod_7kzws5.OXYLABS_COMMON_GEO_LOCATIONS;
-exports.OXYLABS_US_STATES = _mod_7kzws5.OXYLABS_US_STATES;
-exports.OXYLABS_EUROPEAN_COUNTRIES = _mod_7kzws5.OXYLABS_EUROPEAN_COUNTRIES;
-exports.OXYLABS_ASIAN_COUNTRIES = _mod_7kzws5.OXYLABS_ASIAN_COUNTRIES;
-exports.getRandomOxylabsBrowserType = _mod_7kzws5.getRandomBrowserType;
-exports.getRandomOxylabsLocale = _mod_7kzws5.getRandomLocale;
-exports.getRandomOxylabsGeoLocation = _mod_7kzws5.getRandomGeoLocation;;
-const _mod_gdeyg9 = require('./addon/decodo/index.cjs');
-exports.Decodo = _mod_gdeyg9.Decodo;;
-const _mod_jbu0zi = require('./addon/decodo/options.cjs');
-exports.DECODO_DEVICE_TYPES = _mod_jbu0zi.DECODO_DEVICE_TYPES;
-exports.DECODO_HEADLESS_MODES = _mod_jbu0zi.DECODO_HEADLESS_MODES;
-exports.DECODO_COMMON_LOCALES = _mod_jbu0zi.DECODO_COMMON_LOCALES;
-exports.DECODO_COMMON_COUNTRIES = _mod_jbu0zi.DECODO_COMMON_COUNTRIES;
-exports.DECODO_EUROPEAN_COUNTRIES = _mod_jbu0zi.DECODO_EUROPEAN_COUNTRIES;
-exports.DECODO_ASIAN_COUNTRIES = _mod_jbu0zi.DECODO_ASIAN_COUNTRIES;
-exports.DECODO_US_STATES = _mod_jbu0zi.DECODO_US_STATES;
-exports.DECODO_COMMON_CITIES = _mod_jbu0zi.DECODO_COMMON_CITIES;
-exports.getRandomDecodoDeviceType = _mod_jbu0zi.getRandomDeviceType;
-exports.getRandomDecodoLocale = _mod_jbu0zi.getRandomLocale;
-exports.getRandomDecodoCountry = _mod_jbu0zi.getRandomCountry;
-exports.getRandomDecodoCity = _mod_jbu0zi.getRandomCity;
-exports.generateDecodoSessionId = _mod_jbu0zi.generateSessionId;;
+const _mod_6mpp6d = require('./crawler.cjs');
+exports.Crawler = _mod_6mpp6d.Crawler;;
+const _mod_eyxxn5 = require('./crawler-options.cjs');
+exports.CrawlerOptions = _mod_eyxxn5.CrawlerOptions;;
+const _mod_dzyzre = require('./plugin/robots-txt.cjs');
+exports.RobotsTxt = _mod_dzyzre.RobotsTxt;;
+const _mod_8zpwd3 = require('./plugin/file-cacher.cjs');
+exports.FileCacher = _mod_8zpwd3.FileCacher;;
+const _mod_ebtdn9 = require('./plugin/url-store.cjs');
+exports.UrlStore = _mod_ebtdn9.UrlStore;;
+const _mod_heyy3y = require('./plugin/navigation-history.cjs');
+exports.NavigationHistory = _mod_heyy3y.NavigationHistory;;
+const _mod_rv4zsw = require('./addon/oxylabs/index.cjs');
+exports.Oxylabs = _mod_rv4zsw.Oxylabs;;
+const _mod_jj4yar = require('./addon/oxylabs/options.cjs');
+exports.OXYLABS_BROWSER_TYPES = _mod_jj4yar.OXYLABS_BROWSER_TYPES;
+exports.OXYLABS_COMMON_LOCALES = _mod_jj4yar.OXYLABS_COMMON_LOCALES;
+exports.OXYLABS_COMMON_GEO_LOCATIONS = _mod_jj4yar.OXYLABS_COMMON_GEO_LOCATIONS;
+exports.OXYLABS_US_STATES = _mod_jj4yar.OXYLABS_US_STATES;
+exports.OXYLABS_EUROPEAN_COUNTRIES = _mod_jj4yar.OXYLABS_EUROPEAN_COUNTRIES;
+exports.OXYLABS_ASIAN_COUNTRIES = _mod_jj4yar.OXYLABS_ASIAN_COUNTRIES;
+exports.getRandomOxylabsBrowserType = _mod_jj4yar.getRandomBrowserType;
+exports.getRandomOxylabsLocale = _mod_jj4yar.getRandomLocale;
+exports.getRandomOxylabsGeoLocation = _mod_jj4yar.getRandomGeoLocation;;
+const _mod_7fmx1b = require('./addon/decodo/index.cjs');
+exports.Decodo = _mod_7fmx1b.Decodo;;
+const _mod_4dxktf = require('./addon/decodo/options.cjs');
+exports.DECODO_DEVICE_TYPES = _mod_4dxktf.DECODO_DEVICE_TYPES;
+exports.DECODO_HEADLESS_MODES = _mod_4dxktf.DECODO_HEADLESS_MODES;
+exports.DECODO_COMMON_LOCALES = _mod_4dxktf.DECODO_COMMON_LOCALES;
+exports.DECODO_COMMON_COUNTRIES = _mod_4dxktf.DECODO_COMMON_COUNTRIES;
+exports.DECODO_EUROPEAN_COUNTRIES = _mod_4dxktf.DECODO_EUROPEAN_COUNTRIES;
+exports.DECODO_ASIAN_COUNTRIES = _mod_4dxktf.DECODO_ASIAN_COUNTRIES;
+exports.DECODO_US_STATES = _mod_4dxktf.DECODO_US_STATES;
+exports.DECODO_COMMON_CITIES = _mod_4dxktf.DECODO_COMMON_CITIES;
+exports.getRandomDecodoDeviceType = _mod_4dxktf.getRandomDeviceType;
+exports.getRandomDecodoLocale = _mod_4dxktf.getRandomLocale;
+exports.getRandomDecodoCountry = _mod_4dxktf.getRandomCountry;
+exports.getRandomDecodoCity = _mod_4dxktf.getRandomCity;
+exports.generateDecodoSessionId = _mod_4dxktf.generateSessionId;;

package/dist/crawler/plugin/index.cjs

@@ -1 +1 @@
-var e=require("./file-cacher.cjs");exports.FileCacher=e.FileCacher;var r=require("./url-store.cjs");exports.UrlStore=r.UrlStore;var o=require("./result-stream.cjs");exports.ResultStream=o.ResultStream;var t=require("./memory-monitor.cjs");exports.MemoryMonitor=t.MemoryMonitor;var a=require("./health-metrics.cjs");exports.HealthMetrics=a.HealthMetrics;var i=require("./capped-map.cjs");exports.CappedMap=i.CappedMap;var m=require("./capped-array.cjs");exports.CappedArray=m.CappedArray;
+var e=require("./file-cacher.cjs");exports.FileCacher=e.FileCacher;var r=require("./url-store.cjs");exports.UrlStore=r.UrlStore;var o=require("./result-stream.cjs");exports.ResultStream=o.ResultStream;var t=require("./memory-monitor.cjs");exports.MemoryMonitor=t.MemoryMonitor;var a=require("./health-metrics.cjs");exports.HealthMetrics=a.HealthMetrics;var i=require("./capped-map.cjs");exports.CappedMap=i.CappedMap;var d=require("./capped-array.cjs");exports.CappedArray=d.CappedArray;

package/dist/crawler.d.ts

@@ -6504,6 +6504,8 @@ export interface ICrawlerOptions {
 	} | {
 		enable: false;
 	} | undefined | false;
+	/** Enable graceful shutdown handlers for SIGINT/SIGTERM - saves session state (default: false) */
+	enableSignalHandlers?: boolean;
 	/** Maximum concurrent requests for crawler (default: 100) */
 	concurrency?: number;
 	/** Maximum concurrent requests for scraper - separate queue (default: same as concurrency) */
@@ -6606,6 +6608,8 @@ export declare class CrawlerOptions {
 	throwFatalError?: boolean;
 	/** Enable debug logging */
 	debug?: boolean;
+	/** Enable graceful shutdown handlers for SIGINT/SIGTERM - saves session state */
+	enableSignalHandlers: boolean;
 	/** Maximum concurrent requests for crawler (default: 100) */
 	concurrency: number;
 	/** Maximum concurrent requests for scraper (default: same as concurrency) */

package/dist/entries/crawler.cjs

@@ -1,4 +1,4 @@
-const _mod_lzlskh = require('../crawler/crawler.cjs');
-exports.Crawler = _mod_lzlskh.Crawler;;
-const _mod_vbox0l = require('../crawler/crawler-options.cjs');
-exports.CrawlerOptions = _mod_vbox0l.CrawlerOptions;;
+const _mod_yduaec = require('../crawler/crawler.cjs');
+exports.Crawler = _mod_yduaec.Crawler;;
+const _mod_58t9fo = require('../crawler/crawler-options.cjs');
+exports.CrawlerOptions = _mod_58t9fo.CrawlerOptions;;

package/dist/errors/rezo-error.cjs

@@ -553,9 +553,9 @@ function cleanStackTrace(stack) {
 class RezoError extends Error {
   constructor(message, config, code, request, response) {
     super();
-    Object.defineProperty(this, "config", { value: config, enumerable: false });
-    Object.defineProperty(this, "request", { value: request, enumerable: false });
-    Object.defineProperty(this, "response", { value: response, enumerable: false });
+    Object.defineProperty(this, "config", { value: config, enumerable: !!config });
+    Object.defineProperty(this, "request", { value: request, enumerable: !!request });
+    Object.defineProperty(this, "response", { value: response, enumerable: !!response });
     Object.defineProperty(this, "isRezoError", { value: true, enumerable: false });
     if (code) {
       Object.defineProperty(this, "code", { value: code, enumerable: true });
@@ -600,75 +600,6 @@ class RezoError extends Error {
       }
     }
   }
-  [Symbol.for("nodejs.util.inspect.custom")](_depth, options) {
-    const parts = [];
-    const isDebug = this.config?.debug === true;
-    const inspect = options?.stylize ? (v) => require("util").inspect(v, { depth: 3, colors: true }) : JSON.stringify;
-    parts.push(`${this.name}: ${this.message}`);
-    if (this.code)
-      parts.push(`  code: '${this.code}'`);
-    if (this.method)
-      parts.push(`  method: '${this.method}'`);
-    if (this.url)
-      parts.push(`  url: '${this.url}'`);
-    if (this.finalUrl && this.finalUrl !== this.url) {
-      parts.push(`  finalUrl: '${this.finalUrl}'`);
-    }
-    if (this.status)
-      parts.push(`  status: ${this.status}`);
-    if (this.statusText)
-      parts.push(`  statusText: '${this.statusText}'`);
-    if (this.urls && this.urls.length > 1) {
-      parts.push(`  urls: [${this.urls.map((u) => `'${u}'`).join(", ")}]`);
-    }
-    if (this.suggestion)
-      parts.push(`  suggestion: ${this.suggestion}`);
-    if (isDebug) {
-      parts.push("");
-      parts.push("  --- Debug Info ---");
-      if (this.cause) {
-        const causeMsg = typeof this.cause === "string" ? this.cause : this.cause?.message || String(this.cause);
-        parts.push(`  cause: ${causeMsg}`);
-      }
-      if (this.errno)
-        parts.push(`  errno: ${this.errno}`);
-      if (this.hostname)
-        parts.push(`  hostname: '${this.hostname}'`);
-      if (this.port)
-        parts.push(`  port: ${this.port}`);
-      if (this.address)
-        parts.push(`  address: '${this.address}'`);
-      if (this.syscall)
-        parts.push(`  syscall: '${this.syscall}'`);
-      if (this.response) {
-        parts.push("");
-        parts.push("  --- Response ---");
-        parts.push(`  response.status: ${this.response.status}`);
-        parts.push(`  response.statusText: '${this.response.statusText || ""}'`);
-        parts.push(`  response.finalUrl: '${this.response.finalUrl || ""}'`);
-        if (this.response.headers) {
-          parts.push(`  response.headers: ${inspect(this.response.headers)}`);
-        }
-        if (this.response.data !== undefined) {
-          const dataStr = typeof this.response.data === "string" ? this.response.data.substring(0, 500) + (this.response.data.length > 500 ? "..." : "") : inspect(this.response.data);
-          parts.push(`  response.data: ${dataStr}`);
-        }
-      }
-      if (this.response?.config) {
-        parts.push("");
-        parts.push("  --- Request Config ---");
-        const { cookieJar, ...configWithoutJar } = this.response.config;
-        parts.push(`  config: ${inspect(configWithoutJar)}`);
-      }
-      if (this.stack) {
-        parts.push("");
-        parts.push("  --- Stack Trace ---");
-        parts.push(this.stack);
-      }
-    }
-    return parts.join(`
-`);
-  }
   static isRezoError(error) {
     return error instanceof RezoError || error !== null && typeof error === "object" && error.isRezoError === true;
   }

package/dist/errors/rezo-error.js

@@ -553,9 +553,9 @@ function cleanStackTrace(stack) {
 export class RezoError extends Error {
   constructor(message, config, code, request, response) {
     super();
-    Object.defineProperty(this, "config", { value: config, enumerable: false });
-    Object.defineProperty(this, "request", { value: request, enumerable: false });
-    Object.defineProperty(this, "response", { value: response, enumerable: false });
+    Object.defineProperty(this, "config", { value: config, enumerable: !!config });
+    Object.defineProperty(this, "request", { value: request, enumerable: !!request });
+    Object.defineProperty(this, "response", { value: response, enumerable: !!response });
     Object.defineProperty(this, "isRezoError", { value: true, enumerable: false });
     if (code) {
       Object.defineProperty(this, "code", { value: code, enumerable: true });
@@ -600,75 +600,6 @@ export class RezoError extends Error {
       }
     }
   }
-  [Symbol.for("nodejs.util.inspect.custom")](_depth, options) {
-    const parts = [];
-    const isDebug = this.config?.debug === true;
-    const inspect = options?.stylize ? (v) => require("util").inspect(v, { depth: 3, colors: true }) : JSON.stringify;
-    parts.push(`${this.name}: ${this.message}`);
-    if (this.code)
-      parts.push(`  code: '${this.code}'`);
-    if (this.method)
-      parts.push(`  method: '${this.method}'`);
-    if (this.url)
-      parts.push(`  url: '${this.url}'`);
-    if (this.finalUrl && this.finalUrl !== this.url) {
-      parts.push(`  finalUrl: '${this.finalUrl}'`);
-    }
-    if (this.status)
-      parts.push(`  status: ${this.status}`);
-    if (this.statusText)
-      parts.push(`  statusText: '${this.statusText}'`);
-    if (this.urls && this.urls.length > 1) {
-      parts.push(`  urls: [${this.urls.map((u) => `'${u}'`).join(", ")}]`);
-    }
-    if (this.suggestion)
-      parts.push(`  suggestion: ${this.suggestion}`);
-    if (isDebug) {
-      parts.push("");
-      parts.push("  --- Debug Info ---");
-      if (this.cause) {
-        const causeMsg = typeof this.cause === "string" ? this.cause : this.cause?.message || String(this.cause);
-        parts.push(`  cause: ${causeMsg}`);
-      }
-      if (this.errno)
-        parts.push(`  errno: ${this.errno}`);
-      if (this.hostname)
-        parts.push(`  hostname: '${this.hostname}'`);
-      if (this.port)
-        parts.push(`  port: ${this.port}`);
-      if (this.address)
-        parts.push(`  address: '${this.address}'`);
-      if (this.syscall)
-        parts.push(`  syscall: '${this.syscall}'`);
-      if (this.response) {
-        parts.push("");
-        parts.push("  --- Response ---");
-        parts.push(`  response.status: ${this.response.status}`);
-        parts.push(`  response.statusText: '${this.response.statusText || ""}'`);
-        parts.push(`  response.finalUrl: '${this.response.finalUrl || ""}'`);
-        if (this.response.headers) {
-          parts.push(`  response.headers: ${inspect(this.response.headers)}`);
-        }
-        if (this.response.data !== undefined) {
-          const dataStr = typeof this.response.data === "string" ? this.response.data.substring(0, 500) + (this.response.data.length > 500 ? "..." : "") : inspect(this.response.data);
-          parts.push(`  response.data: ${dataStr}`);
-        }
-      }
-      if (this.response?.config) {
-        parts.push("");
-        parts.push("  --- Request Config ---");
-        const { cookieJar, ...configWithoutJar } = this.response.config;
-        parts.push(`  config: ${inspect(configWithoutJar)}`);
-      }
-      if (this.stack) {
-        parts.push("");
-        parts.push("  --- Stack Trace ---");
-        parts.push(this.stack);
-      }
-    }
-    return parts.join(`
-`);
-  }
   static isRezoError(error) {
     return error instanceof RezoError || error !== null && typeof error === "object" && error.isRezoError === true;
   }

package/dist/index.cjs

@@ -1,30 +1,30 @@
-const _mod_nof5gk = require('./core/rezo.cjs');
-exports.Rezo = _mod_nof5gk.Rezo;
-exports.createRezoInstance = _mod_nof5gk.createRezoInstance;
-exports.createDefaultInstance = _mod_nof5gk.createDefaultInstance;;
-const _mod_4m1kqu = require('./errors/rezo-error.cjs');
-exports.RezoError = _mod_4m1kqu.RezoError;
-exports.RezoErrorCode = _mod_4m1kqu.RezoErrorCode;;
-const _mod_p9wchx = require('./utils/headers.cjs');
-exports.RezoHeaders = _mod_p9wchx.RezoHeaders;;
-const _mod_ib1mep = require('./utils/form-data.cjs');
-exports.RezoFormData = _mod_ib1mep.RezoFormData;;
-const _mod_fkghv0 = require('./utils/cookies.cjs');
-exports.RezoCookieJar = _mod_fkghv0.RezoCookieJar;
-exports.Cookie = _mod_fkghv0.Cookie;;
-const _mod_qucbgi = require('./utils/curl.cjs');
-exports.toCurl = _mod_qucbgi.toCurl;
-exports.fromCurl = _mod_qucbgi.fromCurl;;
-const _mod_6whv41 = require('./core/hooks.cjs');
-exports.createDefaultHooks = _mod_6whv41.createDefaultHooks;
-exports.mergeHooks = _mod_6whv41.mergeHooks;;
-const _mod_urv2c9 = require('./proxy/manager.cjs');
-exports.ProxyManager = _mod_urv2c9.ProxyManager;;
-const _mod_hfcxva = require('./queue/index.cjs');
-exports.RezoQueue = _mod_hfcxva.RezoQueue;
-exports.HttpQueue = _mod_hfcxva.HttpQueue;
-exports.Priority = _mod_hfcxva.Priority;
-exports.HttpMethodPriority = _mod_hfcxva.HttpMethodPriority;;
+const _mod_f5e2x4 = require('./core/rezo.cjs');
+exports.Rezo = _mod_f5e2x4.Rezo;
+exports.createRezoInstance = _mod_f5e2x4.createRezoInstance;
+exports.createDefaultInstance = _mod_f5e2x4.createDefaultInstance;;
+const _mod_l2n0t8 = require('./errors/rezo-error.cjs');
+exports.RezoError = _mod_l2n0t8.RezoError;
+exports.RezoErrorCode = _mod_l2n0t8.RezoErrorCode;;
+const _mod_pgi4on = require('./utils/headers.cjs');
+exports.RezoHeaders = _mod_pgi4on.RezoHeaders;;
+const _mod_k82fb8 = require('./utils/form-data.cjs');
+exports.RezoFormData = _mod_k82fb8.RezoFormData;;
+const _mod_u8f16d = require('./utils/cookies.cjs');
+exports.RezoCookieJar = _mod_u8f16d.RezoCookieJar;
+exports.Cookie = _mod_u8f16d.Cookie;;
+const _mod_5km04t = require('./utils/curl.cjs');
+exports.toCurl = _mod_5km04t.toCurl;
+exports.fromCurl = _mod_5km04t.fromCurl;;
+const _mod_c99qva = require('./core/hooks.cjs');
+exports.createDefaultHooks = _mod_c99qva.createDefaultHooks;
+exports.mergeHooks = _mod_c99qva.mergeHooks;;
+const _mod_nlh20q = require('./proxy/manager.cjs');
+exports.ProxyManager = _mod_nlh20q.ProxyManager;;
+const _mod_2j9uz9 = require('./queue/index.cjs');
+exports.RezoQueue = _mod_2j9uz9.RezoQueue;
+exports.HttpQueue = _mod_2j9uz9.HttpQueue;
+exports.Priority = _mod_2j9uz9.Priority;
+exports.HttpMethodPriority = _mod_2j9uz9.HttpMethodPriority;;
 const { RezoError } = require('./errors/rezo-error.cjs');
 const isRezoError = exports.isRezoError = RezoError.isRezoError;
 const Cancel = exports.Cancel = RezoError;
@@ -34,9 +34,9 @@ const isCancel = exports.isCancel = (error) => {
 };
 const all = exports.all = Promise.all.bind(Promise);
 const spread = exports.spread = (callback) => (array) => callback(...array);
-const _mod_wqlmnp = require('./version.cjs');
-exports.VERSION = _mod_wqlmnp.VERSION;
-exports.PACKAGE_NAME = _mod_wqlmnp.PACKAGE_NAME;;
+const _mod_xznh6j = require('./version.cjs');
+exports.VERSION = _mod_xznh6j.VERSION;
+exports.PACKAGE_NAME = _mod_xznh6j.PACKAGE_NAME;;
 const { executeRequest } = require('./adapters/http.cjs');
 const { setGlobalAdapter, createRezoInstance } = require('./core/rezo.cjs');
 setGlobalAdapter(executeRequest);

package/dist/index.d.ts

@@ -4883,7 +4883,7 @@ export declare class HttpQueue extends RezoQueue<any> {
  *
  * IMPORTANT: Update these values when bumping package version.
  */
-export declare const VERSION = "1.0.73";
+export declare const VERSION = "1.0.74";
 export declare const PACKAGE_NAME = "rezo";
 export declare const isRezoError: typeof RezoError.isRezoError;
 export declare const Cancel: typeof RezoError;

package/dist/internal/agents/index.cjs

@@ -1,10 +1,10 @@
-const _mod_z9dkod = require('./base.cjs');
-exports.Agent = _mod_z9dkod.Agent;;
-const _mod_sd66wk = require('./http-proxy.cjs');
-exports.HttpProxyAgent = _mod_sd66wk.HttpProxyAgent;;
-const _mod_pwlo59 = require('./https-proxy.cjs');
-exports.HttpsProxyAgent = _mod_pwlo59.HttpsProxyAgent;;
-const _mod_a4lzyf = require('./socks-proxy.cjs');
-exports.SocksProxyAgent = _mod_a4lzyf.SocksProxyAgent;;
-const _mod_6yne7d = require('./socks-client.cjs');
-exports.SocksClient = _mod_6yne7d.SocksClient;;
+const _mod_0a73ie = require('./base.cjs');
+exports.Agent = _mod_0a73ie.Agent;;
+const _mod_rs317e = require('./http-proxy.cjs');
+exports.HttpProxyAgent = _mod_rs317e.HttpProxyAgent;;
+const _mod_syy64b = require('./https-proxy.cjs');
+exports.HttpsProxyAgent = _mod_syy64b.HttpsProxyAgent;;
+const _mod_rciw1o = require('./socks-proxy.cjs');
+exports.SocksProxyAgent = _mod_rciw1o.SocksProxyAgent;;
+const _mod_w9tiwm = require('./socks-client.cjs');
+exports.SocksClient = _mod_w9tiwm.SocksClient;;

package/dist/platform/browser.d.ts

@@ -4579,7 +4579,7 @@ export interface RezoInstance extends Rezo {
  *
  * IMPORTANT: Update these values when bumping package version.
  */
-export declare const VERSION = "1.0.73";
+export declare const VERSION = "1.0.74";
 export declare const isRezoError: typeof RezoError.isRezoError;
 export declare const Cancel: typeof RezoError;
 export declare const CancelToken: {

package/dist/platform/bun.d.ts

@@ -4579,7 +4579,7 @@ export interface RezoInstance extends Rezo {
  *
  * IMPORTANT: Update these values when bumping package version.
  */
-export declare const VERSION = "1.0.73";
+export declare const VERSION = "1.0.74";
 export declare const isRezoError: typeof RezoError.isRezoError;
 export declare const Cancel: typeof RezoError;
 export declare const CancelToken: {

package/dist/platform/deno.d.ts

@@ -4579,7 +4579,7 @@ export interface RezoInstance extends Rezo {
  *
  * IMPORTANT: Update these values when bumping package version.
  */
-export declare const VERSION = "1.0.73";
+export declare const VERSION = "1.0.74";
 export declare const isRezoError: typeof RezoError.isRezoError;
 export declare const Cancel: typeof RezoError;
 export declare const CancelToken: {

package/dist/platform/node.d.ts

@@ -4579,7 +4579,7 @@ export interface RezoInstance extends Rezo {
  *
  * IMPORTANT: Update these values when bumping package version.
  */
-export declare const VERSION = "1.0.73";
+export declare const VERSION = "1.0.74";
 export declare const isRezoError: typeof RezoError.isRezoError;
 export declare const Cancel: typeof RezoError;
 export declare const CancelToken: {

package/dist/platform/react-native.d.ts

@@ -4579,7 +4579,7 @@ export interface RezoInstance extends Rezo {
  *
  * IMPORTANT: Update these values when bumping package version.
  */
-export declare const VERSION = "1.0.73";
+export declare const VERSION = "1.0.74";
 export declare const isRezoError: typeof RezoError.isRezoError;
 export declare const Cancel: typeof RezoError;
 export declare const CancelToken: {

package/dist/platform/worker.d.ts

@@ -4579,7 +4579,7 @@ export interface RezoInstance extends Rezo {
  *
  * IMPORTANT: Update these values when bumping package version.
  */
-export declare const VERSION = "1.0.73";
+export declare const VERSION = "1.0.74";
 export declare const isRezoError: typeof RezoError.isRezoError;
 export declare const Cancel: typeof RezoError;
 export declare const CancelToken: {

package/dist/proxy/index.cjs

@@ -1,9 +1,9 @@
 const { Agent, HttpProxyAgent, HttpsProxyAgent, SocksProxyAgent } = require('../internal/agents/index.cjs');
 const { parseProxyString } = require('./parse.cjs');
-const _mod_822tv1 = require('./manager.cjs');
-exports.ProxyManager = _mod_822tv1.ProxyManager;;
-const _mod_nv2yd4 = require('./parse.cjs');
-exports.parseProxyString = _mod_nv2yd4.parseProxyString;;
+const _mod_dkmwz9 = require('./manager.cjs');
+exports.ProxyManager = _mod_dkmwz9.ProxyManager;;
+const _mod_0ibmjj = require('./parse.cjs');
+exports.parseProxyString = _mod_0ibmjj.parseProxyString;;
 function createOptions(uri, opts) {
   if (uri instanceof URL || typeof uri === "string") {
     return {

package/dist/queue/index.cjs

@@ -1,8 +1,8 @@
-const _mod_sdhfnz = require('./queue.cjs');
-exports.RezoQueue = _mod_sdhfnz.RezoQueue;;
-const _mod_7ea16t = require('./http-queue.cjs');
-exports.HttpQueue = _mod_7ea16t.HttpQueue;
-exports.extractDomain = _mod_7ea16t.extractDomain;;
-const _mod_juhwej = require('./types.cjs');
-exports.Priority = _mod_juhwej.Priority;
-exports.HttpMethodPriority = _mod_juhwej.HttpMethodPriority;;
+const _mod_rllkog = require('./queue.cjs');
+exports.RezoQueue = _mod_rllkog.RezoQueue;;
+const _mod_2glpb0 = require('./http-queue.cjs');
+exports.HttpQueue = _mod_2glpb0.HttpQueue;
+exports.extractDomain = _mod_2glpb0.extractDomain;;
+const _mod_rpdaro = require('./types.cjs');
+exports.Priority = _mod_rpdaro.Priority;
+exports.HttpMethodPriority = _mod_rpdaro.HttpMethodPriority;;

package/dist/queue/queue.cjs

@@ -36,7 +36,7 @@ class RezoQueue {
       interval: config.interval ?? 0,
       intervalCap: config.intervalCap ?? 1 / 0,
       carryoverConcurrencyCount: config.carryoverConcurrencyCount ?? false,
-      rejectOnError: config.rejectOnError ?? false
+      rejectOnError: config.rejectOnError ?? true
     };
     if (!this.config.autoStart) {
       this.isPausedFlag = true;

package/dist/queue/queue.js

@@ -36,7 +36,7 @@ export class RezoQueue {
       interval: config.interval ?? 0,
       intervalCap: config.intervalCap ?? 1 / 0,
       carryoverConcurrencyCount: config.carryoverConcurrencyCount ?? false,
-      rejectOnError: config.rejectOnError ?? false
+      rejectOnError: config.rejectOnError ?? true
     };
     if (!this.config.autoStart) {
       this.isPausedFlag = true;

package/dist/responses/universal/index.cjs

@@ -1,11 +1,11 @@
-const _mod_inkbi3 = require('./event-emitter.cjs');
-exports.UniversalEventEmitter = _mod_inkbi3.UniversalEventEmitter;;
-const _mod_mpsc08 = require('./stream.cjs');
-exports.UniversalStreamResponse = _mod_mpsc08.UniversalStreamResponse;
-exports.StreamResponse = _mod_mpsc08.StreamResponse;;
-const _mod_c99n0a = require('./download.cjs');
-exports.UniversalDownloadResponse = _mod_c99n0a.UniversalDownloadResponse;
-exports.DownloadResponse = _mod_c99n0a.DownloadResponse;;
-const _mod_c0t3t8 = require('./upload.cjs');
-exports.UniversalUploadResponse = _mod_c0t3t8.UniversalUploadResponse;
-exports.UploadResponse = _mod_c0t3t8.UploadResponse;;
+const _mod_1hy9oe = require('./event-emitter.cjs');
+exports.UniversalEventEmitter = _mod_1hy9oe.UniversalEventEmitter;;
+const _mod_zi59hf = require('./stream.cjs');
+exports.UniversalStreamResponse = _mod_zi59hf.UniversalStreamResponse;
+exports.StreamResponse = _mod_zi59hf.StreamResponse;;
+const _mod_vm1o39 = require('./download.cjs');
+exports.UniversalDownloadResponse = _mod_vm1o39.UniversalDownloadResponse;
+exports.DownloadResponse = _mod_vm1o39.DownloadResponse;;
+const _mod_zh9h9e = require('./upload.cjs');
+exports.UniversalUploadResponse = _mod_zh9h9e.UniversalUploadResponse;
+exports.UploadResponse = _mod_zh9h9e.UploadResponse;;

package/dist/utils/agent-pool.cjs

@@ -1,5 +1,6 @@
 const http = require("node:http");
 const https = require("node:https");
+const tls = require("node:tls");
 const { getGlobalDNSCache } = require('../cache/dns-cache.cjs');
 const DEFAULT_CONFIG = {
   keepAlive: true,
@@ -79,6 +80,39 @@ class AgentPool {
     });
   }
   createHttpsAgent(key, tlsOptions) {
+    const secureContext = tls.createSecureContext({
+      ecdhCurve: "X25519:prime256v1:secp384r1",
+      ciphers: [
+        "TLS_AES_128_GCM_SHA256",
+        "TLS_AES_256_GCM_SHA384",
+        "TLS_CHACHA20_POLY1305_SHA256",
+        "ECDHE-ECDSA-AES128-GCM-SHA256",
+        "ECDHE-RSA-AES128-GCM-SHA256",
+        "ECDHE-ECDSA-AES256-GCM-SHA384",
+        "ECDHE-RSA-AES256-GCM-SHA384",
+        "ECDHE-ECDSA-CHACHA20-POLY1305",
+        "ECDHE-RSA-CHACHA20-POLY1305",
+        "ECDHE-RSA-AES128-SHA",
+        "ECDHE-RSA-AES256-SHA",
+        "AES128-GCM-SHA256",
+        "AES256-GCM-SHA384",
+        "AES128-SHA",
+        "AES256-SHA"
+      ].join(":"),
+      sigalgs: [
+        "ecdsa_secp256r1_sha256",
+        "ecdsa_secp384r1_sha384",
+        "rsa_pss_rsae_sha256",
+        "rsa_pss_rsae_sha384",
+        "rsa_pss_rsae_sha512",
+        "rsa_pkcs1_sha256",
+        "rsa_pkcs1_sha384",
+        "rsa_pkcs1_sha512"
+      ].join(":"),
+      minVersion: "TLSv1.2",
+      maxVersion: "TLSv1.3",
+      sessionTimeout: 3600
+    });
     const agentOptions = {
       keepAlive: this.config.keepAlive,
       keepAliveMsecs: this.config.keepAliveMsecs,
@@ -86,6 +120,7 @@ class AgentPool {
       maxFreeSockets: this.config.maxFreeSockets,
       timeout: this.config.timeout,
       scheduling: this.config.scheduling,
+      secureContext,
       ...tlsOptions
     };
     const lookup = this.createLookupFunction();

package/dist/utils/agent-pool.js

@@ -1,5 +1,6 @@
 import * as http from "node:http";
 import * as https from "node:https";
+import * as tls from "node:tls";
 import { getGlobalDNSCache } from '../cache/dns-cache.js';
 const DEFAULT_CONFIG = {
   keepAlive: true,
@@ -79,6 +80,39 @@ class AgentPool {
     });
   }
   createHttpsAgent(key, tlsOptions) {
+    const secureContext = tls.createSecureContext({
+      ecdhCurve: "X25519:prime256v1:secp384r1",
+      ciphers: [
+        "TLS_AES_128_GCM_SHA256",
+        "TLS_AES_256_GCM_SHA384",
+        "TLS_CHACHA20_POLY1305_SHA256",
+        "ECDHE-ECDSA-AES128-GCM-SHA256",
+        "ECDHE-RSA-AES128-GCM-SHA256",
+        "ECDHE-ECDSA-AES256-GCM-SHA384",
+        "ECDHE-RSA-AES256-GCM-SHA384",
+        "ECDHE-ECDSA-CHACHA20-POLY1305",
+        "ECDHE-RSA-CHACHA20-POLY1305",
+        "ECDHE-RSA-AES128-SHA",
+        "ECDHE-RSA-AES256-SHA",
+        "AES128-GCM-SHA256",
+        "AES256-GCM-SHA384",
+        "AES128-SHA",
+        "AES256-SHA"
+      ].join(":"),
+      sigalgs: [
+        "ecdsa_secp256r1_sha256",
+        "ecdsa_secp384r1_sha384",
+        "rsa_pss_rsae_sha256",
+        "rsa_pss_rsae_sha384",
+        "rsa_pss_rsae_sha512",
+        "rsa_pkcs1_sha256",
+        "rsa_pkcs1_sha384",
+        "rsa_pkcs1_sha512"
+      ].join(":"),
+      minVersion: "TLSv1.2",
+      maxVersion: "TLSv1.3",
+      sessionTimeout: 3600
+    });
     const agentOptions = {
       keepAlive: this.config.keepAlive,
       keepAliveMsecs: this.config.keepAliveMsecs,
@@ -86,6 +120,7 @@ class AgentPool {
       maxFreeSockets: this.config.maxFreeSockets,
       timeout: this.config.timeout,
       scheduling: this.config.scheduling,
+      secureContext,
       ...tlsOptions
     };
     const lookup = this.createLookupFunction();

package/dist/version.cjs

@@ -1,2 +1,2 @@
-const VERSION = exports.VERSION = "1.0.73";
+const VERSION = exports.VERSION = "1.0.74";
 const PACKAGE_NAME = exports.PACKAGE_NAME = "rezo";

package/dist/version.js

@@ -1,2 +1,2 @@
-export const VERSION = "1.0.73";
+export const VERSION = "1.0.74";
 export const PACKAGE_NAME = "rezo";

package/dist/wget/index.cjs

@@ -1,52 +1,52 @@
-const _mod_5ei63p = require('./types.cjs');
-exports.WgetError = _mod_5ei63p.WgetError;;
-const _mod_tutikq = require('./asset-extractor.cjs');
-exports.AssetExtractor = _mod_tutikq.AssetExtractor;;
-const _mod_o2ssmt = require('./url-filter.cjs');
-exports.UrlFilter = _mod_o2ssmt.UrlFilter;;
-const _mod_8pdoow = require('./file-writer.cjs');
-exports.FileWriter = _mod_8pdoow.FileWriter;;
-const _mod_hb8okl = require('./robots.cjs');
-exports.RobotsHandler = _mod_hb8okl.RobotsHandler;;
-const _mod_6w54ca = require('./resume.cjs');
-exports.ResumeHandler = _mod_6w54ca.ResumeHandler;;
-const _mod_ivqayl = require('./progress.cjs');
-exports.ProgressReporter = _mod_ivqayl.ProgressReporter;
-exports.ProgressTracker = _mod_ivqayl.ProgressTracker;
-exports.parseSize = _mod_ivqayl.parseSize;;
-const _mod_75fsyq = require('./link-converter.cjs');
-exports.LinkConverter = _mod_75fsyq.LinkConverter;;
-const _mod_bfrkpp = require('./downloader.cjs');
-exports.Downloader = _mod_bfrkpp.Downloader;;
-const _mod_ym1dyu = require('./asset-organizer.cjs');
-exports.AssetOrganizer = _mod_ym1dyu.AssetOrganizer;
-exports.DEFAULT_ASSET_FOLDERS = _mod_ym1dyu.DEFAULT_ASSET_FOLDERS;;
-const _mod_hwajgz = require('./download-cache.cjs');
-exports.DownloadCache = _mod_hwajgz.DownloadCache;;
-const _mod_kb7vuw = require('./filter-lists.cjs');
-exports.EXECUTABLE_EXTENSIONS = _mod_kb7vuw.EXECUTABLE_EXTENSIONS;
-exports.ARCHIVE_EXTENSIONS = _mod_kb7vuw.ARCHIVE_EXTENSIONS;
-exports.DOCUMENT_EXTENSIONS = _mod_kb7vuw.DOCUMENT_EXTENSIONS;
-exports.IMAGE_EXTENSIONS = _mod_kb7vuw.IMAGE_EXTENSIONS;
-exports.VIDEO_EXTENSIONS = _mod_kb7vuw.VIDEO_EXTENSIONS;
-exports.AUDIO_EXTENSIONS = _mod_kb7vuw.AUDIO_EXTENSIONS;
-exports.FONT_EXTENSIONS = _mod_kb7vuw.FONT_EXTENSIONS;
-exports.WEB_ASSET_EXTENSIONS = _mod_kb7vuw.WEB_ASSET_EXTENSIONS;
-exports.DATA_EXTENSIONS = _mod_kb7vuw.DATA_EXTENSIONS;
-exports.EXECUTABLE_MIME_TYPES = _mod_kb7vuw.EXECUTABLE_MIME_TYPES;
-exports.ARCHIVE_MIME_TYPES = _mod_kb7vuw.ARCHIVE_MIME_TYPES;
-exports.DOCUMENT_MIME_TYPES = _mod_kb7vuw.DOCUMENT_MIME_TYPES;
-exports.IMAGE_MIME_TYPES = _mod_kb7vuw.IMAGE_MIME_TYPES;
-exports.VIDEO_MIME_TYPES = _mod_kb7vuw.VIDEO_MIME_TYPES;
-exports.AUDIO_MIME_TYPES = _mod_kb7vuw.AUDIO_MIME_TYPES;
-exports.FONT_MIME_TYPES = _mod_kb7vuw.FONT_MIME_TYPES;
-exports.WEB_ASSET_MIME_TYPES = _mod_kb7vuw.WEB_ASSET_MIME_TYPES;
-exports.DATA_MIME_TYPES = _mod_kb7vuw.DATA_MIME_TYPES;
-exports.SAFE_WEB_PRESET = _mod_kb7vuw.SAFE_WEB_PRESET;
-exports.DOCUMENTS_ONLY_PRESET = _mod_kb7vuw.DOCUMENTS_ONLY_PRESET;
-exports.NO_MEDIA_PRESET = _mod_kb7vuw.NO_MEDIA_PRESET;
-exports.MINIMAL_MIRROR_PRESET = _mod_kb7vuw.MINIMAL_MIRROR_PRESET;
-exports.TEXT_ONLY_PRESET = _mod_kb7vuw.TEXT_ONLY_PRESET;;
+const _mod_8lpt1q = require('./types.cjs');
+exports.WgetError = _mod_8lpt1q.WgetError;;
+const _mod_3knsix = require('./asset-extractor.cjs');
+exports.AssetExtractor = _mod_3knsix.AssetExtractor;;
+const _mod_hmciaa = require('./url-filter.cjs');
+exports.UrlFilter = _mod_hmciaa.UrlFilter;;
+const _mod_25damj = require('./file-writer.cjs');
+exports.FileWriter = _mod_25damj.FileWriter;;
+const _mod_cp8z2y = require('./robots.cjs');
+exports.RobotsHandler = _mod_cp8z2y.RobotsHandler;;
+const _mod_xx31k2 = require('./resume.cjs');
+exports.ResumeHandler = _mod_xx31k2.ResumeHandler;;
+const _mod_xe7f9y = require('./progress.cjs');
+exports.ProgressReporter = _mod_xe7f9y.ProgressReporter;
+exports.ProgressTracker = _mod_xe7f9y.ProgressTracker;
+exports.parseSize = _mod_xe7f9y.parseSize;;
+const _mod_ju89ay = require('./link-converter.cjs');
+exports.LinkConverter = _mod_ju89ay.LinkConverter;;
+const _mod_hypiea = require('./downloader.cjs');
+exports.Downloader = _mod_hypiea.Downloader;;
+const _mod_haffqa = require('./asset-organizer.cjs');
+exports.AssetOrganizer = _mod_haffqa.AssetOrganizer;
+exports.DEFAULT_ASSET_FOLDERS = _mod_haffqa.DEFAULT_ASSET_FOLDERS;;
+const _mod_ej979a = require('./download-cache.cjs');
+exports.DownloadCache = _mod_ej979a.DownloadCache;;
+const _mod_4n02yz = require('./filter-lists.cjs');
+exports.EXECUTABLE_EXTENSIONS = _mod_4n02yz.EXECUTABLE_EXTENSIONS;
+exports.ARCHIVE_EXTENSIONS = _mod_4n02yz.ARCHIVE_EXTENSIONS;
+exports.DOCUMENT_EXTENSIONS = _mod_4n02yz.DOCUMENT_EXTENSIONS;
+exports.IMAGE_EXTENSIONS = _mod_4n02yz.IMAGE_EXTENSIONS;
+exports.VIDEO_EXTENSIONS = _mod_4n02yz.VIDEO_EXTENSIONS;
+exports.AUDIO_EXTENSIONS = _mod_4n02yz.AUDIO_EXTENSIONS;
+exports.FONT_EXTENSIONS = _mod_4n02yz.FONT_EXTENSIONS;
+exports.WEB_ASSET_EXTENSIONS = _mod_4n02yz.WEB_ASSET_EXTENSIONS;
+exports.DATA_EXTENSIONS = _mod_4n02yz.DATA_EXTENSIONS;
+exports.EXECUTABLE_MIME_TYPES = _mod_4n02yz.EXECUTABLE_MIME_TYPES;
+exports.ARCHIVE_MIME_TYPES = _mod_4n02yz.ARCHIVE_MIME_TYPES;
+exports.DOCUMENT_MIME_TYPES = _mod_4n02yz.DOCUMENT_MIME_TYPES;
+exports.IMAGE_MIME_TYPES = _mod_4n02yz.IMAGE_MIME_TYPES;
+exports.VIDEO_MIME_TYPES = _mod_4n02yz.VIDEO_MIME_TYPES;
+exports.AUDIO_MIME_TYPES = _mod_4n02yz.AUDIO_MIME_TYPES;
+exports.FONT_MIME_TYPES = _mod_4n02yz.FONT_MIME_TYPES;
+exports.WEB_ASSET_MIME_TYPES = _mod_4n02yz.WEB_ASSET_MIME_TYPES;
+exports.DATA_MIME_TYPES = _mod_4n02yz.DATA_MIME_TYPES;
+exports.SAFE_WEB_PRESET = _mod_4n02yz.SAFE_WEB_PRESET;
+exports.DOCUMENTS_ONLY_PRESET = _mod_4n02yz.DOCUMENTS_ONLY_PRESET;
+exports.NO_MEDIA_PRESET = _mod_4n02yz.NO_MEDIA_PRESET;
+exports.MINIMAL_MIRROR_PRESET = _mod_4n02yz.MINIMAL_MIRROR_PRESET;
+exports.TEXT_ONLY_PRESET = _mod_4n02yz.TEXT_ONLY_PRESET;;
 const { Downloader } = require('./downloader.cjs');
 const rezo = require('../index.cjs');
 const { promises: fs } = require("node:fs");

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "rezo",
-  "version": "1.0.73",
+  "version": "1.0.74",
   "description": "Lightning-fast, enterprise-grade HTTP client for modern JavaScript. Full HTTP/2 support, intelligent cookie management, multiple adapters (HTTP, Fetch, cURL, XHR), streaming, proxy support (HTTP/HTTPS/SOCKS), and cross-environment compatibility.",
   "main": "dist/index.cjs",
   "module": "dist/index.js",