rezo 1.0.50 → 1.0.52

package/dist/adapters/http.cjs

@@ -776,7 +776,7 @@ async function request(config, fetchOptions, requestCount, timing, _stats, respo
           if (isRedirected)
             _stats.statusOnNext = "redirect";
           if (isRedirected && location) {
-            const redirectUrlObj = new URL(location, url);
+            const redirectUrlObj = new URL(typeof location === "string" ? location : location.toString(), url);
             if (!redirectUrlObj.hash && url.hash) {
               redirectUrlObj.hash = url.hash;
             }

package/dist/adapters/http.js

@@ -776,7 +776,7 @@ async function request(config, fetchOptions, requestCount, timing, _stats, respo
           if (isRedirected)
             _stats.statusOnNext = "redirect";
           if (isRedirected && location) {
-            const redirectUrlObj = new URL(location, url);
+            const redirectUrlObj = new URL(typeof location === "string" ? location : location.toString(), url);
             if (!redirectUrlObj.hash && url.hash) {
               redirectUrlObj.hash = url.hash;
             }

package/dist/adapters/http2.cjs

@@ -1253,12 +1253,15 @@ async function executeHttp2Stream(config, fetchOptions, requestCount, timing, _s
           resolve(error);
         }
       };
-      session.on("error", sessionErrorHandler);
-      session.on("goaway", (errorCode, lastStreamID) => {
+      session.once("error", sessionErrorHandler);
+      session.once("goaway", (errorCode, lastStreamID) => {
         if (config.debug) {
           console.log(`[Rezo Debug] HTTP/2: Session GOAWAY received (errorCode: ${errorCode}, lastStreamID: ${lastStreamID})`);
         }
       });
+      const cleanupSessionListeners = () => {
+        session.removeListener("error", sessionErrorHandler);
+      };
       if (config.debug) {
         console.log(`[Rezo Debug] HTTP/2: Creating request stream...`);
       }
@@ -1282,6 +1285,7 @@ async function executeHttp2Stream(config, fetchOptions, requestCount, timing, _s
       timeoutId = setTimeout(() => {
         if (!resolved) {
           resolved = true;
+          cleanupSessionListeners();
           if (config.debug) {
             console.log(`[Rezo Debug] HTTP/2: Request timeout after ${requestTimeout}ms (no response received)`);
           }
@@ -1302,6 +1306,7 @@ async function executeHttp2Stream(config, fetchOptions, requestCount, timing, _s
         if (!resolved && status === 0) {
           resolved = true;
           clearTimeout(timeoutId);
+          cleanupSessionListeners();
           if (config.debug) {
             console.log(`[Rezo Debug] HTTP/2: Stream closed without response - retrying with new session`);
           }
@@ -1317,6 +1322,7 @@ async function executeHttp2Stream(config, fetchOptions, requestCount, timing, _s
         if (!resolved) {
           resolved = true;
           clearTimeout(timeoutId);
+          cleanupSessionListeners();
           const error = buildSmartError(config, fetchOptions, new Error("HTTP/2 stream aborted"));
           _stats.statusOnNext = "error";
           resolve(error);
@@ -1329,6 +1335,7 @@ async function executeHttp2Stream(config, fetchOptions, requestCount, timing, _s
         if (!resolved) {
           resolved = true;
           clearTimeout(timeoutId);
+          cleanupSessionListeners();
           const error = buildSmartError(config, fetchOptions, err);
           _stats.statusOnNext = "error";
           resolve(error);
@@ -1427,6 +1434,7 @@ async function executeHttp2Stream(config, fetchOptions, requestCount, timing, _s
         }
         resolved = true;
         clearTimeout(timeoutId);
+        cleanupSessionListeners();
         try {
           updateTiming(config, timing, contentLengthCounter);
           if (!config.transfer) {

package/dist/adapters/http2.js

@@ -1253,12 +1253,15 @@ async function executeHttp2Stream(config, fetchOptions, requestCount, timing, _s
           resolve(error);
         }
       };
-      session.on("error", sessionErrorHandler);
-      session.on("goaway", (errorCode, lastStreamID) => {
+      session.once("error", sessionErrorHandler);
+      session.once("goaway", (errorCode, lastStreamID) => {
         if (config.debug) {
           console.log(`[Rezo Debug] HTTP/2: Session GOAWAY received (errorCode: ${errorCode}, lastStreamID: ${lastStreamID})`);
         }
       });
+      const cleanupSessionListeners = () => {
+        session.removeListener("error", sessionErrorHandler);
+      };
       if (config.debug) {
         console.log(`[Rezo Debug] HTTP/2: Creating request stream...`);
       }
@@ -1282,6 +1285,7 @@ async function executeHttp2Stream(config, fetchOptions, requestCount, timing, _s
       timeoutId = setTimeout(() => {
         if (!resolved) {
           resolved = true;
+          cleanupSessionListeners();
           if (config.debug) {
             console.log(`[Rezo Debug] HTTP/2: Request timeout after ${requestTimeout}ms (no response received)`);
           }
@@ -1302,6 +1306,7 @@ async function executeHttp2Stream(config, fetchOptions, requestCount, timing, _s
         if (!resolved && status === 0) {
           resolved = true;
           clearTimeout(timeoutId);
+          cleanupSessionListeners();
           if (config.debug) {
             console.log(`[Rezo Debug] HTTP/2: Stream closed without response - retrying with new session`);
           }
@@ -1317,6 +1322,7 @@ async function executeHttp2Stream(config, fetchOptions, requestCount, timing, _s
         if (!resolved) {
           resolved = true;
           clearTimeout(timeoutId);
+          cleanupSessionListeners();
           const error = buildSmartError(config, fetchOptions, new Error("HTTP/2 stream aborted"));
           _stats.statusOnNext = "error";
           resolve(error);
@@ -1329,6 +1335,7 @@ async function executeHttp2Stream(config, fetchOptions, requestCount, timing, _s
         if (!resolved) {
           resolved = true;
           clearTimeout(timeoutId);
+          cleanupSessionListeners();
           const error = buildSmartError(config, fetchOptions, err);
           _stats.statusOnNext = "error";
           resolve(error);
@@ -1427,6 +1434,7 @@ async function executeHttp2Stream(config, fetchOptions, requestCount, timing, _s
         }
         resolved = true;
         clearTimeout(timeoutId);
+        cleanupSessionListeners();
         try {
           updateTiming(config, timing, contentLengthCounter);
           if (!config.transfer) {

package/dist/adapters/index.cjs

@@ -1,6 +1,6 @@
-const _mod_gojqf6 = require('./picker.cjs');
-exports.detectRuntime = _mod_gojqf6.detectRuntime;
-exports.getAdapterCapabilities = _mod_gojqf6.getAdapterCapabilities;
-exports.buildAdapterContext = _mod_gojqf6.buildAdapterContext;
-exports.getAvailableAdapters = _mod_gojqf6.getAvailableAdapters;
-exports.selectAdapter = _mod_gojqf6.selectAdapter;;
+const _mod_5w28h8 = require('./picker.cjs');
+exports.detectRuntime = _mod_5w28h8.detectRuntime;
+exports.getAdapterCapabilities = _mod_5w28h8.getAdapterCapabilities;
+exports.buildAdapterContext = _mod_5w28h8.buildAdapterContext;
+exports.getAvailableAdapters = _mod_5w28h8.getAvailableAdapters;
+exports.selectAdapter = _mod_5w28h8.selectAdapter;;

package/dist/cache/index.cjs

@@ -1,9 +1,9 @@
-const _mod_7vxa4h = require('./lru-cache.cjs');
-exports.LRUCache = _mod_7vxa4h.LRUCache;;
-const _mod_erzp5e = require('./dns-cache.cjs');
-exports.DNSCache = _mod_erzp5e.DNSCache;
-exports.getGlobalDNSCache = _mod_erzp5e.getGlobalDNSCache;
-exports.resetGlobalDNSCache = _mod_erzp5e.resetGlobalDNSCache;;
-const _mod_pra7ih = require('./response-cache.cjs');
-exports.ResponseCache = _mod_pra7ih.ResponseCache;
-exports.normalizeResponseCacheConfig = _mod_pra7ih.normalizeResponseCacheConfig;;
+const _mod_xclwey = require('./lru-cache.cjs');
+exports.LRUCache = _mod_xclwey.LRUCache;;
+const _mod_dqcrie = require('./dns-cache.cjs');
+exports.DNSCache = _mod_dqcrie.DNSCache;
+exports.getGlobalDNSCache = _mod_dqcrie.getGlobalDNSCache;
+exports.resetGlobalDNSCache = _mod_dqcrie.resetGlobalDNSCache;;
+const _mod_oqeitx = require('./response-cache.cjs');
+exports.ResponseCache = _mod_oqeitx.ResponseCache;
+exports.normalizeResponseCacheConfig = _mod_oqeitx.normalizeResponseCacheConfig;;

package/dist/crawler/addon/oxylabs/index.cjs

@@ -1 +1 @@
-var{Rezo:p}=require("../../../core/rezo.cjs"),a=require("./options.cjs");exports.OXYLABS_BROWSER_TYPES=a.OXYLABS_BROWSER_TYPES;exports.OXYLABS_COMMON_LOCALES=a.OXYLABS_COMMON_LOCALES;exports.OXYLABS_COMMON_GEO_LOCATIONS=a.OXYLABS_COMMON_GEO_LOCATIONS;exports.OXYLABS_US_STATES=a.OXYLABS_US_STATES;exports.OXYLABS_EUROPEAN_COUNTRIES=a.OXYLABS_EUROPEAN_COUNTRIES;exports.OXYLABS_ASIAN_COUNTRIES=a.OXYLABS_ASIAN_COUNTRIES;exports.getRandomBrowserType=a.getRandomBrowserType;exports.getRandomLocale=a.getRandomLocale;exports.getRandomGeoLocation=a.getRandomGeoLocation;var c="https://realtime.oxylabs.io/v1/queries";class u{config;http;authHeader;constructor(e){if(!e.username||!e.password)throw Error("Oxylabs username and password are required");this.config={username:e.username,password:e.password,browserType:e.browserType??"desktop",locale:e.locale??"en-US",geoLocation:e.geoLocation??"",render:e.render??!1,context:e.context??{},timeout:e.timeout??120000},this.http=new p({baseURL:c,timeout:this.config.timeout}),this.authHeader=`Basic ${Buffer.from(`${e.username}:${e.password}`).toString("base64")}`}async scrape(e,t){let r={...this.config,...t},s=this.buildRequestBody(e,r),o=(await this.http.postJson(c,s,{headers:{Authorization:this.authHeader,"Content-Type":"application/json"}})).data;if(o.error)throw Error(`Oxylabs API error: ${o.error}`);if(!o.results||o.results.length===0)throw Error("Oxylabs API returned no results");let n=o.results[0],d=n._response?.cookies||[],l=n._response?.headers||{};return{statusCode:n.status_code,url:n.url,content:n.content,cookies:d,headers:l,jobId:o.job_id||n.job_id,rendered:r.render,geoLocation:r.geoLocation||void 0,locale:r.locale,browserType:r.browserType,raw:o}}async scrapeMany(e,t,r=1000){let s=[];for(let i=0;i<e.length;i++){let o=await this.scrape(e[i],t);if(s.push(o),i<e.length-1&&r>0)await new Promise((n)=>setTimeout(n,r))}return s}buildRequestBody(e,t){let r=[];if(r.push({key:"return_page_cookies",value:!0}),r.push({key:"return_page_headers",value:!0}),t.context)for(let[i,o]of Object.entries(t.context))r.push({key:i,value:o});let s={source:"universal",url:e,context:r};if(t.render)s.render="html";if(t.browserType)s.user_agent_type=t.browserType;if(t.locale)s.locale=t.locale;if(t.geoLocation)s.geo_location=t.geoLocation;return s}getConfig(){return{...this.config,password:"***"}}withConfig(e){return new u({...this.config,...e})}async testConnection(){try{return await this.scrape("https://httpbin.org/ip"),!0}catch(e){throw Error(`Oxylabs connection test failed: ${e.message}`)}}}exports.Oxylabs=u;exports.default=u;module.exports=Object.assign(u,exports);
+var{Rezo:h}=require("../../../core/rezo.cjs"),a=require("./options.cjs");exports.OXYLABS_BROWSER_TYPES=a.OXYLABS_BROWSER_TYPES;exports.OXYLABS_COMMON_LOCALES=a.OXYLABS_COMMON_LOCALES;exports.OXYLABS_COMMON_GEO_LOCATIONS=a.OXYLABS_COMMON_GEO_LOCATIONS;exports.OXYLABS_US_STATES=a.OXYLABS_US_STATES;exports.OXYLABS_EUROPEAN_COUNTRIES=a.OXYLABS_EUROPEAN_COUNTRIES;exports.OXYLABS_ASIAN_COUNTRIES=a.OXYLABS_ASIAN_COUNTRIES;exports.getRandomBrowserType=a.getRandomBrowserType;exports.getRandomLocale=a.getRandomLocale;exports.getRandomGeoLocation=a.getRandomGeoLocation;var c="https://realtime.oxylabs.io/v1/queries";class u{config;http;authHeader;constructor(e){if(!e.username||!e.password)throw Error("Oxylabs username and password are required");this.config={username:e.username,password:e.password,browserType:e.browserType??"desktop",locale:e.locale??"en-US",geoLocation:e.geoLocation??"",render:e.render??!1,context:e.context??{},timeout:e.timeout??120000},this.http=new h({baseURL:c,timeout:this.config.timeout}),this.authHeader=`Basic ${Buffer.from(`${e.username}:${e.password}`).toString("base64")}`}async scrape(e,t){let r={...this.config,...t},s=this.buildRequestBody(e,r),o=(await this.http.postJson(c,s,{headers:{Authorization:this.authHeader,"Content-Type":"application/json"}})).data;if(o.error)throw Error(`Oxylabs API error: ${o.error}`);if(!o.results||o.results.length===0)throw Error("Oxylabs API returned no results");let n=o.results[0],d=n._response?.cookies||[],l=n._response?.headers||{};return{statusCode:n.status_code,url:n.url,content:n.content,cookies:d,headers:l,jobId:o.job_id||n.job_id,rendered:r.render,geoLocation:r.geoLocation||void 0,locale:r.locale,browserType:r.browserType,raw:o}}async scrapeMany(e,t,r=1000){let s=[];for(let i=0;i<e.length;i++){let o=await this.scrape(e[i],t);if(s.push(o),i<e.length-1&&r>0)await new Promise((n)=>setTimeout(n,r))}return s}buildRequestBody(e,t){let r=[];if(r.push({key:"return_page_cookies",value:!0}),r.push({key:"return_page_headers",value:!0}),t.context)for(let[i,o]of Object.entries(t.context))r.push({key:i,value:o});let s={source:"universal",url:e,context:r};if(t.render)s.render="html";if(t.browserType)s.user_agent_type=t.browserType;if(t.locale)s.locale=t.locale;if(t.geoLocation)s.geo_location=t.geoLocation;return s}getConfig(){return{...this.config,password:"***"}}withConfig(e){return new u({...this.config,...e})}async testConnection(){try{return await this.scrape("https://httpbin.org/ip"),!0}catch(e){throw Error(`Oxylabs connection test failed: ${e.message}`)}}}exports.Oxylabs=u;exports.default=u;module.exports=Object.assign(u,exports);

package/dist/crawler/crawler-options.cjs

@@ -1 +1 @@
-var{defineProperty:f,getOwnPropertyNames:g,getOwnPropertyDescriptor:y}=Object,A=Object.prototype.hasOwnProperty;var m=new WeakMap,v=(e)=>{var t=m.get(e),s;if(t)return t;if(t=f({},"__esModule",{value:!0}),e&&typeof e==="object"||typeof e==="function")g(e).map((a)=>!A.call(t,a)&&f(t,a,{get:()=>e[a],enumerable:!(s=y(e,a))||s.enumerable}));return m.set(e,t),t};var b={};module.exports=v(b);var{RezoQueue:$}=require("../queue/queue.cjs"),{Oxylabs:M}=require("./addon/oxylabs/index.cjs"),w=require("node:path"),O=require("node:os"),{Decodo:p}=require("./addon/decodo/index.cjs");class x{baseUrl;adapter;enableNavigationHistory;sessionId;rejectUnauthorized;userAgent;useRndUserAgent;timeout;maxRedirects;maxRetryAttempts;retryDelay;retryOnStatusCode;forceRevisit;retryWithoutProxyOnStatusCode;retryOnProxyError;maxRetryOnProxyError;allowRevisiting;enableCache;cacheTTL;cacheDir;throwFatalError;debug;maxDepth;maxUrls;maxResponseSize;respectRobotsTxt;followNofollow;autoThrottle;autoThrottleTargetDelay;autoThrottleMinDelay;autoThrottleMaxDelay;maxWaitOn429;alwaysWaitOn429;oxylabs=[];decodo=[];proxies=[];limiters=[];requestHeaders=[];userAgents=D();constructor(e={}){this.baseUrl=e.baseUrl||"",this.adapter=e.adapter??"http",this.enableNavigationHistory=e.enableNavigationHistory??!1,this.sessionId=e.sessionId??`session_${Date.now()}_${Math.random().toString(36).slice(2,8)}`,this.rejectUnauthorized=e.rejectUnauthorized??!0,this.userAgent=e.userAgent,this.useRndUserAgent=e.useRndUserAgent??!1,this.timeout=e.timeout??30000,this.maxRedirects=e.maxRedirects??10,this.maxRetryAttempts=e.maxRetryAttempts??3,this.retryDelay=e.retryDelay??0,this.retryOnStatusCode=e.retryOnStatusCode??[408,429,500,502,503,504],this.forceRevisit=e.forceRevisit??!1,this.retryWithoutProxyOnStatusCode=e.retryWithoutProxyOnStatusCode??[407,403],this.retryOnProxyError=e.retryOnProxyError??!0,this.maxRetryOnProxyError=e.maxRetryOnProxyError??3,this.allowRevisiting=e.allowRevisiting??!1,this.enableCache=e.enableCache??!0,this.cacheTTL=e.cacheTTL??604800000,this.cacheDir=e.cacheDir??w.join(O.tmpdir(),"uiniqhtt_cache"),this.throwFatalError=e.throwFatalError??!1,this.debug=e.debug??!1,this.maxDepth=e.maxDepth??0,this.maxUrls=e.maxUrls??0,this.maxResponseSize=e.maxResponseSize??0,this.respectRobotsTxt=e.respectRobotsTxt??!1,this.followNofollow=e.followNofollow??!1,this.autoThrottle=e.autoThrottle??!0,this.autoThrottleTargetDelay=e.autoThrottleTargetDelay??1000,this.autoThrottleMinDelay=e.autoThrottleMinDelay??100,this.autoThrottleMaxDelay=e.autoThrottleMaxDelay??60000,this.maxWaitOn429=e.maxWaitOn429??1800000,this.alwaysWaitOn429=e.alwaysWaitOn429??!1,this._addHeaders(e.headers),this._addOxylabs(e.oxylabs),this._addDecodo(e.decodo),this._addProxies(e.proxy),this._addLimiters(e.limiter)}getConfiguredDomains(e){return(e==="headers"?this.requestHeaders:e==="limiters"?this.limiters:e==="oxylabs"?this.oxylabs:this.proxies).filter((s)=>s.domain).map((s)=>s.domain).filter((s,a,r)=>r.indexOf(s)===a)}removeDomain(e){return this.requestHeaders=this.requestHeaders.filter((t)=>!t.domain||!this._domainsEqual(t.domain,e)),this.proxies=this.proxies.filter((t)=>!t.domain||!this._domainsEqual(t.domain,e)),this.limiters=this.limiters.filter((t)=>!t.domain||!this._domainsEqual(t.domain,e)),this.oxylabs=this.oxylabs.filter((t)=>!t.domain||!this._domainsEqual(t.domain,e)),this}_domainsEqual(e,t){if(Array.isArray(e)&&Array.isArray(t))return e.length===t.length&&e.every((s,a)=>s===t[a]);return e===t}getConfigurationSummary(){let e=(t)=>({total:t.length,global:t.filter((s)=>s.isGlobal).length,domainSpecific:t.filter((s)=>!s.isGlobal&&s.domain).length});return{headers:e(this.requestHeaders),proxies:e(this.proxies),limiters:e(this.limiters),oxylabs:e(this.oxylabs)}}_addHeaders(e){if(!e||!e.enable)return;for(let t of e.httpHeaders){let{domain:s,isGlobal:a,headers:r}=t;if(!s&&!a)continue;if(r instanceof Headers){let i=Object.fromEntries(r.entries());if(Object.keys(i).length<1)continue;r=i}else if(!r||Object.keys(r).length<1)continue;this.requestHeaders.push({domain:s,isGlobal:a,headers:r})}}_addProxies(e){if(!e||!e.enable)return;for(let t of e.proxies){let{domain:s,isGlobal:a,proxy:r}=t;if(!s&&!a)continue;if(!r||Object.keys(r).length<1)continue;this.proxies.push({domain:s,isGlobal:a,proxy:r})}}_addLimiters(e){if(!e||!e.enable)return;for(let t of e.limiters){let{domain:s,isGlobal:a,options:r}=t;if(!s&&!a)continue;if(!r||Object.keys(r).length<1)continue;this.limiters.push({domain:s,isGlobal:a,pqueue:new $(r)})}}_addOxylabs(e){if(!e||!e.enable)return;for(let t of e.labs){let{domain:s,isGlobal:a,options:r,queueOptions:i}=t;if(!s&&!a)continue;if(!r||Object.keys(r).length<1)continue;this.oxylabs.push({domain:s,isGlobal:a,adaptar:new M(r)})}}_addDecodo(e){if(!e||!e.enable)return;for(let t of e.labs){let{domain:s,isGlobal:a,options:r,queueOptions:i}=t;if(!s&&!a)continue;if(!r||Object.keys(r).length<1)continue;this.decodo.push({domain:s,isGlobal:a,adaptar:new p(r)})}}addHeaders(e){return this._addHeaders({enable:!0,httpHeaders:[e]}),this}addProxy(e){return this._addProxies({enable:!0,proxies:[e]}),this}addLimiter(e){return this._addLimiters({enable:!0,limiters:[e]}),this}addOxylabs(e){return this._addOxylabs({enable:!0,labs:[e]}),this}addDecodo(e){return this._addDecodo({enable:!0,labs:[e]}),this}clearGlobalConfigs(){if(Array.isArray(this.requestHeaders))this.requestHeaders=this.requestHeaders.filter((e)=>!e.isGlobal);if(Array.isArray(this.oxylabs))this.oxylabs=this.oxylabs.filter((e)=>!e.isGlobal);if(Array.isArray(this.limiters))this.limiters=this.limiters.filter((e)=>!e.isGlobal);if(Array.isArray(this.proxies))this.proxies=this.proxies.filter((e)=>!e.isGlobal);return this}getAdapter(e,t,s,a){if(!this.getDomainName(e))return null;let i=[],n=t==="headers"?this.requestHeaders:t==="limiters"?this.limiters:t==="oxylabs"?this.oxylabs:t==="decodo"?this.decodo:this.proxies;for(let o=0;o<n.length;o++)if(this._hasDomain(e,n[o].domain))i.push(o);if(i.length){let o=a?i[this.rnd(0,i.length-1)]:i[0];return t==="headers"?this.requestHeaders[o].headers:t==="limiters"?this.limiters[o].pqueue:t==="oxylabs"?this.oxylabs[o].adaptar:t==="decodo"?this.decodo[o].adaptar:this.proxies[o].proxy}i.length=0;for(let o=0;o<n.length;o++)i.push(o);if(i.length){let o=a?i[this.rnd(0,i.length-1)]:i[0];if(n[o].isGlobal&&s)return t==="headers"?this.requestHeaders[o].headers:t==="limiters"?this.limiters[o].pqueue:t==="oxylabs"?this.oxylabs[o].adaptar:t==="decodo"?this.decodo[o].adaptar:this.proxies[o].proxy}return null}rnd(e=0,t=Number.MAX_VALUE){return Math.floor(Math.random()*(t-e+1))+e}hasDomain(e,t,s){if(!this.getDomainName(e))return!1;let r=t==="headers"?this.requestHeaders:t==="limiters"?this.limiters:t==="oxylabs"?this.oxylabs:t==="decodo"?this.decodo:this.proxies;for(let i=0;i<r.length;i++)if(this._hasDomain(e,r[i].domain))return!0;if(s){for(let i=0;i<r.length;i++)if(r[i].isGlobal)return!0}return!1}pickHeaders(e,t,s,a){let r=this.getAdapter(e,"headers",t),i=new Headers(r??{});if(s&&s instanceof Headers)for(let[n,o]of Object.entries(s.entries()))i.set(n,o);else if(s&&typeof s==="object"){for(let[n,o]of Object.entries(s))if(typeof o==="string")i.set(n,o)}if(a)i.set("user-agent",this.getRandomUserAgent());return Object.fromEntries(i.entries())}_hasDomain(e,t){if(!t)return!1;let s=this.getDomainName(e);if(!s)return!1;let a=(i)=>{return/[\^\$\*\+\?\{\}\[\]\(\)\|\\]/.test(i)||i.startsWith("/")||i.includes(".*")||i.includes(".+")},r=(i)=>{if(i instanceof RegExp)return i.test(s)||i.test(e);let n=i.toString().trim();if(s.toLowerCase()===n.toLowerCase())return!0;if(n.includes("*")){let l=n.replace(/[.*+?^${}()|[\]\\]/g,"\\$&").replace(/\\\*/g,".*"),h=new RegExp(`^${l}$`,"i");return h.test(s)||h.test(e)}if(a(n))try{let l=n,h="i",u=n.match(/^\/(.*)\/(\w*)$/);if(u)l=u[1],h=u[2]||"i";let c=new RegExp(l,h);return c.test(s)||c.test(e)}catch(l){return s.toLowerCase().includes(n.toLowerCase())}let o=s.toLowerCase(),d=n.toLowerCase();return o===d||o.endsWith("."+d)||d.endsWith("."+o)};if(Array.isArray(t)){for(let i of t)if(r(i))return!0;return!1}return r(t)}getDomainName(e){if(this.isValidUrl(e))return new URL(e).hostname;else if(this.isHostName(e))return e;return null}isHostName(e){if(!e)return!1;if(e.length>255)return!1;let t=/^(?:(?:[a-zA-Z0-9](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?\.)+ [a-zA-Z]{2,})$/;return e=e.trim().toLowerCase(),t.test(e)&&!e.startsWith("-")&&!e.endsWith("-")}isValidUrl(e){if(!e)return!1;e=e.trim();try{let t=new URL(e);if(!t.protocol||!["http:","https:"].includes(t.protocol.toLowerCase()))return!1;if(!t.hostname)return!1;if(!/^(?:(?:[a-zA-Z0-9](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?\.)+[a-zA-Z]{2,})$/.test(t.hostname))return!1;return!0}catch{return!1}}getRandomUserAgent(){return this.userAgents[Math.floor(Math.random()*this.userAgents.length)]}}function D(){let e=[{name:"Chrome",version:"91.0.4472.124",engine:"AppleWebKit/537.36"},{name:"Firefox",version:"89.0",engine:"Gecko/20100101"},{name:"Safari",version:"14.1.1",engine:"AppleWebKit/605.1.15"},{name:"Edge",version:"91.0.864.59",engine:"AppleWebKit/537.36"},{name:"Opera",version:"77.0.4054.277",engine:"AppleWebKit/537.36"},{name:"Vivaldi",version:"3.8.2259.42",engine:"AppleWebKit/537.36"},{name:"Brave",version:"1.26.74",engine:"AppleWebKit/537.36"},{name:"Chromium",version:"91.0.4472.101",engine:"AppleWebKit/537.36"},{name:"Yandex",version:"21.5.3.742",engine:"AppleWebKit/537.36"},{name:"Maxthon",version:"5.3.8.2000",engine:"AppleWebKit/537.36"}],t=["Windows NT 10.0","Windows NT 6.1","Macintosh; Intel Mac OS X 10_15_7","Macintosh; Intel Mac OS X 11_4_0","X11; Linux x86_64","X11; Ubuntu; Linux x86_64"],s=[];for(let a=0;a<200;a++){let r=e[Math.floor(Math.random()*e.length)],i=t[Math.floor(Math.random()*t.length)],n="";switch(r.name){case"Chrome":n=`Mozilla/5.0 (${i}) ${r.engine} (KHTML, like Gecko) Chrome/${r.version} Safari/537.36`;break;case"Firefox":n=`Mozilla/5.0 (${i}; rv:${r.version}) ${r.engine} Firefox/${r.version}`;break;case"Safari":n=`Mozilla/5.0 (${i}) ${r.engine} (KHTML, like Gecko) Version/${r.version} Safari/605.1.15`;break;case"Edge":n=`Mozilla/5.0 (${i}) ${r.engine} (KHTML, like Gecko) Chrome/${r.version} Safari/537.36 Edg/${r.version}`;break;case"Opera":n=`Mozilla/5.0 (${i}) ${r.engine} (KHTML, like Gecko) Chrome/${r.version} Safari/537.36 OPR/${r.version}`;break;case"Vivaldi":n=`Mozilla/5.0 (${i}) ${r.engine} (KHTML, like Gecko) Chrome/${r.version} Safari/537.36 Vivaldi/${r.version}`;break;case"Brave":n=`Mozilla/5.0 (${i}) ${r.engine} (KHTML, like Gecko) Chrome/${r.version} Safari/537.36 Brave/${r.version}`;break;case"Chromium":n=`Mozilla/5.0 (${i}) ${r.engine} (KHTML, like Gecko) Chromium/${r.version} Chrome/${r.version} Safari/537.36`;break;case"Yandex":n=`Mozilla/5.0 (${i}) ${r.engine} (KHTML, like Gecko) Chrome/${r.version} YaBrowser/${r.version} Safari/537.36`;break;case"Maxthon":n=`Mozilla/5.0 (${i}) ${r.engine} (KHTML, like Gecko) Chrome/${r.version} Safari/537.36 Maxthon/${r.version}`;break}s.push(n)}return s}b.CrawlerOptions=x;
+var{RezoQueue:m}=require("../queue/queue.cjs"),{Oxylabs:x}=require("./addon/oxylabs/index.cjs"),b=require("node:path"),g=require("node:os"),{Decodo:y}=require("./addon/decodo/index.cjs");class f{baseUrl;adapter;enableNavigationHistory;sessionId;rejectUnauthorized;userAgent;useRndUserAgent;timeout;maxRedirects;maxRetryAttempts;retryDelay;retryOnStatusCode;forceRevisit;retryWithoutProxyOnStatusCode;retryOnProxyError;maxRetryOnProxyError;allowRevisiting;enableCache;cacheTTL;cacheDir;throwFatalError;debug;maxDepth;maxUrls;maxResponseSize;respectRobotsTxt;followNofollow;autoThrottle;autoThrottleTargetDelay;autoThrottleMinDelay;autoThrottleMaxDelay;maxWaitOn429;alwaysWaitOn429;oxylabs=[];decodo=[];proxies=[];limiters=[];requestHeaders=[];userAgents=A();constructor(e={}){this.baseUrl=e.baseUrl||"",this.adapter=e.adapter??"http",this.enableNavigationHistory=e.enableNavigationHistory??!1,this.sessionId=e.sessionId??`session_${Date.now()}_${Math.random().toString(36).slice(2,8)}`,this.rejectUnauthorized=e.rejectUnauthorized??!0,this.userAgent=e.userAgent,this.useRndUserAgent=e.useRndUserAgent??!1,this.timeout=e.timeout??30000,this.maxRedirects=e.maxRedirects??10,this.maxRetryAttempts=e.maxRetryAttempts??3,this.retryDelay=e.retryDelay??0,this.retryOnStatusCode=e.retryOnStatusCode??[408,429,500,502,503,504],this.forceRevisit=e.forceRevisit??!1,this.retryWithoutProxyOnStatusCode=e.retryWithoutProxyOnStatusCode??[407,403],this.retryOnProxyError=e.retryOnProxyError??!0,this.maxRetryOnProxyError=e.maxRetryOnProxyError??3,this.allowRevisiting=e.allowRevisiting??!1,this.enableCache=e.enableCache??!0,this.cacheTTL=e.cacheTTL??604800000,this.cacheDir=e.cacheDir??b.join(g.tmpdir(),"uiniqhtt_cache"),this.throwFatalError=e.throwFatalError??!1,this.debug=e.debug??!1,this.maxDepth=e.maxDepth??0,this.maxUrls=e.maxUrls??0,this.maxResponseSize=e.maxResponseSize??0,this.respectRobotsTxt=e.respectRobotsTxt??!1,this.followNofollow=e.followNofollow??!1,this.autoThrottle=e.autoThrottle??!0,this.autoThrottleTargetDelay=e.autoThrottleTargetDelay??1000,this.autoThrottleMinDelay=e.autoThrottleMinDelay??100,this.autoThrottleMaxDelay=e.autoThrottleMaxDelay??60000,this.maxWaitOn429=e.maxWaitOn429??1800000,this.alwaysWaitOn429=e.alwaysWaitOn429??!1,this._addHeaders(e.headers),this._addOxylabs(e.oxylabs),this._addDecodo(e.decodo),this._addProxies(e.proxy),this._addLimiters(e.limiter)}getConfiguredDomains(e){return(e==="headers"?this.requestHeaders:e==="limiters"?this.limiters:e==="oxylabs"?this.oxylabs:this.proxies).filter((i)=>i.domain).map((i)=>i.domain).filter((i,o,r)=>r.indexOf(i)===o)}removeDomain(e){return this.requestHeaders=this.requestHeaders.filter((t)=>!t.domain||!this._domainsEqual(t.domain,e)),this.proxies=this.proxies.filter((t)=>!t.domain||!this._domainsEqual(t.domain,e)),this.limiters=this.limiters.filter((t)=>!t.domain||!this._domainsEqual(t.domain,e)),this.oxylabs=this.oxylabs.filter((t)=>!t.domain||!this._domainsEqual(t.domain,e)),this}_domainsEqual(e,t){if(Array.isArray(e)&&Array.isArray(t))return e.length===t.length&&e.every((i,o)=>i===t[o]);return e===t}getConfigurationSummary(){let e=(t)=>({total:t.length,global:t.filter((i)=>i.isGlobal).length,domainSpecific:t.filter((i)=>!i.isGlobal&&i.domain).length});return{headers:e(this.requestHeaders),proxies:e(this.proxies),limiters:e(this.limiters),oxylabs:e(this.oxylabs)}}_addHeaders(e){if(!e||!e.enable)return;for(let t of e.httpHeaders){let{domain:i,isGlobal:o,headers:r}=t;if(!i&&!o)continue;if(r instanceof Headers){let s=Object.fromEntries(r.entries());if(Object.keys(s).length<1)continue;r=s}else if(!r||Object.keys(r).length<1)continue;this.requestHeaders.push({domain:i,isGlobal:o,headers:r})}}_addProxies(e){if(!e||!e.enable)return;for(let t of e.proxies){let{domain:i,isGlobal:o,proxy:r}=t;if(!i&&!o)continue;if(!r||Object.keys(r).length<1)continue;this.proxies.push({domain:i,isGlobal:o,proxy:r})}}_addLimiters(e){if(!e||!e.enable)return;for(let t of e.limiters){let{domain:i,isGlobal:o,options:r}=t;if(!i&&!o)continue;if(!r||Object.keys(r).length<1)continue;this.limiters.push({domain:i,isGlobal:o,pqueue:new m(r)})}}_addOxylabs(e){if(!e||!e.enable)return;for(let t of e.labs){let{domain:i,isGlobal:o,options:r,queueOptions:s}=t;if(!i&&!o)continue;if(!r||Object.keys(r).length<1)continue;this.oxylabs.push({domain:i,isGlobal:o,adaptar:new x(r)})}}_addDecodo(e){if(!e||!e.enable)return;for(let t of e.labs){let{domain:i,isGlobal:o,options:r,queueOptions:s}=t;if(!i&&!o)continue;if(!r||Object.keys(r).length<1)continue;this.decodo.push({domain:i,isGlobal:o,adaptar:new y(r)})}}addHeaders(e){return this._addHeaders({enable:!0,httpHeaders:[e]}),this}addProxy(e){return this._addProxies({enable:!0,proxies:[e]}),this}addLimiter(e){return this._addLimiters({enable:!0,limiters:[e]}),this}addOxylabs(e){return this._addOxylabs({enable:!0,labs:[e]}),this}addDecodo(e){return this._addDecodo({enable:!0,labs:[e]}),this}clearGlobalConfigs(){if(Array.isArray(this.requestHeaders))this.requestHeaders=this.requestHeaders.filter((e)=>!e.isGlobal);if(Array.isArray(this.oxylabs))this.oxylabs=this.oxylabs.filter((e)=>!e.isGlobal);if(Array.isArray(this.limiters))this.limiters=this.limiters.filter((e)=>!e.isGlobal);if(Array.isArray(this.proxies))this.proxies=this.proxies.filter((e)=>!e.isGlobal);return this}getAdapter(e,t,i,o){if(!this.getDomainName(e))return null;let s=[],a=t==="headers"?this.requestHeaders:t==="limiters"?this.limiters:t==="oxylabs"?this.oxylabs:t==="decodo"?this.decodo:this.proxies;for(let n=0;n<a.length;n++)if(this._hasDomain(e,a[n].domain))s.push(n);if(s.length){let n=o?s[this.rnd(0,s.length-1)]:s[0];return t==="headers"?this.requestHeaders[n].headers:t==="limiters"?this.limiters[n].pqueue:t==="oxylabs"?this.oxylabs[n].adaptar:t==="decodo"?this.decodo[n].adaptar:this.proxies[n].proxy}s.length=0;for(let n=0;n<a.length;n++)s.push(n);if(s.length){let n=o?s[this.rnd(0,s.length-1)]:s[0];if(a[n].isGlobal&&i)return t==="headers"?this.requestHeaders[n].headers:t==="limiters"?this.limiters[n].pqueue:t==="oxylabs"?this.oxylabs[n].adaptar:t==="decodo"?this.decodo[n].adaptar:this.proxies[n].proxy}return null}rnd(e=0,t=Number.MAX_VALUE){return Math.floor(Math.random()*(t-e+1))+e}hasDomain(e,t,i){if(!this.getDomainName(e))return!1;let r=t==="headers"?this.requestHeaders:t==="limiters"?this.limiters:t==="oxylabs"?this.oxylabs:t==="decodo"?this.decodo:this.proxies;for(let s=0;s<r.length;s++)if(this._hasDomain(e,r[s].domain))return!0;if(i){for(let s=0;s<r.length;s++)if(r[s].isGlobal)return!0}return!1}pickHeaders(e,t,i,o){let r=this.getAdapter(e,"headers",t),s=new Headers(r??{});if(i&&i instanceof Headers)for(let[a,n]of Object.entries(i.entries()))s.set(a,n);else if(i&&typeof i==="object"){for(let[a,n]of Object.entries(i))if(typeof n==="string")s.set(a,n)}if(o)s.set("user-agent",this.getRandomUserAgent());return Object.fromEntries(s.entries())}_hasDomain(e,t){if(!t)return!1;let i=this.getDomainName(e);if(!i)return!1;let o=(s)=>{return/[\^\$\*\+\?\{\}\[\]\(\)\|\\]/.test(s)||s.startsWith("/")||s.includes(".*")||s.includes(".+")},r=(s)=>{if(s instanceof RegExp)return s.test(i)||s.test(e);let a=s.toString().trim();if(i.toLowerCase()===a.toLowerCase())return!0;if(a.includes("*")){let l=a.replace(/[.*+?^${}()|[\]\\]/g,"\\$&").replace(/\\\*/g,".*"),h=new RegExp(`^${l}$`,"i");return h.test(i)||h.test(e)}if(o(a))try{let l=a,h="i",u=a.match(/^\/(.*)\/(\w*)$/);if(u)l=u[1],h=u[2]||"i";let c=new RegExp(l,h);return c.test(i)||c.test(e)}catch(l){return i.toLowerCase().includes(a.toLowerCase())}let n=i.toLowerCase(),d=a.toLowerCase();return n===d||n.endsWith("."+d)||d.endsWith("."+n)};if(Array.isArray(t)){for(let s of t)if(r(s))return!0;return!1}return r(t)}getDomainName(e){if(this.isValidUrl(e))return new URL(e).hostname;else if(this.isHostName(e))return e;return null}isHostName(e){if(!e)return!1;if(e.length>255)return!1;let t=/^(?:(?:[a-zA-Z0-9](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?\.)+ [a-zA-Z]{2,})$/;return e=e.trim().toLowerCase(),t.test(e)&&!e.startsWith("-")&&!e.endsWith("-")}isValidUrl(e){if(!e)return!1;e=e.trim();try{let t=new URL(e);if(!t.protocol||!["http:","https:"].includes(t.protocol.toLowerCase()))return!1;if(!t.hostname)return!1;if(!/^(?:(?:[a-zA-Z0-9](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?\.)+[a-zA-Z]{2,})$/.test(t.hostname))return!1;return!0}catch{return!1}}getRandomUserAgent(){return this.userAgents[Math.floor(Math.random()*this.userAgents.length)]}}function A(){let e=[{name:"Chrome",version:"91.0.4472.124",engine:"AppleWebKit/537.36"},{name:"Firefox",version:"89.0",engine:"Gecko/20100101"},{name:"Safari",version:"14.1.1",engine:"AppleWebKit/605.1.15"},{name:"Edge",version:"91.0.864.59",engine:"AppleWebKit/537.36"},{name:"Opera",version:"77.0.4054.277",engine:"AppleWebKit/537.36"},{name:"Vivaldi",version:"3.8.2259.42",engine:"AppleWebKit/537.36"},{name:"Brave",version:"1.26.74",engine:"AppleWebKit/537.36"},{name:"Chromium",version:"91.0.4472.101",engine:"AppleWebKit/537.36"},{name:"Yandex",version:"21.5.3.742",engine:"AppleWebKit/537.36"},{name:"Maxthon",version:"5.3.8.2000",engine:"AppleWebKit/537.36"}],t=["Windows NT 10.0","Windows NT 6.1","Macintosh; Intel Mac OS X 10_15_7","Macintosh; Intel Mac OS X 11_4_0","X11; Linux x86_64","X11; Ubuntu; Linux x86_64"],i=[];for(let o=0;o<200;o++){let r=e[Math.floor(Math.random()*e.length)],s=t[Math.floor(Math.random()*t.length)],a="";switch(r.name){case"Chrome":a=`Mozilla/5.0 (${s}) ${r.engine} (KHTML, like Gecko) Chrome/${r.version} Safari/537.36`;break;case"Firefox":a=`Mozilla/5.0 (${s}; rv:${r.version}) ${r.engine} Firefox/${r.version}`;break;case"Safari":a=`Mozilla/5.0 (${s}) ${r.engine} (KHTML, like Gecko) Version/${r.version} Safari/605.1.15`;break;case"Edge":a=`Mozilla/5.0 (${s}) ${r.engine} (KHTML, like Gecko) Chrome/${r.version} Safari/537.36 Edg/${r.version}`;break;case"Opera":a=`Mozilla/5.0 (${s}) ${r.engine} (KHTML, like Gecko) Chrome/${r.version} Safari/537.36 OPR/${r.version}`;break;case"Vivaldi":a=`Mozilla/5.0 (${s}) ${r.engine} (KHTML, like Gecko) Chrome/${r.version} Safari/537.36 Vivaldi/${r.version}`;break;case"Brave":a=`Mozilla/5.0 (${s}) ${r.engine} (KHTML, like Gecko) Chrome/${r.version} Safari/537.36 Brave/${r.version}`;break;case"Chromium":a=`Mozilla/5.0 (${s}) ${r.engine} (KHTML, like Gecko) Chromium/${r.version} Chrome/${r.version} Safari/537.36`;break;case"Yandex":a=`Mozilla/5.0 (${s}) ${r.engine} (KHTML, like Gecko) Chrome/${r.version} YaBrowser/${r.version} Safari/537.36`;break;case"Maxthon":a=`Mozilla/5.0 (${s}) ${r.engine} (KHTML, like Gecko) Chrome/${r.version} Safari/537.36 Maxthon/${r.version}`;break}i.push(a)}return i}exports.CrawlerOptions=f;

package/dist/crawler/crawler.cjs

@@ -5,11 +5,12 @@ const { NavigationHistory } = require('./plugin/navigation-history.cjs');
 const { RobotsTxt } = require('./plugin/robots-txt.cjs');
 const { parseHTML } = require("linkedom");
 const path = require("node:path");
-const { Rezo } = require('../core/rezo.cjs');
+const { Rezo } = require('../adapters/entries/http.cjs');
 const { RezoQueue } = require('../queue/queue.cjs');
 const { Scraper } = require('./scraper.cjs');
 const { CrawlerOptions } = require('./crawler-options.cjs');
 const { loadAdapter } = require('../adapters/picker.cjs');
+const { resetGlobalAgentPool } = require('../utils/agent-pool.cjs');
 String.prototype.addBaseUrl = function(url) {
   url = url instanceof URL ? url.href : url;
   const html = this.replace(/<base\b[^>]*?>/gi, "");
@@ -94,6 +95,10 @@ class Crawler {
       }).then((storage) => {
         this.cacher = storage;
         this.isCacheReady = true;
+      }).catch((err) => {
+        if (this.config.debug)
+          console.warn("[Crawler] Failed to initialize cache:", err);
+        this.isCacheReady = true;
       });
       const dit = path.resolve(cacheDir, "urls");
       if (!fs.existsSync(dit))
@@ -105,6 +110,10 @@ class Crawler {
       }).then((storage) => {
         this.urlStorage = storage;
         this.isStorageReady = true;
+      }).catch((err) => {
+        if (this.config.debug)
+          console.warn("[Crawler] Failed to initialize URL storage:", err);
+        this.isStorageReady = true;
       });
     } else {
       const dit = path.resolve(this.config.cacheDir, "./cache/urls");
@@ -117,6 +126,10 @@ class Crawler {
       }).then((storage) => {
         this.urlStorage = storage;
         this.isStorageReady = true;
+      }).catch((err) => {
+        if (this.config.debug)
+          console.warn("[Crawler] Failed to initialize URL storage:", err);
+        this.isStorageReady = true;
       });
     }
     if (this.config.enableNavigationHistory) {
@@ -856,12 +869,15 @@ class Crawler {
   }
   async execute(method, url, body, options = {}, isEmail, forceRevisit, oxylabsOptions, oxylabsInstanse, decodoInstanse, decodoOptions) {
     await this.waitForStorage();
+    console.log("Waiting for storage...");
     if (this.isCacheEnabled) {
       await this.waitForCache();
     }
+    console.log("Waiting for cache...");
     if (this.config.enableNavigationHistory) {
       await this.waitForNavigationHistory();
     }
+    console.log("Waiting for navigation history...");
     const task = this.queue.add(() => this.executeHttp(method, url, body, options, isEmail, forceRevisit, oxylabsOptions, oxylabsInstanse, decodoInstanse, decodoOptions));
     task.finally(() => this.pendingExecutions.delete(task));
   }
@@ -888,7 +904,9 @@ class Crawler {
   }
   async executeHttp(method, url, body, options = {}, isEmail, forceRevisit, oxylabsOptions, oxylabsInstanse, decodoInstanse, decodoOptions, retryCount = 0, parentUrl) {
     try {
+      console.log("Triggering start handlers...");
       await this.triggerStartHandlers();
+      console.log("Checking crawl limits...");
       const limitCheck = await this.checkCrawlLimits(url, parentUrl);
       if (!limitCheck.allowed) {
         if (this.config.debug) {
@@ -903,13 +921,16 @@ class Crawler {
         await new Promise((resolve) => setTimeout(resolve, delay));
       }
       const isVisited = forceRevisit ? false : await this.hasUrlInCache(url);
-      const cache = await this.getCache(url);
-      if (isVisited && !cache)
-        return;
-      if (isVisited && method !== "GET")
-        return;
+      const cache = method.toLowerCase() === "get" ? await this.getCache(url) : undefined;
+      if (method.toLowerCase() === "get") {
+        if (isVisited && !cache)
+          return;
+        if (isVisited)
+          return;
+      }
       const requestStartTime = Date.now();
       const response = cache && method === "GET" ? cache : oxylabsInstanse && oxylabsOptions ? await oxylabsInstanse.scrape(url) : decodoInstanse && decodoOptions ? await decodoInstanse.scrape(url) : await (method === "GET" ? this.http.get(url, options) : method === "PATCH" ? this.http.patch(url, body, options) : method === "POST" ? this.http.post(url, body, options) : this.http.put(url, body, options));
+      console.log("Response received...");
       if (!cache) {
         const responseTime = Date.now() - requestStartTime;
         this.calculateAutoThrottleDelay(domain, responseTime);
@@ -943,9 +964,9 @@ class Crawler {
           statusCode: res.status
         });
       }
-      if (!cache)
+      if (!cache && method === "GET")
         await this.saveCache(url, res);
-      if (!isVisited)
+      if (!isVisited && method === "GET")
         await this.saveUrl(url);
       await this.markUrlVisited(url, {
         status: res.status,
@@ -1039,6 +1060,7 @@ class Crawler {
     }
     await this.queue.onIdle();
     await this.triggerFinishHandlers();
+    await this.destroy();
   }
   async done() {
     return this.waitForAll();
@@ -1055,7 +1077,7 @@ class Crawler {
     } catch {}
   }
   async destroy() {
-    this.queue.clear();
+    this.queue.destroy();
     this.events.length = 0;
     this.jsonEvents.length = 0;
     this.errorEvents.length = 0;
@@ -1064,6 +1086,7 @@ class Crawler {
     this.emailDiscoveredEvents.length = 0;
     this.emailLeadsEvents.length = 0;
     await this.close();
+    resetGlobalAgentPool();
   }
 }
 

package/dist/crawler/crawler.js

@@ -5,11 +5,12 @@ import { NavigationHistory } from './plugin/navigation-history.js';
 import { RobotsTxt } from './plugin/robots-txt.js';
 import { parseHTML } from "linkedom";
 import path from "node:path";
-import { Rezo } from '../core/rezo.js';
+import { Rezo } from '../adapters/entries/http.js';
 import { RezoQueue } from '../queue/queue.js';
 import { Scraper } from './scraper.js';
 import { CrawlerOptions } from './crawler-options.js';
 import { loadAdapter } from '../adapters/picker.js';
+import { resetGlobalAgentPool } from '../utils/agent-pool.js';
 String.prototype.addBaseUrl = function(url) {
   url = url instanceof URL ? url.href : url;
   const html = this.replace(/<base\b[^>]*?>/gi, "");
@@ -94,6 +95,10 @@ export class Crawler {
       }).then((storage) => {
         this.cacher = storage;
         this.isCacheReady = true;
+      }).catch((err) => {
+        if (this.config.debug)
+          console.warn("[Crawler] Failed to initialize cache:", err);
+        this.isCacheReady = true;
       });
       const dit = path.resolve(cacheDir, "urls");
       if (!fs.existsSync(dit))
@@ -105,6 +110,10 @@ export class Crawler {
       }).then((storage) => {
         this.urlStorage = storage;
         this.isStorageReady = true;
+      }).catch((err) => {
+        if (this.config.debug)
+          console.warn("[Crawler] Failed to initialize URL storage:", err);
+        this.isStorageReady = true;
       });
     } else {
       const dit = path.resolve(this.config.cacheDir, "./cache/urls");
@@ -117,6 +126,10 @@ export class Crawler {
       }).then((storage) => {
         this.urlStorage = storage;
         this.isStorageReady = true;
+      }).catch((err) => {
+        if (this.config.debug)
+          console.warn("[Crawler] Failed to initialize URL storage:", err);
+        this.isStorageReady = true;
       });
     }
     if (this.config.enableNavigationHistory) {
@@ -856,12 +869,15 @@ export class Crawler {
   }
   async execute(method, url, body, options = {}, isEmail, forceRevisit, oxylabsOptions, oxylabsInstanse, decodoInstanse, decodoOptions) {
     await this.waitForStorage();
+    console.log("Waiting for storage...");
     if (this.isCacheEnabled) {
       await this.waitForCache();
     }
+    console.log("Waiting for cache...");
     if (this.config.enableNavigationHistory) {
       await this.waitForNavigationHistory();
     }
+    console.log("Waiting for navigation history...");
     const task = this.queue.add(() => this.executeHttp(method, url, body, options, isEmail, forceRevisit, oxylabsOptions, oxylabsInstanse, decodoInstanse, decodoOptions));
     task.finally(() => this.pendingExecutions.delete(task));
   }
@@ -888,7 +904,9 @@ export class Crawler {
   }
   async executeHttp(method, url, body, options = {}, isEmail, forceRevisit, oxylabsOptions, oxylabsInstanse, decodoInstanse, decodoOptions, retryCount = 0, parentUrl) {
     try {
+      console.log("Triggering start handlers...");
       await this.triggerStartHandlers();
+      console.log("Checking crawl limits...");
       const limitCheck = await this.checkCrawlLimits(url, parentUrl);
       if (!limitCheck.allowed) {
         if (this.config.debug) {
@@ -903,13 +921,16 @@ export class Crawler {
         await new Promise((resolve) => setTimeout(resolve, delay));
       }
       const isVisited = forceRevisit ? false : await this.hasUrlInCache(url);
-      const cache = await this.getCache(url);
-      if (isVisited && !cache)
-        return;
-      if (isVisited && method !== "GET")
-        return;
+      const cache = method.toLowerCase() === "get" ? await this.getCache(url) : undefined;
+      if (method.toLowerCase() === "get") {
+        if (isVisited && !cache)
+          return;
+        if (isVisited)
+          return;
+      }
       const requestStartTime = Date.now();
       const response = cache && method === "GET" ? cache : oxylabsInstanse && oxylabsOptions ? await oxylabsInstanse.scrape(url) : decodoInstanse && decodoOptions ? await decodoInstanse.scrape(url) : await (method === "GET" ? this.http.get(url, options) : method === "PATCH" ? this.http.patch(url, body, options) : method === "POST" ? this.http.post(url, body, options) : this.http.put(url, body, options));
+      console.log("Response received...");
       if (!cache) {
         const responseTime = Date.now() - requestStartTime;
         this.calculateAutoThrottleDelay(domain, responseTime);
@@ -943,9 +964,9 @@ export class Crawler {
           statusCode: res.status
         });
       }
-      if (!cache)
+      if (!cache && method === "GET")
         await this.saveCache(url, res);
-      if (!isVisited)
+      if (!isVisited && method === "GET")
         await this.saveUrl(url);
       await this.markUrlVisited(url, {
         status: res.status,
@@ -1039,6 +1060,7 @@ export class Crawler {
     }
     await this.queue.onIdle();
     await this.triggerFinishHandlers();
+    await this.destroy();
   }
   async done() {
     return this.waitForAll();
@@ -1055,7 +1077,7 @@ export class Crawler {
     } catch {}
   }
   async destroy() {
-    this.queue.clear();
+    this.queue.destroy();
     this.events.length = 0;
     this.jsonEvents.length = 0;
     this.errorEvents.length = 0;
@@ -1064,5 +1086,6 @@ export class Crawler {
     this.emailDiscoveredEvents.length = 0;
     this.emailLeadsEvents.length = 0;
     await this.close();
+    resetGlobalAgentPool();
   }
 }

package/dist/crawler/index.cjs

@@ -1,40 +1,40 @@
-const _mod_6qbwtg = require('./crawler.cjs');
-exports.Crawler = _mod_6qbwtg.Crawler;;
-const _mod_9ij1lx = require('./crawler-options.cjs');
-exports.CrawlerOptions = _mod_9ij1lx.CrawlerOptions;;
-const _mod_775rjr = require('./plugin/robots-txt.cjs');
-exports.RobotsTxt = _mod_775rjr.RobotsTxt;;
-const _mod_k9roj6 = require('./plugin/file-cacher.cjs');
-exports.FileCacher = _mod_k9roj6.FileCacher;;
-const _mod_4by88j = require('./plugin/url-store.cjs');
-exports.UrlStore = _mod_4by88j.UrlStore;;
-const _mod_lrvy6d = require('./plugin/navigation-history.cjs');
-exports.NavigationHistory = _mod_lrvy6d.NavigationHistory;;
-const _mod_sr9k75 = require('./addon/oxylabs/index.cjs');
-exports.Oxylabs = _mod_sr9k75.Oxylabs;;
-const _mod_gos8z0 = require('./addon/oxylabs/options.cjs');
-exports.OXYLABS_BROWSER_TYPES = _mod_gos8z0.OXYLABS_BROWSER_TYPES;
-exports.OXYLABS_COMMON_LOCALES = _mod_gos8z0.OXYLABS_COMMON_LOCALES;
-exports.OXYLABS_COMMON_GEO_LOCATIONS = _mod_gos8z0.OXYLABS_COMMON_GEO_LOCATIONS;
-exports.OXYLABS_US_STATES = _mod_gos8z0.OXYLABS_US_STATES;
-exports.OXYLABS_EUROPEAN_COUNTRIES = _mod_gos8z0.OXYLABS_EUROPEAN_COUNTRIES;
-exports.OXYLABS_ASIAN_COUNTRIES = _mod_gos8z0.OXYLABS_ASIAN_COUNTRIES;
-exports.getRandomOxylabsBrowserType = _mod_gos8z0.getRandomBrowserType;
-exports.getRandomOxylabsLocale = _mod_gos8z0.getRandomLocale;
-exports.getRandomOxylabsGeoLocation = _mod_gos8z0.getRandomGeoLocation;;
-const _mod_0ku3ak = require('./addon/decodo/index.cjs');
-exports.Decodo = _mod_0ku3ak.Decodo;;
-const _mod_4jcnuq = require('./addon/decodo/options.cjs');
-exports.DECODO_DEVICE_TYPES = _mod_4jcnuq.DECODO_DEVICE_TYPES;
-exports.DECODO_HEADLESS_MODES = _mod_4jcnuq.DECODO_HEADLESS_MODES;
-exports.DECODO_COMMON_LOCALES = _mod_4jcnuq.DECODO_COMMON_LOCALES;
-exports.DECODO_COMMON_COUNTRIES = _mod_4jcnuq.DECODO_COMMON_COUNTRIES;
-exports.DECODO_EUROPEAN_COUNTRIES = _mod_4jcnuq.DECODO_EUROPEAN_COUNTRIES;
-exports.DECODO_ASIAN_COUNTRIES = _mod_4jcnuq.DECODO_ASIAN_COUNTRIES;
-exports.DECODO_US_STATES = _mod_4jcnuq.DECODO_US_STATES;
-exports.DECODO_COMMON_CITIES = _mod_4jcnuq.DECODO_COMMON_CITIES;
-exports.getRandomDecodoDeviceType = _mod_4jcnuq.getRandomDeviceType;
-exports.getRandomDecodoLocale = _mod_4jcnuq.getRandomLocale;
-exports.getRandomDecodoCountry = _mod_4jcnuq.getRandomCountry;
-exports.getRandomDecodoCity = _mod_4jcnuq.getRandomCity;
-exports.generateDecodoSessionId = _mod_4jcnuq.generateSessionId;;
+const _mod_rvxqq1 = require('./crawler.cjs');
+exports.Crawler = _mod_rvxqq1.Crawler;;
+const _mod_jy5ex9 = require('./crawler-options.cjs');
+exports.CrawlerOptions = _mod_jy5ex9.CrawlerOptions;;
+const _mod_3wi87q = require('./plugin/robots-txt.cjs');
+exports.RobotsTxt = _mod_3wi87q.RobotsTxt;;
+const _mod_fl8p4x = require('./plugin/file-cacher.cjs');
+exports.FileCacher = _mod_fl8p4x.FileCacher;;
+const _mod_mdvmnw = require('./plugin/url-store.cjs');
+exports.UrlStore = _mod_mdvmnw.UrlStore;;
+const _mod_p8gmd0 = require('./plugin/navigation-history.cjs');
+exports.NavigationHistory = _mod_p8gmd0.NavigationHistory;;
+const _mod_e2qt7i = require('./addon/oxylabs/index.cjs');
+exports.Oxylabs = _mod_e2qt7i.Oxylabs;;
+const _mod_5utni8 = require('./addon/oxylabs/options.cjs');
+exports.OXYLABS_BROWSER_TYPES = _mod_5utni8.OXYLABS_BROWSER_TYPES;
+exports.OXYLABS_COMMON_LOCALES = _mod_5utni8.OXYLABS_COMMON_LOCALES;
+exports.OXYLABS_COMMON_GEO_LOCATIONS = _mod_5utni8.OXYLABS_COMMON_GEO_LOCATIONS;
+exports.OXYLABS_US_STATES = _mod_5utni8.OXYLABS_US_STATES;
+exports.OXYLABS_EUROPEAN_COUNTRIES = _mod_5utni8.OXYLABS_EUROPEAN_COUNTRIES;
+exports.OXYLABS_ASIAN_COUNTRIES = _mod_5utni8.OXYLABS_ASIAN_COUNTRIES;
+exports.getRandomOxylabsBrowserType = _mod_5utni8.getRandomBrowserType;
+exports.getRandomOxylabsLocale = _mod_5utni8.getRandomLocale;
+exports.getRandomOxylabsGeoLocation = _mod_5utni8.getRandomGeoLocation;;
+const _mod_gao0f0 = require('./addon/decodo/index.cjs');
+exports.Decodo = _mod_gao0f0.Decodo;;
+const _mod_gzkadc = require('./addon/decodo/options.cjs');
+exports.DECODO_DEVICE_TYPES = _mod_gzkadc.DECODO_DEVICE_TYPES;
+exports.DECODO_HEADLESS_MODES = _mod_gzkadc.DECODO_HEADLESS_MODES;
+exports.DECODO_COMMON_LOCALES = _mod_gzkadc.DECODO_COMMON_LOCALES;
+exports.DECODO_COMMON_COUNTRIES = _mod_gzkadc.DECODO_COMMON_COUNTRIES;
+exports.DECODO_EUROPEAN_COUNTRIES = _mod_gzkadc.DECODO_EUROPEAN_COUNTRIES;
+exports.DECODO_ASIAN_COUNTRIES = _mod_gzkadc.DECODO_ASIAN_COUNTRIES;
+exports.DECODO_US_STATES = _mod_gzkadc.DECODO_US_STATES;
+exports.DECODO_COMMON_CITIES = _mod_gzkadc.DECODO_COMMON_CITIES;
+exports.getRandomDecodoDeviceType = _mod_gzkadc.getRandomDeviceType;
+exports.getRandomDecodoLocale = _mod_gzkadc.getRandomLocale;
+exports.getRandomDecodoCountry = _mod_gzkadc.getRandomCountry;
+exports.getRandomDecodoCity = _mod_gzkadc.getRandomCity;
+exports.generateDecodoSessionId = _mod_gzkadc.generateSessionId;;

package/dist/entries/crawler.cjs

@@ -1,5 +1,5 @@
-const _mod_rv2j5v = require('../crawler/crawler.cjs');
-exports.Crawler = _mod_rv2j5v.Crawler;;
-const _mod_3ot4xw = require('../crawler/crawler-options.cjs');
-exports.CrawlerOptions = _mod_3ot4xw.CrawlerOptions;
-exports.Domain = _mod_3ot4xw.Domain;;
+const _mod_f7dbqn = require('../crawler/crawler.cjs');
+exports.Crawler = _mod_f7dbqn.Crawler;;
+const _mod_h6mpf1 = require('../crawler/crawler-options.cjs');
+exports.CrawlerOptions = _mod_h6mpf1.CrawlerOptions;
+exports.Domain = _mod_h6mpf1.Domain;;

package/dist/index.cjs

@@ -1,30 +1,30 @@
-const _mod_j077ww = require('./core/rezo.cjs');
-exports.Rezo = _mod_j077ww.Rezo;
-exports.createRezoInstance = _mod_j077ww.createRezoInstance;
-exports.createDefaultInstance = _mod_j077ww.createDefaultInstance;;
-const _mod_vwpt1f = require('./errors/rezo-error.cjs');
-exports.RezoError = _mod_vwpt1f.RezoError;
-exports.RezoErrorCode = _mod_vwpt1f.RezoErrorCode;;
-const _mod_dtsggs = require('./utils/headers.cjs');
-exports.RezoHeaders = _mod_dtsggs.RezoHeaders;;
-const _mod_42ww1i = require('./utils/form-data.cjs');
-exports.RezoFormData = _mod_42ww1i.RezoFormData;;
-const _mod_f1n5ji = require('./utils/cookies.cjs');
-exports.RezoCookieJar = _mod_f1n5ji.RezoCookieJar;
-exports.Cookie = _mod_f1n5ji.Cookie;;
-const _mod_oejjlq = require('./utils/curl.cjs');
-exports.toCurl = _mod_oejjlq.toCurl;
-exports.fromCurl = _mod_oejjlq.fromCurl;;
-const _mod_7hseuc = require('./core/hooks.cjs');
-exports.createDefaultHooks = _mod_7hseuc.createDefaultHooks;
-exports.mergeHooks = _mod_7hseuc.mergeHooks;;
-const _mod_gy98dc = require('./proxy/manager.cjs');
-exports.ProxyManager = _mod_gy98dc.ProxyManager;;
-const _mod_s8gstf = require('./queue/index.cjs');
-exports.RezoQueue = _mod_s8gstf.RezoQueue;
-exports.HttpQueue = _mod_s8gstf.HttpQueue;
-exports.Priority = _mod_s8gstf.Priority;
-exports.HttpMethodPriority = _mod_s8gstf.HttpMethodPriority;;
+const _mod_jh14m9 = require('./core/rezo.cjs');
+exports.Rezo = _mod_jh14m9.Rezo;
+exports.createRezoInstance = _mod_jh14m9.createRezoInstance;
+exports.createDefaultInstance = _mod_jh14m9.createDefaultInstance;;
+const _mod_5th6ta = require('./errors/rezo-error.cjs');
+exports.RezoError = _mod_5th6ta.RezoError;
+exports.RezoErrorCode = _mod_5th6ta.RezoErrorCode;;
+const _mod_45ccpv = require('./utils/headers.cjs');
+exports.RezoHeaders = _mod_45ccpv.RezoHeaders;;
+const _mod_4hcsiy = require('./utils/form-data.cjs');
+exports.RezoFormData = _mod_4hcsiy.RezoFormData;;
+const _mod_4l7ckn = require('./utils/cookies.cjs');
+exports.RezoCookieJar = _mod_4l7ckn.RezoCookieJar;
+exports.Cookie = _mod_4l7ckn.Cookie;;
+const _mod_uz0amh = require('./utils/curl.cjs');
+exports.toCurl = _mod_uz0amh.toCurl;
+exports.fromCurl = _mod_uz0amh.fromCurl;;
+const _mod_yngoam = require('./core/hooks.cjs');
+exports.createDefaultHooks = _mod_yngoam.createDefaultHooks;
+exports.mergeHooks = _mod_yngoam.mergeHooks;;
+const _mod_k1cygh = require('./proxy/manager.cjs');
+exports.ProxyManager = _mod_k1cygh.ProxyManager;;
+const _mod_di8wn9 = require('./queue/index.cjs');
+exports.RezoQueue = _mod_di8wn9.RezoQueue;
+exports.HttpQueue = _mod_di8wn9.HttpQueue;
+exports.Priority = _mod_di8wn9.Priority;
+exports.HttpMethodPriority = _mod_di8wn9.HttpMethodPriority;;
 const { RezoError } = require('./errors/rezo-error.cjs');
 const isRezoError = exports.isRezoError = RezoError.isRezoError;
 const Cancel = exports.Cancel = RezoError;

package/dist/internal/agents/index.cjs

@@ -1,10 +1,10 @@
-const _mod_l57fx3 = require('./base.cjs');
-exports.Agent = _mod_l57fx3.Agent;;
-const _mod_n1joag = require('./http-proxy.cjs');
-exports.HttpProxyAgent = _mod_n1joag.HttpProxyAgent;;
-const _mod_r59sx3 = require('./https-proxy.cjs');
-exports.HttpsProxyAgent = _mod_r59sx3.HttpsProxyAgent;;
-const _mod_ektwdy = require('./socks-proxy.cjs');
-exports.SocksProxyAgent = _mod_ektwdy.SocksProxyAgent;;
-const _mod_ajkmvy = require('./socks-client.cjs');
-exports.SocksClient = _mod_ajkmvy.SocksClient;;
+const _mod_yzlkcq = require('./base.cjs');
+exports.Agent = _mod_yzlkcq.Agent;;
+const _mod_ebay39 = require('./http-proxy.cjs');
+exports.HttpProxyAgent = _mod_ebay39.HttpProxyAgent;;
+const _mod_xntect = require('./https-proxy.cjs');
+exports.HttpsProxyAgent = _mod_xntect.HttpsProxyAgent;;
+const _mod_ti239q = require('./socks-proxy.cjs');
+exports.SocksProxyAgent = _mod_ti239q.SocksProxyAgent;;
+const _mod_8g5j5m = require('./socks-client.cjs');
+exports.SocksClient = _mod_8g5j5m.SocksClient;;

package/dist/proxy/index.cjs

@@ -1,9 +1,9 @@
 const { Agent, HttpProxyAgent, HttpsProxyAgent, SocksProxyAgent } = require('../internal/agents/index.cjs');
 const { parseProxyString } = require('./parse.cjs');
-const _mod_vdfb66 = require('./manager.cjs');
-exports.ProxyManager = _mod_vdfb66.ProxyManager;;
-const _mod_tpxocw = require('./parse.cjs');
-exports.parseProxyString = _mod_tpxocw.parseProxyString;;
+const _mod_pkrm9n = require('./manager.cjs');
+exports.ProxyManager = _mod_pkrm9n.ProxyManager;;
+const _mod_glpq6p = require('./parse.cjs');
+exports.parseProxyString = _mod_glpq6p.parseProxyString;;
 function createOptions(uri, opts) {
   if (uri instanceof URL || typeof uri === "string") {
     return {

package/dist/queue/index.cjs

@@ -1,8 +1,8 @@
-const _mod_twpsnc = require('./queue.cjs');
-exports.RezoQueue = _mod_twpsnc.RezoQueue;;
-const _mod_fjmcur = require('./http-queue.cjs');
-exports.HttpQueue = _mod_fjmcur.HttpQueue;
-exports.extractDomain = _mod_fjmcur.extractDomain;;
-const _mod_r7ktc8 = require('./types.cjs');
-exports.Priority = _mod_r7ktc8.Priority;
-exports.HttpMethodPriority = _mod_r7ktc8.HttpMethodPriority;;
+const _mod_kc1i35 = require('./queue.cjs');
+exports.RezoQueue = _mod_kc1i35.RezoQueue;;
+const _mod_ajadj5 = require('./http-queue.cjs');
+exports.HttpQueue = _mod_ajadj5.HttpQueue;
+exports.extractDomain = _mod_ajadj5.extractDomain;;
+const _mod_6ghr69 = require('./types.cjs');
+exports.Priority = _mod_6ghr69.Priority;
+exports.HttpMethodPriority = _mod_6ghr69.HttpMethodPriority;;

package/dist/responses/universal/index.cjs

@@ -1,11 +1,11 @@
-const _mod_pmc4pz = require('./event-emitter.cjs');
-exports.UniversalEventEmitter = _mod_pmc4pz.UniversalEventEmitter;;
-const _mod_r9i55d = require('./stream.cjs');
-exports.UniversalStreamResponse = _mod_r9i55d.UniversalStreamResponse;
-exports.StreamResponse = _mod_r9i55d.StreamResponse;;
-const _mod_6diyad = require('./download.cjs');
-exports.UniversalDownloadResponse = _mod_6diyad.UniversalDownloadResponse;
-exports.DownloadResponse = _mod_6diyad.DownloadResponse;;
-const _mod_lyyos6 = require('./upload.cjs');
-exports.UniversalUploadResponse = _mod_lyyos6.UniversalUploadResponse;
-exports.UploadResponse = _mod_lyyos6.UploadResponse;;
+const _mod_yo50yl = require('./event-emitter.cjs');
+exports.UniversalEventEmitter = _mod_yo50yl.UniversalEventEmitter;;
+const _mod_l4durm = require('./stream.cjs');
+exports.UniversalStreamResponse = _mod_l4durm.UniversalStreamResponse;
+exports.StreamResponse = _mod_l4durm.StreamResponse;;
+const _mod_0wvtwo = require('./download.cjs');
+exports.UniversalDownloadResponse = _mod_0wvtwo.UniversalDownloadResponse;
+exports.DownloadResponse = _mod_0wvtwo.DownloadResponse;;
+const _mod_c7py98 = require('./upload.cjs');
+exports.UniversalUploadResponse = _mod_c7py98.UniversalUploadResponse;
+exports.UploadResponse = _mod_c7py98.UploadResponse;;

package/dist/utils/compression.cjs

@@ -1,23 +1,4 @@
-const { createGunzip, createInflate, createBrotliDecompress } = require("node:zlib");
-const { Transform } = require("node:stream");
-let createZstdDecompress = null;
-let zstdAvailable = null;
-function getZstdDecompressor() {
-  if (zstdAvailable === null) {
-    try {
-      const zlib = require("node:zlib");
-      if (typeof zlib.createZstdDecompress === "function") {
-        createZstdDecompress = zlib.createZstdDecompress;
-        zstdAvailable = true;
-      } else {
-        zstdAvailable = false;
-      }
-    } catch {
-      zstdAvailable = false;
-    }
-  }
-  return createZstdDecompress;
-}
+const { createGunzip, createInflate, createBrotliDecompress, createZstdDecompress } = require("node:zlib");
 
 class CompressionUtil {
   static decompressStream(response, contentEncoding, config) {
@@ -41,7 +22,7 @@ class CompressionUtil {
       case "brotli":
         return response.pipe(createBrotliDecompress());
       case "zstd":
-        return this.createZstdDecompressStream(response);
+        return response.pipe(createZstdDecompress());
       default:
         return response;
     }
@@ -61,18 +42,6 @@ class CompressionUtil {
     }
     return true;
   }
-  static createZstdDecompressStream(response) {
-    const decompressor = getZstdDecompressor();
-    if (!decompressor) {
-      const passthrough = new Transform({
-        transform(chunk, encoding, callback) {
-          callback(new Error("zstd decompression not available: requires Node.js 22.15+ with native zstd support"));
-        }
-      });
-      return response.pipe(passthrough);
-    }
-    return response.pipe(decompressor());
-  }
   static getSupportedAlgorithms() {
     return ["gzip", "x-gzip", "deflate", "x-deflate", "gzip-raw", "br", "brotli", "zstd"];
   }

package/dist/utils/compression.js

@@ -1,23 +1,4 @@
-import { createGunzip, createInflate, createBrotliDecompress } from "node:zlib";
-import { Transform } from "node:stream";
-let createZstdDecompress = null;
-let zstdAvailable = null;
-function getZstdDecompressor() {
-  if (zstdAvailable === null) {
-    try {
-      const zlib = require("node:zlib");
-      if (typeof zlib.createZstdDecompress === "function") {
-        createZstdDecompress = zlib.createZstdDecompress;
-        zstdAvailable = true;
-      } else {
-        zstdAvailable = false;
-      }
-    } catch {
-      zstdAvailable = false;
-    }
-  }
-  return createZstdDecompress;
-}
+import { createGunzip, createInflate, createBrotliDecompress, createZstdDecompress } from "node:zlib";
 
 export class CompressionUtil {
   static decompressStream(response, contentEncoding, config) {
@@ -41,7 +22,7 @@ export class CompressionUtil {
       case "brotli":
         return response.pipe(createBrotliDecompress());
       case "zstd":
-        return this.createZstdDecompressStream(response);
+        return response.pipe(createZstdDecompress());
       default:
         return response;
     }
@@ -61,18 +42,6 @@ export class CompressionUtil {
     }
     return true;
   }
-  static createZstdDecompressStream(response) {
-    const decompressor = getZstdDecompressor();
-    if (!decompressor) {
-      const passthrough = new Transform({
-        transform(chunk, encoding, callback) {
-          callback(new Error("zstd decompression not available: requires Node.js 22.15+ with native zstd support"));
-        }
-      });
-      return response.pipe(passthrough);
-    }
-    return response.pipe(decompressor());
-  }
   static getSupportedAlgorithms() {
     return ["gzip", "x-gzip", "deflate", "x-deflate", "gzip-raw", "br", "brotli", "zstd"];
   }

package/dist/utils/http-config.cjs

@@ -173,7 +173,7 @@ function setSignal() {
     clearTimeout(this.timeoutClearInstanse);
   if (this.timeout && typeof this.timeout === "number" && this.timeout > 100) {
     const controller = new AbortController;
-    const timer = setTimeout(() => controller.abort(), this.timeout);
+    const timer = setTimeout(() => controller.abort(), this.timeout).unref();
     this.timeoutClearInstanse = timer;
     this.signal = controller.signal;
   }

package/dist/utils/http-config.js

@@ -173,7 +173,7 @@ function setSignal() {
     clearTimeout(this.timeoutClearInstanse);
   if (this.timeout && typeof this.timeout === "number" && this.timeout > 100) {
     const controller = new AbortController;
-    const timer = setTimeout(() => controller.abort(), this.timeout);
+    const timer = setTimeout(() => controller.abort(), this.timeout).unref();
     this.timeoutClearInstanse = timer;
     this.signal = controller.signal;
   }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "rezo",
-  "version": "1.0.50",
+  "version": "1.0.52",
   "description": "Lightning-fast, enterprise-grade HTTP client for modern JavaScript. Full HTTP/2 support, intelligent cookie management, multiple adapters (HTTP, Fetch, cURL, XHR), streaming, proxy support (HTTP/HTTPS/SOCKS), and cross-environment compatibility.",
   "main": "dist/index.cjs",
   "module": "dist/index.js",