rezo 1.0.76 → 1.0.78

package/dist/adapters/entries/curl.d.ts

@@ -4582,7 +4582,7 @@ export interface RezoInstance extends Rezo {
  *
  * IMPORTANT: Update these values when bumping package version.
  */
-export declare const VERSION = "1.0.76";
+export declare const VERSION = "1.0.78";
 /**
  * cURL Options Configuration
  *

package/dist/adapters/entries/fetch.d.ts

@@ -4582,7 +4582,7 @@ export interface RezoInstance extends Rezo {
  *
  * IMPORTANT: Update these values when bumping package version.
  */
-export declare const VERSION = "1.0.76";
+export declare const VERSION = "1.0.78";
 export declare const isRezoError: typeof RezoError.isRezoError;
 export declare const Cancel: typeof RezoError;
 export declare const CancelToken: {

package/dist/adapters/entries/http.d.ts

@@ -4582,7 +4582,7 @@ export interface RezoInstance extends Rezo {
  *
  * IMPORTANT: Update these values when bumping package version.
  */
-export declare const VERSION = "1.0.76";
+export declare const VERSION = "1.0.78";
 /**
  * Type guard to check if an error is a RezoError instance.
  */

package/dist/adapters/entries/http2.d.ts

@@ -4582,7 +4582,7 @@ export interface RezoInstance extends Rezo {
  *
  * IMPORTANT: Update these values when bumping package version.
  */
-export declare const VERSION = "1.0.76";
+export declare const VERSION = "1.0.78";
 export declare const isRezoError: typeof RezoError.isRezoError;
 export declare const Cancel: typeof RezoError;
 export declare const CancelToken: {

package/dist/adapters/entries/react-native.d.ts

@@ -4582,7 +4582,7 @@ export interface RezoInstance extends Rezo {
  *
  * IMPORTANT: Update these values when bumping package version.
  */
-export declare const VERSION = "1.0.76";
+export declare const VERSION = "1.0.78";
 export declare const isRezoError: typeof RezoError.isRezoError;
 export declare const Cancel: typeof RezoError;
 export declare const CancelToken: {

package/dist/adapters/entries/xhr.d.ts

@@ -4582,7 +4582,7 @@ export interface RezoInstance extends Rezo {
  *
  * IMPORTANT: Update these values when bumping package version.
  */
-export declare const VERSION = "1.0.76";
+export declare const VERSION = "1.0.78";
 export declare const isRezoError: typeof RezoError.isRezoError;
 export declare const Cancel: typeof RezoError;
 export declare const CancelToken: {

package/dist/adapters/index.cjs

@@ -1,6 +1,6 @@
-const _mod_p2iotn = require('./picker.cjs');
-exports.detectRuntime = _mod_p2iotn.detectRuntime;
-exports.getAdapterCapabilities = _mod_p2iotn.getAdapterCapabilities;
-exports.buildAdapterContext = _mod_p2iotn.buildAdapterContext;
-exports.getAvailableAdapters = _mod_p2iotn.getAvailableAdapters;
-exports.selectAdapter = _mod_p2iotn.selectAdapter;;
+const _mod_qvllzl = require('./picker.cjs');
+exports.detectRuntime = _mod_qvllzl.detectRuntime;
+exports.getAdapterCapabilities = _mod_qvllzl.getAdapterCapabilities;
+exports.buildAdapterContext = _mod_qvllzl.buildAdapterContext;
+exports.getAvailableAdapters = _mod_qvllzl.getAvailableAdapters;
+exports.selectAdapter = _mod_qvllzl.selectAdapter;;

package/dist/cache/index.cjs

@@ -1,9 +1,9 @@
-const _mod_zvxqrg = require('./lru-cache.cjs');
-exports.LRUCache = _mod_zvxqrg.LRUCache;;
-const _mod_u7bdfq = require('./dns-cache.cjs');
-exports.DNSCache = _mod_u7bdfq.DNSCache;
-exports.getGlobalDNSCache = _mod_u7bdfq.getGlobalDNSCache;
-exports.resetGlobalDNSCache = _mod_u7bdfq.resetGlobalDNSCache;;
-const _mod_cjcq8q = require('./response-cache.cjs');
-exports.ResponseCache = _mod_cjcq8q.ResponseCache;
-exports.normalizeResponseCacheConfig = _mod_cjcq8q.normalizeResponseCacheConfig;;
+const _mod_g8wjr4 = require('./lru-cache.cjs');
+exports.LRUCache = _mod_g8wjr4.LRUCache;;
+const _mod_m9iqi3 = require('./dns-cache.cjs');
+exports.DNSCache = _mod_m9iqi3.DNSCache;
+exports.getGlobalDNSCache = _mod_m9iqi3.getGlobalDNSCache;
+exports.resetGlobalDNSCache = _mod_m9iqi3.resetGlobalDNSCache;;
+const _mod_2mwone = require('./response-cache.cjs');
+exports.ResponseCache = _mod_2mwone.ResponseCache;
+exports.normalizeResponseCacheConfig = _mod_2mwone.normalizeResponseCacheConfig;;

package/dist/crawler/crawler.cjs

@@ -45,6 +45,7 @@ class Crawler {
   rawResponseEvents = [];
   emailDiscoveredEvents = [];
   emailLeadsEvents = [];
+  eventCount = 0;
   cacher = null;
   queue;
   scraperQueue;
@@ -728,8 +729,35 @@ class Crawler {
       }
     }
   }
+  _runHandler(handler, arg) {
+    this.eventCount++;
+    new Promise(async (resolve) => {
+      try {
+        await handler(arg);
+      } catch (err) {
+        if (this.config.debug)
+          console.error("[Crawler] Handler error:", err);
+      }
+      resolve();
+    }).finally(() => {
+      this.eventCount--;
+    });
+  }
+  _runBoundHandler(fn) {
+    this.eventCount++;
+    Promise.resolve().then(async () => {
+      try {
+        await fn();
+      } catch (err) {
+        if (this.config.debug)
+          console.error("[Crawler] Handler error:", err);
+      }
+    }).finally(() => {
+      this.eventCount--;
+    });
+  }
   _onBody(handler, document) {
-    this.queue.add(() => handler(document.body));
+    this._runHandler(handler, document.body);
   }
   _onAttribute(selection, attribute, handler, document) {
     const isSimpleForm = typeof attribute === "function";
@@ -741,7 +769,7 @@ class Crawler {
       const el = elements[i];
       if (el.hasAttribute(actualAttribute)) {
         const value = el.getAttribute(actualAttribute);
-        this.queue.add(() => actualHandler.call(el, value, actualAttribute));
+        this._runBoundHandler(() => actualHandler.call(el, value, actualAttribute));
       }
     }
   }
@@ -750,19 +778,19 @@ class Crawler {
     for (let i = 0;i < elements.length; i++) {
       const el = elements[i];
       const text = el.textContent;
-      this.queue.add(() => handler.call(el, text));
+      this._runBoundHandler(() => handler.call(el, text));
     }
   }
   _onSelection(selection, handler, document) {
     const elements = document.querySelectorAll(selection);
     for (let i = 0;i < elements.length; i++) {
-      this.queue.add(() => handler(elements[i]));
+      this._runHandler(handler, elements[i]);
     }
   }
   _onElement(handler, document) {
     const elements = document.querySelectorAll("*");
     for (let i = 0;i < elements.length; i++) {
-      this.queue.add(() => handler(elements[i]));
+      this._runHandler(handler, elements[i]);
     }
   }
   _onHref(handler, document) {
@@ -771,7 +799,7 @@ class Crawler {
       const el = elements[i];
       if (el.hasAttribute("href")) {
         const href = new URL(el.getAttribute("href"), document.URL).href;
-        this.queue.add(() => handler.call(el, href));
+        this._runBoundHandler(() => handler.call(el, href));
       }
     }
   }
@@ -782,29 +810,29 @@ class Crawler {
     for (let i = 0;i < elements.length; i++) {
       if (elements[i]?.href && document.baseURI)
         elements[i].href = new URL(elements[i].getAttribute("href"), document.baseURI).href;
-      this.queue.add(() => handler(elements[i]));
+      this._runHandler(handler, elements[i]);
     }
   }
   _onDocument(handler, document) {
-    this.queue.add(() => handler(document));
+    this._runHandler(handler, document);
   }
   _onJson(handler, json) {
-    this.queue.add(() => handler(json));
+    this._runHandler(handler, json);
   }
   _onError(handler, error) {
-    this.queue.add(() => handler(error));
+    this._runHandler(handler, error);
   }
   async _onEmailDiscovered(handler, email) {
-    this.queue.add(() => handler(email));
+    this._runHandler(handler, email);
   }
   async _onEmailLeads(handler, emails) {
-    this.queue.add(() => handler(emails));
+    this._runHandler(handler, emails);
   }
   _onRawResponse(handler, rawResponse) {
-    this.queue.add(() => handler(rawResponse));
+    this._runHandler(handler, rawResponse);
   }
   _onResponse(handler, response) {
-    this.queue.add(() => handler(response));
+    this._runHandler(handler, response);
   }
   calculateAutoThrottleDelay(domain, responseTime) {
     if (!this.config.autoThrottle)
@@ -1015,13 +1043,7 @@ class Crawler {
       this.crawlStats.startTime = Date.now();
       this.startPeriodicCleanup();
       for (const handler of this.startHandlers) {
-        try {
-          this.queue.add(() => handler());
-        } catch (error) {
-          if (this.config.debug) {
-            console.error("[Crawler] onStart handler error:", error);
-          }
-        }
+        this._runBoundHandler(() => handler());
       }
     })();
     return this.startHandlersPromise;
@@ -1029,24 +1051,12 @@ class Crawler {
   async triggerFinishHandlers() {
     this.crawlStats.endTime = Date.now();
     for (const handler of this.finishHandlers) {
-      try {
-        this.queue.add(() => handler(this.crawlStats));
-      } catch (error) {
-        if (this.config.debug) {
-          console.error("[Crawler] onFinish handler error:", error);
-        }
-      }
+      this._runHandler(handler, this.crawlStats);
     }
   }
   async triggerRedirectHandlers(event) {
     for (const handler of this.redirectHandlers) {
-      try {
-        this.queue.add(() => handler(event));
-      } catch (error) {
-        if (this.config.debug) {
-          console.error("[Crawler] onRedirect handler error:", error);
-        }
-      }
+      this._runHandler(handler, event);
     }
   }
   buildUrl(url, params) {
@@ -1362,7 +1372,7 @@ class Crawler {
       if (scraperSize === 0 && scraperPending === 0) {
         const queueSize = this.queue.size;
         const queuePending = this.queue.pending;
-        if (queueSize === 0 && queuePending === 0) {
+        if (queueSize === 0 && queuePending === 0 && this.eventCount === 0) {
           consecutiveIdleChecks++;
           if (consecutiveIdleChecks >= REQUIRED_IDLE_CHECKS) {
             break;

package/dist/crawler/crawler.js

@@ -45,6 +45,7 @@ export class Crawler {
   rawResponseEvents = [];
   emailDiscoveredEvents = [];
   emailLeadsEvents = [];
+  eventCount = 0;
   cacher = null;
   queue;
   scraperQueue;
@@ -728,8 +729,35 @@ export class Crawler {
       }
     }
   }
+  _runHandler(handler, arg) {
+    this.eventCount++;
+    new Promise(async (resolve) => {
+      try {
+        await handler(arg);
+      } catch (err) {
+        if (this.config.debug)
+          console.error("[Crawler] Handler error:", err);
+      }
+      resolve();
+    }).finally(() => {
+      this.eventCount--;
+    });
+  }
+  _runBoundHandler(fn) {
+    this.eventCount++;
+    Promise.resolve().then(async () => {
+      try {
+        await fn();
+      } catch (err) {
+        if (this.config.debug)
+          console.error("[Crawler] Handler error:", err);
+      }
+    }).finally(() => {
+      this.eventCount--;
+    });
+  }
   _onBody(handler, document) {
-    this.queue.add(() => handler(document.body));
+    this._runHandler(handler, document.body);
   }
   _onAttribute(selection, attribute, handler, document) {
     const isSimpleForm = typeof attribute === "function";
@@ -741,7 +769,7 @@ export class Crawler {
       const el = elements[i];
       if (el.hasAttribute(actualAttribute)) {
         const value = el.getAttribute(actualAttribute);
-        this.queue.add(() => actualHandler.call(el, value, actualAttribute));
+        this._runBoundHandler(() => actualHandler.call(el, value, actualAttribute));
       }
     }
   }
@@ -750,19 +778,19 @@ export class Crawler {
     for (let i = 0;i < elements.length; i++) {
       const el = elements[i];
       const text = el.textContent;
-      this.queue.add(() => handler.call(el, text));
+      this._runBoundHandler(() => handler.call(el, text));
     }
   }
   _onSelection(selection, handler, document) {
     const elements = document.querySelectorAll(selection);
     for (let i = 0;i < elements.length; i++) {
-      this.queue.add(() => handler(elements[i]));
+      this._runHandler(handler, elements[i]);
     }
   }
   _onElement(handler, document) {
     const elements = document.querySelectorAll("*");
     for (let i = 0;i < elements.length; i++) {
-      this.queue.add(() => handler(elements[i]));
+      this._runHandler(handler, elements[i]);
     }
   }
   _onHref(handler, document) {
@@ -771,7 +799,7 @@ export class Crawler {
       const el = elements[i];
       if (el.hasAttribute("href")) {
         const href = new URL(el.getAttribute("href"), document.URL).href;
-        this.queue.add(() => handler.call(el, href));
+        this._runBoundHandler(() => handler.call(el, href));
       }
     }
   }
@@ -782,29 +810,29 @@ export class Crawler {
     for (let i = 0;i < elements.length; i++) {
       if (elements[i]?.href && document.baseURI)
         elements[i].href = new URL(elements[i].getAttribute("href"), document.baseURI).href;
-      this.queue.add(() => handler(elements[i]));
+      this._runHandler(handler, elements[i]);
     }
   }
   _onDocument(handler, document) {
-    this.queue.add(() => handler(document));
+    this._runHandler(handler, document);
   }
   _onJson(handler, json) {
-    this.queue.add(() => handler(json));
+    this._runHandler(handler, json);
   }
   _onError(handler, error) {
-    this.queue.add(() => handler(error));
+    this._runHandler(handler, error);
   }
   async _onEmailDiscovered(handler, email) {
-    this.queue.add(() => handler(email));
+    this._runHandler(handler, email);
   }
   async _onEmailLeads(handler, emails) {
-    this.queue.add(() => handler(emails));
+    this._runHandler(handler, emails);
   }
   _onRawResponse(handler, rawResponse) {
-    this.queue.add(() => handler(rawResponse));
+    this._runHandler(handler, rawResponse);
   }
   _onResponse(handler, response) {
-    this.queue.add(() => handler(response));
+    this._runHandler(handler, response);
   }
   calculateAutoThrottleDelay(domain, responseTime) {
     if (!this.config.autoThrottle)
@@ -1015,13 +1043,7 @@ export class Crawler {
       this.crawlStats.startTime = Date.now();
       this.startPeriodicCleanup();
       for (const handler of this.startHandlers) {
-        try {
-          this.queue.add(() => handler());
-        } catch (error) {
-          if (this.config.debug) {
-            console.error("[Crawler] onStart handler error:", error);
-          }
-        }
+        this._runBoundHandler(() => handler());
       }
     })();
     return this.startHandlersPromise;
@@ -1029,24 +1051,12 @@ export class Crawler {
   async triggerFinishHandlers() {
     this.crawlStats.endTime = Date.now();
     for (const handler of this.finishHandlers) {
-      try {
-        this.queue.add(() => handler(this.crawlStats));
-      } catch (error) {
-        if (this.config.debug) {
-          console.error("[Crawler] onFinish handler error:", error);
-        }
-      }
+      this._runHandler(handler, this.crawlStats);
     }
   }
   async triggerRedirectHandlers(event) {
     for (const handler of this.redirectHandlers) {
-      try {
-        this.queue.add(() => handler(event));
-      } catch (error) {
-        if (this.config.debug) {
-          console.error("[Crawler] onRedirect handler error:", error);
-        }
-      }
+      this._runHandler(handler, event);
     }
   }
   buildUrl(url, params) {
@@ -1362,7 +1372,7 @@ export class Crawler {
       if (scraperSize === 0 && scraperPending === 0) {
         const queueSize = this.queue.size;
         const queuePending = this.queue.pending;
-        if (queueSize === 0 && queuePending === 0) {
+        if (queueSize === 0 && queuePending === 0 && this.eventCount === 0) {
           consecutiveIdleChecks++;
           if (consecutiveIdleChecks >= REQUIRED_IDLE_CHECKS) {
             break;

package/dist/crawler/index.cjs

@@ -1,40 +1,42 @@
-const _mod_lzbr9j = require('./crawler.cjs');
-exports.Crawler = _mod_lzbr9j.Crawler;;
-const _mod_u60zhc = require('./crawler-options.cjs');
-exports.CrawlerOptions = _mod_u60zhc.CrawlerOptions;;
-const _mod_1vebd3 = require('./plugin/robots-txt.cjs');
-exports.RobotsTxt = _mod_1vebd3.RobotsTxt;;
-const _mod_bhjqtv = require('./plugin/file-cacher.cjs');
-exports.FileCacher = _mod_bhjqtv.FileCacher;;
-const _mod_04rd64 = require('./plugin/url-store.cjs');
-exports.UrlStore = _mod_04rd64.UrlStore;;
-const _mod_0e89e3 = require('./plugin/navigation-history.cjs');
-exports.NavigationHistory = _mod_0e89e3.NavigationHistory;;
-const _mod_y6ssi0 = require('./addon/oxylabs/index.cjs');
-exports.Oxylabs = _mod_y6ssi0.Oxylabs;;
-const _mod_3tpb6j = require('./addon/oxylabs/options.cjs');
-exports.OXYLABS_BROWSER_TYPES = _mod_3tpb6j.OXYLABS_BROWSER_TYPES;
-exports.OXYLABS_COMMON_LOCALES = _mod_3tpb6j.OXYLABS_COMMON_LOCALES;
-exports.OXYLABS_COMMON_GEO_LOCATIONS = _mod_3tpb6j.OXYLABS_COMMON_GEO_LOCATIONS;
-exports.OXYLABS_US_STATES = _mod_3tpb6j.OXYLABS_US_STATES;
-exports.OXYLABS_EUROPEAN_COUNTRIES = _mod_3tpb6j.OXYLABS_EUROPEAN_COUNTRIES;
-exports.OXYLABS_ASIAN_COUNTRIES = _mod_3tpb6j.OXYLABS_ASIAN_COUNTRIES;
-exports.getRandomOxylabsBrowserType = _mod_3tpb6j.getRandomBrowserType;
-exports.getRandomOxylabsLocale = _mod_3tpb6j.getRandomLocale;
-exports.getRandomOxylabsGeoLocation = _mod_3tpb6j.getRandomGeoLocation;;
-const _mod_74k0au = require('./addon/decodo/index.cjs');
-exports.Decodo = _mod_74k0au.Decodo;;
-const _mod_hibwsw = require('./addon/decodo/options.cjs');
-exports.DECODO_DEVICE_TYPES = _mod_hibwsw.DECODO_DEVICE_TYPES;
-exports.DECODO_HEADLESS_MODES = _mod_hibwsw.DECODO_HEADLESS_MODES;
-exports.DECODO_COMMON_LOCALES = _mod_hibwsw.DECODO_COMMON_LOCALES;
-exports.DECODO_COMMON_COUNTRIES = _mod_hibwsw.DECODO_COMMON_COUNTRIES;
-exports.DECODO_EUROPEAN_COUNTRIES = _mod_hibwsw.DECODO_EUROPEAN_COUNTRIES;
-exports.DECODO_ASIAN_COUNTRIES = _mod_hibwsw.DECODO_ASIAN_COUNTRIES;
-exports.DECODO_US_STATES = _mod_hibwsw.DECODO_US_STATES;
-exports.DECODO_COMMON_CITIES = _mod_hibwsw.DECODO_COMMON_CITIES;
-exports.getRandomDecodoDeviceType = _mod_hibwsw.getRandomDeviceType;
-exports.getRandomDecodoLocale = _mod_hibwsw.getRandomLocale;
-exports.getRandomDecodoCountry = _mod_hibwsw.getRandomCountry;
-exports.getRandomDecodoCity = _mod_hibwsw.getRandomCity;
-exports.generateDecodoSessionId = _mod_hibwsw.generateSessionId;;
+const _mod_yqtvmc = require('./crawler.cjs');
+exports.Crawler = _mod_yqtvmc.Crawler;;
+const _mod_iokp0e = require('./crawler-options.cjs');
+exports.CrawlerOptions = _mod_iokp0e.CrawlerOptions;;
+const _mod_eu2nlx = require('./plugin/robots-txt.cjs');
+exports.RobotsTxt = _mod_eu2nlx.RobotsTxt;;
+const _mod_pdez3o = require('./plugin/file-cacher.cjs');
+exports.FileCacher = _mod_pdez3o.FileCacher;;
+const _mod_q7ukwr = require('./plugin/url-store.cjs');
+exports.UrlStore = _mod_q7ukwr.UrlStore;;
+const _mod_bzuoem = require('./plugin/navigation-history.cjs');
+exports.NavigationHistory = _mod_bzuoem.NavigationHistory;;
+const _mod_ayoad0 = require('./addon/oxylabs/index.cjs');
+exports.Oxylabs = _mod_ayoad0.Oxylabs;;
+const _mod_6bfl5x = require('./addon/oxylabs/options.cjs');
+exports.OXYLABS_BROWSER_TYPES = _mod_6bfl5x.OXYLABS_BROWSER_TYPES;
+exports.OXYLABS_COMMON_LOCALES = _mod_6bfl5x.OXYLABS_COMMON_LOCALES;
+exports.OXYLABS_COMMON_GEO_LOCATIONS = _mod_6bfl5x.OXYLABS_COMMON_GEO_LOCATIONS;
+exports.OXYLABS_US_STATES = _mod_6bfl5x.OXYLABS_US_STATES;
+exports.OXYLABS_EUROPEAN_COUNTRIES = _mod_6bfl5x.OXYLABS_EUROPEAN_COUNTRIES;
+exports.OXYLABS_ASIAN_COUNTRIES = _mod_6bfl5x.OXYLABS_ASIAN_COUNTRIES;
+exports.getRandomOxylabsBrowserType = _mod_6bfl5x.getRandomBrowserType;
+exports.getRandomOxylabsLocale = _mod_6bfl5x.getRandomLocale;
+exports.getRandomOxylabsGeoLocation = _mod_6bfl5x.getRandomGeoLocation;;
+const _mod_2z2x6m = require('./scraper.cjs');
+exports.isRestrictedDomain = _mod_2z2x6m.isRestrictedDomain;;
+const _mod_1mya5t = require('./addon/decodo/index.cjs');
+exports.Decodo = _mod_1mya5t.Decodo;;
+const _mod_ila1ru = require('./addon/decodo/options.cjs');
+exports.DECODO_DEVICE_TYPES = _mod_ila1ru.DECODO_DEVICE_TYPES;
+exports.DECODO_HEADLESS_MODES = _mod_ila1ru.DECODO_HEADLESS_MODES;
+exports.DECODO_COMMON_LOCALES = _mod_ila1ru.DECODO_COMMON_LOCALES;
+exports.DECODO_COMMON_COUNTRIES = _mod_ila1ru.DECODO_COMMON_COUNTRIES;
+exports.DECODO_EUROPEAN_COUNTRIES = _mod_ila1ru.DECODO_EUROPEAN_COUNTRIES;
+exports.DECODO_ASIAN_COUNTRIES = _mod_ila1ru.DECODO_ASIAN_COUNTRIES;
+exports.DECODO_US_STATES = _mod_ila1ru.DECODO_US_STATES;
+exports.DECODO_COMMON_CITIES = _mod_ila1ru.DECODO_COMMON_CITIES;
+exports.getRandomDecodoDeviceType = _mod_ila1ru.getRandomDeviceType;
+exports.getRandomDecodoLocale = _mod_ila1ru.getRandomLocale;
+exports.getRandomDecodoCountry = _mod_ila1ru.getRandomCountry;
+exports.getRandomDecodoCity = _mod_ila1ru.getRandomCity;
+exports.generateDecodoSessionId = _mod_ila1ru.generateSessionId;;

package/dist/crawler/index.js

@@ -16,6 +16,7 @@ export {
   getRandomLocale as getRandomOxylabsLocale,
   getRandomGeoLocation as getRandomOxylabsGeoLocation
 } from './addon/oxylabs/options.js';
+export { isRestrictedDomain } from './scraper.js';
 export { Decodo } from './addon/decodo/index.js';
 export {
   DECODO_DEVICE_TYPES,

package/dist/crawler/plugin/index.cjs

@@ -1 +1 @@
-var e=require("./file-cacher.cjs");exports.FileCacher=e.FileCacher;var r=require("./url-store.cjs");exports.UrlStore=r.UrlStore;var o=require("./result-stream.cjs");exports.ResultStream=o.ResultStream;var t=require("./memory-monitor.cjs");exports.MemoryMonitor=t.MemoryMonitor;var a=require("./health-metrics.cjs");exports.HealthMetrics=a.HealthMetrics;var i=require("./capped-map.cjs");exports.CappedMap=i.CappedMap;var s=require("./capped-array.cjs");exports.CappedArray=s.CappedArray;
+var e=require("./file-cacher.cjs");exports.FileCacher=e.FileCacher;var r=require("./url-store.cjs");exports.UrlStore=r.UrlStore;var o=require("./result-stream.cjs");exports.ResultStream=o.ResultStream;var t=require("./memory-monitor.cjs");exports.MemoryMonitor=t.MemoryMonitor;var a=require("./health-metrics.cjs");exports.HealthMetrics=a.HealthMetrics;var d=require("./capped-map.cjs");exports.CappedMap=d.CappedMap;var i=require("./capped-array.cjs");exports.CappedArray=i.CappedArray;

package/dist/crawler/scraper.cjs

@@ -1 +1 @@
-var{parseHTML:k}=require("linkedom");class w extends Set{maxSize;evictionCount;constructor(e,a=0.1){super();this.maxSize=e,this.evictionCount=Math.max(1,Math.ceil(e*a))}add(e){if(this.has(e))return this;if(this.size>=this.maxSize){let a=this.values();for(let n=0;n<this.evictionCount;n++){let t=a.next();if(t.done)break;this.delete(t.value)}}return super.add(e)}}class b{http;httpOptions;onEmailLeads;onEmailDiscovered;debug;static CLEAN_CONTENT_REGEX=/[^\w@.-\s]/g;static EMAIL_REGEX=/\b[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,}\b/g;static EMAIL_VALIDATION_REGEX=/^[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,}$/;discoveredEmails=new w(1e4);userAgents=[];fileExtensions=[];restrictedDomains=C();forbiddenProtocols=["mailto:","tel:","javascript:","data:","sms:","ftp:","file:","irc:","blob:","chrome:","about:","intent:"];constructor(e,a,n,t,o=!1){this.http=e,this.httpOptions=a,this.onEmailLeads=n,this.onEmailDiscovered=t,this.debug=o,this.userAgents=$()}sleep(e){return new Promise((a)=>setTimeout(a,e))}async executeHttp(e,a,n,t,o,i=0){let{getCache:s,saveCache:r,hasUrlInCache:m,saveUrl:u,httpConfig:l={}}=t;if(!e||e.length<3||this.forbiddenProtocols.some((f)=>e.startsWith(f)))return;try{let f=o?!1:await m(e),h=await s(e);if(f&&!h)return!1;if(f&&a!=="GET")return!1;let c=h&&a==="GET"?h:await(a==="GET"?this.http.get(e,l):a==="PATCH"?this.http.patch(e,n,l):a==="POST"?this.http.post(e,n,l):this.http.put(e,n,l));if(!h)await r(e,{data:c.data,contentType:c.contentType,finalUrl:c.finalUrl});if(!f)await u(e);if(!c.contentType||!c.contentType.includes("/html")||!c.contentType.includes("text/")||typeof c.data!=="string")return null;return{data:c.data,contentType:c.contentType,finalUrl:c.finalUrl}}catch(f){let h=f,c=this.httpOptions;if(h&&h.response){let g=h.response.status,p=c.retryDelay||100,x=c.maxRetryAttempts||3,d=c.retryWithoutProxyOnStatusCode||void 0,v=c.maxRetryOnProxyError||3;if(d&&l.proxy&&d.includes(g)&&i<x)return await this.sleep(p),delete l.proxy,await this.executeHttp(e,a,n,t,o,i+1);else if(c.retryOnStatusCode&&l.proxy&&c.retryOnStatusCode.includes(g)&&i<x)return await this.sleep(p),await this.executeHttp(e,a,n,t,o,i+1);else if(c.retryOnProxyError&&l.proxy&&i<v)return await this.sleep(p),await this.executeHttp(e,a,n,t,o,i+1)}if(this.debug){if(this.debug)console.log(`Error: unable to ${a} ${e}: ${f.message}`)}return null}}extractEmails(e,a,n,t,o,i){let s=this.extractEmailsFromContent(e?.replaceAll("mailto:","  ")),r=[];for(let m of s)if(this.handleEmailDiscovery(m,a,n,o,i))r.push(m);if(t&&t.length>0&&r.length>0)o.add(async()=>Promise.all(t.map((m)=>m(r))));s.length=0,r.length=0}async parseExternalWebsite(e,a,n,t,o,i=!0,s,r){let m=t.httpConfig?.headers?t.httpConfig.headers instanceof Headers?Object.fromEntries(t.httpConfig.headers.entries()):t.httpConfig.headers:{};t.httpConfig=t.httpConfig||{},t.httpConfig.headers={"user-agent":this.getRandomUserAgent(),accept:"text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8","accept-language":"en-US,en;q=0.9","cache-control":"no-cache",pragma:"no-cache",...m},t.httpConfig.timeout=t.httpConfig.timeout||15000,t.depth=t.depth||0,t.allowCrossDomainTravel=t.allowCrossDomainTravel||!1,o=o&&i;let u=[];try{let l=new URL(e),f=this.extractRootDomain(e);if(this.isLinktreeUrl(e))return await this.parseLinktreeProfile(e,t,o);if(this.isRestrictedDomain(e)){if(this.debug)console.warn(`⚠️ Skipped URL (restricted url): ${e}`);return u}let h=await this.executeHttp(e,a,n,t,o);if(!h){if(this.debug&&h===null)console.warn(`⚠️ Failed to fetch page content: ${e}`);if(this.debug&&h===!1)console.warn(`⚠️ Skipped URL (already visited): ${e}`);return u}let c=this.extractEmailsFromContent(h.data?.replaceAll("mailto:","  "));for(let g of c)if(this.handleEmailDiscovery(g,e,t.onEmailDiscovered,t.queue,t.emailMetadata))u.push(g);if(t.depth>0||!s){let g;{let{document:d}=k(h.data);g=this.extractRelevantLinks(d,l,f,t.depth,t.allowCrossDomainTravel)}t.depth--;let p=10,x=[];for(let d=0;d<g.length;d+=p){let v=g.slice(d,d+p),y=await Promise.allSettled(v.map((E)=>this.parseExternalWebsite(E,"GET",null,{...t,depth:t.depth},o,!1,!0)));x.push(...y)}for(let d of x)if(d.status==="fulfilled")u.push(...d.value);else if(this.debug)console.warn("⚠️ Failed to parse child URL:",d.reason?.message)}}catch(l){if(this.debug)console.error(`❌ Error parsing external website: ${e}`,l?.message)}if(i){if(t.onEmails&&t.onEmails.length>0)t.queue.add(async()=>Promise.all(t.onEmails.map((l)=>l(u))))}return u}async parseLinktreeProfile(e,a,n){let t=[];try{let o=await this.executeHttp(e,"GET",null,a,n);if(!o){if(this.debug)console.warn(`⚠️ Failed to fetch Linktree profile: ${e}`);return t}let i;{let{document:r}=k(o),m=r.getElementById("links-container");if(!m){if(this.debug)console.warn(`\uD83D\uDD0D No links container found in Linktree profile: ${e}`);return t}i=this.extractLinktreeExternalUrls(m,e)}if(i.length===0){if(this.debug)console.info("\uD83D\uDCED No valid external links found in Linktree profile");return t}if(this.debug)console.info(`\uD83C\uDFAF Found ${i.length} external links in Linktree profile`);let s=await Promise.allSettled(i.map((r)=>this.parseExternalWebsite(r,"GET",null,a,n,!1)));for(let r of s)if(r.status==="fulfilled")t.push(...r.value);else if(this.debug)console.warn("⚠️ Failed to parse Linktree external URL:",r.reason?.message)}catch(o){if(this.debug)console.error(`❌ Error parsing Linktree profile: ${e}`,o?.message)}return t}extractLinktreeExternalUrls(e,a){let n=new Set,t=e.querySelectorAll("a[href][target='_blank']");for(let o of t){let i=o.getAttribute("href");if(!i||i.length<3||this.forbiddenProtocols.some((s)=>i.startsWith(s)))continue;try{let s=new URL(i,a).href,r=this.extractRootDomain(s);if(r!=="linktr.ee"&&!this.isRestrictedDomain(s)&&r.length>3)n.add(s)}catch(s){if(this.debug)console.warn(`\uD83D\uDD17 Invalid URL in Linktree: ${i}`)}}return Array.from(n)}handleEmailDiscovery(e,a,n,t,o){if(!this.discoveredEmails.has(e)){this.discoveredEmails.add(e);let i={email:e,discoveredAt:a,timestamp:new Date,metadata:o||{}};if(n&&n.length>0)t.add(async()=>Promise.all(n.map((s)=>s(i))));if(this.debug)console.info(`\uD83D\uDCE7 New email discovered: ${e} at ${a}`);return!0}return!1}isDomainAccessAllowed(e,a,n,t){if(t)return!0;if(n===0)return e===a;return e===a||e.endsWith(`.${a}`)||a.endsWith(`.${e}`)}extractRelevantLinks(e,a,n,t,o){let i=[],s=["about","contact","help","support","reach","email","mail","message","company","team","staff","info","inquiry","feedback","service","assistance","connect","touch"],r=e.querySelectorAll("a[href]");for(let m of r){let u=m.getAttribute("href");if(!u||u.length<2)continue;try{let l=this.normalizeUrl(u,a),f=this.extractRootDomain(l);if(!this.isDomainAccessAllowed(f,n,t,o))continue;if(s.some((c)=>l.toLowerCase().includes(c))||this.isLinktreeUrl(l))i.push(l)}catch(l){if(this.debug)console.warn(`\uD83D\uDD17 Invalid link found: ${u}`,l?.message)}}return i}extractEmailsFromContent(e){let a=e.replace(b.CLEAN_CONTENT_REGEX," "),n=(s)=>{let r=s.split("@")[1]?.toLowerCase(),m=s.split(".").pop()?.toLowerCase();return b.EMAIL_VALIDATION_REGEX.test(s)&&r!==void 0&&m!==void 0&&!this.fileExtensions.includes(`.${m}`)&&!this.isRestrictedDomain(`https://${r}`)},t=(s)=>{return b.EMAIL_REGEX.lastIndex=0,(s.match(b.EMAIL_REGEX)||[]).filter(n)},o=e.replace(/<[^>]*>/g," "),i=[...t(o),...t(a)];return[...new Set(i)]}isRestrictedDomain(e){try{let a=new URL(e).host.toLowerCase();return this.restrictedDomains.some((n)=>a===n.toLowerCase()||a.endsWith(`.${n.toLowerCase()}`))}catch{return!0}}isLinktreeUrl(e){try{return this.extractRootDomain(e)==="linktr.ee"}catch{return!1}}extractRootDomain(e){try{let n=new URL(e).hostname.toLowerCase();return n.startsWith("www.")?n.slice(4):n}catch{return""}}normalizeUrl(e,a){if(e.startsWith("http://")||e.startsWith("https://"))return e;if(e.startsWith("//"))return`${a.protocol}${e}`;return new URL(e,a.href).href}getRandomUserAgent(){return this.userAgents[Math.floor(Math.random()*this.userAgents.length)]}}function $(){let e=[{name:"Chrome",version:"91.0.4472.124",engine:"AppleWebKit/537.36"},{name:"Firefox",version:"89.0",engine:"Gecko/20100101"},{name:"Safari",version:"14.1.1",engine:"AppleWebKit/605.1.15"},{name:"Edge",version:"91.0.864.59",engine:"AppleWebKit/537.36"},{name:"Opera",version:"77.0.4054.277",engine:"AppleWebKit/537.36"},{name:"Vivaldi",version:"3.8.2259.42",engine:"AppleWebKit/537.36"},{name:"Brave",version:"1.26.74",engine:"AppleWebKit/537.36"},{name:"Chromium",version:"91.0.4472.101",engine:"AppleWebKit/537.36"},{name:"Yandex",version:"21.5.3.742",engine:"AppleWebKit/537.36"},{name:"Maxthon",version:"5.3.8.2000",engine:"AppleWebKit/537.36"}],a=["Windows NT 10.0","Windows NT 6.1","Macintosh; Intel Mac OS X 10_15_7","Macintosh; Intel Mac OS X 11_4_0","X11; Linux x86_64","X11; Ubuntu; Linux x86_64"],n=[];for(let t=0;t<200;t++){let o=e[Math.floor(Math.random()*e.length)],i=a[Math.floor(Math.random()*a.length)],s="";switch(o.name){case"Chrome":s=`Mozilla/5.0 (${i}) ${o.engine} (KHTML, like Gecko) Chrome/${o.version} Safari/537.36`;break;case"Firefox":s=`Mozilla/5.0 (${i}; rv:${o.version}) ${o.engine} Firefox/${o.version}`;break;case"Safari":s=`Mozilla/5.0 (${i}) ${o.engine} (KHTML, like Gecko) Version/${o.version} Safari/605.1.15`;break;case"Edge":s=`Mozilla/5.0 (${i}) ${o.engine} (KHTML, like Gecko) Chrome/${o.version} Safari/537.36 Edg/${o.version}`;break;case"Opera":s=`Mozilla/5.0 (${i}) ${o.engine} (KHTML, like Gecko) Chrome/${o.version} Safari/537.36 OPR/${o.version}`;break;case"Vivaldi":s=`Mozilla/5.0 (${i}) ${o.engine} (KHTML, like Gecko) Chrome/${o.version} Safari/537.36 Vivaldi/${o.version}`;break;case"Brave":s=`Mozilla/5.0 (${i}) ${o.engine} (KHTML, like Gecko) Chrome/${o.version} Safari/537.36 Brave/${o.version}`;break;case"Chromium":s=`Mozilla/5.0 (${i}) ${o.engine} (KHTML, like Gecko) Chromium/${o.version} Chrome/${o.version} Safari/537.36`;break;case"Yandex":s=`Mozilla/5.0 (${i}) ${o.engine} (KHTML, like Gecko) Chrome/${o.version} YaBrowser/${o.version} Safari/537.36`;break;case"Maxthon":s=`Mozilla/5.0 (${i}) ${o.engine} (KHTML, like Gecko) Chrome/${o.version} Safari/537.36 Maxthon/${o.version}`;break}n.push(s)}return n}function C(){return["facebook.com","fb.com","messenger.com","instagram.com","threads.net","twitter.com","x.com","linkedin.com","pinterest.com","pin.it","reddit.com","tumblr.com","snapchat.com","tiktok.com","douyin.com","youtube.com","youtu.be","whatsapp.com","telegram.org","t.me","medium.com","quora.com","flickr.com","vimeo.com","vk.com","weibo.com","sina.com.cn","line.me","discord.com","discordapp.com","twitch.tv","meetup.com","nextdoor.com","xing.com","yelp.com","zalo.me","mastodon.social","clubhouse.com","patreon.com","onlyfans.com","douban.com","goodreads.com","soundcloud.com","spotify.com","last.fm","behance.net","dribbble.com","deviantart.com","pixiv.net","slideshare.net","tinder.com","bumble.com","etsy.com","indeed.com","glassdoor.com","monster.com","careerbuilder.com","dice.com","ziprecruiter.com","simplyhired.com","upwork.com","freelancer.com","fiverr.com","stackoverflow.com","stackoverflow.co","angel.co","wellfound.com","quora.com","stackexchange.com","yahoo.com","answers.microsoft.com","askubuntu.com","superuser.com","serverfault.com","mathoverflow.net","xda-developers.com","gamespot.com","ign.com","4chan.org","9gag.com","gizmodo.com","slashdot.org","hacker-news.news","ycombinator.com","producthunt.com","discourse.org","google.com","google.co.uk","google.de","google.fr","google.co.jp","bing.com","yahoo.com","search.yahoo.com","duckduckgo.com","baidu.com","yandex.com","yandex.ru","ask.com","wolframalpha.com","ecosia.org","startpage.com","qwant.com","searx.me","gibiru.com","swisscows.com","gmail.com","googlemail.com","outlook.com","hotmail.com","live.com","msn.com","yahoo.com","ymail.com","aol.com","icloud.com","me.com","mac.com","protonmail.com","pm.me","zoho.com","mail.com","gmx.com","gmx.net","yandex.com","yandex.ru","tutanota.com","tutanota.de","fastmail.com","hushmail.com","mailbox.org","posteo.de","runbox.com","disroot.org","163.com","qq.com","rambler.ru","mail.ru","yelp.com","yelp.ca","yelp.co.uk","yelp.com.au","yellowpages.com","yellowpages.ca","yell.com","tripadvisor.com","tripadvisor.co.uk","tripadvisor.ca","foursquare.com","angieslist.com","bbb.org","manta.com","thumbtack.com","homeadvisor.com","superpages.com","whitepages.com","local.com","citysearch.com","merchantcircle.com","insiderpages.com","kudzu.com","hotfrog.com","buildzoom.com","houzz.com","porch.com","mapquest.com","zagat.com","zomato.com","opentable.com","viator.com","expedia.com","booking.com","airbnb.com","vrbo.com","homeaway.com","craigslist.org","nextdoor.com","patch.com","meetup.com","eventbrite.com","groupon.com","livingsocial.com","gumtree.com","gumtree.com.au","kijiji.ca","leboncoin.fr","finn.no","blocket.se","58.com","dianping.com","tabelog.com","ypcdn.com"]}exports.Scraper=b;exports.CappedSet=w;
+var{parseHTML:k}=require("linkedom");class v extends Set{maxSize;evictionCount;constructor(e,n=0.1){super();this.maxSize=e,this.evictionCount=Math.max(1,Math.ceil(e*n))}add(e){if(this.has(e))return this;if(this.size>=this.maxSize){let n=this.values();for(let a=0;a<this.evictionCount;a++){let t=n.next();if(t.done)break;this.delete(t.value)}}return super.add(e)}}class b{http;httpOptions;onEmailLeads;onEmailDiscovered;debug;static CLEAN_CONTENT_REGEX=/[^\w@.-\s]/g;static EMAIL_REGEX=/\b[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,}\b/g;static EMAIL_VALIDATION_REGEX=/^[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,}$/;discoveredEmails=new v(1e4);userAgents=[];fileExtensions=[];restrictedDomains=y();forbiddenProtocols=["mailto:","tel:","javascript:","data:","sms:","ftp:","file:","irc:","blob:","chrome:","about:","intent:"];constructor(e,n,a,t,o=!1){this.http=e,this.httpOptions=n,this.onEmailLeads=a,this.onEmailDiscovered=t,this.debug=o,this.userAgents=C()}sleep(e){return new Promise((n)=>setTimeout(n,e))}async executeHttp(e,n,a,t,o,i=0){let{getCache:s,saveCache:r,hasUrlInCache:m,saveUrl:u,httpConfig:l={}}=t;if(!e||e.length<3||this.forbiddenProtocols.some((f)=>e.startsWith(f)))return;try{let f=o?!1:await m(e),h=await s(e);if(f&&!h)return!1;if(f&&n!=="GET")return!1;let c=h&&n==="GET"?h:await(n==="GET"?this.http.get(e,l):n==="PATCH"?this.http.patch(e,a,l):n==="POST"?this.http.post(e,a,l):this.http.put(e,a,l));if(!h)await r(e,{data:c.data,contentType:c.contentType,finalUrl:c.finalUrl});if(!f)await u(e);if(!c.contentType||!c.contentType.includes("/html")||!c.contentType.includes("text/")||typeof c.data!=="string")return null;return{data:c.data,contentType:c.contentType,finalUrl:c.finalUrl}}catch(f){let h=f,c=this.httpOptions;if(h&&h.response){let g=h.response.status,p=c.retryDelay||100,x=c.maxRetryAttempts||3,d=c.retryWithoutProxyOnStatusCode||void 0,w=c.maxRetryOnProxyError||3;if(d&&l.proxy&&d.includes(g)&&i<x)return await this.sleep(p),delete l.proxy,await this.executeHttp(e,n,a,t,o,i+1);else if(c.retryOnStatusCode&&l.proxy&&c.retryOnStatusCode.includes(g)&&i<x)return await this.sleep(p),await this.executeHttp(e,n,a,t,o,i+1);else if(c.retryOnProxyError&&l.proxy&&i<w)return await this.sleep(p),await this.executeHttp(e,n,a,t,o,i+1)}if(this.debug){if(this.debug)console.log(`Error: unable to ${n} ${e}: ${f.message}`)}return null}}extractEmails(e,n,a,t,o,i){let s=this.extractEmailsFromContent(e?.replaceAll("mailto:","  ")),r=[];for(let m of s)if(this.handleEmailDiscovery(m,n,a,o,i))r.push(m);if(t&&t.length>0&&r.length>0)o.add(async()=>Promise.all(t.map((m)=>m(r))));s.length=0,r.length=0}async parseExternalWebsite(e,n,a,t,o,i=!0,s,r){let m=t.httpConfig?.headers?t.httpConfig.headers instanceof Headers?Object.fromEntries(t.httpConfig.headers.entries()):t.httpConfig.headers:{};t.httpConfig=t.httpConfig||{},t.httpConfig.headers={"user-agent":this.getRandomUserAgent(),accept:"text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8","accept-language":"en-US,en;q=0.9","cache-control":"no-cache",pragma:"no-cache",...m},t.httpConfig.timeout=t.httpConfig.timeout||15000,t.depth=t.depth||0,t.allowCrossDomainTravel=t.allowCrossDomainTravel||!1,o=o&&i;let u=[];try{let l=new URL(e),f=this.extractRootDomain(e);if(this.isLinktreeUrl(e))return await this.parseLinktreeProfile(e,t,o);if(this.isRestrictedDomain(e)){if(this.debug)console.warn(`⚠️ Skipped URL (restricted url): ${e}`);return u}let h=await this.executeHttp(e,n,a,t,o);if(!h){if(this.debug&&h===null)console.warn(`⚠️ Failed to fetch page content: ${e}`);if(this.debug&&h===!1)console.warn(`⚠️ Skipped URL (already visited): ${e}`);return u}let c=this.extractEmailsFromContent(h.data?.replaceAll("mailto:","  "));for(let g of c)if(this.handleEmailDiscovery(g,e,t.onEmailDiscovered,t.queue,t.emailMetadata))u.push(g);if(t.depth>0||!s){let g;{let{document:d}=k(h.data);g=this.extractRelevantLinks(d,l,f,t.depth,t.allowCrossDomainTravel)}t.depth--;let p=10,x=[];for(let d=0;d<g.length;d+=p){let w=g.slice(d,d+p),E=await Promise.allSettled(w.map((L)=>this.parseExternalWebsite(L,"GET",null,{...t,depth:t.depth},o,!1,!0)));x.push(...E)}for(let d of x)if(d.status==="fulfilled")u.push(...d.value);else if(this.debug)console.warn("⚠️ Failed to parse child URL:",d.reason?.message)}}catch(l){if(this.debug)console.error(`❌ Error parsing external website: ${e}`,l?.message)}if(i){if(t.onEmails&&t.onEmails.length>0)t.queue.add(async()=>Promise.all(t.onEmails.map((l)=>l(u))))}return u}async parseLinktreeProfile(e,n,a){let t=[];try{let o=await this.executeHttp(e,"GET",null,n,a);if(!o){if(this.debug)console.warn(`⚠️ Failed to fetch Linktree profile: ${e}`);return t}let i;{let{document:r}=k(o),m=r.getElementById("links-container");if(!m){if(this.debug)console.warn(`\uD83D\uDD0D No links container found in Linktree profile: ${e}`);return t}i=this.extractLinktreeExternalUrls(m,e)}if(i.length===0){if(this.debug)console.info("\uD83D\uDCED No valid external links found in Linktree profile");return t}if(this.debug)console.info(`\uD83C\uDFAF Found ${i.length} external links in Linktree profile`);let s=await Promise.allSettled(i.map((r)=>this.parseExternalWebsite(r,"GET",null,n,a,!1)));for(let r of s)if(r.status==="fulfilled")t.push(...r.value);else if(this.debug)console.warn("⚠️ Failed to parse Linktree external URL:",r.reason?.message)}catch(o){if(this.debug)console.error(`❌ Error parsing Linktree profile: ${e}`,o?.message)}return t}extractLinktreeExternalUrls(e,n){let a=new Set,t=e.querySelectorAll("a[href][target='_blank']");for(let o of t){let i=o.getAttribute("href");if(!i||i.length<3||this.forbiddenProtocols.some((s)=>i.startsWith(s)))continue;try{let s=new URL(i,n).href,r=this.extractRootDomain(s);if(r!=="linktr.ee"&&!this.isRestrictedDomain(s)&&r.length>3)a.add(s)}catch(s){if(this.debug)console.warn(`\uD83D\uDD17 Invalid URL in Linktree: ${i}`)}}return Array.from(a)}handleEmailDiscovery(e,n,a,t,o){if(!this.discoveredEmails.has(e)){this.discoveredEmails.add(e);let i={email:e,discoveredAt:n,timestamp:new Date,metadata:o||{}};if(a&&a.length>0)t.add(async()=>Promise.all(a.map((s)=>s(i))));if(this.debug)console.info(`\uD83D\uDCE7 New email discovered: ${e} at ${n}`);return!0}return!1}isDomainAccessAllowed(e,n,a,t){if(t)return!0;if(a===0)return e===n;return e===n||e.endsWith(`.${n}`)||n.endsWith(`.${e}`)}extractRelevantLinks(e,n,a,t,o){let i=[],s=["about","contact","help","support","reach","email","mail","message","company","team","staff","info","inquiry","feedback","service","assistance","connect","touch"],r=e.querySelectorAll("a[href]");for(let m of r){let u=m.getAttribute("href");if(!u||u.length<2)continue;try{let l=this.normalizeUrl(u,n),f=this.extractRootDomain(l);if(!this.isDomainAccessAllowed(f,a,t,o))continue;if(s.some((c)=>l.toLowerCase().includes(c))||this.isLinktreeUrl(l))i.push(l)}catch(l){if(this.debug)console.warn(`\uD83D\uDD17 Invalid link found: ${u}`,l?.message)}}return i}extractEmailsFromContent(e){let n=e.replace(b.CLEAN_CONTENT_REGEX," "),a=(s)=>{let r=s.split("@")[1]?.toLowerCase(),m=s.split(".").pop()?.toLowerCase();return b.EMAIL_VALIDATION_REGEX.test(s)&&r!==void 0&&m!==void 0&&!this.fileExtensions.includes(`.${m}`)&&!this.isRestrictedDomain(`https://${r}`)},t=(s)=>{return b.EMAIL_REGEX.lastIndex=0,(s.match(b.EMAIL_REGEX)||[]).filter(a)},o=e.replace(/<[^>]*>/g," "),i=[...t(o),...t(n)];return[...new Set(i)]}isRestrictedDomain(e){try{let n=new URL(e).host.toLowerCase();return this.restrictedDomains.some((a)=>n===a.toLowerCase()||n.endsWith(`.${a.toLowerCase()}`))}catch{return!0}}isLinktreeUrl(e){try{return this.extractRootDomain(e)==="linktr.ee"}catch{return!1}}extractRootDomain(e){try{let a=new URL(e).hostname.toLowerCase();return a.startsWith("www.")?a.slice(4):a}catch{return""}}normalizeUrl(e,n){if(e.startsWith("http://")||e.startsWith("https://"))return e;if(e.startsWith("//"))return`${n.protocol}${e}`;return new URL(e,n.href).href}getRandomUserAgent(){return this.userAgents[Math.floor(Math.random()*this.userAgents.length)]}}function C(){let e=[{name:"Chrome",version:"91.0.4472.124",engine:"AppleWebKit/537.36"},{name:"Firefox",version:"89.0",engine:"Gecko/20100101"},{name:"Safari",version:"14.1.1",engine:"AppleWebKit/605.1.15"},{name:"Edge",version:"91.0.864.59",engine:"AppleWebKit/537.36"},{name:"Opera",version:"77.0.4054.277",engine:"AppleWebKit/537.36"},{name:"Vivaldi",version:"3.8.2259.42",engine:"AppleWebKit/537.36"},{name:"Brave",version:"1.26.74",engine:"AppleWebKit/537.36"},{name:"Chromium",version:"91.0.4472.101",engine:"AppleWebKit/537.36"},{name:"Yandex",version:"21.5.3.742",engine:"AppleWebKit/537.36"},{name:"Maxthon",version:"5.3.8.2000",engine:"AppleWebKit/537.36"}],n=["Windows NT 10.0","Windows NT 6.1","Macintosh; Intel Mac OS X 10_15_7","Macintosh; Intel Mac OS X 11_4_0","X11; Linux x86_64","X11; Ubuntu; Linux x86_64"],a=[];for(let t=0;t<200;t++){let o=e[Math.floor(Math.random()*e.length)],i=n[Math.floor(Math.random()*n.length)],s="";switch(o.name){case"Chrome":s=`Mozilla/5.0 (${i}) ${o.engine} (KHTML, like Gecko) Chrome/${o.version} Safari/537.36`;break;case"Firefox":s=`Mozilla/5.0 (${i}; rv:${o.version}) ${o.engine} Firefox/${o.version}`;break;case"Safari":s=`Mozilla/5.0 (${i}) ${o.engine} (KHTML, like Gecko) Version/${o.version} Safari/605.1.15`;break;case"Edge":s=`Mozilla/5.0 (${i}) ${o.engine} (KHTML, like Gecko) Chrome/${o.version} Safari/537.36 Edg/${o.version}`;break;case"Opera":s=`Mozilla/5.0 (${i}) ${o.engine} (KHTML, like Gecko) Chrome/${o.version} Safari/537.36 OPR/${o.version}`;break;case"Vivaldi":s=`Mozilla/5.0 (${i}) ${o.engine} (KHTML, like Gecko) Chrome/${o.version} Safari/537.36 Vivaldi/${o.version}`;break;case"Brave":s=`Mozilla/5.0 (${i}) ${o.engine} (KHTML, like Gecko) Chrome/${o.version} Safari/537.36 Brave/${o.version}`;break;case"Chromium":s=`Mozilla/5.0 (${i}) ${o.engine} (KHTML, like Gecko) Chromium/${o.version} Chrome/${o.version} Safari/537.36`;break;case"Yandex":s=`Mozilla/5.0 (${i}) ${o.engine} (KHTML, like Gecko) Chrome/${o.version} YaBrowser/${o.version} Safari/537.36`;break;case"Maxthon":s=`Mozilla/5.0 (${i}) ${o.engine} (KHTML, like Gecko) Chrome/${o.version} Safari/537.36 Maxthon/${o.version}`;break}a.push(s)}return a}function y(){return["facebook.com","fb.com","messenger.com","instagram.com","threads.net","twitter.com","x.com","linkedin.com","pinterest.com","pin.it","reddit.com","tumblr.com","snapchat.com","tiktok.com","douyin.com","youtube.com","youtu.be","whatsapp.com","telegram.org","t.me","medium.com","quora.com","flickr.com","vimeo.com","vk.com","weibo.com","sina.com.cn","line.me","discord.com","discordapp.com","twitch.tv","meetup.com","nextdoor.com","xing.com","yelp.com","zalo.me","mastodon.social","clubhouse.com","patreon.com","onlyfans.com","douban.com","goodreads.com","soundcloud.com","spotify.com","last.fm","behance.net","dribbble.com","deviantart.com","pixiv.net","slideshare.net","tinder.com","bumble.com","etsy.com","indeed.com","glassdoor.com","monster.com","careerbuilder.com","dice.com","ziprecruiter.com","simplyhired.com","upwork.com","freelancer.com","fiverr.com","stackoverflow.com","stackoverflow.co","angel.co","wellfound.com","quora.com","stackexchange.com","yahoo.com","answers.microsoft.com","askubuntu.com","superuser.com","serverfault.com","mathoverflow.net","xda-developers.com","gamespot.com","ign.com","4chan.org","9gag.com","gizmodo.com","slashdot.org","hacker-news.news","ycombinator.com","producthunt.com","discourse.org","google.com","google.co.uk","google.de","google.fr","google.co.jp","bing.com","yahoo.com","search.yahoo.com","duckduckgo.com","baidu.com","yandex.com","yandex.ru","ask.com","wolframalpha.com","ecosia.org","startpage.com","qwant.com","searx.me","gibiru.com","swisscows.com","gmail.com","googlemail.com","outlook.com","hotmail.com","live.com","msn.com","yahoo.com","ymail.com","aol.com","icloud.com","me.com","mac.com","protonmail.com","pm.me","zoho.com","mail.com","gmx.com","gmx.net","yandex.com","yandex.ru","tutanota.com","tutanota.de","fastmail.com","hushmail.com","mailbox.org","posteo.de","runbox.com","disroot.org","163.com","qq.com","rambler.ru","mail.ru","yelp.com","yelp.ca","yelp.co.uk","yelp.com.au","yellowpages.com","yellowpages.ca","yell.com","tripadvisor.com","tripadvisor.co.uk","tripadvisor.ca","foursquare.com","angieslist.com","bbb.org","manta.com","thumbtack.com","homeadvisor.com","superpages.com","whitepages.com","local.com","citysearch.com","merchantcircle.com","insiderpages.com","kudzu.com","hotfrog.com","buildzoom.com","houzz.com","porch.com","mapquest.com","zagat.com","zomato.com","opentable.com","viator.com","expedia.com","booking.com","airbnb.com","vrbo.com","homeaway.com","craigslist.org","nextdoor.com","patch.com","meetup.com","eventbrite.com","groupon.com","livingsocial.com","gumtree.com","gumtree.com.au","kijiji.ca","leboncoin.fr","finn.no","blocket.se","58.com","dianping.com","tabelog.com","ypcdn.com"]}function $(e){try{let n=new URL(e).host.toLowerCase();return y().some((a)=>n===a.toLowerCase()||n.endsWith(`.${a.toLowerCase()}`))}catch{return!0}}exports.Scraper=b;exports.CappedSet=v;exports.isRestrictedDomain=$;

package/dist/crawler/scraper.js

@@ -1 +1 @@
-import{parseHTML as w}from"linkedom";class k extends Set{maxSize;evictionCount;constructor(e,a=0.1){super();this.maxSize=e,this.evictionCount=Math.max(1,Math.ceil(e*a))}add(e){if(this.has(e))return this;if(this.size>=this.maxSize){let a=this.values();for(let n=0;n<this.evictionCount;n++){let t=a.next();if(t.done)break;this.delete(t.value)}}return super.add(e)}}class x{http;httpOptions;onEmailLeads;onEmailDiscovered;debug;static CLEAN_CONTENT_REGEX=/[^\w@.-\s]/g;static EMAIL_REGEX=/\b[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,}\b/g;static EMAIL_VALIDATION_REGEX=/^[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,}$/;discoveredEmails=new k(1e4);userAgents=[];fileExtensions=[];restrictedDomains=L();forbiddenProtocols=["mailto:","tel:","javascript:","data:","sms:","ftp:","file:","irc:","blob:","chrome:","about:","intent:"];constructor(e,a,n,t,o=!1){this.http=e,this.httpOptions=a,this.onEmailLeads=n,this.onEmailDiscovered=t,this.debug=o,this.userAgents=$()}sleep(e){return new Promise((a)=>setTimeout(a,e))}async executeHttp(e,a,n,t,o,i=0){let{getCache:s,saveCache:r,hasUrlInCache:m,saveUrl:u,httpConfig:l={}}=t;if(!e||e.length<3||this.forbiddenProtocols.some((f)=>e.startsWith(f)))return;try{let f=o?!1:await m(e),h=await s(e);if(f&&!h)return!1;if(f&&a!=="GET")return!1;let c=h&&a==="GET"?h:await(a==="GET"?this.http.get(e,l):a==="PATCH"?this.http.patch(e,n,l):a==="POST"?this.http.post(e,n,l):this.http.put(e,n,l));if(!h)await r(e,{data:c.data,contentType:c.contentType,finalUrl:c.finalUrl});if(!f)await u(e);if(!c.contentType||!c.contentType.includes("/html")||!c.contentType.includes("text/")||typeof c.data!=="string")return null;return{data:c.data,contentType:c.contentType,finalUrl:c.finalUrl}}catch(f){let h=f,c=this.httpOptions;if(h&&h.response){let g=h.response.status,p=c.retryDelay||100,b=c.maxRetryAttempts||3,d=c.retryWithoutProxyOnStatusCode||void 0,v=c.maxRetryOnProxyError||3;if(d&&l.proxy&&d.includes(g)&&i<b)return await this.sleep(p),delete l.proxy,await this.executeHttp(e,a,n,t,o,i+1);else if(c.retryOnStatusCode&&l.proxy&&c.retryOnStatusCode.includes(g)&&i<b)return await this.sleep(p),await this.executeHttp(e,a,n,t,o,i+1);else if(c.retryOnProxyError&&l.proxy&&i<v)return await this.sleep(p),await this.executeHttp(e,a,n,t,o,i+1)}if(this.debug){if(this.debug)console.log(`Error: unable to ${a} ${e}: ${f.message}`)}return null}}extractEmails(e,a,n,t,o,i){let s=this.extractEmailsFromContent(e?.replaceAll("mailto:","  ")),r=[];for(let m of s)if(this.handleEmailDiscovery(m,a,n,o,i))r.push(m);if(t&&t.length>0&&r.length>0)o.add(async()=>Promise.all(t.map((m)=>m(r))));s.length=0,r.length=0}async parseExternalWebsite(e,a,n,t,o,i=!0,s,r){let m=t.httpConfig?.headers?t.httpConfig.headers instanceof Headers?Object.fromEntries(t.httpConfig.headers.entries()):t.httpConfig.headers:{};t.httpConfig=t.httpConfig||{},t.httpConfig.headers={"user-agent":this.getRandomUserAgent(),accept:"text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8","accept-language":"en-US,en;q=0.9","cache-control":"no-cache",pragma:"no-cache",...m},t.httpConfig.timeout=t.httpConfig.timeout||15000,t.depth=t.depth||0,t.allowCrossDomainTravel=t.allowCrossDomainTravel||!1,o=o&&i;let u=[];try{let l=new URL(e),f=this.extractRootDomain(e);if(this.isLinktreeUrl(e))return await this.parseLinktreeProfile(e,t,o);if(this.isRestrictedDomain(e)){if(this.debug)console.warn(`⚠️ Skipped URL (restricted url): ${e}`);return u}let h=await this.executeHttp(e,a,n,t,o);if(!h){if(this.debug&&h===null)console.warn(`⚠️ Failed to fetch page content: ${e}`);if(this.debug&&h===!1)console.warn(`⚠️ Skipped URL (already visited): ${e}`);return u}let c=this.extractEmailsFromContent(h.data?.replaceAll("mailto:","  "));for(let g of c)if(this.handleEmailDiscovery(g,e,t.onEmailDiscovered,t.queue,t.emailMetadata))u.push(g);if(t.depth>0||!s){let g;{let{document:d}=w(h.data);g=this.extractRelevantLinks(d,l,f,t.depth,t.allowCrossDomainTravel)}t.depth--;let p=10,b=[];for(let d=0;d<g.length;d+=p){let v=g.slice(d,d+p),y=await Promise.allSettled(v.map((E)=>this.parseExternalWebsite(E,"GET",null,{...t,depth:t.depth},o,!1,!0)));b.push(...y)}for(let d of b)if(d.status==="fulfilled")u.push(...d.value);else if(this.debug)console.warn("⚠️ Failed to parse child URL:",d.reason?.message)}}catch(l){if(this.debug)console.error(`❌ Error parsing external website: ${e}`,l?.message)}if(i){if(t.onEmails&&t.onEmails.length>0)t.queue.add(async()=>Promise.all(t.onEmails.map((l)=>l(u))))}return u}async parseLinktreeProfile(e,a,n){let t=[];try{let o=await this.executeHttp(e,"GET",null,a,n);if(!o){if(this.debug)console.warn(`⚠️ Failed to fetch Linktree profile: ${e}`);return t}let i;{let{document:r}=w(o),m=r.getElementById("links-container");if(!m){if(this.debug)console.warn(`\uD83D\uDD0D No links container found in Linktree profile: ${e}`);return t}i=this.extractLinktreeExternalUrls(m,e)}if(i.length===0){if(this.debug)console.info("\uD83D\uDCED No valid external links found in Linktree profile");return t}if(this.debug)console.info(`\uD83C\uDFAF Found ${i.length} external links in Linktree profile`);let s=await Promise.allSettled(i.map((r)=>this.parseExternalWebsite(r,"GET",null,a,n,!1)));for(let r of s)if(r.status==="fulfilled")t.push(...r.value);else if(this.debug)console.warn("⚠️ Failed to parse Linktree external URL:",r.reason?.message)}catch(o){if(this.debug)console.error(`❌ Error parsing Linktree profile: ${e}`,o?.message)}return t}extractLinktreeExternalUrls(e,a){let n=new Set,t=e.querySelectorAll("a[href][target='_blank']");for(let o of t){let i=o.getAttribute("href");if(!i||i.length<3||this.forbiddenProtocols.some((s)=>i.startsWith(s)))continue;try{let s=new URL(i,a).href,r=this.extractRootDomain(s);if(r!=="linktr.ee"&&!this.isRestrictedDomain(s)&&r.length>3)n.add(s)}catch(s){if(this.debug)console.warn(`\uD83D\uDD17 Invalid URL in Linktree: ${i}`)}}return Array.from(n)}handleEmailDiscovery(e,a,n,t,o){if(!this.discoveredEmails.has(e)){this.discoveredEmails.add(e);let i={email:e,discoveredAt:a,timestamp:new Date,metadata:o||{}};if(n&&n.length>0)t.add(async()=>Promise.all(n.map((s)=>s(i))));if(this.debug)console.info(`\uD83D\uDCE7 New email discovered: ${e} at ${a}`);return!0}return!1}isDomainAccessAllowed(e,a,n,t){if(t)return!0;if(n===0)return e===a;return e===a||e.endsWith(`.${a}`)||a.endsWith(`.${e}`)}extractRelevantLinks(e,a,n,t,o){let i=[],s=["about","contact","help","support","reach","email","mail","message","company","team","staff","info","inquiry","feedback","service","assistance","connect","touch"],r=e.querySelectorAll("a[href]");for(let m of r){let u=m.getAttribute("href");if(!u||u.length<2)continue;try{let l=this.normalizeUrl(u,a),f=this.extractRootDomain(l);if(!this.isDomainAccessAllowed(f,n,t,o))continue;if(s.some((c)=>l.toLowerCase().includes(c))||this.isLinktreeUrl(l))i.push(l)}catch(l){if(this.debug)console.warn(`\uD83D\uDD17 Invalid link found: ${u}`,l?.message)}}return i}extractEmailsFromContent(e){let a=e.replace(x.CLEAN_CONTENT_REGEX," "),n=(s)=>{let r=s.split("@")[1]?.toLowerCase(),m=s.split(".").pop()?.toLowerCase();return x.EMAIL_VALIDATION_REGEX.test(s)&&r!==void 0&&m!==void 0&&!this.fileExtensions.includes(`.${m}`)&&!this.isRestrictedDomain(`https://${r}`)},t=(s)=>{return x.EMAIL_REGEX.lastIndex=0,(s.match(x.EMAIL_REGEX)||[]).filter(n)},o=e.replace(/<[^>]*>/g," "),i=[...t(o),...t(a)];return[...new Set(i)]}isRestrictedDomain(e){try{let a=new URL(e).host.toLowerCase();return this.restrictedDomains.some((n)=>a===n.toLowerCase()||a.endsWith(`.${n.toLowerCase()}`))}catch{return!0}}isLinktreeUrl(e){try{return this.extractRootDomain(e)==="linktr.ee"}catch{return!1}}extractRootDomain(e){try{let n=new URL(e).hostname.toLowerCase();return n.startsWith("www.")?n.slice(4):n}catch{return""}}normalizeUrl(e,a){if(e.startsWith("http://")||e.startsWith("https://"))return e;if(e.startsWith("//"))return`${a.protocol}${e}`;return new URL(e,a.href).href}getRandomUserAgent(){return this.userAgents[Math.floor(Math.random()*this.userAgents.length)]}}function $(){let e=[{name:"Chrome",version:"91.0.4472.124",engine:"AppleWebKit/537.36"},{name:"Firefox",version:"89.0",engine:"Gecko/20100101"},{name:"Safari",version:"14.1.1",engine:"AppleWebKit/605.1.15"},{name:"Edge",version:"91.0.864.59",engine:"AppleWebKit/537.36"},{name:"Opera",version:"77.0.4054.277",engine:"AppleWebKit/537.36"},{name:"Vivaldi",version:"3.8.2259.42",engine:"AppleWebKit/537.36"},{name:"Brave",version:"1.26.74",engine:"AppleWebKit/537.36"},{name:"Chromium",version:"91.0.4472.101",engine:"AppleWebKit/537.36"},{name:"Yandex",version:"21.5.3.742",engine:"AppleWebKit/537.36"},{name:"Maxthon",version:"5.3.8.2000",engine:"AppleWebKit/537.36"}],a=["Windows NT 10.0","Windows NT 6.1","Macintosh; Intel Mac OS X 10_15_7","Macintosh; Intel Mac OS X 11_4_0","X11; Linux x86_64","X11; Ubuntu; Linux x86_64"],n=[];for(let t=0;t<200;t++){let o=e[Math.floor(Math.random()*e.length)],i=a[Math.floor(Math.random()*a.length)],s="";switch(o.name){case"Chrome":s=`Mozilla/5.0 (${i}) ${o.engine} (KHTML, like Gecko) Chrome/${o.version} Safari/537.36`;break;case"Firefox":s=`Mozilla/5.0 (${i}; rv:${o.version}) ${o.engine} Firefox/${o.version}`;break;case"Safari":s=`Mozilla/5.0 (${i}) ${o.engine} (KHTML, like Gecko) Version/${o.version} Safari/605.1.15`;break;case"Edge":s=`Mozilla/5.0 (${i}) ${o.engine} (KHTML, like Gecko) Chrome/${o.version} Safari/537.36 Edg/${o.version}`;break;case"Opera":s=`Mozilla/5.0 (${i}) ${o.engine} (KHTML, like Gecko) Chrome/${o.version} Safari/537.36 OPR/${o.version}`;break;case"Vivaldi":s=`Mozilla/5.0 (${i}) ${o.engine} (KHTML, like Gecko) Chrome/${o.version} Safari/537.36 Vivaldi/${o.version}`;break;case"Brave":s=`Mozilla/5.0 (${i}) ${o.engine} (KHTML, like Gecko) Chrome/${o.version} Safari/537.36 Brave/${o.version}`;break;case"Chromium":s=`Mozilla/5.0 (${i}) ${o.engine} (KHTML, like Gecko) Chromium/${o.version} Chrome/${o.version} Safari/537.36`;break;case"Yandex":s=`Mozilla/5.0 (${i}) ${o.engine} (KHTML, like Gecko) Chrome/${o.version} YaBrowser/${o.version} Safari/537.36`;break;case"Maxthon":s=`Mozilla/5.0 (${i}) ${o.engine} (KHTML, like Gecko) Chrome/${o.version} Safari/537.36 Maxthon/${o.version}`;break}n.push(s)}return n}function L(){return["facebook.com","fb.com","messenger.com","instagram.com","threads.net","twitter.com","x.com","linkedin.com","pinterest.com","pin.it","reddit.com","tumblr.com","snapchat.com","tiktok.com","douyin.com","youtube.com","youtu.be","whatsapp.com","telegram.org","t.me","medium.com","quora.com","flickr.com","vimeo.com","vk.com","weibo.com","sina.com.cn","line.me","discord.com","discordapp.com","twitch.tv","meetup.com","nextdoor.com","xing.com","yelp.com","zalo.me","mastodon.social","clubhouse.com","patreon.com","onlyfans.com","douban.com","goodreads.com","soundcloud.com","spotify.com","last.fm","behance.net","dribbble.com","deviantart.com","pixiv.net","slideshare.net","tinder.com","bumble.com","etsy.com","indeed.com","glassdoor.com","monster.com","careerbuilder.com","dice.com","ziprecruiter.com","simplyhired.com","upwork.com","freelancer.com","fiverr.com","stackoverflow.com","stackoverflow.co","angel.co","wellfound.com","quora.com","stackexchange.com","yahoo.com","answers.microsoft.com","askubuntu.com","superuser.com","serverfault.com","mathoverflow.net","xda-developers.com","gamespot.com","ign.com","4chan.org","9gag.com","gizmodo.com","slashdot.org","hacker-news.news","ycombinator.com","producthunt.com","discourse.org","google.com","google.co.uk","google.de","google.fr","google.co.jp","bing.com","yahoo.com","search.yahoo.com","duckduckgo.com","baidu.com","yandex.com","yandex.ru","ask.com","wolframalpha.com","ecosia.org","startpage.com","qwant.com","searx.me","gibiru.com","swisscows.com","gmail.com","googlemail.com","outlook.com","hotmail.com","live.com","msn.com","yahoo.com","ymail.com","aol.com","icloud.com","me.com","mac.com","protonmail.com","pm.me","zoho.com","mail.com","gmx.com","gmx.net","yandex.com","yandex.ru","tutanota.com","tutanota.de","fastmail.com","hushmail.com","mailbox.org","posteo.de","runbox.com","disroot.org","163.com","qq.com","rambler.ru","mail.ru","yelp.com","yelp.ca","yelp.co.uk","yelp.com.au","yellowpages.com","yellowpages.ca","yell.com","tripadvisor.com","tripadvisor.co.uk","tripadvisor.ca","foursquare.com","angieslist.com","bbb.org","manta.com","thumbtack.com","homeadvisor.com","superpages.com","whitepages.com","local.com","citysearch.com","merchantcircle.com","insiderpages.com","kudzu.com","hotfrog.com","buildzoom.com","houzz.com","porch.com","mapquest.com","zagat.com","zomato.com","opentable.com","viator.com","expedia.com","booking.com","airbnb.com","vrbo.com","homeaway.com","craigslist.org","nextdoor.com","patch.com","meetup.com","eventbrite.com","groupon.com","livingsocial.com","gumtree.com","gumtree.com.au","kijiji.ca","leboncoin.fr","finn.no","blocket.se","58.com","dianping.com","tabelog.com","ypcdn.com"]}export{x as Scraper,k as CappedSet};
+import{parseHTML as v}from"linkedom";class k extends Set{maxSize;evictionCount;constructor(e,n=0.1){super();this.maxSize=e,this.evictionCount=Math.max(1,Math.ceil(e*n))}add(e){if(this.has(e))return this;if(this.size>=this.maxSize){let n=this.values();for(let a=0;a<this.evictionCount;a++){let t=n.next();if(t.done)break;this.delete(t.value)}}return super.add(e)}}class x{http;httpOptions;onEmailLeads;onEmailDiscovered;debug;static CLEAN_CONTENT_REGEX=/[^\w@.-\s]/g;static EMAIL_REGEX=/\b[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,}\b/g;static EMAIL_VALIDATION_REGEX=/^[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,}$/;discoveredEmails=new k(1e4);userAgents=[];fileExtensions=[];restrictedDomains=y();forbiddenProtocols=["mailto:","tel:","javascript:","data:","sms:","ftp:","file:","irc:","blob:","chrome:","about:","intent:"];constructor(e,n,a,t,o=!1){this.http=e,this.httpOptions=n,this.onEmailLeads=a,this.onEmailDiscovered=t,this.debug=o,this.userAgents=C()}sleep(e){return new Promise((n)=>setTimeout(n,e))}async executeHttp(e,n,a,t,o,i=0){let{getCache:s,saveCache:r,hasUrlInCache:m,saveUrl:u,httpConfig:l={}}=t;if(!e||e.length<3||this.forbiddenProtocols.some((f)=>e.startsWith(f)))return;try{let f=o?!1:await m(e),h=await s(e);if(f&&!h)return!1;if(f&&n!=="GET")return!1;let c=h&&n==="GET"?h:await(n==="GET"?this.http.get(e,l):n==="PATCH"?this.http.patch(e,a,l):n==="POST"?this.http.post(e,a,l):this.http.put(e,a,l));if(!h)await r(e,{data:c.data,contentType:c.contentType,finalUrl:c.finalUrl});if(!f)await u(e);if(!c.contentType||!c.contentType.includes("/html")||!c.contentType.includes("text/")||typeof c.data!=="string")return null;return{data:c.data,contentType:c.contentType,finalUrl:c.finalUrl}}catch(f){let h=f,c=this.httpOptions;if(h&&h.response){let g=h.response.status,p=c.retryDelay||100,b=c.maxRetryAttempts||3,d=c.retryWithoutProxyOnStatusCode||void 0,w=c.maxRetryOnProxyError||3;if(d&&l.proxy&&d.includes(g)&&i<b)return await this.sleep(p),delete l.proxy,await this.executeHttp(e,n,a,t,o,i+1);else if(c.retryOnStatusCode&&l.proxy&&c.retryOnStatusCode.includes(g)&&i<b)return await this.sleep(p),await this.executeHttp(e,n,a,t,o,i+1);else if(c.retryOnProxyError&&l.proxy&&i<w)return await this.sleep(p),await this.executeHttp(e,n,a,t,o,i+1)}if(this.debug){if(this.debug)console.log(`Error: unable to ${n} ${e}: ${f.message}`)}return null}}extractEmails(e,n,a,t,o,i){let s=this.extractEmailsFromContent(e?.replaceAll("mailto:","  ")),r=[];for(let m of s)if(this.handleEmailDiscovery(m,n,a,o,i))r.push(m);if(t&&t.length>0&&r.length>0)o.add(async()=>Promise.all(t.map((m)=>m(r))));s.length=0,r.length=0}async parseExternalWebsite(e,n,a,t,o,i=!0,s,r){let m=t.httpConfig?.headers?t.httpConfig.headers instanceof Headers?Object.fromEntries(t.httpConfig.headers.entries()):t.httpConfig.headers:{};t.httpConfig=t.httpConfig||{},t.httpConfig.headers={"user-agent":this.getRandomUserAgent(),accept:"text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8","accept-language":"en-US,en;q=0.9","cache-control":"no-cache",pragma:"no-cache",...m},t.httpConfig.timeout=t.httpConfig.timeout||15000,t.depth=t.depth||0,t.allowCrossDomainTravel=t.allowCrossDomainTravel||!1,o=o&&i;let u=[];try{let l=new URL(e),f=this.extractRootDomain(e);if(this.isLinktreeUrl(e))return await this.parseLinktreeProfile(e,t,o);if(this.isRestrictedDomain(e)){if(this.debug)console.warn(`⚠️ Skipped URL (restricted url): ${e}`);return u}let h=await this.executeHttp(e,n,a,t,o);if(!h){if(this.debug&&h===null)console.warn(`⚠️ Failed to fetch page content: ${e}`);if(this.debug&&h===!1)console.warn(`⚠️ Skipped URL (already visited): ${e}`);return u}let c=this.extractEmailsFromContent(h.data?.replaceAll("mailto:","  "));for(let g of c)if(this.handleEmailDiscovery(g,e,t.onEmailDiscovered,t.queue,t.emailMetadata))u.push(g);if(t.depth>0||!s){let g;{let{document:d}=v(h.data);g=this.extractRelevantLinks(d,l,f,t.depth,t.allowCrossDomainTravel)}t.depth--;let p=10,b=[];for(let d=0;d<g.length;d+=p){let w=g.slice(d,d+p),E=await Promise.allSettled(w.map((L)=>this.parseExternalWebsite(L,"GET",null,{...t,depth:t.depth},o,!1,!0)));b.push(...E)}for(let d of b)if(d.status==="fulfilled")u.push(...d.value);else if(this.debug)console.warn("⚠️ Failed to parse child URL:",d.reason?.message)}}catch(l){if(this.debug)console.error(`❌ Error parsing external website: ${e}`,l?.message)}if(i){if(t.onEmails&&t.onEmails.length>0)t.queue.add(async()=>Promise.all(t.onEmails.map((l)=>l(u))))}return u}async parseLinktreeProfile(e,n,a){let t=[];try{let o=await this.executeHttp(e,"GET",null,n,a);if(!o){if(this.debug)console.warn(`⚠️ Failed to fetch Linktree profile: ${e}`);return t}let i;{let{document:r}=v(o),m=r.getElementById("links-container");if(!m){if(this.debug)console.warn(`\uD83D\uDD0D No links container found in Linktree profile: ${e}`);return t}i=this.extractLinktreeExternalUrls(m,e)}if(i.length===0){if(this.debug)console.info("\uD83D\uDCED No valid external links found in Linktree profile");return t}if(this.debug)console.info(`\uD83C\uDFAF Found ${i.length} external links in Linktree profile`);let s=await Promise.allSettled(i.map((r)=>this.parseExternalWebsite(r,"GET",null,n,a,!1)));for(let r of s)if(r.status==="fulfilled")t.push(...r.value);else if(this.debug)console.warn("⚠️ Failed to parse Linktree external URL:",r.reason?.message)}catch(o){if(this.debug)console.error(`❌ Error parsing Linktree profile: ${e}`,o?.message)}return t}extractLinktreeExternalUrls(e,n){let a=new Set,t=e.querySelectorAll("a[href][target='_blank']");for(let o of t){let i=o.getAttribute("href");if(!i||i.length<3||this.forbiddenProtocols.some((s)=>i.startsWith(s)))continue;try{let s=new URL(i,n).href,r=this.extractRootDomain(s);if(r!=="linktr.ee"&&!this.isRestrictedDomain(s)&&r.length>3)a.add(s)}catch(s){if(this.debug)console.warn(`\uD83D\uDD17 Invalid URL in Linktree: ${i}`)}}return Array.from(a)}handleEmailDiscovery(e,n,a,t,o){if(!this.discoveredEmails.has(e)){this.discoveredEmails.add(e);let i={email:e,discoveredAt:n,timestamp:new Date,metadata:o||{}};if(a&&a.length>0)t.add(async()=>Promise.all(a.map((s)=>s(i))));if(this.debug)console.info(`\uD83D\uDCE7 New email discovered: ${e} at ${n}`);return!0}return!1}isDomainAccessAllowed(e,n,a,t){if(t)return!0;if(a===0)return e===n;return e===n||e.endsWith(`.${n}`)||n.endsWith(`.${e}`)}extractRelevantLinks(e,n,a,t,o){let i=[],s=["about","contact","help","support","reach","email","mail","message","company","team","staff","info","inquiry","feedback","service","assistance","connect","touch"],r=e.querySelectorAll("a[href]");for(let m of r){let u=m.getAttribute("href");if(!u||u.length<2)continue;try{let l=this.normalizeUrl(u,n),f=this.extractRootDomain(l);if(!this.isDomainAccessAllowed(f,a,t,o))continue;if(s.some((c)=>l.toLowerCase().includes(c))||this.isLinktreeUrl(l))i.push(l)}catch(l){if(this.debug)console.warn(`\uD83D\uDD17 Invalid link found: ${u}`,l?.message)}}return i}extractEmailsFromContent(e){let n=e.replace(x.CLEAN_CONTENT_REGEX," "),a=(s)=>{let r=s.split("@")[1]?.toLowerCase(),m=s.split(".").pop()?.toLowerCase();return x.EMAIL_VALIDATION_REGEX.test(s)&&r!==void 0&&m!==void 0&&!this.fileExtensions.includes(`.${m}`)&&!this.isRestrictedDomain(`https://${r}`)},t=(s)=>{return x.EMAIL_REGEX.lastIndex=0,(s.match(x.EMAIL_REGEX)||[]).filter(a)},o=e.replace(/<[^>]*>/g," "),i=[...t(o),...t(n)];return[...new Set(i)]}isRestrictedDomain(e){try{let n=new URL(e).host.toLowerCase();return this.restrictedDomains.some((a)=>n===a.toLowerCase()||n.endsWith(`.${a.toLowerCase()}`))}catch{return!0}}isLinktreeUrl(e){try{return this.extractRootDomain(e)==="linktr.ee"}catch{return!1}}extractRootDomain(e){try{let a=new URL(e).hostname.toLowerCase();return a.startsWith("www.")?a.slice(4):a}catch{return""}}normalizeUrl(e,n){if(e.startsWith("http://")||e.startsWith("https://"))return e;if(e.startsWith("//"))return`${n.protocol}${e}`;return new URL(e,n.href).href}getRandomUserAgent(){return this.userAgents[Math.floor(Math.random()*this.userAgents.length)]}}function C(){let e=[{name:"Chrome",version:"91.0.4472.124",engine:"AppleWebKit/537.36"},{name:"Firefox",version:"89.0",engine:"Gecko/20100101"},{name:"Safari",version:"14.1.1",engine:"AppleWebKit/605.1.15"},{name:"Edge",version:"91.0.864.59",engine:"AppleWebKit/537.36"},{name:"Opera",version:"77.0.4054.277",engine:"AppleWebKit/537.36"},{name:"Vivaldi",version:"3.8.2259.42",engine:"AppleWebKit/537.36"},{name:"Brave",version:"1.26.74",engine:"AppleWebKit/537.36"},{name:"Chromium",version:"91.0.4472.101",engine:"AppleWebKit/537.36"},{name:"Yandex",version:"21.5.3.742",engine:"AppleWebKit/537.36"},{name:"Maxthon",version:"5.3.8.2000",engine:"AppleWebKit/537.36"}],n=["Windows NT 10.0","Windows NT 6.1","Macintosh; Intel Mac OS X 10_15_7","Macintosh; Intel Mac OS X 11_4_0","X11; Linux x86_64","X11; Ubuntu; Linux x86_64"],a=[];for(let t=0;t<200;t++){let o=e[Math.floor(Math.random()*e.length)],i=n[Math.floor(Math.random()*n.length)],s="";switch(o.name){case"Chrome":s=`Mozilla/5.0 (${i}) ${o.engine} (KHTML, like Gecko) Chrome/${o.version} Safari/537.36`;break;case"Firefox":s=`Mozilla/5.0 (${i}; rv:${o.version}) ${o.engine} Firefox/${o.version}`;break;case"Safari":s=`Mozilla/5.0 (${i}) ${o.engine} (KHTML, like Gecko) Version/${o.version} Safari/605.1.15`;break;case"Edge":s=`Mozilla/5.0 (${i}) ${o.engine} (KHTML, like Gecko) Chrome/${o.version} Safari/537.36 Edg/${o.version}`;break;case"Opera":s=`Mozilla/5.0 (${i}) ${o.engine} (KHTML, like Gecko) Chrome/${o.version} Safari/537.36 OPR/${o.version}`;break;case"Vivaldi":s=`Mozilla/5.0 (${i}) ${o.engine} (KHTML, like Gecko) Chrome/${o.version} Safari/537.36 Vivaldi/${o.version}`;break;case"Brave":s=`Mozilla/5.0 (${i}) ${o.engine} (KHTML, like Gecko) Chrome/${o.version} Safari/537.36 Brave/${o.version}`;break;case"Chromium":s=`Mozilla/5.0 (${i}) ${o.engine} (KHTML, like Gecko) Chromium/${o.version} Chrome/${o.version} Safari/537.36`;break;case"Yandex":s=`Mozilla/5.0 (${i}) ${o.engine} (KHTML, like Gecko) Chrome/${o.version} YaBrowser/${o.version} Safari/537.36`;break;case"Maxthon":s=`Mozilla/5.0 (${i}) ${o.engine} (KHTML, like Gecko) Chrome/${o.version} Safari/537.36 Maxthon/${o.version}`;break}a.push(s)}return a}function y(){return["facebook.com","fb.com","messenger.com","instagram.com","threads.net","twitter.com","x.com","linkedin.com","pinterest.com","pin.it","reddit.com","tumblr.com","snapchat.com","tiktok.com","douyin.com","youtube.com","youtu.be","whatsapp.com","telegram.org","t.me","medium.com","quora.com","flickr.com","vimeo.com","vk.com","weibo.com","sina.com.cn","line.me","discord.com","discordapp.com","twitch.tv","meetup.com","nextdoor.com","xing.com","yelp.com","zalo.me","mastodon.social","clubhouse.com","patreon.com","onlyfans.com","douban.com","goodreads.com","soundcloud.com","spotify.com","last.fm","behance.net","dribbble.com","deviantart.com","pixiv.net","slideshare.net","tinder.com","bumble.com","etsy.com","indeed.com","glassdoor.com","monster.com","careerbuilder.com","dice.com","ziprecruiter.com","simplyhired.com","upwork.com","freelancer.com","fiverr.com","stackoverflow.com","stackoverflow.co","angel.co","wellfound.com","quora.com","stackexchange.com","yahoo.com","answers.microsoft.com","askubuntu.com","superuser.com","serverfault.com","mathoverflow.net","xda-developers.com","gamespot.com","ign.com","4chan.org","9gag.com","gizmodo.com","slashdot.org","hacker-news.news","ycombinator.com","producthunt.com","discourse.org","google.com","google.co.uk","google.de","google.fr","google.co.jp","bing.com","yahoo.com","search.yahoo.com","duckduckgo.com","baidu.com","yandex.com","yandex.ru","ask.com","wolframalpha.com","ecosia.org","startpage.com","qwant.com","searx.me","gibiru.com","swisscows.com","gmail.com","googlemail.com","outlook.com","hotmail.com","live.com","msn.com","yahoo.com","ymail.com","aol.com","icloud.com","me.com","mac.com","protonmail.com","pm.me","zoho.com","mail.com","gmx.com","gmx.net","yandex.com","yandex.ru","tutanota.com","tutanota.de","fastmail.com","hushmail.com","mailbox.org","posteo.de","runbox.com","disroot.org","163.com","qq.com","rambler.ru","mail.ru","yelp.com","yelp.ca","yelp.co.uk","yelp.com.au","yellowpages.com","yellowpages.ca","yell.com","tripadvisor.com","tripadvisor.co.uk","tripadvisor.ca","foursquare.com","angieslist.com","bbb.org","manta.com","thumbtack.com","homeadvisor.com","superpages.com","whitepages.com","local.com","citysearch.com","merchantcircle.com","insiderpages.com","kudzu.com","hotfrog.com","buildzoom.com","houzz.com","porch.com","mapquest.com","zagat.com","zomato.com","opentable.com","viator.com","expedia.com","booking.com","airbnb.com","vrbo.com","homeaway.com","craigslist.org","nextdoor.com","patch.com","meetup.com","eventbrite.com","groupon.com","livingsocial.com","gumtree.com","gumtree.com.au","kijiji.ca","leboncoin.fr","finn.no","blocket.se","58.com","dianping.com","tabelog.com","ypcdn.com"]}function A(e){try{let n=new URL(e).host.toLowerCase();return y().some((a)=>n===a.toLowerCase()||n.endsWith(`.${a.toLowerCase()}`))}catch{return!0}}export{A as isRestrictedDomain,x as Scraper,k as CappedSet};

package/dist/crawler.d.ts

@@ -7200,6 +7200,8 @@ export declare class Crawler {
 	private readonly rawResponseEvents;
 	private emailDiscoveredEvents;
 	private emailLeadsEvents;
+	/** Counter for active event handlers (not queued, runs directly) */
+	private eventCount;
 	/**
 	 * Key-value cache instance for storing HTTP responses.
 	 * Uses SQLite as the underlying storage mechanism.
@@ -7751,6 +7753,16 @@ export declare class Crawler {
 	 * Called when onQueueChange handlers are registered to ensure limiter events are captured.
 	 */
 	private _subscribeToLimiterQueues;
+	/**
+	 * Run a handler with event tracking (not through queue).
+	 * Increments eventCount before running, decrements after completion.
+	 * This allows waitForAll() to wait for handlers without queue bloat.
+	 */
+	private _runHandler;
+	/**
+	 * Run a bound handler (using .call()) with event tracking.
+	 */
+	private _runBoundHandler;
 	private _onBody;
 	private _onAttribute;
 	private _onText;

package/dist/entries/crawler.cjs

@@ -1,4 +1,4 @@
-const _mod_6bmlfo = require('../crawler/crawler.cjs');
-exports.Crawler = _mod_6bmlfo.Crawler;;
-const _mod_4nabp1 = require('../crawler/crawler-options.cjs');
-exports.CrawlerOptions = _mod_4nabp1.CrawlerOptions;;
+const _mod_qcl1xk = require('../crawler/crawler.cjs');
+exports.Crawler = _mod_qcl1xk.Crawler;;
+const _mod_m4i5gz = require('../crawler/crawler-options.cjs');
+exports.CrawlerOptions = _mod_m4i5gz.CrawlerOptions;;

package/dist/index.cjs

@@ -1,30 +1,30 @@
-const _mod_uyga9f = require('./core/rezo.cjs');
-exports.Rezo = _mod_uyga9f.Rezo;
-exports.createRezoInstance = _mod_uyga9f.createRezoInstance;
-exports.createDefaultInstance = _mod_uyga9f.createDefaultInstance;;
-const _mod_w3fnjx = require('./errors/rezo-error.cjs');
-exports.RezoError = _mod_w3fnjx.RezoError;
-exports.RezoErrorCode = _mod_w3fnjx.RezoErrorCode;;
-const _mod_cfn1g3 = require('./utils/headers.cjs');
-exports.RezoHeaders = _mod_cfn1g3.RezoHeaders;;
-const _mod_ld3867 = require('./utils/form-data.cjs');
-exports.RezoFormData = _mod_ld3867.RezoFormData;;
-const _mod_um6gnq = require('./utils/cookies.cjs');
-exports.RezoCookieJar = _mod_um6gnq.RezoCookieJar;
-exports.Cookie = _mod_um6gnq.Cookie;;
-const _mod_2u2x4t = require('./utils/curl.cjs');
-exports.toCurl = _mod_2u2x4t.toCurl;
-exports.fromCurl = _mod_2u2x4t.fromCurl;;
-const _mod_l9fvot = require('./core/hooks.cjs');
-exports.createDefaultHooks = _mod_l9fvot.createDefaultHooks;
-exports.mergeHooks = _mod_l9fvot.mergeHooks;;
-const _mod_3we0pp = require('./proxy/manager.cjs');
-exports.ProxyManager = _mod_3we0pp.ProxyManager;;
-const _mod_m23nko = require('./queue/index.cjs');
-exports.RezoQueue = _mod_m23nko.RezoQueue;
-exports.HttpQueue = _mod_m23nko.HttpQueue;
-exports.Priority = _mod_m23nko.Priority;
-exports.HttpMethodPriority = _mod_m23nko.HttpMethodPriority;;
+const _mod_vfaphs = require('./core/rezo.cjs');
+exports.Rezo = _mod_vfaphs.Rezo;
+exports.createRezoInstance = _mod_vfaphs.createRezoInstance;
+exports.createDefaultInstance = _mod_vfaphs.createDefaultInstance;;
+const _mod_of2frn = require('./errors/rezo-error.cjs');
+exports.RezoError = _mod_of2frn.RezoError;
+exports.RezoErrorCode = _mod_of2frn.RezoErrorCode;;
+const _mod_340nsi = require('./utils/headers.cjs');
+exports.RezoHeaders = _mod_340nsi.RezoHeaders;;
+const _mod_h8tc2h = require('./utils/form-data.cjs');
+exports.RezoFormData = _mod_h8tc2h.RezoFormData;;
+const _mod_nogecb = require('./utils/cookies.cjs');
+exports.RezoCookieJar = _mod_nogecb.RezoCookieJar;
+exports.Cookie = _mod_nogecb.Cookie;;
+const _mod_dem7ed = require('./utils/curl.cjs');
+exports.toCurl = _mod_dem7ed.toCurl;
+exports.fromCurl = _mod_dem7ed.fromCurl;;
+const _mod_ajli2g = require('./core/hooks.cjs');
+exports.createDefaultHooks = _mod_ajli2g.createDefaultHooks;
+exports.mergeHooks = _mod_ajli2g.mergeHooks;;
+const _mod_n46ifj = require('./proxy/manager.cjs');
+exports.ProxyManager = _mod_n46ifj.ProxyManager;;
+const _mod_c88szv = require('./queue/index.cjs');
+exports.RezoQueue = _mod_c88szv.RezoQueue;
+exports.HttpQueue = _mod_c88szv.HttpQueue;
+exports.Priority = _mod_c88szv.Priority;
+exports.HttpMethodPriority = _mod_c88szv.HttpMethodPriority;;
 const { RezoError } = require('./errors/rezo-error.cjs');
 const isRezoError = exports.isRezoError = RezoError.isRezoError;
 const Cancel = exports.Cancel = RezoError;
@@ -34,9 +34,9 @@ const isCancel = exports.isCancel = (error) => {
 };
 const all = exports.all = Promise.all.bind(Promise);
 const spread = exports.spread = (callback) => (array) => callback(...array);
-const _mod_zq73m7 = require('./version.cjs');
-exports.VERSION = _mod_zq73m7.VERSION;
-exports.PACKAGE_NAME = _mod_zq73m7.PACKAGE_NAME;;
+const _mod_rw2too = require('./version.cjs');
+exports.VERSION = _mod_rw2too.VERSION;
+exports.PACKAGE_NAME = _mod_rw2too.PACKAGE_NAME;;
 const { executeRequest } = require('./adapters/http.cjs');
 const { setGlobalAdapter, createRezoInstance } = require('./core/rezo.cjs');
 setGlobalAdapter(executeRequest);

package/dist/index.d.ts

@@ -4886,7 +4886,7 @@ export declare class HttpQueue extends RezoQueue<any> {
  *
  * IMPORTANT: Update these values when bumping package version.
  */
-export declare const VERSION = "1.0.76";
+export declare const VERSION = "1.0.78";
 export declare const PACKAGE_NAME = "rezo";
 export declare const isRezoError: typeof RezoError.isRezoError;
 export declare const Cancel: typeof RezoError;

package/dist/internal/agents/index.cjs

@@ -1,14 +1,14 @@
-const _mod_gg6p0c = require('./base.cjs');
-exports.Agent = _mod_gg6p0c.Agent;;
-const _mod_yk29a5 = require('./http-proxy.cjs');
-exports.HttpProxyAgent = _mod_yk29a5.HttpProxyAgent;;
-const _mod_gwxwmp = require('./https-proxy.cjs');
-exports.HttpsProxyAgent = _mod_gwxwmp.HttpsProxyAgent;;
-const _mod_hyjr95 = require('./socks-proxy.cjs');
-exports.SocksProxyAgent = _mod_hyjr95.SocksProxyAgent;;
-const _mod_f1pn5c = require('./socks-client.cjs');
-exports.SocksClient = _mod_f1pn5c.SocksClient;;
-const _mod_1yajei = require('./bun-socks-http.cjs');
-exports.bunHttp = _mod_1yajei.bunHttp;
-exports.isBunRuntime = _mod_1yajei.isBunRuntime;
-exports.isBunSocksRequest = _mod_1yajei.isBunSocksRequest;;
+const _mod_w3wpi5 = require('./base.cjs');
+exports.Agent = _mod_w3wpi5.Agent;;
+const _mod_tkhiqu = require('./http-proxy.cjs');
+exports.HttpProxyAgent = _mod_tkhiqu.HttpProxyAgent;;
+const _mod_9x6z7a = require('./https-proxy.cjs');
+exports.HttpsProxyAgent = _mod_9x6z7a.HttpsProxyAgent;;
+const _mod_qt4do7 = require('./socks-proxy.cjs');
+exports.SocksProxyAgent = _mod_qt4do7.SocksProxyAgent;;
+const _mod_k5s6py = require('./socks-client.cjs');
+exports.SocksClient = _mod_k5s6py.SocksClient;;
+const _mod_6uihcw = require('./bun-socks-http.cjs');
+exports.bunHttp = _mod_6uihcw.bunHttp;
+exports.isBunRuntime = _mod_6uihcw.isBunRuntime;
+exports.isBunSocksRequest = _mod_6uihcw.isBunSocksRequest;;

package/dist/platform/browser.d.ts

@@ -4582,7 +4582,7 @@ export interface RezoInstance extends Rezo {
  *
  * IMPORTANT: Update these values when bumping package version.
  */
-export declare const VERSION = "1.0.76";
+export declare const VERSION = "1.0.78";
 export declare const isRezoError: typeof RezoError.isRezoError;
 export declare const Cancel: typeof RezoError;
 export declare const CancelToken: {

package/dist/platform/bun.d.ts

@@ -4582,7 +4582,7 @@ export interface RezoInstance extends Rezo {
  *
  * IMPORTANT: Update these values when bumping package version.
  */
-export declare const VERSION = "1.0.76";
+export declare const VERSION = "1.0.78";
 export declare const isRezoError: typeof RezoError.isRezoError;
 export declare const Cancel: typeof RezoError;
 export declare const CancelToken: {

package/dist/platform/deno.d.ts

@@ -4582,7 +4582,7 @@ export interface RezoInstance extends Rezo {
  *
  * IMPORTANT: Update these values when bumping package version.
  */
-export declare const VERSION = "1.0.76";
+export declare const VERSION = "1.0.78";
 export declare const isRezoError: typeof RezoError.isRezoError;
 export declare const Cancel: typeof RezoError;
 export declare const CancelToken: {

package/dist/platform/node.d.ts

@@ -4582,7 +4582,7 @@ export interface RezoInstance extends Rezo {
  *
  * IMPORTANT: Update these values when bumping package version.
  */
-export declare const VERSION = "1.0.76";
+export declare const VERSION = "1.0.78";
 export declare const isRezoError: typeof RezoError.isRezoError;
 export declare const Cancel: typeof RezoError;
 export declare const CancelToken: {

package/dist/platform/react-native.d.ts

@@ -4582,7 +4582,7 @@ export interface RezoInstance extends Rezo {
  *
  * IMPORTANT: Update these values when bumping package version.
  */
-export declare const VERSION = "1.0.76";
+export declare const VERSION = "1.0.78";
 export declare const isRezoError: typeof RezoError.isRezoError;
 export declare const Cancel: typeof RezoError;
 export declare const CancelToken: {

package/dist/platform/worker.d.ts

@@ -4582,7 +4582,7 @@ export interface RezoInstance extends Rezo {
  *
  * IMPORTANT: Update these values when bumping package version.
  */
-export declare const VERSION = "1.0.76";
+export declare const VERSION = "1.0.78";
 export declare const isRezoError: typeof RezoError.isRezoError;
 export declare const Cancel: typeof RezoError;
 export declare const CancelToken: {

package/dist/proxy/index.cjs

@@ -1,9 +1,9 @@
 const { Agent, HttpProxyAgent, HttpsProxyAgent, SocksProxyAgent } = require('../internal/agents/index.cjs');
 const { parseProxyString } = require('./parse.cjs');
-const _mod_g5wdrq = require('./manager.cjs');
-exports.ProxyManager = _mod_g5wdrq.ProxyManager;;
-const _mod_mcuozi = require('./parse.cjs');
-exports.parseProxyString = _mod_mcuozi.parseProxyString;;
+const _mod_vkqfol = require('./manager.cjs');
+exports.ProxyManager = _mod_vkqfol.ProxyManager;;
+const _mod_9fbcpj = require('./parse.cjs');
+exports.parseProxyString = _mod_9fbcpj.parseProxyString;;
 function createOptions(uri, opts) {
   if (uri instanceof URL || typeof uri === "string") {
     return {

package/dist/queue/index.cjs

@@ -1,8 +1,8 @@
-const _mod_vg5z50 = require('./queue.cjs');
-exports.RezoQueue = _mod_vg5z50.RezoQueue;;
-const _mod_ibwlue = require('./http-queue.cjs');
-exports.HttpQueue = _mod_ibwlue.HttpQueue;
-exports.extractDomain = _mod_ibwlue.extractDomain;;
-const _mod_jnmptv = require('./types.cjs');
-exports.Priority = _mod_jnmptv.Priority;
-exports.HttpMethodPriority = _mod_jnmptv.HttpMethodPriority;;
+const _mod_fq49yc = require('./queue.cjs');
+exports.RezoQueue = _mod_fq49yc.RezoQueue;;
+const _mod_l43il2 = require('./http-queue.cjs');
+exports.HttpQueue = _mod_l43il2.HttpQueue;
+exports.extractDomain = _mod_l43il2.extractDomain;;
+const _mod_brf3na = require('./types.cjs');
+exports.Priority = _mod_brf3na.Priority;
+exports.HttpMethodPriority = _mod_brf3na.HttpMethodPriority;;

package/dist/responses/universal/index.cjs

@@ -1,11 +1,11 @@
-const _mod_wl7wnx = require('./event-emitter.cjs');
-exports.UniversalEventEmitter = _mod_wl7wnx.UniversalEventEmitter;;
-const _mod_a6km22 = require('./stream.cjs');
-exports.UniversalStreamResponse = _mod_a6km22.UniversalStreamResponse;
-exports.StreamResponse = _mod_a6km22.StreamResponse;;
-const _mod_0dk9cp = require('./download.cjs');
-exports.UniversalDownloadResponse = _mod_0dk9cp.UniversalDownloadResponse;
-exports.DownloadResponse = _mod_0dk9cp.DownloadResponse;;
-const _mod_v3kk6n = require('./upload.cjs');
-exports.UniversalUploadResponse = _mod_v3kk6n.UniversalUploadResponse;
-exports.UploadResponse = _mod_v3kk6n.UploadResponse;;
+const _mod_jimq7c = require('./event-emitter.cjs');
+exports.UniversalEventEmitter = _mod_jimq7c.UniversalEventEmitter;;
+const _mod_m88wcs = require('./stream.cjs');
+exports.UniversalStreamResponse = _mod_m88wcs.UniversalStreamResponse;
+exports.StreamResponse = _mod_m88wcs.StreamResponse;;
+const _mod_u873ev = require('./download.cjs');
+exports.UniversalDownloadResponse = _mod_u873ev.UniversalDownloadResponse;
+exports.DownloadResponse = _mod_u873ev.DownloadResponse;;
+const _mod_x1pelx = require('./upload.cjs');
+exports.UniversalUploadResponse = _mod_x1pelx.UniversalUploadResponse;
+exports.UploadResponse = _mod_x1pelx.UploadResponse;;

package/dist/version.cjs

@@ -1,2 +1,2 @@
-const VERSION = exports.VERSION = "1.0.76";
+const VERSION = exports.VERSION = "1.0.78";
 const PACKAGE_NAME = exports.PACKAGE_NAME = "rezo";

package/dist/version.js

@@ -1,2 +1,2 @@
-export const VERSION = "1.0.76";
+export const VERSION = "1.0.78";
 export const PACKAGE_NAME = "rezo";

package/dist/wget/index.cjs

@@ -1,52 +1,52 @@
-const _mod_bwlia3 = require('./types.cjs');
-exports.WgetError = _mod_bwlia3.WgetError;;
-const _mod_ior70c = require('./asset-extractor.cjs');
-exports.AssetExtractor = _mod_ior70c.AssetExtractor;;
-const _mod_4gs7wl = require('./url-filter.cjs');
-exports.UrlFilter = _mod_4gs7wl.UrlFilter;;
-const _mod_pqaqrk = require('./file-writer.cjs');
-exports.FileWriter = _mod_pqaqrk.FileWriter;;
-const _mod_1gmuq7 = require('./robots.cjs');
-exports.RobotsHandler = _mod_1gmuq7.RobotsHandler;;
-const _mod_jtwega = require('./resume.cjs');
-exports.ResumeHandler = _mod_jtwega.ResumeHandler;;
-const _mod_kxdayw = require('./progress.cjs');
-exports.ProgressReporter = _mod_kxdayw.ProgressReporter;
-exports.ProgressTracker = _mod_kxdayw.ProgressTracker;
-exports.parseSize = _mod_kxdayw.parseSize;;
-const _mod_eetcw2 = require('./link-converter.cjs');
-exports.LinkConverter = _mod_eetcw2.LinkConverter;;
-const _mod_kvfx8b = require('./downloader.cjs');
-exports.Downloader = _mod_kvfx8b.Downloader;;
-const _mod_tl3x2q = require('./asset-organizer.cjs');
-exports.AssetOrganizer = _mod_tl3x2q.AssetOrganizer;
-exports.DEFAULT_ASSET_FOLDERS = _mod_tl3x2q.DEFAULT_ASSET_FOLDERS;;
-const _mod_d60ii8 = require('./download-cache.cjs');
-exports.DownloadCache = _mod_d60ii8.DownloadCache;;
-const _mod_exqqlg = require('./filter-lists.cjs');
-exports.EXECUTABLE_EXTENSIONS = _mod_exqqlg.EXECUTABLE_EXTENSIONS;
-exports.ARCHIVE_EXTENSIONS = _mod_exqqlg.ARCHIVE_EXTENSIONS;
-exports.DOCUMENT_EXTENSIONS = _mod_exqqlg.DOCUMENT_EXTENSIONS;
-exports.IMAGE_EXTENSIONS = _mod_exqqlg.IMAGE_EXTENSIONS;
-exports.VIDEO_EXTENSIONS = _mod_exqqlg.VIDEO_EXTENSIONS;
-exports.AUDIO_EXTENSIONS = _mod_exqqlg.AUDIO_EXTENSIONS;
-exports.FONT_EXTENSIONS = _mod_exqqlg.FONT_EXTENSIONS;
-exports.WEB_ASSET_EXTENSIONS = _mod_exqqlg.WEB_ASSET_EXTENSIONS;
-exports.DATA_EXTENSIONS = _mod_exqqlg.DATA_EXTENSIONS;
-exports.EXECUTABLE_MIME_TYPES = _mod_exqqlg.EXECUTABLE_MIME_TYPES;
-exports.ARCHIVE_MIME_TYPES = _mod_exqqlg.ARCHIVE_MIME_TYPES;
-exports.DOCUMENT_MIME_TYPES = _mod_exqqlg.DOCUMENT_MIME_TYPES;
-exports.IMAGE_MIME_TYPES = _mod_exqqlg.IMAGE_MIME_TYPES;
-exports.VIDEO_MIME_TYPES = _mod_exqqlg.VIDEO_MIME_TYPES;
-exports.AUDIO_MIME_TYPES = _mod_exqqlg.AUDIO_MIME_TYPES;
-exports.FONT_MIME_TYPES = _mod_exqqlg.FONT_MIME_TYPES;
-exports.WEB_ASSET_MIME_TYPES = _mod_exqqlg.WEB_ASSET_MIME_TYPES;
-exports.DATA_MIME_TYPES = _mod_exqqlg.DATA_MIME_TYPES;
-exports.SAFE_WEB_PRESET = _mod_exqqlg.SAFE_WEB_PRESET;
-exports.DOCUMENTS_ONLY_PRESET = _mod_exqqlg.DOCUMENTS_ONLY_PRESET;
-exports.NO_MEDIA_PRESET = _mod_exqqlg.NO_MEDIA_PRESET;
-exports.MINIMAL_MIRROR_PRESET = _mod_exqqlg.MINIMAL_MIRROR_PRESET;
-exports.TEXT_ONLY_PRESET = _mod_exqqlg.TEXT_ONLY_PRESET;;
+const _mod_zyus1m = require('./types.cjs');
+exports.WgetError = _mod_zyus1m.WgetError;;
+const _mod_wv8msk = require('./asset-extractor.cjs');
+exports.AssetExtractor = _mod_wv8msk.AssetExtractor;;
+const _mod_bphj0k = require('./url-filter.cjs');
+exports.UrlFilter = _mod_bphj0k.UrlFilter;;
+const _mod_ht925z = require('./file-writer.cjs');
+exports.FileWriter = _mod_ht925z.FileWriter;;
+const _mod_ldwidd = require('./robots.cjs');
+exports.RobotsHandler = _mod_ldwidd.RobotsHandler;;
+const _mod_8v1w99 = require('./resume.cjs');
+exports.ResumeHandler = _mod_8v1w99.ResumeHandler;;
+const _mod_pzzxi6 = require('./progress.cjs');
+exports.ProgressReporter = _mod_pzzxi6.ProgressReporter;
+exports.ProgressTracker = _mod_pzzxi6.ProgressTracker;
+exports.parseSize = _mod_pzzxi6.parseSize;;
+const _mod_xb50un = require('./link-converter.cjs');
+exports.LinkConverter = _mod_xb50un.LinkConverter;;
+const _mod_wqutce = require('./downloader.cjs');
+exports.Downloader = _mod_wqutce.Downloader;;
+const _mod_7xd59t = require('./asset-organizer.cjs');
+exports.AssetOrganizer = _mod_7xd59t.AssetOrganizer;
+exports.DEFAULT_ASSET_FOLDERS = _mod_7xd59t.DEFAULT_ASSET_FOLDERS;;
+const _mod_obq3yt = require('./download-cache.cjs');
+exports.DownloadCache = _mod_obq3yt.DownloadCache;;
+const _mod_svuqx6 = require('./filter-lists.cjs');
+exports.EXECUTABLE_EXTENSIONS = _mod_svuqx6.EXECUTABLE_EXTENSIONS;
+exports.ARCHIVE_EXTENSIONS = _mod_svuqx6.ARCHIVE_EXTENSIONS;
+exports.DOCUMENT_EXTENSIONS = _mod_svuqx6.DOCUMENT_EXTENSIONS;
+exports.IMAGE_EXTENSIONS = _mod_svuqx6.IMAGE_EXTENSIONS;
+exports.VIDEO_EXTENSIONS = _mod_svuqx6.VIDEO_EXTENSIONS;
+exports.AUDIO_EXTENSIONS = _mod_svuqx6.AUDIO_EXTENSIONS;
+exports.FONT_EXTENSIONS = _mod_svuqx6.FONT_EXTENSIONS;
+exports.WEB_ASSET_EXTENSIONS = _mod_svuqx6.WEB_ASSET_EXTENSIONS;
+exports.DATA_EXTENSIONS = _mod_svuqx6.DATA_EXTENSIONS;
+exports.EXECUTABLE_MIME_TYPES = _mod_svuqx6.EXECUTABLE_MIME_TYPES;
+exports.ARCHIVE_MIME_TYPES = _mod_svuqx6.ARCHIVE_MIME_TYPES;
+exports.DOCUMENT_MIME_TYPES = _mod_svuqx6.DOCUMENT_MIME_TYPES;
+exports.IMAGE_MIME_TYPES = _mod_svuqx6.IMAGE_MIME_TYPES;
+exports.VIDEO_MIME_TYPES = _mod_svuqx6.VIDEO_MIME_TYPES;
+exports.AUDIO_MIME_TYPES = _mod_svuqx6.AUDIO_MIME_TYPES;
+exports.FONT_MIME_TYPES = _mod_svuqx6.FONT_MIME_TYPES;
+exports.WEB_ASSET_MIME_TYPES = _mod_svuqx6.WEB_ASSET_MIME_TYPES;
+exports.DATA_MIME_TYPES = _mod_svuqx6.DATA_MIME_TYPES;
+exports.SAFE_WEB_PRESET = _mod_svuqx6.SAFE_WEB_PRESET;
+exports.DOCUMENTS_ONLY_PRESET = _mod_svuqx6.DOCUMENTS_ONLY_PRESET;
+exports.NO_MEDIA_PRESET = _mod_svuqx6.NO_MEDIA_PRESET;
+exports.MINIMAL_MIRROR_PRESET = _mod_svuqx6.MINIMAL_MIRROR_PRESET;
+exports.TEXT_ONLY_PRESET = _mod_svuqx6.TEXT_ONLY_PRESET;;
 const { Downloader } = require('./downloader.cjs');
 const rezo = require('../index.cjs');
 const { promises: fs } = require("node:fs");

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "rezo",
-  "version": "1.0.76",
+  "version": "1.0.78",
   "description": "Lightning-fast, enterprise-grade HTTP client for modern JavaScript. Full HTTP/2 support, intelligent cookie management, multiple adapters (HTTP, Fetch, cURL, XHR), streaming, proxy support (HTTP/HTTPS/SOCKS), and cross-environment compatibility.",
   "main": "dist/index.cjs",
   "module": "dist/index.js",