rezo 1.0.71 → 1.0.73

package/dist/crawler/crawler.js

@@ -3,6 +3,10 @@ import { FileCacher } from './plugin/file-cacher.js';
 import { UrlStore } from './plugin/url-store.js';
 import { NavigationHistory } from './plugin/navigation-history.js';
 import { RobotsTxt } from './plugin/robots-txt.js';
+import { MemoryMonitor } from './plugin/memory-monitor.js';
+import { HealthMetrics } from './plugin/health-metrics.js';
+import { CappedMap } from './plugin/capped-map.js';
+import { CappedArray } from './plugin/capped-array.js';
 import { parseHTML } from "linkedom";
 import path from "node:path";
 import { Rezo } from '../adapters/entries/http.js';
@@ -43,6 +47,7 @@ export class Crawler {
   emailLeadsEvents = [];
   cacher = null;
   queue;
+  scraperQueue;
   isCacheEnabled;
   config;
   urlStorage;
@@ -56,13 +61,12 @@ export class Crawler {
   navigationHistoryInitPromise = null;
   adapterExecutor = null;
   adapterType;
-  pendingExecutions = new Set;
-  pendingVisitCount = 0;
   isDestroyed = false;
-  queueOptions = { concurrency: 1000 };
+  shutdownRequested = false;
+  queueOptions = { concurrency: 100 };
   robotsTxt;
-  domainResponseTimes = new Map;
-  domainCurrentDelay = new Map;
+  domainResponseTimes = new CappedMap({ maxSize: 500 });
+  domainCurrentDelay = new CappedMap({ maxSize: 500 });
   crawlStats = {
     urlsVisited: 0,
     urlsQueued: 0,
@@ -70,19 +74,42 @@ export class Crawler {
     startTime: 0,
     currentDepth: 0
   };
-  urlDepthMap = new Map;
+  urlDepthMap = new CappedMap({ maxSize: 50000 });
+  cleanupInterval;
+  checkpointInterval;
+  lastCheckpointTime = 0;
+  memoryMonitor;
+  healthMetrics;
+  originalConcurrency = 100;
+  shutdownHandler = null;
   startHandlers = [];
   finishHandlers = [];
   redirectHandlers = [];
-  collectedData = [];
+  collectedData = new CappedArray({
+    maxSize: 1e5,
+    evictionRatio: 0.1,
+    onEviction: (evicted, remaining) => {
+      console.warn(`[Crawler] collectedData auto-evicted ${evicted.length} oldest entries. ${remaining} entries remaining. Consider using exportData() more frequently.`);
+    }
+  });
   crawlStarted = false;
+  startHandlersPromise = null;
   constructor(crawlerOptions, http = new Rezo) {
     this.http = http;
-    this.queue = new RezoQueue({
-      concurrency: 1000
-    });
     this.config = new CrawlerOptions(crawlerOptions);
     this.adapterType = this.config.adapter;
+    const concurrency = this.config.concurrency;
+    this.queue = new RezoQueue({
+      concurrency,
+      timeout: 60000
+    });
+    this.originalConcurrency = concurrency;
+    this.scraperQueue = new RezoQueue({
+      concurrency: this.config.scraperConcurrency,
+      timeout: 60000
+    });
+    this.memoryMonitor = new MemoryMonitor({ warningRatio: 0.7, criticalRatio: 0.85 });
+    this.healthMetrics = new HealthMetrics({ windowSize: 60000 });
     const enableCache = this.config.enableCache;
     this.isCacheEnabled = enableCache;
     if (enableCache) {
@@ -94,7 +121,7 @@ export class Crawler {
       FileCacher.create({
         cacheDir: dbUrl,
         ttl: cacheTTL,
-        encryptNamespace: true
+        maxEntries: 1e5
       }).then((storage) => {
         this.cacher = storage;
         this.isCacheReady = true;
@@ -151,6 +178,55 @@ export class Crawler {
     if (this.config.baseUrl) {
       this.urlDepthMap.set(this.config.baseUrl, 0);
     }
+    this.registerShutdownHandlers();
+  }
+  registerShutdownHandlers() {
+    if (this.shutdownHandler)
+      return;
+    this.shutdownHandler = () => this.gracefulShutdown();
+    process.on("SIGINT", this.shutdownHandler);
+    process.on("SIGTERM", this.shutdownHandler);
+  }
+  removeShutdownHandlers() {
+    if (this.shutdownHandler) {
+      process.off("SIGINT", this.shutdownHandler);
+      process.off("SIGTERM", this.shutdownHandler);
+      this.shutdownHandler = null;
+    }
+  }
+  async gracefulShutdown() {
+    if (this.shutdownRequested || this.isDestroyed)
+      return;
+    this.shutdownRequested = true;
+    console.log(`
+[Crawler] Shutdown requested, finishing current tasks...`);
+    this.queue.pause();
+    this.scraperQueue.pause();
+    const timeoutPromise = new Promise((resolve) => {
+      setTimeout(() => {
+        console.log("[Crawler] Shutdown timeout (5s), forcing exit...");
+        resolve();
+      }, 5000);
+    });
+    await Promise.race([
+      Promise.all([this.queue.onIdle(), this.scraperQueue.onIdle()]),
+      timeoutPromise
+    ]);
+    if (this.navigationHistory && this.currentSession) {
+      try {
+        await this.navigationHistory.updateSessionStatus(this.currentSession.sessionId, "paused");
+        await this.navigationHistory.updateSessionStats(this.currentSession.sessionId, {
+          urlsVisited: this.crawlStats.urlsVisited,
+          urlsQueued: this.queue.size,
+          urlsFailed: this.crawlStats.urlsFailed
+        });
+        console.log(`[Crawler] Session saved: ${this.currentSession.sessionId}`);
+      } catch (err) {
+        console.warn("[Crawler] Failed to save session state:", err);
+      }
+    }
+    await this.destroy();
+    console.log("[Crawler] Graceful shutdown complete");
   }
   async initializeAdapter() {
     try {
@@ -166,7 +242,6 @@ export class Crawler {
     if (!this.isDestroyed)
       return;
     this.queue = new RezoQueue(this.queueOptions);
-    this.pendingExecutions.clear();
     this.isDestroyed = false;
     if (this.config.debug) {
       console.log("[Crawler] Restored from destroyed state");
@@ -342,17 +417,110 @@ export class Crawler {
       handler(data);
     });
   }
-  async waitForCache() {
+  async waitForCache(timeoutMs = 30000) {
     if (this.isCacheReady)
       return;
-    await this.sleep(this.rnd(50, 200));
-    await this.waitForCache();
+    const start = Date.now();
+    while (!this.isCacheReady) {
+      if (Date.now() - start > timeoutMs) {
+        console.warn("[Crawler] Cache initialization timeout, continuing without cache");
+        this.isCacheReady = true;
+        return;
+      }
+      await new Promise((resolve) => setTimeout(resolve, 100));
+    }
   }
-  async waitForStorage() {
+  async waitForStorage(timeoutMs = 30000) {
     if (this.isStorageReady)
       return;
-    await this.sleep(this.rnd(50, 200));
-    await this.waitForStorage();
+    const start = Date.now();
+    while (!this.isStorageReady) {
+      if (Date.now() - start > timeoutMs) {
+        console.warn("[Crawler] Storage initialization timeout, continuing without URL tracking");
+        this.isStorageReady = true;
+        return;
+      }
+      await new Promise((resolve) => setTimeout(resolve, 100));
+    }
+  }
+  startPeriodicCleanup() {
+    if (this.cleanupInterval)
+      return;
+    this.cleanupInterval = setInterval(() => {
+      const memStatus = this.memoryMonitor.check();
+      if (memStatus === "critical") {
+        this.queue.pause();
+        this.memoryMonitor.forceGC();
+        if (this.config.debug) {
+          const report = this.memoryMonitor.getReport();
+          console.warn(`[Crawler] CRITICAL memory (${report.usagePercent}%), pausing...`);
+        }
+        setTimeout(() => {
+          this.queue.concurrency = Math.max(5, Math.floor(this.originalConcurrency * 0.25));
+          this.queue.start();
+        }, 3000);
+      } else if (memStatus === "warning") {
+        const newConcurrency = Math.max(10, Math.floor(this.originalConcurrency * 0.5));
+        if (this.queue.concurrency > newConcurrency) {
+          this.queue.concurrency = newConcurrency;
+          if (this.config.debug) {
+            const report = this.memoryMonitor.getReport();
+            console.warn(`[Crawler] High memory (${report.usagePercent}%), reducing concurrency to ${newConcurrency}`);
+          }
+        }
+      } else {
+        if (this.queue.concurrency < this.originalConcurrency) {
+          this.queue.concurrency = Math.min(this.originalConcurrency, this.queue.concurrency + 10);
+        }
+      }
+    }, 30000);
+    if (this.cleanupInterval.unref) {
+      this.cleanupInterval.unref();
+    }
+    if (this.config.enableNavigationHistory) {
+      this.startAutoCheckpoint();
+    }
+  }
+  startAutoCheckpoint() {
+    if (this.checkpointInterval)
+      return;
+    const CHECKPOINT_INTERVAL = 5 * 60 * 1000;
+    this.checkpointInterval = setInterval(async () => {
+      if (this.shutdownRequested || this.isDestroyed)
+        return;
+      try {
+        await this.saveCheckpoint();
+      } catch (error) {
+        if (this.config.debug) {
+          console.error("[Crawler] Checkpoint save failed:", error);
+        }
+      }
+    }, CHECKPOINT_INTERVAL);
+    if (this.checkpointInterval.unref) {
+      this.checkpointInterval.unref();
+    }
+  }
+  async saveCheckpoint() {
+    if (!this.navigationHistory || !this.currentSession)
+      return;
+    const now = Date.now();
+    if (now - this.lastCheckpointTime < 60000)
+      return;
+    try {
+      await this.navigationHistory.updateSessionStats(this.currentSession.sessionId, {
+        urlsVisited: this.crawlStats.urlsVisited,
+        urlsQueued: this.crawlStats.urlsQueued,
+        urlsFailed: this.crawlStats.urlsFailed
+      });
+      this.lastCheckpointTime = now;
+      if (this.config.debug) {
+        console.log(`[Crawler] Checkpoint saved: ${this.crawlStats.urlsVisited} visited, ${this.crawlStats.urlsFailed} failed`);
+      }
+    } catch (error) {
+      if (this.config.debug) {
+        console.error("[Crawler] Failed to save checkpoint:", error);
+      }
+    }
   }
   async saveUrl(url) {
     await this.waitForStorage();
@@ -696,14 +864,14 @@ export class Crawler {
     return this;
   }
   getCollectedData() {
-    return [...this.collectedData];
+    return this.collectedData.toArray();
   }
   clearCollectedData() {
-    this.collectedData = [];
+    this.collectedData.clear();
     return this;
   }
   async exportData(filePath, format = "json") {
-    const data = this.collectedData;
+    const data = this.collectedData.toArray();
     if (data.length === 0) {
       if (this.config.debug) {
         console.warn("[Crawler] No data to export");
@@ -764,20 +932,36 @@ export class Crawler {
   getStats() {
     return { ...this.crawlStats };
   }
+  getHealthSnapshot() {
+    return this.healthMetrics.getSnapshot(this.queue.size, this.queue.pending);
+  }
+  isHealthy(options) {
+    return this.healthMetrics.isHealthy(options);
+  }
+  getPrometheusMetrics(prefix = "crawler") {
+    return this.healthMetrics.toPrometheusFormat(prefix);
+  }
   async triggerStartHandlers() {
     if (this.crawlStarted)
       return;
+    if (this.startHandlersPromise) {
+      return this.startHandlersPromise;
+    }
     this.crawlStarted = true;
-    this.crawlStats.startTime = Date.now();
-    for (const handler of this.startHandlers) {
-      try {
-        this.queue.add(() => handler());
-      } catch (error) {
-        if (this.config.debug) {
-          console.error("[Crawler] onStart handler error:", error);
+    this.startHandlersPromise = (async () => {
+      this.crawlStats.startTime = Date.now();
+      this.startPeriodicCleanup();
+      for (const handler of this.startHandlers) {
+        try {
+          this.queue.add(() => handler());
+        } catch (error) {
+          if (this.config.debug) {
+            console.error("[Crawler] onStart handler error:", error);
+          }
         }
       }
-    }
+    })();
+    return this.startHandlersPromise;
   }
   async triggerFinishHandlers() {
     this.crawlStats.endTime = Date.now();
@@ -874,53 +1058,55 @@ export class Crawler {
     }
     if (url.includes(`/www.yellowpages.com/search?`))
       console.log("Visiting: ", url);
+    this.crawlStarted = true;
     if (deepEmailFinder) {
-      this.pendingVisitCount++;
-      const p = this.execute2(method, url, body, _options, forceRevisit, emailMetadata);
-      this.pendingExecutions.add(p);
-      p.finally(() => this.pendingExecutions.delete(p));
+      this.execute2(method, url, body, _options, forceRevisit, emailMetadata);
       return this;
     }
-    this.pendingVisitCount++;
-    const p = this.execute(method, url, body, _options, extractLeads, forceRevisit, oxylabsOptions, oxylabsInstanse, decodoInstanse, decodoOptions, skipCache, emailMetadata);
-    this.pendingExecutions.add(p);
-    p.finally(() => this.pendingExecutions.delete(p));
+    this.execute(method, url, body, _options, extractLeads, forceRevisit, oxylabsOptions, oxylabsInstanse, decodoInstanse, decodoOptions, skipCache, emailMetadata);
     return this;
   }
   async execute(method, url, body, options = {}, isEmail, forceRevisit, oxylabsOptions, oxylabsInstanse, decodoInstanse, decodoOptions, skipCache, emailMetadata) {
-    await this.waitForStorage();
-    if (this.isCacheEnabled) {
-      await this.waitForCache();
-    }
-    if (this.config.enableNavigationHistory) {
-      await this.waitForNavigationHistory();
-    }
-    const task = this.queue.add(() => this.executeHttp(method, url, body, options, isEmail, forceRevisit, oxylabsOptions, oxylabsInstanse, decodoInstanse, decodoOptions, 0, undefined, skipCache, emailMetadata));
-    this.pendingVisitCount--;
-    task.finally(() => this.pendingExecutions.delete(task));
+    this.queue.add(async () => {
+      await this.waitForStorage();
+      if (this.isCacheEnabled) {
+        await this.waitForCache();
+      }
+      if (this.config.enableNavigationHistory) {
+        await this.waitForNavigationHistory();
+      }
+      await this.executeHttp(method, url, body, options, isEmail, forceRevisit, oxylabsOptions, oxylabsInstanse, decodoInstanse, decodoOptions, 0, undefined, skipCache, emailMetadata);
+    }).catch((err) => {
+      if (this.config.debug)
+        console.warn("[Crawler] execute() task error:", err?.message);
+    });
   }
   async execute2(method, url, body, options = {}, forceRevisit, emailMetadata) {
-    await this.waitForStorage();
-    if (this.isCacheEnabled) {
-      await this.waitForCache();
-    }
-    if (this.config.enableNavigationHistory) {
-      await this.waitForNavigationHistory();
-    }
-    this.queue.add(() => this.leadsFinder.parseExternalWebsite(url, method, body, {
-      httpConfig: options,
-      saveCache: this.saveCache.bind(this),
-      saveUrl: this.saveUrl.bind(this),
-      getCache: this.getCache.bind(this),
-      hasUrlInCache: this.hasUrlInCache.bind(this),
-      onEmailDiscovered: this.emailDiscoveredEvents,
-      onEmails: this.emailLeadsEvents,
-      queue: this.queue,
-      depth: 1,
-      allowCrossDomainTravel: true,
-      emailMetadata
-    }, forceRevisit, true)).then();
-    this.pendingVisitCount--;
+    this.scraperQueue.add(async () => {
+      await this.waitForStorage();
+      if (this.isCacheEnabled) {
+        await this.waitForCache();
+      }
+      if (this.config.enableNavigationHistory) {
+        await this.waitForNavigationHistory();
+      }
+      await this.leadsFinder.parseExternalWebsite(url, method, body, {
+        httpConfig: options,
+        saveCache: this.saveCache.bind(this),
+        saveUrl: this.saveUrl.bind(this),
+        getCache: this.getCache.bind(this),
+        hasUrlInCache: this.hasUrlInCache.bind(this),
+        onEmailDiscovered: this.emailDiscoveredEvents,
+        onEmails: this.emailLeadsEvents,
+        queue: this.scraperQueue,
+        depth: 1,
+        allowCrossDomainTravel: true,
+        emailMetadata
+      }, forceRevisit, true);
+    }).catch((err) => {
+      if (this.config.debug)
+        console.warn("[Crawler] execute2() task error:", err?.message);
+    });
   }
   async executeHttp(method, url, body, options = {}, isEmail, forceRevisit, oxylabsOptions, oxylabsInstanse, decodoInstanse, decodoOptions, retryCount = 0, parentUrl, skipCache, emailMetadata) {
     try {
@@ -948,6 +1134,14 @@ export class Crawler {
       }
       const requestStartTime = Date.now();
       const response = cache && method === "GET" && !skipCache ? cache : oxylabsInstanse && oxylabsOptions ? await oxylabsInstanse.scrape(url) : decodoInstanse && decodoOptions ? await decodoInstanse.scrape(url) : await (method === "GET" ? this.http.get(url, options) : method === "PATCH" ? this.http.patch(url, body, options) : method === "POST" ? this.http.post(url, body, options) : this.http.put(url, body, options));
+      if (!response) {
+        this.crawlStats.urlsFailed++;
+        this.healthMetrics.recordRequest(Date.now() - requestStartTime, false);
+        if (this.config.debug) {
+          console.log(`[Crawler] Request failed for ${url}`);
+        }
+        return;
+      }
       if (!cache) {
         const responseTime = Date.now() - requestStartTime;
         this.calculateAutoThrottleDelay(domain, responseTime);
@@ -973,6 +1167,8 @@ export class Crawler {
         }
       }
       this.crawlStats.urlsVisited++;
+      const finalResponseTime = cache ? 0 : Date.now() - requestStartTime;
+      this.healthMetrics.recordRequest(finalResponseTime, true);
       if (res.finalUrl && res.finalUrl !== url && this.redirectHandlers.length > 0) {
         await this.triggerRedirectHandlers({
           originalUrl: url,
@@ -992,7 +1188,7 @@ export class Crawler {
       });
       if (res.contentType && res.contentType.includes("/json")) {
         if (this.emailDiscoveredEvents.length > 0 || this.emailLeadsEvents.length > 0) {
-          this.leadsFinder.extractEmails(JSON.stringify(res.data), res.finalUrl, this.emailDiscoveredEvents, this.emailLeadsEvents, this.queue, emailMetadata);
+          this.leadsFinder.extractEmails(JSON.stringify(res.data), res.finalUrl, this.emailDiscoveredEvents, this.emailLeadsEvents, this.scraperQueue, emailMetadata);
         }
         for (let i = 0;i < this.jsonEvents.length; i++) {
           const event = this.jsonEvents[i];
@@ -1007,7 +1203,7 @@ export class Crawler {
       if (!res.contentType || !res.contentType.includes("/html") || typeof res.data !== "string")
         return;
       if ((this.emailDiscoveredEvents.length > 0 || this.emailLeadsEvents.length > 0) && isEmail) {
-        this.leadsFinder.extractEmails(res.data, res.finalUrl, this.emailDiscoveredEvents, this.emailLeadsEvents, this.queue, emailMetadata);
+        this.leadsFinder.extractEmails(res.data, res.finalUrl, this.emailDiscoveredEvents, this.emailLeadsEvents, this.scraperQueue, emailMetadata);
       }
       const { document } = parseHTML(res.data.addBaseUrl(res.finalUrl));
       document.URL = res.finalUrl;
@@ -1072,23 +1268,33 @@ export class Crawler {
     }
   }
   async waitForAll() {
-    if (this.pendingVisitCount === 0 && this.pendingExecutions.size === 0 && this.queue.size === 0 && this.queue.pending === 0 && !this.crawlStarted) {
-      await this.triggerFinishHandlers();
-      await this.destroy();
-      return;
-    }
+    const MIN_DELAY = 50;
+    const MAX_DELAY = 500;
+    let currentDelay = MIN_DELAY;
+    let consecutiveIdleChecks = 0;
+    const REQUIRED_IDLE_CHECKS = 3;
     while (true) {
-      while (this.pendingVisitCount > 0) {
-        await new Promise((resolve) => setTimeout(resolve, 10));
-      }
-      if (this.pendingExecutions.size > 0) {
-        await Promise.allSettled([...this.pendingExecutions]);
-      }
-      if (this.queue.size > 0 || this.queue.pending > 0) {
-        await this.queue.onIdle();
-      }
-      if (this.pendingVisitCount === 0 && this.pendingExecutions.size === 0 && this.queue.size === 0 && this.queue.pending === 0) {
-        break;
+      await this.queue.onIdle();
+      await new Promise((resolve) => setTimeout(resolve, currentDelay));
+      const scraperSize = this.scraperQueue.size;
+      const scraperPending = this.scraperQueue.pending;
+      if (scraperSize === 0 && scraperPending === 0) {
+        const queueSize = this.queue.size;
+        const queuePending = this.queue.pending;
+        if (queueSize === 0 && queuePending === 0) {
+          consecutiveIdleChecks++;
+          if (consecutiveIdleChecks >= REQUIRED_IDLE_CHECKS) {
+            break;
+          }
+          currentDelay = Math.max(MIN_DELAY, currentDelay / 2);
+        } else {
+          consecutiveIdleChecks = 0;
+          currentDelay = Math.min(MAX_DELAY, currentDelay * 1.5);
+        }
+      } else {
+        consecutiveIdleChecks = 0;
+        currentDelay = Math.min(MAX_DELAY, currentDelay * 1.5);
+        await this.scraperQueue.onIdle();
       }
     }
     await this.triggerFinishHandlers();
@@ -1112,7 +1318,18 @@ export class Crawler {
     if (this.isDestroyed)
       return;
     this.isDestroyed = true;
+    this.removeShutdownHandlers();
+    if (this.cleanupInterval) {
+      clearInterval(this.cleanupInterval);
+      this.cleanupInterval = undefined;
+    }
+    if (this.checkpointInterval) {
+      clearInterval(this.checkpointInterval);
+      this.checkpointInterval = undefined;
+    }
+    this.memoryMonitor.destroy();
     this.queue.destroy();
+    this.scraperQueue.destroy();
     this.config.destroyLimiters();
     this.events.length = 0;
     this.jsonEvents.length = 0;
@@ -1121,6 +1338,10 @@ export class Crawler {
     this.rawResponseEvents.length = 0;
     this.emailDiscoveredEvents.length = 0;
     this.emailLeadsEvents.length = 0;
+    this.domainResponseTimes.clear();
+    this.domainCurrentDelay.clear();
+    this.urlDepthMap.clear();
+    this.collectedData.clear();
     await this.close();
     resetGlobalAgentPool();
   }

package/dist/crawler/index.cjs

@@ -1,40 +1,40 @@
-const _mod_qcm6x2 = require('./crawler.cjs');
-exports.Crawler = _mod_qcm6x2.Crawler;;
-const _mod_732kra = require('./crawler-options.cjs');
-exports.CrawlerOptions = _mod_732kra.CrawlerOptions;;
-const _mod_z5v0eg = require('./plugin/robots-txt.cjs');
-exports.RobotsTxt = _mod_z5v0eg.RobotsTxt;;
-const _mod_cfyi08 = require('./plugin/file-cacher.cjs');
-exports.FileCacher = _mod_cfyi08.FileCacher;;
-const _mod_noooi1 = require('./plugin/url-store.cjs');
-exports.UrlStore = _mod_noooi1.UrlStore;;
-const _mod_i5bk7z = require('./plugin/navigation-history.cjs');
-exports.NavigationHistory = _mod_i5bk7z.NavigationHistory;;
-const _mod_b7gkfk = require('./addon/oxylabs/index.cjs');
-exports.Oxylabs = _mod_b7gkfk.Oxylabs;;
-const _mod_38cltn = require('./addon/oxylabs/options.cjs');
-exports.OXYLABS_BROWSER_TYPES = _mod_38cltn.OXYLABS_BROWSER_TYPES;
-exports.OXYLABS_COMMON_LOCALES = _mod_38cltn.OXYLABS_COMMON_LOCALES;
-exports.OXYLABS_COMMON_GEO_LOCATIONS = _mod_38cltn.OXYLABS_COMMON_GEO_LOCATIONS;
-exports.OXYLABS_US_STATES = _mod_38cltn.OXYLABS_US_STATES;
-exports.OXYLABS_EUROPEAN_COUNTRIES = _mod_38cltn.OXYLABS_EUROPEAN_COUNTRIES;
-exports.OXYLABS_ASIAN_COUNTRIES = _mod_38cltn.OXYLABS_ASIAN_COUNTRIES;
-exports.getRandomOxylabsBrowserType = _mod_38cltn.getRandomBrowserType;
-exports.getRandomOxylabsLocale = _mod_38cltn.getRandomLocale;
-exports.getRandomOxylabsGeoLocation = _mod_38cltn.getRandomGeoLocation;;
-const _mod_hg233z = require('./addon/decodo/index.cjs');
-exports.Decodo = _mod_hg233z.Decodo;;
-const _mod_5tt6b2 = require('./addon/decodo/options.cjs');
-exports.DECODO_DEVICE_TYPES = _mod_5tt6b2.DECODO_DEVICE_TYPES;
-exports.DECODO_HEADLESS_MODES = _mod_5tt6b2.DECODO_HEADLESS_MODES;
-exports.DECODO_COMMON_LOCALES = _mod_5tt6b2.DECODO_COMMON_LOCALES;
-exports.DECODO_COMMON_COUNTRIES = _mod_5tt6b2.DECODO_COMMON_COUNTRIES;
-exports.DECODO_EUROPEAN_COUNTRIES = _mod_5tt6b2.DECODO_EUROPEAN_COUNTRIES;
-exports.DECODO_ASIAN_COUNTRIES = _mod_5tt6b2.DECODO_ASIAN_COUNTRIES;
-exports.DECODO_US_STATES = _mod_5tt6b2.DECODO_US_STATES;
-exports.DECODO_COMMON_CITIES = _mod_5tt6b2.DECODO_COMMON_CITIES;
-exports.getRandomDecodoDeviceType = _mod_5tt6b2.getRandomDeviceType;
-exports.getRandomDecodoLocale = _mod_5tt6b2.getRandomLocale;
-exports.getRandomDecodoCountry = _mod_5tt6b2.getRandomCountry;
-exports.getRandomDecodoCity = _mod_5tt6b2.getRandomCity;
-exports.generateDecodoSessionId = _mod_5tt6b2.generateSessionId;;
+const _mod_7qsaj1 = require('./crawler.cjs');
+exports.Crawler = _mod_7qsaj1.Crawler;;
+const _mod_5rlu0e = require('./crawler-options.cjs');
+exports.CrawlerOptions = _mod_5rlu0e.CrawlerOptions;;
+const _mod_skia2o = require('./plugin/robots-txt.cjs');
+exports.RobotsTxt = _mod_skia2o.RobotsTxt;;
+const _mod_w2aaxb = require('./plugin/file-cacher.cjs');
+exports.FileCacher = _mod_w2aaxb.FileCacher;;
+const _mod_f9de8f = require('./plugin/url-store.cjs');
+exports.UrlStore = _mod_f9de8f.UrlStore;;
+const _mod_wqtwlm = require('./plugin/navigation-history.cjs');
+exports.NavigationHistory = _mod_wqtwlm.NavigationHistory;;
+const _mod_hj9vi3 = require('./addon/oxylabs/index.cjs');
+exports.Oxylabs = _mod_hj9vi3.Oxylabs;;
+const _mod_7kzws5 = require('./addon/oxylabs/options.cjs');
+exports.OXYLABS_BROWSER_TYPES = _mod_7kzws5.OXYLABS_BROWSER_TYPES;
+exports.OXYLABS_COMMON_LOCALES = _mod_7kzws5.OXYLABS_COMMON_LOCALES;
+exports.OXYLABS_COMMON_GEO_LOCATIONS = _mod_7kzws5.OXYLABS_COMMON_GEO_LOCATIONS;
+exports.OXYLABS_US_STATES = _mod_7kzws5.OXYLABS_US_STATES;
+exports.OXYLABS_EUROPEAN_COUNTRIES = _mod_7kzws5.OXYLABS_EUROPEAN_COUNTRIES;
+exports.OXYLABS_ASIAN_COUNTRIES = _mod_7kzws5.OXYLABS_ASIAN_COUNTRIES;
+exports.getRandomOxylabsBrowserType = _mod_7kzws5.getRandomBrowserType;
+exports.getRandomOxylabsLocale = _mod_7kzws5.getRandomLocale;
+exports.getRandomOxylabsGeoLocation = _mod_7kzws5.getRandomGeoLocation;;
+const _mod_gdeyg9 = require('./addon/decodo/index.cjs');
+exports.Decodo = _mod_gdeyg9.Decodo;;
+const _mod_jbu0zi = require('./addon/decodo/options.cjs');
+exports.DECODO_DEVICE_TYPES = _mod_jbu0zi.DECODO_DEVICE_TYPES;
+exports.DECODO_HEADLESS_MODES = _mod_jbu0zi.DECODO_HEADLESS_MODES;
+exports.DECODO_COMMON_LOCALES = _mod_jbu0zi.DECODO_COMMON_LOCALES;
+exports.DECODO_COMMON_COUNTRIES = _mod_jbu0zi.DECODO_COMMON_COUNTRIES;
+exports.DECODO_EUROPEAN_COUNTRIES = _mod_jbu0zi.DECODO_EUROPEAN_COUNTRIES;
+exports.DECODO_ASIAN_COUNTRIES = _mod_jbu0zi.DECODO_ASIAN_COUNTRIES;
+exports.DECODO_US_STATES = _mod_jbu0zi.DECODO_US_STATES;
+exports.DECODO_COMMON_CITIES = _mod_jbu0zi.DECODO_COMMON_CITIES;
+exports.getRandomDecodoDeviceType = _mod_jbu0zi.getRandomDeviceType;
+exports.getRandomDecodoLocale = _mod_jbu0zi.getRandomLocale;
+exports.getRandomDecodoCountry = _mod_jbu0zi.getRandomCountry;
+exports.getRandomDecodoCity = _mod_jbu0zi.getRandomCity;
+exports.generateDecodoSessionId = _mod_jbu0zi.generateSessionId;;

package/dist/crawler/plugin/capped-array.cjs

@@ -0,0 +1 @@
+class i{items=[];maxSize;evictionCount;onEviction;constructor(t={}){this.maxSize=t.maxSize??1e5;let e=t.evictionRatio??0.1;this.evictionCount=Math.max(1,Math.ceil(this.maxSize*e)),this.onEviction=t.onEviction}push(t){if(this.items.length>=this.maxSize)this.evict();return this.items.push(t)}pushMany(t){for(let e of t)this.push(e);return this.items.length}get(t){return this.items[t]}get length(){return this.items.length}clear(){this.items.length=0}toArray(){return[...this.items]}[Symbol.iterator](){return this.items[Symbol.iterator]()}forEach(t){this.items.forEach(t)}map(t){return this.items.map(t)}filter(t){return this.items.filter(t)}evict(){let t=this.items.splice(0,this.evictionCount);if(this.onEviction&&t.length>0)this.onEviction(t,this.items.length)}get maxCapacity(){return this.maxSize}get isAtCapacity(){return this.items.length>=this.maxSize}}exports.CappedArray=i;exports.default=i;module.exports=Object.assign(i,exports);

package/dist/crawler/plugin/capped-array.js

@@ -0,0 +1 @@
+class e{items=[];maxSize;evictionCount;onEviction;constructor(t={}){this.maxSize=t.maxSize??1e5;let i=t.evictionRatio??0.1;this.evictionCount=Math.max(1,Math.ceil(this.maxSize*i)),this.onEviction=t.onEviction}push(t){if(this.items.length>=this.maxSize)this.evict();return this.items.push(t)}pushMany(t){for(let i of t)this.push(i);return this.items.length}get(t){return this.items[t]}get length(){return this.items.length}clear(){this.items.length=0}toArray(){return[...this.items]}[Symbol.iterator](){return this.items[Symbol.iterator]()}forEach(t){this.items.forEach(t)}map(t){return this.items.map(t)}filter(t){return this.items.filter(t)}evict(){let t=this.items.splice(0,this.evictionCount);if(this.onEviction&&t.length>0)this.onEviction(t,this.items.length)}get maxCapacity(){return this.maxSize}get isAtCapacity(){return this.items.length>=this.maxSize}}var h=e;export{h as default,e as CappedArray};

package/dist/crawler/plugin/capped-map.cjs

@@ -0,0 +1 @@
+class i{map;maxSize;evictionCount;constructor(t={}){this.maxSize=t.maxSize??1e4,this.evictionRatio=t.evictionRatio??0.1,this.evictionCount=Math.max(1,Math.ceil(this.maxSize*this.evictionRatio)),this.map=new Map}evictionRatio;set(t,e){if(this.map.has(t))this.map.delete(t);else if(this.map.size>=this.maxSize)this.evict();return this.map.set(t,e),this}get(t){return this.map.get(t)}getAndTouch(t){let e=this.map.get(t);if(e!==void 0)this.map.delete(t),this.map.set(t,e);return e}has(t){return this.map.has(t)}delete(t){return this.map.delete(t)}clear(){this.map.clear()}get size(){return this.map.size}entries(){return this.map.entries()}keys(){return this.map.keys()}values(){return this.map.values()}forEach(t){this.map.forEach(t)}evict(){let t=this.map.keys();for(let e=0;e<this.evictionCount;e++){let s=t.next();if(s.done)break;this.map.delete(s.value)}}toMap(){return new Map(this.map)}}exports.CappedMap=i;exports.default=i;module.exports=Object.assign(i,exports);

package/dist/crawler/plugin/capped-map.js

@@ -0,0 +1 @@
+class s{map;maxSize;evictionCount;constructor(t={}){this.maxSize=t.maxSize??1e4,this.evictionRatio=t.evictionRatio??0.1,this.evictionCount=Math.max(1,Math.ceil(this.maxSize*this.evictionRatio)),this.map=new Map}evictionRatio;set(t,e){if(this.map.has(t))this.map.delete(t);else if(this.map.size>=this.maxSize)this.evict();return this.map.set(t,e),this}get(t){return this.map.get(t)}getAndTouch(t){let e=this.map.get(t);if(e!==void 0)this.map.delete(t),this.map.set(t,e);return e}has(t){return this.map.has(t)}delete(t){return this.map.delete(t)}clear(){this.map.clear()}get size(){return this.map.size}entries(){return this.map.entries()}keys(){return this.map.keys()}values(){return this.map.values()}forEach(t){this.map.forEach(t)}evict(){let t=this.map.keys();for(let e=0;e<this.evictionCount;e++){let i=t.next();if(i.done)break;this.map.delete(i.value)}}toMap(){return new Map(this.map)}}var h=s;export{h as default,s as CappedMap};