rezo 1.0.72 → 1.0.73

package/dist/crawler/crawler.cjs

@@ -3,6 +3,10 @@ const { FileCacher } = require('./plugin/file-cacher.cjs');
 const { UrlStore } = require('./plugin/url-store.cjs');
 const { NavigationHistory } = require('./plugin/navigation-history.cjs');
 const { RobotsTxt } = require('./plugin/robots-txt.cjs');
+const { MemoryMonitor } = require('./plugin/memory-monitor.cjs');
+const { HealthMetrics } = require('./plugin/health-metrics.cjs');
+const { CappedMap } = require('./plugin/capped-map.cjs');
+const { CappedArray } = require('./plugin/capped-array.cjs');
 const { parseHTML } = require("linkedom");
 const path = require("node:path");
 const { Rezo } = require('../adapters/entries/http.cjs');
@@ -43,6 +47,7 @@ class Crawler {
   emailLeadsEvents = [];
   cacher = null;
   queue;
+  scraperQueue;
   isCacheEnabled;
   config;
   urlStorage;
@@ -56,13 +61,12 @@ class Crawler {
   navigationHistoryInitPromise = null;
   adapterExecutor = null;
   adapterType;
-  pendingExecutions = new Set;
-  pendingVisitCount = 0;
   isDestroyed = false;
-  queueOptions = { concurrency: 1000 };
+  shutdownRequested = false;
+  queueOptions = { concurrency: 100 };
   robotsTxt;
-  domainResponseTimes = new Map;
-  domainCurrentDelay = new Map;
+  domainResponseTimes = new CappedMap({ maxSize: 500 });
+  domainCurrentDelay = new CappedMap({ maxSize: 500 });
   crawlStats = {
     urlsVisited: 0,
     urlsQueued: 0,
@@ -70,19 +74,42 @@ class Crawler {
     startTime: 0,
     currentDepth: 0
   };
-  urlDepthMap = new Map;
+  urlDepthMap = new CappedMap({ maxSize: 50000 });
+  cleanupInterval;
+  checkpointInterval;
+  lastCheckpointTime = 0;
+  memoryMonitor;
+  healthMetrics;
+  originalConcurrency = 100;
+  shutdownHandler = null;
   startHandlers = [];
   finishHandlers = [];
   redirectHandlers = [];
-  collectedData = [];
+  collectedData = new CappedArray({
+    maxSize: 1e5,
+    evictionRatio: 0.1,
+    onEviction: (evicted, remaining) => {
+      console.warn(`[Crawler] collectedData auto-evicted ${evicted.length} oldest entries. ${remaining} entries remaining. Consider using exportData() more frequently.`);
+    }
+  });
   crawlStarted = false;
+  startHandlersPromise = null;
   constructor(crawlerOptions, http = new Rezo) {
     this.http = http;
-    this.queue = new RezoQueue({
-      concurrency: 1000
-    });
     this.config = new CrawlerOptions(crawlerOptions);
     this.adapterType = this.config.adapter;
+    const concurrency = this.config.concurrency;
+    this.queue = new RezoQueue({
+      concurrency,
+      timeout: 60000
+    });
+    this.originalConcurrency = concurrency;
+    this.scraperQueue = new RezoQueue({
+      concurrency: this.config.scraperConcurrency,
+      timeout: 60000
+    });
+    this.memoryMonitor = new MemoryMonitor({ warningRatio: 0.7, criticalRatio: 0.85 });
+    this.healthMetrics = new HealthMetrics({ windowSize: 60000 });
     const enableCache = this.config.enableCache;
     this.isCacheEnabled = enableCache;
     if (enableCache) {
@@ -94,7 +121,7 @@ class Crawler {
       FileCacher.create({
         cacheDir: dbUrl,
         ttl: cacheTTL,
-        encryptNamespace: true
+        maxEntries: 1e5
       }).then((storage) => {
         this.cacher = storage;
         this.isCacheReady = true;
@@ -151,6 +178,55 @@ class Crawler {
     if (this.config.baseUrl) {
       this.urlDepthMap.set(this.config.baseUrl, 0);
     }
+    this.registerShutdownHandlers();
+  }
+  registerShutdownHandlers() {
+    if (this.shutdownHandler)
+      return;
+    this.shutdownHandler = () => this.gracefulShutdown();
+    process.on("SIGINT", this.shutdownHandler);
+    process.on("SIGTERM", this.shutdownHandler);
+  }
+  removeShutdownHandlers() {
+    if (this.shutdownHandler) {
+      process.off("SIGINT", this.shutdownHandler);
+      process.off("SIGTERM", this.shutdownHandler);
+      this.shutdownHandler = null;
+    }
+  }
+  async gracefulShutdown() {
+    if (this.shutdownRequested || this.isDestroyed)
+      return;
+    this.shutdownRequested = true;
+    console.log(`
+[Crawler] Shutdown requested, finishing current tasks...`);
+    this.queue.pause();
+    this.scraperQueue.pause();
+    const timeoutPromise = new Promise((resolve) => {
+      setTimeout(() => {
+        console.log("[Crawler] Shutdown timeout (5s), forcing exit...");
+        resolve();
+      }, 5000);
+    });
+    await Promise.race([
+      Promise.all([this.queue.onIdle(), this.scraperQueue.onIdle()]),
+      timeoutPromise
+    ]);
+    if (this.navigationHistory && this.currentSession) {
+      try {
+        await this.navigationHistory.updateSessionStatus(this.currentSession.sessionId, "paused");
+        await this.navigationHistory.updateSessionStats(this.currentSession.sessionId, {
+          urlsVisited: this.crawlStats.urlsVisited,
+          urlsQueued: this.queue.size,
+          urlsFailed: this.crawlStats.urlsFailed
+        });
+        console.log(`[Crawler] Session saved: ${this.currentSession.sessionId}`);
+      } catch (err) {
+        console.warn("[Crawler] Failed to save session state:", err);
+      }
+    }
+    await this.destroy();
+    console.log("[Crawler] Graceful shutdown complete");
   }
   async initializeAdapter() {
     try {
@@ -166,7 +242,6 @@ class Crawler {
     if (!this.isDestroyed)
       return;
     this.queue = new RezoQueue(this.queueOptions);
-    this.pendingExecutions.clear();
     this.isDestroyed = false;
     if (this.config.debug) {
       console.log("[Crawler] Restored from destroyed state");
@@ -342,17 +417,110 @@ class Crawler {
       handler(data);
     });
   }
-  async waitForCache() {
+  async waitForCache(timeoutMs = 30000) {
     if (this.isCacheReady)
       return;
-    await this.sleep(this.rnd(50, 200));
-    await this.waitForCache();
+    const start = Date.now();
+    while (!this.isCacheReady) {
+      if (Date.now() - start > timeoutMs) {
+        console.warn("[Crawler] Cache initialization timeout, continuing without cache");
+        this.isCacheReady = true;
+        return;
+      }
+      await new Promise((resolve) => setTimeout(resolve, 100));
+    }
   }
-  async waitForStorage() {
+  async waitForStorage(timeoutMs = 30000) {
     if (this.isStorageReady)
       return;
-    await this.sleep(this.rnd(50, 200));
-    await this.waitForStorage();
+    const start = Date.now();
+    while (!this.isStorageReady) {
+      if (Date.now() - start > timeoutMs) {
+        console.warn("[Crawler] Storage initialization timeout, continuing without URL tracking");
+        this.isStorageReady = true;
+        return;
+      }
+      await new Promise((resolve) => setTimeout(resolve, 100));
+    }
+  }
+  startPeriodicCleanup() {
+    if (this.cleanupInterval)
+      return;
+    this.cleanupInterval = setInterval(() => {
+      const memStatus = this.memoryMonitor.check();
+      if (memStatus === "critical") {
+        this.queue.pause();
+        this.memoryMonitor.forceGC();
+        if (this.config.debug) {
+          const report = this.memoryMonitor.getReport();
+          console.warn(`[Crawler] CRITICAL memory (${report.usagePercent}%), pausing...`);
+        }
+        setTimeout(() => {
+          this.queue.concurrency = Math.max(5, Math.floor(this.originalConcurrency * 0.25));
+          this.queue.start();
+        }, 3000);
+      } else if (memStatus === "warning") {
+        const newConcurrency = Math.max(10, Math.floor(this.originalConcurrency * 0.5));
+        if (this.queue.concurrency > newConcurrency) {
+          this.queue.concurrency = newConcurrency;
+          if (this.config.debug) {
+            const report = this.memoryMonitor.getReport();
+            console.warn(`[Crawler] High memory (${report.usagePercent}%), reducing concurrency to ${newConcurrency}`);
+          }
+        }
+      } else {
+        if (this.queue.concurrency < this.originalConcurrency) {
+          this.queue.concurrency = Math.min(this.originalConcurrency, this.queue.concurrency + 10);
+        }
+      }
+    }, 30000);
+    if (this.cleanupInterval.unref) {
+      this.cleanupInterval.unref();
+    }
+    if (this.config.enableNavigationHistory) {
+      this.startAutoCheckpoint();
+    }
+  }
+  startAutoCheckpoint() {
+    if (this.checkpointInterval)
+      return;
+    const CHECKPOINT_INTERVAL = 5 * 60 * 1000;
+    this.checkpointInterval = setInterval(async () => {
+      if (this.shutdownRequested || this.isDestroyed)
+        return;
+      try {
+        await this.saveCheckpoint();
+      } catch (error) {
+        if (this.config.debug) {
+          console.error("[Crawler] Checkpoint save failed:", error);
+        }
+      }
+    }, CHECKPOINT_INTERVAL);
+    if (this.checkpointInterval.unref) {
+      this.checkpointInterval.unref();
+    }
+  }
+  async saveCheckpoint() {
+    if (!this.navigationHistory || !this.currentSession)
+      return;
+    const now = Date.now();
+    if (now - this.lastCheckpointTime < 60000)
+      return;
+    try {
+      await this.navigationHistory.updateSessionStats(this.currentSession.sessionId, {
+        urlsVisited: this.crawlStats.urlsVisited,
+        urlsQueued: this.crawlStats.urlsQueued,
+        urlsFailed: this.crawlStats.urlsFailed
+      });
+      this.lastCheckpointTime = now;
+      if (this.config.debug) {
+        console.log(`[Crawler] Checkpoint saved: ${this.crawlStats.urlsVisited} visited, ${this.crawlStats.urlsFailed} failed`);
+      }
+    } catch (error) {
+      if (this.config.debug) {
+        console.error("[Crawler] Failed to save checkpoint:", error);
+      }
+    }
   }
   async saveUrl(url) {
     await this.waitForStorage();
@@ -696,14 +864,14 @@ class Crawler {
     return this;
   }
   getCollectedData() {
-    return [...this.collectedData];
+    return this.collectedData.toArray();
   }
   clearCollectedData() {
-    this.collectedData = [];
+    this.collectedData.clear();
     return this;
   }
   async exportData(filePath, format = "json") {
-    const data = this.collectedData;
+    const data = this.collectedData.toArray();
     if (data.length === 0) {
       if (this.config.debug) {
         console.warn("[Crawler] No data to export");
@@ -764,20 +932,36 @@ class Crawler {
   getStats() {
     return { ...this.crawlStats };
   }
+  getHealthSnapshot() {
+    return this.healthMetrics.getSnapshot(this.queue.size, this.queue.pending);
+  }
+  isHealthy(options) {
+    return this.healthMetrics.isHealthy(options);
+  }
+  getPrometheusMetrics(prefix = "crawler") {
+    return this.healthMetrics.toPrometheusFormat(prefix);
+  }
   async triggerStartHandlers() {
     if (this.crawlStarted)
       return;
+    if (this.startHandlersPromise) {
+      return this.startHandlersPromise;
+    }
     this.crawlStarted = true;
-    this.crawlStats.startTime = Date.now();
-    for (const handler of this.startHandlers) {
-      try {
-        this.queue.add(() => handler());
-      } catch (error) {
-        if (this.config.debug) {
-          console.error("[Crawler] onStart handler error:", error);
+    this.startHandlersPromise = (async () => {
+      this.crawlStats.startTime = Date.now();
+      this.startPeriodicCleanup();
+      for (const handler of this.startHandlers) {
+        try {
+          this.queue.add(() => handler());
+        } catch (error) {
+          if (this.config.debug) {
+            console.error("[Crawler] onStart handler error:", error);
+          }
         }
       }
-    }
+    })();
+    return this.startHandlersPromise;
   }
   async triggerFinishHandlers() {
     this.crawlStats.endTime = Date.now();
@@ -874,53 +1058,55 @@ class Crawler {
     }
     if (url.includes(`/www.yellowpages.com/search?`))
       console.log("Visiting: ", url);
+    this.crawlStarted = true;
     if (deepEmailFinder) {
-      this.pendingVisitCount++;
-      const p = this.execute2(method, url, body, _options, forceRevisit, emailMetadata);
-      this.pendingExecutions.add(p);
-      p.finally(() => this.pendingExecutions.delete(p));
+      this.execute2(method, url, body, _options, forceRevisit, emailMetadata);
       return this;
     }
-    this.pendingVisitCount++;
-    const p = this.execute(method, url, body, _options, extractLeads, forceRevisit, oxylabsOptions, oxylabsInstanse, decodoInstanse, decodoOptions, skipCache, emailMetadata);
-    this.pendingExecutions.add(p);
-    p.finally(() => this.pendingExecutions.delete(p));
+    this.execute(method, url, body, _options, extractLeads, forceRevisit, oxylabsOptions, oxylabsInstanse, decodoInstanse, decodoOptions, skipCache, emailMetadata);
     return this;
   }
   async execute(method, url, body, options = {}, isEmail, forceRevisit, oxylabsOptions, oxylabsInstanse, decodoInstanse, decodoOptions, skipCache, emailMetadata) {
-    await this.waitForStorage();
-    if (this.isCacheEnabled) {
-      await this.waitForCache();
-    }
-    if (this.config.enableNavigationHistory) {
-      await this.waitForNavigationHistory();
-    }
-    const task = this.queue.add(() => this.executeHttp(method, url, body, options, isEmail, forceRevisit, oxylabsOptions, oxylabsInstanse, decodoInstanse, decodoOptions, 0, undefined, skipCache, emailMetadata));
-    this.pendingVisitCount--;
-    task.finally(() => this.pendingExecutions.delete(task));
+    this.queue.add(async () => {
+      await this.waitForStorage();
+      if (this.isCacheEnabled) {
+        await this.waitForCache();
+      }
+      if (this.config.enableNavigationHistory) {
+        await this.waitForNavigationHistory();
+      }
+      await this.executeHttp(method, url, body, options, isEmail, forceRevisit, oxylabsOptions, oxylabsInstanse, decodoInstanse, decodoOptions, 0, undefined, skipCache, emailMetadata);
+    }).catch((err) => {
+      if (this.config.debug)
+        console.warn("[Crawler] execute() task error:", err?.message);
+    });
   }
   async execute2(method, url, body, options = {}, forceRevisit, emailMetadata) {
-    await this.waitForStorage();
-    if (this.isCacheEnabled) {
-      await this.waitForCache();
-    }
-    if (this.config.enableNavigationHistory) {
-      await this.waitForNavigationHistory();
-    }
-    this.queue.add(() => this.leadsFinder.parseExternalWebsite(url, method, body, {
-      httpConfig: options,
-      saveCache: this.saveCache.bind(this),
-      saveUrl: this.saveUrl.bind(this),
-      getCache: this.getCache.bind(this),
-      hasUrlInCache: this.hasUrlInCache.bind(this),
-      onEmailDiscovered: this.emailDiscoveredEvents,
-      onEmails: this.emailLeadsEvents,
-      queue: this.queue,
-      depth: 1,
-      allowCrossDomainTravel: true,
-      emailMetadata
-    }, forceRevisit, true)).then();
-    this.pendingVisitCount--;
+    this.scraperQueue.add(async () => {
+      await this.waitForStorage();
+      if (this.isCacheEnabled) {
+        await this.waitForCache();
+      }
+      if (this.config.enableNavigationHistory) {
+        await this.waitForNavigationHistory();
+      }
+      await this.leadsFinder.parseExternalWebsite(url, method, body, {
+        httpConfig: options,
+        saveCache: this.saveCache.bind(this),
+        saveUrl: this.saveUrl.bind(this),
+        getCache: this.getCache.bind(this),
+        hasUrlInCache: this.hasUrlInCache.bind(this),
+        onEmailDiscovered: this.emailDiscoveredEvents,
+        onEmails: this.emailLeadsEvents,
+        queue: this.scraperQueue,
+        depth: 1,
+        allowCrossDomainTravel: true,
+        emailMetadata
+      }, forceRevisit, true);
+    }).catch((err) => {
+      if (this.config.debug)
+        console.warn("[Crawler] execute2() task error:", err?.message);
+    });
   }
   async executeHttp(method, url, body, options = {}, isEmail, forceRevisit, oxylabsOptions, oxylabsInstanse, decodoInstanse, decodoOptions, retryCount = 0, parentUrl, skipCache, emailMetadata) {
     try {
@@ -948,6 +1134,14 @@ class Crawler {
       }
       const requestStartTime = Date.now();
       const response = cache && method === "GET" && !skipCache ? cache : oxylabsInstanse && oxylabsOptions ? await oxylabsInstanse.scrape(url) : decodoInstanse && decodoOptions ? await decodoInstanse.scrape(url) : await (method === "GET" ? this.http.get(url, options) : method === "PATCH" ? this.http.patch(url, body, options) : method === "POST" ? this.http.post(url, body, options) : this.http.put(url, body, options));
+      if (!response) {
+        this.crawlStats.urlsFailed++;
+        this.healthMetrics.recordRequest(Date.now() - requestStartTime, false);
+        if (this.config.debug) {
+          console.log(`[Crawler] Request failed for ${url}`);
+        }
+        return;
+      }
       if (!cache) {
         const responseTime = Date.now() - requestStartTime;
         this.calculateAutoThrottleDelay(domain, responseTime);
@@ -973,6 +1167,8 @@ class Crawler {
         }
       }
       this.crawlStats.urlsVisited++;
+      const finalResponseTime = cache ? 0 : Date.now() - requestStartTime;
+      this.healthMetrics.recordRequest(finalResponseTime, true);
       if (res.finalUrl && res.finalUrl !== url && this.redirectHandlers.length > 0) {
         await this.triggerRedirectHandlers({
           originalUrl: url,
@@ -992,7 +1188,7 @@ class Crawler {
       });
       if (res.contentType && res.contentType.includes("/json")) {
         if (this.emailDiscoveredEvents.length > 0 || this.emailLeadsEvents.length > 0) {
-          this.leadsFinder.extractEmails(JSON.stringify(res.data), res.finalUrl, this.emailDiscoveredEvents, this.emailLeadsEvents, this.queue, emailMetadata);
+          this.leadsFinder.extractEmails(JSON.stringify(res.data), res.finalUrl, this.emailDiscoveredEvents, this.emailLeadsEvents, this.scraperQueue, emailMetadata);
         }
         for (let i = 0;i < this.jsonEvents.length; i++) {
           const event = this.jsonEvents[i];
@@ -1007,7 +1203,7 @@ class Crawler {
       if (!res.contentType || !res.contentType.includes("/html") || typeof res.data !== "string")
         return;
       if ((this.emailDiscoveredEvents.length > 0 || this.emailLeadsEvents.length > 0) && isEmail) {
-        this.leadsFinder.extractEmails(res.data, res.finalUrl, this.emailDiscoveredEvents, this.emailLeadsEvents, this.queue, emailMetadata);
+        this.leadsFinder.extractEmails(res.data, res.finalUrl, this.emailDiscoveredEvents, this.emailLeadsEvents, this.scraperQueue, emailMetadata);
       }
       const { document } = parseHTML(res.data.addBaseUrl(res.finalUrl));
       document.URL = res.finalUrl;
@@ -1072,23 +1268,33 @@ class Crawler {
     }
   }
   async waitForAll() {
-    if (this.pendingVisitCount === 0 && this.pendingExecutions.size === 0 && this.queue.size === 0 && this.queue.pending === 0 && !this.crawlStarted) {
-      await this.triggerFinishHandlers();
-      await this.destroy();
-      return;
-    }
+    const MIN_DELAY = 50;
+    const MAX_DELAY = 500;
+    let currentDelay = MIN_DELAY;
+    let consecutiveIdleChecks = 0;
+    const REQUIRED_IDLE_CHECKS = 3;
     while (true) {
-      while (this.pendingVisitCount > 0) {
-        await new Promise((resolve) => setTimeout(resolve, 10));
-      }
-      if (this.pendingExecutions.size > 0) {
-        await Promise.allSettled([...this.pendingExecutions]);
-      }
-      if (this.queue.size > 0 || this.queue.pending > 0) {
-        await this.queue.onIdle();
-      }
-      if (this.pendingVisitCount === 0 && this.pendingExecutions.size === 0 && this.queue.size === 0 && this.queue.pending === 0) {
-        break;
+      await this.queue.onIdle();
+      await new Promise((resolve) => setTimeout(resolve, currentDelay));
+      const scraperSize = this.scraperQueue.size;
+      const scraperPending = this.scraperQueue.pending;
+      if (scraperSize === 0 && scraperPending === 0) {
+        const queueSize = this.queue.size;
+        const queuePending = this.queue.pending;
+        if (queueSize === 0 && queuePending === 0) {
+          consecutiveIdleChecks++;
+          if (consecutiveIdleChecks >= REQUIRED_IDLE_CHECKS) {
+            break;
+          }
+          currentDelay = Math.max(MIN_DELAY, currentDelay / 2);
+        } else {
+          consecutiveIdleChecks = 0;
+          currentDelay = Math.min(MAX_DELAY, currentDelay * 1.5);
+        }
+      } else {
+        consecutiveIdleChecks = 0;
+        currentDelay = Math.min(MAX_DELAY, currentDelay * 1.5);
+        await this.scraperQueue.onIdle();
       }
     }
     await this.triggerFinishHandlers();
@@ -1112,7 +1318,18 @@ class Crawler {
     if (this.isDestroyed)
       return;
     this.isDestroyed = true;
+    this.removeShutdownHandlers();
+    if (this.cleanupInterval) {
+      clearInterval(this.cleanupInterval);
+      this.cleanupInterval = undefined;
+    }
+    if (this.checkpointInterval) {
+      clearInterval(this.checkpointInterval);
+      this.checkpointInterval = undefined;
+    }
+    this.memoryMonitor.destroy();
     this.queue.destroy();
+    this.scraperQueue.destroy();
     this.config.destroyLimiters();
     this.events.length = 0;
     this.jsonEvents.length = 0;
@@ -1121,6 +1338,10 @@ class Crawler {
     this.rawResponseEvents.length = 0;
     this.emailDiscoveredEvents.length = 0;
     this.emailLeadsEvents.length = 0;
+    this.domainResponseTimes.clear();
+    this.domainCurrentDelay.clear();
+    this.urlDepthMap.clear();
+    this.collectedData.clear();
     await this.close();
     resetGlobalAgentPool();
   }