rezo 1.0.43 → 1.0.44

package/dist/{plugin → crawler}/crawler.cjs

@@ -1,7 +1,8 @@
 const fs = require("node:fs");
-const { FileCacher } = require('../cache/file-cacher.cjs');
-const { UrlStore } = require('../cache/url-store.cjs');
-const { NavigationHistory } = require('../cache/navigation-history.cjs');
+const { FileCacher } = require('./plugin/file-cacher.cjs');
+const { UrlStore } = require('./plugin/url-store.cjs');
+const { NavigationHistory } = require('./plugin/navigation-history.cjs');
+const { RobotsTxt } = require('./plugin/robots-txt.cjs');
 const { parseHTML } = require("linkedom");
 const path = require("node:path");
 const { Rezo } = require('../core/rezo.cjs');
@@ -54,6 +55,23 @@ class Crawler {
   navigationHistoryInitPromise = null;
   adapterExecutor = null;
   adapterType;
+  pendingExecutions = new Set;
+  robotsTxt;
+  domainResponseTimes = new Map;
+  domainCurrentDelay = new Map;
+  crawlStats = {
+    urlsVisited: 0,
+    urlsQueued: 0,
+    urlsFailed: 0,
+    startTime: 0,
+    currentDepth: 0
+  };
+  urlDepthMap = new Map;
+  startHandlers = [];
+  finishHandlers = [];
+  redirectHandlers = [];
+  collectedData = [];
+  crawlStarted = false;
   constructor(crawlerOptions, http = new Rezo) {
     this.http = http;
     this.queue = new RezoQueue({
@@ -71,7 +89,6 @@ class Crawler {
         fs.mkdirSync(path.dirname(dbUrl), { recursive: true });
       FileCacher.create({
         cacheDir: dbUrl,
-        softDelete: false,
         ttl: cacheTTL,
         encryptNamespace: true
       }).then((storage) => {
@@ -110,6 +127,14 @@ class Crawler {
     }
     this.initializeAdapter();
     this.leadsFinder = new Scraper(this.http, this.config, this._onEmailLeads.bind(this), this._onEmailDiscovered.bind(this), this.config.debug);
+    this.robotsTxt = new RobotsTxt({
+      userAgent: this.config.userAgent || "RezoBot",
+      cacheTTL: 24 * 60 * 60 * 1000
+    });
+    this.crawlStats.startTime = Date.now();
+    if (this.config.baseUrl) {
+      this.urlDepthMap.set(this.config.baseUrl, 0);
+    }
   }
   async initializeAdapter() {
     try {
@@ -148,6 +173,8 @@ class Crawler {
   async waitForNavigationHistory() {
     if (!this.config.enableNavigationHistory)
       return;
+    if (this.isNavigationHistoryReady && this.isSessionReady)
+      return;
     if (this.navigationHistoryInitPromise) {
       await this.navigationHistoryInitPromise;
     }
@@ -361,6 +388,18 @@ class Crawler {
     this.emailLeadsEvents.push(handler);
     return this;
   }
+  onStart(handler) {
+    this.startHandlers.push(handler);
+    return this;
+  }
+  onFinish(handler) {
+    this.finishHandlers.push(handler);
+    return this;
+  }
+  onRedirect(handler) {
+    this.redirectHandlers.push(handler);
+    return this;
+  }
   onRawData(handler) {
     this.rawResponseEvents.push({
       handler: "_onRawResponse",
@@ -435,20 +474,25 @@ class Crawler {
     this.queue.add(() => handler(document.body));
   }
   _onAttribute(selection, attribute, handler, document) {
-    selection = typeof attribute === "function" ? selection : null;
-    attribute = typeof attribute === "function" ? selection : attribute;
-    handler = typeof attribute === "function" ? attribute : handler;
-    selection = selection || `[${attribute}]`;
-    const elements = document.querySelectorAll(selection);
+    const isSimpleForm = typeof attribute === "function";
+    const actualAttribute = isSimpleForm ? selection : attribute;
+    const actualHandler = isSimpleForm ? attribute : handler;
+    const actualSelection = isSimpleForm ? `[${selection}]` : selection || `[${attribute}]`;
+    const elements = document.querySelectorAll(actualSelection);
     for (let i = 0;i < elements.length; i++) {
-      if (elements[i].hasAttribute(attribute))
-        this.queue.add(() => handler(elements[i].getAttribute(attribute)));
+      const el = elements[i];
+      if (el.hasAttribute(actualAttribute)) {
+        const value = el.getAttribute(actualAttribute);
+        this.queue.add(() => actualHandler.call(el, value, actualAttribute));
+      }
     }
   }
   _onText(selection, handler, document) {
     const elements = document.querySelectorAll(selection);
     for (let i = 0;i < elements.length; i++) {
-      this.queue.add(() => handler(elements[i].textContent));
+      const el = elements[i];
+      const text = el.textContent;
+      this.queue.add(() => handler.call(el, text));
     }
   }
   _onSelection(selection, handler, document) {
@@ -466,8 +510,11 @@ class Crawler {
   _onHref(handler, document) {
     const elements = document.querySelectorAll("a, link");
     for (let i = 0;i < elements.length; i++) {
-      if (elements[i].hasAttribute("href"))
-        this.queue.add(() => handler(new URL(elements[i].getAttribute("href"), document.URL).href));
+      const el = elements[i];
+      if (el.hasAttribute("href")) {
+        const href = new URL(el.getAttribute("href"), document.URL).href;
+        this.queue.add(() => handler.call(el, href));
+      }
     }
   }
   _onAnchor(selection, handler, document) {
@@ -501,6 +548,233 @@ class Crawler {
   _onResponse(handler, response) {
     this.queue.add(() => handler(response));
   }
+  calculateAutoThrottleDelay(domain, responseTime) {
+    if (!this.config.autoThrottle)
+      return 0;
+    let times = this.domainResponseTimes.get(domain) || [];
+    times.push(responseTime);
+    if (times.length > 10) {
+      times = times.slice(-10);
+    }
+    this.domainResponseTimes.set(domain, times);
+    const avgResponseTime = times.reduce((a, b) => a + b, 0) / times.length;
+    const targetDelay = this.config.autoThrottleTargetDelay;
+    const loadFactor = avgResponseTime / 200;
+    let newDelay = Math.round(targetDelay * loadFactor);
+    newDelay = Math.max(this.config.autoThrottleMinDelay, newDelay);
+    newDelay = Math.min(this.config.autoThrottleMaxDelay, newDelay);
+    this.domainCurrentDelay.set(domain, newDelay);
+    if (this.config.debug) {
+      console.log(`[AutoThrottle] ${domain}: avgRT=${avgResponseTime.toFixed(0)}ms, delay=${newDelay}ms`);
+    }
+    return newDelay;
+  }
+  getAutoThrottleDelay(domain) {
+    if (!this.config.autoThrottle)
+      return 0;
+    return this.domainCurrentDelay.get(domain) || this.config.autoThrottleMinDelay;
+  }
+  async handle429Response(url, response) {
+    let retryAfter = 0;
+    const retryAfterHeader = response?.headers?.["retry-after"] || response?.headers?.get?.("retry-after");
+    if (retryAfterHeader) {
+      const parsed = parseInt(retryAfterHeader, 10);
+      if (!isNaN(parsed)) {
+        retryAfter = parsed * 1000;
+      } else {
+        const date = new Date(retryAfterHeader);
+        if (!isNaN(date.getTime())) {
+          retryAfter = date.getTime() - Date.now();
+        }
+      }
+    }
+    if (retryAfter <= 0) {
+      retryAfter = 60000;
+    }
+    const maxWait = this.config.maxWaitOn429;
+    const alwaysWait = this.config.alwaysWaitOn429;
+    if (retryAfter > maxWait && !alwaysWait) {
+      const waitMinutes = Math.round(retryAfter / 60000);
+      const error = new Error(`Rate limited: Server requested wait time of ${waitMinutes} minutes, which exceeds maxWaitOn429 (${Math.round(maxWait / 60000)} minutes). Set alwaysWaitOn429: true to wait regardless.`);
+      error.code = "REZ_RATE_LIMIT_EXCEEDED";
+      error.url = url;
+      error.status = 429;
+      throw error;
+    }
+    if (retryAfter > maxWait && alwaysWait) {
+      const waitMinutes = Math.round(retryAfter / 60000);
+      console.warn(`[Crawler] WARNING: Rate limited on ${url}. Server requested ${waitMinutes} minute wait. Waiting because alwaysWaitOn429 is enabled.`);
+    }
+    if (this.config.debug) {
+      console.log(`[Crawler] 429 Rate Limited: waiting ${Math.round(retryAfter / 1000)}s before retry`);
+    }
+    return { shouldRetry: true, waitTime: retryAfter };
+  }
+  async checkCrawlLimits(url, parentUrl) {
+    if (this.config.maxUrls > 0 && this.crawlStats.urlsVisited >= this.config.maxUrls) {
+      return { allowed: false, reason: `maxUrls limit reached (${this.config.maxUrls})` };
+    }
+    if (this.config.maxDepth > 0) {
+      const parentDepth = parentUrl ? this.urlDepthMap.get(parentUrl) ?? 0 : 0;
+      const urlDepth = this.urlDepthMap.get(url) ?? parentDepth + 1;
+      if (urlDepth > this.config.maxDepth) {
+        return { allowed: false, reason: `maxDepth limit reached (depth ${urlDepth} > ${this.config.maxDepth})` };
+      }
+      if (!this.urlDepthMap.has(url)) {
+        this.urlDepthMap.set(url, urlDepth);
+        this.crawlStats.currentDepth = Math.max(this.crawlStats.currentDepth, urlDepth);
+      }
+    }
+    if (this.config.respectRobotsTxt) {
+      try {
+        if (!this.robotsTxt.isCached(url)) {
+          await this.robotsTxt.fetch(url, async (robotsUrl) => {
+            const response = await this.http.get(robotsUrl, { timeout: 1e4 });
+            return { status: response.status, data: response.data };
+          });
+        }
+        const allowed = this.robotsTxt.isAllowed(url);
+        if (!allowed) {
+          return { allowed: false, reason: "Blocked by robots.txt" };
+        }
+      } catch (error) {
+        if (this.config.debug) {
+          console.warn(`[Crawler] Failed to check robots.txt for ${url}:`, error);
+        }
+      }
+    }
+    return { allowed: true };
+  }
+  shouldFollowLink(element) {
+    if (this.config.followNofollow) {
+      return true;
+    }
+    const rel = element.getAttribute("rel");
+    if (rel && rel.toLowerCase().includes("nofollow")) {
+      return false;
+    }
+    return true;
+  }
+  checkResponseSize(contentLength) {
+    if (this.config.maxResponseSize > 0 && contentLength > this.config.maxResponseSize) {
+      return {
+        allowed: false,
+        reason: `Response size (${contentLength} bytes) exceeds maxResponseSize (${this.config.maxResponseSize} bytes)`
+      };
+    }
+    return { allowed: true };
+  }
+  collect(data) {
+    this.collectedData.push(data);
+    return this;
+  }
+  getCollectedData() {
+    return [...this.collectedData];
+  }
+  clearCollectedData() {
+    this.collectedData = [];
+    return this;
+  }
+  async exportData(filePath, format = "json") {
+    const data = this.collectedData;
+    if (data.length === 0) {
+      if (this.config.debug) {
+        console.warn("[Crawler] No data to export");
+      }
+      return;
+    }
+    let content;
+    switch (format) {
+      case "json":
+        content = JSON.stringify(data, null, 2);
+        break;
+      case "jsonl":
+        content = data.map((item) => JSON.stringify(item)).join(`
+`);
+        break;
+      case "csv":
+        const keys = new Set;
+        data.forEach((item) => {
+          if (typeof item === "object" && item !== null) {
+            Object.keys(item).forEach((key) => keys.add(key));
+          }
+        });
+        const headers = Array.from(keys);
+        const escapeCSV = (val) => {
+          if (val === null || val === undefined)
+            return "";
+          const str = String(val);
+          if (str.includes(",") || str.includes('"') || str.includes(`
+`)) {
+            return `"${str.replace(/"/g, '""')}"`;
+          }
+          return str;
+        };
+        const rows = [
+          headers.join(","),
+          ...data.map((item) => {
+            if (typeof item !== "object" || item === null) {
+              return escapeCSV(item);
+            }
+            return headers.map((key) => escapeCSV(item[key])).join(",");
+          })
+        ];
+        content = rows.join(`
+`);
+        break;
+      default:
+        throw new Error(`Unsupported export format: ${format}`);
+    }
+    const dir = path.dirname(filePath);
+    if (!fs.existsSync(dir)) {
+      fs.mkdirSync(dir, { recursive: true });
+    }
+    fs.writeFileSync(filePath, content, "utf-8");
+    if (this.config.debug) {
+      console.log(`[Crawler] Exported ${data.length} items to ${filePath} (${format})`);
+    }
+  }
+  getStats() {
+    return { ...this.crawlStats };
+  }
+  async triggerStartHandlers() {
+    if (this.crawlStarted)
+      return;
+    this.crawlStarted = true;
+    this.crawlStats.startTime = Date.now();
+    for (const handler of this.startHandlers) {
+      try {
+        await handler();
+      } catch (error) {
+        if (this.config.debug) {
+          console.error("[Crawler] onStart handler error:", error);
+        }
+      }
+    }
+  }
+  async triggerFinishHandlers() {
+    this.crawlStats.endTime = Date.now();
+    for (const handler of this.finishHandlers) {
+      try {
+        await handler(this.crawlStats);
+      } catch (error) {
+        if (this.config.debug) {
+          console.error("[Crawler] onFinish handler error:", error);
+        }
+      }
+    }
+  }
+  async triggerRedirectHandlers(event) {
+    for (const handler of this.redirectHandlers) {
+      try {
+        await handler(event);
+      } catch (error) {
+        if (this.config.debug) {
+          console.error("[Crawler] onRedirect handler error:", error);
+        }
+      }
+    }
+  }
   buildUrl(url, params) {
     if (params) {
       const u = new URL(url, this.config.baseUrl);
@@ -570,16 +844,35 @@ class Crawler {
       this.addToNavigationQueue(url, method, body, headersObj);
     }
     if (deepEmailFinder) {
-      this.execute2(method, url, body, _options, forceRevisit).then();
+      const p = this.execute2(method, url, body, _options, forceRevisit);
+      this.pendingExecutions.add(p);
+      p.finally(() => this.pendingExecutions.delete(p));
       return this;
     }
-    this.execute(method, url, body, _options, extractLeads, forceRevisit, oxylabsOptions, oxylabsInstanse, decodoInstanse, decodoOptions).then();
+    const p = this.execute(method, url, body, _options, extractLeads, forceRevisit, oxylabsOptions, oxylabsInstanse, decodoInstanse, decodoOptions);
+    this.pendingExecutions.add(p);
+    p.finally(() => this.pendingExecutions.delete(p));
     return this;
   }
   async execute(method, url, body, options = {}, isEmail, forceRevisit, oxylabsOptions, oxylabsInstanse, decodoInstanse, decodoOptions) {
-    this.queue.add(() => this.executeHttp(method, url, body, options, isEmail, forceRevisit, oxylabsOptions, oxylabsInstanse, decodoInstanse, decodoOptions)).then();
+    await this.waitForStorage();
+    if (this.isCacheEnabled) {
+      await this.waitForCache();
+    }
+    if (this.config.enableNavigationHistory) {
+      await this.waitForNavigationHistory();
+    }
+    const task = this.queue.add(() => this.executeHttp(method, url, body, options, isEmail, forceRevisit, oxylabsOptions, oxylabsInstanse, decodoInstanse, decodoOptions));
+    task.finally(() => this.pendingExecutions.delete(task));
   }
   async execute2(method, url, body, options = {}, forceRevisit) {
+    await this.waitForStorage();
+    if (this.isCacheEnabled) {
+      await this.waitForCache();
+    }
+    if (this.config.enableNavigationHistory) {
+      await this.waitForNavigationHistory();
+    }
     this.queue.add(() => this.leadsFinder.parseExternalWebsite(url, method, body, {
       httpConfig: options,
       saveCache: this.saveCache.bind(this),
@@ -593,21 +886,34 @@ class Crawler {
       allowCrossDomainTravel: true
     }, forceRevisit, true)).then();
   }
-  async executeHttp(method, url, body, options = {}, isEmail, forceRevisit, oxylabsOptions, oxylabsInstanse, decodoInstanse, decodoOptions, retryCount = 0) {
+  async executeHttp(method, url, body, options = {}, isEmail, forceRevisit, oxylabsOptions, oxylabsInstanse, decodoInstanse, decodoOptions, retryCount = 0, parentUrl) {
     try {
-      console.log({
-        oxylabsOptions: typeof oxylabsOptions,
-        oxylabsInstanse: typeof oxylabsInstanse,
-        decodoInstanse: typeof decodoInstanse,
-        decodoOptions: typeof decodoOptions
-      });
+      await this.triggerStartHandlers();
+      const limitCheck = await this.checkCrawlLimits(url, parentUrl);
+      if (!limitCheck.allowed) {
+        if (this.config.debug) {
+          console.log(`[Crawler] Skipping ${url}: ${limitCheck.reason}`);
+        }
+        return;
+      }
+      this.crawlStats.urlsQueued++;
+      const domain = new URL(url).hostname;
+      const delay = this.getAutoThrottleDelay(domain);
+      if (delay > 0) {
+        await new Promise((resolve) => setTimeout(resolve, delay));
+      }
       const isVisited = forceRevisit ? false : await this.hasUrlInCache(url);
       const cache = await this.getCache(url);
       if (isVisited && !cache)
         return;
       if (isVisited && method !== "GET")
         return;
+      const requestStartTime = Date.now();
       const response = cache && method === "GET" ? cache : oxylabsInstanse && oxylabsOptions ? await oxylabsInstanse.scrape(url) : decodoInstanse && decodoOptions ? await decodoInstanse.scrape(url) : await (method === "GET" ? this.http.get(url, options) : method === "PATCH" ? this.http.patch(url, body, options) : method === "POST" ? this.http.post(url, body, options) : this.http.put(url, body, options));
+      if (!cache) {
+        const responseTime = Date.now() - requestStartTime;
+        this.calculateAutoThrottleDelay(domain, responseTime);
+      }
       const res = {
         data: response.data || response.content || "",
         contentType: response.contentType || "",
@@ -619,11 +925,29 @@ class Crawler {
         cookies: response?.cookies?.serialized || response?.cookies,
         contentLength: response.contentLength || 0
       };
+      if (res.contentLength && res.contentLength > 0) {
+        const sizeCheck = this.checkResponseSize(res.contentLength);
+        if (!sizeCheck.allowed) {
+          if (this.config.debug) {
+            console.log(`[Crawler] Skipping ${url}: ${sizeCheck.reason}`);
+          }
+          return;
+        }
+      }
+      this.crawlStats.urlsVisited++;
+      if (res.finalUrl && res.finalUrl !== url && this.redirectHandlers.length > 0) {
+        await this.triggerRedirectHandlers({
+          originalUrl: url,
+          finalUrl: res.finalUrl,
+          redirectCount: response.redirectCount || 1,
+          statusCode: res.status
+        });
+      }
       if (!cache)
         await this.saveCache(url, res);
       if (!isVisited)
         await this.saveUrl(url);
-      this.markUrlVisited(url, {
+      await this.markUrlVisited(url, {
         status: res.status,
         finalUrl: res.finalUrl,
         contentType: res.contentType
@@ -655,6 +979,24 @@ class Crawler {
       }
     } catch (e) {
       const error = e;
+      if (error?.response?.status === 429 || error?.status === 429) {
+        try {
+          const { shouldRetry, waitTime } = await this.handle429Response(url, error.response || error);
+          if (shouldRetry) {
+            await this.sleep(waitTime);
+            return await this.executeHttp(method, url, body, options, isEmail, forceRevisit, oxylabsOptions, oxylabsInstanse, decodoInstanse, decodoOptions, retryCount + 1, parentUrl);
+          }
+        } catch (rateLimitError) {
+          this.crawlStats.urlsFailed++;
+          if (this.config.throwFatalError)
+            throw rateLimitError;
+          for (let i = 0;i < this.errorEvents.length; i++) {
+            const event = this.errorEvents[i];
+            this[event.handler](...event.attr, rateLimitError);
+          }
+          return;
+        }
+      }
       if (error && error.response) {
         const status = error.response.status;
         const retryDelay = this.config.retryDelay || 1000;
@@ -666,16 +1008,17 @@ class Crawler {
         if (retryWithoutProxyOnStatusCode && options.proxy && retryWithoutProxyOnStatusCode.includes(status) && retryCount < maxRetryAttempts) {
           await this.sleep(retryDelay);
           delete options.proxy;
-          return await this.executeHttp(method, url, body, options, isEmail, forceRevisit, oxylabsOptions, oxylabsInstanse, decodoInstanse, decodoOptions, retryCount + 1);
+          return await this.executeHttp(method, url, body, options, isEmail, forceRevisit, oxylabsOptions, oxylabsInstanse, decodoInstanse, decodoOptions, retryCount + 1, parentUrl);
         } else if (retryOnStatusCode && options.proxy && retryOnStatusCode.includes(status) && retryCount < maxRetryAttempts) {
           await this.sleep(retryDelay);
-          return await this.executeHttp(method, url, body, options, isEmail, forceRevisit, oxylabsOptions, oxylabsInstanse, decodoInstanse, decodoOptions, retryCount + 1);
+          return await this.executeHttp(method, url, body, options, isEmail, forceRevisit, oxylabsOptions, oxylabsInstanse, decodoInstanse, decodoOptions, retryCount + 1, parentUrl);
         } else if (retryOnProxyError && options.proxy && retryCount < maxRetryOnProxyError) {
           await this.sleep(retryDelay);
-          return await this.executeHttp(method, url, body, options, isEmail, forceRevisit, oxylabsOptions, oxylabsInstanse, decodoInstanse, decodoOptions, retryCount + 1);
+          return await this.executeHttp(method, url, body, options, isEmail, forceRevisit, oxylabsOptions, oxylabsInstanse, decodoInstanse, decodoOptions, retryCount + 1, parentUrl);
         }
       }
-      this.markUrlVisited(url, {
+      this.crawlStats.urlsFailed++;
+      await this.markUrlVisited(url, {
         status: error?.response?.status || 0,
         errorMessage: e.message || "Unknown error"
       });
@@ -684,7 +1027,6 @@ class Crawler {
       if (this.config.debug) {
         console.log(`Error visiting ${url}: ${e.message}`);
       }
-      console.log(error);
       for (let i = 0;i < this.errorEvents.length; i++) {
         const event = this.errorEvents[i];
         this[event.handler](...event.attr, e);
@@ -692,18 +1034,36 @@ class Crawler {
     }
   }
   async waitForAll() {
+    if (this.pendingExecutions.size > 0) {
+      await Promise.allSettled([...this.pendingExecutions]);
+    }
     await this.queue.onIdle();
+    await this.triggerFinishHandlers();
   }
   async done() {
     return this.waitForAll();
   }
   async close() {
     try {
-      await this.cacher.close();
+      await this.cacher?.close();
     } catch {}
     try {
-      await this.urlStorage.close();
+      await this.urlStorage?.close();
     } catch {}
+    try {
+      await this.navigationHistory?.close();
+    } catch {}
+  }
+  async destroy() {
+    this.queue.clear();
+    this.events.length = 0;
+    this.jsonEvents.length = 0;
+    this.errorEvents.length = 0;
+    this.responseEvents.length = 0;
+    this.rawResponseEvents.length = 0;
+    this.emailDiscoveredEvents.length = 0;
+    this.emailLeadsEvents.length = 0;
+    await this.close();
   }
 }