rezo 1.0.43 → 1.0.44

package/dist/{plugin → crawler}/crawler.js

@@ -1,7 +1,8 @@
 import fs from "node:fs";
-import { FileCacher } from '../cache/file-cacher.js';
-import { UrlStore } from '../cache/url-store.js';
-import { NavigationHistory } from '../cache/navigation-history.js';
+import { FileCacher } from './plugin/file-cacher.js';
+import { UrlStore } from './plugin/url-store.js';
+import { NavigationHistory } from './plugin/navigation-history.js';
+import { RobotsTxt } from './plugin/robots-txt.js';
 import { parseHTML } from "linkedom";
 import path from "node:path";
 import { Rezo } from '../core/rezo.js';
@@ -54,6 +55,23 @@ export class Crawler {
   navigationHistoryInitPromise = null;
   adapterExecutor = null;
   adapterType;
+  pendingExecutions = new Set;
+  robotsTxt;
+  domainResponseTimes = new Map;
+  domainCurrentDelay = new Map;
+  crawlStats = {
+    urlsVisited: 0,
+    urlsQueued: 0,
+    urlsFailed: 0,
+    startTime: 0,
+    currentDepth: 0
+  };
+  urlDepthMap = new Map;
+  startHandlers = [];
+  finishHandlers = [];
+  redirectHandlers = [];
+  collectedData = [];
+  crawlStarted = false;
   constructor(crawlerOptions, http = new Rezo) {
     this.http = http;
     this.queue = new RezoQueue({
@@ -71,7 +89,6 @@ export class Crawler {
         fs.mkdirSync(path.dirname(dbUrl), { recursive: true });
       FileCacher.create({
         cacheDir: dbUrl,
-        softDelete: false,
         ttl: cacheTTL,
         encryptNamespace: true
       }).then((storage) => {
@@ -110,6 +127,14 @@ export class Crawler {
     }
     this.initializeAdapter();
     this.leadsFinder = new Scraper(this.http, this.config, this._onEmailLeads.bind(this), this._onEmailDiscovered.bind(this), this.config.debug);
+    this.robotsTxt = new RobotsTxt({
+      userAgent: this.config.userAgent || "RezoBot",
+      cacheTTL: 24 * 60 * 60 * 1000
+    });
+    this.crawlStats.startTime = Date.now();
+    if (this.config.baseUrl) {
+      this.urlDepthMap.set(this.config.baseUrl, 0);
+    }
   }
   async initializeAdapter() {
     try {
@@ -148,6 +173,8 @@ export class Crawler {
   async waitForNavigationHistory() {
     if (!this.config.enableNavigationHistory)
       return;
+    if (this.isNavigationHistoryReady && this.isSessionReady)
+      return;
     if (this.navigationHistoryInitPromise) {
       await this.navigationHistoryInitPromise;
     }
@@ -361,6 +388,18 @@ export class Crawler {
     this.emailLeadsEvents.push(handler);
     return this;
   }
+  onStart(handler) {
+    this.startHandlers.push(handler);
+    return this;
+  }
+  onFinish(handler) {
+    this.finishHandlers.push(handler);
+    return this;
+  }
+  onRedirect(handler) {
+    this.redirectHandlers.push(handler);
+    return this;
+  }
   onRawData(handler) {
     this.rawResponseEvents.push({
       handler: "_onRawResponse",
@@ -435,20 +474,25 @@ export class Crawler {
     this.queue.add(() => handler(document.body));
   }
   _onAttribute(selection, attribute, handler, document) {
-    selection = typeof attribute === "function" ? selection : null;
-    attribute = typeof attribute === "function" ? selection : attribute;
-    handler = typeof attribute === "function" ? attribute : handler;
-    selection = selection || `[${attribute}]`;
-    const elements = document.querySelectorAll(selection);
+    const isSimpleForm = typeof attribute === "function";
+    const actualAttribute = isSimpleForm ? selection : attribute;
+    const actualHandler = isSimpleForm ? attribute : handler;
+    const actualSelection = isSimpleForm ? `[${selection}]` : selection || `[${attribute}]`;
+    const elements = document.querySelectorAll(actualSelection);
     for (let i = 0;i < elements.length; i++) {
-      if (elements[i].hasAttribute(attribute))
-        this.queue.add(() => handler(elements[i].getAttribute(attribute)));
+      const el = elements[i];
+      if (el.hasAttribute(actualAttribute)) {
+        const value = el.getAttribute(actualAttribute);
+        this.queue.add(() => actualHandler.call(el, value, actualAttribute));
+      }
     }
   }
   _onText(selection, handler, document) {
     const elements = document.querySelectorAll(selection);
     for (let i = 0;i < elements.length; i++) {
-      this.queue.add(() => handler(elements[i].textContent));
+      const el = elements[i];
+      const text = el.textContent;
+      this.queue.add(() => handler.call(el, text));
     }
   }
   _onSelection(selection, handler, document) {
@@ -466,8 +510,11 @@ export class Crawler {
   _onHref(handler, document) {
     const elements = document.querySelectorAll("a, link");
     for (let i = 0;i < elements.length; i++) {
-      if (elements[i].hasAttribute("href"))
-        this.queue.add(() => handler(new URL(elements[i].getAttribute("href"), document.URL).href));
+      const el = elements[i];
+      if (el.hasAttribute("href")) {
+        const href = new URL(el.getAttribute("href"), document.URL).href;
+        this.queue.add(() => handler.call(el, href));
+      }
     }
   }
   _onAnchor(selection, handler, document) {
@@ -501,6 +548,233 @@ export class Crawler {
   _onResponse(handler, response) {
     this.queue.add(() => handler(response));
   }
+  calculateAutoThrottleDelay(domain, responseTime) {
+    if (!this.config.autoThrottle)
+      return 0;
+    let times = this.domainResponseTimes.get(domain) || [];
+    times.push(responseTime);
+    if (times.length > 10) {
+      times = times.slice(-10);
+    }
+    this.domainResponseTimes.set(domain, times);
+    const avgResponseTime = times.reduce((a, b) => a + b, 0) / times.length;
+    const targetDelay = this.config.autoThrottleTargetDelay;
+    const loadFactor = avgResponseTime / 200;
+    let newDelay = Math.round(targetDelay * loadFactor);
+    newDelay = Math.max(this.config.autoThrottleMinDelay, newDelay);
+    newDelay = Math.min(this.config.autoThrottleMaxDelay, newDelay);
+    this.domainCurrentDelay.set(domain, newDelay);
+    if (this.config.debug) {
+      console.log(`[AutoThrottle] ${domain}: avgRT=${avgResponseTime.toFixed(0)}ms, delay=${newDelay}ms`);
+    }
+    return newDelay;
+  }
+  getAutoThrottleDelay(domain) {
+    if (!this.config.autoThrottle)
+      return 0;
+    return this.domainCurrentDelay.get(domain) || this.config.autoThrottleMinDelay;
+  }
+  async handle429Response(url, response) {
+    let retryAfter = 0;
+    const retryAfterHeader = response?.headers?.["retry-after"] || response?.headers?.get?.("retry-after");
+    if (retryAfterHeader) {
+      const parsed = parseInt(retryAfterHeader, 10);
+      if (!isNaN(parsed)) {
+        retryAfter = parsed * 1000;
+      } else {
+        const date = new Date(retryAfterHeader);
+        if (!isNaN(date.getTime())) {
+          retryAfter = date.getTime() - Date.now();
+        }
+      }
+    }
+    if (retryAfter <= 0) {
+      retryAfter = 60000;
+    }
+    const maxWait = this.config.maxWaitOn429;
+    const alwaysWait = this.config.alwaysWaitOn429;
+    if (retryAfter > maxWait && !alwaysWait) {
+      const waitMinutes = Math.round(retryAfter / 60000);
+      const error = new Error(`Rate limited: Server requested wait time of ${waitMinutes} minutes, which exceeds maxWaitOn429 (${Math.round(maxWait / 60000)} minutes). Set alwaysWaitOn429: true to wait regardless.`);
+      error.code = "REZ_RATE_LIMIT_EXCEEDED";
+      error.url = url;
+      error.status = 429;
+      throw error;
+    }
+    if (retryAfter > maxWait && alwaysWait) {
+      const waitMinutes = Math.round(retryAfter / 60000);
+      console.warn(`[Crawler] WARNING: Rate limited on ${url}. Server requested ${waitMinutes} minute wait. Waiting because alwaysWaitOn429 is enabled.`);
+    }
+    if (this.config.debug) {
+      console.log(`[Crawler] 429 Rate Limited: waiting ${Math.round(retryAfter / 1000)}s before retry`);
+    }
+    return { shouldRetry: true, waitTime: retryAfter };
+  }
+  async checkCrawlLimits(url, parentUrl) {
+    if (this.config.maxUrls > 0 && this.crawlStats.urlsVisited >= this.config.maxUrls) {
+      return { allowed: false, reason: `maxUrls limit reached (${this.config.maxUrls})` };
+    }
+    if (this.config.maxDepth > 0) {
+      const parentDepth = parentUrl ? this.urlDepthMap.get(parentUrl) ?? 0 : 0;
+      const urlDepth = this.urlDepthMap.get(url) ?? parentDepth + 1;
+      if (urlDepth > this.config.maxDepth) {
+        return { allowed: false, reason: `maxDepth limit reached (depth ${urlDepth} > ${this.config.maxDepth})` };
+      }
+      if (!this.urlDepthMap.has(url)) {
+        this.urlDepthMap.set(url, urlDepth);
+        this.crawlStats.currentDepth = Math.max(this.crawlStats.currentDepth, urlDepth);
+      }
+    }
+    if (this.config.respectRobotsTxt) {
+      try {
+        if (!this.robotsTxt.isCached(url)) {
+          await this.robotsTxt.fetch(url, async (robotsUrl) => {
+            const response = await this.http.get(robotsUrl, { timeout: 1e4 });
+            return { status: response.status, data: response.data };
+          });
+        }
+        const allowed = this.robotsTxt.isAllowed(url);
+        if (!allowed) {
+          return { allowed: false, reason: "Blocked by robots.txt" };
+        }
+      } catch (error) {
+        if (this.config.debug) {
+          console.warn(`[Crawler] Failed to check robots.txt for ${url}:`, error);
+        }
+      }
+    }
+    return { allowed: true };
+  }
+  shouldFollowLink(element) {
+    if (this.config.followNofollow) {
+      return true;
+    }
+    const rel = element.getAttribute("rel");
+    if (rel && rel.toLowerCase().includes("nofollow")) {
+      return false;
+    }
+    return true;
+  }
+  checkResponseSize(contentLength) {
+    if (this.config.maxResponseSize > 0 && contentLength > this.config.maxResponseSize) {
+      return {
+        allowed: false,
+        reason: `Response size (${contentLength} bytes) exceeds maxResponseSize (${this.config.maxResponseSize} bytes)`
+      };
+    }
+    return { allowed: true };
+  }
+  collect(data) {
+    this.collectedData.push(data);
+    return this;
+  }
+  getCollectedData() {
+    return [...this.collectedData];
+  }
+  clearCollectedData() {
+    this.collectedData = [];
+    return this;
+  }
+  async exportData(filePath, format = "json") {
+    const data = this.collectedData;
+    if (data.length === 0) {
+      if (this.config.debug) {
+        console.warn("[Crawler] No data to export");
+      }
+      return;
+    }
+    let content;
+    switch (format) {
+      case "json":
+        content = JSON.stringify(data, null, 2);
+        break;
+      case "jsonl":
+        content = data.map((item) => JSON.stringify(item)).join(`
+`);
+        break;
+      case "csv":
+        const keys = new Set;
+        data.forEach((item) => {
+          if (typeof item === "object" && item !== null) {
+            Object.keys(item).forEach((key) => keys.add(key));
+          }
+        });
+        const headers = Array.from(keys);
+        const escapeCSV = (val) => {
+          if (val === null || val === undefined)
+            return "";
+          const str = String(val);
+          if (str.includes(",") || str.includes('"') || str.includes(`
+`)) {
+            return `"${str.replace(/"/g, '""')}"`;
+          }
+          return str;
+        };
+        const rows = [
+          headers.join(","),
+          ...data.map((item) => {
+            if (typeof item !== "object" || item === null) {
+              return escapeCSV(item);
+            }
+            return headers.map((key) => escapeCSV(item[key])).join(",");
+          })
+        ];
+        content = rows.join(`
+`);
+        break;
+      default:
+        throw new Error(`Unsupported export format: ${format}`);
+    }
+    const dir = path.dirname(filePath);
+    if (!fs.existsSync(dir)) {
+      fs.mkdirSync(dir, { recursive: true });
+    }
+    fs.writeFileSync(filePath, content, "utf-8");
+    if (this.config.debug) {
+      console.log(`[Crawler] Exported ${data.length} items to ${filePath} (${format})`);
+    }
+  }
+  getStats() {
+    return { ...this.crawlStats };
+  }
+  async triggerStartHandlers() {
+    if (this.crawlStarted)
+      return;
+    this.crawlStarted = true;
+    this.crawlStats.startTime = Date.now();
+    for (const handler of this.startHandlers) {
+      try {
+        await handler();
+      } catch (error) {
+        if (this.config.debug) {
+          console.error("[Crawler] onStart handler error:", error);
+        }
+      }
+    }
+  }
+  async triggerFinishHandlers() {
+    this.crawlStats.endTime = Date.now();
+    for (const handler of this.finishHandlers) {
+      try {
+        await handler(this.crawlStats);
+      } catch (error) {
+        if (this.config.debug) {
+          console.error("[Crawler] onFinish handler error:", error);
+        }
+      }
+    }
+  }
+  async triggerRedirectHandlers(event) {
+    for (const handler of this.redirectHandlers) {
+      try {
+        await handler(event);
+      } catch (error) {
+        if (this.config.debug) {
+          console.error("[Crawler] onRedirect handler error:", error);
+        }
+      }
+    }
+  }
   buildUrl(url, params) {
     if (params) {
       const u = new URL(url, this.config.baseUrl);
@@ -570,16 +844,35 @@ export class Crawler {
       this.addToNavigationQueue(url, method, body, headersObj);
     }
     if (deepEmailFinder) {
-      this.execute2(method, url, body, _options, forceRevisit).then();
+      const p = this.execute2(method, url, body, _options, forceRevisit);
+      this.pendingExecutions.add(p);
+      p.finally(() => this.pendingExecutions.delete(p));
       return this;
     }
-    this.execute(method, url, body, _options, extractLeads, forceRevisit, oxylabsOptions, oxylabsInstanse, decodoInstanse, decodoOptions).then();
+    const p = this.execute(method, url, body, _options, extractLeads, forceRevisit, oxylabsOptions, oxylabsInstanse, decodoInstanse, decodoOptions);
+    this.pendingExecutions.add(p);
+    p.finally(() => this.pendingExecutions.delete(p));
     return this;
   }
   async execute(method, url, body, options = {}, isEmail, forceRevisit, oxylabsOptions, oxylabsInstanse, decodoInstanse, decodoOptions) {
-    this.queue.add(() => this.executeHttp(method, url, body, options, isEmail, forceRevisit, oxylabsOptions, oxylabsInstanse, decodoInstanse, decodoOptions)).then();
+    await this.waitForStorage();
+    if (this.isCacheEnabled) {
+      await this.waitForCache();
+    }
+    if (this.config.enableNavigationHistory) {
+      await this.waitForNavigationHistory();
+    }
+    const task = this.queue.add(() => this.executeHttp(method, url, body, options, isEmail, forceRevisit, oxylabsOptions, oxylabsInstanse, decodoInstanse, decodoOptions));
+    task.finally(() => this.pendingExecutions.delete(task));
   }
   async execute2(method, url, body, options = {}, forceRevisit) {
+    await this.waitForStorage();
+    if (this.isCacheEnabled) {
+      await this.waitForCache();
+    }
+    if (this.config.enableNavigationHistory) {
+      await this.waitForNavigationHistory();
+    }
     this.queue.add(() => this.leadsFinder.parseExternalWebsite(url, method, body, {
       httpConfig: options,
       saveCache: this.saveCache.bind(this),
@@ -593,21 +886,34 @@ export class Crawler {
       allowCrossDomainTravel: true
     }, forceRevisit, true)).then();
   }
-  async executeHttp(method, url, body, options = {}, isEmail, forceRevisit, oxylabsOptions, oxylabsInstanse, decodoInstanse, decodoOptions, retryCount = 0) {
+  async executeHttp(method, url, body, options = {}, isEmail, forceRevisit, oxylabsOptions, oxylabsInstanse, decodoInstanse, decodoOptions, retryCount = 0, parentUrl) {
     try {
-      console.log({
-        oxylabsOptions: typeof oxylabsOptions,
-        oxylabsInstanse: typeof oxylabsInstanse,
-        decodoInstanse: typeof decodoInstanse,
-        decodoOptions: typeof decodoOptions
-      });
+      await this.triggerStartHandlers();
+      const limitCheck = await this.checkCrawlLimits(url, parentUrl);
+      if (!limitCheck.allowed) {
+        if (this.config.debug) {
+          console.log(`[Crawler] Skipping ${url}: ${limitCheck.reason}`);
+        }
+        return;
+      }
+      this.crawlStats.urlsQueued++;
+      const domain = new URL(url).hostname;
+      const delay = this.getAutoThrottleDelay(domain);
+      if (delay > 0) {
+        await new Promise((resolve) => setTimeout(resolve, delay));
+      }
       const isVisited = forceRevisit ? false : await this.hasUrlInCache(url);
       const cache = await this.getCache(url);
       if (isVisited && !cache)
         return;
       if (isVisited && method !== "GET")
         return;
+      const requestStartTime = Date.now();
       const response = cache && method === "GET" ? cache : oxylabsInstanse && oxylabsOptions ? await oxylabsInstanse.scrape(url) : decodoInstanse && decodoOptions ? await decodoInstanse.scrape(url) : await (method === "GET" ? this.http.get(url, options) : method === "PATCH" ? this.http.patch(url, body, options) : method === "POST" ? this.http.post(url, body, options) : this.http.put(url, body, options));
+      if (!cache) {
+        const responseTime = Date.now() - requestStartTime;
+        this.calculateAutoThrottleDelay(domain, responseTime);
+      }
       const res = {
         data: response.data || response.content || "",
         contentType: response.contentType || "",
@@ -619,11 +925,29 @@ export class Crawler {
         cookies: response?.cookies?.serialized || response?.cookies,
         contentLength: response.contentLength || 0
       };
+      if (res.contentLength && res.contentLength > 0) {
+        const sizeCheck = this.checkResponseSize(res.contentLength);
+        if (!sizeCheck.allowed) {
+          if (this.config.debug) {
+            console.log(`[Crawler] Skipping ${url}: ${sizeCheck.reason}`);
+          }
+          return;
+        }
+      }
+      this.crawlStats.urlsVisited++;
+      if (res.finalUrl && res.finalUrl !== url && this.redirectHandlers.length > 0) {
+        await this.triggerRedirectHandlers({
+          originalUrl: url,
+          finalUrl: res.finalUrl,
+          redirectCount: response.redirectCount || 1,
+          statusCode: res.status
+        });
+      }
       if (!cache)
         await this.saveCache(url, res);
       if (!isVisited)
         await this.saveUrl(url);
-      this.markUrlVisited(url, {
+      await this.markUrlVisited(url, {
         status: res.status,
         finalUrl: res.finalUrl,
         contentType: res.contentType
@@ -655,6 +979,24 @@ export class Crawler {
       }
     } catch (e) {
       const error = e;
+      if (error?.response?.status === 429 || error?.status === 429) {
+        try {
+          const { shouldRetry, waitTime } = await this.handle429Response(url, error.response || error);
+          if (shouldRetry) {
+            await this.sleep(waitTime);
+            return await this.executeHttp(method, url, body, options, isEmail, forceRevisit, oxylabsOptions, oxylabsInstanse, decodoInstanse, decodoOptions, retryCount + 1, parentUrl);
+          }
+        } catch (rateLimitError) {
+          this.crawlStats.urlsFailed++;
+          if (this.config.throwFatalError)
+            throw rateLimitError;
+          for (let i = 0;i < this.errorEvents.length; i++) {
+            const event = this.errorEvents[i];
+            this[event.handler](...event.attr, rateLimitError);
+          }
+          return;
+        }
+      }
       if (error && error.response) {
         const status = error.response.status;
         const retryDelay = this.config.retryDelay || 1000;
@@ -666,16 +1008,17 @@ export class Crawler {
         if (retryWithoutProxyOnStatusCode && options.proxy && retryWithoutProxyOnStatusCode.includes(status) && retryCount < maxRetryAttempts) {
           await this.sleep(retryDelay);
           delete options.proxy;
-          return await this.executeHttp(method, url, body, options, isEmail, forceRevisit, oxylabsOptions, oxylabsInstanse, decodoInstanse, decodoOptions, retryCount + 1);
+          return await this.executeHttp(method, url, body, options, isEmail, forceRevisit, oxylabsOptions, oxylabsInstanse, decodoInstanse, decodoOptions, retryCount + 1, parentUrl);
         } else if (retryOnStatusCode && options.proxy && retryOnStatusCode.includes(status) && retryCount < maxRetryAttempts) {
           await this.sleep(retryDelay);
-          return await this.executeHttp(method, url, body, options, isEmail, forceRevisit, oxylabsOptions, oxylabsInstanse, decodoInstanse, decodoOptions, retryCount + 1);
+          return await this.executeHttp(method, url, body, options, isEmail, forceRevisit, oxylabsOptions, oxylabsInstanse, decodoInstanse, decodoOptions, retryCount + 1, parentUrl);
         } else if (retryOnProxyError && options.proxy && retryCount < maxRetryOnProxyError) {
           await this.sleep(retryDelay);
-          return await this.executeHttp(method, url, body, options, isEmail, forceRevisit, oxylabsOptions, oxylabsInstanse, decodoInstanse, decodoOptions, retryCount + 1);
+          return await this.executeHttp(method, url, body, options, isEmail, forceRevisit, oxylabsOptions, oxylabsInstanse, decodoInstanse, decodoOptions, retryCount + 1, parentUrl);
         }
       }
-      this.markUrlVisited(url, {
+      this.crawlStats.urlsFailed++;
+      await this.markUrlVisited(url, {
         status: error?.response?.status || 0,
         errorMessage: e.message || "Unknown error"
       });
@@ -684,7 +1027,6 @@ export class Crawler {
       if (this.config.debug) {
         console.log(`Error visiting ${url}: ${e.message}`);
       }
-      console.log(error);
       for (let i = 0;i < this.errorEvents.length; i++) {
         const event = this.errorEvents[i];
         this[event.handler](...event.attr, e);
@@ -692,17 +1034,35 @@ export class Crawler {
     }
   }
   async waitForAll() {
+    if (this.pendingExecutions.size > 0) {
+      await Promise.allSettled([...this.pendingExecutions]);
+    }
     await this.queue.onIdle();
+    await this.triggerFinishHandlers();
   }
   async done() {
     return this.waitForAll();
   }
   async close() {
     try {
-      await this.cacher.close();
+      await this.cacher?.close();
     } catch {}
     try {
-      await this.urlStorage.close();
+      await this.urlStorage?.close();
     } catch {}
+    try {
+      await this.navigationHistory?.close();
+    } catch {}
+  }
+  async destroy() {
+    this.queue.clear();
+    this.events.length = 0;
+    this.jsonEvents.length = 0;
+    this.errorEvents.length = 0;
+    this.responseEvents.length = 0;
+    this.rawResponseEvents.length = 0;
+    this.emailDiscoveredEvents.length = 0;
+    this.emailLeadsEvents.length = 0;
+    await this.close();
   }
 }

package/dist/crawler/index.cjs

@@ -0,0 +1,40 @@
+const _mod_uzdw07 = require('./crawler.cjs');
+exports.Crawler = _mod_uzdw07.Crawler;;
+const _mod_fk2i9s = require('./crawler-options.cjs');
+exports.CrawlerOptions = _mod_fk2i9s.CrawlerOptions;;
+const _mod_y9972f = require('./plugin/robots-txt.cjs');
+exports.RobotsTxt = _mod_y9972f.RobotsTxt;;
+const _mod_y7t1zo = require('./plugin/file-cacher.cjs');
+exports.FileCacher = _mod_y7t1zo.FileCacher;;
+const _mod_ecu03i = require('./plugin/url-store.cjs');
+exports.UrlStore = _mod_ecu03i.UrlStore;;
+const _mod_kxfnq7 = require('./plugin/navigation-history.cjs');
+exports.NavigationHistory = _mod_kxfnq7.NavigationHistory;;
+const _mod_bg9vts = require('./addon/oxylabs/index.cjs');
+exports.Oxylabs = _mod_bg9vts.Oxylabs;;
+const _mod_zd2i1t = require('./addon/oxylabs/options.cjs');
+exports.OXYLABS_BROWSER_TYPES = _mod_zd2i1t.OXYLABS_BROWSER_TYPES;
+exports.OXYLABS_COMMON_LOCALES = _mod_zd2i1t.OXYLABS_COMMON_LOCALES;
+exports.OXYLABS_COMMON_GEO_LOCATIONS = _mod_zd2i1t.OXYLABS_COMMON_GEO_LOCATIONS;
+exports.OXYLABS_US_STATES = _mod_zd2i1t.OXYLABS_US_STATES;
+exports.OXYLABS_EUROPEAN_COUNTRIES = _mod_zd2i1t.OXYLABS_EUROPEAN_COUNTRIES;
+exports.OXYLABS_ASIAN_COUNTRIES = _mod_zd2i1t.OXYLABS_ASIAN_COUNTRIES;
+exports.getRandomOxylabsBrowserType = _mod_zd2i1t.getRandomBrowserType;
+exports.getRandomOxylabsLocale = _mod_zd2i1t.getRandomLocale;
+exports.getRandomOxylabsGeoLocation = _mod_zd2i1t.getRandomGeoLocation;;
+const _mod_gr8483 = require('./addon/decodo/index.cjs');
+exports.Decodo = _mod_gr8483.Decodo;;
+const _mod_km63k6 = require('./addon/decodo/options.cjs');
+exports.DECODO_DEVICE_TYPES = _mod_km63k6.DECODO_DEVICE_TYPES;
+exports.DECODO_HEADLESS_MODES = _mod_km63k6.DECODO_HEADLESS_MODES;
+exports.DECODO_COMMON_LOCALES = _mod_km63k6.DECODO_COMMON_LOCALES;
+exports.DECODO_COMMON_COUNTRIES = _mod_km63k6.DECODO_COMMON_COUNTRIES;
+exports.DECODO_EUROPEAN_COUNTRIES = _mod_km63k6.DECODO_EUROPEAN_COUNTRIES;
+exports.DECODO_ASIAN_COUNTRIES = _mod_km63k6.DECODO_ASIAN_COUNTRIES;
+exports.DECODO_US_STATES = _mod_km63k6.DECODO_US_STATES;
+exports.DECODO_COMMON_CITIES = _mod_km63k6.DECODO_COMMON_CITIES;
+exports.getRandomDecodoDeviceType = _mod_km63k6.getRandomDeviceType;
+exports.getRandomDecodoLocale = _mod_km63k6.getRandomLocale;
+exports.getRandomDecodoCountry = _mod_km63k6.getRandomCountry;
+exports.getRandomDecodoCity = _mod_km63k6.getRandomCity;
+exports.generateDecodoSessionId = _mod_km63k6.generateSessionId;;

package/dist/{plugin → crawler}/index.js

@@ -1,7 +1,9 @@
 export { Crawler } from './crawler.js';
 export { CrawlerOptions } from './crawler-options.js';
-export { FileCacher } from '../cache/file-cacher.js';
-export { UrlStore } from '../cache/url-store.js';
+export { RobotsTxt } from './plugin/robots-txt.js';
+export { FileCacher } from './plugin/file-cacher.js';
+export { UrlStore } from './plugin/url-store.js';
+export { NavigationHistory } from './plugin/navigation-history.js';
 export { Oxylabs } from './addon/oxylabs/index.js';
 export {
   OXYLABS_BROWSER_TYPES,

package/dist/crawler/plugin/file-cacher.cjs

@@ -0,0 +1,19 @@
+var x=Object.create;var{getPrototypeOf:A,defineProperty:d,getOwnPropertyNames:b}=Object;var m=Object.prototype.hasOwnProperty;var p=(e,r,c)=>{c=e!=null?x(A(e)):{};let t=r||!e||!e.__esModule?d(c,"default",{value:e,enumerable:!0}):c;for(let s of b(e))if(!m.call(t,s))d(t,s,{get:()=>e[s],enumerable:!0});return t};var y=require("node:fs"),R=require("node:path"),{createHash:O}=require("node:crypto"),f=require("node:zlib"),D=typeof globalThis.Bun<"u",E=typeof f.zstdCompressSync==="function";async function w(e){if(D){let{Database:t}=await import("bun:sqlite"),s=new t(e);return{run:(a,...n)=>s.run(a,...n),get:(a,...n)=>s.query(a).get(...n),all:(a,...n)=>s.query(a).all(...n),exec:(a)=>s.exec(a),close:()=>s.close()}}let{DatabaseSync:r}=await import("node:sqlite"),c=new r(e);return{run:(t,...s)=>{if(s.length===0)c.exec(t);else c.prepare(t).run(...s)},get:(t,...s)=>{return c.prepare(t).get(...s)},all:(t,...s)=>{return c.prepare(t).all(...s)},exec:(t)=>c.exec(t),close:()=>c.close()}}function T(e){if(E)return f.zstdCompressSync(e);return e}function g(e){if(E)return f.zstdDecompressSync(e);return e}class l{databases=new Map;options;cacheDir;closed=!1;constructor(e={}){if(this.options={cacheDir:e.cacheDir||"/tmp/rezo-crawler/cache",ttl:e.ttl||604800000,compression:e.compression??!1,encryptNamespace:e.encryptNamespace??!1,maxEntries:e.maxEntries??0},this.cacheDir=R.resolve(this.options.cacheDir),!y.existsSync(this.cacheDir))y.mkdirSync(this.cacheDir,{recursive:!0})}static async create(e={}){return new l(e)}async getDatabase(e){let r=this.options.encryptNamespace?O("md5").update(e).digest("hex"):e.replace(/[^a-zA-Z0-9_-]/g,"_");if(this.databases.has(r))return this.databases.get(r);let c=R.join(this.cacheDir,`${r}.db`),t=await w(c);return t.exec("PRAGMA journal_mode = WAL"),t.exec("PRAGMA synchronous = NORMAL"),t.exec("PRAGMA cache_size = -64000"),t.exec("PRAGMA temp_store = MEMORY"),t.exec("PRAGMA mmap_size = 268435456"),t.exec(`
+      CREATE TABLE IF NOT EXISTS cache (
+        key TEXT PRIMARY KEY,
+        value BLOB NOT NULL,
+        expiresAt INTEGER NOT NULL,
+        createdAt INTEGER NOT NULL,
+        compressed INTEGER DEFAULT 0
+      ) WITHOUT ROWID
+    `),t.exec("CREATE INDEX IF NOT EXISTS idx_expires ON cache(expiresAt)"),this.databases.set(r,t),t}async set(e,r,c,t="default"){if(this.closed)throw Error("FileCacher is closed");let s=await this.getDatabase(t),a=Date.now(),n=a+(c??this.options.ttl),o=Buffer.from(JSON.stringify(r),"utf-8"),i=0;if(this.options.compression&&E)try{o=T(o),i=1}catch{}if(s.run(`
+      INSERT OR REPLACE INTO cache (key, value, expiresAt, createdAt, compressed)
+      VALUES (?, ?, ?, ?, ?)
+    `,e,Buffer.from(o).toString("base64"),n,a,i),this.options.maxEntries>0){let u=s.get("SELECT COUNT(*) as cnt FROM cache");if(u&&u.cnt>this.options.maxEntries){let h=u.cnt-this.options.maxEntries;s.run(`
+          DELETE FROM cache WHERE key IN (
+            SELECT key FROM cache ORDER BY createdAt ASC LIMIT ?
+          )
+        `,h)}}}async setMany(e,r="default"){if(this.closed)throw Error("FileCacher is closed");if(e.length===0)return;let c=await this.getDatabase(r),t=Date.now(),s=t+this.options.ttl;c.exec("BEGIN TRANSACTION");try{for(let a of e){let n=a.ttl?t+a.ttl:s,o=Buffer.from(JSON.stringify(a.value),"utf-8"),i=0;if(this.options.compression&&E)try{o=T(o),i=1}catch{}c.run(`
+          INSERT OR REPLACE INTO cache (key, value, expiresAt, createdAt, compressed)
+          VALUES (?, ?, ?, ?, ?)
+        `,a.key,Buffer.from(o).toString("base64"),n,t,i)}c.exec("COMMIT")}catch(a){throw c.exec("ROLLBACK"),a}}async get(e,r="default"){if(this.closed)throw Error("FileCacher is closed");let c=await this.getDatabase(r),t=c.get("SELECT value, expiresAt, compressed FROM cache WHERE key = ?",e);if(!t)return null;if(t.expiresAt<Date.now())return c.run("DELETE FROM cache WHERE key = ?",e),null;let s=Buffer.from(t.value,"base64");if(t.compressed)try{s=g(s)}catch{return null}try{return JSON.parse(Buffer.from(s).toString("utf-8"))}catch{return null}}async has(e,r="default"){if(this.closed)return!1;let t=(await this.getDatabase(r)).get("SELECT expiresAt FROM cache WHERE key = ?",e);if(!t)return!1;return t.expiresAt>=Date.now()}async hasMany(e,r="default"){if(this.closed)return new Set;if(e.length===0)return new Set;let c=await this.getDatabase(r),t=Date.now(),s=new Set,a=500;for(let n=0;n<e.length;n+=a){let o=e.slice(n,n+a),i=o.map(()=>"?").join(","),u=c.all(`SELECT key, expiresAt FROM cache WHERE key IN (${i})`,...o);for(let h of u)if(h.expiresAt>=t)s.add(h.key)}return s}async delete(e,r="default"){if(this.closed)return!1;return(await this.getDatabase(r)).run("DELETE FROM cache WHERE key = ?",e),!0}async clear(e="default"){if(this.closed)return;(await this.getDatabase(e)).exec("DELETE FROM cache")}async cleanup(e="default"){if(this.closed)return 0;let r=await this.getDatabase(e),c=Date.now(),t=r.get("SELECT COUNT(*) as cnt FROM cache");r.run("DELETE FROM cache WHERE expiresAt < ?",c);let s=r.get("SELECT COUNT(*) as cnt FROM cache");return(t?.cnt||0)-(s?.cnt||0)}async stats(e="default"){if(this.closed)return{count:0,expired:0};let r=await this.getDatabase(e),c=Date.now(),t=r.get("SELECT COUNT(*) as cnt FROM cache"),s=r.get("SELECT COUNT(*) as cnt FROM cache WHERE expiresAt < ?",c);return{count:t?.cnt||0,expired:s?.cnt||0}}async close(){if(this.closed)return;this.closed=!0;for(let e of this.databases.values())try{e.close()}catch{}this.databases.clear()}get isClosed(){return this.closed}get directory(){return this.cacheDir}}exports.FileCacher=l;exports.default=l;module.exports=Object.assign(l,exports);