npm - rezo - Versions diffs - 1.0.42 → 1.0.44 - Mend

rezo 1.0.42 → 1.0.44

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Files changed (89) hide show

package/dist/adapters/curl.cjs +131 -29
package/dist/adapters/curl.js +131 -29
package/dist/adapters/entries/curl.d.ts +65 -0
package/dist/adapters/entries/fetch.d.ts +65 -0
package/dist/adapters/entries/http.d.ts +65 -0
package/dist/adapters/entries/http2.d.ts +65 -0
package/dist/adapters/entries/react-native.d.ts +65 -0
package/dist/adapters/entries/xhr.d.ts +65 -0
package/dist/adapters/http2.cjs +209 -22
package/dist/adapters/http2.js +209 -22
package/dist/adapters/index.cjs +6 -6
package/dist/cache/index.cjs +9 -13
package/dist/cache/index.js +0 -2
package/dist/core/rezo.cjs +7 -0
package/dist/core/rezo.js +7 -0
package/dist/crawler/addon/decodo/index.cjs +1 -0
package/dist/crawler/addon/decodo/index.js +1 -0
package/dist/crawler/crawler-options.cjs +1 -0
package/dist/crawler/crawler-options.js +1 -0
package/dist/crawler/crawler.cjs +1070 -0
package/dist/crawler/crawler.js +1068 -0
package/dist/crawler/index.cjs +40 -0
package/dist/{plugin → crawler}/index.js +4 -2
package/dist/crawler/plugin/file-cacher.cjs +19 -0
package/dist/crawler/plugin/file-cacher.js +19 -0
package/dist/crawler/plugin/index.cjs +1 -0
package/dist/crawler/plugin/index.js +1 -0
package/dist/crawler/plugin/navigation-history.cjs +43 -0
package/dist/crawler/plugin/navigation-history.js +43 -0
package/dist/crawler/plugin/robots-txt.cjs +2 -0
package/dist/crawler/plugin/robots-txt.js +2 -0
package/dist/crawler/plugin/url-store.cjs +18 -0
package/dist/crawler/plugin/url-store.js +18 -0
package/dist/crawler.d.ts +511 -183
package/dist/entries/crawler.cjs +5 -5
package/dist/entries/crawler.js +2 -2
package/dist/index.cjs +27 -24
package/dist/index.d.ts +73 -0
package/dist/index.js +1 -0
package/dist/internal/agents/base.cjs +113 -0
package/dist/internal/agents/base.js +110 -0
package/dist/internal/agents/http-proxy.cjs +89 -0
package/dist/internal/agents/http-proxy.js +86 -0
package/dist/internal/agents/https-proxy.cjs +176 -0
package/dist/internal/agents/https-proxy.js +173 -0
package/dist/internal/agents/index.cjs +10 -0
package/dist/internal/agents/index.js +5 -0
package/dist/internal/agents/socks-client.cjs +571 -0
package/dist/internal/agents/socks-client.js +567 -0
package/dist/internal/agents/socks-proxy.cjs +75 -0
package/dist/internal/agents/socks-proxy.js +72 -0
package/dist/platform/browser.d.ts +65 -0
package/dist/platform/bun.d.ts +65 -0
package/dist/platform/deno.d.ts +65 -0
package/dist/platform/node.d.ts +65 -0
package/dist/platform/react-native.d.ts +65 -0
package/dist/platform/worker.d.ts +65 -0
package/dist/proxy/index.cjs +18 -16
package/dist/proxy/index.js +17 -12
package/dist/queue/index.cjs +8 -8
package/dist/responses/buildError.cjs +11 -2
package/dist/responses/buildError.js +11 -2
package/dist/responses/universal/index.cjs +11 -11
package/dist/utils/curl.cjs +317 -0
package/dist/utils/curl.js +314 -0
package/package.json +2 -6
package/dist/cache/file-cacher.cjs +0 -264
package/dist/cache/file-cacher.js +0 -261
package/dist/cache/url-store.cjs +0 -288
package/dist/cache/url-store.js +0 -285
package/dist/plugin/addon/decodo/index.cjs +0 -1
package/dist/plugin/addon/decodo/index.js +0 -1
package/dist/plugin/crawler-options.cjs +0 -1
package/dist/plugin/crawler-options.js +0 -1
package/dist/plugin/crawler.cjs +0 -519
package/dist/plugin/crawler.js +0 -517
package/dist/plugin/index.cjs +0 -36
/package/dist/{plugin → crawler}/addon/decodo/options.cjs +0 -0
/package/dist/{plugin → crawler}/addon/decodo/options.js +0 -0
/package/dist/{plugin → crawler}/addon/decodo/types.cjs +0 -0
/package/dist/{plugin → crawler}/addon/decodo/types.js +0 -0
/package/dist/{plugin → crawler}/addon/oxylabs/index.cjs +0 -0
/package/dist/{plugin → crawler}/addon/oxylabs/index.js +0 -0
/package/dist/{plugin → crawler}/addon/oxylabs/options.cjs +0 -0
/package/dist/{plugin → crawler}/addon/oxylabs/options.js +0 -0
/package/dist/{plugin → crawler}/addon/oxylabs/types.cjs +0 -0
/package/dist/{plugin → crawler}/addon/oxylabs/types.js +0 -0
/package/dist/{plugin → crawler}/scraper.cjs +0 -0
/package/dist/{plugin → crawler}/scraper.js +0 -0

package/dist/crawler/crawler.cjs ADDED Viewed

@@ -0,0 +1,1070 @@
+const fs = require("node:fs");
+const { FileCacher } = require('./plugin/file-cacher.cjs');
+const { UrlStore } = require('./plugin/url-store.cjs');
+const { NavigationHistory } = require('./plugin/navigation-history.cjs');
+const { RobotsTxt } = require('./plugin/robots-txt.cjs');
+const { parseHTML } = require("linkedom");
+const path = require("node:path");
+const { Rezo } = require('../core/rezo.cjs');
+const { RezoQueue } = require('../queue/queue.cjs');
+const { Scraper } = require('./scraper.cjs');
+const { CrawlerOptions } = require('./crawler-options.cjs');
+const { loadAdapter } = require('../adapters/picker.cjs');
+String.prototype.addBaseUrl = function(url) {
+  url = url instanceof URL ? url.href : url;
+  const html = this.replace(/<base\b[^>]*?>/gi, "");
+  if (/<head[^>]*>/i.test(html)) {
+    return html.replace(/<head[^>]*>/i, (match) => `${match}
+<base href="${url}">`);
+  }
+  const baseTag = `<head>
+<base href="${url}">
+</head>
+`;
+  if (/<body[^>]*>/i.test(html)) {
+    return html.replace(/<body[^>]*>/i, baseTag + "$&");
+  }
+  if (/<html[^>]*>/i.test(html)) {
+    return html.replace(/<html[^>]*>/i, `$&
+` + baseTag);
+  }
+  return this;
+};
+class Crawler {
+  http;
+  events = [];
+  jsonEvents = [];
+  errorEvents = [];
+  responseEvents = [];
+  rawResponseEvents = [];
+  emailDiscoveredEvents = [];
+  emailLeadsEvents = [];
+  cacher = null;
+  queue;
+  isCacheEnabled;
+  config;
+  urlStorage;
+  isStorageReady = false;
+  isCacheReady = false;
+  leadsFinder;
+  navigationHistory = null;
+  isNavigationHistoryReady = false;
+  isSessionReady = false;
+  currentSession = null;
+  navigationHistoryInitPromise = null;
+  adapterExecutor = null;
+  adapterType;
+  pendingExecutions = new Set;
+  robotsTxt;
+  domainResponseTimes = new Map;
+  domainCurrentDelay = new Map;
+  crawlStats = {
+    urlsVisited: 0,
+    urlsQueued: 0,
+    urlsFailed: 0,
+    startTime: 0,
+    currentDepth: 0
+  };
+  urlDepthMap = new Map;
+  startHandlers = [];
+  finishHandlers = [];
+  redirectHandlers = [];
+  collectedData = [];
+  crawlStarted = false;
+  constructor(crawlerOptions, http = new Rezo) {
+    this.http = http;
+    this.queue = new RezoQueue({
+      concurrency: 1000
+    });
+    this.config = new CrawlerOptions(crawlerOptions);
+    this.adapterType = this.config.adapter;
+    const enableCache = this.config.enableCache;
+    this.isCacheEnabled = enableCache;
+    if (enableCache) {
+      const cacheDir = this.config.cacheDir;
+      const cacheTTL = this.config.cacheTTL;
+      const dbUrl = cacheDir && (cacheDir.startsWith("./") || cacheDir.startsWith("/")) ? `${cacheDir}${cacheDir.endsWith("/") ? "" : "/"}` : cacheDir ? `./${cacheDir}${cacheDir.endsWith("/") ? "" : "/"}` : `./cache/`;
+      if (!fs.existsSync(path.dirname(dbUrl)))
+        fs.mkdirSync(path.dirname(dbUrl), { recursive: true });
+      FileCacher.create({
+        cacheDir: dbUrl,
+        ttl: cacheTTL,
+        encryptNamespace: true
+      }).then((storage) => {
+        this.cacher = storage;
+        this.isCacheReady = true;
+      });
+      const dit = path.resolve(cacheDir, "urls");
+      if (!fs.existsSync(dit))
+        fs.mkdirSync(dit, { recursive: true });
+      UrlStore.create({
+        storeDir: dit,
+        dbFileName: ".url_cache.db",
+        ttl: 1000 * 60 * 60 * 24 * 7
+      }).then((storage) => {
+        this.urlStorage = storage;
+        this.isStorageReady = true;
+      });
+    } else {
+      const dit = path.resolve(this.config.cacheDir, "./cache/urls");
+      if (!fs.existsSync(dit))
+        fs.mkdirSync(dit, { recursive: true });
+      UrlStore.create({
+        storeDir: dit,
+        dbFileName: ".url_cache.db",
+        ttl: 1000 * 60 * 60 * 24 * 7
+      }).then((storage) => {
+        this.urlStorage = storage;
+        this.isStorageReady = true;
+      });
+    }
+    if (this.config.enableNavigationHistory) {
+      const navHistoryDir = path.resolve(this.config.cacheDir, "navigation");
+      if (!fs.existsSync(navHistoryDir))
+        fs.mkdirSync(navHistoryDir, { recursive: true });
+      this.navigationHistoryInitPromise = this.initializeNavigationHistory(navHistoryDir);
+    }
+    this.initializeAdapter();
+    this.leadsFinder = new Scraper(this.http, this.config, this._onEmailLeads.bind(this), this._onEmailDiscovered.bind(this), this.config.debug);
+    this.robotsTxt = new RobotsTxt({
+      userAgent: this.config.userAgent || "RezoBot",
+      cacheTTL: 24 * 60 * 60 * 1000
+    });
+    this.crawlStats.startTime = Date.now();
+    if (this.config.baseUrl) {
+      this.urlDepthMap.set(this.config.baseUrl, 0);
+    }
+  }
+  async initializeAdapter() {
+    try {
+      const adapterModule = await loadAdapter(this.adapterType);
+      this.adapterExecutor = adapterModule.executeRequest.bind(adapterModule);
+    } catch (error) {
+      if (this.config.debug) {
+        console.warn(`[Crawler] Failed to load adapter '${this.adapterType}', falling back to http instance`);
+      }
+    }
+  }
+  async initializeNavigationHistory(navHistoryDir) {
+    try {
+      const history = await NavigationHistory.create({
+        storeDir: navHistoryDir,
+        dbFileName: "navigation.db"
+      });
+      this.navigationHistory = history;
+      this.isNavigationHistoryReady = true;
+      const session = await history.getSession(this.config.sessionId);
+      if (session && (session.status === "running" || session.status === "paused")) {
+        this.currentSession = session;
+        await history.updateSessionStatus(this.config.sessionId, "running");
+      } else if (!session) {
+        this.currentSession = await history.createSession(this.config.sessionId, this.config.baseUrl, { adapter: this.adapterType });
+      }
+      this.isSessionReady = true;
+    } catch (error) {
+      if (this.config.debug) {
+        console.error(`[Crawler] Failed to initialize navigation history:`, error);
+      }
+      this.isNavigationHistoryReady = false;
+      this.isSessionReady = false;
+    }
+  }
+  async waitForNavigationHistory() {
+    if (!this.config.enableNavigationHistory)
+      return;
+    if (this.isNavigationHistoryReady && this.isSessionReady)
+      return;
+    if (this.navigationHistoryInitPromise) {
+      await this.navigationHistoryInitPromise;
+    }
+  }
+  async ensureNavigationHistoryReady() {
+    if (!this.config.enableNavigationHistory)
+      return null;
+    await this.waitForNavigationHistory();
+    return this.navigationHistory;
+  }
+  async addToNavigationQueue(url, method, body, headers) {
+    const history = await this.ensureNavigationHistoryReady();
+    if (!history || !this.currentSession)
+      return;
+    try {
+      await history.addToQueue(this.currentSession.sessionId, url, {
+        method,
+        body,
+        headers
+      });
+    } catch (error) {
+      if (this.config.debug) {
+        console.warn(`[Crawler] Failed to add URL to navigation queue: ${url}`, error);
+      }
+    }
+  }
+  async markUrlVisited(url, result) {
+    const history = await this.ensureNavigationHistoryReady();
+    if (!history || !this.currentSession)
+      return;
+    try {
+      await history.markVisited(this.currentSession.sessionId, url, result);
+    } catch (error) {
+      if (this.config.debug) {
+        console.warn(`[Crawler] Failed to mark URL as visited: ${url}`, error);
+      }
+    }
+  }
+  getSession() {
+    return this.currentSession;
+  }
+  getSessionId() {
+    return this.config.sessionId;
+  }
+  async resume(sessionId) {
+    if (!this.config.enableNavigationHistory) {
+      throw new Error("Navigation history is not enabled. Set enableNavigationHistory: true in options.");
+    }
+    await this.waitForNavigationHistory();
+    if (!this.navigationHistory) {
+      throw new Error("Navigation history failed to initialize.");
+    }
+    await this.waitForStorage();
+    if (this.isCacheEnabled) {
+      await this.waitForCache();
+    }
+    const targetSessionId = sessionId || this.config.sessionId;
+    const session = await this.navigationHistory.getSession(targetSessionId);
+    if (!session) {
+      throw new Error(`Session '${targetSessionId}' not found`);
+    }
+    if (session.status === "completed") {
+      throw new Error(`Session '${targetSessionId}' is already completed`);
+    }
+    this.currentSession = session;
+    await this.navigationHistory.updateSessionStatus(targetSessionId, "running");
+    const queuedUrls = await this.navigationHistory.getAllQueuedUrls(targetSessionId);
+    if (this.config.debug) {
+      console.log(`[Crawler] Resuming session '${targetSessionId}' with ${queuedUrls.length} queued URLs`);
+    }
+    const scheduledUrls = new Set;
+    for (const item of queuedUrls) {
+      if (scheduledUrls.has(item.url)) {
+        continue;
+      }
+      scheduledUrls.add(item.url);
+      const body = item.body ? JSON.parse(item.body) : undefined;
+      const headers = item.headers ? JSON.parse(item.headers) : undefined;
+      this.visit(item.url, {
+        method: item.method,
+        body,
+        headers,
+        forceRevisit: false
+      });
+    }
+    return this;
+  }
+  async getResumableSessions() {
+    if (!this.config.enableNavigationHistory) {
+      return [];
+    }
+    await this.waitForNavigationHistory();
+    if (!this.navigationHistory) {
+      return [];
+    }
+    return this.navigationHistory.getResumableSessions();
+  }
+  async pause() {
+    await this.waitForNavigationHistory();
+    if (!this.navigationHistory || !this.currentSession) {
+      return;
+    }
+    await this.navigationHistory.updateSessionStatus(this.currentSession.sessionId, "paused");
+    this.currentSession.status = "paused";
+  }
+  async complete() {
+    await this.waitForNavigationHistory();
+    if (!this.navigationHistory || !this.currentSession) {
+      return;
+    }
+    await this.navigationHistory.updateSessionStatus(this.currentSession.sessionId, "completed");
+    this.currentSession.status = "completed";
+  }
+  getAdapterType() {
+    return this.adapterType;
+  }
+  async setAdapter(adapter) {
+    this.adapterType = adapter;
+    await this.initializeAdapter();
+  }
+  rawResponseHandler(data) {
+    if (this.rawResponseEvents.length === 0)
+      return;
+    const isBuffer = Buffer.isBuffer(data);
+    if (!isBuffer) {
+      if (data instanceof ArrayBuffer) {
+        data = Buffer.from(new Uint8Array(data));
+      } else if (data instanceof Uint8Array) {
+        data = Buffer.from(data);
+      } else if (typeof data === "string") {
+        data = Buffer.from(data, "utf8");
+      } else if (typeof data === "object") {
+        data = Buffer.from(JSON.stringify(data), "utf8");
+      }
+    }
+    this.rawResponseEvents.forEach((e) => {
+      const handler = e.attr[0];
+      handler(data);
+    });
+  }
+  async waitForCache() {
+    if (this.isCacheReady)
+      return;
+    await this.sleep(this.rnd(50, 200));
+    await this.waitForCache();
+  }
+  async waitForStorage() {
+    if (this.isStorageReady)
+      return;
+    await this.sleep(this.rnd(50, 200));
+    await this.waitForStorage();
+  }
+  async saveUrl(url) {
+    await this.waitForStorage();
+    await this.urlStorage.set(url);
+  }
+  async hasUrlInCache(url) {
+    await this.waitForStorage();
+    return await this.urlStorage.has(url);
+  }
+  async saveCache(url, value) {
+    if (!this.isCacheEnabled)
+      return;
+    await this.waitForCache();
+    return this.cacher.set(url, value, this.config.cacheTTL, this.getNamespace(url));
+  }
+  getNamespace(url) {
+    try {
+      return new URL(url).hostname;
+    } catch {
+      return;
+    }
+  }
+  async hasCache(url) {
+    if (!this.isCacheEnabled)
+      return false;
+    await this.waitForCache();
+    return this.cacher.has(url, this.getNamespace(url));
+  }
+  async getCache(url) {
+    if (!this.isCacheEnabled)
+      return null;
+    await this.waitForCache();
+    return this.cacher.get(url, this.getNamespace(url));
+  }
+  sleep(ms) {
+    return new Promise((resolve) => setTimeout(resolve, ms));
+  }
+  rnd(min = 0, max = Number.MAX_VALUE) {
+    return Math.floor(Math.random() * (max - min + 1)) + min;
+  }
+  onError(handler) {
+    this.errorEvents.push({
+      handler: "_onError",
+      attr: [handler]
+    });
+    return this;
+  }
+  onJson(handler) {
+    this.jsonEvents.push({
+      handler: "_onJson",
+      attr: [handler]
+    });
+    return this;
+  }
+  onEmailDiscovered(handler) {
+    this.emailDiscoveredEvents.push(handler);
+    return this;
+  }
+  onEmailLeads(handler) {
+    this.emailLeadsEvents.push(handler);
+    return this;
+  }
+  onStart(handler) {
+    this.startHandlers.push(handler);
+    return this;
+  }
+  onFinish(handler) {
+    this.finishHandlers.push(handler);
+    return this;
+  }
+  onRedirect(handler) {
+    this.redirectHandlers.push(handler);
+    return this;
+  }
+  onRawData(handler) {
+    this.rawResponseEvents.push({
+      handler: "_onRawResponse",
+      attr: [handler]
+    });
+    return this;
+  }
+  onDocument(handler) {
+    this.events.push({
+      handler: "_onDocument",
+      attr: [handler]
+    });
+    return this;
+  }
+  onBody(handler) {
+    this.events.push({
+      handler: "_onBody",
+      attr: [handler]
+    });
+    return this;
+  }
+  onElement(handler) {
+    this.events.push({
+      handler: "_onElement",
+      attr: [handler]
+    });
+    return this;
+  }
+  onAnchor(selection, handler) {
+    this.events.push({
+      handler: "_onAnchor",
+      attr: [selection, handler]
+    });
+    return this;
+  }
+  onHref(handler) {
+    this.events.push({
+      handler: "_onHref",
+      attr: [handler]
+    });
+    return this;
+  }
+  onSelection(selection, handler) {
+    this.events.push({
+      handler: "_onSelection",
+      attr: [selection, handler]
+    });
+    return this;
+  }
+  onResponse(handler) {
+    this.responseEvents.push({
+      handler: "_onResponse",
+      attr: [handler]
+    });
+    return this;
+  }
+  onAttribute(selection, attribute, handler) {
+    this.events.push({
+      handler: "_onAttribute",
+      attr: [selection, attribute, handler]
+    });
+    return this;
+  }
+  onText(selection, handler) {
+    this.events.push({
+      handler: "_onText",
+      attr: [selection, handler]
+    });
+    return this;
+  }
+  _onBody(handler, document) {
+    this.queue.add(() => handler(document.body));
+  }
+  _onAttribute(selection, attribute, handler, document) {
+    const isSimpleForm = typeof attribute === "function";
+    const actualAttribute = isSimpleForm ? selection : attribute;
+    const actualHandler = isSimpleForm ? attribute : handler;
+    const actualSelection = isSimpleForm ? `[${selection}]` : selection || `[${attribute}]`;
+    const elements = document.querySelectorAll(actualSelection);
+    for (let i = 0;i < elements.length; i++) {
+      const el = elements[i];
+      if (el.hasAttribute(actualAttribute)) {
+        const value = el.getAttribute(actualAttribute);
+        this.queue.add(() => actualHandler.call(el, value, actualAttribute));
+      }
+    }
+  }
+  _onText(selection, handler, document) {
+    const elements = document.querySelectorAll(selection);
+    for (let i = 0;i < elements.length; i++) {
+      const el = elements[i];
+      const text = el.textContent;
+      this.queue.add(() => handler.call(el, text));
+    }
+  }
+  _onSelection(selection, handler, document) {
+    const elements = document.querySelectorAll(selection);
+    for (let i = 0;i < elements.length; i++) {
+      this.queue.add(() => handler(elements[i]));
+    }
+  }
+  _onElement(handler, document) {
+    const elements = document.querySelectorAll("*");
+    for (let i = 0;i < elements.length; i++) {
+      this.queue.add(() => handler(elements[i]));
+    }
+  }
+  _onHref(handler, document) {
+    const elements = document.querySelectorAll("a, link");
+    for (let i = 0;i < elements.length; i++) {
+      const el = elements[i];
+      if (el.hasAttribute("href")) {
+        const href = new URL(el.getAttribute("href"), document.URL).href;
+        this.queue.add(() => handler.call(el, href));
+      }
+    }
+  }
+  _onAnchor(selection, handler, document) {
+    handler = typeof selection === "function" ? selection : handler;
+    selection = typeof selection === "function" ? "a" : selection;
+    const elements = document.querySelectorAll(selection);
+    for (let i = 0;i < elements.length; i++) {
+      if (elements[i]?.href && document.baseURI)
+        elements[i].href = new URL(elements[i].getAttribute("href"), document.baseURI).href;
+      this.queue.add(() => handler(elements[i]));
+    }
+  }
+  _onDocument(handler, document) {
+    this.queue.add(() => handler(document));
+  }
+  _onJson(handler, json) {
+    this.queue.add(() => handler(json));
+  }
+  _onError(handler, error) {
+    this.queue.add(() => handler(error));
+  }
+  async _onEmailDiscovered(handler, email) {
+    await handler(email);
+  }
+  async _onEmailLeads(handler, emails) {
+    await handler(emails);
+  }
+  _onRawResponse(handler, rawResponse) {
+    this.queue.add(() => handler(rawResponse));
+  }
+  _onResponse(handler, response) {
+    this.queue.add(() => handler(response));
+  }
+  calculateAutoThrottleDelay(domain, responseTime) {
+    if (!this.config.autoThrottle)
+      return 0;
+    let times = this.domainResponseTimes.get(domain) || [];
+    times.push(responseTime);
+    if (times.length > 10) {
+      times = times.slice(-10);
+    }
+    this.domainResponseTimes.set(domain, times);
+    const avgResponseTime = times.reduce((a, b) => a + b, 0) / times.length;
+    const targetDelay = this.config.autoThrottleTargetDelay;
+    const loadFactor = avgResponseTime / 200;
+    let newDelay = Math.round(targetDelay * loadFactor);
+    newDelay = Math.max(this.config.autoThrottleMinDelay, newDelay);
+    newDelay = Math.min(this.config.autoThrottleMaxDelay, newDelay);
+    this.domainCurrentDelay.set(domain, newDelay);
+    if (this.config.debug) {
+      console.log(`[AutoThrottle] ${domain}: avgRT=${avgResponseTime.toFixed(0)}ms, delay=${newDelay}ms`);
+    }
+    return newDelay;
+  }
+  getAutoThrottleDelay(domain) {
+    if (!this.config.autoThrottle)
+      return 0;
+    return this.domainCurrentDelay.get(domain) || this.config.autoThrottleMinDelay;
+  }
+  async handle429Response(url, response) {
+    let retryAfter = 0;
+    const retryAfterHeader = response?.headers?.["retry-after"] || response?.headers?.get?.("retry-after");
+    if (retryAfterHeader) {
+      const parsed = parseInt(retryAfterHeader, 10);
+      if (!isNaN(parsed)) {
+        retryAfter = parsed * 1000;
+      } else {
+        const date = new Date(retryAfterHeader);
+        if (!isNaN(date.getTime())) {
+          retryAfter = date.getTime() - Date.now();
+        }
+      }
+    }
+    if (retryAfter <= 0) {
+      retryAfter = 60000;
+    }
+    const maxWait = this.config.maxWaitOn429;
+    const alwaysWait = this.config.alwaysWaitOn429;
+    if (retryAfter > maxWait && !alwaysWait) {
+      const waitMinutes = Math.round(retryAfter / 60000);
+      const error = new Error(`Rate limited: Server requested wait time of ${waitMinutes} minutes, which exceeds maxWaitOn429 (${Math.round(maxWait / 60000)} minutes). Set alwaysWaitOn429: true to wait regardless.`);
+      error.code = "REZ_RATE_LIMIT_EXCEEDED";
+      error.url = url;
+      error.status = 429;
+      throw error;
+    }
+    if (retryAfter > maxWait && alwaysWait) {
+      const waitMinutes = Math.round(retryAfter / 60000);
+      console.warn(`[Crawler] WARNING: Rate limited on ${url}. Server requested ${waitMinutes} minute wait. Waiting because alwaysWaitOn429 is enabled.`);
+    }
+    if (this.config.debug) {
+      console.log(`[Crawler] 429 Rate Limited: waiting ${Math.round(retryAfter / 1000)}s before retry`);
+    }
+    return { shouldRetry: true, waitTime: retryAfter };
+  }
+  async checkCrawlLimits(url, parentUrl) {
+    if (this.config.maxUrls > 0 && this.crawlStats.urlsVisited >= this.config.maxUrls) {
+      return { allowed: false, reason: `maxUrls limit reached (${this.config.maxUrls})` };
+    }
+    if (this.config.maxDepth > 0) {
+      const parentDepth = parentUrl ? this.urlDepthMap.get(parentUrl) ?? 0 : 0;
+      const urlDepth = this.urlDepthMap.get(url) ?? parentDepth + 1;
+      if (urlDepth > this.config.maxDepth) {
+        return { allowed: false, reason: `maxDepth limit reached (depth ${urlDepth} > ${this.config.maxDepth})` };
+      }
+      if (!this.urlDepthMap.has(url)) {
+        this.urlDepthMap.set(url, urlDepth);
+        this.crawlStats.currentDepth = Math.max(this.crawlStats.currentDepth, urlDepth);
+      }
+    }
+    if (this.config.respectRobotsTxt) {
+      try {
+        if (!this.robotsTxt.isCached(url)) {
+          await this.robotsTxt.fetch(url, async (robotsUrl) => {
+            const response = await this.http.get(robotsUrl, { timeout: 1e4 });
+            return { status: response.status, data: response.data };
+          });
+        }
+        const allowed = this.robotsTxt.isAllowed(url);
+        if (!allowed) {
+          return { allowed: false, reason: "Blocked by robots.txt" };
+        }
+      } catch (error) {
+        if (this.config.debug) {
+          console.warn(`[Crawler] Failed to check robots.txt for ${url}:`, error);
+        }
+      }
+    }
+    return { allowed: true };
+  }
+  shouldFollowLink(element) {
+    if (this.config.followNofollow) {
+      return true;
+    }
+    const rel = element.getAttribute("rel");
+    if (rel && rel.toLowerCase().includes("nofollow")) {
+      return false;
+    }
+    return true;
+  }
+  checkResponseSize(contentLength) {
+    if (this.config.maxResponseSize > 0 && contentLength > this.config.maxResponseSize) {
+      return {
+        allowed: false,
+        reason: `Response size (${contentLength} bytes) exceeds maxResponseSize (${this.config.maxResponseSize} bytes)`
+      };
+    }
+    return { allowed: true };
+  }
+  collect(data) {
+    this.collectedData.push(data);
+    return this;
+  }
+  getCollectedData() {
+    return [...this.collectedData];
+  }
+  clearCollectedData() {
+    this.collectedData = [];
+    return this;
+  }
+  async exportData(filePath, format = "json") {
+    const data = this.collectedData;
+    if (data.length === 0) {
+      if (this.config.debug) {
+        console.warn("[Crawler] No data to export");
+      }
+      return;
+    }
+    let content;
+    switch (format) {
+      case "json":
+        content = JSON.stringify(data, null, 2);
+        break;
+      case "jsonl":
+        content = data.map((item) => JSON.stringify(item)).join(`
+`);
+        break;
+      case "csv":
+        const keys = new Set;
+        data.forEach((item) => {
+          if (typeof item === "object" && item !== null) {
+            Object.keys(item).forEach((key) => keys.add(key));
+          }
+        });
+        const headers = Array.from(keys);
+        const escapeCSV = (val) => {
+          if (val === null || val === undefined)
+            return "";
+          const str = String(val);
+          if (str.includes(",") || str.includes('"') || str.includes(`
+`)) {
+            return `"${str.replace(/"/g, '""')}"`;
+          }
+          return str;
+        };
+        const rows = [
+          headers.join(","),
+          ...data.map((item) => {
+            if (typeof item !== "object" || item === null) {
+              return escapeCSV(item);
+            }
+            return headers.map((key) => escapeCSV(item[key])).join(",");
+          })
+        ];
+        content = rows.join(`
+`);
+        break;
+      default:
+        throw new Error(`Unsupported export format: ${format}`);
+    }
+    const dir = path.dirname(filePath);
+    if (!fs.existsSync(dir)) {
+      fs.mkdirSync(dir, { recursive: true });
+    }
+    fs.writeFileSync(filePath, content, "utf-8");
+    if (this.config.debug) {
+      console.log(`[Crawler] Exported ${data.length} items to ${filePath} (${format})`);
+    }
+  }
+  getStats() {
+    return { ...this.crawlStats };
+  }
+  async triggerStartHandlers() {
+    if (this.crawlStarted)
+      return;
+    this.crawlStarted = true;
+    this.crawlStats.startTime = Date.now();
+    for (const handler of this.startHandlers) {
+      try {
+        await handler();
+      } catch (error) {
+        if (this.config.debug) {
+          console.error("[Crawler] onStart handler error:", error);
+        }
+      }
+    }
+  }
+  async triggerFinishHandlers() {
+    this.crawlStats.endTime = Date.now();
+    for (const handler of this.finishHandlers) {
+      try {
+        await handler(this.crawlStats);
+      } catch (error) {
+        if (this.config.debug) {
+          console.error("[Crawler] onFinish handler error:", error);
+        }
+      }
+    }
+  }
+  async triggerRedirectHandlers(event) {
+    for (const handler of this.redirectHandlers) {
+      try {
+        await handler(event);
+      } catch (error) {
+        if (this.config.debug) {
+          console.error("[Crawler] onRedirect handler error:", error);
+        }
+      }
+    }
+  }
+  buildUrl(url, params) {
+    if (params) {
+      const u = new URL(url, this.config.baseUrl);
+      for (const [key, value] of Object.entries(params)) {
+        u.searchParams.set(key, value.toString());
+      }
+      url = u.href;
+    }
+    return url;
+  }
+  visit(url, options) {
+    if (this.config.baseUrl)
+      url = new URL(url, this.config.baseUrl).href;
+    if (options?.params && (options.useOxylabsScraperAi || this.config.hasDomain(url, "oxylabs"))) {
+      url = this.buildUrl(url, options.params);
+    }
+    const {
+      method = "GET",
+      headers = new Headers,
+      forceRevisit = this.config.forceRevisit,
+      body = "",
+      timeout = this.config.timeout,
+      maxRedirects = this.config.maxRedirects,
+      useProxy = this.config.hasDomain(url, "proxies", options?.useProxy),
+      extractLeads = false,
+      params,
+      rejectUnauthorized,
+      useQueue = false,
+      deepEmailFinder = false,
+      useOxylabsScraperAi = false,
+      useOxylabsRotation = true,
+      useDecodo = false
+    } = options || {};
+    const _options = {
+      headers: this.config.pickHeaders(url, true, headers, true),
+      timeout,
+      maxRedirects,
+      params,
+      proxy: useProxy ? this.config.getAdapter(url, "proxies", true, true) || undefined : undefined,
+      rejectUnauthorized: typeof rejectUnauthorized === "boolean" ? rejectUnauthorized : this.config.rejectUnauthorized,
+      pqueue: this.config.getAdapter(url, "limiters", useQueue, useQueue) || undefined
+    };
+    let oxylabsOptions = {};
+    let oxylabsInstanse = undefined;
+    if (useOxylabsScraperAi && this.config.hasDomain(url, "oxylabs")) {
+      oxylabsOptions = {
+        method: method === "POST" ? "post" : "get",
+        headers: this.config.pickHeaders(url, true, headers, true),
+        pqueue: this.config.getAdapter(url, "limiters", useQueue, useQueue) || undefined,
+        base64Body: typeof body === "string" ? Buffer.from(body).toString("base64") : undefined
+      };
+      oxylabsInstanse = this.config.getAdapter(url, "oxylabs", false, useOxylabsRotation) || undefined;
+    }
+    let decodoOptions = {};
+    let decodoInstanse = undefined;
+    if (useDecodo && this.config.hasDomain(url, "decodo")) {
+      decodoOptions = {
+        method: method === "POST" ? "post" : "get",
+        headers: this.config.pickHeaders(url, true, headers, true),
+        pqueue: this.config.getAdapter(url, "limiters", useQueue, useQueue) || undefined,
+        base64Body: typeof body === "string" ? Buffer.from(body).toString("base64") : undefined
+      };
+      decodoInstanse = this.config.getAdapter(url, "decodo", false, useOxylabsRotation) || undefined;
+    }
+    if (this.config.enableNavigationHistory) {
+      const headersObj = headers instanceof Headers ? Object.fromEntries(headers.entries()) : headers;
+      this.addToNavigationQueue(url, method, body, headersObj);
+    }
+    if (deepEmailFinder) {
+      const p = this.execute2(method, url, body, _options, forceRevisit);
+      this.pendingExecutions.add(p);
+      p.finally(() => this.pendingExecutions.delete(p));
+      return this;
+    }
+    const p = this.execute(method, url, body, _options, extractLeads, forceRevisit, oxylabsOptions, oxylabsInstanse, decodoInstanse, decodoOptions);
+    this.pendingExecutions.add(p);
+    p.finally(() => this.pendingExecutions.delete(p));
+    return this;
+  }
+  async execute(method, url, body, options = {}, isEmail, forceRevisit, oxylabsOptions, oxylabsInstanse, decodoInstanse, decodoOptions) {
+    await this.waitForStorage();
+    if (this.isCacheEnabled) {
+      await this.waitForCache();
+    }
+    if (this.config.enableNavigationHistory) {
+      await this.waitForNavigationHistory();
+    }
+    const task = this.queue.add(() => this.executeHttp(method, url, body, options, isEmail, forceRevisit, oxylabsOptions, oxylabsInstanse, decodoInstanse, decodoOptions));
+    task.finally(() => this.pendingExecutions.delete(task));
+  }
+  async execute2(method, url, body, options = {}, forceRevisit) {
+    await this.waitForStorage();
+    if (this.isCacheEnabled) {
+      await this.waitForCache();
+    }
+    if (this.config.enableNavigationHistory) {
+      await this.waitForNavigationHistory();
+    }
+    this.queue.add(() => this.leadsFinder.parseExternalWebsite(url, method, body, {
+      httpConfig: options,
+      saveCache: this.saveCache.bind(this),
+      saveUrl: this.saveUrl.bind(this),
+      getCache: this.getCache.bind(this),
+      hasUrlInCache: this.hasUrlInCache.bind(this),
+      onEmailDiscovered: this.emailDiscoveredEvents,
+      onEmails: this.emailLeadsEvents,
+      queue: this.queue,
+      depth: 1,
+      allowCrossDomainTravel: true
+    }, forceRevisit, true)).then();
+  }
+  async executeHttp(method, url, body, options = {}, isEmail, forceRevisit, oxylabsOptions, oxylabsInstanse, decodoInstanse, decodoOptions, retryCount = 0, parentUrl) {
+    try {
+      await this.triggerStartHandlers();
+      const limitCheck = await this.checkCrawlLimits(url, parentUrl);
+      if (!limitCheck.allowed) {
+        if (this.config.debug) {
+          console.log(`[Crawler] Skipping ${url}: ${limitCheck.reason}`);
+        }
+        return;
+      }
+      this.crawlStats.urlsQueued++;
+      const domain = new URL(url).hostname;
+      const delay = this.getAutoThrottleDelay(domain);
+      if (delay > 0) {
+        await new Promise((resolve) => setTimeout(resolve, delay));
+      }
+      const isVisited = forceRevisit ? false : await this.hasUrlInCache(url);
+      const cache = await this.getCache(url);
+      if (isVisited && !cache)
+        return;
+      if (isVisited && method !== "GET")
+        return;
+      const requestStartTime = Date.now();
+      const response = cache && method === "GET" ? cache : oxylabsInstanse && oxylabsOptions ? await oxylabsInstanse.scrape(url) : decodoInstanse && decodoOptions ? await decodoInstanse.scrape(url) : await (method === "GET" ? this.http.get(url, options) : method === "PATCH" ? this.http.patch(url, body, options) : method === "POST" ? this.http.post(url, body, options) : this.http.put(url, body, options));
+      if (!cache) {
+        const responseTime = Date.now() - requestStartTime;
+        this.calculateAutoThrottleDelay(domain, responseTime);
+      }
+      const res = {
+        data: response.data || response.content || "",
+        contentType: response.contentType || "",
+        finalUrl: response.finalUrl || response.url || url,
+        url: response?.urls?.[0] || response.url || this.buildUrl(url, options.params),
+        headers: response.headers || {},
+        status: response.status || response.statusCode || 200,
+        statusText: response.statusText || "",
+        cookies: response?.cookies?.serialized || response?.cookies,
+        contentLength: response.contentLength || 0
+      };
+      if (res.contentLength && res.contentLength > 0) {
+        const sizeCheck = this.checkResponseSize(res.contentLength);
+        if (!sizeCheck.allowed) {
+          if (this.config.debug) {
+            console.log(`[Crawler] Skipping ${url}: ${sizeCheck.reason}`);
+          }
+          return;
+        }
+      }
+      this.crawlStats.urlsVisited++;
+      if (res.finalUrl && res.finalUrl !== url && this.redirectHandlers.length > 0) {
+        await this.triggerRedirectHandlers({
+          originalUrl: url,
+          finalUrl: res.finalUrl,
+          redirectCount: response.redirectCount || 1,
+          statusCode: res.status
+        });
+      }
+      if (!cache)
+        await this.saveCache(url, res);
+      if (!isVisited)
+        await this.saveUrl(url);
+      await this.markUrlVisited(url, {
+        status: res.status,
+        finalUrl: res.finalUrl,
+        contentType: res.contentType
+      });
+      if (res.contentType && res.contentType.includes("/json")) {
+        if (this.emailDiscoveredEvents.length > 0 || this.emailLeadsEvents.length > 0) {
+          this.leadsFinder.extractEmails(JSON.stringify(res.data), res.finalUrl, this.emailDiscoveredEvents, this.emailLeadsEvents, this.queue);
+        }
+        for (let i = 0;i < this.jsonEvents.length; i++) {
+          const event = this.jsonEvents[i];
+          this[event.handler](...event.attr, res.data);
+        }
+      }
+      for (let i = 0;i < this.responseEvents.length; i++) {
+        const event = this.responseEvents[i];
+        this[event.handler](...event.attr, res);
+      }
+      this.rawResponseHandler(res.data);
+      if (!res.contentType || !res.contentType.includes("/html") || typeof res.data !== "string")
+        return;
+      if ((this.emailDiscoveredEvents.length > 0 || this.emailLeadsEvents.length > 0) && isEmail) {
+        this.leadsFinder.extractEmails(res.data, res.finalUrl, this.emailDiscoveredEvents, this.emailLeadsEvents, this.queue);
+      }
+      const { document } = parseHTML(res.data.addBaseUrl(res.finalUrl));
+      document.URL = res.finalUrl;
+      for (let i = 0;i < this.events.length; i++) {
+        const event = this.events[i];
+        this[event.handler](...event.attr, document);
+      }
+    } catch (e) {
+      const error = e;
+      if (error?.response?.status === 429 || error?.status === 429) {
+        try {
+          const { shouldRetry, waitTime } = await this.handle429Response(url, error.response || error);
+          if (shouldRetry) {
+            await this.sleep(waitTime);
+            return await this.executeHttp(method, url, body, options, isEmail, forceRevisit, oxylabsOptions, oxylabsInstanse, decodoInstanse, decodoOptions, retryCount + 1, parentUrl);
+          }
+        } catch (rateLimitError) {
+          this.crawlStats.urlsFailed++;
+          if (this.config.throwFatalError)
+            throw rateLimitError;
+          for (let i = 0;i < this.errorEvents.length; i++) {
+            const event = this.errorEvents[i];
+            this[event.handler](...event.attr, rateLimitError);
+          }
+          return;
+        }
+      }
+      if (error && error.response) {
+        const status = error.response.status;
+        const retryDelay = this.config.retryDelay || 1000;
+        const maxRetryAttempts = this.config.maxRetryAttempts || 3;
+        const maxRetryOnProxyError = this.config.maxRetryOnProxyError || 3;
+        const retryWithoutProxyOnStatusCode = this.config.retryWithoutProxyOnStatusCode || undefined;
+        const retryOnStatusCode = this.config.retryOnStatusCode || undefined;
+        const retryOnProxyError = this.config.retryOnProxyError || undefined;
+        if (retryWithoutProxyOnStatusCode && options.proxy && retryWithoutProxyOnStatusCode.includes(status) && retryCount < maxRetryAttempts) {
+          await this.sleep(retryDelay);
+          delete options.proxy;
+          return await this.executeHttp(method, url, body, options, isEmail, forceRevisit, oxylabsOptions, oxylabsInstanse, decodoInstanse, decodoOptions, retryCount + 1, parentUrl);
+        } else if (retryOnStatusCode && options.proxy && retryOnStatusCode.includes(status) && retryCount < maxRetryAttempts) {
+          await this.sleep(retryDelay);
+          return await this.executeHttp(method, url, body, options, isEmail, forceRevisit, oxylabsOptions, oxylabsInstanse, decodoInstanse, decodoOptions, retryCount + 1, parentUrl);
+        } else if (retryOnProxyError && options.proxy && retryCount < maxRetryOnProxyError) {
+          await this.sleep(retryDelay);
+          return await this.executeHttp(method, url, body, options, isEmail, forceRevisit, oxylabsOptions, oxylabsInstanse, decodoInstanse, decodoOptions, retryCount + 1, parentUrl);
+        }
+      }
+      this.crawlStats.urlsFailed++;
+      await this.markUrlVisited(url, {
+        status: error?.response?.status || 0,
+        errorMessage: e.message || "Unknown error"
+      });
+      if (this.config.throwFatalError)
+        throw e;
+      if (this.config.debug) {
+        console.log(`Error visiting ${url}: ${e.message}`);
+      }
+      for (let i = 0;i < this.errorEvents.length; i++) {
+        const event = this.errorEvents[i];
+        this[event.handler](...event.attr, e);
+      }
+    }
+  }
+  async waitForAll() {
+    if (this.pendingExecutions.size > 0) {
+      await Promise.allSettled([...this.pendingExecutions]);
+    }
+    await this.queue.onIdle();
+    await this.triggerFinishHandlers();
+  }
+  async done() {
+    return this.waitForAll();
+  }
+  async close() {
+    try {
+      await this.cacher?.close();
+    } catch {}
+    try {
+      await this.urlStorage?.close();
+    } catch {}
+    try {
+      await this.navigationHistory?.close();
+    } catch {}
+  }
+  async destroy() {
+    this.queue.clear();
+    this.events.length = 0;
+    this.jsonEvents.length = 0;
+    this.errorEvents.length = 0;
+    this.responseEvents.length = 0;
+    this.rawResponseEvents.length = 0;
+    this.emailDiscoveredEvents.length = 0;
+    this.emailLeadsEvents.length = 0;
+    await this.close();
+  }
+}
+exports.Crawler = Crawler;