npm - rezo - Versions diffs - 1.0.0 - Mend

rezo 1.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Files changed (135) hide show

package/LICENSE +202 -0
package/README.md +1507 -0
package/assets/icon.svg +37 -0
package/assets/logo-dark.svg +47 -0
package/assets/logo.svg +58 -0
package/dist/adapters/curl.cjs +1034 -0
package/dist/adapters/curl.js +1031 -0
package/dist/adapters/entries/curl.cjs +4 -0
package/dist/adapters/entries/curl.d.ts +2136 -0
package/dist/adapters/entries/curl.js +2 -0
package/dist/adapters/entries/fetch.cjs +2 -0
package/dist/adapters/entries/fetch.d.ts +2127 -0
package/dist/adapters/entries/fetch.js +1 -0
package/dist/adapters/entries/http.cjs +2 -0
package/dist/adapters/entries/http.d.ts +2126 -0
package/dist/adapters/entries/http.js +1 -0
package/dist/adapters/entries/http2.cjs +4 -0
package/dist/adapters/entries/http2.d.ts +2136 -0
package/dist/adapters/entries/http2.js +2 -0
package/dist/adapters/entries/react-native.cjs +2 -0
package/dist/adapters/entries/react-native.d.ts +2126 -0
package/dist/adapters/entries/react-native.js +1 -0
package/dist/adapters/entries/xhr.cjs +2 -0
package/dist/adapters/entries/xhr.d.ts +2127 -0
package/dist/adapters/entries/xhr.js +1 -0
package/dist/adapters/fetch.cjs +740 -0
package/dist/adapters/fetch.js +739 -0
package/dist/adapters/http.cjs +1153 -0
package/dist/adapters/http.js +1151 -0
package/dist/adapters/http2.cjs +957 -0
package/dist/adapters/http2.js +956 -0
package/dist/adapters/index.cjs +6 -0
package/dist/adapters/index.js +7 -0
package/dist/adapters/picker.cjs +342 -0
package/dist/adapters/picker.js +331 -0
package/dist/adapters/react-native.cjs +545 -0
package/dist/adapters/react-native.js +544 -0
package/dist/adapters/xhr.cjs +622 -0
package/dist/adapters/xhr.js +621 -0
package/dist/cache/dns-cache.cjs +118 -0
package/dist/cache/dns-cache.js +113 -0
package/dist/cache/file-cacher.cjs +264 -0
package/dist/cache/file-cacher.js +261 -0
package/dist/cache/index.cjs +13 -0
package/dist/cache/index.js +5 -0
package/dist/cache/lru-cache.cjs +96 -0
package/dist/cache/lru-cache.js +93 -0
package/dist/cache/response-cache.cjs +314 -0
package/dist/cache/response-cache.js +310 -0
package/dist/cache/url-store.cjs +288 -0
package/dist/cache/url-store.js +285 -0
package/dist/core/hooks.cjs +133 -0
package/dist/core/hooks.js +120 -0
package/dist/core/rezo.cjs +464 -0
package/dist/core/rezo.js +458 -0
package/dist/crawler.d.ts +6255 -0
package/dist/dom/index.cjs +1 -0
package/dist/dom/index.d.ts +23 -0
package/dist/dom/index.js +1 -0
package/dist/entries/crawler.cjs +5 -0
package/dist/entries/crawler.js +2 -0
package/dist/errors/rezo-error.cjs +722 -0
package/dist/errors/rezo-error.js +716 -0
package/dist/index.cjs +34 -0
package/dist/index.d.ts +3335 -0
package/dist/index.js +26 -0
package/dist/platform/browser.cjs +9 -0
package/dist/platform/browser.d.ts +3203 -0
package/dist/platform/browser.js +7 -0
package/dist/platform/bun.cjs +9 -0
package/dist/platform/bun.d.ts +3203 -0
package/dist/platform/bun.js +7 -0
package/dist/platform/deno.cjs +9 -0
package/dist/platform/deno.d.ts +3203 -0
package/dist/platform/deno.js +7 -0
package/dist/platform/node.cjs +9 -0
package/dist/platform/node.d.ts +3203 -0
package/dist/platform/node.js +7 -0
package/dist/platform/react-native.cjs +9 -0
package/dist/platform/react-native.d.ts +3203 -0
package/dist/platform/react-native.js +7 -0
package/dist/platform/worker.cjs +9 -0
package/dist/platform/worker.d.ts +3203 -0
package/dist/platform/worker.js +7 -0
package/dist/plugin/addon/decodo/index.cjs +1 -0
package/dist/plugin/addon/decodo/index.js +1 -0
package/dist/plugin/addon/decodo/options.cjs +1 -0
package/dist/plugin/addon/decodo/options.js +1 -0
package/dist/plugin/addon/oxylabs/index.cjs +1 -0
package/dist/plugin/addon/oxylabs/index.js +1 -0
package/dist/plugin/addon/oxylabs/options.cjs +1 -0
package/dist/plugin/addon/oxylabs/options.js +1 -0
package/dist/plugin/crawler-options.cjs +1 -0
package/dist/plugin/crawler-options.js +1 -0
package/dist/plugin/crawler.cjs +519 -0
package/dist/plugin/crawler.js +517 -0
package/dist/plugin/index.cjs +36 -0
package/dist/plugin/index.js +32 -0
package/dist/proxy/index.cjs +142 -0
package/dist/proxy/index.js +139 -0
package/dist/responses/buildError.cjs +452 -0
package/dist/responses/buildError.js +441 -0
package/dist/responses/buildResponse.cjs +365 -0
package/dist/responses/buildResponse.js +361 -0
package/dist/responses/download.cjs +54 -0
package/dist/responses/download.js +52 -0
package/dist/responses/stream.cjs +60 -0
package/dist/responses/stream.js +58 -0
package/dist/responses/upload.cjs +54 -0
package/dist/responses/upload.js +52 -0
package/dist/types/cookies.cjs +394 -0
package/dist/types/cookies.js +391 -0
package/dist/types/download.cjs +10 -0
package/dist/types/download.js +10 -0
package/dist/types/rezo-request.cjs +131 -0
package/dist/types/rezo-request.js +131 -0
package/dist/utils/agent-merger.cjs +111 -0
package/dist/utils/agent-merger.js +108 -0
package/dist/utils/compression.cjs +84 -0
package/dist/utils/compression.js +82 -0
package/dist/utils/cookies.cjs +514 -0
package/dist/utils/cookies.js +511 -0
package/dist/utils/data-operations.cjs +75 -0
package/dist/utils/data-operations.js +73 -0
package/dist/utils/form-data.cjs +164 -0
package/dist/utils/form-data.js +161 -0
package/dist/utils/headers.cjs +162 -0
package/dist/utils/headers.js +161 -0
package/dist/utils/http-config.cjs +723 -0
package/dist/utils/http-config.js +718 -0
package/dist/utils/index.cjs +8 -0
package/dist/utils/index.js +8 -0
package/dist/utils/tools.cjs +18 -0
package/dist/utils/tools.js +15 -0
package/package.json +172 -0

package/dist/plugin/crawler.js ADDED Viewed

@@ -0,0 +1,517 @@
+import fs from "node:fs";
+import { FileCacher } from '../cache/file-cacher.js';
+import { UrlStore } from '../cache/url-store.js';
+import { parseHTML } from "linkedom";
+import path from "node:path";
+import PQueue from "p-queue";
+import { Scraper } from './scraper.js';
+import { CrawlerOptions } from './crawler-options.js';
+String.prototype.addBaseUrl = function(url) {
+  url = url instanceof URL ? url.href : url;
+  const html = this.replace(/<base\b[^>]*?>/gi, "");
+  if (/<head[^>]*>/i.test(html)) {
+    return html.replace(/<head[^>]*>/i, (match) => `${match}
+<base href="${url}">`);
+  }
+  const baseTag = `<head>
+<base href="${url}">
+</head>
+`;
+  if (/<body[^>]*>/i.test(html)) {
+    return html.replace(/<body[^>]*>/i, baseTag + "$&");
+  }
+  if (/<html[^>]*>/i.test(html)) {
+    return html.replace(/<html[^>]*>/i, `$&
+` + baseTag);
+  }
+  return this;
+};
+export class Crawler {
+  http;
+  events = [];
+  jsonEvents = [];
+  errorEvents = [];
+  responseEvents = [];
+  rawResponseEvents = [];
+  emailDiscoveredEvents = [];
+  emailLeadsEvents = [];
+  cacher = null;
+  queue;
+  isCacheEnabled;
+  config;
+  urlStorage;
+  isStorageReady = false;
+  isCacheReady = false;
+  leadsFinder;
+  constructor(crawlerOptions, http) {
+    this.http = http;
+    this.queue = new PQueue({
+      concurrency: 1000
+    });
+    this.config = new CrawlerOptions(crawlerOptions);
+    const enableCache = this.config.enableCache;
+    this.isCacheEnabled = enableCache;
+    if (enableCache) {
+      const cacheDir = this.config.cacheDir;
+      const cacheTTL = this.config.cacheTTL;
+      const dbUrl = cacheDir && (cacheDir.startsWith("./") || cacheDir.startsWith("/")) ? `${cacheDir}${cacheDir.endsWith("/") ? "" : "/"}` : cacheDir ? `./${cacheDir}${cacheDir.endsWith("/") ? "" : "/"}` : `./cache/`;
+      if (!fs.existsSync(path.dirname(dbUrl)))
+        fs.mkdirSync(path.dirname(dbUrl), { recursive: true });
+      FileCacher.create({
+        cacheDir: dbUrl,
+        softDelete: false,
+        ttl: cacheTTL,
+        encryptNamespace: true
+      }).then((storage) => {
+        this.cacher = storage;
+        this.isCacheReady = true;
+      });
+      const dit = path.resolve(cacheDir, "urls");
+      if (!fs.existsSync(dit))
+        fs.mkdirSync(dit, { recursive: true });
+      UrlStore.create({
+        storeDir: dit,
+        dbFileName: ".url_cache.db",
+        ttl: 1000 * 60 * 60 * 24 * 7
+      }).then((storage) => {
+        this.urlStorage = storage;
+        this.isStorageReady = true;
+      });
+    } else {
+      const dit = path.resolve(this.config.cacheDir, "./cache/urls");
+      if (!fs.existsSync(dit))
+        fs.mkdirSync(dit, { recursive: true });
+      UrlStore.create({
+        storeDir: dit,
+        dbFileName: ".url_cache.db",
+        ttl: 1000 * 60 * 60 * 24 * 7
+      }).then((storage) => {
+        this.urlStorage = storage;
+        this.isStorageReady = true;
+      });
+    }
+    this.leadsFinder = new Scraper(this.http, this.config, this._onEmailLeads.bind(this), this._onEmailDiscovered.bind(this), this.config.debug);
+  }
+  rawResponseHandler(data) {
+    if (this.rawResponseEvents.length === 0)
+      return;
+    const isBuffer = data instanceof Buffer;
+    if (!isBuffer) {
+      if (data instanceof ArrayBuffer) {
+        data = Buffer.from(new Uint8Array(data));
+      } else if (data instanceof Uint8Array) {
+        data = Buffer.from(data);
+      } else if (typeof data === "string") {
+        data = Buffer.from(data, "utf8");
+      } else if (typeof data === "object") {
+        data = Buffer.from(JSON.stringify(data), "utf8");
+      }
+    }
+    this.rawResponseEvents.forEach((e) => {
+      const handler = e.attr[0];
+      handler(data);
+    });
+  }
+  async waitForCache() {
+    if (this.isCacheReady)
+      return;
+    await this.sleep(this.rnd(50, 200));
+    await this.waitForCache();
+  }
+  async waitForStorage() {
+    if (this.isStorageReady)
+      return;
+    await this.sleep(this.rnd(50, 200));
+    await this.waitForStorage();
+  }
+  async saveUrl(url) {
+    await this.waitForStorage();
+    await this.urlStorage.set(url);
+  }
+  async hasUrlInCache(url) {
+    await this.waitForStorage();
+    return await this.urlStorage.has(url);
+  }
+  async saveCache(url, value) {
+    if (!this.isCacheEnabled)
+      return;
+    await this.waitForCache();
+    return this.cacher.set(url, value, this.config.cacheTTL, this.getNamespace(url));
+  }
+  getNamespace(url) {
+    try {
+      return new URL(url).hostname;
+    } catch {
+      return;
+    }
+  }
+  async hasCache(url) {
+    if (!this.isCacheEnabled)
+      return false;
+    await this.waitForCache();
+    return this.cacher.has(url, this.getNamespace(url));
+  }
+  async getCache(url) {
+    if (!this.isCacheEnabled)
+      return null;
+    await this.waitForCache();
+    return this.cacher.get(url, this.getNamespace(url));
+  }
+  sleep(ms) {
+    return new Promise((resolve) => setTimeout(resolve, ms));
+  }
+  rnd(min = 0, max = Number.MAX_VALUE) {
+    return Math.floor(Math.random() * (max - min + 1)) + min;
+  }
+  onError(handler) {
+    this.errorEvents.push({
+      handler: "_onError",
+      attr: [handler]
+    });
+    return this;
+  }
+  onJson(handler) {
+    this.jsonEvents.push({
+      handler: "_onJson",
+      attr: [handler]
+    });
+    return this;
+  }
+  onEmailDiscovered(handler) {
+    this.emailDiscoveredEvents.push(handler);
+    return this;
+  }
+  onEmailLeads(handler) {
+    this.emailLeadsEvents.push(handler);
+    return this;
+  }
+  onRawData(handler) {
+    this.rawResponseEvents.push({
+      handler: "_onRawResponse",
+      attr: [handler]
+    });
+    return this;
+  }
+  onDocument(handler) {
+    this.events.push({
+      handler: "_onDocument",
+      attr: [handler]
+    });
+    return this;
+  }
+  onBody(handler) {
+    this.events.push({
+      handler: "_onBody",
+      attr: [handler]
+    });
+    return this;
+  }
+  onElement(handler) {
+    this.events.push({
+      handler: "_onElement",
+      attr: [handler]
+    });
+    return this;
+  }
+  onAnchor(selection, handler) {
+    this.events.push({
+      handler: "_onAnchor",
+      attr: [selection, handler]
+    });
+    return this;
+  }
+  onHref(handler) {
+    this.events.push({
+      handler: "_onHref",
+      attr: [handler]
+    });
+    return this;
+  }
+  onSelection(selection, handler) {
+    this.events.push({
+      handler: "_onSelection",
+      attr: [selection, handler]
+    });
+    return this;
+  }
+  onResponse(handler) {
+    this.responseEvents.push({
+      handler: "_onResponse",
+      attr: [handler]
+    });
+    return this;
+  }
+  onAttribute(selection, attribute, handler) {
+    this.events.push({
+      handler: "_onAttribute",
+      attr: [selection, attribute, handler]
+    });
+    return this;
+  }
+  onText(selection, handler) {
+    this.events.push({
+      handler: "_onText",
+      attr: [selection, handler]
+    });
+    return this;
+  }
+  _onBody(handler, document) {
+    this.queue.add(() => handler(document.body));
+  }
+  _onAttribute(selection, attribute, handler, document) {
+    selection = typeof attribute === "function" ? selection : null;
+    attribute = typeof attribute === "function" ? selection : attribute;
+    handler = typeof attribute === "function" ? attribute : handler;
+    selection = selection || `[${attribute}]`;
+    const elements = document.querySelectorAll(selection);
+    for (let i = 0;i < elements.length; i++) {
+      if (elements[i].hasAttribute(attribute))
+        this.queue.add(() => handler(elements[i].getAttribute(attribute)));
+    }
+  }
+  _onText(selection, handler, document) {
+    const elements = document.querySelectorAll(selection);
+    for (let i = 0;i < elements.length; i++) {
+      this.queue.add(() => handler(elements[i].textContent));
+    }
+  }
+  _onSelection(selection, handler, document) {
+    const elements = document.querySelectorAll(selection);
+    for (let i = 0;i < elements.length; i++) {
+      this.queue.add(() => handler(elements[i]));
+    }
+  }
+  _onElement(handler, document) {
+    const elements = document.querySelectorAll("*");
+    for (let i = 0;i < elements.length; i++) {
+      this.queue.add(() => handler(elements[i]));
+    }
+  }
+  _onHref(handler, document) {
+    const elements = document.querySelectorAll("a, link");
+    for (let i = 0;i < elements.length; i++) {
+      if (elements[i].hasAttribute("href"))
+        this.queue.add(() => handler(new URL(elements[i].getAttribute("href"), document.URL).href));
+    }
+  }
+  _onAnchor(selection, handler, document) {
+    handler = typeof selection === "function" ? selection : handler;
+    selection = typeof selection === "function" ? "a" : selection;
+    const elements = document.querySelectorAll(selection);
+    for (let i = 0;i < elements.length; i++) {
+      if (elements[i]?.href && document.baseURI)
+        elements[i].href = new URL(elements[i].getAttribute("href"), document.baseURI).href;
+      this.queue.add(() => handler(elements[i]));
+    }
+  }
+  _onDocument(handler, document) {
+    this.queue.add(() => handler(document));
+  }
+  _onJson(handler, json) {
+    this.queue.add(() => handler(json));
+  }
+  _onError(handler, error) {
+    this.queue.add(() => handler(error));
+  }
+  async _onEmailDiscovered(handler, email) {
+    await handler(email);
+  }
+  async _onEmailLeads(handler, emails) {
+    await handler(emails);
+  }
+  _onRawResponse(handler, rawResponse) {
+    this.queue.add(() => handler(rawResponse));
+  }
+  _onResponse(handler, response) {
+    this.queue.add(() => handler(response));
+  }
+  buildUrl(url, params) {
+    if (params) {
+      const u = new URL(url, this.config.baseUrl);
+      for (const [key, value] of Object.entries(params)) {
+        u.searchParams.set(key, value.toString());
+      }
+      url = u.href;
+    }
+    return url;
+  }
+  visit(url, options) {
+    if (this.config.baseUrl)
+      url = new URL(url, this.config.baseUrl).href;
+    if (options?.params && (options.useOxylabsScraperAi || this.config.hasDomain(url, "oxylabs"))) {
+      url = this.buildUrl(url, options.params);
+    }
+    const {
+      method = "GET",
+      headers = new Headers,
+      forceRevisit = this.config.forceRevisit,
+      body = "",
+      timeout = this.config.timeout,
+      maxRedirects = this.config.maxRedirects,
+      useProxy = this.config.hasDomain(url, "proxies", options?.useProxy),
+      extractLeads = false,
+      params,
+      rejectUnauthorized,
+      useQueue = false,
+      deepEmailFinder = false,
+      useOxylabsScraperAi = false,
+      useOxylabsRotation = true,
+      useDecodo = false
+    } = options || {};
+    const _options = {
+      headers: this.config.pickHeaders(url, true, headers, true),
+      timeout,
+      maxRedirects,
+      params,
+      proxy: useProxy ? this.config.getAdapter(url, "proxies", true, true) || undefined : undefined,
+      rejectUnauthorized: typeof rejectUnauthorized === "boolean" ? rejectUnauthorized : this.config.rejectUnauthorized,
+      pqueue: this.config.getAdapter(url, "limiters", useQueue, useQueue) || undefined
+    };
+    let oxylabsOptions = {};
+    let oxylabsInstanse = undefined;
+    if (useOxylabsScraperAi && this.config.hasDomain(url, "oxylabs")) {
+      oxylabsOptions = {
+        method: method === "POST" ? "post" : "get",
+        headers: this.config.pickHeaders(url, true, headers, true),
+        pqueue: this.config.getAdapter(url, "limiters", useQueue, useQueue) || undefined,
+        base64Body: typeof body === "string" ? Buffer.from(body).toString("base64") : undefined
+      };
+      oxylabsInstanse = this.config.getAdapter(url, "oxylabs", false, useOxylabsRotation) || undefined;
+    }
+    let decodoOptions = {};
+    let decodoInstanse = undefined;
+    if (useDecodo && this.config.hasDomain(url, "decodo")) {
+      decodoOptions = {
+        method: method === "POST" ? "post" : "get",
+        headers: this.config.pickHeaders(url, true, headers, true),
+        pqueue: this.config.getAdapter(url, "limiters", useQueue, useQueue) || undefined,
+        base64Body: typeof body === "string" ? Buffer.from(body).toString("base64") : undefined
+      };
+      decodoInstanse = this.config.getAdapter(url, "decodo", false, useOxylabsRotation) || undefined;
+    }
+    if (deepEmailFinder) {
+      this.execute2(method, url, body, _options, forceRevisit).then();
+      return this;
+    }
+    this.execute(method, url, body, _options, extractLeads, forceRevisit, oxylabsOptions, oxylabsInstanse, decodoInstanse, decodoOptions).then();
+    return this;
+  }
+  async execute(method, url, body, options = {}, isEmail, forceRevisit, oxylabsOptions, oxylabsInstanse, decodoInstanse, decodoOptions) {
+    this.queue.add(() => this.executeHttp(method, url, body, options, isEmail, forceRevisit, oxylabsOptions, oxylabsInstanse, decodoInstanse, decodoOptions)).then();
+  }
+  async execute2(method, url, body, options = {}, forceRevisit) {
+    this.queue.add(() => this.leadsFinder.parseExternalWebsite(url, method, body, {
+      httpConfig: options,
+      saveCache: this.saveCache.bind(this),
+      saveUrl: this.saveUrl.bind(this),
+      getCache: this.getCache.bind(this),
+      hasUrlInCache: this.hasUrlInCache.bind(this),
+      onEmailDiscovered: this.emailDiscoveredEvents,
+      onEmails: this.emailLeadsEvents,
+      queue: this.queue,
+      depth: 1,
+      allowCrossDomainTravel: true
+    }, forceRevisit, true)).then();
+  }
+  async executeHttp(method, url, body, options = {}, isEmail, forceRevisit, oxylabsOptions, oxylabsInstanse, decodoInstanse, decodoOptions, retryCount = 0) {
+    try {
+      console.log({
+        oxylabsOptions: typeof oxylabsOptions,
+        oxylabsInstanse: typeof oxylabsInstanse,
+        decodoInstanse: typeof decodoInstanse,
+        decodoOptions: typeof decodoOptions
+      });
+      const isVisited = forceRevisit ? false : await this.hasUrlInCache(url);
+      const cache = await this.getCache(url);
+      if (isVisited && !cache)
+        return;
+      if (isVisited && method !== "GET")
+        return;
+      const response = cache && method === "GET" ? cache : oxylabsInstanse && oxylabsOptions ? await oxylabsInstanse.scrape(url) : decodoInstanse && decodoOptions ? await decodoInstanse.scrape(url) : await (method === "GET" ? this.http.get(url, options) : method === "PATCH" ? this.http.patch(url, body, options) : method === "POST" ? this.http.post(url, body, options) : this.http.put(url, body, options));
+      const res = {
+        data: response.data || response.content || "",
+        contentType: response.contentType || "",
+        finalUrl: response.finalUrl || response.url || url,
+        url: response?.urls?.[0] || response.url || this.buildUrl(url, options.params),
+        headers: response.headers || {},
+        status: response.status || response.statusCode || 200,
+        statusText: response.statusText || "",
+        cookies: response?.cookies?.serialized || response?.cookies,
+        contentLength: response.contentLength || 0
+      };
+      if (!cache)
+        await this.saveCache(url, res);
+      if (!isVisited)
+        await this.saveUrl(url);
+      if (res.contentType && res.contentType.includes("/json")) {
+        if (this.emailDiscoveredEvents.length > 0 || this.emailLeadsEvents.length > 0) {
+          this.leadsFinder.extractEmails(JSON.stringify(res.data), res.finalUrl, this.emailDiscoveredEvents, this.emailLeadsEvents, this.queue);
+        }
+        for (let i = 0;i < this.jsonEvents.length; i++) {
+          const event = this.jsonEvents[i];
+          this[event.handler](...event.attr, res.data);
+        }
+      }
+      for (let i = 0;i < this.responseEvents.length; i++) {
+        const event = this.responseEvents[i];
+        this[event.handler](...event.attr, res);
+      }
+      this.rawResponseHandler(res.data);
+      if (!res.contentType || !res.contentType.includes("/html") || typeof res.data !== "string")
+        return;
+      if ((this.emailDiscoveredEvents.length > 0 || this.emailLeadsEvents.length > 0) && isEmail) {
+        this.leadsFinder.extractEmails(res.data, res.finalUrl, this.emailDiscoveredEvents, this.emailLeadsEvents, this.queue);
+      }
+      const { document } = parseHTML(res.data.addBaseUrl(res.finalUrl));
+      document.URL = res.finalUrl;
+      for (let i = 0;i < this.events.length; i++) {
+        const event = this.events[i];
+        this[event.handler](...event.attr, document);
+      }
+    } catch (e) {
+      const error = e;
+      if (error && error.response) {
+        const status = error.response.status;
+        const retryDelay = this.config.retryDelay || 1000;
+        const maxRetryAttempts = this.config.maxRetryAttempts || 3;
+        const maxRetryOnProxyError = this.config.maxRetryOnProxyError || 3;
+        const retryWithoutProxyOnStatusCode = this.config.retryWithoutProxyOnStatusCode || undefined;
+        const retryOnStatusCode = this.config.retryOnStatusCode || undefined;
+        const retryOnProxyError = this.config.retryOnProxyError || undefined;
+        if (retryWithoutProxyOnStatusCode && options.proxy && retryWithoutProxyOnStatusCode.includes(status) && retryCount < maxRetryAttempts) {
+          await this.sleep(retryDelay);
+          delete options.proxy;
+          return await this.executeHttp(method, url, body, options, isEmail, forceRevisit, oxylabsOptions, oxylabsInstanse, decodoInstanse, decodoOptions, retryCount + 1);
+        } else if (retryOnStatusCode && options.proxy && retryOnStatusCode.includes(status) && retryCount < maxRetryAttempts) {
+          await this.sleep(retryDelay);
+          return await this.executeHttp(method, url, body, options, isEmail, forceRevisit, oxylabsOptions, oxylabsInstanse, decodoInstanse, decodoOptions, retryCount + 1);
+        } else if (retryOnProxyError && options.proxy && retryCount < maxRetryOnProxyError) {
+          await this.sleep(retryDelay);
+          return await this.executeHttp(method, url, body, options, isEmail, forceRevisit, oxylabsOptions, oxylabsInstanse, decodoInstanse, decodoOptions, retryCount + 1);
+        }
+      }
+      if (this.config.throwFatalError)
+        throw e;
+      if (this.config.debug) {
+        console.log(`Error visiting ${url}: ${e.message}`);
+      }
+      console.log(error);
+      for (let i = 0;i < this.errorEvents.length; i++) {
+        const event = this.errorEvents[i];
+        this[event.handler](...event.attr, e);
+      }
+    }
+  }
+  async waitForAll() {
+    await this.queue.onIdle();
+  }
+  async close() {
+    try {
+      await this.cacher.close();
+    } catch {}
+    try {
+      await this.urlStorage.close();
+    } catch {}
+  }
+}

package/dist/plugin/index.cjs ADDED Viewed

@@ -0,0 +1,36 @@
+const _mod_4lomw8 = require('./crawler.cjs');
+exports.Crawler = _mod_4lomw8.Crawler;;
+const _mod_w4j7fu = require('./crawler-options.cjs');
+exports.CrawlerOptions = _mod_w4j7fu.CrawlerOptions;;
+const _mod_ek7ks1 = require('../cache/file-cacher.cjs');
+exports.FileCacher = _mod_ek7ks1.FileCacher;;
+const _mod_0txncg = require('../cache/url-store.cjs');
+exports.UrlStore = _mod_0txncg.UrlStore;;
+const _mod_y87964 = require('./addon/oxylabs/index.cjs');
+exports.Oxylabs = _mod_y87964.Oxylabs;;
+const _mod_z2fqld = require('./addon/oxylabs/options.cjs');
+exports.OXYLABS_BROWSER_TYPES = _mod_z2fqld.OXYLABS_BROWSER_TYPES;
+exports.OXYLABS_COMMON_LOCALES = _mod_z2fqld.OXYLABS_COMMON_LOCALES;
+exports.OXYLABS_COMMON_GEO_LOCATIONS = _mod_z2fqld.OXYLABS_COMMON_GEO_LOCATIONS;
+exports.OXYLABS_US_STATES = _mod_z2fqld.OXYLABS_US_STATES;
+exports.OXYLABS_EUROPEAN_COUNTRIES = _mod_z2fqld.OXYLABS_EUROPEAN_COUNTRIES;
+exports.OXYLABS_ASIAN_COUNTRIES = _mod_z2fqld.OXYLABS_ASIAN_COUNTRIES;
+exports.getRandomOxylabsBrowserType = _mod_z2fqld.getRandomBrowserType;
+exports.getRandomOxylabsLocale = _mod_z2fqld.getRandomLocale;
+exports.getRandomOxylabsGeoLocation = _mod_z2fqld.getRandomGeoLocation;;
+const _mod_v2ufiz = require('./addon/decodo/index.cjs');
+exports.Decodo = _mod_v2ufiz.Decodo;;
+const _mod_3pykq7 = require('./addon/decodo/options.cjs');
+exports.DECODO_DEVICE_TYPES = _mod_3pykq7.DECODO_DEVICE_TYPES;
+exports.DECODO_HEADLESS_MODES = _mod_3pykq7.DECODO_HEADLESS_MODES;
+exports.DECODO_COMMON_LOCALES = _mod_3pykq7.DECODO_COMMON_LOCALES;
+exports.DECODO_COMMON_COUNTRIES = _mod_3pykq7.DECODO_COMMON_COUNTRIES;
+exports.DECODO_EUROPEAN_COUNTRIES = _mod_3pykq7.DECODO_EUROPEAN_COUNTRIES;
+exports.DECODO_ASIAN_COUNTRIES = _mod_3pykq7.DECODO_ASIAN_COUNTRIES;
+exports.DECODO_US_STATES = _mod_3pykq7.DECODO_US_STATES;
+exports.DECODO_COMMON_CITIES = _mod_3pykq7.DECODO_COMMON_CITIES;
+exports.getRandomDecodoDeviceType = _mod_3pykq7.getRandomDeviceType;
+exports.getRandomDecodoLocale = _mod_3pykq7.getRandomLocale;
+exports.getRandomDecodoCountry = _mod_3pykq7.getRandomCountry;
+exports.getRandomDecodoCity = _mod_3pykq7.getRandomCity;
+exports.generateDecodoSessionId = _mod_3pykq7.generateSessionId;;

package/dist/plugin/index.js ADDED Viewed

@@ -0,0 +1,32 @@
+export { Crawler } from './crawler.js';
+export { CrawlerOptions } from './crawler-options.js';
+export { FileCacher } from '../cache/file-cacher.js';
+export { UrlStore } from '../cache/url-store.js';
+export { Oxylabs } from './addon/oxylabs/index.js';
+export {
+  OXYLABS_BROWSER_TYPES,
+  OXYLABS_COMMON_LOCALES,
+  OXYLABS_COMMON_GEO_LOCATIONS,
+  OXYLABS_US_STATES,
+  OXYLABS_EUROPEAN_COUNTRIES,
+  OXYLABS_ASIAN_COUNTRIES,
+  getRandomBrowserType as getRandomOxylabsBrowserType,
+  getRandomLocale as getRandomOxylabsLocale,
+  getRandomGeoLocation as getRandomOxylabsGeoLocation
+} from './addon/oxylabs/options.js';
+export { Decodo } from './addon/decodo/index.js';
+export {
+  DECODO_DEVICE_TYPES,
+  DECODO_HEADLESS_MODES,
+  DECODO_COMMON_LOCALES,
+  DECODO_COMMON_COUNTRIES,
+  DECODO_EUROPEAN_COUNTRIES,
+  DECODO_ASIAN_COUNTRIES,
+  DECODO_US_STATES,
+  DECODO_COMMON_CITIES,
+  getRandomDeviceType as getRandomDecodoDeviceType,
+  getRandomLocale as getRandomDecodoLocale,
+  getRandomCountry as getRandomDecodoCountry,
+  getRandomCity as getRandomDecodoCity,
+  generateSessionId as generateDecodoSessionId
+} from './addon/decodo/options.js';