rezo 1.0.130 → 1.0.132

package/dist/adapters/entries/curl.d.ts

@@ -6027,7 +6027,7 @@ export interface RezoInstance extends Rezo, RezoCallable {
  *
  * IMPORTANT: Update these values when bumping package version.
  */
-export declare const VERSION = "1.0.130";
+export declare const VERSION = "1.0.132";
 /**
  * cURL Options Configuration
  *

package/dist/adapters/entries/fetch.d.ts

@@ -6027,7 +6027,7 @@ export interface RezoInstance extends Rezo, RezoCallable {
  *
  * IMPORTANT: Update these values when bumping package version.
  */
-export declare const VERSION = "1.0.130";
+export declare const VERSION = "1.0.132";
 export declare const isRezoError: typeof RezoError.isRezoError;
 export declare const Cancel: typeof RezoError;
 export declare const CancelToken: {

package/dist/adapters/entries/http.d.ts

@@ -6027,7 +6027,7 @@ export interface RezoInstance extends Rezo, RezoCallable {
  *
  * IMPORTANT: Update these values when bumping package version.
  */
-export declare const VERSION = "1.0.130";
+export declare const VERSION = "1.0.132";
 /**
  * Type guard to check if an error is a RezoError instance.
  */

package/dist/adapters/entries/http2.d.ts

@@ -6027,7 +6027,7 @@ export interface RezoInstance extends Rezo, RezoCallable {
  *
  * IMPORTANT: Update these values when bumping package version.
  */
-export declare const VERSION = "1.0.130";
+export declare const VERSION = "1.0.132";
 export declare const isRezoError: typeof RezoError.isRezoError;
 export declare const Cancel: typeof RezoError;
 export declare const CancelToken: {

package/dist/adapters/entries/react-native.cjs

@@ -6,12 +6,12 @@ const { RezoFormData } = require('../../utils/form-data.cjs');
 const { RezoCookieJar, Cookie } = require('../../cookies/cookie-jar.cjs');
 const { createDefaultHooks, mergeHooks } = require('../../core/hooks.cjs');
 const { VERSION } = require('../../version.cjs');
-const _mod_fft01t = require('../../platform/react-native-providers.cjs');
-exports.createFetchStreamTransport = _mod_fft01t.createFetchStreamTransport;
-exports.createExpoFileSystemAdapter = _mod_fft01t.createExpoFileSystemAdapter;
-exports.createReactNativeFsAdapter = _mod_fft01t.createReactNativeFsAdapter;
-exports.createNetInfoProvider = _mod_fft01t.createNetInfoProvider;
-exports.createExpoBackgroundTaskProvider = _mod_fft01t.createExpoBackgroundTaskProvider;;
+const _mod_jspfje = require('../../platform/react-native-providers.cjs');
+exports.createFetchStreamTransport = _mod_jspfje.createFetchStreamTransport;
+exports.createExpoFileSystemAdapter = _mod_jspfje.createExpoFileSystemAdapter;
+exports.createReactNativeFsAdapter = _mod_jspfje.createReactNativeFsAdapter;
+exports.createNetInfoProvider = _mod_jspfje.createNetInfoProvider;
+exports.createExpoBackgroundTaskProvider = _mod_jspfje.createExpoBackgroundTaskProvider;;
 exports.Rezo = Rezo;
 exports.RezoError = RezoError;
 exports.RezoErrorCode = RezoErrorCode;

package/dist/adapters/entries/react-native.d.ts

@@ -6027,7 +6027,7 @@ export interface RezoInstance extends Rezo, RezoCallable {
  *
  * IMPORTANT: Update these values when bumping package version.
  */
-export declare const VERSION = "1.0.130";
+export declare const VERSION = "1.0.132";
 export interface ExpoFileSystemFileLike {
 	uri?: string;
 	size?: number;

package/dist/adapters/entries/xhr.d.ts

@@ -6027,7 +6027,7 @@ export interface RezoInstance extends Rezo, RezoCallable {
  *
  * IMPORTANT: Update these values when bumping package version.
  */
-export declare const VERSION = "1.0.130";
+export declare const VERSION = "1.0.132";
 export declare const isRezoError: typeof RezoError.isRezoError;
 export declare const Cancel: typeof RezoError;
 export declare const CancelToken: {

package/dist/adapters/index.cjs

@@ -1,6 +1,6 @@
-const _mod_qhux44 = require('./picker.cjs');
-exports.detectRuntime = _mod_qhux44.detectRuntime;
-exports.getAdapterCapabilities = _mod_qhux44.getAdapterCapabilities;
-exports.buildAdapterContext = _mod_qhux44.buildAdapterContext;
-exports.getAvailableAdapters = _mod_qhux44.getAvailableAdapters;
-exports.selectAdapter = _mod_qhux44.selectAdapter;;
+const _mod_pq418y = require('./picker.cjs');
+exports.detectRuntime = _mod_pq418y.detectRuntime;
+exports.getAdapterCapabilities = _mod_pq418y.getAdapterCapabilities;
+exports.buildAdapterContext = _mod_pq418y.buildAdapterContext;
+exports.getAvailableAdapters = _mod_pq418y.getAvailableAdapters;
+exports.selectAdapter = _mod_pq418y.selectAdapter;;

package/dist/cache/dns-cache.cjs

@@ -1,5 +1,5 @@
 const { LRUCache } = require('./lru-cache.cjs');
-const dns = require("node:dns");
+const { requireNodeModule } = require('../utils/node-runtime.cjs');
 const DEFAULT_DNS_TTL = 60000;
 const DEFAULT_DNS_MAX_ENTRIES = 1000;
 
@@ -59,6 +59,11 @@ class DNSCache {
   }
   resolveDNS(hostname, family) {
     return new Promise((resolve) => {
+      const dns = requireNodeModule("node:dns");
+      if (!dns?.lookup) {
+        resolve(undefined);
+        return;
+      }
       const options = { family: family ?? 0 };
       dns.lookup(hostname, options, (err, address, resultFamily) => {
         if (err || !address || typeof address !== "string") {
@@ -71,6 +76,11 @@ class DNSCache {
   }
   resolveAllDNS(hostname, family) {
     return new Promise((resolve) => {
+      const dns = requireNodeModule("node:dns");
+      if (!dns?.lookup) {
+        resolve([]);
+        return;
+      }
       const options = { family: family ?? 0, all: true };
       dns.lookup(hostname, options, (err, addresses) => {
         if (err || !addresses || !Array.isArray(addresses)) {

package/dist/cache/dns-cache.js

@@ -1,5 +1,5 @@
 import { LRUCache } from './lru-cache.js';
-import dns from "node:dns";
+import { requireNodeModule } from '../utils/node-runtime.js';
 const DEFAULT_DNS_TTL = 60000;
 const DEFAULT_DNS_MAX_ENTRIES = 1000;
 
@@ -59,6 +59,11 @@ export class DNSCache {
   }
   resolveDNS(hostname, family) {
     return new Promise((resolve) => {
+      const dns = requireNodeModule("node:dns");
+      if (!dns?.lookup) {
+        resolve(undefined);
+        return;
+      }
       const options = { family: family ?? 0 };
       dns.lookup(hostname, options, (err, address, resultFamily) => {
         if (err || !address || typeof address !== "string") {
@@ -71,6 +76,11 @@ export class DNSCache {
   }
   resolveAllDNS(hostname, family) {
     return new Promise((resolve) => {
+      const dns = requireNodeModule("node:dns");
+      if (!dns?.lookup) {
+        resolve([]);
+        return;
+      }
       const options = { family: family ?? 0, all: true };
       dns.lookup(hostname, options, (err, addresses) => {
         if (err || !addresses || !Array.isArray(addresses)) {

package/dist/cache/index.cjs

@@ -1,9 +1,9 @@
-const _mod_66d5y0 = require('./lru-cache.cjs');
-exports.LRUCache = _mod_66d5y0.LRUCache;;
-const _mod_fiylx5 = require('./dns-cache.cjs');
-exports.DNSCache = _mod_fiylx5.DNSCache;
-exports.getGlobalDNSCache = _mod_fiylx5.getGlobalDNSCache;
-exports.resetGlobalDNSCache = _mod_fiylx5.resetGlobalDNSCache;;
-const _mod_t0ymbs = require('./response-cache.cjs');
-exports.ResponseCache = _mod_t0ymbs.ResponseCache;
-exports.normalizeResponseCacheConfig = _mod_t0ymbs.normalizeResponseCacheConfig;;
+const _mod_cryjmu = require('./lru-cache.cjs');
+exports.LRUCache = _mod_cryjmu.LRUCache;;
+const _mod_g9gezo = require('./dns-cache.cjs');
+exports.DNSCache = _mod_g9gezo.DNSCache;
+exports.getGlobalDNSCache = _mod_g9gezo.getGlobalDNSCache;
+exports.resetGlobalDNSCache = _mod_g9gezo.resetGlobalDNSCache;;
+const _mod_lqa0tk = require('./response-cache.cjs');
+exports.ResponseCache = _mod_lqa0tk.ResponseCache;
+exports.normalizeResponseCacheConfig = _mod_lqa0tk.normalizeResponseCacheConfig;;

package/dist/cache/response-cache.cjs

@@ -1,7 +1,11 @@
 const { LRUCache } = require('./lru-cache.cjs');
-const fs = require("node:fs");
-const path = require("node:path");
-const fsPromises = fs.promises;
+const { requireNodeModule } = require('../utils/node-runtime.cjs');
+function getFsModule() {
+  return requireNodeModule("node:fs");
+}
+function getPathModule() {
+  return requireNodeModule("node:path");
+}
 const DEFAULT_TTL = 3000000;
 const DEFAULT_MAX_ENTRIES = 500;
 const DEFAULT_METHODS = ["GET", "HEAD"];
@@ -22,7 +26,7 @@ class ResponseCache {
       methods: config.methods ?? DEFAULT_METHODS,
       respectHeaders: config.respectHeaders ?? true
     };
-    this.persistenceEnabled = !!this.config.cacheDir;
+    this.persistenceEnabled = !!this.config.cacheDir && !!getFsModule() && !!getPathModule();
     this.memoryCache = new LRUCache({
       maxEntries: this.config.maxEntries,
       ttl: this.config.ttl,
@@ -42,8 +46,13 @@ class ResponseCache {
   async initializePersistenceAsync() {
     if (!this.config.cacheDir)
       return;
+    const fs = getFsModule();
+    if (!fs?.promises) {
+      this.persistenceEnabled = false;
+      return;
+    }
     try {
-      await fsPromises.mkdir(this.config.cacheDir, { recursive: true });
+      await fs.promises.mkdir(this.config.cacheDir, { recursive: true });
       await this.loadFromDiskAsync();
       this.initialized = true;
     } catch {
@@ -51,34 +60,42 @@ class ResponseCache {
     }
   }
   getCacheFilePath(key) {
+    const path = getPathModule();
     const safeKey = Buffer.from(key).toString("base64url");
-    return path.join(this.config.cacheDir, `${safeKey}.json`);
+    return path && this.config.cacheDir ? path.join(this.config.cacheDir, `${safeKey}.json`) : `${this.config.cacheDir || ""}/${safeKey}.json`;
   }
   persistToDisk(key, entry) {
     if (!this.persistenceEnabled || !this.config.cacheDir)
       return;
+    const fs = getFsModule();
+    if (!fs?.promises)
+      return;
     const filePath = this.getCacheFilePath(key);
-    fsPromises.writeFile(filePath, JSON.stringify(entry), "utf-8").catch(() => {});
+    fs.promises.writeFile(filePath, JSON.stringify(entry), "utf-8").catch(() => {});
   }
   async loadFromDiskAsync() {
     if (!this.persistenceEnabled || !this.config.cacheDir)
       return;
+    const fs = getFsModule();
+    const path = getPathModule();
+    if (!fs?.promises || !path)
+      return;
     try {
-      const files = await fsPromises.readdir(this.config.cacheDir);
+      const files = await fs.promises.readdir(this.config.cacheDir);
       const now = Date.now();
       for (const file of files) {
         if (!file.endsWith(".json"))
           continue;
         try {
           const filePath = path.join(this.config.cacheDir, file);
-          const content = await fsPromises.readFile(filePath, "utf-8");
+          const content = await fs.promises.readFile(filePath, "utf-8");
           const entry = JSON.parse(content);
           if (entry.timestamp + entry.ttl > now) {
             const key = Buffer.from(file.replace(".json", ""), "base64url").toString("utf-8");
             const remainingTTL = entry.timestamp + entry.ttl - now;
             this.memoryCache.set(key, entry, remainingTTL);
           } else {
-            fsPromises.unlink(filePath).catch(() => {});
+            fs.promises.unlink(filePath).catch(() => {});
           }
         } catch {}
       }
@@ -145,6 +162,9 @@ class ResponseCache {
   loadSingleFromDisk(key) {
     if (!this.persistenceEnabled || !this.config.cacheDir)
       return;
+    const fs = getFsModule();
+    if (!fs)
+      return;
     try {
       const filePath = this.getCacheFilePath(key);
       if (!fs.existsSync(filePath))
@@ -241,7 +261,8 @@ class ResponseCache {
         this.memoryCache.delete(key);
         if (this.persistenceEnabled) {
           const filePath = this.getCacheFilePath(key);
-          fsPromises.unlink(filePath).catch(() => {});
+          const fs = getFsModule();
+          fs?.promises?.unlink(filePath).catch(() => {});
         }
         return;
       }
@@ -270,18 +291,23 @@ class ResponseCache {
       this.memoryCache.delete(key);
       if (this.persistenceEnabled) {
         const filePath = this.getCacheFilePath(key);
-        fsPromises.unlink(filePath).catch(() => {});
+        const fs = getFsModule();
+        fs?.promises?.unlink(filePath).catch(() => {});
       }
     }
   }
   clear() {
     this.memoryCache.clear();
     if (this.persistenceEnabled && this.config.cacheDir) {
+      const fs = getFsModule();
+      const path = getPathModule();
+      if (!fs?.promises || !path)
+        return;
       const cacheDir = this.config.cacheDir;
-      fsPromises.readdir(cacheDir).then((files) => {
+      fs.promises.readdir(cacheDir).then((files) => {
         for (const file of files) {
           if (file.endsWith(".json")) {
-            fsPromises.unlink(path.join(cacheDir, file)).catch(() => {});
+            fs.promises.unlink(path.join(cacheDir, file)).catch(() => {});
           }
         }
       }).catch(() => {});

package/dist/cache/response-cache.js

@@ -1,7 +1,11 @@
 import { LRUCache } from './lru-cache.js';
-import fs from "node:fs";
-import path from "node:path";
-const fsPromises = fs.promises;
+import { requireNodeModule } from '../utils/node-runtime.js';
+function getFsModule() {
+  return requireNodeModule("node:fs");
+}
+function getPathModule() {
+  return requireNodeModule("node:path");
+}
 const DEFAULT_TTL = 3000000;
 const DEFAULT_MAX_ENTRIES = 500;
 const DEFAULT_METHODS = ["GET", "HEAD"];
@@ -22,7 +26,7 @@ export class ResponseCache {
       methods: config.methods ?? DEFAULT_METHODS,
       respectHeaders: config.respectHeaders ?? true
     };
-    this.persistenceEnabled = !!this.config.cacheDir;
+    this.persistenceEnabled = !!this.config.cacheDir && !!getFsModule() && !!getPathModule();
     this.memoryCache = new LRUCache({
       maxEntries: this.config.maxEntries,
       ttl: this.config.ttl,
@@ -42,8 +46,13 @@ export class ResponseCache {
   async initializePersistenceAsync() {
     if (!this.config.cacheDir)
       return;
+    const fs = getFsModule();
+    if (!fs?.promises) {
+      this.persistenceEnabled = false;
+      return;
+    }
     try {
-      await fsPromises.mkdir(this.config.cacheDir, { recursive: true });
+      await fs.promises.mkdir(this.config.cacheDir, { recursive: true });
       await this.loadFromDiskAsync();
       this.initialized = true;
     } catch {
@@ -51,34 +60,42 @@ export class ResponseCache {
     }
   }
   getCacheFilePath(key) {
+    const path = getPathModule();
     const safeKey = Buffer.from(key).toString("base64url");
-    return path.join(this.config.cacheDir, `${safeKey}.json`);
+    return path && this.config.cacheDir ? path.join(this.config.cacheDir, `${safeKey}.json`) : `${this.config.cacheDir || ""}/${safeKey}.json`;
   }
   persistToDisk(key, entry) {
     if (!this.persistenceEnabled || !this.config.cacheDir)
       return;
+    const fs = getFsModule();
+    if (!fs?.promises)
+      return;
     const filePath = this.getCacheFilePath(key);
-    fsPromises.writeFile(filePath, JSON.stringify(entry), "utf-8").catch(() => {});
+    fs.promises.writeFile(filePath, JSON.stringify(entry), "utf-8").catch(() => {});
   }
   async loadFromDiskAsync() {
     if (!this.persistenceEnabled || !this.config.cacheDir)
       return;
+    const fs = getFsModule();
+    const path = getPathModule();
+    if (!fs?.promises || !path)
+      return;
     try {
-      const files = await fsPromises.readdir(this.config.cacheDir);
+      const files = await fs.promises.readdir(this.config.cacheDir);
       const now = Date.now();
       for (const file of files) {
         if (!file.endsWith(".json"))
           continue;
         try {
           const filePath = path.join(this.config.cacheDir, file);
-          const content = await fsPromises.readFile(filePath, "utf-8");
+          const content = await fs.promises.readFile(filePath, "utf-8");
           const entry = JSON.parse(content);
           if (entry.timestamp + entry.ttl > now) {
             const key = Buffer.from(file.replace(".json", ""), "base64url").toString("utf-8");
             const remainingTTL = entry.timestamp + entry.ttl - now;
             this.memoryCache.set(key, entry, remainingTTL);
           } else {
-            fsPromises.unlink(filePath).catch(() => {});
+            fs.promises.unlink(filePath).catch(() => {});
           }
         } catch {}
       }
@@ -145,6 +162,9 @@ export class ResponseCache {
   loadSingleFromDisk(key) {
     if (!this.persistenceEnabled || !this.config.cacheDir)
       return;
+    const fs = getFsModule();
+    if (!fs)
+      return;
     try {
       const filePath = this.getCacheFilePath(key);
       if (!fs.existsSync(filePath))
@@ -241,7 +261,8 @@ export class ResponseCache {
         this.memoryCache.delete(key);
         if (this.persistenceEnabled) {
           const filePath = this.getCacheFilePath(key);
-          fsPromises.unlink(filePath).catch(() => {});
+          const fs = getFsModule();
+          fs?.promises?.unlink(filePath).catch(() => {});
         }
         return;
       }
@@ -270,18 +291,23 @@ export class ResponseCache {
       this.memoryCache.delete(key);
       if (this.persistenceEnabled) {
         const filePath = this.getCacheFilePath(key);
-        fsPromises.unlink(filePath).catch(() => {});
+        const fs = getFsModule();
+        fs?.promises?.unlink(filePath).catch(() => {});
       }
     }
   }
   clear() {
     this.memoryCache.clear();
     if (this.persistenceEnabled && this.config.cacheDir) {
+      const fs = getFsModule();
+      const path = getPathModule();
+      if (!fs?.promises || !path)
+        return;
       const cacheDir = this.config.cacheDir;
-      fsPromises.readdir(cacheDir).then((files) => {
+      fs.promises.readdir(cacheDir).then((files) => {
         for (const file of files) {
           if (file.endsWith(".json")) {
-            fsPromises.unlink(path.join(cacheDir, file)).catch(() => {});
+            fs.promises.unlink(path.join(cacheDir, file)).catch(() => {});
           }
         }
       }).catch(() => {});

package/dist/cookies/cookie-jar.cjs

@@ -1,7 +1,8 @@
 const { CookieJar: TouchCookieJar } = require("tough-cookie");
 const { Cookie } = require('./cookie.cjs');
-const _mod_y8mmg9 = require('./cookie.cjs');
-exports.Cookie = _mod_y8mmg9.Cookie;;
+const { requireNodeModule } = require('../utils/node-runtime.cjs');
+const _mod_ojn3wu = require('./cookie.cjs');
+exports.Cookie = _mod_ojn3wu.Cookie;;
 
 class RezoCookieJar extends TouchCookieJar {
   constructor(store, options) {
@@ -402,10 +403,8 @@ class RezoCookieJar extends TouchCookieJar {
     return this._cookieFile;
   }
   loadFromFile(filePath, _defaultUrl) {
-    let fs;
-    try {
-      fs = require("node:fs");
-    } catch {
+    const fs = requireNodeModule("node:fs");
+    if (!fs) {
       throw new Error("loadFromFile() requires Node.js, Bun, or Deno. Not available in browsers or React Native.");
     }
     if (!fs.existsSync(filePath)) {
@@ -456,10 +455,8 @@ class RezoCookieJar extends TouchCookieJar {
     if (!targetPath) {
       throw new Error("No cookie file path specified. Provide a path or load from a file first.");
     }
-    let fs;
-    try {
-      fs = require("node:fs");
-    } catch {
+    const fs = requireNodeModule("node:fs");
+    if (!fs) {
       throw new Error("saveToFile() requires Node.js, Bun, or Deno. Not available in browsers or React Native.");
     }
     const isJson = targetPath.toLowerCase().endsWith(".json");
@@ -484,7 +481,7 @@ class RezoCookieJar extends TouchCookieJar {
   }
 }
 const CookieJar = exports.CookieJar = RezoCookieJar;
-const _mod_2m96de = require("tough-cookie");
-exports.Store = _mod_2m96de.Store;;
+const _mod_3lg6nf = require("tough-cookie");
+exports.Store = _mod_3lg6nf.Store;;
 
 exports.RezoCookieJar = RezoCookieJar;

package/dist/cookies/cookie-jar.js

@@ -1,5 +1,6 @@
 import { CookieJar as TouchCookieJar } from "tough-cookie";
 import { Cookie } from './cookie.js';
+import { requireNodeModule } from '../utils/node-runtime.js';
 export { Cookie } from './cookie.js';
 
 export class RezoCookieJar extends TouchCookieJar {
@@ -401,10 +402,8 @@ export class RezoCookieJar extends TouchCookieJar {
     return this._cookieFile;
   }
   loadFromFile(filePath, _defaultUrl) {
-    let fs;
-    try {
-      fs = require("node:fs");
-    } catch {
+    const fs = requireNodeModule("node:fs");
+    if (!fs) {
       throw new Error("loadFromFile() requires Node.js, Bun, or Deno. Not available in browsers or React Native.");
     }
     if (!fs.existsSync(filePath)) {
@@ -455,10 +454,8 @@ export class RezoCookieJar extends TouchCookieJar {
     if (!targetPath) {
       throw new Error("No cookie file path specified. Provide a path or load from a file first.");
     }
-    let fs;
-    try {
-      fs = require("node:fs");
-    } catch {
+    const fs = requireNodeModule("node:fs");
+    if (!fs) {
       throw new Error("saveToFile() requires Node.js, Bun, or Deno. Not available in browsers or React Native.");
     }
     const isJson = targetPath.toLowerCase().endsWith(".json");

package/dist/cookies/index.cjs

@@ -1,10 +1,10 @@
-const _mod_pjz10g = require('./cookie.cjs');
-exports.Cookie = _mod_pjz10g.Cookie;
-exports.RezoCookie = _mod_pjz10g.RezoCookie;;
-const _mod_r06e2k = require('./cookie-store.cjs');
-exports.RezoCookieStore = _mod_r06e2k.RezoCookieStore;;
-const _mod_8gcdod = require('./cookie-jar.cjs');
-exports.RezoCookieJar = _mod_8gcdod.RezoCookieJar;
-exports.CookieJar = _mod_8gcdod.CookieJar;;
-const _mod_jxvzsj = require("tough-cookie");
-exports.Store = _mod_jxvzsj.Store;;
+const _mod_meju1m = require('./cookie.cjs');
+exports.Cookie = _mod_meju1m.Cookie;
+exports.RezoCookie = _mod_meju1m.RezoCookie;;
+const _mod_mh08zd = require('./cookie-store.cjs');
+exports.RezoCookieStore = _mod_mh08zd.RezoCookieStore;;
+const _mod_kqcrba = require('./cookie-jar.cjs');
+exports.RezoCookieJar = _mod_kqcrba.RezoCookieJar;
+exports.CookieJar = _mod_kqcrba.CookieJar;;
+const _mod_mg6dea = require("tough-cookie");
+exports.Store = _mod_mg6dea.Store;;

package/dist/crawler/addon/oxylabs/index.cjs

@@ -1 +1 @@
-var{Rezo:h}=require("../../../core/rezo.cjs"),a=require("./options.cjs");exports.OXYLABS_BROWSER_TYPES=a.OXYLABS_BROWSER_TYPES;exports.OXYLABS_COMMON_LOCALES=a.OXYLABS_COMMON_LOCALES;exports.OXYLABS_COMMON_GEO_LOCATIONS=a.OXYLABS_COMMON_GEO_LOCATIONS;exports.OXYLABS_US_STATES=a.OXYLABS_US_STATES;exports.OXYLABS_EUROPEAN_COUNTRIES=a.OXYLABS_EUROPEAN_COUNTRIES;exports.OXYLABS_ASIAN_COUNTRIES=a.OXYLABS_ASIAN_COUNTRIES;exports.getRandomBrowserType=a.getRandomBrowserType;exports.getRandomLocale=a.getRandomLocale;exports.getRandomGeoLocation=a.getRandomGeoLocation;var i="https://realtime.oxylabs.io/v1/queries";class d{config;http;authHeader;constructor(s){if(!s.username||!s.password)throw Error("Oxylabs username and password are required");this.config={username:s.username,password:s.password,browserType:s.browserType??"desktop",locale:s.locale??"en-US",geoLocation:s.geoLocation??"",render:s.render??!1,context:s.context??{},timeout:s.timeout??120000},this.http=new h({baseURL:i,timeout:this.config.timeout}),this.authHeader=`Basic ${Buffer.from(`${s.username}:${s.password}`).toString("base64")}`}async scrape(s,e){let t={...this.config,...e},r=this.buildRequestBody(s,t),o=(await this.http.postJson(i,r,{headers:{Authorization:this.authHeader,"Content-Type":"application/json"}})).data;if(o.error)throw Error(`Oxylabs API error: ${o.error}`);if(!o.results||o.results.length===0)throw Error("Oxylabs API returned no results");let n=o.results[0],l=n._response?.cookies||[],_=n._response?.headers||{};return{statusCode:n.status_code,url:n.url,content:n.content,cookies:l,headers:_,jobId:o.job_id||n.job_id,rendered:t.render,geoLocation:t.geoLocation||void 0,locale:t.locale,browserType:t.browserType,raw:o}}async scrapeMany(s,e,t=1000){let r=[];for(let u=0;u<s.length;u++){let o=await this.scrape(s[u],e);if(r.push(o),u<s.length-1&&t>0)await new Promise((n)=>setTimeout(n,t))}return r}buildRequestBody(s,e){let t=[];if(t.push({key:"return_page_cookies",value:!0}),t.push({key:"return_page_headers",value:!0}),e.context)for(let[u,o]of Object.entries(e.context))t.push({key:u,value:o});if(e.headers&&Object.keys(e.headers).length>0)t.push({key:"headers",value:e.headers});if(e.cookies&&e.cookies.length>0)t.push({key:"cookies",value:e.cookies});if(e.session_id)t.push({key:"session_id",value:e.session_id});if(e.http_method)t.push({key:"http_method",value:e.http_method});if(e.base64Body)t.push({key:"content",value:e.base64Body});if(typeof e.follow_redirects==="boolean")t.push({key:"follow_redirects",value:e.follow_redirects});if(e.successful_status_codes&&e.successful_status_codes.length>0)t.push({key:"successful_status_codes",value:e.successful_status_codes});let r={source:"universal",url:s,context:t};if(e.render||e.javascript_rendering)r.render="html";if(e.browserType)r.user_agent_type=e.browserType;if(e.locale)r.locale=e.locale;if(e.geoLocation)r.geo_location=e.geoLocation;if(e.returnAsBase64)r.content_encoding="base64";return r}getConfig(){return{...this.config,password:"***"}}withConfig(s){return new d({...this.config,...s})}async testConnection(){try{return await this.scrape("https://httpbin.org/ip"),!0}catch(s){throw Error(`Oxylabs connection test failed: ${s.message}`)}}}exports.Oxylabs=d;exports.default=d;module.exports=Object.assign(d,exports);
+var{Rezo:c}=require("../../../core/rezo.cjs"),a=require("./options.cjs");exports.OXYLABS_BROWSER_TYPES=a.OXYLABS_BROWSER_TYPES;exports.OXYLABS_COMMON_LOCALES=a.OXYLABS_COMMON_LOCALES;exports.OXYLABS_COMMON_GEO_LOCATIONS=a.OXYLABS_COMMON_GEO_LOCATIONS;exports.OXYLABS_US_STATES=a.OXYLABS_US_STATES;exports.OXYLABS_EUROPEAN_COUNTRIES=a.OXYLABS_EUROPEAN_COUNTRIES;exports.OXYLABS_ASIAN_COUNTRIES=a.OXYLABS_ASIAN_COUNTRIES;exports.getRandomBrowserType=a.getRandomBrowserType;exports.getRandomLocale=a.getRandomLocale;exports.getRandomGeoLocation=a.getRandomGeoLocation;var i="https://realtime.oxylabs.io/v1/queries";class d{config;http;authHeader;constructor(s){if(!s.username||!s.password)throw Error("Oxylabs username and password are required");this.config={username:s.username,password:s.password,browserType:s.browserType??"desktop",locale:s.locale??"en-US",geoLocation:s.geoLocation??"",render:s.render??!1,context:s.context??{},timeout:s.timeout??120000},this.http=new c({baseURL:i,timeout:this.config.timeout}),this.authHeader=`Basic ${Buffer.from(`${s.username}:${s.password}`).toString("base64")}`}async scrape(s,e){let t={...this.config,...e},r=this.buildRequestBody(s,t),o=(await this.http.postJson(i,r,{headers:{Authorization:this.authHeader,"Content-Type":"application/json"}})).data;if(o.error)throw Error(`Oxylabs API error: ${o.error}`);if(!o.results||o.results.length===0)throw Error("Oxylabs API returned no results");let n=o.results[0],l=n._response?.cookies||[],h=n._response?.headers||{};return{statusCode:n.status_code,url:n.url,content:n.content,cookies:l,headers:h,jobId:o.job_id||n.job_id,rendered:t.render,geoLocation:t.geoLocation||void 0,locale:t.locale,browserType:t.browserType,raw:o}}async scrapeMany(s,e,t=1000){let r=[];for(let u=0;u<s.length;u++){let o=await this.scrape(s[u],e);if(r.push(o),u<s.length-1&&t>0)await new Promise((n)=>setTimeout(n,t))}return r}buildRequestBody(s,e){let t=[];if(t.push({key:"return_page_cookies",value:!0}),t.push({key:"return_page_headers",value:!0}),e.context)for(let[u,o]of Object.entries(e.context))t.push({key:u,value:o});if(e.headers&&Object.keys(e.headers).length>0)t.push({key:"headers",value:e.headers});if(e.cookies&&e.cookies.length>0)t.push({key:"cookies",value:e.cookies});if(e.session_id)t.push({key:"session_id",value:e.session_id});if(e.http_method)t.push({key:"http_method",value:e.http_method});if(e.base64Body)t.push({key:"content",value:e.base64Body});if(typeof e.follow_redirects==="boolean")t.push({key:"follow_redirects",value:e.follow_redirects});if(e.successful_status_codes&&e.successful_status_codes.length>0)t.push({key:"successful_status_codes",value:e.successful_status_codes});let r={source:"universal",url:s,context:t};if(e.render||e.javascript_rendering)r.render="html";if(e.browserType)r.user_agent_type=e.browserType;if(e.locale)r.locale=e.locale;if(e.geoLocation)r.geo_location=e.geoLocation;if(e.returnAsBase64)r.content_encoding="base64";return r}getConfig(){return{...this.config,password:"***"}}withConfig(s){return new d({...this.config,...s})}async testConnection(){try{return await this.scrape("https://httpbin.org/ip"),!0}catch(s){throw Error(`Oxylabs connection test failed: ${s.message}`)}}}exports.Oxylabs=d;exports.default=d;module.exports=Object.assign(d,exports);

package/dist/crawler/crawler-options.cjs

@@ -1 +1 @@
-var{RezoQueue:y}=require("../queue/queue.cjs"),{Oxylabs:x}=require("./addon/oxylabs/index.cjs"),v=require("node:path"),A=require("node:os"),{Decodo:p}=require("./addon/decodo/index.cjs"),{Rezo:D}=require("../core/rezo.cjs");class l{static destroyQueue(e){if(e&&typeof e.destroy==="function")e.destroy()}static destroyConfigQueues(e){for(let r of e)l.destroyQueue(r.pqueue)}static splitConfigs(e,r){let s=[],a=[];for(let t of e)if(r(t))a.push(t);else s.push(t);return{kept:s,removed:a}}static getHostname(e){try{return new URL(e).hostname||void 0}catch{return}}static createStableThroughputOptions(e){let{baseUrl:r,concurrency:s=40,scraperConcurrency:a=10,retryDelay:t=1000,maxRetryAttempts:i=2,retryOnStatusCode:o=[408,500,502,503,504],maxWaitOn429:d=15000,alwaysWaitOn429:n=!1,globalLimiter:h={concurrency:8},domain:u=l.getHostname(r),domainLimiter:c={concurrency:2,interval:1000,intervalCap:2,randomDelay:150},domainRetry:m={enable:!0,max429Retries:2,retryDelay:1000,maxRetryAttempts:2,backoff:!0},extraLimiters:g=[],overrides:b={}}=e,f=[];if(h)f.push({isGlobal:!0,options:h});if(u&&c)f.push({domain:u,options:c,retry:m||void 0});return f.push(...g),{...{baseUrl:r,autoThrottle:!1,concurrency:s,scraperConcurrency:a,retryDelay:t,maxRetryAttempts:i,retryOnStatusCode:o,maxWaitOn429:d,alwaysWaitOn429:n,limiter:f.length>0?{enable:!0,limiters:f}:!1},...b,baseUrl:r}}baseUrl;adapter;enableNavigationHistory;sessionId;rejectUnauthorized;userAgent;useRndUserAgent;timeout;maxRedirects;maxRetryAttempts;retryDelay;retryOnStatusCode;forceRevisit;retryWithoutProxyOnStatusCode;retryOnProxyError;maxRetryOnProxyError;allowRevisiting;enableCache;cacheTTL;cacheDir;throwFatalError;debug;enableSignalHandlers;concurrency;scraperConcurrency;maxDepth;maxUrls;maxResponseSize;respectRobotsTxt;followNofollow;autoThrottle;autoThrottleTargetDelay;autoThrottleMinDelay;autoThrottleMaxDelay;maxWaitOn429;alwaysWaitOn429;stealth;oxylabs=[];decodo=[];stealthProfiles=[];proxies=[];limiters=[];onLimiterAdded;onProviderAdded;requestHeaders=[];userAgents=$();constructor(e={}){this.baseUrl=e.baseUrl||"",this.adapter=e.adapter??"http",this.enableNavigationHistory=e.enableNavigationHistory??!1,this.sessionId=e.sessionId??`session_${Date.now()}_${Math.random().toString(36).slice(2,8)}`,this.rejectUnauthorized=e.rejectUnauthorized??!0,this.userAgent=e.userAgent,this.useRndUserAgent=e.useRndUserAgent??!1,this.timeout=e.timeout??30000,this.maxRedirects=e.maxRedirects??10,this.maxRetryAttempts=e.maxRetryAttempts??3,this.retryDelay=e.retryDelay??0,this.retryOnStatusCode=e.retryOnStatusCode??[408,429,500,502,503,504],this.forceRevisit=e.forceRevisit??!1,this.retryWithoutProxyOnStatusCode=e.retryWithoutProxyOnStatusCode??[407,403],this.retryOnProxyError=e.retryOnProxyError??!0,this.maxRetryOnProxyError=e.maxRetryOnProxyError??3,this.allowRevisiting=e.allowRevisiting??!1,this.enableCache=e.enableCache??!0,this.cacheTTL=e.cacheTTL??604800000,this.cacheDir=e.cacheDir??v.join(A.tmpdir(),"rezo_cache"),this.throwFatalError=e.throwFatalError??!1,this.debug=e.debug??!1,this.enableSignalHandlers=e.enableSignalHandlers??!1,this.concurrency=e.concurrency??100,this.scraperConcurrency=e.scraperConcurrency??this.concurrency,this.maxDepth=e.maxDepth??0,this.maxUrls=e.maxUrls??0,this.maxResponseSize=e.maxResponseSize??0,this.respectRobotsTxt=e.respectRobotsTxt??!1,this.followNofollow=e.followNofollow??!1,this.autoThrottle=e.autoThrottle??!0,this.autoThrottleTargetDelay=e.autoThrottleTargetDelay??1000,this.autoThrottleMinDelay=e.autoThrottleMinDelay??100,this.autoThrottleMaxDelay=e.autoThrottleMaxDelay??60000,this.maxWaitOn429=e.maxWaitOn429??1800000,this.alwaysWaitOn429=e.alwaysWaitOn429??!1,this._addHeaders(e.headers),this._addOxylabs(e.oxylabs),this._addDecodo(e.decodo),this._addProxies(e.proxy),this._addLimiters(e.limiter),this._addStealths(e.stealths),this.stealth=e.stealth}getConfiguredDomains(e){return(e==="headers"?this.requestHeaders:e==="limiters"?this.limiters:e==="oxylabs"?this.oxylabs:e==="decodo"?this.decodo:e==="stealth"?this.stealthProfiles:this.proxies).filter((s)=>s.domain).map((s)=>s.domain).filter((s,a,t)=>t.indexOf(s)===a)}removeDomain(e){this.requestHeaders=this.requestHeaders.filter((t)=>!t.domain||!this._domainsEqual(t.domain,e)),this.proxies=this.proxies.filter((t)=>!t.domain||!this._domainsEqual(t.domain,e));let r=l.splitConfigs(this.limiters,(t)=>!!t.domain&&this._domainsEqual(t.domain,e));l.destroyConfigQueues(r.removed),this.limiters=r.kept;let s=l.splitConfigs(this.oxylabs,(t)=>!!t.domain&&this._domainsEqual(t.domain,e));l.destroyConfigQueues(s.removed),this.oxylabs=s.kept;let a=l.splitConfigs(this.decodo,(t)=>!!t.domain&&this._domainsEqual(t.domain,e));return l.destroyConfigQueues(a.removed),this.decodo=a.kept,this.stealthProfiles=this.stealthProfiles.filter((t)=>!t.domain||!this._domainsEqual(t.domain,e)),this}_domainsEqual(e,r){if(Array.isArray(e)&&Array.isArray(r))return e.length===r.length&&e.every((s,a)=>s===r[a]);return e===r}getConfigurationSummary(){let e=(r)=>({total:r.length,global:r.filter((s)=>s.isGlobal).length,domainSpecific:r.filter((s)=>!s.isGlobal&&s.domain).length});return{headers:e(this.requestHeaders),proxies:e(this.proxies),limiters:e(this.limiters),oxylabs:e(this.oxylabs),decodo:e(this.decodo)}}_addHeaders(e){if(!e||!e.enable)return;for(let r of e.httpHeaders){let{domain:s,isGlobal:a,headers:t}=r;if(!s&&!a)continue;if(t instanceof Headers){let i=Object.fromEntries(t.entries());if(Object.keys(i).length<1)continue;t=i}else if(!t||Object.keys(t).length<1)continue;this.requestHeaders.push({domain:s,isGlobal:a,headers:t})}}_addProxies(e){if(!e||!e.enable)return;for(let r of e.proxies){let{domain:s,isGlobal:a,proxy:t,rotating:i}=r;if(!s&&!a)continue;if(!t||Object.keys(t).length<1)continue;this.proxies.push({domain:s,isGlobal:a,proxy:t,rotating:!!i})}}_addLimiters(e){if(!e||!e.enable)return;for(let r of e.limiters){let{domain:s,isGlobal:a,options:t,retry:i}=r;if(!s&&!a)continue;if(!t&&!i)continue;let o=t?new y(t):new y({name:"limiter"});if(this.limiters.push({domain:s,isGlobal:a,pqueue:o,randomDelay:t?.randomDelay,retry:i}),this.onLimiterAdded)this.onLimiterAdded(o)}}_addOxylabs(e){if(!e||!e.enable)return;for(let r of e.labs){let{domain:s,isGlobal:a,options:t,queueOptions:i}=r;if(!s&&!a)continue;if(!t||Object.keys(t).length<1)continue;let o=i?new y(i):void 0;if(this.oxylabs.push({domain:s,isGlobal:a,adaptar:new x(t),pqueue:o}),o&&this.onProviderAdded)this.onProviderAdded(o)}}_addDecodo(e){if(!e||!e.enable)return;for(let r of e.labs){let{domain:s,isGlobal:a,options:t,queueOptions:i}=r;if(!s&&!a)continue;if(!t||Object.keys(t).length<1)continue;let o=i?new y(i):void 0;if(this.decodo.push({domain:s,isGlobal:a,adaptar:new p(t),pqueue:o}),o&&this.onProviderAdded)this.onProviderAdded(o)}}_addStealths(e){if(!e||!e.enable)return;for(let r of e.profiles){let{domain:s,isGlobal:a,stealth:t}=r;if(!s&&!a)continue;if(!t)continue;this.stealthProfiles.push({domain:s,isGlobal:a,adaptar:new D({stealth:t})})}}addHeaders(e){return this._addHeaders({enable:!0,httpHeaders:[e]}),this}addProxy(e){return this._addProxies({enable:!0,proxies:[e]}),this}addProxies(e){return this._addProxies({enable:!0,proxies:e}),this}getProxyConfig(e,r,s){let a=this.selectConfigEntry(e,this.proxies,r,s);if(!a)return null;return{proxy:a.proxy,rotating:!!a.rotating}}addLimiter(e){return this._addLimiters({enable:!0,limiters:[e]}),this}addOxylabs(e){return this._addOxylabs({enable:!0,labs:[e]}),this}addDecodo(e){return this._addDecodo({enable:!0,labs:[e]}),this}addStealth(e){return this._addStealths({enable:!0,profiles:[e]}),this}destroyLimiters(){for(let e of this.limiters)l.destroyQueue(e.pqueue);this.limiters=[]}destroyProviderQueues(){for(let e of this.oxylabs)l.destroyQueue(e.pqueue),delete e.pqueue;for(let e of this.decodo)l.destroyQueue(e.pqueue),delete e.pqueue}getLimiters(){return this.limiters}getRandomDelay(e,r){if(!this.getDomainName(e))return;for(let a of this.limiters)if(this._hasDomain(e,a.domain)&&a.randomDelay!==void 0)return a.randomDelay;if(r){for(let a of this.limiters)if(a.isGlobal&&a.randomDelay!==void 0)return a.randomDelay}return}clearGlobalConfigs(){if(Array.isArray(this.requestHeaders))this.requestHeaders=this.requestHeaders.filter((e)=>!e.isGlobal);if(Array.isArray(this.oxylabs)){let e=l.splitConfigs(this.oxylabs,(r)=>!!r.isGlobal);l.destroyConfigQueues(e.removed),this.oxylabs=e.kept}if(Array.isArray(this.decodo)){let e=l.splitConfigs(this.decodo,(r)=>!!r.isGlobal);l.destroyConfigQueues(e.removed),this.decodo=e.kept}if(Array.isArray(this.limiters)){let e=l.splitConfigs(this.limiters,(r)=>!!r.isGlobal);l.destroyConfigQueues(e.removed),this.limiters=e.kept}if(Array.isArray(this.proxies))this.proxies=this.proxies.filter((e)=>!e.isGlobal);if(Array.isArray(this.stealthProfiles))this.stealthProfiles=this.stealthProfiles.filter((e)=>!e.isGlobal);return this}getAdapter(e,r,s,a){if(!this.getDomainName(e))return null;let i=r==="headers"?this.requestHeaders:r==="limiters"?this.limiters:r==="oxylabs"?this.oxylabs:r==="decodo"?this.decodo:r==="stealth"?this.stealthProfiles:this.proxies,o=this.selectConfigEntry(e,i,s,a);if(!o)return null;return r==="headers"?o.headers:r==="limiters"?o.pqueue:r==="oxylabs"?o.adaptar:r==="decodo"?o.adaptar:r==="stealth"?o.adaptar:o.proxy}getRetryOptions(e){if(!this.getDomainName(e))return null;for(let s=0;s<this.limiters.length;s++)if(this._hasDomain(e,this.limiters[s].domain))return this.limiters[s].retry||null;for(let s=0;s<this.limiters.length;s++)if(this.limiters[s].isGlobal&&this.limiters[s].retry)return this.limiters[s].retry;return null}rnd(e=0,r=Number.MAX_VALUE){return Math.floor(Math.random()*(r-e+1))+e}hasDomain(e,r,s){if(!this.getDomainName(e))return!1;let t=r==="headers"?this.requestHeaders:r==="limiters"?this.limiters:r==="oxylabs"?this.oxylabs:r==="decodo"?this.decodo:r==="stealth"?this.stealthProfiles:this.proxies;for(let i=0;i<t.length;i++)if(this._hasDomain(e,t[i].domain))return!0;if(s){for(let i=0;i<t.length;i++)if(t[i].isGlobal)return!0}return!1}pickHeaders(e,r,s,a){let t=this.getAdapter(e,"headers",r),i=new Headers(t??{}),o=i.count;if(s&&s instanceof Headers)for(let[d,n]of Object.entries(s.entries()))i.set(d,n);else if(s&&typeof s==="object"){for(let[d,n]of Object.entries(s))if(typeof n==="string")i.set(d,n)}if(a&&o===0&&!this.stealth)i.set("user-agent",this.getRandomUserAgent());return Object.fromEntries(i.entries())}_hasDomain(e,r){if(!r)return!1;let s=this.getDomainName(e);if(!s)return!1;let a=(i)=>{return/[\^\$\*\+\?\{\}\[\]\(\)\|\\]/.test(i)||i.startsWith("/")||i.includes(".*")||i.includes(".+")},t=(i)=>{if(i instanceof RegExp)return i.test(s)||i.test(e);let o=i.toString().trim();if(s.toLowerCase()===o.toLowerCase())return!0;if(o.includes("*")){let h=o.replace(/[.*+?^${}()|[\]\\]/g,"\\$&").replace(/\\\*/g,".*"),u=new RegExp(`^${h}$`,"i");return u.test(s)||u.test(e)}if(a(o))try{let h=o,u="i",c=o.match(/^\/(.*)\/(\w*)$/);if(c)h=c[1],u=c[2]||"i";let m=new RegExp(h,u);return m.test(s)||m.test(e)}catch(h){return s.toLowerCase().includes(o.toLowerCase())}let d=s.toLowerCase(),n=o.toLowerCase();return d===n||d.endsWith("."+n)||n.endsWith("."+d)};if(Array.isArray(r)){for(let i of r)if(t(i))return!0;return!1}return t(r)}selectConfigEntry(e,r,s,a){if(!this.getDomainName(e))return null;let i=(n)=>{if(!n.length)return null;let h=a?this.rnd(0,n.length-1):0;return n[h]},o=[];for(let n of r)if(this._hasDomain(e,n.domain))o.push(n);let d=i(o);if(d)return d;if(s){let n=[];for(let h of r)if(h.isGlobal)n.push(h);return i(n)}return null}getDomainName(e){if(this.isValidUrl(e))return new URL(e).hostname;else if(this.isHostName(e))return e;return null}isHostName(e){if(!e)return!1;if(e.length>255)return!1;let r=/^(?:(?:[a-zA-Z0-9](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?\.)+ [a-zA-Z]{2,})$/;return e=e.trim().toLowerCase(),r.test(e)&&!e.startsWith("-")&&!e.endsWith("-")}isValidUrl(e){if(!e)return!1;e=e.trim();try{let r=new URL(e);if(!r.protocol||!["http:","https:"].includes(r.protocol.toLowerCase()))return!1;if(!r.hostname)return!1;if(!/^(?:(?:[a-zA-Z0-9](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?\.)+[a-zA-Z]{2,})$/.test(r.hostname))return!1;return!0}catch{return!1}}getRandomUserAgent(){return this.userAgents[Math.floor(Math.random()*this.userAgents.length)]}}function $(){let e=[{name:"Chrome",version:"91.0.4472.124",engine:"AppleWebKit/537.36"},{name:"Firefox",version:"89.0",engine:"Gecko/20100101"},{name:"Safari",version:"14.1.1",engine:"AppleWebKit/605.1.15"},{name:"Edge",version:"91.0.864.59",engine:"AppleWebKit/537.36"},{name:"Opera",version:"77.0.4054.277",engine:"AppleWebKit/537.36"},{name:"Vivaldi",version:"3.8.2259.42",engine:"AppleWebKit/537.36"},{name:"Brave",version:"1.26.74",engine:"AppleWebKit/537.36"},{name:"Chromium",version:"91.0.4472.101",engine:"AppleWebKit/537.36"},{name:"Yandex",version:"21.5.3.742",engine:"AppleWebKit/537.36"},{name:"Maxthon",version:"5.3.8.2000",engine:"AppleWebKit/537.36"}],r=["Windows NT 10.0","Windows NT 6.1","Macintosh; Intel Mac OS X 10_15_7","Macintosh; Intel Mac OS X 11_4_0","X11; Linux x86_64","X11; Ubuntu; Linux x86_64"],s=[];for(let a=0;a<200;a++){let t=e[Math.floor(Math.random()*e.length)],i=r[Math.floor(Math.random()*r.length)],o="";switch(t.name){case"Chrome":o=`Mozilla/5.0 (${i}) ${t.engine} (KHTML, like Gecko) Chrome/${t.version} Safari/537.36`;break;case"Firefox":o=`Mozilla/5.0 (${i}; rv:${t.version}) ${t.engine} Firefox/${t.version}`;break;case"Safari":o=`Mozilla/5.0 (${i}) ${t.engine} (KHTML, like Gecko) Version/${t.version} Safari/605.1.15`;break;case"Edge":o=`Mozilla/5.0 (${i}) ${t.engine} (KHTML, like Gecko) Chrome/${t.version} Safari/537.36 Edg/${t.version}`;break;case"Opera":o=`Mozilla/5.0 (${i}) ${t.engine} (KHTML, like Gecko) Chrome/${t.version} Safari/537.36 OPR/${t.version}`;break;case"Vivaldi":o=`Mozilla/5.0 (${i}) ${t.engine} (KHTML, like Gecko) Chrome/${t.version} Safari/537.36 Vivaldi/${t.version}`;break;case"Brave":o=`Mozilla/5.0 (${i}) ${t.engine} (KHTML, like Gecko) Chrome/${t.version} Safari/537.36 Brave/${t.version}`;break;case"Chromium":o=`Mozilla/5.0 (${i}) ${t.engine} (KHTML, like Gecko) Chromium/${t.version} Chrome/${t.version} Safari/537.36`;break;case"Yandex":o=`Mozilla/5.0 (${i}) ${t.engine} (KHTML, like Gecko) Chrome/${t.version} YaBrowser/${t.version} Safari/537.36`;break;case"Maxthon":o=`Mozilla/5.0 (${i}) ${t.engine} (KHTML, like Gecko) Chrome/${t.version} Safari/537.36 Maxthon/${t.version}`;break}s.push(o)}return s}exports.CrawlerOptions=l;
+var{RezoQueue:y}=require("../queue/queue.cjs"),{Oxylabs:x}=require("./addon/oxylabs/index.cjs"),v=require("node:path"),A=require("node:os"),{Decodo:p}=require("./addon/decodo/index.cjs"),{Rezo:D}=require("../core/rezo.cjs");class l{static destroyQueue(e){if(e&&typeof e.destroy==="function")e.destroy()}static destroyConfigQueues(e){for(let r of e)l.destroyQueue(r.pqueue)}static splitConfigs(e,r){let s=[],a=[];for(let t of e)if(r(t))a.push(t);else s.push(t);return{kept:s,removed:a}}static getHostname(e){try{return new URL(e).hostname||void 0}catch{return}}static createStableThroughputOptions(e){let{baseUrl:r,concurrency:s=40,scraperConcurrency:a=10,retryDelay:t=1000,maxRetryAttempts:i=2,retryOnStatusCode:o=[408,500,502,503,504],maxWaitOn429:d=15000,alwaysWaitOn429:n=!1,globalLimiter:h={concurrency:8},domain:u=l.getHostname(r),domainLimiter:c={concurrency:2,interval:1000,intervalCap:2,randomDelay:150},domainRetry:m={enable:!0,max429Retries:2,retryDelay:1000,maxRetryAttempts:2,backoff:!0},extraLimiters:g=[],overrides:b={}}=e,f=[];if(h)f.push({isGlobal:!0,options:h});if(u&&c)f.push({domain:u,options:c,retry:m||void 0});return f.push(...g),{...{baseUrl:r,autoThrottle:!1,concurrency:s,scraperConcurrency:a,retryDelay:t,maxRetryAttempts:i,retryOnStatusCode:o,maxWaitOn429:d,alwaysWaitOn429:n,limiter:f.length>0?{enable:!0,limiters:f}:!1},...b,baseUrl:r}}baseUrl;adapter;enableNavigationHistory;sessionId;rejectUnauthorized;userAgent;useRndUserAgent;timeout;maxRedirects;maxRetryAttempts;retryDelay;retryOnStatusCode;forceRevisit;retryWithoutProxyOnStatusCode;retryOnProxyError;maxRetryOnProxyError;allowRevisiting;enableCache;cacheTTL;cacheDir;throwFatalError;debug;enableSignalHandlers;concurrency;scraperConcurrency;maxDepth;maxUrls;maxResponseSize;respectRobotsTxt;followNofollow;autoThrottle;autoThrottleTargetDelay;autoThrottleMinDelay;autoThrottleMaxDelay;maxWaitOn429;alwaysWaitOn429;stealth;oxylabs=[];decodo=[];stealthProfiles=[];proxies=[];limiters=[];onLimiterAdded;onProviderAdded;requestHeaders=[];userAgents=$();constructor(e={}){this.baseUrl=e.baseUrl||"",this.adapter=e.adapter??"http",this.enableNavigationHistory=e.enableNavigationHistory??!1,this.sessionId=e.sessionId??`session_${Date.now()}_${Math.random().toString(36).slice(2,8)}`,this.rejectUnauthorized=e.rejectUnauthorized??!0,this.userAgent=e.userAgent,this.useRndUserAgent=e.useRndUserAgent??!1,this.timeout=e.timeout??30000,this.maxRedirects=e.maxRedirects??10,this.maxRetryAttempts=e.maxRetryAttempts??3,this.retryDelay=e.retryDelay??0,this.retryOnStatusCode=e.retryOnStatusCode??[408,429,500,502,503,504],this.forceRevisit=e.forceRevisit??!1,this.retryWithoutProxyOnStatusCode=e.retryWithoutProxyOnStatusCode??[407,403],this.retryOnProxyError=e.retryOnProxyError??!0,this.maxRetryOnProxyError=e.maxRetryOnProxyError??3,this.allowRevisiting=e.allowRevisiting??!1,this.enableCache=e.enableCache??!0,this.cacheTTL=e.cacheTTL??604800000,this.cacheDir=e.cacheDir??v.join(A.tmpdir(),"rezo_cache"),this.throwFatalError=e.throwFatalError??!1,this.debug=e.debug??!1,this.enableSignalHandlers=e.enableSignalHandlers??!1,this.concurrency=e.concurrency??100,this.scraperConcurrency=e.scraperConcurrency??this.concurrency,this.maxDepth=e.maxDepth??0,this.maxUrls=e.maxUrls??0,this.maxResponseSize=e.maxResponseSize??0,this.respectRobotsTxt=e.respectRobotsTxt??!1,this.followNofollow=e.followNofollow??!1,this.autoThrottle=e.autoThrottle??!0,this.autoThrottleTargetDelay=e.autoThrottleTargetDelay??1000,this.autoThrottleMinDelay=e.autoThrottleMinDelay??100,this.autoThrottleMaxDelay=e.autoThrottleMaxDelay??60000,this.maxWaitOn429=e.maxWaitOn429??1800000,this.alwaysWaitOn429=e.alwaysWaitOn429??!1,this._addHeaders(e.headers),this._addOxylabs(e.oxylabs),this._addDecodo(e.decodo),this._addProxies(e.proxy),this._addLimiters(e.limiter),this._addStealths(e.stealths),this.stealth=e.stealth}getConfiguredDomains(e){return(e==="headers"?this.requestHeaders:e==="limiters"?this.limiters:e==="oxylabs"?this.oxylabs:e==="decodo"?this.decodo:e==="stealth"?this.stealthProfiles:this.proxies).filter((s)=>s.domain).map((s)=>s.domain).filter((s,a,t)=>t.indexOf(s)===a)}removeDomain(e){this.requestHeaders=this.requestHeaders.filter((t)=>!t.domain||!this._domainsEqual(t.domain,e)),this.proxies=this.proxies.filter((t)=>!t.domain||!this._domainsEqual(t.domain,e));let r=l.splitConfigs(this.limiters,(t)=>!!t.domain&&this._domainsEqual(t.domain,e));l.destroyConfigQueues(r.removed),this.limiters=r.kept;let s=l.splitConfigs(this.oxylabs,(t)=>!!t.domain&&this._domainsEqual(t.domain,e));l.destroyConfigQueues(s.removed),this.oxylabs=s.kept;let a=l.splitConfigs(this.decodo,(t)=>!!t.domain&&this._domainsEqual(t.domain,e));return l.destroyConfigQueues(a.removed),this.decodo=a.kept,this.stealthProfiles=this.stealthProfiles.filter((t)=>!t.domain||!this._domainsEqual(t.domain,e)),this}_domainsEqual(e,r){if(Array.isArray(e)&&Array.isArray(r))return e.length===r.length&&e.every((s,a)=>s===r[a]);return e===r}getConfigurationSummary(){let e=(r)=>({total:r.length,global:r.filter((s)=>s.isGlobal).length,domainSpecific:r.filter((s)=>!s.isGlobal&&s.domain).length});return{headers:e(this.requestHeaders),proxies:e(this.proxies),limiters:e(this.limiters),oxylabs:e(this.oxylabs),decodo:e(this.decodo)}}_addHeaders(e){if(!e||!e.enable)return;for(let r of e.httpHeaders){let{domain:s,isGlobal:a,headers:t}=r;if(!s&&!a)continue;if(t instanceof Headers){let i=Object.fromEntries(t.entries());if(Object.keys(i).length<1)continue;t=i}else if(!t||Object.keys(t).length<1)continue;this.requestHeaders.push({domain:s,isGlobal:a,headers:t})}}_addProxies(e){if(!e||!e.enable)return;for(let r of e.proxies){let{domain:s,isGlobal:a,proxy:t,rotating:i}=r;if(!s&&!a)continue;if(!t||Object.keys(t).length<1)continue;this.proxies.push({domain:s,isGlobal:a,proxy:t,rotating:!!i})}}_addLimiters(e){if(!e||!e.enable)return;for(let r of e.limiters){let{domain:s,isGlobal:a,options:t,retry:i}=r;if(!s&&!a)continue;if(!t&&!i)continue;let o=t?new y(t):new y({name:"limiter"});if(this.limiters.push({domain:s,isGlobal:a,pqueue:o,randomDelay:t?.randomDelay,retry:i}),this.onLimiterAdded)this.onLimiterAdded(o)}}_addOxylabs(e){if(!e||!e.enable)return;for(let r of e.labs){let{domain:s,isGlobal:a,options:t,queueOptions:i}=r;if(!s&&!a)continue;if(!t||Object.keys(t).length<1)continue;let o=i?new y(i):void 0;if(this.oxylabs.push({domain:s,isGlobal:a,adaptar:new x(t),pqueue:o}),o&&this.onProviderAdded)this.onProviderAdded(o)}}_addDecodo(e){if(!e||!e.enable)return;for(let r of e.labs){let{domain:s,isGlobal:a,options:t,queueOptions:i}=r;if(!s&&!a)continue;if(!t||Object.keys(t).length<1)continue;let o=i?new y(i):void 0;if(this.decodo.push({domain:s,isGlobal:a,adaptar:new p(t),pqueue:o}),o&&this.onProviderAdded)this.onProviderAdded(o)}}_addStealths(e){if(!e||!e.enable)return;for(let r of e.profiles){let{domain:s,isGlobal:a,stealth:t}=r;if(!s&&!a)continue;if(!t)continue;this.stealthProfiles.push({domain:s,isGlobal:a,adaptar:new D({stealth:t})})}}addHeaders(e){return this._addHeaders({enable:!0,httpHeaders:[e]}),this}addProxy(e){return this._addProxies({enable:!0,proxies:[e]}),this}addProxies(e){return this._addProxies({enable:!0,proxies:e}),this}getProxyConfig(e,r,s){let a=this.selectConfigEntry(e,this.proxies,r,s);if(!a)return null;return{proxy:a.proxy,rotating:!!a.rotating}}addLimiter(e){return this._addLimiters({enable:!0,limiters:[e]}),this}addOxylabs(e){return this._addOxylabs({enable:!0,labs:[e]}),this}addDecodo(e){return this._addDecodo({enable:!0,labs:[e]}),this}addStealth(e){return this._addStealths({enable:!0,profiles:[e]}),this}destroyLimiters(){for(let e of this.limiters)l.destroyQueue(e.pqueue);this.limiters=[]}destroyProviderQueues(){for(let e of this.oxylabs)l.destroyQueue(e.pqueue),delete e.pqueue;for(let e of this.decodo)l.destroyQueue(e.pqueue),delete e.pqueue}getLimiters(){return this.limiters}getRandomDelay(e,r){if(!this.getDomainName(e))return;for(let a of this.limiters)if(this._hasDomain(e,a.domain)&&a.randomDelay!==void 0)return a.randomDelay;if(r){for(let a of this.limiters)if(a.isGlobal&&a.randomDelay!==void 0)return a.randomDelay}return}clearGlobalConfigs(){if(Array.isArray(this.requestHeaders))this.requestHeaders=this.requestHeaders.filter((e)=>!e.isGlobal);if(Array.isArray(this.oxylabs)){let e=l.splitConfigs(this.oxylabs,(r)=>!!r.isGlobal);l.destroyConfigQueues(e.removed),this.oxylabs=e.kept}if(Array.isArray(this.decodo)){let e=l.splitConfigs(this.decodo,(r)=>!!r.isGlobal);l.destroyConfigQueues(e.removed),this.decodo=e.kept}if(Array.isArray(this.limiters)){let e=l.splitConfigs(this.limiters,(r)=>!!r.isGlobal);l.destroyConfigQueues(e.removed),this.limiters=e.kept}if(Array.isArray(this.proxies))this.proxies=this.proxies.filter((e)=>!e.isGlobal);if(Array.isArray(this.stealthProfiles))this.stealthProfiles=this.stealthProfiles.filter((e)=>!e.isGlobal);return this}getAdapter(e,r,s,a){if(!this.getDomainName(e))return null;let i=r==="headers"?this.requestHeaders:r==="limiters"?this.limiters:r==="oxylabs"?this.oxylabs:r==="decodo"?this.decodo:r==="stealth"?this.stealthProfiles:this.proxies,o=this.selectConfigEntry(e,i,s,a);if(!o)return null;return r==="headers"?o.headers:r==="limiters"?o.pqueue:r==="oxylabs"?o.adaptar:r==="decodo"?o.adaptar:r==="stealth"?o.adaptar:o.proxy}getRetryOptions(e){if(!this.getDomainName(e))return null;for(let s=0;s<this.limiters.length;s++)if(this._hasDomain(e,this.limiters[s].domain))return this.limiters[s].retry||null;for(let s=0;s<this.limiters.length;s++)if(this.limiters[s].isGlobal&&this.limiters[s].retry)return this.limiters[s].retry;return null}rnd(e=0,r=Number.MAX_VALUE){return Math.floor(Math.random()*(r-e+1))+e}hasDomain(e,r,s){if(!this.getDomainName(e))return!1;let t=r==="headers"?this.requestHeaders:r==="limiters"?this.limiters:r==="oxylabs"?this.oxylabs:r==="decodo"?this.decodo:r==="stealth"?this.stealthProfiles:this.proxies;for(let i=0;i<t.length;i++)if(this._hasDomain(e,t[i].domain))return!0;if(s){for(let i=0;i<t.length;i++)if(t[i].isGlobal)return!0}return!1}pickHeaders(e,r,s,a){let t=this.getAdapter(e,"headers",r),i=new Headers(t??{}),o=[...i.keys()].length;if(s&&s instanceof Headers)for(let[d,n]of Object.entries(s.entries()))i.set(d,n);else if(s&&typeof s==="object"){for(let[d,n]of Object.entries(s))if(typeof n==="string")i.set(d,n)}if(a&&o===0&&!this.stealth)i.set("user-agent",this.getRandomUserAgent());return Object.fromEntries(i.entries())}_hasDomain(e,r){if(!r)return!1;let s=this.getDomainName(e);if(!s)return!1;let a=(i)=>{return/[\^\$\*\+\?\{\}\[\]\(\)\|\\]/.test(i)||i.startsWith("/")||i.includes(".*")||i.includes(".+")},t=(i)=>{if(i instanceof RegExp)return i.test(s)||i.test(e);let o=i.toString().trim();if(s.toLowerCase()===o.toLowerCase())return!0;if(o.includes("*")){let h=o.replace(/[.*+?^${}()|[\]\\]/g,"\\$&").replace(/\\\*/g,".*"),u=new RegExp(`^${h}$`,"i");return u.test(s)||u.test(e)}if(a(o))try{let h=o,u="i",c=o.match(/^\/(.*)\/(\w*)$/);if(c)h=c[1],u=c[2]||"i";let m=new RegExp(h,u);return m.test(s)||m.test(e)}catch(h){return s.toLowerCase().includes(o.toLowerCase())}let d=s.toLowerCase(),n=o.toLowerCase();return d===n||d.endsWith("."+n)||n.endsWith("."+d)};if(Array.isArray(r)){for(let i of r)if(t(i))return!0;return!1}return t(r)}selectConfigEntry(e,r,s,a){if(!this.getDomainName(e))return null;let i=(n)=>{if(!n.length)return null;let h=a?this.rnd(0,n.length-1):0;return n[h]},o=[];for(let n of r)if(this._hasDomain(e,n.domain))o.push(n);let d=i(o);if(d)return d;if(s){let n=[];for(let h of r)if(h.isGlobal)n.push(h);return i(n)}return null}getDomainName(e){if(this.isValidUrl(e))return new URL(e).hostname;else if(this.isHostName(e))return e;return null}isHostName(e){if(!e)return!1;if(e.length>255)return!1;let r=/^(?:(?:[a-zA-Z0-9](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?\.)+ [a-zA-Z]{2,})$/;return e=e.trim().toLowerCase(),r.test(e)&&!e.startsWith("-")&&!e.endsWith("-")}isValidUrl(e){if(!e)return!1;e=e.trim();try{let r=new URL(e);if(!r.protocol||!["http:","https:"].includes(r.protocol.toLowerCase()))return!1;if(!r.hostname)return!1;if(!/^(?:(?:[a-zA-Z0-9](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?\.)+[a-zA-Z]{2,})$/.test(r.hostname))return!1;return!0}catch{return!1}}getRandomUserAgent(){return this.userAgents[Math.floor(Math.random()*this.userAgents.length)]}}function $(){let e=[{name:"Chrome",version:"91.0.4472.124",engine:"AppleWebKit/537.36"},{name:"Firefox",version:"89.0",engine:"Gecko/20100101"},{name:"Safari",version:"14.1.1",engine:"AppleWebKit/605.1.15"},{name:"Edge",version:"91.0.864.59",engine:"AppleWebKit/537.36"},{name:"Opera",version:"77.0.4054.277",engine:"AppleWebKit/537.36"},{name:"Vivaldi",version:"3.8.2259.42",engine:"AppleWebKit/537.36"},{name:"Brave",version:"1.26.74",engine:"AppleWebKit/537.36"},{name:"Chromium",version:"91.0.4472.101",engine:"AppleWebKit/537.36"},{name:"Yandex",version:"21.5.3.742",engine:"AppleWebKit/537.36"},{name:"Maxthon",version:"5.3.8.2000",engine:"AppleWebKit/537.36"}],r=["Windows NT 10.0","Windows NT 6.1","Macintosh; Intel Mac OS X 10_15_7","Macintosh; Intel Mac OS X 11_4_0","X11; Linux x86_64","X11; Ubuntu; Linux x86_64"],s=[];for(let a=0;a<200;a++){let t=e[Math.floor(Math.random()*e.length)],i=r[Math.floor(Math.random()*r.length)],o="";switch(t.name){case"Chrome":o=`Mozilla/5.0 (${i}) ${t.engine} (KHTML, like Gecko) Chrome/${t.version} Safari/537.36`;break;case"Firefox":o=`Mozilla/5.0 (${i}; rv:${t.version}) ${t.engine} Firefox/${t.version}`;break;case"Safari":o=`Mozilla/5.0 (${i}) ${t.engine} (KHTML, like Gecko) Version/${t.version} Safari/605.1.15`;break;case"Edge":o=`Mozilla/5.0 (${i}) ${t.engine} (KHTML, like Gecko) Chrome/${t.version} Safari/537.36 Edg/${t.version}`;break;case"Opera":o=`Mozilla/5.0 (${i}) ${t.engine} (KHTML, like Gecko) Chrome/${t.version} Safari/537.36 OPR/${t.version}`;break;case"Vivaldi":o=`Mozilla/5.0 (${i}) ${t.engine} (KHTML, like Gecko) Chrome/${t.version} Safari/537.36 Vivaldi/${t.version}`;break;case"Brave":o=`Mozilla/5.0 (${i}) ${t.engine} (KHTML, like Gecko) Chrome/${t.version} Safari/537.36 Brave/${t.version}`;break;case"Chromium":o=`Mozilla/5.0 (${i}) ${t.engine} (KHTML, like Gecko) Chromium/${t.version} Chrome/${t.version} Safari/537.36`;break;case"Yandex":o=`Mozilla/5.0 (${i}) ${t.engine} (KHTML, like Gecko) Chrome/${t.version} YaBrowser/${t.version} Safari/537.36`;break;case"Maxthon":o=`Mozilla/5.0 (${i}) ${t.engine} (KHTML, like Gecko) Chrome/${t.version} Safari/537.36 Maxthon/${t.version}`;break}s.push(o)}return s}exports.CrawlerOptions=l;