rezo 1.0.2 → 1.0.4

package/dist/platform/worker.js

@@ -1,7 +1,29 @@
 import { executeRequest } from '../adapters/fetch.js';
 import { setGlobalAdapter, createRezoInstance, Rezo } from '../core/rezo.js';
+import { RezoError, RezoErrorCode } from '../errors/rezo-error.js';
+import { RezoHeaders } from '../utils/headers.js';
+import { RezoFormData } from '../utils/form-data.js';
+import { RezoCookieJar } from '../utils/cookies.js';
+import { createDefaultHooks, mergeHooks } from '../core/hooks.js';
+import packageJson from "../../package.json" with { type: 'json' };
+
+export { Rezo };
+export { RezoError };
+export { RezoErrorCode };
+export { RezoHeaders };
+export { RezoFormData };
+export { RezoCookieJar };
+export { createDefaultHooks };
+export { mergeHooks };
+export const isRezoError = RezoError.isRezoError;
+export const Cancel = RezoError;
+export const CancelToken = AbortController;
+export const isCancel = (error) => {
+  return error instanceof RezoError && error.code === "ECONNABORTED";
+};
+export const all = Promise.all.bind(Promise);
+export const spread = (callback) => (array) => callback(...array);
+export const VERSION = packageJson.version;
 setGlobalAdapter(executeRequest);
 const rezo = createRezoInstance(executeRequest);
-
-export { Rezo, rezo };
 export default rezo;

package/dist/plugin/index.cjs

@@ -1,36 +1,36 @@
-const _mod_khxd8a = require('./crawler.cjs');
-exports.Crawler = _mod_khxd8a.Crawler;;
-const _mod_vm96it = require('./crawler-options.cjs');
-exports.CrawlerOptions = _mod_vm96it.CrawlerOptions;;
-const _mod_enfdda = require('../cache/file-cacher.cjs');
-exports.FileCacher = _mod_enfdda.FileCacher;;
-const _mod_p51bkd = require('../cache/url-store.cjs');
-exports.UrlStore = _mod_p51bkd.UrlStore;;
-const _mod_q47c3j = require('./addon/oxylabs/index.cjs');
-exports.Oxylabs = _mod_q47c3j.Oxylabs;;
-const _mod_rfrfyo = require('./addon/oxylabs/options.cjs');
-exports.OXYLABS_BROWSER_TYPES = _mod_rfrfyo.OXYLABS_BROWSER_TYPES;
-exports.OXYLABS_COMMON_LOCALES = _mod_rfrfyo.OXYLABS_COMMON_LOCALES;
-exports.OXYLABS_COMMON_GEO_LOCATIONS = _mod_rfrfyo.OXYLABS_COMMON_GEO_LOCATIONS;
-exports.OXYLABS_US_STATES = _mod_rfrfyo.OXYLABS_US_STATES;
-exports.OXYLABS_EUROPEAN_COUNTRIES = _mod_rfrfyo.OXYLABS_EUROPEAN_COUNTRIES;
-exports.OXYLABS_ASIAN_COUNTRIES = _mod_rfrfyo.OXYLABS_ASIAN_COUNTRIES;
-exports.getRandomOxylabsBrowserType = _mod_rfrfyo.getRandomBrowserType;
-exports.getRandomOxylabsLocale = _mod_rfrfyo.getRandomLocale;
-exports.getRandomOxylabsGeoLocation = _mod_rfrfyo.getRandomGeoLocation;;
-const _mod_5qyxa5 = require('./addon/decodo/index.cjs');
-exports.Decodo = _mod_5qyxa5.Decodo;;
-const _mod_38klqg = require('./addon/decodo/options.cjs');
-exports.DECODO_DEVICE_TYPES = _mod_38klqg.DECODO_DEVICE_TYPES;
-exports.DECODO_HEADLESS_MODES = _mod_38klqg.DECODO_HEADLESS_MODES;
-exports.DECODO_COMMON_LOCALES = _mod_38klqg.DECODO_COMMON_LOCALES;
-exports.DECODO_COMMON_COUNTRIES = _mod_38klqg.DECODO_COMMON_COUNTRIES;
-exports.DECODO_EUROPEAN_COUNTRIES = _mod_38klqg.DECODO_EUROPEAN_COUNTRIES;
-exports.DECODO_ASIAN_COUNTRIES = _mod_38klqg.DECODO_ASIAN_COUNTRIES;
-exports.DECODO_US_STATES = _mod_38klqg.DECODO_US_STATES;
-exports.DECODO_COMMON_CITIES = _mod_38klqg.DECODO_COMMON_CITIES;
-exports.getRandomDecodoDeviceType = _mod_38klqg.getRandomDeviceType;
-exports.getRandomDecodoLocale = _mod_38klqg.getRandomLocale;
-exports.getRandomDecodoCountry = _mod_38klqg.getRandomCountry;
-exports.getRandomDecodoCity = _mod_38klqg.getRandomCity;
-exports.generateDecodoSessionId = _mod_38klqg.generateSessionId;;
+const _mod_er88yk = require('./crawler.cjs');
+exports.Crawler = _mod_er88yk.Crawler;;
+const _mod_fi3kva = require('./crawler-options.cjs');
+exports.CrawlerOptions = _mod_fi3kva.CrawlerOptions;;
+const _mod_pgkdtt = require('../cache/file-cacher.cjs');
+exports.FileCacher = _mod_pgkdtt.FileCacher;;
+const _mod_6h8hob = require('../cache/url-store.cjs');
+exports.UrlStore = _mod_6h8hob.UrlStore;;
+const _mod_9aknyh = require('./addon/oxylabs/index.cjs');
+exports.Oxylabs = _mod_9aknyh.Oxylabs;;
+const _mod_8drluh = require('./addon/oxylabs/options.cjs');
+exports.OXYLABS_BROWSER_TYPES = _mod_8drluh.OXYLABS_BROWSER_TYPES;
+exports.OXYLABS_COMMON_LOCALES = _mod_8drluh.OXYLABS_COMMON_LOCALES;
+exports.OXYLABS_COMMON_GEO_LOCATIONS = _mod_8drluh.OXYLABS_COMMON_GEO_LOCATIONS;
+exports.OXYLABS_US_STATES = _mod_8drluh.OXYLABS_US_STATES;
+exports.OXYLABS_EUROPEAN_COUNTRIES = _mod_8drluh.OXYLABS_EUROPEAN_COUNTRIES;
+exports.OXYLABS_ASIAN_COUNTRIES = _mod_8drluh.OXYLABS_ASIAN_COUNTRIES;
+exports.getRandomOxylabsBrowserType = _mod_8drluh.getRandomBrowserType;
+exports.getRandomOxylabsLocale = _mod_8drluh.getRandomLocale;
+exports.getRandomOxylabsGeoLocation = _mod_8drluh.getRandomGeoLocation;;
+const _mod_hco42s = require('./addon/decodo/index.cjs');
+exports.Decodo = _mod_hco42s.Decodo;;
+const _mod_ly422g = require('./addon/decodo/options.cjs');
+exports.DECODO_DEVICE_TYPES = _mod_ly422g.DECODO_DEVICE_TYPES;
+exports.DECODO_HEADLESS_MODES = _mod_ly422g.DECODO_HEADLESS_MODES;
+exports.DECODO_COMMON_LOCALES = _mod_ly422g.DECODO_COMMON_LOCALES;
+exports.DECODO_COMMON_COUNTRIES = _mod_ly422g.DECODO_COMMON_COUNTRIES;
+exports.DECODO_EUROPEAN_COUNTRIES = _mod_ly422g.DECODO_EUROPEAN_COUNTRIES;
+exports.DECODO_ASIAN_COUNTRIES = _mod_ly422g.DECODO_ASIAN_COUNTRIES;
+exports.DECODO_US_STATES = _mod_ly422g.DECODO_US_STATES;
+exports.DECODO_COMMON_CITIES = _mod_ly422g.DECODO_COMMON_CITIES;
+exports.getRandomDecodoDeviceType = _mod_ly422g.getRandomDeviceType;
+exports.getRandomDecodoLocale = _mod_ly422g.getRandomLocale;
+exports.getRandomDecodoCountry = _mod_ly422g.getRandomCountry;
+exports.getRandomDecodoCity = _mod_ly422g.getRandomCity;
+exports.generateDecodoSessionId = _mod_ly422g.generateSessionId;;

package/dist/proxy/index.cjs

@@ -1,6 +1,8 @@
 const { SocksProxyAgent: RezoSocksProxy } = require("socks-proxy-agent");
 const { HttpsProxyAgent: RezoHttpsSocks } = require("https-proxy-agent");
 const { HttpProxyAgent: RezoHttpSocks } = require("http-proxy-agent");
+const _mod_jzt7z5 = require('./manager.cjs');
+exports.ProxyManager = _mod_jzt7z5.ProxyManager;;
 function createOptions(uri, opts) {
   if (uri instanceof URL || typeof uri === "string") {
     return {

package/dist/proxy/index.js

@@ -1,6 +1,7 @@
 import { SocksProxyAgent as RezoSocksProxy } from "socks-proxy-agent";
 import { HttpsProxyAgent as RezoHttpsSocks } from "https-proxy-agent";
 import { HttpProxyAgent as RezoHttpSocks } from "http-proxy-agent";
+export { ProxyManager } from './manager.js';
 function createOptions(uri, opts) {
   if (uri instanceof URL || typeof uri === "string") {
     return {

package/dist/proxy/manager.cjs

@@ -0,0 +1,446 @@
+function generateProxyId() {
+  return `proxy_${Date.now()}_${Math.random().toString(36).substring(2, 9)}`;
+}
+
+class ProxyManager {
+  config;
+  states = new Map;
+  currentIndex = 0;
+  currentProxyRequests = 0;
+  lastSelectedProxy = null;
+  cooldownTimers = new Map;
+  _totalRequests = 0;
+  _totalSuccesses = 0;
+  _totalFailures = 0;
+  hooks = {
+    beforeProxySelect: [],
+    afterProxySelect: [],
+    beforeProxyError: [],
+    afterProxyError: [],
+    beforeProxyDisable: [],
+    afterProxyDisable: [],
+    afterProxyRotate: [],
+    afterProxyEnable: []
+  };
+  constructor(config) {
+    this.config = {
+      failWithoutProxy: true,
+      autoDisableDeadProxies: false,
+      maxFailures: 3,
+      retryWithNextProxy: false,
+      maxProxyRetries: 3,
+      ...config
+    };
+    for (const proxy of this.config.proxies) {
+      if (!proxy.id) {
+        proxy.id = generateProxyId();
+      }
+      this.states.set(proxy.id, this.createInitialState(proxy));
+    }
+  }
+  createInitialState(proxy) {
+    return {
+      proxy,
+      requestCount: 0,
+      failureCount: 0,
+      successCount: 0,
+      totalFailures: 0,
+      isActive: true
+    };
+  }
+  shouldProxy(url) {
+    const { whitelist, blacklist } = this.config;
+    if (whitelist && whitelist.length > 0) {
+      const matches = whitelist.some((pattern) => this.matchPattern(pattern, url));
+      if (!matches) {
+        return false;
+      }
+    }
+    if (blacklist && blacklist.length > 0) {
+      const matches = blacklist.some((pattern) => this.matchPattern(pattern, url));
+      if (matches) {
+        return false;
+      }
+    }
+    return true;
+  }
+  matchPattern(pattern, url) {
+    if (pattern instanceof RegExp) {
+      return pattern.test(url);
+    }
+    try {
+      const urlObj = new URL(url);
+      const hostname = urlObj.hostname;
+      if (hostname === pattern) {
+        return true;
+      }
+      if (hostname.endsWith("." + pattern)) {
+        return true;
+      }
+      if (pattern.startsWith("*.")) {
+        const domain = pattern.slice(2);
+        if (hostname === domain || hostname.endsWith("." + domain)) {
+          return true;
+        }
+      }
+      return false;
+    } catch {
+      return url.includes(pattern);
+    }
+  }
+  getActive() {
+    this.processExpiredCooldowns();
+    return Array.from(this.states.values()).filter((state) => state.isActive).map((state) => state.proxy);
+  }
+  getDisabled() {
+    return Array.from(this.states.values()).filter((state) => !state.isActive && !state.reenableAt).map((state) => state.proxy);
+  }
+  getCooldown() {
+    const now = Date.now();
+    return Array.from(this.states.values()).filter((state) => !state.isActive && state.reenableAt && state.reenableAt > now).map((state) => state.proxy);
+  }
+  processExpiredCooldowns() {
+    const now = Date.now();
+    for (const state of this.states.values()) {
+      if (!state.isActive && state.reenableAt && state.reenableAt <= now) {
+        this.enableProxy(state.proxy, "cooldown-expired");
+      }
+    }
+  }
+  next(url) {
+    this._totalRequests++;
+    const activeProxies = this.getActive();
+    this.runBeforeProxySelectHooksSync({
+      url,
+      proxies: activeProxies,
+      isRetry: false,
+      retryCount: 0
+    });
+    if (!this.shouldProxy(url)) {
+      this.runAfterProxySelectHooksSync({
+        url,
+        proxy: null,
+        reason: this.config.blacklist?.some((p) => this.matchPattern(p, url)) ? "blacklist-match" : "whitelist-no-match"
+      });
+      return null;
+    }
+    if (activeProxies.length === 0) {
+      this.runAfterProxySelectHooksSync({
+        url,
+        proxy: null,
+        reason: "no-proxies-available"
+      });
+      return null;
+    }
+    const selected = this.selectProxy(activeProxies);
+    if (selected) {
+      const state = this.states.get(selected.id);
+      if (state) {
+        state.requestCount++;
+        if (this.config.rotation === "per-proxy-limit") {
+          const limit = this.config.limit;
+          if (state.requestCount >= limit) {
+            this.disableProxy(selected, "limit-reached");
+          }
+        }
+      }
+      if (this.lastSelectedProxy && this.lastSelectedProxy.id !== selected.id) {
+        this.runAfterProxyRotateHooks({
+          from: this.lastSelectedProxy,
+          to: selected,
+          reason: "scheduled"
+        });
+      }
+      this.lastSelectedProxy = selected;
+    }
+    this.runAfterProxySelectHooksSync({
+      url,
+      proxy: selected,
+      reason: selected ? "selected" : "no-proxies-available"
+    });
+    return selected;
+  }
+  select(url) {
+    this._totalRequests++;
+    if (!this.shouldProxy(url)) {
+      const { whitelist, blacklist } = this.config;
+      const reason = blacklist?.some((p) => this.matchPattern(p, url)) ? "blacklist-match" : "whitelist-no-match";
+      return { proxy: null, reason };
+    }
+    const activeProxies = this.getActive();
+    if (activeProxies.length === 0) {
+      return { proxy: null, reason: "no-proxies-available" };
+    }
+    const selected = this.selectProxy(activeProxies);
+    if (selected) {
+      const state = this.states.get(selected.id);
+      if (state) {
+        state.requestCount++;
+        if (this.config.rotation === "per-proxy-limit") {
+          const limit = this.config.limit;
+          if (state.requestCount >= limit) {
+            this.disableProxy(selected, "limit-reached");
+          }
+        }
+      }
+      this.lastSelectedProxy = selected;
+      return { proxy: selected, reason: "selected" };
+    }
+    return { proxy: null, reason: "no-proxies-available" };
+  }
+  selectProxy(activeProxies) {
+    if (activeProxies.length === 0) {
+      return null;
+    }
+    const rotation = this.config.rotation;
+    if (rotation === "random") {
+      const index = Math.floor(Math.random() * activeProxies.length);
+      return activeProxies[index];
+    }
+    if (rotation === "sequential") {
+      const requestsPerProxy = this.config.requestsPerProxy ?? 1;
+      if (this.currentProxyRequests >= requestsPerProxy) {
+        this.currentIndex = (this.currentIndex + 1) % activeProxies.length;
+        this.currentProxyRequests = 0;
+      }
+      if (this.currentIndex >= activeProxies.length) {
+        this.currentIndex = 0;
+      }
+      this.currentProxyRequests++;
+      return activeProxies[this.currentIndex];
+    }
+    if (rotation === "per-proxy-limit") {
+      for (const proxy of activeProxies) {
+        const state = this.states.get(proxy.id);
+        if (state && state.isActive) {
+          return proxy;
+        }
+      }
+      return null;
+    }
+    return activeProxies[0];
+  }
+  reportSuccess(proxy) {
+    this._totalSuccesses++;
+    const state = this.states.get(proxy.id);
+    if (state) {
+      state.successCount++;
+      state.failureCount = 0;
+      state.lastSuccessAt = Date.now();
+    }
+  }
+  reportFailure(proxy, error, url) {
+    this._totalFailures++;
+    const state = this.states.get(proxy.id);
+    if (!state)
+      return;
+    const willBeDisabled = !!(this.config.autoDisableDeadProxies && state.failureCount + 1 >= (this.config.maxFailures ?? 3));
+    this.runBeforeProxyErrorHooksSync({
+      proxy,
+      error,
+      url: url || "",
+      failureCount: state.failureCount + 1,
+      willBeDisabled
+    });
+    state.failureCount++;
+    state.totalFailures++;
+    state.lastFailureAt = Date.now();
+    state.lastError = error.message;
+    let action = "continue";
+    if (this.config.autoDisableDeadProxies) {
+      const maxFailures = this.config.maxFailures ?? 3;
+      if (state.failureCount >= maxFailures) {
+        this.disableProxy(proxy, "dead");
+        action = "disabled";
+      }
+    }
+    this.runAfterProxyErrorHooksSync({
+      proxy,
+      error,
+      action
+    });
+  }
+  disableProxy(proxy, reason = "manual") {
+    const state = this.states.get(proxy.id);
+    if (!state || !state.isActive)
+      return;
+    state.isActive = false;
+    state.disabledReason = reason;
+    state.disabledAt = Date.now();
+    const { cooldown } = this.config;
+    let hasCooldown = false;
+    let reenableAt;
+    if (cooldown?.enabled && cooldown.durationMs > 0) {
+      hasCooldown = true;
+      reenableAt = Date.now() + cooldown.durationMs;
+      state.reenableAt = reenableAt;
+      const timerId = setTimeout(() => {
+        this.enableProxy(proxy, "cooldown-expired");
+        this.cooldownTimers.delete(proxy.id);
+      }, cooldown.durationMs);
+      this.cooldownTimers.set(proxy.id, timerId);
+    }
+    this.runAfterProxyDisableHooks({
+      proxy,
+      reason,
+      hasCooldown,
+      reenableAt
+    });
+  }
+  enableProxy(proxy, reason = "manual") {
+    const state = this.states.get(proxy.id);
+    if (!state || state.isActive)
+      return;
+    state.isActive = true;
+    state.failureCount = 0;
+    state.disabledReason = undefined;
+    state.disabledAt = undefined;
+    state.reenableAt = undefined;
+    const timerId = this.cooldownTimers.get(proxy.id);
+    if (timerId) {
+      clearTimeout(timerId);
+      this.cooldownTimers.delete(proxy.id);
+    }
+    this.runAfterProxyEnableHooks({
+      proxy,
+      reason
+    });
+  }
+  add(proxies) {
+    const toAdd = Array.isArray(proxies) ? proxies : [proxies];
+    for (const proxy of toAdd) {
+      if (!proxy.id) {
+        proxy.id = generateProxyId();
+      }
+      if (!this.states.has(proxy.id)) {
+        this.states.set(proxy.id, this.createInitialState(proxy));
+        this.config.proxies.push(proxy);
+      }
+    }
+  }
+  remove(proxies) {
+    const toRemove = Array.isArray(proxies) ? proxies : [proxies];
+    for (const proxy of toRemove) {
+      if (proxy.id) {
+        const timerId = this.cooldownTimers.get(proxy.id);
+        if (timerId) {
+          clearTimeout(timerId);
+          this.cooldownTimers.delete(proxy.id);
+        }
+        this.states.delete(proxy.id);
+        const index = this.config.proxies.findIndex((p) => p.id === proxy.id);
+        if (index !== -1) {
+          this.config.proxies.splice(index, 1);
+        }
+      }
+    }
+  }
+  reset() {
+    for (const timerId of this.cooldownTimers.values()) {
+      clearTimeout(timerId);
+    }
+    this.cooldownTimers.clear();
+    for (const state of this.states.values()) {
+      state.requestCount = 0;
+      state.failureCount = 0;
+      state.isActive = true;
+      state.disabledReason = undefined;
+      state.disabledAt = undefined;
+      state.reenableAt = undefined;
+    }
+    this.currentIndex = 0;
+    this.currentProxyRequests = 0;
+    this.lastSelectedProxy = null;
+  }
+  getStatus() {
+    this.processExpiredCooldowns();
+    return {
+      active: this.getActive(),
+      disabled: this.getDisabled(),
+      cooldown: this.getCooldown(),
+      total: this.states.size,
+      rotation: this.config.rotation,
+      totalRequests: this._totalRequests,
+      totalSuccesses: this._totalSuccesses,
+      totalFailures: this._totalFailures
+    };
+  }
+  getProxyState(proxy) {
+    return this.states.get(proxy.id);
+  }
+  hasAvailableProxies() {
+    return this.getActive().length > 0;
+  }
+  destroy() {
+    for (const timerId of this.cooldownTimers.values()) {
+      clearTimeout(timerId);
+    }
+    this.cooldownTimers.clear();
+    this.states.clear();
+  }
+  runBeforeProxySelectHooksSync(context) {
+    for (const hook of this.hooks.beforeProxySelect) {
+      try {
+        hook(context);
+      } catch (error) {
+        console.error("[ProxyManager] beforeProxySelect hook error:", error);
+      }
+    }
+  }
+  runAfterProxySelectHooksSync(context) {
+    for (const hook of this.hooks.afterProxySelect) {
+      try {
+        hook(context);
+      } catch (error) {
+        console.error("[ProxyManager] afterProxySelect hook error:", error);
+      }
+    }
+  }
+  runBeforeProxyErrorHooksSync(context) {
+    for (const hook of this.hooks.beforeProxyError) {
+      try {
+        hook(context);
+      } catch (error) {
+        console.error("[ProxyManager] beforeProxyError hook error:", error);
+      }
+    }
+  }
+  runAfterProxyErrorHooksSync(context) {
+    for (const hook of this.hooks.afterProxyError) {
+      try {
+        hook(context);
+      } catch (error) {
+        console.error("[ProxyManager] afterProxyError hook error:", error);
+      }
+    }
+  }
+  runAfterProxyRotateHooks(context) {
+    for (const hook of this.hooks.afterProxyRotate) {
+      try {
+        hook(context);
+      } catch (error) {
+        console.error("[ProxyManager] afterProxyRotate hook error:", error);
+      }
+    }
+  }
+  runAfterProxyDisableHooks(context) {
+    for (const hook of this.hooks.afterProxyDisable) {
+      try {
+        hook(context);
+      } catch (error) {
+        console.error("[ProxyManager] afterProxyDisable hook error:", error);
+      }
+    }
+  }
+  runAfterProxyEnableHooks(context) {
+    for (const hook of this.hooks.afterProxyEnable) {
+      try {
+        hook(context);
+      } catch (error) {
+        console.error("[ProxyManager] afterProxyEnable hook error:", error);
+      }
+    }
+  }
+}
+
+exports.ProxyManager = ProxyManager;