rezo 1.0.5 → 1.0.7

package/dist/plugin/index.cjs

@@ -1,36 +1,36 @@
-const _mod_9qwi8e = require('./crawler.cjs');
-exports.Crawler = _mod_9qwi8e.Crawler;;
-const _mod_kpl4t4 = require('./crawler-options.cjs');
-exports.CrawlerOptions = _mod_kpl4t4.CrawlerOptions;;
-const _mod_o6bmw3 = require('../cache/file-cacher.cjs');
-exports.FileCacher = _mod_o6bmw3.FileCacher;;
-const _mod_9vlknl = require('../cache/url-store.cjs');
-exports.UrlStore = _mod_9vlknl.UrlStore;;
-const _mod_31b8nd = require('./addon/oxylabs/index.cjs');
-exports.Oxylabs = _mod_31b8nd.Oxylabs;;
-const _mod_63rr5e = require('./addon/oxylabs/options.cjs');
-exports.OXYLABS_BROWSER_TYPES = _mod_63rr5e.OXYLABS_BROWSER_TYPES;
-exports.OXYLABS_COMMON_LOCALES = _mod_63rr5e.OXYLABS_COMMON_LOCALES;
-exports.OXYLABS_COMMON_GEO_LOCATIONS = _mod_63rr5e.OXYLABS_COMMON_GEO_LOCATIONS;
-exports.OXYLABS_US_STATES = _mod_63rr5e.OXYLABS_US_STATES;
-exports.OXYLABS_EUROPEAN_COUNTRIES = _mod_63rr5e.OXYLABS_EUROPEAN_COUNTRIES;
-exports.OXYLABS_ASIAN_COUNTRIES = _mod_63rr5e.OXYLABS_ASIAN_COUNTRIES;
-exports.getRandomOxylabsBrowserType = _mod_63rr5e.getRandomBrowserType;
-exports.getRandomOxylabsLocale = _mod_63rr5e.getRandomLocale;
-exports.getRandomOxylabsGeoLocation = _mod_63rr5e.getRandomGeoLocation;;
-const _mod_h4ktbx = require('./addon/decodo/index.cjs');
-exports.Decodo = _mod_h4ktbx.Decodo;;
-const _mod_t9ve3n = require('./addon/decodo/options.cjs');
-exports.DECODO_DEVICE_TYPES = _mod_t9ve3n.DECODO_DEVICE_TYPES;
-exports.DECODO_HEADLESS_MODES = _mod_t9ve3n.DECODO_HEADLESS_MODES;
-exports.DECODO_COMMON_LOCALES = _mod_t9ve3n.DECODO_COMMON_LOCALES;
-exports.DECODO_COMMON_COUNTRIES = _mod_t9ve3n.DECODO_COMMON_COUNTRIES;
-exports.DECODO_EUROPEAN_COUNTRIES = _mod_t9ve3n.DECODO_EUROPEAN_COUNTRIES;
-exports.DECODO_ASIAN_COUNTRIES = _mod_t9ve3n.DECODO_ASIAN_COUNTRIES;
-exports.DECODO_US_STATES = _mod_t9ve3n.DECODO_US_STATES;
-exports.DECODO_COMMON_CITIES = _mod_t9ve3n.DECODO_COMMON_CITIES;
-exports.getRandomDecodoDeviceType = _mod_t9ve3n.getRandomDeviceType;
-exports.getRandomDecodoLocale = _mod_t9ve3n.getRandomLocale;
-exports.getRandomDecodoCountry = _mod_t9ve3n.getRandomCountry;
-exports.getRandomDecodoCity = _mod_t9ve3n.getRandomCity;
-exports.generateDecodoSessionId = _mod_t9ve3n.generateSessionId;;
+const _mod_gtcptp = require('./crawler.cjs');
+exports.Crawler = _mod_gtcptp.Crawler;;
+const _mod_3p3u6d = require('./crawler-options.cjs');
+exports.CrawlerOptions = _mod_3p3u6d.CrawlerOptions;;
+const _mod_umel97 = require('../cache/file-cacher.cjs');
+exports.FileCacher = _mod_umel97.FileCacher;;
+const _mod_kjgxkr = require('../cache/url-store.cjs');
+exports.UrlStore = _mod_kjgxkr.UrlStore;;
+const _mod_u41p6l = require('./addon/oxylabs/index.cjs');
+exports.Oxylabs = _mod_u41p6l.Oxylabs;;
+const _mod_6hmgeg = require('./addon/oxylabs/options.cjs');
+exports.OXYLABS_BROWSER_TYPES = _mod_6hmgeg.OXYLABS_BROWSER_TYPES;
+exports.OXYLABS_COMMON_LOCALES = _mod_6hmgeg.OXYLABS_COMMON_LOCALES;
+exports.OXYLABS_COMMON_GEO_LOCATIONS = _mod_6hmgeg.OXYLABS_COMMON_GEO_LOCATIONS;
+exports.OXYLABS_US_STATES = _mod_6hmgeg.OXYLABS_US_STATES;
+exports.OXYLABS_EUROPEAN_COUNTRIES = _mod_6hmgeg.OXYLABS_EUROPEAN_COUNTRIES;
+exports.OXYLABS_ASIAN_COUNTRIES = _mod_6hmgeg.OXYLABS_ASIAN_COUNTRIES;
+exports.getRandomOxylabsBrowserType = _mod_6hmgeg.getRandomBrowserType;
+exports.getRandomOxylabsLocale = _mod_6hmgeg.getRandomLocale;
+exports.getRandomOxylabsGeoLocation = _mod_6hmgeg.getRandomGeoLocation;;
+const _mod_7v9zq0 = require('./addon/decodo/index.cjs');
+exports.Decodo = _mod_7v9zq0.Decodo;;
+const _mod_i0yci3 = require('./addon/decodo/options.cjs');
+exports.DECODO_DEVICE_TYPES = _mod_i0yci3.DECODO_DEVICE_TYPES;
+exports.DECODO_HEADLESS_MODES = _mod_i0yci3.DECODO_HEADLESS_MODES;
+exports.DECODO_COMMON_LOCALES = _mod_i0yci3.DECODO_COMMON_LOCALES;
+exports.DECODO_COMMON_COUNTRIES = _mod_i0yci3.DECODO_COMMON_COUNTRIES;
+exports.DECODO_EUROPEAN_COUNTRIES = _mod_i0yci3.DECODO_EUROPEAN_COUNTRIES;
+exports.DECODO_ASIAN_COUNTRIES = _mod_i0yci3.DECODO_ASIAN_COUNTRIES;
+exports.DECODO_US_STATES = _mod_i0yci3.DECODO_US_STATES;
+exports.DECODO_COMMON_CITIES = _mod_i0yci3.DECODO_COMMON_CITIES;
+exports.getRandomDecodoDeviceType = _mod_i0yci3.getRandomDeviceType;
+exports.getRandomDecodoLocale = _mod_i0yci3.getRandomLocale;
+exports.getRandomDecodoCountry = _mod_i0yci3.getRandomCountry;
+exports.getRandomDecodoCity = _mod_i0yci3.getRandomCity;
+exports.generateDecodoSessionId = _mod_i0yci3.generateSessionId;;

package/dist/proxy/index.cjs

@@ -1,8 +1,8 @@
 const { SocksProxyAgent: RezoSocksProxy } = require("socks-proxy-agent");
 const { HttpsProxyAgent: RezoHttpsSocks } = require("https-proxy-agent");
 const { HttpProxyAgent: RezoHttpSocks } = require("http-proxy-agent");
-const _mod_4nmwf9 = require('./manager.cjs');
-exports.ProxyManager = _mod_4nmwf9.ProxyManager;;
+const _mod_0g3tce = require('./manager.cjs');
+exports.ProxyManager = _mod_0g3tce.ProxyManager;;
 function createOptions(uri, opts) {
   if (uri instanceof URL || typeof uri === "string") {
     return {

package/dist/proxy/manager.cjs

@@ -1,3 +1,4 @@
+const { parseProxyString } = require('./index.cjs');
 function generateProxyId() {
   return `proxy_${Date.now()}_${Math.random().toString(36).substring(2, 9)}`;
 }
@@ -20,16 +21,29 @@ class ProxyManager {
     beforeProxyDisable: [],
     afterProxyDisable: [],
     afterProxyRotate: [],
-    afterProxyEnable: []
+    afterProxyEnable: [],
+    onNoProxiesAvailable: []
   };
   constructor(config) {
+    const parsedProxies = [];
+    for (const proxy of config.proxies) {
+      if (typeof proxy === "string") {
+        const parsed = parseProxyString(proxy);
+        if (parsed) {
+          parsedProxies.push(parsed);
+        }
+      } else {
+        parsedProxies.push(proxy);
+      }
+    }
     this.config = {
       failWithoutProxy: true,
       autoDisableDeadProxies: false,
       maxFailures: 3,
       retryWithNextProxy: false,
       maxProxyRetries: 3,
-      ...config
+      ...config,
+      proxies: parsedProxies
     };
     for (const proxy of this.config.proxies) {
       if (!proxy.id) {
@@ -441,6 +455,47 @@ class ProxyManager {
       }
     }
   }
+  runOnNoProxiesAvailableHooksSync(context) {
+    for (const hook of this.hooks.onNoProxiesAvailable) {
+      try {
+        hook(context);
+      } catch (error) {
+        console.error("[ProxyManager] onNoProxiesAvailable hook error:", error);
+      }
+    }
+  }
+  async runOnNoProxiesAvailableHooks(context) {
+    for (const hook of this.hooks.onNoProxiesAvailable) {
+      try {
+        await hook(context);
+      } catch (error) {
+        console.error("[ProxyManager] onNoProxiesAvailable hook error:", error);
+      }
+    }
+  }
+  notifyNoProxiesAvailable(url, error) {
+    const allProxies = Array.from(this.states.values());
+    const active = allProxies.filter((s) => s.isActive);
+    const disabled = allProxies.filter((s) => !s.isActive && !s.reenableAt);
+    const cooldown = allProxies.filter((s) => !s.isActive && s.reenableAt);
+    const disabledReasons = {
+      dead: allProxies.filter((s) => s.disabledReason === "dead").length,
+      limitReached: allProxies.filter((s) => s.disabledReason === "limit-reached").length,
+      manual: allProxies.filter((s) => s.disabledReason === "manual").length
+    };
+    const context = {
+      url,
+      error,
+      allProxies,
+      activeCount: active.length,
+      disabledCount: disabled.length,
+      cooldownCount: cooldown.length,
+      disabledReasons,
+      timestamp: Date.now()
+    };
+    this.runOnNoProxiesAvailableHooksSync(context);
+    return context;
+  }
 }
 
 exports.ProxyManager = ProxyManager;

package/dist/proxy/manager.js

@@ -1,3 +1,4 @@
+import { parseProxyString } from './index.js';
 function generateProxyId() {
   return `proxy_${Date.now()}_${Math.random().toString(36).substring(2, 9)}`;
 }
@@ -20,16 +21,29 @@ export class ProxyManager {
     beforeProxyDisable: [],
     afterProxyDisable: [],
     afterProxyRotate: [],
-    afterProxyEnable: []
+    afterProxyEnable: [],
+    onNoProxiesAvailable: []
   };
   constructor(config) {
+    const parsedProxies = [];
+    for (const proxy of config.proxies) {
+      if (typeof proxy === "string") {
+        const parsed = parseProxyString(proxy);
+        if (parsed) {
+          parsedProxies.push(parsed);
+        }
+      } else {
+        parsedProxies.push(proxy);
+      }
+    }
     this.config = {
       failWithoutProxy: true,
       autoDisableDeadProxies: false,
       maxFailures: 3,
       retryWithNextProxy: false,
       maxProxyRetries: 3,
-      ...config
+      ...config,
+      proxies: parsedProxies
     };
     for (const proxy of this.config.proxies) {
       if (!proxy.id) {
@@ -441,4 +455,45 @@ export class ProxyManager {
       }
     }
   }
+  runOnNoProxiesAvailableHooksSync(context) {
+    for (const hook of this.hooks.onNoProxiesAvailable) {
+      try {
+        hook(context);
+      } catch (error) {
+        console.error("[ProxyManager] onNoProxiesAvailable hook error:", error);
+      }
+    }
+  }
+  async runOnNoProxiesAvailableHooks(context) {
+    for (const hook of this.hooks.onNoProxiesAvailable) {
+      try {
+        await hook(context);
+      } catch (error) {
+        console.error("[ProxyManager] onNoProxiesAvailable hook error:", error);
+      }
+    }
+  }
+  notifyNoProxiesAvailable(url, error) {
+    const allProxies = Array.from(this.states.values());
+    const active = allProxies.filter((s) => s.isActive);
+    const disabled = allProxies.filter((s) => !s.isActive && !s.reenableAt);
+    const cooldown = allProxies.filter((s) => !s.isActive && s.reenableAt);
+    const disabledReasons = {
+      dead: allProxies.filter((s) => s.disabledReason === "dead").length,
+      limitReached: allProxies.filter((s) => s.disabledReason === "limit-reached").length,
+      manual: allProxies.filter((s) => s.disabledReason === "manual").length
+    };
+    const context = {
+      url,
+      error,
+      allProxies,
+      activeCount: active.length,
+      disabledCount: disabled.length,
+      cooldownCount: cooldown.length,
+      disabledReasons,
+      timestamp: Date.now()
+    };
+    this.runOnNoProxiesAvailableHooksSync(context);
+    return context;
+  }
 }

package/dist/queue/http-queue.cjs

@@ -0,0 +1,313 @@
+const { RezoQueue } = require('./queue.cjs');
+const { HttpMethodPriority } = require('./types.cjs');
+function extractDomain(url) {
+  try {
+    const parsed = new URL(url);
+    return parsed.hostname;
+  } catch {
+    const match = url.match(/^(?:https?:\/\/)?([^/:]+)/i);
+    return match?.[1] ?? "unknown";
+  }
+}
+function generateId() {
+  return `${Date.now().toString(36)}-${Math.random().toString(36).slice(2, 9)}`;
+}
+
+class HttpQueue extends RezoQueue {
+  domainQueues = new Map;
+  domainPending = new Map;
+  domainPaused = new Map;
+  domainRateLimited = new Map;
+  domainConcurrencyLimits = new Map;
+  httpStatsData;
+  httpEventHandlers = new Map;
+  httpConfig;
+  constructor(config = {}) {
+    super(config);
+    this.httpConfig = {
+      ...this.config,
+      domainConcurrency: config.domainConcurrency ?? 1 / 0,
+      requestsPerSecond: config.requestsPerSecond ?? 0,
+      respectRetryAfter: config.respectRetryAfter ?? true,
+      respectRateLimitHeaders: config.respectRateLimitHeaders ?? true,
+      autoRetry: config.autoRetry ?? false,
+      maxRetries: config.maxRetries ?? 3,
+      retryDelay: config.retryDelay ?? 1000,
+      retryStatusCodes: config.retryStatusCodes ?? [429, 500, 502, 503, 504]
+    };
+    if (typeof config.domainConcurrency === "object") {
+      for (const [domain, limit] of Object.entries(config.domainConcurrency)) {
+        this.domainConcurrencyLimits.set(domain, limit);
+      }
+    }
+    this.httpStatsData = {
+      ...this.stats,
+      byDomain: {},
+      retries: 0,
+      rateLimitHits: 0
+    };
+    if (this.httpConfig.requestsPerSecond > 0) {
+      this.config.interval = 1000;
+      this.config.intervalCap = this.httpConfig.requestsPerSecond;
+    }
+  }
+  start() {
+    super.start();
+    this.tryRunHttpNext();
+  }
+  get httpStats() {
+    return {
+      ...this.stats,
+      byDomain: { ...this.httpStatsData.byDomain },
+      retries: this.httpStatsData.retries,
+      rateLimitHits: this.httpStatsData.rateLimitHits
+    };
+  }
+  addHttp(fn, options = {}) {
+    return new Promise((resolve, reject) => {
+      const domain = options.domain ?? "default";
+      const method = (options.method ?? "GET").toUpperCase();
+      const effectivePriority = options.priority ?? (HttpMethodPriority[method] ?? HttpMethodPriority.GET);
+      const maxRetries = typeof options.retry === "number" ? options.retry : options.retry === true ? this.httpConfig.maxRetries : 0;
+      const task = {
+        id: options.id ?? generateId(),
+        fn,
+        priority: effectivePriority,
+        timeout: options.timeout ?? this.config.timeout,
+        signal: options.signal,
+        resolve,
+        reject,
+        addedAt: Date.now(),
+        domain,
+        method,
+        retryCount: 0,
+        maxRetries: this.httpConfig.autoRetry ? Math.max(maxRetries, this.httpConfig.maxRetries) : maxRetries,
+        retryDelay: options.retryDelay
+      };
+      if (options.signal?.aborted) {
+        reject(new Error("Task was cancelled before starting"));
+        return;
+      }
+      options.signal?.addEventListener("abort", () => {
+        this.cancelHttp(task.id);
+      });
+      this.ensureDomainStats(domain);
+      this.insertHttpTask(task);
+      this.emitHttp("add", { id: task.id, priority: task.priority });
+      if (!this.isPaused) {
+        this.tryRunHttpNext();
+      }
+    });
+  }
+  pauseDomain(domain) {
+    this.domainPaused.set(domain, true);
+  }
+  resumeDomain(domain) {
+    this.domainPaused.delete(domain);
+    this.emitHttp("domainAvailable", { domain });
+    this.tryRunHttpNext();
+  }
+  setDomainConcurrency(domain, limit) {
+    this.domainConcurrencyLimits.set(domain, limit);
+    this.tryRunHttpNext();
+  }
+  getDomainState(domain) {
+    return {
+      pending: this.domainPending.get(domain) ?? 0,
+      size: this.domainQueues.get(domain)?.length ?? 0,
+      isPaused: this.domainPaused.get(domain) ?? false,
+      rateLimitedUntil: this.domainRateLimited.get(domain)
+    };
+  }
+  handleRateLimit(domain, retryAfter) {
+    const until = Date.now() + retryAfter * 1000;
+    this.domainRateLimited.set(domain, until);
+    this.httpStatsData.rateLimitHits++;
+    this.ensureDomainStats(domain);
+    this.httpStatsData.byDomain[domain].rateLimited++;
+    this.emitHttp("rateLimited", { domain, retryAfter });
+    setTimeout(() => {
+      this.domainRateLimited.delete(domain);
+      this.emitHttp("domainAvailable", { domain });
+      this.tryRunHttpNext();
+    }, retryAfter * 1000);
+  }
+  cancelHttp(id) {
+    for (const [domain, queue] of this.domainQueues.entries()) {
+      const index = queue.findIndex((t) => t.id === id);
+      if (index !== -1) {
+        const [task] = queue.splice(index, 1);
+        task.reject(new Error("Task was cancelled"));
+        this.ensureDomainStats(domain);
+        this.httpStatsData.byDomain[domain].failed++;
+        this.emitHttp("cancelled", { id });
+        return true;
+      }
+    }
+    return false;
+  }
+  onHttp(event, handler) {
+    if (!this.httpEventHandlers.has(event)) {
+      this.httpEventHandlers.set(event, new Set);
+    }
+    this.httpEventHandlers.get(event).add(handler);
+  }
+  offHttp(event, handler) {
+    this.httpEventHandlers.get(event)?.delete(handler);
+  }
+  clearHttp() {
+    for (const [domain, queue] of this.domainQueues.entries()) {
+      this.ensureDomainStats(domain);
+      for (const task of queue) {
+        task.reject(new Error("Queue was cleared"));
+        this.httpStatsData.byDomain[domain].failed++;
+        this.emitHttp("cancelled", { id: task.id });
+      }
+    }
+    this.domainQueues.clear();
+    this.domainPending.clear();
+  }
+  destroy() {
+    this.clearHttp();
+    this.httpEventHandlers.clear();
+    super.destroy();
+  }
+  insertHttpTask(task) {
+    const domain = task.domain ?? "default";
+    if (!this.domainQueues.has(domain)) {
+      this.domainQueues.set(domain, []);
+    }
+    const queue = this.domainQueues.get(domain);
+    let insertIndex = queue.length;
+    for (let i = 0;i < queue.length; i++) {
+      if (task.priority > queue[i].priority) {
+        insertIndex = i;
+        break;
+      }
+    }
+    queue.splice(insertIndex, 0, task);
+  }
+  getDomainLimit(domain) {
+    const specificLimit = this.domainConcurrencyLimits.get(domain);
+    if (specificLimit !== undefined)
+      return specificLimit;
+    if (typeof this.httpConfig.domainConcurrency === "number") {
+      return this.httpConfig.domainConcurrency;
+    }
+    return 1 / 0;
+  }
+  canRunDomain(domain) {
+    if (this.domainPaused.get(domain))
+      return false;
+    const rateLimitedUntil = this.domainRateLimited.get(domain);
+    if (rateLimitedUntil && Date.now() < rateLimitedUntil)
+      return false;
+    const pending = this.domainPending.get(domain) ?? 0;
+    const limit = this.getDomainLimit(domain);
+    return pending < limit;
+  }
+  tryRunHttpNext() {
+    if (this.isPaused)
+      return;
+    for (const [domain, queue] of this.domainQueues.entries()) {
+      if (queue.length === 0)
+        continue;
+      if (!this.canRunDomain(domain))
+        continue;
+      const task = queue.shift();
+      this.runHttpTask(task);
+    }
+  }
+  async runHttpTask(task) {
+    const domain = task.domain ?? "default";
+    const pending = (this.domainPending.get(domain) ?? 0) + 1;
+    this.domainPending.set(domain, pending);
+    this.ensureDomainStats(domain);
+    this.httpStatsData.byDomain[domain].pending++;
+    this.emitHttp("start", { id: task.id });
+    const startTime = Date.now();
+    let timeoutId;
+    try {
+      let result;
+      if (task.timeout && task.timeout > 0) {
+        result = await Promise.race([
+          task.fn(),
+          new Promise((_, reject) => {
+            timeoutId = setTimeout(() => {
+              this.emitHttp("timeout", { id: task.id });
+              reject(new Error(`Task ${task.id} timed out after ${task.timeout}ms`));
+            }, task.timeout);
+          })
+        ]);
+      } else {
+        result = await task.fn();
+      }
+      if (timeoutId)
+        clearTimeout(timeoutId);
+      const duration = Date.now() - startTime;
+      this.httpStatsData.byDomain[domain].completed++;
+      this.emitHttp("completed", { id: task.id, result, duration });
+      task.resolve(result);
+    } catch (error) {
+      if (timeoutId)
+        clearTimeout(timeoutId);
+      const shouldRetry = task.retryCount < task.maxRetries;
+      if (shouldRetry) {
+        task.retryCount++;
+        this.httpStatsData.retries++;
+        const delay = this.getRetryDelay(task);
+        this.emitHttp("retry", {
+          id: task.id,
+          attempt: task.retryCount,
+          error
+        });
+        setTimeout(() => {
+          this.insertHttpTask(task);
+          this.tryRunHttpNext();
+        }, delay);
+      } else {
+        this.httpStatsData.byDomain[domain].failed++;
+        this.emitHttp("error", { id: task.id, error });
+        task.reject(error);
+      }
+    } finally {
+      const newPending = (this.domainPending.get(domain) ?? 1) - 1;
+      this.domainPending.set(domain, newPending);
+      this.httpStatsData.byDomain[domain].pending = newPending;
+      this.tryRunHttpNext();
+    }
+  }
+  getRetryDelay(task) {
+    if (task.retryDelay !== undefined) {
+      return task.retryDelay;
+    }
+    const baseDelay = typeof this.httpConfig.retryDelay === "function" ? this.httpConfig.retryDelay(task.retryCount) : this.httpConfig.retryDelay;
+    return baseDelay * Math.pow(2, task.retryCount - 1);
+  }
+  ensureDomainStats(domain) {
+    if (!this.httpStatsData.byDomain[domain]) {
+      this.httpStatsData.byDomain[domain] = {
+        pending: 0,
+        completed: 0,
+        failed: 0,
+        rateLimited: 0
+      };
+    }
+  }
+  emitHttp(event, data) {
+    const handlers = this.httpEventHandlers.get(event);
+    if (handlers) {
+      for (const handler of handlers) {
+        try {
+          handler(data);
+        } catch {}
+      }
+    }
+    if (event in this.config) {
+      this.emit(event, data);
+    }
+  }
+}
+
+exports.extractDomain = extractDomain;
+exports.HttpQueue = HttpQueue;