rezo 1.0.43 → 1.0.44

package/dist/cache/url-store.js

@@ -1,291 +0,0 @@
-import fs from "node:fs";
-import path from "node:path";
-import { createHash } from "node:crypto";
-function detectRuntime() {
-  if (typeof globalThis.Bun !== "undefined") {
-    return "bun";
-  }
-  if (typeof globalThis.Deno !== "undefined") {
-    return "deno";
-  }
-  return "node";
-}
-async function createDatabase(dbPath) {
-  const runtime = detectRuntime();
-  if (runtime === "bun") {
-    const { Database } = await import("bun:sqlite");
-    const db = new Database(dbPath);
-    return {
-      run: (sql, ...params) => db.run(sql, ...params),
-      get: (sql, ...params) => db.query(sql).get(...params),
-      all: (sql, ...params) => db.query(sql).all(...params),
-      close: () => db.close()
-    };
-  }
-  if (runtime === "deno") {
-    try {
-      const { Database } = await import("node:sqlite");
-      const db = new Database(dbPath);
-      return {
-        run: (sql, ...params) => db.exec(sql, params),
-        get: (sql, ...params) => {
-          const stmt = db.prepare(sql);
-          return stmt.get(...params);
-        },
-        all: (sql, ...params) => {
-          const stmt = db.prepare(sql);
-          return stmt.all(...params);
-        },
-        close: () => db.close()
-      };
-    } catch {
-      throw new Error("Deno SQLite support requires Node.js compatibility mode");
-    }
-  }
-  const { DatabaseSync } = await import("node:sqlite");
-  const db = new DatabaseSync(dbPath);
-  return {
-    run: (sql, ...params) => {
-      if (params.length === 0) {
-        db.exec(sql);
-      } else {
-        db.prepare(sql).run(...params);
-      }
-    },
-    get: (sql, ...params) => {
-      const stmt = db.prepare(sql);
-      return stmt.get(...params);
-    },
-    all: (sql, ...params) => {
-      const stmt = db.prepare(sql);
-      return stmt.all(...params);
-    },
-    close: () => db.close()
-  };
-}
-
-export class UrlStore {
-  db = null;
-  options;
-  storeDir;
-  dbPath;
-  closed = false;
-  initPromise = null;
-  constructor(options = {}) {
-    this.options = {
-      storeDir: options.storeDir || "./url-store",
-      dbFileName: options.dbFileName || "urls.db",
-      ttl: options.ttl || 604800000,
-      maxUrls: options.maxUrls ?? 0,
-      hashUrls: options.hashUrls ?? false
-    };
-    this.storeDir = path.resolve(this.options.storeDir);
-    this.dbPath = path.join(this.storeDir, this.options.dbFileName);
-    if (!fs.existsSync(this.storeDir)) {
-      fs.mkdirSync(this.storeDir, { recursive: true });
-    }
-  }
-  static async create(options = {}) {
-    const store = new UrlStore(options);
-    await store.initialize();
-    return store;
-  }
-  async initialize() {
-    if (this.initPromise)
-      return this.initPromise;
-    this.initPromise = (async () => {
-      this.db = await createDatabase(this.dbPath);
-      this.db.run(`
-        CREATE TABLE IF NOT EXISTS urls (
-          url TEXT PRIMARY KEY,
-          visitedAt INTEGER NOT NULL,
-          expiresAt INTEGER NOT NULL,
-          namespace TEXT DEFAULT 'default',
-          metadata TEXT
-        )
-      `);
-      this.db.run("CREATE INDEX IF NOT EXISTS idx_expires ON urls(expiresAt)");
-      this.db.run("CREATE INDEX IF NOT EXISTS idx_namespace ON urls(namespace)");
-      this.db.run("CREATE INDEX IF NOT EXISTS idx_visited ON urls(visitedAt)");
-    })();
-    return this.initPromise;
-  }
-  getUrlKey(url) {
-    if (this.options.hashUrls) {
-      return createHash("sha256").update(url).digest("hex");
-    }
-    return url;
-  }
-  async set(url, namespace = "default", metadata, ttl) {
-    if (this.closed)
-      throw new Error("UrlStore is closed");
-    await this.initialize();
-    const key = this.getUrlKey(url);
-    const now = Date.now();
-    const expiresAt = now + (ttl ?? this.options.ttl);
-    const metaStr = metadata ? JSON.stringify(metadata) : null;
-    this.db.run(`
-      INSERT OR REPLACE INTO urls (url, visitedAt, expiresAt, namespace, metadata)
-      VALUES (?, ?, ?, ?, ?)
-    `, key, now, expiresAt, namespace, metaStr);
-    if (this.options.maxUrls > 0) {
-      const count = this.db.get("SELECT COUNT(*) as cnt FROM urls");
-      if (count && count.cnt > this.options.maxUrls) {
-        const excess = count.cnt - this.options.maxUrls;
-        this.db.run(`
-          DELETE FROM urls WHERE url IN (
-            SELECT url FROM urls ORDER BY visitedAt ASC LIMIT ?
-          )
-        `, excess);
-      }
-    }
-  }
-  async has(url, namespace) {
-    if (this.closed)
-      return false;
-    await this.initialize();
-    const key = this.getUrlKey(url);
-    const now = Date.now();
-    let entry;
-    if (namespace) {
-      entry = this.db.get("SELECT url, expiresAt FROM urls WHERE url = ? AND namespace = ?", key, namespace);
-    } else {
-      entry = this.db.get("SELECT url, expiresAt FROM urls WHERE url = ?", key);
-    }
-    if (!entry)
-      return false;
-    return entry.expiresAt >= now;
-  }
-  async hasMany(urls, namespace) {
-    if (this.closed)
-      return new Set;
-    await this.initialize();
-    const result = new Set;
-    const now = Date.now();
-    const batchSize = 100;
-    for (let i = 0;i < urls.length; i += batchSize) {
-      const batch = urls.slice(i, i + batchSize);
-      const keys = batch.map((u) => this.getUrlKey(u));
-      const placeholders = keys.map(() => "?").join(",");
-      let entries;
-      if (namespace) {
-        entries = this.db.all(`SELECT url, expiresAt FROM urls WHERE url IN (${placeholders}) AND namespace = ?`, ...keys, namespace);
-      } else {
-        entries = this.db.all(`SELECT url, expiresAt FROM urls WHERE url IN (${placeholders})`, ...keys);
-      }
-      for (const entry of entries) {
-        if (entry.expiresAt >= now) {
-          const idx = keys.indexOf(entry.url);
-          if (idx !== -1) {
-            result.add(batch[idx]);
-          }
-        }
-      }
-    }
-    return result;
-  }
-  async getMetadata(url, namespace) {
-    if (this.closed)
-      return null;
-    await this.initialize();
-    const key = this.getUrlKey(url);
-    let entry;
-    if (namespace) {
-      entry = this.db.get("SELECT metadata FROM urls WHERE url = ? AND namespace = ?", key, namespace);
-    } else {
-      entry = this.db.get("SELECT metadata FROM urls WHERE url = ?", key);
-    }
-    if (!entry?.metadata)
-      return null;
-    try {
-      return JSON.parse(entry.metadata);
-    } catch {
-      return null;
-    }
-  }
-  async delete(url, namespace) {
-    if (this.closed)
-      return false;
-    await this.initialize();
-    const key = this.getUrlKey(url);
-    if (namespace) {
-      this.db.run("DELETE FROM urls WHERE url = ? AND namespace = ?", key, namespace);
-    } else {
-      this.db.run("DELETE FROM urls WHERE url = ?", key);
-    }
-    return true;
-  }
-  async clear(namespace) {
-    if (this.closed)
-      return;
-    await this.initialize();
-    if (namespace) {
-      this.db.run("DELETE FROM urls WHERE namespace = ?", namespace);
-    } else {
-      this.db.run("DELETE FROM urls");
-    }
-  }
-  async cleanup() {
-    if (this.closed)
-      return 0;
-    await this.initialize();
-    const now = Date.now();
-    const countBefore = this.db.get("SELECT COUNT(*) as cnt FROM urls");
-    this.db.run("DELETE FROM urls WHERE expiresAt < ?", now);
-    const countAfter = this.db.get("SELECT COUNT(*) as cnt FROM urls");
-    return (countBefore?.cnt || 0) - (countAfter?.cnt || 0);
-  }
-  async getAll(namespace = "default", includeExpired = false) {
-    if (this.closed)
-      return [];
-    await this.initialize();
-    const now = Date.now();
-    let entries;
-    if (includeExpired) {
-      entries = this.db.all("SELECT url FROM urls WHERE namespace = ?", namespace);
-    } else {
-      entries = this.db.all("SELECT url FROM urls WHERE namespace = ? AND expiresAt >= ?", namespace, now);
-    }
-    return entries.map((e) => e.url);
-  }
-  async stats(namespace) {
-    if (this.closed)
-      return { total: 0, expired: 0, namespaces: 0 };
-    await this.initialize();
-    const now = Date.now();
-    let total;
-    let expired;
-    if (namespace) {
-      total = this.db.get("SELECT COUNT(*) as cnt FROM urls WHERE namespace = ?", namespace);
-      expired = this.db.get("SELECT COUNT(*) as cnt FROM urls WHERE namespace = ? AND expiresAt < ?", namespace, now);
-    } else {
-      total = this.db.get("SELECT COUNT(*) as cnt FROM urls");
-      expired = this.db.get("SELECT COUNT(*) as cnt FROM urls WHERE expiresAt < ?", now);
-    }
-    const namespaceCount = this.db.get("SELECT COUNT(DISTINCT namespace) as cnt FROM urls");
-    return {
-      total: total?.cnt || 0,
-      expired: expired?.cnt || 0,
-      namespaces: namespaceCount?.cnt || 0
-    };
-  }
-  async close() {
-    if (this.closed)
-      return;
-    this.closed = true;
-    await this.initPromise;
-    if (this.db) {
-      try {
-        this.db.close();
-      } catch {}
-      this.db = null;
-    }
-  }
-  get isClosed() {
-    return this.closed;
-  }
-  get path() {
-    return this.dbPath;
-  }
-}
-export default UrlStore;

package/dist/plugin/addon/decodo/index.cjs

@@ -1 +0,0 @@
-var{Rezo:O}=require("../../../core/rezo.cjs"),r=require("./options.cjs");exports.DECODO_DEVICE_TYPES=r.DECODO_DEVICE_TYPES;exports.DECODO_HEADLESS_MODES=r.DECODO_HEADLESS_MODES;exports.DECODO_COMMON_LOCALES=r.DECODO_COMMON_LOCALES;exports.DECODO_COMMON_COUNTRIES=r.DECODO_COMMON_COUNTRIES;exports.DECODO_EUROPEAN_COUNTRIES=r.DECODO_EUROPEAN_COUNTRIES;exports.DECODO_ASIAN_COUNTRIES=r.DECODO_ASIAN_COUNTRIES;exports.DECODO_US_STATES=r.DECODO_US_STATES;exports.DECODO_COMMON_CITIES=r.DECODO_COMMON_CITIES;exports.getRandomDeviceType=r.getRandomDeviceType;exports.getRandomLocale=r.getRandomLocale;exports.getRandomCountry=r.getRandomCountry;exports.getRandomCity=r.getRandomCity;exports.generateSessionId=r.generateSessionId;var u="https://scraper-api.smartproxy.com/v2/scrape";class d{config;http;authHeader;constructor(e){if(!e.username||!e.password)throw Error("Decodo username and password are required");this.config={username:e.username,password:e.password,deviceType:e.deviceType??"desktop",locale:e.locale??"en-US",country:e.country??"",state:e.state??"",city:e.city??"",headless:e.headless??void 0,headers:e.headers??{},sessionId:e.sessionId??"",sessionDuration:e.sessionDuration??0,javascript:e.javascript??"",javascriptWait:e.javascriptWait??0,waitForCss:e.waitForCss??"",timeout:e.timeout??120000},this.http=new O({baseURL:u,timeout:this.config.timeout}),this.authHeader=`Basic ${Buffer.from(`${e.username}:${e.password}`).toString("base64")}`}async scrape(e,t){let s={...this.config,...t,headers:{...this.config.headers,...t?.headers||{}}},n=this.buildRequestBody(e,s),a=(await this.http.postJson(u,n,{headers:{Authorization:this.authHeader,"Content-Type":"application/json"}})).data;if(a.error)throw Error(`Decodo API error: ${a.error} (${a.error_code||"unknown"})`);if(!a.results||a.results.length===0)throw Error("Decodo API returned no results");let o=a.results[0];return{statusCode:o.status_code,url:o.url,content:o.body,cookies:o.cookies||[],headers:o.headers||{},taskId:a.id,rendered:!!s.headless,country:s.country||void 0,city:s.city||void 0,state:s.state||void 0,deviceType:s.deviceType,contentType:o.content_type,contentLength:o.content_length,raw:a}}async scrapeMany(e,t,s=1000){let n=[];for(let i=0;i<e.length;i++){let a=await this.scrape(e[i],t);if(n.push(a),i<e.length-1&&s>0)await new Promise((o)=>setTimeout(o,s))}return n}async scrapeWithSession(e,t,s=10){let n=`session_${Date.now()}_${Math.random().toString(36).substring(2,11)}`;return this.scrapeMany(e,{...t,sessionId:n,sessionDuration:s},500)}buildRequestBody(e,t){let s={url:e,return_cookies:!0,return_headers:!0};if(t.deviceType)s.device_type=t.deviceType;if(t.headless)s.headless=t.headless;if(t.locale)s.locale=t.locale;if(t.country)s.country=t.country;if(t.state)s.state=t.state;if(t.city)s.city=t.city;if(t.sessionId){if(s.session=t.sessionId,t.sessionDuration)s.session_duration=t.sessionDuration}if(t.headers&&Object.keys(t.headers).length>0)s.headers=t.headers;if(t.javascript){if(s.javascript=t.javascript,t.javascriptWait)s.javascript_wait=t.javascriptWait}if(t.waitForCss)s.wait_for_css=t.waitForCss;return s}getConfig(){return{...this.config,password:"***"}}withConfig(e){return new d({...this.config,...e})}async testConnection(){try{return await this.scrape("https://httpbin.org/ip"),!0}catch(e){throw Error(`Decodo connection test failed: ${e.message}`)}}}exports.Decodo=d;exports.default=d;module.exports=Object.assign(d,exports);

package/dist/plugin/addon/decodo/index.js

@@ -1 +0,0 @@
-import{Rezo as u}from"../../../core/rezo.js";import{DECODO_DEVICE_TYPES as y,DECODO_HEADLESS_MODES as l,DECODO_COMMON_LOCALES as w,DECODO_COMMON_COUNTRIES as O,DECODO_EUROPEAN_COUNTRIES as _,DECODO_ASIAN_COUNTRIES as D,DECODO_US_STATES as C,DECODO_COMMON_CITIES as E,getRandomDeviceType as m,getRandomLocale as v,getRandomCountry as S,getRandomCity as I,generateSessionId as T}from"./options.js";var d="https://scraper-api.smartproxy.com/v2/scrape";class n{config;http;authHeader;constructor(e){if(!e.username||!e.password)throw Error("Decodo username and password are required");this.config={username:e.username,password:e.password,deviceType:e.deviceType??"desktop",locale:e.locale??"en-US",country:e.country??"",state:e.state??"",city:e.city??"",headless:e.headless??void 0,headers:e.headers??{},sessionId:e.sessionId??"",sessionDuration:e.sessionDuration??0,javascript:e.javascript??"",javascriptWait:e.javascriptWait??0,waitForCss:e.waitForCss??"",timeout:e.timeout??120000},this.http=new u({baseURL:d,timeout:this.config.timeout}),this.authHeader=`Basic ${Buffer.from(`${e.username}:${e.password}`).toString("base64")}`}async scrape(e,t){let s={...this.config,...t,headers:{...this.config.headers,...t?.headers||{}}},o=this.buildRequestBody(e,s),r=(await this.http.postJson(d,o,{headers:{Authorization:this.authHeader,"Content-Type":"application/json"}})).data;if(r.error)throw Error(`Decodo API error: ${r.error} (${r.error_code||"unknown"})`);if(!r.results||r.results.length===0)throw Error("Decodo API returned no results");let a=r.results[0];return{statusCode:a.status_code,url:a.url,content:a.body,cookies:a.cookies||[],headers:a.headers||{},taskId:r.id,rendered:!!s.headless,country:s.country||void 0,city:s.city||void 0,state:s.state||void 0,deviceType:s.deviceType,contentType:a.content_type,contentLength:a.content_length,raw:r}}async scrapeMany(e,t,s=1000){let o=[];for(let i=0;i<e.length;i++){let r=await this.scrape(e[i],t);if(o.push(r),i<e.length-1&&s>0)await new Promise((a)=>setTimeout(a,s))}return o}async scrapeWithSession(e,t,s=10){let o=`session_${Date.now()}_${Math.random().toString(36).substring(2,11)}`;return this.scrapeMany(e,{...t,sessionId:o,sessionDuration:s},500)}buildRequestBody(e,t){let s={url:e,return_cookies:!0,return_headers:!0};if(t.deviceType)s.device_type=t.deviceType;if(t.headless)s.headless=t.headless;if(t.locale)s.locale=t.locale;if(t.country)s.country=t.country;if(t.state)s.state=t.state;if(t.city)s.city=t.city;if(t.sessionId){if(s.session=t.sessionId,t.sessionDuration)s.session_duration=t.sessionDuration}if(t.headers&&Object.keys(t.headers).length>0)s.headers=t.headers;if(t.javascript){if(s.javascript=t.javascript,t.javascriptWait)s.javascript_wait=t.javascriptWait}if(t.waitForCss)s.wait_for_css=t.waitForCss;return s}getConfig(){return{...this.config,password:"***"}}withConfig(e){return new n({...this.config,...e})}async testConnection(){try{return await this.scrape("https://httpbin.org/ip"),!0}catch(e){throw Error(`Decodo connection test failed: ${e.message}`)}}}var R=n;export{v as getRandomLocale,m as getRandomDeviceType,S as getRandomCountry,I as getRandomCity,T as generateSessionId,R as default,n as Decodo,C as DECODO_US_STATES,l as DECODO_HEADLESS_MODES,_ as DECODO_EUROPEAN_COUNTRIES,y as DECODO_DEVICE_TYPES,w as DECODO_COMMON_LOCALES,O as DECODO_COMMON_COUNTRIES,E as DECODO_COMMON_CITIES,D as DECODO_ASIAN_COUNTRIES};

package/dist/plugin/crawler-options.cjs

@@ -1 +0,0 @@
-var{defineProperty:f,getOwnPropertyNames:x,getOwnPropertyDescriptor:y}=Object,A=Object.prototype.hasOwnProperty;var m=new WeakMap,v=(e)=>{var r=m.get(e),i;if(r)return r;if(r=f({},"__esModule",{value:!0}),e&&typeof e==="object"||typeof e==="function")x(e).map((a)=>!A.call(r,a)&&f(r,a,{get:()=>e[a],enumerable:!(i=y(e,a))||i.enumerable}));return m.set(e,r),r};var g={};module.exports=v(g);var{RezoQueue:$}=require("../queue/queue.cjs"),{Oxylabs:p}=require("./addon/oxylabs/index.cjs"),k=require("node:path"),M=require("node:os"),{Decodo:O}=require("./addon/decodo/index.cjs");class b{baseUrl;adapter;enableNavigationHistory;sessionId;rejectUnauthorized;userAgent;useRndUserAgent;timeout;maxRedirects;maxRetryAttempts;retryDelay;retryOnStatusCode;forceRevisit;retryWithoutProxyOnStatusCode;retryOnProxyError;maxRetryOnProxyError;allowRevisiting;enableCache;cacheTTL;cacheDir;throwFatalError;debug;oxylabs=[];decodo=[];proxies=[];limiters=[];requestHeaders=[];userAgents=_();constructor(e={}){this.baseUrl=e.baseUrl||"",this.adapter=e.adapter??"http",this.enableNavigationHistory=e.enableNavigationHistory??!1,this.sessionId=e.sessionId??`session_${Date.now()}_${Math.random().toString(36).slice(2,8)}`,this.rejectUnauthorized=e.rejectUnauthorized??!0,this.userAgent=e.userAgent,this.useRndUserAgent=e.useRndUserAgent??!1,this.timeout=e.timeout??30000,this.maxRedirects=e.maxRedirects??10,this.maxRetryAttempts=e.maxRetryAttempts??3,this.retryDelay=e.retryDelay??0,this.retryOnStatusCode=e.retryOnStatusCode??[408,429,500,502,503,504],this.forceRevisit=e.forceRevisit??!1,this.retryWithoutProxyOnStatusCode=e.retryWithoutProxyOnStatusCode??[407,403],this.retryOnProxyError=e.retryOnProxyError??!0,this.maxRetryOnProxyError=e.maxRetryOnProxyError??3,this.allowRevisiting=e.allowRevisiting??!1,this.enableCache=e.enableCache??!0,this.cacheTTL=e.cacheTTL??604800000,this.cacheDir=e.cacheDir??k.join(M.tmpdir(),"uiniqhtt_cache"),this.throwFatalError=e.throwFatalError??!1,this.debug=e.debug??!1,this._addHeaders(e.headers),this._addOxylabs(e.oxylabs),this._addProxies(e.proxy),this._addLimiters(e.limiter)}getConfiguredDomains(e){return(e==="headers"?this.requestHeaders:e==="limiters"?this.limiters:e==="oxylabs"?this.oxylabs:this.proxies).filter((i)=>i.domain).map((i)=>i.domain).filter((i,a,t)=>t.indexOf(i)===a)}removeDomain(e){return this.requestHeaders=this.requestHeaders.filter((r)=>!r.domain||!this._domainsEqual(r.domain,e)),this.proxies=this.proxies.filter((r)=>!r.domain||!this._domainsEqual(r.domain,e)),this.limiters=this.limiters.filter((r)=>!r.domain||!this._domainsEqual(r.domain,e)),this.oxylabs=this.oxylabs.filter((r)=>!r.domain||!this._domainsEqual(r.domain,e)),this}_domainsEqual(e,r){if(Array.isArray(e)&&Array.isArray(r))return e.length===r.length&&e.every((i,a)=>i===r[a]);return e===r}getConfigurationSummary(){let e=(r)=>({total:r.length,global:r.filter((i)=>i.isGlobal).length,domainSpecific:r.filter((i)=>!i.isGlobal&&i.domain).length});return{headers:e(this.requestHeaders),proxies:e(this.proxies),limiters:e(this.limiters),oxylabs:e(this.oxylabs)}}_addHeaders(e){if(!e||!e.enable)return;for(let r of e.httpHeaders){let{domain:i,isGlobal:a,headers:t}=r;if(!i&&!a)continue;if(r instanceof Headers&&Object.keys(Object.fromEntries(r.entries())).length<1)continue;else if(Object.keys(t).length<1)continue;t=r instanceof Headers?Object.fromEntries(r.entries()):t,this.requestHeaders.push({domain:i,isGlobal:a,headers:t})}}_addProxies(e){if(!e||!e.enable)return;for(let r of e.proxies){let{domain:i,isGlobal:a,proxy:t}=r;if(!i&&!a)continue;if(!t||Object.keys(t).length<1)continue;this.proxies.push({domain:i,isGlobal:a,proxy:t})}}_addLimiters(e){if(!e||!e.enable)return;for(let r of e.limiters){let{domain:i,isGlobal:a,options:t}=r;if(!i&&!a)continue;if(!t||Object.keys(t).length<1)continue;this.limiters.push({domain:i,isGlobal:a,pqueue:new $(t)})}}_addOxylabs(e){if(!e||!e.enable)return;for(let r of e.labs){let{domain:i,isGlobal:a,options:t,queueOptions:s}=r;if(!i&&!a)continue;if(!t||Object.keys(t).length<1)continue;this.oxylabs.push({domain:i,isGlobal:a,adaptar:new p(t)})}}_addDecodo(e){if(!e||!e.enable)return;for(let r of e.labs){let{domain:i,isGlobal:a,options:t,queueOptions:s}=r;if(!i&&!a)continue;if(!t||Object.keys(t).length<1)continue;this.decodo.push({domain:i,isGlobal:a,adaptar:new O(t)})}}addHeaders(e){return this._addHeaders({enable:!0,httpHeaders:[e]}),this}addProxy(e){return this._addProxies({enable:!0,proxies:[e]}),this}addLimiter(e){return this._addLimiters({enable:!0,limiters:[e]}),this}addOxylabs(e){return this._addOxylabs({enable:!0,labs:[e]}),this}addDecodo(e){return this._addDecodo({enable:!0,labs:[e]}),this}clearGlobalConfigs(){if(Array.isArray(this.requestHeaders))this.requestHeaders=this.requestHeaders.filter((e)=>!e.isGlobal);if(Array.isArray(this.oxylabs))this.oxylabs=this.oxylabs.filter((e)=>!e.isGlobal);if(Array.isArray(this.limiters))this.limiters=this.limiters.filter((e)=>!e.isGlobal);if(Array.isArray(this.proxies))this.proxies=this.proxies.filter((e)=>!e.isGlobal);return this}getAdapter(e,r,i,a){if(!this.getDomainName(e))return null;let s=[],n=r==="headers"?this.requestHeaders:r==="limiters"?this.limiters:r==="oxylabs"?this.oxylabs:r==="decodo"?this.decodo:this.proxies;for(let o=0;o<n.length;o++)if(this._hasDomain(e,n[o].domain))s.push(o);if(s.length){let o=a?s[this.rnd(0,s.length-1)]:s[0];return r==="headers"?this.requestHeaders[o].headers:r==="limiters"?this.limiters[o].pqueue:r==="oxylabs"?this.oxylabs[o].adaptar:r==="decodo"?this.decodo[o].adaptar:this.proxies[o].proxy}s.length=0;for(let o=0;o<n.length;o++)s.push(o);if(s.length){let o=a?s[this.rnd(0,s.length-1)]:s[0];if(n[o].isGlobal&&i)return r==="headers"?this.requestHeaders[o].headers:r==="limiters"?this.limiters[o].pqueue:r==="oxylabs"?this.oxylabs[o].adaptar:r==="decodo"?this.decodo[o].adaptar:this.proxies[o].proxy}return null}rnd(e=0,r=Number.MAX_VALUE){return Math.floor(Math.random()*(r-e+1))+e}hasDomain(e,r,i){if(!this.getDomainName(e))return!1;let t=r==="headers"?this.requestHeaders:r==="limiters"?this.limiters:r==="oxylabs"?this.oxylabs:r==="decodo"?this.decodo:this.proxies;for(let s=0;s<t.length;s++)if(this._hasDomain(e,t[s].domain))return!0;if(i){for(let s=0;s<t.length;s++)if(t[s].isGlobal)return!0}return!1}pickHeaders(e,r,i,a){let t=this.getAdapter(e,"headers",r),s=new Headers(t??{});if(i&&i instanceof Headers)for(let[n,o]of Object.entries(i.entries()))s.set(n,o);else if(i&&typeof i==="object"){for(let[n,o]of Object.entries(i))if(typeof o==="string")s.set(n,o)}if(a)s.set("user-agent",this.getRandomUserAgent());return Object.fromEntries(s.entries())}_hasDomain(e,r){if(!r)return!1;let i=this.getDomainName(e);if(!i)return!1;let a=(s)=>{return/[\^\$\*\+\?\{\}\[\]\(\)\|\\]/.test(s)||s.startsWith("/")||s.includes(".*")||s.includes(".+")},t=(s)=>{if(s instanceof RegExp)return s.test(i)||s.test(e);let n=s.toString().trim();if(i.toLowerCase()===n.toLowerCase())return!0;if(n.includes("*")){let l=n.replace(/[.*+?^${}()|[\]\\]/g,"\\$&").replace(/\\\*/g,".*"),h=new RegExp(`^${l}$`,"i");return h.test(i)||h.test(e)}if(a(n))try{let l=n,h="i",c=n.match(/^\/(.*)\/(\w*)$/);if(c)l=c[1],h=c[2]||"i";let u=new RegExp(l,h);return u.test(i)||u.test(e)}catch(l){return i.toLowerCase().includes(n.toLowerCase())}let o=i.toLowerCase(),d=n.toLowerCase();return o===d||o.endsWith("."+d)||d.endsWith("."+o)};if(Array.isArray(r)){for(let s of r)if(t(s))return!0;return!1}return t(r)}getDomainName(e){if(this.isValidUrl(e))return new URL(e).hostname;else if(this.isHostName(e))return e;return null}isHostName(e){if(!e)return!1;if(e.length>255)return!1;let r=/^(?:(?:[a-zA-Z0-9](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?\.)+ [a-zA-Z]{2,})$/;return e=e.trim().toLowerCase(),r.test(e)&&!e.startsWith("-")&&!e.endsWith("-")}isValidUrl(e){if(!e)return!1;e=e.trim();try{let r=new URL(e);if(!r.protocol||!["http:","https:"].includes(r.protocol.toLowerCase()))return!1;if(!r.hostname)return!1;if(!/^(?:(?:[a-zA-Z0-9](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?\.)+[a-zA-Z]{2,})$/.test(r.hostname))return!1;return!0}catch{return!1}}getRandomUserAgent(){return this.userAgents[Math.floor(Math.random()*this.userAgents.length)]}}function _(){let e=[{name:"Chrome",version:"91.0.4472.124",engine:"AppleWebKit/537.36"},{name:"Firefox",version:"89.0",engine:"Gecko/20100101"},{name:"Safari",version:"14.1.1",engine:"AppleWebKit/605.1.15"},{name:"Edge",version:"91.0.864.59",engine:"AppleWebKit/537.36"},{name:"Opera",version:"77.0.4054.277",engine:"AppleWebKit/537.36"},{name:"Vivaldi",version:"3.8.2259.42",engine:"AppleWebKit/537.36"},{name:"Brave",version:"1.26.74",engine:"AppleWebKit/537.36"},{name:"Chromium",version:"91.0.4472.101",engine:"AppleWebKit/537.36"},{name:"Yandex",version:"21.5.3.742",engine:"AppleWebKit/537.36"},{name:"Maxthon",version:"5.3.8.2000",engine:"AppleWebKit/537.36"}],r=["Windows NT 10.0","Windows NT 6.1","Macintosh; Intel Mac OS X 10_15_7","Macintosh; Intel Mac OS X 11_4_0","X11; Linux x86_64","X11; Ubuntu; Linux x86_64"],i=[];for(let a=0;a<200;a++){let t=e[Math.floor(Math.random()*e.length)],s=r[Math.floor(Math.random()*r.length)],n="";switch(t.name){case"Chrome":n=`Mozilla/5.0 (${s}) ${t.engine} (KHTML, like Gecko) Chrome/${t.version} Safari/537.36`;break;case"Firefox":n=`Mozilla/5.0 (${s}; rv:${t.version}) ${t.engine} Firefox/${t.version}`;break;case"Safari":n=`Mozilla/5.0 (${s}) ${t.engine} (KHTML, like Gecko) Version/${t.version} Safari/605.1.15`;break;case"Edge":n=`Mozilla/5.0 (${s}) ${t.engine} (KHTML, like Gecko) Chrome/${t.version} Safari/537.36 Edg/${t.version}`;break;case"Opera":n=`Mozilla/5.0 (${s}) ${t.engine} (KHTML, like Gecko) Chrome/${t.version} Safari/537.36 OPR/${t.version}`;break;case"Vivaldi":n=`Mozilla/5.0 (${s}) ${t.engine} (KHTML, like Gecko) Chrome/${t.version} Safari/537.36 Vivaldi/${t.version}`;break;case"Brave":n=`Mozilla/5.0 (${s}) ${t.engine} (KHTML, like Gecko) Chrome/${t.version} Safari/537.36 Brave/${t.version}`;break;case"Chromium":n=`Mozilla/5.0 (${s}) ${t.engine} (KHTML, like Gecko) Chromium/${t.version} Chrome/${t.version} Safari/537.36`;break;case"Yandex":n=`Mozilla/5.0 (${s}) ${t.engine} (KHTML, like Gecko) Chrome/${t.version} YaBrowser/${t.version} Safari/537.36`;break;case"Maxthon":n=`Mozilla/5.0 (${s}) ${t.engine} (KHTML, like Gecko) Chrome/${t.version} Safari/537.36 Maxthon/${t.version}`;break}i.push(n)}return i}g.CrawlerOptions=b;

package/dist/plugin/crawler-options.js

@@ -1 +0,0 @@
-import{RezoQueue as f}from"../queue/queue.js";import{Oxylabs as m}from"./addon/oxylabs/index.js";import b from"node:path";import g from"node:os";import{Decodo as x}from"./addon/decodo/index.js";class y{baseUrl;adapter;enableNavigationHistory;sessionId;rejectUnauthorized;userAgent;useRndUserAgent;timeout;maxRedirects;maxRetryAttempts;retryDelay;retryOnStatusCode;forceRevisit;retryWithoutProxyOnStatusCode;retryOnProxyError;maxRetryOnProxyError;allowRevisiting;enableCache;cacheTTL;cacheDir;throwFatalError;debug;oxylabs=[];decodo=[];proxies=[];limiters=[];requestHeaders=[];userAgents=A();constructor(e={}){this.baseUrl=e.baseUrl||"",this.adapter=e.adapter??"http",this.enableNavigationHistory=e.enableNavigationHistory??!1,this.sessionId=e.sessionId??`session_${Date.now()}_${Math.random().toString(36).slice(2,8)}`,this.rejectUnauthorized=e.rejectUnauthorized??!0,this.userAgent=e.userAgent,this.useRndUserAgent=e.useRndUserAgent??!1,this.timeout=e.timeout??30000,this.maxRedirects=e.maxRedirects??10,this.maxRetryAttempts=e.maxRetryAttempts??3,this.retryDelay=e.retryDelay??0,this.retryOnStatusCode=e.retryOnStatusCode??[408,429,500,502,503,504],this.forceRevisit=e.forceRevisit??!1,this.retryWithoutProxyOnStatusCode=e.retryWithoutProxyOnStatusCode??[407,403],this.retryOnProxyError=e.retryOnProxyError??!0,this.maxRetryOnProxyError=e.maxRetryOnProxyError??3,this.allowRevisiting=e.allowRevisiting??!1,this.enableCache=e.enableCache??!0,this.cacheTTL=e.cacheTTL??604800000,this.cacheDir=e.cacheDir??b.join(g.tmpdir(),"uiniqhtt_cache"),this.throwFatalError=e.throwFatalError??!1,this.debug=e.debug??!1,this._addHeaders(e.headers),this._addOxylabs(e.oxylabs),this._addProxies(e.proxy),this._addLimiters(e.limiter)}getConfiguredDomains(e){return(e==="headers"?this.requestHeaders:e==="limiters"?this.limiters:e==="oxylabs"?this.oxylabs:this.proxies).filter((i)=>i.domain).map((i)=>i.domain).filter((i,o,t)=>t.indexOf(i)===o)}removeDomain(e){return this.requestHeaders=this.requestHeaders.filter((r)=>!r.domain||!this._domainsEqual(r.domain,e)),this.proxies=this.proxies.filter((r)=>!r.domain||!this._domainsEqual(r.domain,e)),this.limiters=this.limiters.filter((r)=>!r.domain||!this._domainsEqual(r.domain,e)),this.oxylabs=this.oxylabs.filter((r)=>!r.domain||!this._domainsEqual(r.domain,e)),this}_domainsEqual(e,r){if(Array.isArray(e)&&Array.isArray(r))return e.length===r.length&&e.every((i,o)=>i===r[o]);return e===r}getConfigurationSummary(){let e=(r)=>({total:r.length,global:r.filter((i)=>i.isGlobal).length,domainSpecific:r.filter((i)=>!i.isGlobal&&i.domain).length});return{headers:e(this.requestHeaders),proxies:e(this.proxies),limiters:e(this.limiters),oxylabs:e(this.oxylabs)}}_addHeaders(e){if(!e||!e.enable)return;for(let r of e.httpHeaders){let{domain:i,isGlobal:o,headers:t}=r;if(!i&&!o)continue;if(r instanceof Headers&&Object.keys(Object.fromEntries(r.entries())).length<1)continue;else if(Object.keys(t).length<1)continue;t=r instanceof Headers?Object.fromEntries(r.entries()):t,this.requestHeaders.push({domain:i,isGlobal:o,headers:t})}}_addProxies(e){if(!e||!e.enable)return;for(let r of e.proxies){let{domain:i,isGlobal:o,proxy:t}=r;if(!i&&!o)continue;if(!t||Object.keys(t).length<1)continue;this.proxies.push({domain:i,isGlobal:o,proxy:t})}}_addLimiters(e){if(!e||!e.enable)return;for(let r of e.limiters){let{domain:i,isGlobal:o,options:t}=r;if(!i&&!o)continue;if(!t||Object.keys(t).length<1)continue;this.limiters.push({domain:i,isGlobal:o,pqueue:new f(t)})}}_addOxylabs(e){if(!e||!e.enable)return;for(let r of e.labs){let{domain:i,isGlobal:o,options:t,queueOptions:s}=r;if(!i&&!o)continue;if(!t||Object.keys(t).length<1)continue;this.oxylabs.push({domain:i,isGlobal:o,adaptar:new m(t)})}}_addDecodo(e){if(!e||!e.enable)return;for(let r of e.labs){let{domain:i,isGlobal:o,options:t,queueOptions:s}=r;if(!i&&!o)continue;if(!t||Object.keys(t).length<1)continue;this.decodo.push({domain:i,isGlobal:o,adaptar:new x(t)})}}addHeaders(e){return this._addHeaders({enable:!0,httpHeaders:[e]}),this}addProxy(e){return this._addProxies({enable:!0,proxies:[e]}),this}addLimiter(e){return this._addLimiters({enable:!0,limiters:[e]}),this}addOxylabs(e){return this._addOxylabs({enable:!0,labs:[e]}),this}addDecodo(e){return this._addDecodo({enable:!0,labs:[e]}),this}clearGlobalConfigs(){if(Array.isArray(this.requestHeaders))this.requestHeaders=this.requestHeaders.filter((e)=>!e.isGlobal);if(Array.isArray(this.oxylabs))this.oxylabs=this.oxylabs.filter((e)=>!e.isGlobal);if(Array.isArray(this.limiters))this.limiters=this.limiters.filter((e)=>!e.isGlobal);if(Array.isArray(this.proxies))this.proxies=this.proxies.filter((e)=>!e.isGlobal);return this}getAdapter(e,r,i,o){if(!this.getDomainName(e))return null;let s=[],a=r==="headers"?this.requestHeaders:r==="limiters"?this.limiters:r==="oxylabs"?this.oxylabs:r==="decodo"?this.decodo:this.proxies;for(let n=0;n<a.length;n++)if(this._hasDomain(e,a[n].domain))s.push(n);if(s.length){let n=o?s[this.rnd(0,s.length-1)]:s[0];return r==="headers"?this.requestHeaders[n].headers:r==="limiters"?this.limiters[n].pqueue:r==="oxylabs"?this.oxylabs[n].adaptar:r==="decodo"?this.decodo[n].adaptar:this.proxies[n].proxy}s.length=0;for(let n=0;n<a.length;n++)s.push(n);if(s.length){let n=o?s[this.rnd(0,s.length-1)]:s[0];if(a[n].isGlobal&&i)return r==="headers"?this.requestHeaders[n].headers:r==="limiters"?this.limiters[n].pqueue:r==="oxylabs"?this.oxylabs[n].adaptar:r==="decodo"?this.decodo[n].adaptar:this.proxies[n].proxy}return null}rnd(e=0,r=Number.MAX_VALUE){return Math.floor(Math.random()*(r-e+1))+e}hasDomain(e,r,i){if(!this.getDomainName(e))return!1;let t=r==="headers"?this.requestHeaders:r==="limiters"?this.limiters:r==="oxylabs"?this.oxylabs:r==="decodo"?this.decodo:this.proxies;for(let s=0;s<t.length;s++)if(this._hasDomain(e,t[s].domain))return!0;if(i){for(let s=0;s<t.length;s++)if(t[s].isGlobal)return!0}return!1}pickHeaders(e,r,i,o){let t=this.getAdapter(e,"headers",r),s=new Headers(t??{});if(i&&i instanceof Headers)for(let[a,n]of Object.entries(i.entries()))s.set(a,n);else if(i&&typeof i==="object"){for(let[a,n]of Object.entries(i))if(typeof n==="string")s.set(a,n)}if(o)s.set("user-agent",this.getRandomUserAgent());return Object.fromEntries(s.entries())}_hasDomain(e,r){if(!r)return!1;let i=this.getDomainName(e);if(!i)return!1;let o=(s)=>{return/[\^\$\*\+\?\{\}\[\]\(\)\|\\]/.test(s)||s.startsWith("/")||s.includes(".*")||s.includes(".+")},t=(s)=>{if(s instanceof RegExp)return s.test(i)||s.test(e);let a=s.toString().trim();if(i.toLowerCase()===a.toLowerCase())return!0;if(a.includes("*")){let l=a.replace(/[.*+?^${}()|[\]\\]/g,"\\$&").replace(/\\\*/g,".*"),h=new RegExp(`^${l}$`,"i");return h.test(i)||h.test(e)}if(o(a))try{let l=a,h="i",c=a.match(/^\/(.*)\/(\w*)$/);if(c)l=c[1],h=c[2]||"i";let u=new RegExp(l,h);return u.test(i)||u.test(e)}catch(l){return i.toLowerCase().includes(a.toLowerCase())}let n=i.toLowerCase(),d=a.toLowerCase();return n===d||n.endsWith("."+d)||d.endsWith("."+n)};if(Array.isArray(r)){for(let s of r)if(t(s))return!0;return!1}return t(r)}getDomainName(e){if(this.isValidUrl(e))return new URL(e).hostname;else if(this.isHostName(e))return e;return null}isHostName(e){if(!e)return!1;if(e.length>255)return!1;let r=/^(?:(?:[a-zA-Z0-9](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?\.)+ [a-zA-Z]{2,})$/;return e=e.trim().toLowerCase(),r.test(e)&&!e.startsWith("-")&&!e.endsWith("-")}isValidUrl(e){if(!e)return!1;e=e.trim();try{let r=new URL(e);if(!r.protocol||!["http:","https:"].includes(r.protocol.toLowerCase()))return!1;if(!r.hostname)return!1;if(!/^(?:(?:[a-zA-Z0-9](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?\.)+[a-zA-Z]{2,})$/.test(r.hostname))return!1;return!0}catch{return!1}}getRandomUserAgent(){return this.userAgents[Math.floor(Math.random()*this.userAgents.length)]}}function A(){let e=[{name:"Chrome",version:"91.0.4472.124",engine:"AppleWebKit/537.36"},{name:"Firefox",version:"89.0",engine:"Gecko/20100101"},{name:"Safari",version:"14.1.1",engine:"AppleWebKit/605.1.15"},{name:"Edge",version:"91.0.864.59",engine:"AppleWebKit/537.36"},{name:"Opera",version:"77.0.4054.277",engine:"AppleWebKit/537.36"},{name:"Vivaldi",version:"3.8.2259.42",engine:"AppleWebKit/537.36"},{name:"Brave",version:"1.26.74",engine:"AppleWebKit/537.36"},{name:"Chromium",version:"91.0.4472.101",engine:"AppleWebKit/537.36"},{name:"Yandex",version:"21.5.3.742",engine:"AppleWebKit/537.36"},{name:"Maxthon",version:"5.3.8.2000",engine:"AppleWebKit/537.36"}],r=["Windows NT 10.0","Windows NT 6.1","Macintosh; Intel Mac OS X 10_15_7","Macintosh; Intel Mac OS X 11_4_0","X11; Linux x86_64","X11; Ubuntu; Linux x86_64"],i=[];for(let o=0;o<200;o++){let t=e[Math.floor(Math.random()*e.length)],s=r[Math.floor(Math.random()*r.length)],a="";switch(t.name){case"Chrome":a=`Mozilla/5.0 (${s}) ${t.engine} (KHTML, like Gecko) Chrome/${t.version} Safari/537.36`;break;case"Firefox":a=`Mozilla/5.0 (${s}; rv:${t.version}) ${t.engine} Firefox/${t.version}`;break;case"Safari":a=`Mozilla/5.0 (${s}) ${t.engine} (KHTML, like Gecko) Version/${t.version} Safari/605.1.15`;break;case"Edge":a=`Mozilla/5.0 (${s}) ${t.engine} (KHTML, like Gecko) Chrome/${t.version} Safari/537.36 Edg/${t.version}`;break;case"Opera":a=`Mozilla/5.0 (${s}) ${t.engine} (KHTML, like Gecko) Chrome/${t.version} Safari/537.36 OPR/${t.version}`;break;case"Vivaldi":a=`Mozilla/5.0 (${s}) ${t.engine} (KHTML, like Gecko) Chrome/${t.version} Safari/537.36 Vivaldi/${t.version}`;break;case"Brave":a=`Mozilla/5.0 (${s}) ${t.engine} (KHTML, like Gecko) Chrome/${t.version} Safari/537.36 Brave/${t.version}`;break;case"Chromium":a=`Mozilla/5.0 (${s}) ${t.engine} (KHTML, like Gecko) Chromium/${t.version} Chrome/${t.version} Safari/537.36`;break;case"Yandex":a=`Mozilla/5.0 (${s}) ${t.engine} (KHTML, like Gecko) Chrome/${t.version} YaBrowser/${t.version} Safari/537.36`;break;case"Maxthon":a=`Mozilla/5.0 (${s}) ${t.engine} (KHTML, like Gecko) Chrome/${t.version} Safari/537.36 Maxthon/${t.version}`;break}i.push(a)}return i}export{y as CrawlerOptions};

package/dist/plugin/index.cjs

@@ -1,36 +0,0 @@
-const _mod_pwsonp = require('./crawler.cjs');
-exports.Crawler = _mod_pwsonp.Crawler;;
-const _mod_f0a514 = require('./crawler-options.cjs');
-exports.CrawlerOptions = _mod_f0a514.CrawlerOptions;;
-const _mod_0j6c45 = require('../cache/file-cacher.cjs');
-exports.FileCacher = _mod_0j6c45.FileCacher;;
-const _mod_0ys1f7 = require('../cache/url-store.cjs');
-exports.UrlStore = _mod_0ys1f7.UrlStore;;
-const _mod_bs9mae = require('./addon/oxylabs/index.cjs');
-exports.Oxylabs = _mod_bs9mae.Oxylabs;;
-const _mod_afta57 = require('./addon/oxylabs/options.cjs');
-exports.OXYLABS_BROWSER_TYPES = _mod_afta57.OXYLABS_BROWSER_TYPES;
-exports.OXYLABS_COMMON_LOCALES = _mod_afta57.OXYLABS_COMMON_LOCALES;
-exports.OXYLABS_COMMON_GEO_LOCATIONS = _mod_afta57.OXYLABS_COMMON_GEO_LOCATIONS;
-exports.OXYLABS_US_STATES = _mod_afta57.OXYLABS_US_STATES;
-exports.OXYLABS_EUROPEAN_COUNTRIES = _mod_afta57.OXYLABS_EUROPEAN_COUNTRIES;
-exports.OXYLABS_ASIAN_COUNTRIES = _mod_afta57.OXYLABS_ASIAN_COUNTRIES;
-exports.getRandomOxylabsBrowserType = _mod_afta57.getRandomBrowserType;
-exports.getRandomOxylabsLocale = _mod_afta57.getRandomLocale;
-exports.getRandomOxylabsGeoLocation = _mod_afta57.getRandomGeoLocation;;
-const _mod_8woeb1 = require('./addon/decodo/index.cjs');
-exports.Decodo = _mod_8woeb1.Decodo;;
-const _mod_rpxa81 = require('./addon/decodo/options.cjs');
-exports.DECODO_DEVICE_TYPES = _mod_rpxa81.DECODO_DEVICE_TYPES;
-exports.DECODO_HEADLESS_MODES = _mod_rpxa81.DECODO_HEADLESS_MODES;
-exports.DECODO_COMMON_LOCALES = _mod_rpxa81.DECODO_COMMON_LOCALES;
-exports.DECODO_COMMON_COUNTRIES = _mod_rpxa81.DECODO_COMMON_COUNTRIES;
-exports.DECODO_EUROPEAN_COUNTRIES = _mod_rpxa81.DECODO_EUROPEAN_COUNTRIES;
-exports.DECODO_ASIAN_COUNTRIES = _mod_rpxa81.DECODO_ASIAN_COUNTRIES;
-exports.DECODO_US_STATES = _mod_rpxa81.DECODO_US_STATES;
-exports.DECODO_COMMON_CITIES = _mod_rpxa81.DECODO_COMMON_CITIES;
-exports.getRandomDecodoDeviceType = _mod_rpxa81.getRandomDeviceType;
-exports.getRandomDecodoLocale = _mod_rpxa81.getRandomLocale;
-exports.getRandomDecodoCountry = _mod_rpxa81.getRandomCountry;
-exports.getRandomDecodoCity = _mod_rpxa81.getRandomCity;
-exports.generateDecodoSessionId = _mod_rpxa81.generateSessionId;;