rezo 1.0.49 → 1.0.51

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -776,7 +776,7 @@ async function request(config, fetchOptions, requestCount, timing, _stats, respo
776
776
  if (isRedirected)
777
777
  _stats.statusOnNext = "redirect";
778
778
  if (isRedirected && location) {
779
- const redirectUrlObj = new URL(location, url);
779
+ const redirectUrlObj = new URL(typeof location === "string" ? location : location.toString(), url);
780
780
  if (!redirectUrlObj.hash && url.hash) {
781
781
  redirectUrlObj.hash = url.hash;
782
782
  }
@@ -776,7 +776,7 @@ async function request(config, fetchOptions, requestCount, timing, _stats, respo
776
776
  if (isRedirected)
777
777
  _stats.statusOnNext = "redirect";
778
778
  if (isRedirected && location) {
779
- const redirectUrlObj = new URL(location, url);
779
+ const redirectUrlObj = new URL(typeof location === "string" ? location : location.toString(), url);
780
780
  if (!redirectUrlObj.hash && url.hash) {
781
781
  redirectUrlObj.hash = url.hash;
782
782
  }
@@ -1253,12 +1253,15 @@ async function executeHttp2Stream(config, fetchOptions, requestCount, timing, _s
1253
1253
  resolve(error);
1254
1254
  }
1255
1255
  };
1256
- session.on("error", sessionErrorHandler);
1257
- session.on("goaway", (errorCode, lastStreamID) => {
1256
+ session.once("error", sessionErrorHandler);
1257
+ session.once("goaway", (errorCode, lastStreamID) => {
1258
1258
  if (config.debug) {
1259
1259
  console.log(`[Rezo Debug] HTTP/2: Session GOAWAY received (errorCode: ${errorCode}, lastStreamID: ${lastStreamID})`);
1260
1260
  }
1261
1261
  });
1262
+ const cleanupSessionListeners = () => {
1263
+ session.removeListener("error", sessionErrorHandler);
1264
+ };
1262
1265
  if (config.debug) {
1263
1266
  console.log(`[Rezo Debug] HTTP/2: Creating request stream...`);
1264
1267
  }
@@ -1282,6 +1285,7 @@ async function executeHttp2Stream(config, fetchOptions, requestCount, timing, _s
1282
1285
  timeoutId = setTimeout(() => {
1283
1286
  if (!resolved) {
1284
1287
  resolved = true;
1288
+ cleanupSessionListeners();
1285
1289
  if (config.debug) {
1286
1290
  console.log(`[Rezo Debug] HTTP/2: Request timeout after ${requestTimeout}ms (no response received)`);
1287
1291
  }
@@ -1302,6 +1306,7 @@ async function executeHttp2Stream(config, fetchOptions, requestCount, timing, _s
1302
1306
  if (!resolved && status === 0) {
1303
1307
  resolved = true;
1304
1308
  clearTimeout(timeoutId);
1309
+ cleanupSessionListeners();
1305
1310
  if (config.debug) {
1306
1311
  console.log(`[Rezo Debug] HTTP/2: Stream closed without response - retrying with new session`);
1307
1312
  }
@@ -1317,6 +1322,7 @@ async function executeHttp2Stream(config, fetchOptions, requestCount, timing, _s
1317
1322
  if (!resolved) {
1318
1323
  resolved = true;
1319
1324
  clearTimeout(timeoutId);
1325
+ cleanupSessionListeners();
1320
1326
  const error = buildSmartError(config, fetchOptions, new Error("HTTP/2 stream aborted"));
1321
1327
  _stats.statusOnNext = "error";
1322
1328
  resolve(error);
@@ -1329,6 +1335,7 @@ async function executeHttp2Stream(config, fetchOptions, requestCount, timing, _s
1329
1335
  if (!resolved) {
1330
1336
  resolved = true;
1331
1337
  clearTimeout(timeoutId);
1338
+ cleanupSessionListeners();
1332
1339
  const error = buildSmartError(config, fetchOptions, err);
1333
1340
  _stats.statusOnNext = "error";
1334
1341
  resolve(error);
@@ -1427,6 +1434,7 @@ async function executeHttp2Stream(config, fetchOptions, requestCount, timing, _s
1427
1434
  }
1428
1435
  resolved = true;
1429
1436
  clearTimeout(timeoutId);
1437
+ cleanupSessionListeners();
1430
1438
  try {
1431
1439
  updateTiming(config, timing, contentLengthCounter);
1432
1440
  if (!config.transfer) {
@@ -1253,12 +1253,15 @@ async function executeHttp2Stream(config, fetchOptions, requestCount, timing, _s
1253
1253
  resolve(error);
1254
1254
  }
1255
1255
  };
1256
- session.on("error", sessionErrorHandler);
1257
- session.on("goaway", (errorCode, lastStreamID) => {
1256
+ session.once("error", sessionErrorHandler);
1257
+ session.once("goaway", (errorCode, lastStreamID) => {
1258
1258
  if (config.debug) {
1259
1259
  console.log(`[Rezo Debug] HTTP/2: Session GOAWAY received (errorCode: ${errorCode}, lastStreamID: ${lastStreamID})`);
1260
1260
  }
1261
1261
  });
1262
+ const cleanupSessionListeners = () => {
1263
+ session.removeListener("error", sessionErrorHandler);
1264
+ };
1262
1265
  if (config.debug) {
1263
1266
  console.log(`[Rezo Debug] HTTP/2: Creating request stream...`);
1264
1267
  }
@@ -1282,6 +1285,7 @@ async function executeHttp2Stream(config, fetchOptions, requestCount, timing, _s
1282
1285
  timeoutId = setTimeout(() => {
1283
1286
  if (!resolved) {
1284
1287
  resolved = true;
1288
+ cleanupSessionListeners();
1285
1289
  if (config.debug) {
1286
1290
  console.log(`[Rezo Debug] HTTP/2: Request timeout after ${requestTimeout}ms (no response received)`);
1287
1291
  }
@@ -1302,6 +1306,7 @@ async function executeHttp2Stream(config, fetchOptions, requestCount, timing, _s
1302
1306
  if (!resolved && status === 0) {
1303
1307
  resolved = true;
1304
1308
  clearTimeout(timeoutId);
1309
+ cleanupSessionListeners();
1305
1310
  if (config.debug) {
1306
1311
  console.log(`[Rezo Debug] HTTP/2: Stream closed without response - retrying with new session`);
1307
1312
  }
@@ -1317,6 +1322,7 @@ async function executeHttp2Stream(config, fetchOptions, requestCount, timing, _s
1317
1322
  if (!resolved) {
1318
1323
  resolved = true;
1319
1324
  clearTimeout(timeoutId);
1325
+ cleanupSessionListeners();
1320
1326
  const error = buildSmartError(config, fetchOptions, new Error("HTTP/2 stream aborted"));
1321
1327
  _stats.statusOnNext = "error";
1322
1328
  resolve(error);
@@ -1329,6 +1335,7 @@ async function executeHttp2Stream(config, fetchOptions, requestCount, timing, _s
1329
1335
  if (!resolved) {
1330
1336
  resolved = true;
1331
1337
  clearTimeout(timeoutId);
1338
+ cleanupSessionListeners();
1332
1339
  const error = buildSmartError(config, fetchOptions, err);
1333
1340
  _stats.statusOnNext = "error";
1334
1341
  resolve(error);
@@ -1427,6 +1434,7 @@ async function executeHttp2Stream(config, fetchOptions, requestCount, timing, _s
1427
1434
  }
1428
1435
  resolved = true;
1429
1436
  clearTimeout(timeoutId);
1437
+ cleanupSessionListeners();
1430
1438
  try {
1431
1439
  updateTiming(config, timing, contentLengthCounter);
1432
1440
  if (!config.transfer) {
@@ -1,6 +1,6 @@
1
- const _mod_s1uryh = require('./picker.cjs');
2
- exports.detectRuntime = _mod_s1uryh.detectRuntime;
3
- exports.getAdapterCapabilities = _mod_s1uryh.getAdapterCapabilities;
4
- exports.buildAdapterContext = _mod_s1uryh.buildAdapterContext;
5
- exports.getAvailableAdapters = _mod_s1uryh.getAvailableAdapters;
6
- exports.selectAdapter = _mod_s1uryh.selectAdapter;;
1
+ const _mod_an14fc = require('./picker.cjs');
2
+ exports.detectRuntime = _mod_an14fc.detectRuntime;
3
+ exports.getAdapterCapabilities = _mod_an14fc.getAdapterCapabilities;
4
+ exports.buildAdapterContext = _mod_an14fc.buildAdapterContext;
5
+ exports.getAvailableAdapters = _mod_an14fc.getAvailableAdapters;
6
+ exports.selectAdapter = _mod_an14fc.selectAdapter;;
@@ -1,9 +1,9 @@
1
- const _mod_ydlfsa = require('./lru-cache.cjs');
2
- exports.LRUCache = _mod_ydlfsa.LRUCache;;
3
- const _mod_16n4h5 = require('./dns-cache.cjs');
4
- exports.DNSCache = _mod_16n4h5.DNSCache;
5
- exports.getGlobalDNSCache = _mod_16n4h5.getGlobalDNSCache;
6
- exports.resetGlobalDNSCache = _mod_16n4h5.resetGlobalDNSCache;;
7
- const _mod_v39s7i = require('./response-cache.cjs');
8
- exports.ResponseCache = _mod_v39s7i.ResponseCache;
9
- exports.normalizeResponseCacheConfig = _mod_v39s7i.normalizeResponseCacheConfig;;
1
+ const _mod_c06h7j = require('./lru-cache.cjs');
2
+ exports.LRUCache = _mod_c06h7j.LRUCache;;
3
+ const _mod_7lzpm2 = require('./dns-cache.cjs');
4
+ exports.DNSCache = _mod_7lzpm2.DNSCache;
5
+ exports.getGlobalDNSCache = _mod_7lzpm2.getGlobalDNSCache;
6
+ exports.resetGlobalDNSCache = _mod_7lzpm2.resetGlobalDNSCache;;
7
+ const _mod_2v893e = require('./response-cache.cjs');
8
+ exports.ResponseCache = _mod_2v893e.ResponseCache;
9
+ exports.normalizeResponseCacheConfig = _mod_2v893e.normalizeResponseCacheConfig;;
@@ -5,11 +5,12 @@ const { NavigationHistory } = require('./plugin/navigation-history.cjs');
5
5
  const { RobotsTxt } = require('./plugin/robots-txt.cjs');
6
6
  const { parseHTML } = require("linkedom");
7
7
  const path = require("node:path");
8
- const { Rezo } = require('../core/rezo.cjs');
8
+ const { Rezo } = require('../adapters/entries/http.cjs');
9
9
  const { RezoQueue } = require('../queue/queue.cjs');
10
10
  const { Scraper } = require('./scraper.cjs');
11
11
  const { CrawlerOptions } = require('./crawler-options.cjs');
12
12
  const { loadAdapter } = require('../adapters/picker.cjs');
13
+ const { resetGlobalAgentPool } = require('../utils/agent-pool.cjs');
13
14
  String.prototype.addBaseUrl = function(url) {
14
15
  url = url instanceof URL ? url.href : url;
15
16
  const html = this.replace(/<base\b[^>]*?>/gi, "");
@@ -94,6 +95,10 @@ class Crawler {
94
95
  }).then((storage) => {
95
96
  this.cacher = storage;
96
97
  this.isCacheReady = true;
98
+ }).catch((err) => {
99
+ if (this.config.debug)
100
+ console.warn("[Crawler] Failed to initialize cache:", err);
101
+ this.isCacheReady = true;
97
102
  });
98
103
  const dit = path.resolve(cacheDir, "urls");
99
104
  if (!fs.existsSync(dit))
@@ -105,6 +110,10 @@ class Crawler {
105
110
  }).then((storage) => {
106
111
  this.urlStorage = storage;
107
112
  this.isStorageReady = true;
113
+ }).catch((err) => {
114
+ if (this.config.debug)
115
+ console.warn("[Crawler] Failed to initialize URL storage:", err);
116
+ this.isStorageReady = true;
108
117
  });
109
118
  } else {
110
119
  const dit = path.resolve(this.config.cacheDir, "./cache/urls");
@@ -117,6 +126,10 @@ class Crawler {
117
126
  }).then((storage) => {
118
127
  this.urlStorage = storage;
119
128
  this.isStorageReady = true;
129
+ }).catch((err) => {
130
+ if (this.config.debug)
131
+ console.warn("[Crawler] Failed to initialize URL storage:", err);
132
+ this.isStorageReady = true;
120
133
  });
121
134
  }
122
135
  if (this.config.enableNavigationHistory) {
@@ -856,12 +869,15 @@ class Crawler {
856
869
  }
857
870
  async execute(method, url, body, options = {}, isEmail, forceRevisit, oxylabsOptions, oxylabsInstanse, decodoInstanse, decodoOptions) {
858
871
  await this.waitForStorage();
872
+ console.log("Waiting for storage...");
859
873
  if (this.isCacheEnabled) {
860
874
  await this.waitForCache();
861
875
  }
876
+ console.log("Waiting for cache...");
862
877
  if (this.config.enableNavigationHistory) {
863
878
  await this.waitForNavigationHistory();
864
879
  }
880
+ console.log("Waiting for navigation history...");
865
881
  const task = this.queue.add(() => this.executeHttp(method, url, body, options, isEmail, forceRevisit, oxylabsOptions, oxylabsInstanse, decodoInstanse, decodoOptions));
866
882
  task.finally(() => this.pendingExecutions.delete(task));
867
883
  }
@@ -888,7 +904,9 @@ class Crawler {
888
904
  }
889
905
  async executeHttp(method, url, body, options = {}, isEmail, forceRevisit, oxylabsOptions, oxylabsInstanse, decodoInstanse, decodoOptions, retryCount = 0, parentUrl) {
890
906
  try {
907
+ console.log("Triggering start handlers...");
891
908
  await this.triggerStartHandlers();
909
+ console.log("Checking crawl limits...");
892
910
  const limitCheck = await this.checkCrawlLimits(url, parentUrl);
893
911
  if (!limitCheck.allowed) {
894
912
  if (this.config.debug) {
@@ -903,13 +921,16 @@ class Crawler {
903
921
  await new Promise((resolve) => setTimeout(resolve, delay));
904
922
  }
905
923
  const isVisited = forceRevisit ? false : await this.hasUrlInCache(url);
906
- const cache = await this.getCache(url);
907
- if (isVisited && !cache)
908
- return;
909
- if (isVisited && method !== "GET")
910
- return;
924
+ const cache = method.toLowerCase() === "get" ? await this.getCache(url) : undefined;
925
+ if (method.toLowerCase() === "get") {
926
+ if (isVisited && !cache)
927
+ return;
928
+ if (isVisited)
929
+ return;
930
+ }
911
931
  const requestStartTime = Date.now();
912
932
  const response = cache && method === "GET" ? cache : oxylabsInstanse && oxylabsOptions ? await oxylabsInstanse.scrape(url) : decodoInstanse && decodoOptions ? await decodoInstanse.scrape(url) : await (method === "GET" ? this.http.get(url, options) : method === "PATCH" ? this.http.patch(url, body, options) : method === "POST" ? this.http.post(url, body, options) : this.http.put(url, body, options));
933
+ console.log("Response received...");
913
934
  if (!cache) {
914
935
  const responseTime = Date.now() - requestStartTime;
915
936
  this.calculateAutoThrottleDelay(domain, responseTime);
@@ -943,9 +964,9 @@ class Crawler {
943
964
  statusCode: res.status
944
965
  });
945
966
  }
946
- if (!cache)
967
+ if (!cache && method === "GET")
947
968
  await this.saveCache(url, res);
948
- if (!isVisited)
969
+ if (!isVisited && method === "GET")
949
970
  await this.saveUrl(url);
950
971
  await this.markUrlVisited(url, {
951
972
  status: res.status,
@@ -1039,6 +1060,7 @@ class Crawler {
1039
1060
  }
1040
1061
  await this.queue.onIdle();
1041
1062
  await this.triggerFinishHandlers();
1063
+ await this.destroy();
1042
1064
  }
1043
1065
  async done() {
1044
1066
  return this.waitForAll();
@@ -1055,7 +1077,7 @@ class Crawler {
1055
1077
  } catch {}
1056
1078
  }
1057
1079
  async destroy() {
1058
- this.queue.clear();
1080
+ this.queue.destroy();
1059
1081
  this.events.length = 0;
1060
1082
  this.jsonEvents.length = 0;
1061
1083
  this.errorEvents.length = 0;
@@ -1064,6 +1086,7 @@ class Crawler {
1064
1086
  this.emailDiscoveredEvents.length = 0;
1065
1087
  this.emailLeadsEvents.length = 0;
1066
1088
  await this.close();
1089
+ resetGlobalAgentPool();
1067
1090
  }
1068
1091
  }
1069
1092
 
@@ -5,11 +5,12 @@ import { NavigationHistory } from './plugin/navigation-history.js';
5
5
  import { RobotsTxt } from './plugin/robots-txt.js';
6
6
  import { parseHTML } from "linkedom";
7
7
  import path from "node:path";
8
- import { Rezo } from '../core/rezo.js';
8
+ import { Rezo } from '../adapters/entries/http.js';
9
9
  import { RezoQueue } from '../queue/queue.js';
10
10
  import { Scraper } from './scraper.js';
11
11
  import { CrawlerOptions } from './crawler-options.js';
12
12
  import { loadAdapter } from '../adapters/picker.js';
13
+ import { resetGlobalAgentPool } from '../utils/agent-pool.js';
13
14
  String.prototype.addBaseUrl = function(url) {
14
15
  url = url instanceof URL ? url.href : url;
15
16
  const html = this.replace(/<base\b[^>]*?>/gi, "");
@@ -94,6 +95,10 @@ export class Crawler {
94
95
  }).then((storage) => {
95
96
  this.cacher = storage;
96
97
  this.isCacheReady = true;
98
+ }).catch((err) => {
99
+ if (this.config.debug)
100
+ console.warn("[Crawler] Failed to initialize cache:", err);
101
+ this.isCacheReady = true;
97
102
  });
98
103
  const dit = path.resolve(cacheDir, "urls");
99
104
  if (!fs.existsSync(dit))
@@ -105,6 +110,10 @@ export class Crawler {
105
110
  }).then((storage) => {
106
111
  this.urlStorage = storage;
107
112
  this.isStorageReady = true;
113
+ }).catch((err) => {
114
+ if (this.config.debug)
115
+ console.warn("[Crawler] Failed to initialize URL storage:", err);
116
+ this.isStorageReady = true;
108
117
  });
109
118
  } else {
110
119
  const dit = path.resolve(this.config.cacheDir, "./cache/urls");
@@ -117,6 +126,10 @@ export class Crawler {
117
126
  }).then((storage) => {
118
127
  this.urlStorage = storage;
119
128
  this.isStorageReady = true;
129
+ }).catch((err) => {
130
+ if (this.config.debug)
131
+ console.warn("[Crawler] Failed to initialize URL storage:", err);
132
+ this.isStorageReady = true;
120
133
  });
121
134
  }
122
135
  if (this.config.enableNavigationHistory) {
@@ -856,12 +869,15 @@ export class Crawler {
856
869
  }
857
870
  async execute(method, url, body, options = {}, isEmail, forceRevisit, oxylabsOptions, oxylabsInstanse, decodoInstanse, decodoOptions) {
858
871
  await this.waitForStorage();
872
+ console.log("Waiting for storage...");
859
873
  if (this.isCacheEnabled) {
860
874
  await this.waitForCache();
861
875
  }
876
+ console.log("Waiting for cache...");
862
877
  if (this.config.enableNavigationHistory) {
863
878
  await this.waitForNavigationHistory();
864
879
  }
880
+ console.log("Waiting for navigation history...");
865
881
  const task = this.queue.add(() => this.executeHttp(method, url, body, options, isEmail, forceRevisit, oxylabsOptions, oxylabsInstanse, decodoInstanse, decodoOptions));
866
882
  task.finally(() => this.pendingExecutions.delete(task));
867
883
  }
@@ -888,7 +904,9 @@ export class Crawler {
888
904
  }
889
905
  async executeHttp(method, url, body, options = {}, isEmail, forceRevisit, oxylabsOptions, oxylabsInstanse, decodoInstanse, decodoOptions, retryCount = 0, parentUrl) {
890
906
  try {
907
+ console.log("Triggering start handlers...");
891
908
  await this.triggerStartHandlers();
909
+ console.log("Checking crawl limits...");
892
910
  const limitCheck = await this.checkCrawlLimits(url, parentUrl);
893
911
  if (!limitCheck.allowed) {
894
912
  if (this.config.debug) {
@@ -903,13 +921,16 @@ export class Crawler {
903
921
  await new Promise((resolve) => setTimeout(resolve, delay));
904
922
  }
905
923
  const isVisited = forceRevisit ? false : await this.hasUrlInCache(url);
906
- const cache = await this.getCache(url);
907
- if (isVisited && !cache)
908
- return;
909
- if (isVisited && method !== "GET")
910
- return;
924
+ const cache = method.toLowerCase() === "get" ? await this.getCache(url) : undefined;
925
+ if (method.toLowerCase() === "get") {
926
+ if (isVisited && !cache)
927
+ return;
928
+ if (isVisited)
929
+ return;
930
+ }
911
931
  const requestStartTime = Date.now();
912
932
  const response = cache && method === "GET" ? cache : oxylabsInstanse && oxylabsOptions ? await oxylabsInstanse.scrape(url) : decodoInstanse && decodoOptions ? await decodoInstanse.scrape(url) : await (method === "GET" ? this.http.get(url, options) : method === "PATCH" ? this.http.patch(url, body, options) : method === "POST" ? this.http.post(url, body, options) : this.http.put(url, body, options));
933
+ console.log("Response received...");
913
934
  if (!cache) {
914
935
  const responseTime = Date.now() - requestStartTime;
915
936
  this.calculateAutoThrottleDelay(domain, responseTime);
@@ -943,9 +964,9 @@ export class Crawler {
943
964
  statusCode: res.status
944
965
  });
945
966
  }
946
- if (!cache)
967
+ if (!cache && method === "GET")
947
968
  await this.saveCache(url, res);
948
- if (!isVisited)
969
+ if (!isVisited && method === "GET")
949
970
  await this.saveUrl(url);
950
971
  await this.markUrlVisited(url, {
951
972
  status: res.status,
@@ -1039,6 +1060,7 @@ export class Crawler {
1039
1060
  }
1040
1061
  await this.queue.onIdle();
1041
1062
  await this.triggerFinishHandlers();
1063
+ await this.destroy();
1042
1064
  }
1043
1065
  async done() {
1044
1066
  return this.waitForAll();
@@ -1055,7 +1077,7 @@ export class Crawler {
1055
1077
  } catch {}
1056
1078
  }
1057
1079
  async destroy() {
1058
- this.queue.clear();
1080
+ this.queue.destroy();
1059
1081
  this.events.length = 0;
1060
1082
  this.jsonEvents.length = 0;
1061
1083
  this.errorEvents.length = 0;
@@ -1064,5 +1086,6 @@ export class Crawler {
1064
1086
  this.emailDiscoveredEvents.length = 0;
1065
1087
  this.emailLeadsEvents.length = 0;
1066
1088
  await this.close();
1089
+ resetGlobalAgentPool();
1067
1090
  }
1068
1091
  }
@@ -1,40 +1,40 @@
1
- const _mod_v2m6d4 = require('./crawler.cjs');
2
- exports.Crawler = _mod_v2m6d4.Crawler;;
3
- const _mod_p0s83p = require('./crawler-options.cjs');
4
- exports.CrawlerOptions = _mod_p0s83p.CrawlerOptions;;
5
- const _mod_2buxts = require('./plugin/robots-txt.cjs');
6
- exports.RobotsTxt = _mod_2buxts.RobotsTxt;;
7
- const _mod_my5gmu = require('./plugin/file-cacher.cjs');
8
- exports.FileCacher = _mod_my5gmu.FileCacher;;
9
- const _mod_x80ejx = require('./plugin/url-store.cjs');
10
- exports.UrlStore = _mod_x80ejx.UrlStore;;
11
- const _mod_fczuvz = require('./plugin/navigation-history.cjs');
12
- exports.NavigationHistory = _mod_fczuvz.NavigationHistory;;
13
- const _mod_drvz1f = require('./addon/oxylabs/index.cjs');
14
- exports.Oxylabs = _mod_drvz1f.Oxylabs;;
15
- const _mod_u57pv7 = require('./addon/oxylabs/options.cjs');
16
- exports.OXYLABS_BROWSER_TYPES = _mod_u57pv7.OXYLABS_BROWSER_TYPES;
17
- exports.OXYLABS_COMMON_LOCALES = _mod_u57pv7.OXYLABS_COMMON_LOCALES;
18
- exports.OXYLABS_COMMON_GEO_LOCATIONS = _mod_u57pv7.OXYLABS_COMMON_GEO_LOCATIONS;
19
- exports.OXYLABS_US_STATES = _mod_u57pv7.OXYLABS_US_STATES;
20
- exports.OXYLABS_EUROPEAN_COUNTRIES = _mod_u57pv7.OXYLABS_EUROPEAN_COUNTRIES;
21
- exports.OXYLABS_ASIAN_COUNTRIES = _mod_u57pv7.OXYLABS_ASIAN_COUNTRIES;
22
- exports.getRandomOxylabsBrowserType = _mod_u57pv7.getRandomBrowserType;
23
- exports.getRandomOxylabsLocale = _mod_u57pv7.getRandomLocale;
24
- exports.getRandomOxylabsGeoLocation = _mod_u57pv7.getRandomGeoLocation;;
25
- const _mod_rtdy24 = require('./addon/decodo/index.cjs');
26
- exports.Decodo = _mod_rtdy24.Decodo;;
27
- const _mod_098wxj = require('./addon/decodo/options.cjs');
28
- exports.DECODO_DEVICE_TYPES = _mod_098wxj.DECODO_DEVICE_TYPES;
29
- exports.DECODO_HEADLESS_MODES = _mod_098wxj.DECODO_HEADLESS_MODES;
30
- exports.DECODO_COMMON_LOCALES = _mod_098wxj.DECODO_COMMON_LOCALES;
31
- exports.DECODO_COMMON_COUNTRIES = _mod_098wxj.DECODO_COMMON_COUNTRIES;
32
- exports.DECODO_EUROPEAN_COUNTRIES = _mod_098wxj.DECODO_EUROPEAN_COUNTRIES;
33
- exports.DECODO_ASIAN_COUNTRIES = _mod_098wxj.DECODO_ASIAN_COUNTRIES;
34
- exports.DECODO_US_STATES = _mod_098wxj.DECODO_US_STATES;
35
- exports.DECODO_COMMON_CITIES = _mod_098wxj.DECODO_COMMON_CITIES;
36
- exports.getRandomDecodoDeviceType = _mod_098wxj.getRandomDeviceType;
37
- exports.getRandomDecodoLocale = _mod_098wxj.getRandomLocale;
38
- exports.getRandomDecodoCountry = _mod_098wxj.getRandomCountry;
39
- exports.getRandomDecodoCity = _mod_098wxj.getRandomCity;
40
- exports.generateDecodoSessionId = _mod_098wxj.generateSessionId;;
1
+ const _mod_oe2h32 = require('./crawler.cjs');
2
+ exports.Crawler = _mod_oe2h32.Crawler;;
3
+ const _mod_ask6qa = require('./crawler-options.cjs');
4
+ exports.CrawlerOptions = _mod_ask6qa.CrawlerOptions;;
5
+ const _mod_olcke0 = require('./plugin/robots-txt.cjs');
6
+ exports.RobotsTxt = _mod_olcke0.RobotsTxt;;
7
+ const _mod_4422cp = require('./plugin/file-cacher.cjs');
8
+ exports.FileCacher = _mod_4422cp.FileCacher;;
9
+ const _mod_vrdxs5 = require('./plugin/url-store.cjs');
10
+ exports.UrlStore = _mod_vrdxs5.UrlStore;;
11
+ const _mod_0n4r2a = require('./plugin/navigation-history.cjs');
12
+ exports.NavigationHistory = _mod_0n4r2a.NavigationHistory;;
13
+ const _mod_ttzg9i = require('./addon/oxylabs/index.cjs');
14
+ exports.Oxylabs = _mod_ttzg9i.Oxylabs;;
15
+ const _mod_qztowj = require('./addon/oxylabs/options.cjs');
16
+ exports.OXYLABS_BROWSER_TYPES = _mod_qztowj.OXYLABS_BROWSER_TYPES;
17
+ exports.OXYLABS_COMMON_LOCALES = _mod_qztowj.OXYLABS_COMMON_LOCALES;
18
+ exports.OXYLABS_COMMON_GEO_LOCATIONS = _mod_qztowj.OXYLABS_COMMON_GEO_LOCATIONS;
19
+ exports.OXYLABS_US_STATES = _mod_qztowj.OXYLABS_US_STATES;
20
+ exports.OXYLABS_EUROPEAN_COUNTRIES = _mod_qztowj.OXYLABS_EUROPEAN_COUNTRIES;
21
+ exports.OXYLABS_ASIAN_COUNTRIES = _mod_qztowj.OXYLABS_ASIAN_COUNTRIES;
22
+ exports.getRandomOxylabsBrowserType = _mod_qztowj.getRandomBrowserType;
23
+ exports.getRandomOxylabsLocale = _mod_qztowj.getRandomLocale;
24
+ exports.getRandomOxylabsGeoLocation = _mod_qztowj.getRandomGeoLocation;;
25
+ const _mod_vf4tt7 = require('./addon/decodo/index.cjs');
26
+ exports.Decodo = _mod_vf4tt7.Decodo;;
27
+ const _mod_5ztbfj = require('./addon/decodo/options.cjs');
28
+ exports.DECODO_DEVICE_TYPES = _mod_5ztbfj.DECODO_DEVICE_TYPES;
29
+ exports.DECODO_HEADLESS_MODES = _mod_5ztbfj.DECODO_HEADLESS_MODES;
30
+ exports.DECODO_COMMON_LOCALES = _mod_5ztbfj.DECODO_COMMON_LOCALES;
31
+ exports.DECODO_COMMON_COUNTRIES = _mod_5ztbfj.DECODO_COMMON_COUNTRIES;
32
+ exports.DECODO_EUROPEAN_COUNTRIES = _mod_5ztbfj.DECODO_EUROPEAN_COUNTRIES;
33
+ exports.DECODO_ASIAN_COUNTRIES = _mod_5ztbfj.DECODO_ASIAN_COUNTRIES;
34
+ exports.DECODO_US_STATES = _mod_5ztbfj.DECODO_US_STATES;
35
+ exports.DECODO_COMMON_CITIES = _mod_5ztbfj.DECODO_COMMON_CITIES;
36
+ exports.getRandomDecodoDeviceType = _mod_5ztbfj.getRandomDeviceType;
37
+ exports.getRandomDecodoLocale = _mod_5ztbfj.getRandomLocale;
38
+ exports.getRandomDecodoCountry = _mod_5ztbfj.getRandomCountry;
39
+ exports.getRandomDecodoCity = _mod_5ztbfj.getRandomCity;
40
+ exports.generateDecodoSessionId = _mod_5ztbfj.generateSessionId;;
@@ -1,5 +1,5 @@
1
- const _mod_12ia6t = require('../crawler/crawler.cjs');
2
- exports.Crawler = _mod_12ia6t.Crawler;;
3
- const _mod_ii9l9h = require('../crawler/crawler-options.cjs');
4
- exports.CrawlerOptions = _mod_ii9l9h.CrawlerOptions;
5
- exports.Domain = _mod_ii9l9h.Domain;;
1
+ const _mod_anwt38 = require('../crawler/crawler.cjs');
2
+ exports.Crawler = _mod_anwt38.Crawler;;
3
+ const _mod_uvsh93 = require('../crawler/crawler-options.cjs');
4
+ exports.CrawlerOptions = _mod_uvsh93.CrawlerOptions;
5
+ exports.Domain = _mod_uvsh93.Domain;;
package/dist/index.cjs CHANGED
@@ -1,30 +1,30 @@
1
- const _mod_zdke9f = require('./core/rezo.cjs');
2
- exports.Rezo = _mod_zdke9f.Rezo;
3
- exports.createRezoInstance = _mod_zdke9f.createRezoInstance;
4
- exports.createDefaultInstance = _mod_zdke9f.createDefaultInstance;;
5
- const _mod_22ekkf = require('./errors/rezo-error.cjs');
6
- exports.RezoError = _mod_22ekkf.RezoError;
7
- exports.RezoErrorCode = _mod_22ekkf.RezoErrorCode;;
8
- const _mod_kawotj = require('./utils/headers.cjs');
9
- exports.RezoHeaders = _mod_kawotj.RezoHeaders;;
10
- const _mod_7y5q1d = require('./utils/form-data.cjs');
11
- exports.RezoFormData = _mod_7y5q1d.RezoFormData;;
12
- const _mod_zpias0 = require('./utils/cookies.cjs');
13
- exports.RezoCookieJar = _mod_zpias0.RezoCookieJar;
14
- exports.Cookie = _mod_zpias0.Cookie;;
15
- const _mod_ihvyxg = require('./utils/curl.cjs');
16
- exports.toCurl = _mod_ihvyxg.toCurl;
17
- exports.fromCurl = _mod_ihvyxg.fromCurl;;
18
- const _mod_i6to8y = require('./core/hooks.cjs');
19
- exports.createDefaultHooks = _mod_i6to8y.createDefaultHooks;
20
- exports.mergeHooks = _mod_i6to8y.mergeHooks;;
21
- const _mod_35bvtb = require('./proxy/manager.cjs');
22
- exports.ProxyManager = _mod_35bvtb.ProxyManager;;
23
- const _mod_xyqrz8 = require('./queue/index.cjs');
24
- exports.RezoQueue = _mod_xyqrz8.RezoQueue;
25
- exports.HttpQueue = _mod_xyqrz8.HttpQueue;
26
- exports.Priority = _mod_xyqrz8.Priority;
27
- exports.HttpMethodPriority = _mod_xyqrz8.HttpMethodPriority;;
1
+ const _mod_piprwa = require('./core/rezo.cjs');
2
+ exports.Rezo = _mod_piprwa.Rezo;
3
+ exports.createRezoInstance = _mod_piprwa.createRezoInstance;
4
+ exports.createDefaultInstance = _mod_piprwa.createDefaultInstance;;
5
+ const _mod_ily76y = require('./errors/rezo-error.cjs');
6
+ exports.RezoError = _mod_ily76y.RezoError;
7
+ exports.RezoErrorCode = _mod_ily76y.RezoErrorCode;;
8
+ const _mod_q4s6h5 = require('./utils/headers.cjs');
9
+ exports.RezoHeaders = _mod_q4s6h5.RezoHeaders;;
10
+ const _mod_lhxcz7 = require('./utils/form-data.cjs');
11
+ exports.RezoFormData = _mod_lhxcz7.RezoFormData;;
12
+ const _mod_ae3n62 = require('./utils/cookies.cjs');
13
+ exports.RezoCookieJar = _mod_ae3n62.RezoCookieJar;
14
+ exports.Cookie = _mod_ae3n62.Cookie;;
15
+ const _mod_c0ipkb = require('./utils/curl.cjs');
16
+ exports.toCurl = _mod_c0ipkb.toCurl;
17
+ exports.fromCurl = _mod_c0ipkb.fromCurl;;
18
+ const _mod_b2lvov = require('./core/hooks.cjs');
19
+ exports.createDefaultHooks = _mod_b2lvov.createDefaultHooks;
20
+ exports.mergeHooks = _mod_b2lvov.mergeHooks;;
21
+ const _mod_7znktr = require('./proxy/manager.cjs');
22
+ exports.ProxyManager = _mod_7znktr.ProxyManager;;
23
+ const _mod_qpvzjb = require('./queue/index.cjs');
24
+ exports.RezoQueue = _mod_qpvzjb.RezoQueue;
25
+ exports.HttpQueue = _mod_qpvzjb.HttpQueue;
26
+ exports.Priority = _mod_qpvzjb.Priority;
27
+ exports.HttpMethodPriority = _mod_qpvzjb.HttpMethodPriority;;
28
28
  const { RezoError } = require('./errors/rezo-error.cjs');
29
29
  const isRezoError = exports.isRezoError = RezoError.isRezoError;
30
30
  const Cancel = exports.Cancel = RezoError;
@@ -1,10 +1,10 @@
1
- const _mod_pzxtgo = require('./base.cjs');
2
- exports.Agent = _mod_pzxtgo.Agent;;
3
- const _mod_v39yki = require('./http-proxy.cjs');
4
- exports.HttpProxyAgent = _mod_v39yki.HttpProxyAgent;;
5
- const _mod_1hwv9a = require('./https-proxy.cjs');
6
- exports.HttpsProxyAgent = _mod_1hwv9a.HttpsProxyAgent;;
7
- const _mod_qy9ium = require('./socks-proxy.cjs');
8
- exports.SocksProxyAgent = _mod_qy9ium.SocksProxyAgent;;
9
- const _mod_gr2yxg = require('./socks-client.cjs');
10
- exports.SocksClient = _mod_gr2yxg.SocksClient;;
1
+ const _mod_hw46n8 = require('./base.cjs');
2
+ exports.Agent = _mod_hw46n8.Agent;;
3
+ const _mod_an9r9f = require('./http-proxy.cjs');
4
+ exports.HttpProxyAgent = _mod_an9r9f.HttpProxyAgent;;
5
+ const _mod_6u3c2s = require('./https-proxy.cjs');
6
+ exports.HttpsProxyAgent = _mod_6u3c2s.HttpsProxyAgent;;
7
+ const _mod_06xizn = require('./socks-proxy.cjs');
8
+ exports.SocksProxyAgent = _mod_06xizn.SocksProxyAgent;;
9
+ const _mod_ap8s94 = require('./socks-client.cjs');
10
+ exports.SocksClient = _mod_ap8s94.SocksClient;;
@@ -1,9 +1,9 @@
1
1
  const { Agent, HttpProxyAgent, HttpsProxyAgent, SocksProxyAgent } = require('../internal/agents/index.cjs');
2
2
  const { parseProxyString } = require('./parse.cjs');
3
- const _mod_17bmdj = require('./manager.cjs');
4
- exports.ProxyManager = _mod_17bmdj.ProxyManager;;
5
- const _mod_pp8thq = require('./parse.cjs');
6
- exports.parseProxyString = _mod_pp8thq.parseProxyString;;
3
+ const _mod_pfdaeo = require('./manager.cjs');
4
+ exports.ProxyManager = _mod_pfdaeo.ProxyManager;;
5
+ const _mod_9wdrd3 = require('./parse.cjs');
6
+ exports.parseProxyString = _mod_9wdrd3.parseProxyString;;
7
7
  function createOptions(uri, opts) {
8
8
  if (uri instanceof URL || typeof uri === "string") {
9
9
  return {
@@ -1,8 +1,8 @@
1
- const _mod_uoy76r = require('./queue.cjs');
2
- exports.RezoQueue = _mod_uoy76r.RezoQueue;;
3
- const _mod_74hy11 = require('./http-queue.cjs');
4
- exports.HttpQueue = _mod_74hy11.HttpQueue;
5
- exports.extractDomain = _mod_74hy11.extractDomain;;
6
- const _mod_ovg3ej = require('./types.cjs');
7
- exports.Priority = _mod_ovg3ej.Priority;
8
- exports.HttpMethodPriority = _mod_ovg3ej.HttpMethodPriority;;
1
+ const _mod_9zllky = require('./queue.cjs');
2
+ exports.RezoQueue = _mod_9zllky.RezoQueue;;
3
+ const _mod_acgz0f = require('./http-queue.cjs');
4
+ exports.HttpQueue = _mod_acgz0f.HttpQueue;
5
+ exports.extractDomain = _mod_acgz0f.extractDomain;;
6
+ const _mod_a6xcwz = require('./types.cjs');
7
+ exports.Priority = _mod_a6xcwz.Priority;
8
+ exports.HttpMethodPriority = _mod_a6xcwz.HttpMethodPriority;;
@@ -1,11 +1,11 @@
1
- const _mod_xb6lhw = require('./event-emitter.cjs');
2
- exports.UniversalEventEmitter = _mod_xb6lhw.UniversalEventEmitter;;
3
- const _mod_anre5b = require('./stream.cjs');
4
- exports.UniversalStreamResponse = _mod_anre5b.UniversalStreamResponse;
5
- exports.StreamResponse = _mod_anre5b.StreamResponse;;
6
- const _mod_z3kroc = require('./download.cjs');
7
- exports.UniversalDownloadResponse = _mod_z3kroc.UniversalDownloadResponse;
8
- exports.DownloadResponse = _mod_z3kroc.DownloadResponse;;
9
- const _mod_x4d8ls = require('./upload.cjs');
10
- exports.UniversalUploadResponse = _mod_x4d8ls.UniversalUploadResponse;
11
- exports.UploadResponse = _mod_x4d8ls.UploadResponse;;
1
+ const _mod_ylkelc = require('./event-emitter.cjs');
2
+ exports.UniversalEventEmitter = _mod_ylkelc.UniversalEventEmitter;;
3
+ const _mod_ngrtq8 = require('./stream.cjs');
4
+ exports.UniversalStreamResponse = _mod_ngrtq8.UniversalStreamResponse;
5
+ exports.StreamResponse = _mod_ngrtq8.StreamResponse;;
6
+ const _mod_4er8ge = require('./download.cjs');
7
+ exports.UniversalDownloadResponse = _mod_4er8ge.UniversalDownloadResponse;
8
+ exports.DownloadResponse = _mod_4er8ge.DownloadResponse;;
9
+ const _mod_mxbnd3 = require('./upload.cjs');
10
+ exports.UniversalUploadResponse = _mod_mxbnd3.UniversalUploadResponse;
11
+ exports.UploadResponse = _mod_mxbnd3.UploadResponse;;
@@ -1,23 +1,4 @@
1
- const { createGunzip, createInflate, createBrotliDecompress } = require("node:zlib");
2
- const { Transform } = require("node:stream");
3
- let createZstdDecompress = null;
4
- let zstdAvailable = null;
5
- function getZstdDecompressor() {
6
- if (zstdAvailable === null) {
7
- try {
8
- const zlib = require("node:zlib");
9
- if (typeof zlib.createZstdDecompress === "function") {
10
- createZstdDecompress = zlib.createZstdDecompress;
11
- zstdAvailable = true;
12
- } else {
13
- zstdAvailable = false;
14
- }
15
- } catch {
16
- zstdAvailable = false;
17
- }
18
- }
19
- return createZstdDecompress;
20
- }
1
+ const { createGunzip, createInflate, createBrotliDecompress, createZstdDecompress } = require("node:zlib");
21
2
 
22
3
  class CompressionUtil {
23
4
  static decompressStream(response, contentEncoding, config) {
@@ -41,7 +22,7 @@ class CompressionUtil {
41
22
  case "brotli":
42
23
  return response.pipe(createBrotliDecompress());
43
24
  case "zstd":
44
- return this.createZstdDecompressStream(response);
25
+ return response.pipe(createZstdDecompress());
45
26
  default:
46
27
  return response;
47
28
  }
@@ -61,18 +42,6 @@ class CompressionUtil {
61
42
  }
62
43
  return true;
63
44
  }
64
- static createZstdDecompressStream(response) {
65
- const decompressor = getZstdDecompressor();
66
- if (!decompressor) {
67
- const passthrough = new Transform({
68
- transform(chunk, encoding, callback) {
69
- callback(new Error("zstd decompression not available: requires Node.js 22.15+ with native zstd support"));
70
- }
71
- });
72
- return response.pipe(passthrough);
73
- }
74
- return response.pipe(decompressor());
75
- }
76
45
  static getSupportedAlgorithms() {
77
46
  return ["gzip", "x-gzip", "deflate", "x-deflate", "gzip-raw", "br", "brotli", "zstd"];
78
47
  }
@@ -1,23 +1,4 @@
1
- import { createGunzip, createInflate, createBrotliDecompress } from "node:zlib";
2
- import { Transform } from "node:stream";
3
- let createZstdDecompress = null;
4
- let zstdAvailable = null;
5
- function getZstdDecompressor() {
6
- if (zstdAvailable === null) {
7
- try {
8
- const zlib = require("node:zlib");
9
- if (typeof zlib.createZstdDecompress === "function") {
10
- createZstdDecompress = zlib.createZstdDecompress;
11
- zstdAvailable = true;
12
- } else {
13
- zstdAvailable = false;
14
- }
15
- } catch {
16
- zstdAvailable = false;
17
- }
18
- }
19
- return createZstdDecompress;
20
- }
1
+ import { createGunzip, createInflate, createBrotliDecompress, createZstdDecompress } from "node:zlib";
21
2
 
22
3
  export class CompressionUtil {
23
4
  static decompressStream(response, contentEncoding, config) {
@@ -41,7 +22,7 @@ export class CompressionUtil {
41
22
  case "brotli":
42
23
  return response.pipe(createBrotliDecompress());
43
24
  case "zstd":
44
- return this.createZstdDecompressStream(response);
25
+ return response.pipe(createZstdDecompress());
45
26
  default:
46
27
  return response;
47
28
  }
@@ -61,18 +42,6 @@ export class CompressionUtil {
61
42
  }
62
43
  return true;
63
44
  }
64
- static createZstdDecompressStream(response) {
65
- const decompressor = getZstdDecompressor();
66
- if (!decompressor) {
67
- const passthrough = new Transform({
68
- transform(chunk, encoding, callback) {
69
- callback(new Error("zstd decompression not available: requires Node.js 22.15+ with native zstd support"));
70
- }
71
- });
72
- return response.pipe(passthrough);
73
- }
74
- return response.pipe(decompressor());
75
- }
76
45
  static getSupportedAlgorithms() {
77
46
  return ["gzip", "x-gzip", "deflate", "x-deflate", "gzip-raw", "br", "brotli", "zstd"];
78
47
  }
@@ -173,7 +173,7 @@ function setSignal() {
173
173
  clearTimeout(this.timeoutClearInstanse);
174
174
  if (this.timeout && typeof this.timeout === "number" && this.timeout > 100) {
175
175
  const controller = new AbortController;
176
- const timer = setTimeout(() => controller.abort(), this.timeout);
176
+ const timer = setTimeout(() => controller.abort(), this.timeout).unref();
177
177
  this.timeoutClearInstanse = timer;
178
178
  this.signal = controller.signal;
179
179
  }
@@ -173,7 +173,7 @@ function setSignal() {
173
173
  clearTimeout(this.timeoutClearInstanse);
174
174
  if (this.timeout && typeof this.timeout === "number" && this.timeout > 100) {
175
175
  const controller = new AbortController;
176
- const timer = setTimeout(() => controller.abort(), this.timeout);
176
+ const timer = setTimeout(() => controller.abort(), this.timeout).unref();
177
177
  this.timeoutClearInstanse = timer;
178
178
  this.signal = controller.signal;
179
179
  }
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "rezo",
3
- "version": "1.0.49",
3
+ "version": "1.0.51",
4
4
  "description": "Lightning-fast, enterprise-grade HTTP client for modern JavaScript. Full HTTP/2 support, intelligent cookie management, multiple adapters (HTTP, Fetch, cURL, XHR), streaming, proxy support (HTTP/HTTPS/SOCKS), and cross-environment compatibility.",
5
5
  "main": "dist/index.cjs",
6
6
  "module": "dist/index.js",