rezo 1.0.82 → 1.0.84

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (38) hide show
  1. package/dist/adapters/entries/curl.d.ts +1 -1
  2. package/dist/adapters/entries/fetch.d.ts +1 -1
  3. package/dist/adapters/entries/http.d.ts +1 -1
  4. package/dist/adapters/entries/http2.d.ts +1 -1
  5. package/dist/adapters/entries/react-native.d.ts +1 -1
  6. package/dist/adapters/entries/xhr.d.ts +1 -1
  7. package/dist/adapters/http.cjs +25 -19
  8. package/dist/adapters/http.js +25 -19
  9. package/dist/adapters/http2.cjs +25 -0
  10. package/dist/adapters/http2.js +25 -0
  11. package/dist/adapters/index.cjs +6 -6
  12. package/dist/cache/index.cjs +9 -9
  13. package/dist/crawler/index.cjs +42 -42
  14. package/dist/crawler.d.ts +6 -0
  15. package/dist/entries/crawler.cjs +6 -4
  16. package/dist/entries/crawler.js +1 -0
  17. package/dist/index.cjs +31 -31
  18. package/dist/index.d.ts +1 -1
  19. package/dist/internal/agents/bun-socks-http.cjs +3 -1
  20. package/dist/internal/agents/bun-socks-http.js +3 -1
  21. package/dist/internal/agents/index.cjs +14 -14
  22. package/dist/platform/browser.d.ts +1 -1
  23. package/dist/platform/bun.d.ts +1 -1
  24. package/dist/platform/deno.d.ts +1 -1
  25. package/dist/platform/node.d.ts +1 -1
  26. package/dist/platform/react-native.d.ts +1 -1
  27. package/dist/platform/worker.d.ts +1 -1
  28. package/dist/proxy/index.cjs +4 -4
  29. package/dist/proxy/parse.cjs +1 -1
  30. package/dist/proxy/parse.js +1 -1
  31. package/dist/queue/index.cjs +9 -9
  32. package/dist/responses/universal/index.cjs +11 -11
  33. package/dist/utils/compression.cjs +169 -14
  34. package/dist/utils/compression.js +169 -14
  35. package/dist/version.cjs +1 -1
  36. package/dist/version.js +1 -1
  37. package/dist/wget/index.cjs +49 -49
  38. package/package.json +1 -1
@@ -5039,7 +5039,7 @@ export interface RezoInstance extends Rezo, RezoCallable {
5039
5039
  *
5040
5040
  * IMPORTANT: Update these values when bumping package version.
5041
5041
  */
5042
- export declare const VERSION = "1.0.82";
5042
+ export declare const VERSION = "1.0.84";
5043
5043
  /**
5044
5044
  * cURL Options Configuration
5045
5045
  *
@@ -5039,7 +5039,7 @@ export interface RezoInstance extends Rezo, RezoCallable {
5039
5039
  *
5040
5040
  * IMPORTANT: Update these values when bumping package version.
5041
5041
  */
5042
- export declare const VERSION = "1.0.82";
5042
+ export declare const VERSION = "1.0.84";
5043
5043
  export declare const isRezoError: typeof RezoError.isRezoError;
5044
5044
  export declare const Cancel: typeof RezoError;
5045
5045
  export declare const CancelToken: {
@@ -5039,7 +5039,7 @@ export interface RezoInstance extends Rezo, RezoCallable {
5039
5039
  *
5040
5040
  * IMPORTANT: Update these values when bumping package version.
5041
5041
  */
5042
- export declare const VERSION = "1.0.82";
5042
+ export declare const VERSION = "1.0.84";
5043
5043
  /**
5044
5044
  * Type guard to check if an error is a RezoError instance.
5045
5045
  */
@@ -5039,7 +5039,7 @@ export interface RezoInstance extends Rezo, RezoCallable {
5039
5039
  *
5040
5040
  * IMPORTANT: Update these values when bumping package version.
5041
5041
  */
5042
- export declare const VERSION = "1.0.82";
5042
+ export declare const VERSION = "1.0.84";
5043
5043
  export declare const isRezoError: typeof RezoError.isRezoError;
5044
5044
  export declare const Cancel: typeof RezoError;
5045
5045
  export declare const CancelToken: {
@@ -5039,7 +5039,7 @@ export interface RezoInstance extends Rezo, RezoCallable {
5039
5039
  *
5040
5040
  * IMPORTANT: Update these values when bumping package version.
5041
5041
  */
5042
- export declare const VERSION = "1.0.82";
5042
+ export declare const VERSION = "1.0.84";
5043
5043
  export declare const isRezoError: typeof RezoError.isRezoError;
5044
5044
  export declare const Cancel: typeof RezoError;
5045
5045
  export declare const CancelToken: {
@@ -5039,7 +5039,7 @@ export interface RezoInstance extends Rezo, RezoCallable {
5039
5039
  *
5040
5040
  * IMPORTANT: Update these values when bumping package version.
5041
5041
  */
5042
- export declare const VERSION = "1.0.82";
5042
+ export declare const VERSION = "1.0.84";
5043
5043
  export declare const isRezoError: typeof RezoError.isRezoError;
5044
5044
  export declare const Cancel: typeof RezoError;
5045
5045
  export declare const CancelToken: {
@@ -66,16 +66,14 @@ const debugLog = {
66
66
  maxRetries: (config, maxRetries) => {
67
67
  if (config.debug) {
68
68
  console.log(`[Rezo Debug] Max retries (${maxRetries}) reached, throwing error`);
69
- }
70
- if (config.trackUrl) {
69
+ } else if (config.trackUrl) {
71
70
  console.log(`[Rezo Track] ✗ Max retries reached`);
72
71
  }
73
72
  },
74
73
  response: (config, status, statusText, duration) => {
75
74
  if (config.debug) {
76
75
  console.log(`[Rezo Debug] Response: ${status} ${statusText} (${duration.toFixed(2)}ms)`);
77
- }
78
- if (config.trackUrl) {
76
+ } else if (config.trackUrl) {
79
77
  console.log(`[Rezo Track] ✓ ${status} ${statusText}`);
80
78
  }
81
79
  },
@@ -999,25 +997,33 @@ async function request(config, fetchOptions, requestCount, timing, _stats, respo
999
997
  timeoutManager.clearAll();
1000
998
  _stats.statusOnNext = "error";
1001
999
  updateTiming(config, timing, contentLength || "", contentLengthCounter, res.rawHeaders);
1000
+ const data = Buffer.concat(chunks);
1002
1001
  if (_stats.redirectUrl) {
1003
- const partialResponse = buildResponseFromIncoming(res, Buffer.concat(chunks), config, url.toString(), buildUrlTree(config, url.toString()), undefined, undefined, contentLengthCounter);
1002
+ const partialResponse = buildResponseFromIncoming(res, data, config, url.toString(), buildUrlTree(config, url.toString()), undefined, undefined, contentLengthCounter);
1004
1003
  resolve(partialResponse);
1005
1004
  return;
1006
1005
  }
1007
- const error = buildDecompressionError({
1008
- statusCode: res.statusCode || 500,
1009
- headers,
1010
- contentType,
1011
- contentLength: contentLength || contentLengthCounter.toString(),
1012
- cookies: cookies || [],
1013
- statusText: err.message || "Decompression failed",
1014
- url: res.url || url.toString(),
1015
- body: Buffer.concat(chunks),
1016
- finalUrl: url.toString(),
1017
- config,
1018
- request: fetchOptions
1019
- });
1020
- resolve(error);
1006
+ const isNetworkError = err.code && ["ECONNRESET", "ECONNABORTED", "ETIMEDOUT", "EPIPE", "ENOTFOUND", "ECONNREFUSED", "EHOSTUNREACH", "ENETUNREACH", "ERR_STREAM_PREMATURE_CLOSE"].includes(err.code);
1007
+ if (isNetworkError) {
1008
+ const partialResponse = buildResponseFromIncoming(res, data, config, url.toString(), buildUrlTree(config, url.toString()), undefined, undefined, contentLengthCounter);
1009
+ const error = new RezoError(err.message, config, err.code, fetchOptions, partialResponse);
1010
+ resolve(error);
1011
+ } else {
1012
+ const error = buildDecompressionError({
1013
+ statusCode: res.statusCode || 500,
1014
+ headers,
1015
+ contentType,
1016
+ contentLength: contentLength || contentLengthCounter.toString(),
1017
+ cookies: cookies || [],
1018
+ statusText: err.message || "Decompression failed",
1019
+ url: res.url || url.toString(),
1020
+ body: data,
1021
+ finalUrl: url.toString(),
1022
+ config,
1023
+ request: fetchOptions
1024
+ });
1025
+ resolve(error);
1026
+ }
1021
1027
  });
1022
1028
  }
1023
1029
  });
@@ -66,16 +66,14 @@ const debugLog = {
66
66
  maxRetries: (config, maxRetries) => {
67
67
  if (config.debug) {
68
68
  console.log(`[Rezo Debug] Max retries (${maxRetries}) reached, throwing error`);
69
- }
70
- if (config.trackUrl) {
69
+ } else if (config.trackUrl) {
71
70
  console.log(`[Rezo Track] ✗ Max retries reached`);
72
71
  }
73
72
  },
74
73
  response: (config, status, statusText, duration) => {
75
74
  if (config.debug) {
76
75
  console.log(`[Rezo Debug] Response: ${status} ${statusText} (${duration.toFixed(2)}ms)`);
77
- }
78
- if (config.trackUrl) {
76
+ } else if (config.trackUrl) {
79
77
  console.log(`[Rezo Track] ✓ ${status} ${statusText}`);
80
78
  }
81
79
  },
@@ -999,25 +997,33 @@ async function request(config, fetchOptions, requestCount, timing, _stats, respo
999
997
  timeoutManager.clearAll();
1000
998
  _stats.statusOnNext = "error";
1001
999
  updateTiming(config, timing, contentLength || "", contentLengthCounter, res.rawHeaders);
1000
+ const data = Buffer.concat(chunks);
1002
1001
  if (_stats.redirectUrl) {
1003
- const partialResponse = buildResponseFromIncoming(res, Buffer.concat(chunks), config, url.toString(), buildUrlTree(config, url.toString()), undefined, undefined, contentLengthCounter);
1002
+ const partialResponse = buildResponseFromIncoming(res, data, config, url.toString(), buildUrlTree(config, url.toString()), undefined, undefined, contentLengthCounter);
1004
1003
  resolve(partialResponse);
1005
1004
  return;
1006
1005
  }
1007
- const error = buildDecompressionError({
1008
- statusCode: res.statusCode || 500,
1009
- headers,
1010
- contentType,
1011
- contentLength: contentLength || contentLengthCounter.toString(),
1012
- cookies: cookies || [],
1013
- statusText: err.message || "Decompression failed",
1014
- url: res.url || url.toString(),
1015
- body: Buffer.concat(chunks),
1016
- finalUrl: url.toString(),
1017
- config,
1018
- request: fetchOptions
1019
- });
1020
- resolve(error);
1006
+ const isNetworkError = err.code && ["ECONNRESET", "ECONNABORTED", "ETIMEDOUT", "EPIPE", "ENOTFOUND", "ECONNREFUSED", "EHOSTUNREACH", "ENETUNREACH", "ERR_STREAM_PREMATURE_CLOSE"].includes(err.code);
1007
+ if (isNetworkError) {
1008
+ const partialResponse = buildResponseFromIncoming(res, data, config, url.toString(), buildUrlTree(config, url.toString()), undefined, undefined, contentLengthCounter);
1009
+ const error = new RezoError(err.message, config, err.code, fetchOptions, partialResponse);
1010
+ resolve(error);
1011
+ } else {
1012
+ const error = buildDecompressionError({
1013
+ statusCode: res.statusCode || 500,
1014
+ headers,
1015
+ contentType,
1016
+ contentLength: contentLength || contentLengthCounter.toString(),
1017
+ cookies: cookies || [],
1018
+ statusText: err.message || "Decompression failed",
1019
+ url: res.url || url.toString(),
1020
+ body: data,
1021
+ finalUrl: url.toString(),
1022
+ config,
1023
+ request: fetchOptions
1024
+ });
1025
+ resolve(error);
1026
+ }
1021
1027
  });
1022
1028
  }
1023
1029
  });
@@ -115,8 +115,33 @@ const debugLog = {
115
115
  }
116
116
  }
117
117
  };
118
+ function looksCompressed(buffer, encoding) {
119
+ if (buffer.length < 2)
120
+ return false;
121
+ const enc = encoding.toLowerCase();
122
+ if (enc === "gzip" || enc === "x-gzip") {
123
+ return buffer[0] === 31 && buffer[1] === 139;
124
+ }
125
+ if (enc === "deflate" || enc === "x-deflate") {
126
+ return buffer[0] === 120;
127
+ }
128
+ if (enc === "zstd") {
129
+ return buffer.length >= 4 && buffer[0] === 40 && buffer[1] === 181 && buffer[2] === 47 && buffer[3] === 253;
130
+ }
131
+ if (enc === "br" || enc === "brotli") {
132
+ const firstByte = buffer[0];
133
+ if (firstByte === 123 || firstByte === 91 || firstByte === 60) {
134
+ return false;
135
+ }
136
+ return true;
137
+ }
138
+ return true;
139
+ }
118
140
  async function decompressBuffer(buffer, contentEncoding) {
119
141
  const encoding = contentEncoding.toLowerCase();
142
+ if (!looksCompressed(buffer, encoding)) {
143
+ return buffer;
144
+ }
120
145
  switch (encoding) {
121
146
  case "gzip":
122
147
  case "x-gzip":
@@ -115,8 +115,33 @@ const debugLog = {
115
115
  }
116
116
  }
117
117
  };
118
+ function looksCompressed(buffer, encoding) {
119
+ if (buffer.length < 2)
120
+ return false;
121
+ const enc = encoding.toLowerCase();
122
+ if (enc === "gzip" || enc === "x-gzip") {
123
+ return buffer[0] === 31 && buffer[1] === 139;
124
+ }
125
+ if (enc === "deflate" || enc === "x-deflate") {
126
+ return buffer[0] === 120;
127
+ }
128
+ if (enc === "zstd") {
129
+ return buffer.length >= 4 && buffer[0] === 40 && buffer[1] === 181 && buffer[2] === 47 && buffer[3] === 253;
130
+ }
131
+ if (enc === "br" || enc === "brotli") {
132
+ const firstByte = buffer[0];
133
+ if (firstByte === 123 || firstByte === 91 || firstByte === 60) {
134
+ return false;
135
+ }
136
+ return true;
137
+ }
138
+ return true;
139
+ }
118
140
  async function decompressBuffer(buffer, contentEncoding) {
119
141
  const encoding = contentEncoding.toLowerCase();
142
+ if (!looksCompressed(buffer, encoding)) {
143
+ return buffer;
144
+ }
120
145
  switch (encoding) {
121
146
  case "gzip":
122
147
  case "x-gzip":
@@ -1,6 +1,6 @@
1
- const _mod_ae1t5y = require('./picker.cjs');
2
- exports.detectRuntime = _mod_ae1t5y.detectRuntime;
3
- exports.getAdapterCapabilities = _mod_ae1t5y.getAdapterCapabilities;
4
- exports.buildAdapterContext = _mod_ae1t5y.buildAdapterContext;
5
- exports.getAvailableAdapters = _mod_ae1t5y.getAvailableAdapters;
6
- exports.selectAdapter = _mod_ae1t5y.selectAdapter;;
1
+ const _mod_3p5i7l = require('./picker.cjs');
2
+ exports.detectRuntime = _mod_3p5i7l.detectRuntime;
3
+ exports.getAdapterCapabilities = _mod_3p5i7l.getAdapterCapabilities;
4
+ exports.buildAdapterContext = _mod_3p5i7l.buildAdapterContext;
5
+ exports.getAvailableAdapters = _mod_3p5i7l.getAvailableAdapters;
6
+ exports.selectAdapter = _mod_3p5i7l.selectAdapter;;
@@ -1,9 +1,9 @@
1
- const _mod_xniowe = require('./lru-cache.cjs');
2
- exports.LRUCache = _mod_xniowe.LRUCache;;
3
- const _mod_3fsuc1 = require('./dns-cache.cjs');
4
- exports.DNSCache = _mod_3fsuc1.DNSCache;
5
- exports.getGlobalDNSCache = _mod_3fsuc1.getGlobalDNSCache;
6
- exports.resetGlobalDNSCache = _mod_3fsuc1.resetGlobalDNSCache;;
7
- const _mod_yrs7x8 = require('./response-cache.cjs');
8
- exports.ResponseCache = _mod_yrs7x8.ResponseCache;
9
- exports.normalizeResponseCacheConfig = _mod_yrs7x8.normalizeResponseCacheConfig;;
1
+ const _mod_ci3rei = require('./lru-cache.cjs');
2
+ exports.LRUCache = _mod_ci3rei.LRUCache;;
3
+ const _mod_5hg6xp = require('./dns-cache.cjs');
4
+ exports.DNSCache = _mod_5hg6xp.DNSCache;
5
+ exports.getGlobalDNSCache = _mod_5hg6xp.getGlobalDNSCache;
6
+ exports.resetGlobalDNSCache = _mod_5hg6xp.resetGlobalDNSCache;;
7
+ const _mod_ychnao = require('./response-cache.cjs');
8
+ exports.ResponseCache = _mod_ychnao.ResponseCache;
9
+ exports.normalizeResponseCacheConfig = _mod_ychnao.normalizeResponseCacheConfig;;
@@ -1,42 +1,42 @@
1
- const _mod_zpmi1q = require('./crawler.cjs');
2
- exports.Crawler = _mod_zpmi1q.Crawler;;
3
- const _mod_0yghwo = require('./crawler-options.cjs');
4
- exports.CrawlerOptions = _mod_0yghwo.CrawlerOptions;;
5
- const _mod_tzjuxk = require('./plugin/robots-txt.cjs');
6
- exports.RobotsTxt = _mod_tzjuxk.RobotsTxt;;
7
- const _mod_3v8k3g = require('./plugin/file-cacher.cjs');
8
- exports.FileCacher = _mod_3v8k3g.FileCacher;;
9
- const _mod_y54lvl = require('./plugin/url-store.cjs');
10
- exports.UrlStore = _mod_y54lvl.UrlStore;;
11
- const _mod_m8p0kx = require('./plugin/navigation-history.cjs');
12
- exports.NavigationHistory = _mod_m8p0kx.NavigationHistory;;
13
- const _mod_tapdx1 = require('./addon/oxylabs/index.cjs');
14
- exports.Oxylabs = _mod_tapdx1.Oxylabs;;
15
- const _mod_5c1g1g = require('./addon/oxylabs/options.cjs');
16
- exports.OXYLABS_BROWSER_TYPES = _mod_5c1g1g.OXYLABS_BROWSER_TYPES;
17
- exports.OXYLABS_COMMON_LOCALES = _mod_5c1g1g.OXYLABS_COMMON_LOCALES;
18
- exports.OXYLABS_COMMON_GEO_LOCATIONS = _mod_5c1g1g.OXYLABS_COMMON_GEO_LOCATIONS;
19
- exports.OXYLABS_US_STATES = _mod_5c1g1g.OXYLABS_US_STATES;
20
- exports.OXYLABS_EUROPEAN_COUNTRIES = _mod_5c1g1g.OXYLABS_EUROPEAN_COUNTRIES;
21
- exports.OXYLABS_ASIAN_COUNTRIES = _mod_5c1g1g.OXYLABS_ASIAN_COUNTRIES;
22
- exports.getRandomOxylabsBrowserType = _mod_5c1g1g.getRandomBrowserType;
23
- exports.getRandomOxylabsLocale = _mod_5c1g1g.getRandomLocale;
24
- exports.getRandomOxylabsGeoLocation = _mod_5c1g1g.getRandomGeoLocation;;
25
- const _mod_gpd5te = require('./scraper.cjs');
26
- exports.isRestrictedDomain = _mod_gpd5te.isRestrictedDomain;;
27
- const _mod_f3bk19 = require('./addon/decodo/index.cjs');
28
- exports.Decodo = _mod_f3bk19.Decodo;;
29
- const _mod_xexlpg = require('./addon/decodo/options.cjs');
30
- exports.DECODO_DEVICE_TYPES = _mod_xexlpg.DECODO_DEVICE_TYPES;
31
- exports.DECODO_HEADLESS_MODES = _mod_xexlpg.DECODO_HEADLESS_MODES;
32
- exports.DECODO_COMMON_LOCALES = _mod_xexlpg.DECODO_COMMON_LOCALES;
33
- exports.DECODO_COMMON_COUNTRIES = _mod_xexlpg.DECODO_COMMON_COUNTRIES;
34
- exports.DECODO_EUROPEAN_COUNTRIES = _mod_xexlpg.DECODO_EUROPEAN_COUNTRIES;
35
- exports.DECODO_ASIAN_COUNTRIES = _mod_xexlpg.DECODO_ASIAN_COUNTRIES;
36
- exports.DECODO_US_STATES = _mod_xexlpg.DECODO_US_STATES;
37
- exports.DECODO_COMMON_CITIES = _mod_xexlpg.DECODO_COMMON_CITIES;
38
- exports.getRandomDecodoDeviceType = _mod_xexlpg.getRandomDeviceType;
39
- exports.getRandomDecodoLocale = _mod_xexlpg.getRandomLocale;
40
- exports.getRandomDecodoCountry = _mod_xexlpg.getRandomCountry;
41
- exports.getRandomDecodoCity = _mod_xexlpg.getRandomCity;
42
- exports.generateDecodoSessionId = _mod_xexlpg.generateSessionId;;
1
+ const _mod_duk9tf = require('./crawler.cjs');
2
+ exports.Crawler = _mod_duk9tf.Crawler;;
3
+ const _mod_akzmlw = require('./crawler-options.cjs');
4
+ exports.CrawlerOptions = _mod_akzmlw.CrawlerOptions;;
5
+ const _mod_6op5fj = require('./plugin/robots-txt.cjs');
6
+ exports.RobotsTxt = _mod_6op5fj.RobotsTxt;;
7
+ const _mod_dtkqtw = require('./plugin/file-cacher.cjs');
8
+ exports.FileCacher = _mod_dtkqtw.FileCacher;;
9
+ const _mod_7vu350 = require('./plugin/url-store.cjs');
10
+ exports.UrlStore = _mod_7vu350.UrlStore;;
11
+ const _mod_f8xhaf = require('./plugin/navigation-history.cjs');
12
+ exports.NavigationHistory = _mod_f8xhaf.NavigationHistory;;
13
+ const _mod_ytibwi = require('./addon/oxylabs/index.cjs');
14
+ exports.Oxylabs = _mod_ytibwi.Oxylabs;;
15
+ const _mod_1sgs87 = require('./addon/oxylabs/options.cjs');
16
+ exports.OXYLABS_BROWSER_TYPES = _mod_1sgs87.OXYLABS_BROWSER_TYPES;
17
+ exports.OXYLABS_COMMON_LOCALES = _mod_1sgs87.OXYLABS_COMMON_LOCALES;
18
+ exports.OXYLABS_COMMON_GEO_LOCATIONS = _mod_1sgs87.OXYLABS_COMMON_GEO_LOCATIONS;
19
+ exports.OXYLABS_US_STATES = _mod_1sgs87.OXYLABS_US_STATES;
20
+ exports.OXYLABS_EUROPEAN_COUNTRIES = _mod_1sgs87.OXYLABS_EUROPEAN_COUNTRIES;
21
+ exports.OXYLABS_ASIAN_COUNTRIES = _mod_1sgs87.OXYLABS_ASIAN_COUNTRIES;
22
+ exports.getRandomOxylabsBrowserType = _mod_1sgs87.getRandomBrowserType;
23
+ exports.getRandomOxylabsLocale = _mod_1sgs87.getRandomLocale;
24
+ exports.getRandomOxylabsGeoLocation = _mod_1sgs87.getRandomGeoLocation;;
25
+ const _mod_5mf9qe = require('./scraper.cjs');
26
+ exports.isRestrictedDomain = _mod_5mf9qe.isRestrictedDomain;;
27
+ const _mod_4si1yx = require('./addon/decodo/index.cjs');
28
+ exports.Decodo = _mod_4si1yx.Decodo;;
29
+ const _mod_quv5x1 = require('./addon/decodo/options.cjs');
30
+ exports.DECODO_DEVICE_TYPES = _mod_quv5x1.DECODO_DEVICE_TYPES;
31
+ exports.DECODO_HEADLESS_MODES = _mod_quv5x1.DECODO_HEADLESS_MODES;
32
+ exports.DECODO_COMMON_LOCALES = _mod_quv5x1.DECODO_COMMON_LOCALES;
33
+ exports.DECODO_COMMON_COUNTRIES = _mod_quv5x1.DECODO_COMMON_COUNTRIES;
34
+ exports.DECODO_EUROPEAN_COUNTRIES = _mod_quv5x1.DECODO_EUROPEAN_COUNTRIES;
35
+ exports.DECODO_ASIAN_COUNTRIES = _mod_quv5x1.DECODO_ASIAN_COUNTRIES;
36
+ exports.DECODO_US_STATES = _mod_quv5x1.DECODO_US_STATES;
37
+ exports.DECODO_COMMON_CITIES = _mod_quv5x1.DECODO_COMMON_CITIES;
38
+ exports.getRandomDecodoDeviceType = _mod_quv5x1.getRandomDeviceType;
39
+ exports.getRandomDecodoLocale = _mod_quv5x1.getRandomLocale;
40
+ exports.getRandomDecodoCountry = _mod_quv5x1.getRandomCountry;
41
+ exports.getRandomDecodoCity = _mod_quv5x1.getRandomCity;
42
+ exports.generateDecodoSessionId = _mod_quv5x1.generateSessionId;;
package/dist/crawler.d.ts CHANGED
@@ -7494,6 +7494,12 @@ export interface EmailDiscoveryEvent<T = Record<string, any>> {
7494
7494
  timestamp: Date;
7495
7495
  metadata: T;
7496
7496
  }
7497
+ /**
7498
+ * Check if URL belongs to a restricted domain
7499
+ * @param url - URL to check
7500
+ * @returns Boolean indicating if domain is restricted
7501
+ */
7502
+ export declare function isRestrictedDomain(url: string): boolean;
7497
7503
  export interface HealthSnapshot {
7498
7504
  /** Timestamp of snapshot */
7499
7505
  timestamp: number;
@@ -1,4 +1,6 @@
1
- const _mod_e7lebt = require('../crawler/crawler.cjs');
2
- exports.Crawler = _mod_e7lebt.Crawler;;
3
- const _mod_ryspn0 = require('../crawler/crawler-options.cjs');
4
- exports.CrawlerOptions = _mod_ryspn0.CrawlerOptions;;
1
+ const _mod_19fgvq = require('../crawler/crawler.cjs');
2
+ exports.Crawler = _mod_19fgvq.Crawler;;
3
+ const _mod_upiha4 = require('../crawler/crawler-options.cjs');
4
+ exports.CrawlerOptions = _mod_upiha4.CrawlerOptions;;
5
+ const _mod_p1t2de = require('../crawler/scraper.cjs');
6
+ exports.isRestrictedDomain = _mod_p1t2de.isRestrictedDomain;;
@@ -1,2 +1,3 @@
1
1
  export { Crawler } from '../crawler/crawler.js';
2
2
  export { CrawlerOptions } from '../crawler/crawler-options.js';
3
+ export { isRestrictedDomain } from '../crawler/scraper.js';
package/dist/index.cjs CHANGED
@@ -1,31 +1,31 @@
1
- const _mod_wolme3 = require('./core/rezo.cjs');
2
- exports.Rezo = _mod_wolme3.Rezo;
3
- exports.createRezoInstance = _mod_wolme3.createRezoInstance;
4
- exports.createDefaultInstance = _mod_wolme3.createDefaultInstance;;
5
- const _mod_0gbzem = require('./errors/rezo-error.cjs');
6
- exports.RezoError = _mod_0gbzem.RezoError;
7
- exports.RezoErrorCode = _mod_0gbzem.RezoErrorCode;;
8
- const _mod_igkidr = require('./utils/headers.cjs');
9
- exports.RezoHeaders = _mod_igkidr.RezoHeaders;;
10
- const _mod_um43lo = require('./utils/form-data.cjs');
11
- exports.RezoFormData = _mod_um43lo.RezoFormData;;
12
- const _mod_s7nv7c = require('./utils/cookies.cjs');
13
- exports.RezoCookieJar = _mod_s7nv7c.RezoCookieJar;
14
- exports.Cookie = _mod_s7nv7c.Cookie;;
15
- const _mod_r0n94a = require('./utils/curl.cjs');
16
- exports.toCurl = _mod_r0n94a.toCurl;
17
- exports.fromCurl = _mod_r0n94a.fromCurl;;
18
- const _mod_ka6f0h = require('./core/hooks.cjs');
19
- exports.createDefaultHooks = _mod_ka6f0h.createDefaultHooks;
20
- exports.mergeHooks = _mod_ka6f0h.mergeHooks;;
21
- const _mod_p3pm5o = require('./proxy/manager.cjs');
22
- exports.ProxyManager = _mod_p3pm5o.ProxyManager;;
23
- const _mod_o6neu2 = require('./queue/index.cjs');
24
- exports.RezoQueue = _mod_o6neu2.RezoQueue;
25
- exports.HttpQueue = _mod_o6neu2.HttpQueue;
26
- exports.RezoHttpQueue = _mod_o6neu2.RezoHttpQueue;
27
- exports.Priority = _mod_o6neu2.Priority;
28
- exports.HttpMethodPriority = _mod_o6neu2.HttpMethodPriority;;
1
+ const _mod_srihcf = require('./core/rezo.cjs');
2
+ exports.Rezo = _mod_srihcf.Rezo;
3
+ exports.createRezoInstance = _mod_srihcf.createRezoInstance;
4
+ exports.createDefaultInstance = _mod_srihcf.createDefaultInstance;;
5
+ const _mod_uokgel = require('./errors/rezo-error.cjs');
6
+ exports.RezoError = _mod_uokgel.RezoError;
7
+ exports.RezoErrorCode = _mod_uokgel.RezoErrorCode;;
8
+ const _mod_rxz6f7 = require('./utils/headers.cjs');
9
+ exports.RezoHeaders = _mod_rxz6f7.RezoHeaders;;
10
+ const _mod_aiv919 = require('./utils/form-data.cjs');
11
+ exports.RezoFormData = _mod_aiv919.RezoFormData;;
12
+ const _mod_aox6y7 = require('./utils/cookies.cjs');
13
+ exports.RezoCookieJar = _mod_aox6y7.RezoCookieJar;
14
+ exports.Cookie = _mod_aox6y7.Cookie;;
15
+ const _mod_x4c884 = require('./utils/curl.cjs');
16
+ exports.toCurl = _mod_x4c884.toCurl;
17
+ exports.fromCurl = _mod_x4c884.fromCurl;;
18
+ const _mod_50bd90 = require('./core/hooks.cjs');
19
+ exports.createDefaultHooks = _mod_50bd90.createDefaultHooks;
20
+ exports.mergeHooks = _mod_50bd90.mergeHooks;;
21
+ const _mod_zdpxrf = require('./proxy/manager.cjs');
22
+ exports.ProxyManager = _mod_zdpxrf.ProxyManager;;
23
+ const _mod_gqx7wa = require('./queue/index.cjs');
24
+ exports.RezoQueue = _mod_gqx7wa.RezoQueue;
25
+ exports.HttpQueue = _mod_gqx7wa.HttpQueue;
26
+ exports.RezoHttpQueue = _mod_gqx7wa.RezoHttpQueue;
27
+ exports.Priority = _mod_gqx7wa.Priority;
28
+ exports.HttpMethodPriority = _mod_gqx7wa.HttpMethodPriority;;
29
29
  const { RezoError } = require('./errors/rezo-error.cjs');
30
30
  const isRezoError = exports.isRezoError = RezoError.isRezoError;
31
31
  const Cancel = exports.Cancel = RezoError;
@@ -35,9 +35,9 @@ const isCancel = exports.isCancel = (error) => {
35
35
  };
36
36
  const all = exports.all = Promise.all.bind(Promise);
37
37
  const spread = exports.spread = (callback) => (array) => callback(...array);
38
- const _mod_qjpu0j = require('./version.cjs');
39
- exports.VERSION = _mod_qjpu0j.VERSION;
40
- exports.PACKAGE_NAME = _mod_qjpu0j.PACKAGE_NAME;;
38
+ const _mod_vtpksw = require('./version.cjs');
39
+ exports.VERSION = _mod_vtpksw.VERSION;
40
+ exports.PACKAGE_NAME = _mod_vtpksw.PACKAGE_NAME;;
41
41
  const { executeRequest } = require('./adapters/http.cjs');
42
42
  const { setGlobalAdapter, createRezoInstance } = require('./core/rezo.cjs');
43
43
  setGlobalAdapter(executeRequest);
package/dist/index.d.ts CHANGED
@@ -5168,7 +5168,7 @@ export declare function createDefaultInstance(config?: RezoDefaultOptions): Rezo
5168
5168
  *
5169
5169
  * IMPORTANT: Update these values when bumping package version.
5170
5170
  */
5171
- export declare const VERSION = "1.0.82";
5171
+ export declare const VERSION = "1.0.84";
5172
5172
  export declare const PACKAGE_NAME = "rezo";
5173
5173
  export declare const isRezoError: typeof RezoError.isRezoError;
5174
5174
  export declare const Cancel: typeof RezoError;
@@ -3,6 +3,7 @@ const { Readable } = require("node:stream");
3
3
  const net = require("node:net");
4
4
  const tls = require("node:tls");
5
5
  const { SocksClient } = require('./socks-client.cjs');
6
+ const { parseProxyString } = require('../../proxy/parse.cjs');
6
7
  function isBunRuntime() {
7
8
  return typeof globalThis.Bun !== "undefined";
8
9
  }
@@ -10,7 +11,8 @@ function isBunSocksRequest(proxy) {
10
11
  if (!isBunRuntime() || !proxy)
11
12
  return false;
12
13
  if (typeof proxy === "string") {
13
- return proxy.startsWith("socks");
14
+ const proxyObject = parseProxyString(proxy);
15
+ return proxyObject !== null && proxyObject.protocol.startsWith("socks");
14
16
  }
15
17
  return proxy.protocol?.startsWith("socks") ?? false;
16
18
  }
@@ -3,6 +3,7 @@ import { Readable } from "node:stream";
3
3
  import * as net from "node:net";
4
4
  import * as tls from "node:tls";
5
5
  import { SocksClient } from './socks-client.js';
6
+ import { parseProxyString } from '../../proxy/parse.js';
6
7
  export function isBunRuntime() {
7
8
  return typeof globalThis.Bun !== "undefined";
8
9
  }
@@ -10,7 +11,8 @@ export function isBunSocksRequest(proxy) {
10
11
  if (!isBunRuntime() || !proxy)
11
12
  return false;
12
13
  if (typeof proxy === "string") {
13
- return proxy.startsWith("socks");
14
+ const proxyObject = parseProxyString(proxy);
15
+ return proxyObject !== null && proxyObject.protocol.startsWith("socks");
14
16
  }
15
17
  return proxy.protocol?.startsWith("socks") ?? false;
16
18
  }
@@ -1,14 +1,14 @@
1
- const _mod_w8wl28 = require('./base.cjs');
2
- exports.Agent = _mod_w8wl28.Agent;;
3
- const _mod_ebezo6 = require('./http-proxy.cjs');
4
- exports.HttpProxyAgent = _mod_ebezo6.HttpProxyAgent;;
5
- const _mod_5llp66 = require('./https-proxy.cjs');
6
- exports.HttpsProxyAgent = _mod_5llp66.HttpsProxyAgent;;
7
- const _mod_gjlau0 = require('./socks-proxy.cjs');
8
- exports.SocksProxyAgent = _mod_gjlau0.SocksProxyAgent;;
9
- const _mod_yp647h = require('./socks-client.cjs');
10
- exports.SocksClient = _mod_yp647h.SocksClient;;
11
- const _mod_6cq7ui = require('./bun-socks-http.cjs');
12
- exports.bunHttp = _mod_6cq7ui.bunHttp;
13
- exports.isBunRuntime = _mod_6cq7ui.isBunRuntime;
14
- exports.isBunSocksRequest = _mod_6cq7ui.isBunSocksRequest;;
1
+ const _mod_rxywkg = require('./base.cjs');
2
+ exports.Agent = _mod_rxywkg.Agent;;
3
+ const _mod_2fiyim = require('./http-proxy.cjs');
4
+ exports.HttpProxyAgent = _mod_2fiyim.HttpProxyAgent;;
5
+ const _mod_lz2x8w = require('./https-proxy.cjs');
6
+ exports.HttpsProxyAgent = _mod_lz2x8w.HttpsProxyAgent;;
7
+ const _mod_m3uybg = require('./socks-proxy.cjs');
8
+ exports.SocksProxyAgent = _mod_m3uybg.SocksProxyAgent;;
9
+ const _mod_p7s7zs = require('./socks-client.cjs');
10
+ exports.SocksClient = _mod_p7s7zs.SocksClient;;
11
+ const _mod_1b54qx = require('./bun-socks-http.cjs');
12
+ exports.bunHttp = _mod_1b54qx.bunHttp;
13
+ exports.isBunRuntime = _mod_1b54qx.isBunRuntime;
14
+ exports.isBunSocksRequest = _mod_1b54qx.isBunSocksRequest;;
@@ -5039,7 +5039,7 @@ export interface RezoInstance extends Rezo, RezoCallable {
5039
5039
  *
5040
5040
  * IMPORTANT: Update these values when bumping package version.
5041
5041
  */
5042
- export declare const VERSION = "1.0.82";
5042
+ export declare const VERSION = "1.0.84";
5043
5043
  export declare const isRezoError: typeof RezoError.isRezoError;
5044
5044
  export declare const Cancel: typeof RezoError;
5045
5045
  export declare const CancelToken: {
@@ -5039,7 +5039,7 @@ export interface RezoInstance extends Rezo, RezoCallable {
5039
5039
  *
5040
5040
  * IMPORTANT: Update these values when bumping package version.
5041
5041
  */
5042
- export declare const VERSION = "1.0.82";
5042
+ export declare const VERSION = "1.0.84";
5043
5043
  export declare const isRezoError: typeof RezoError.isRezoError;
5044
5044
  export declare const Cancel: typeof RezoError;
5045
5045
  export declare const CancelToken: {
@@ -5039,7 +5039,7 @@ export interface RezoInstance extends Rezo, RezoCallable {
5039
5039
  *
5040
5040
  * IMPORTANT: Update these values when bumping package version.
5041
5041
  */
5042
- export declare const VERSION = "1.0.82";
5042
+ export declare const VERSION = "1.0.84";
5043
5043
  export declare const isRezoError: typeof RezoError.isRezoError;
5044
5044
  export declare const Cancel: typeof RezoError;
5045
5045
  export declare const CancelToken: {
@@ -5039,7 +5039,7 @@ export interface RezoInstance extends Rezo, RezoCallable {
5039
5039
  *
5040
5040
  * IMPORTANT: Update these values when bumping package version.
5041
5041
  */
5042
- export declare const VERSION = "1.0.82";
5042
+ export declare const VERSION = "1.0.84";
5043
5043
  export declare const isRezoError: typeof RezoError.isRezoError;
5044
5044
  export declare const Cancel: typeof RezoError;
5045
5045
  export declare const CancelToken: {
@@ -5039,7 +5039,7 @@ export interface RezoInstance extends Rezo, RezoCallable {
5039
5039
  *
5040
5040
  * IMPORTANT: Update these values when bumping package version.
5041
5041
  */
5042
- export declare const VERSION = "1.0.82";
5042
+ export declare const VERSION = "1.0.84";
5043
5043
  export declare const isRezoError: typeof RezoError.isRezoError;
5044
5044
  export declare const Cancel: typeof RezoError;
5045
5045
  export declare const CancelToken: {
@@ -5039,7 +5039,7 @@ export interface RezoInstance extends Rezo, RezoCallable {
5039
5039
  *
5040
5040
  * IMPORTANT: Update these values when bumping package version.
5041
5041
  */
5042
- export declare const VERSION = "1.0.82";
5042
+ export declare const VERSION = "1.0.84";
5043
5043
  export declare const isRezoError: typeof RezoError.isRezoError;
5044
5044
  export declare const Cancel: typeof RezoError;
5045
5045
  export declare const CancelToken: {
@@ -1,9 +1,9 @@
1
1
  const { Agent, HttpProxyAgent, HttpsProxyAgent, SocksProxyAgent } = require('../internal/agents/index.cjs');
2
2
  const { parseProxyString } = require('./parse.cjs');
3
- const _mod_ty950w = require('./manager.cjs');
4
- exports.ProxyManager = _mod_ty950w.ProxyManager;;
5
- const _mod_fnl1ag = require('./parse.cjs');
6
- exports.parseProxyString = _mod_fnl1ag.parseProxyString;;
3
+ const _mod_6lnuph = require('./manager.cjs');
4
+ exports.ProxyManager = _mod_6lnuph.ProxyManager;;
5
+ const _mod_3zaqo2 = require('./parse.cjs');
6
+ exports.parseProxyString = _mod_3zaqo2.parseProxyString;;
7
7
  function createOptions(uri, opts) {
8
8
  if (uri instanceof URL || typeof uri === "string") {
9
9
  return {
@@ -22,7 +22,7 @@ function parseProxyString(input) {
22
22
  let username;
23
23
  let password;
24
24
  const proto = input.includes("://") ? input.split("://")[0].toLowerCase() : "";
25
- const protocol = proto.startsWith("http") ? "http" : proto.startsWith("https") ? "https" : "socks5";
25
+ const protocol = proto.startsWith("http") ? "http" : proto.startsWith("https") ? "https" : proto.startsWith("socks4") ? "socks4" : "socks5";
26
26
  if (protocol !== "socks5" || input.includes("://")) {
27
27
  input = input.split("://")[1];
28
28
  }
@@ -22,7 +22,7 @@ export function parseProxyString(input) {
22
22
  let username;
23
23
  let password;
24
24
  const proto = input.includes("://") ? input.split("://")[0].toLowerCase() : "";
25
- const protocol = proto.startsWith("http") ? "http" : proto.startsWith("https") ? "https" : "socks5";
25
+ const protocol = proto.startsWith("http") ? "http" : proto.startsWith("https") ? "https" : proto.startsWith("socks4") ? "socks4" : "socks5";
26
26
  if (protocol !== "socks5" || input.includes("://")) {
27
27
  input = input.split("://")[1];
28
28
  }
@@ -1,9 +1,9 @@
1
- const _mod_pp38d9 = require('./queue.cjs');
2
- exports.RezoQueue = _mod_pp38d9.RezoQueue;;
3
- const _mod_7jany9 = require('./http-queue.cjs');
4
- exports.HttpQueue = _mod_7jany9.HttpQueue;
5
- exports.RezoHttpQueue = _mod_7jany9.HttpQueue;
6
- exports.extractDomain = _mod_7jany9.extractDomain;;
7
- const _mod_6b157h = require('./types.cjs');
8
- exports.Priority = _mod_6b157h.Priority;
9
- exports.HttpMethodPriority = _mod_6b157h.HttpMethodPriority;;
1
+ const _mod_90q066 = require('./queue.cjs');
2
+ exports.RezoQueue = _mod_90q066.RezoQueue;;
3
+ const _mod_rc94f3 = require('./http-queue.cjs');
4
+ exports.HttpQueue = _mod_rc94f3.HttpQueue;
5
+ exports.RezoHttpQueue = _mod_rc94f3.HttpQueue;
6
+ exports.extractDomain = _mod_rc94f3.extractDomain;;
7
+ const _mod_wjpsb8 = require('./types.cjs');
8
+ exports.Priority = _mod_wjpsb8.Priority;
9
+ exports.HttpMethodPriority = _mod_wjpsb8.HttpMethodPriority;;
@@ -1,11 +1,11 @@
1
- const _mod_9kvlje = require('./event-emitter.cjs');
2
- exports.UniversalEventEmitter = _mod_9kvlje.UniversalEventEmitter;;
3
- const _mod_2844ml = require('./stream.cjs');
4
- exports.UniversalStreamResponse = _mod_2844ml.UniversalStreamResponse;
5
- exports.StreamResponse = _mod_2844ml.StreamResponse;;
6
- const _mod_foynas = require('./download.cjs');
7
- exports.UniversalDownloadResponse = _mod_foynas.UniversalDownloadResponse;
8
- exports.DownloadResponse = _mod_foynas.DownloadResponse;;
9
- const _mod_relzt0 = require('./upload.cjs');
10
- exports.UniversalUploadResponse = _mod_relzt0.UniversalUploadResponse;
11
- exports.UploadResponse = _mod_relzt0.UploadResponse;;
1
+ const _mod_yv8m3r = require('./event-emitter.cjs');
2
+ exports.UniversalEventEmitter = _mod_yv8m3r.UniversalEventEmitter;;
3
+ const _mod_3pkgs6 = require('./stream.cjs');
4
+ exports.UniversalStreamResponse = _mod_3pkgs6.UniversalStreamResponse;
5
+ exports.StreamResponse = _mod_3pkgs6.StreamResponse;;
6
+ const _mod_ggx77h = require('./download.cjs');
7
+ exports.UniversalDownloadResponse = _mod_ggx77h.UniversalDownloadResponse;
8
+ exports.DownloadResponse = _mod_ggx77h.DownloadResponse;;
9
+ const _mod_ovm7cr = require('./upload.cjs');
10
+ exports.UniversalUploadResponse = _mod_ovm7cr.UniversalUploadResponse;
11
+ exports.UploadResponse = _mod_ovm7cr.UploadResponse;;
@@ -1,31 +1,186 @@
1
1
  const zlib = require("node:zlib");
2
+ const { Transform } = require("node:stream");
3
+ function looksCompressed(data, encoding) {
4
+ if (data.length < 2)
5
+ return false;
6
+ const enc = encoding.toLowerCase();
7
+ if (enc === "gzip" || enc === "x-gzip") {
8
+ return data[0] === 31 && data[1] === 139;
9
+ }
10
+ if (enc === "deflate" || enc === "x-deflate") {
11
+ return data[0] === 120;
12
+ }
13
+ if (enc === "zstd") {
14
+ return data[0] === 40 && data[1] === 181 && data[2] === 47 && data[3] === 253;
15
+ }
16
+ if (enc === "br" || enc === "brotli") {
17
+ let i = 0;
18
+ while (i < data.length && (data[i] === 32 || data[i] === 9 || data[i] === 10 || data[i] === 13)) {
19
+ i++;
20
+ }
21
+ if (i >= data.length) {
22
+ return false;
23
+ }
24
+ const firstNonWhitespace = data[i];
25
+ const textStarts = [
26
+ 123,
27
+ 91,
28
+ 34,
29
+ 39,
30
+ 60,
31
+ 48,
32
+ 49,
33
+ 50,
34
+ 51,
35
+ 52,
36
+ 53,
37
+ 54,
38
+ 55,
39
+ 56,
40
+ 57,
41
+ 45,
42
+ 43,
43
+ 46,
44
+ 116,
45
+ 102,
46
+ 110,
47
+ 84,
48
+ 70,
49
+ 78
50
+ ];
51
+ if (textStarts.includes(firstNonWhitespace)) {
52
+ return false;
53
+ }
54
+ if (data[0] === 239 && data.length >= 3 && data[1] === 187 && data[2] === 191) {
55
+ return false;
56
+ }
57
+ const checkLen = Math.min(16, data.length);
58
+ let printableCount = 0;
59
+ for (let j = 0;j < checkLen; j++) {
60
+ if (data[j] >= 32 && data[j] <= 126) {
61
+ printableCount++;
62
+ }
63
+ }
64
+ if (printableCount >= checkLen * 0.8) {
65
+ return false;
66
+ }
67
+ return true;
68
+ }
69
+ return true;
70
+ }
2
71
 
3
- class CompressionUtil {
4
- static decompressStream(response, contentEncoding, config) {
5
- if (!contentEncoding) {
6
- return response;
72
+ class SmartDecompressStream extends Transform {
73
+ encoding;
74
+ decompressor = null;
75
+ isCompressed = null;
76
+ buffer = Buffer.alloc(0);
77
+ headerChecked = false;
78
+ passThrough = false;
79
+ constructor(encoding) {
80
+ super();
81
+ this.encoding = encoding.toLowerCase();
82
+ }
83
+ _transform(chunk, _encoding, callback) {
84
+ if (!this.headerChecked) {
85
+ this.buffer = Buffer.concat([this.buffer, chunk]);
86
+ if (this.buffer.length >= 4) {
87
+ this.headerChecked = true;
88
+ this.isCompressed = looksCompressed(this.buffer, this.encoding);
89
+ if (this.isCompressed) {
90
+ this.decompressor = this.createDecompressor();
91
+ if (this.decompressor) {
92
+ this.decompressor.on("data", (data) => this.push(data));
93
+ this.decompressor.on("error", (err) => {
94
+ this.destroy(err);
95
+ });
96
+ this.decompressor.write(this.buffer);
97
+ } else {
98
+ this.passThrough = true;
99
+ this.push(this.buffer);
100
+ }
101
+ } else {
102
+ this.passThrough = true;
103
+ this.push(this.buffer);
104
+ }
105
+ this.buffer = Buffer.alloc(0);
106
+ callback();
107
+ return;
108
+ }
109
+ callback();
110
+ return;
7
111
  }
8
- if (!this.shouldDecompress(contentEncoding, config)) {
9
- return response;
112
+ if (this.passThrough) {
113
+ this.push(chunk);
114
+ callback();
115
+ } else if (this.decompressor) {
116
+ this.decompressor.write(chunk, callback);
117
+ } else {
118
+ callback();
10
119
  }
11
- const encoding = contentEncoding.toLowerCase();
12
- switch (encoding) {
120
+ }
121
+ _flush(callback) {
122
+ if (!this.headerChecked && this.buffer.length > 0) {
123
+ this.isCompressed = looksCompressed(this.buffer, this.encoding);
124
+ if (this.isCompressed && this.buffer.length > 0) {
125
+ const decompressor = this.createDecompressor();
126
+ if (decompressor) {
127
+ const chunks = [];
128
+ decompressor.on("data", (data) => chunks.push(data));
129
+ decompressor.on("end", () => {
130
+ this.push(Buffer.concat(chunks));
131
+ callback();
132
+ });
133
+ decompressor.on("error", () => {
134
+ this.push(this.buffer);
135
+ callback();
136
+ });
137
+ decompressor.end(this.buffer);
138
+ return;
139
+ }
140
+ }
141
+ this.push(this.buffer);
142
+ callback();
143
+ return;
144
+ }
145
+ if (this.decompressor) {
146
+ this.decompressor.end();
147
+ this.decompressor.once("end", () => callback());
148
+ } else {
149
+ callback();
150
+ }
151
+ }
152
+ createDecompressor() {
153
+ switch (this.encoding) {
13
154
  case "gzip":
14
155
  case "x-gzip":
15
- return response.pipe(zlib.createGunzip());
156
+ return zlib.createGunzip();
16
157
  case "deflate":
17
158
  case "x-deflate":
18
- return response.pipe(zlib.createInflate());
159
+ return zlib.createInflate();
19
160
  case "gzip-raw":
20
- return response.pipe(zlib.createInflate({ windowBits: 15 }));
161
+ return zlib.createInflate({ windowBits: 15 });
21
162
  case "br":
22
163
  case "brotli":
23
- return response.pipe(zlib.createBrotliDecompress());
164
+ return zlib.createBrotliDecompress();
24
165
  case "zstd":
25
- return response.pipe(zlib.createZstdDecompress());
166
+ return zlib.createZstdDecompress();
26
167
  default:
27
- return response;
168
+ return null;
169
+ }
170
+ }
171
+ }
172
+
173
+ class CompressionUtil {
174
+ static decompressStream(response, contentEncoding, config) {
175
+ if (!contentEncoding) {
176
+ return response;
28
177
  }
178
+ if (!this.shouldDecompress(contentEncoding, config)) {
179
+ return response;
180
+ }
181
+ const encoding = contentEncoding.toLowerCase();
182
+ const smartStream = new SmartDecompressStream(encoding);
183
+ return response.pipe(smartStream);
29
184
  }
30
185
  static shouldDecompress(contentEncoding, config) {
31
186
  if (!config) {
@@ -1,31 +1,186 @@
1
1
  import * as zlib from "node:zlib";
2
+ import { Transform } from "node:stream";
3
+ function looksCompressed(data, encoding) {
4
+ if (data.length < 2)
5
+ return false;
6
+ const enc = encoding.toLowerCase();
7
+ if (enc === "gzip" || enc === "x-gzip") {
8
+ return data[0] === 31 && data[1] === 139;
9
+ }
10
+ if (enc === "deflate" || enc === "x-deflate") {
11
+ return data[0] === 120;
12
+ }
13
+ if (enc === "zstd") {
14
+ return data[0] === 40 && data[1] === 181 && data[2] === 47 && data[3] === 253;
15
+ }
16
+ if (enc === "br" || enc === "brotli") {
17
+ let i = 0;
18
+ while (i < data.length && (data[i] === 32 || data[i] === 9 || data[i] === 10 || data[i] === 13)) {
19
+ i++;
20
+ }
21
+ if (i >= data.length) {
22
+ return false;
23
+ }
24
+ const firstNonWhitespace = data[i];
25
+ const textStarts = [
26
+ 123,
27
+ 91,
28
+ 34,
29
+ 39,
30
+ 60,
31
+ 48,
32
+ 49,
33
+ 50,
34
+ 51,
35
+ 52,
36
+ 53,
37
+ 54,
38
+ 55,
39
+ 56,
40
+ 57,
41
+ 45,
42
+ 43,
43
+ 46,
44
+ 116,
45
+ 102,
46
+ 110,
47
+ 84,
48
+ 70,
49
+ 78
50
+ ];
51
+ if (textStarts.includes(firstNonWhitespace)) {
52
+ return false;
53
+ }
54
+ if (data[0] === 239 && data.length >= 3 && data[1] === 187 && data[2] === 191) {
55
+ return false;
56
+ }
57
+ const checkLen = Math.min(16, data.length);
58
+ let printableCount = 0;
59
+ for (let j = 0;j < checkLen; j++) {
60
+ if (data[j] >= 32 && data[j] <= 126) {
61
+ printableCount++;
62
+ }
63
+ }
64
+ if (printableCount >= checkLen * 0.8) {
65
+ return false;
66
+ }
67
+ return true;
68
+ }
69
+ return true;
70
+ }
2
71
 
3
- export class CompressionUtil {
4
- static decompressStream(response, contentEncoding, config) {
5
- if (!contentEncoding) {
6
- return response;
72
+ class SmartDecompressStream extends Transform {
73
+ encoding;
74
+ decompressor = null;
75
+ isCompressed = null;
76
+ buffer = Buffer.alloc(0);
77
+ headerChecked = false;
78
+ passThrough = false;
79
+ constructor(encoding) {
80
+ super();
81
+ this.encoding = encoding.toLowerCase();
82
+ }
83
+ _transform(chunk, _encoding, callback) {
84
+ if (!this.headerChecked) {
85
+ this.buffer = Buffer.concat([this.buffer, chunk]);
86
+ if (this.buffer.length >= 4) {
87
+ this.headerChecked = true;
88
+ this.isCompressed = looksCompressed(this.buffer, this.encoding);
89
+ if (this.isCompressed) {
90
+ this.decompressor = this.createDecompressor();
91
+ if (this.decompressor) {
92
+ this.decompressor.on("data", (data) => this.push(data));
93
+ this.decompressor.on("error", (err) => {
94
+ this.destroy(err);
95
+ });
96
+ this.decompressor.write(this.buffer);
97
+ } else {
98
+ this.passThrough = true;
99
+ this.push(this.buffer);
100
+ }
101
+ } else {
102
+ this.passThrough = true;
103
+ this.push(this.buffer);
104
+ }
105
+ this.buffer = Buffer.alloc(0);
106
+ callback();
107
+ return;
108
+ }
109
+ callback();
110
+ return;
7
111
  }
8
- if (!this.shouldDecompress(contentEncoding, config)) {
9
- return response;
112
+ if (this.passThrough) {
113
+ this.push(chunk);
114
+ callback();
115
+ } else if (this.decompressor) {
116
+ this.decompressor.write(chunk, callback);
117
+ } else {
118
+ callback();
10
119
  }
11
- const encoding = contentEncoding.toLowerCase();
12
- switch (encoding) {
120
+ }
121
+ _flush(callback) {
122
+ if (!this.headerChecked && this.buffer.length > 0) {
123
+ this.isCompressed = looksCompressed(this.buffer, this.encoding);
124
+ if (this.isCompressed && this.buffer.length > 0) {
125
+ const decompressor = this.createDecompressor();
126
+ if (decompressor) {
127
+ const chunks = [];
128
+ decompressor.on("data", (data) => chunks.push(data));
129
+ decompressor.on("end", () => {
130
+ this.push(Buffer.concat(chunks));
131
+ callback();
132
+ });
133
+ decompressor.on("error", () => {
134
+ this.push(this.buffer);
135
+ callback();
136
+ });
137
+ decompressor.end(this.buffer);
138
+ return;
139
+ }
140
+ }
141
+ this.push(this.buffer);
142
+ callback();
143
+ return;
144
+ }
145
+ if (this.decompressor) {
146
+ this.decompressor.end();
147
+ this.decompressor.once("end", () => callback());
148
+ } else {
149
+ callback();
150
+ }
151
+ }
152
+ createDecompressor() {
153
+ switch (this.encoding) {
13
154
  case "gzip":
14
155
  case "x-gzip":
15
- return response.pipe(zlib.createGunzip());
156
+ return zlib.createGunzip();
16
157
  case "deflate":
17
158
  case "x-deflate":
18
- return response.pipe(zlib.createInflate());
159
+ return zlib.createInflate();
19
160
  case "gzip-raw":
20
- return response.pipe(zlib.createInflate({ windowBits: 15 }));
161
+ return zlib.createInflate({ windowBits: 15 });
21
162
  case "br":
22
163
  case "brotli":
23
- return response.pipe(zlib.createBrotliDecompress());
164
+ return zlib.createBrotliDecompress();
24
165
  case "zstd":
25
- return response.pipe(zlib.createZstdDecompress());
166
+ return zlib.createZstdDecompress();
26
167
  default:
27
- return response;
168
+ return null;
169
+ }
170
+ }
171
+ }
172
+
173
+ export class CompressionUtil {
174
+ static decompressStream(response, contentEncoding, config) {
175
+ if (!contentEncoding) {
176
+ return response;
28
177
  }
178
+ if (!this.shouldDecompress(contentEncoding, config)) {
179
+ return response;
180
+ }
181
+ const encoding = contentEncoding.toLowerCase();
182
+ const smartStream = new SmartDecompressStream(encoding);
183
+ return response.pipe(smartStream);
29
184
  }
30
185
  static shouldDecompress(contentEncoding, config) {
31
186
  if (!config) {
package/dist/version.cjs CHANGED
@@ -1,2 +1,2 @@
1
- const VERSION = exports.VERSION = "1.0.82";
1
+ const VERSION = exports.VERSION = "1.0.84";
2
2
  const PACKAGE_NAME = exports.PACKAGE_NAME = "rezo";
package/dist/version.js CHANGED
@@ -1,2 +1,2 @@
1
- export const VERSION = "1.0.82";
1
+ export const VERSION = "1.0.84";
2
2
  export const PACKAGE_NAME = "rezo";
@@ -1,52 +1,52 @@
1
- const _mod_kajqav = require('./types.cjs');
2
- exports.WgetError = _mod_kajqav.WgetError;;
3
- const _mod_zgb5zc = require('./asset-extractor.cjs');
4
- exports.AssetExtractor = _mod_zgb5zc.AssetExtractor;;
5
- const _mod_87yt9d = require('./url-filter.cjs');
6
- exports.UrlFilter = _mod_87yt9d.UrlFilter;;
7
- const _mod_1hbefm = require('./file-writer.cjs');
8
- exports.FileWriter = _mod_1hbefm.FileWriter;;
9
- const _mod_min03d = require('./robots.cjs');
10
- exports.RobotsHandler = _mod_min03d.RobotsHandler;;
11
- const _mod_2yiljd = require('./resume.cjs');
12
- exports.ResumeHandler = _mod_2yiljd.ResumeHandler;;
13
- const _mod_8xl488 = require('./progress.cjs');
14
- exports.ProgressReporter = _mod_8xl488.ProgressReporter;
15
- exports.ProgressTracker = _mod_8xl488.ProgressTracker;
16
- exports.parseSize = _mod_8xl488.parseSize;;
17
- const _mod_gpwl4q = require('./link-converter.cjs');
18
- exports.LinkConverter = _mod_gpwl4q.LinkConverter;;
19
- const _mod_ztmsq1 = require('./downloader.cjs');
20
- exports.Downloader = _mod_ztmsq1.Downloader;;
21
- const _mod_za9dzp = require('./asset-organizer.cjs');
22
- exports.AssetOrganizer = _mod_za9dzp.AssetOrganizer;
23
- exports.DEFAULT_ASSET_FOLDERS = _mod_za9dzp.DEFAULT_ASSET_FOLDERS;;
24
- const _mod_vxpafq = require('./download-cache.cjs');
25
- exports.DownloadCache = _mod_vxpafq.DownloadCache;;
26
- const _mod_uyszlp = require('./filter-lists.cjs');
27
- exports.EXECUTABLE_EXTENSIONS = _mod_uyszlp.EXECUTABLE_EXTENSIONS;
28
- exports.ARCHIVE_EXTENSIONS = _mod_uyszlp.ARCHIVE_EXTENSIONS;
29
- exports.DOCUMENT_EXTENSIONS = _mod_uyszlp.DOCUMENT_EXTENSIONS;
30
- exports.IMAGE_EXTENSIONS = _mod_uyszlp.IMAGE_EXTENSIONS;
31
- exports.VIDEO_EXTENSIONS = _mod_uyszlp.VIDEO_EXTENSIONS;
32
- exports.AUDIO_EXTENSIONS = _mod_uyszlp.AUDIO_EXTENSIONS;
33
- exports.FONT_EXTENSIONS = _mod_uyszlp.FONT_EXTENSIONS;
34
- exports.WEB_ASSET_EXTENSIONS = _mod_uyszlp.WEB_ASSET_EXTENSIONS;
35
- exports.DATA_EXTENSIONS = _mod_uyszlp.DATA_EXTENSIONS;
36
- exports.EXECUTABLE_MIME_TYPES = _mod_uyszlp.EXECUTABLE_MIME_TYPES;
37
- exports.ARCHIVE_MIME_TYPES = _mod_uyszlp.ARCHIVE_MIME_TYPES;
38
- exports.DOCUMENT_MIME_TYPES = _mod_uyszlp.DOCUMENT_MIME_TYPES;
39
- exports.IMAGE_MIME_TYPES = _mod_uyszlp.IMAGE_MIME_TYPES;
40
- exports.VIDEO_MIME_TYPES = _mod_uyszlp.VIDEO_MIME_TYPES;
41
- exports.AUDIO_MIME_TYPES = _mod_uyszlp.AUDIO_MIME_TYPES;
42
- exports.FONT_MIME_TYPES = _mod_uyszlp.FONT_MIME_TYPES;
43
- exports.WEB_ASSET_MIME_TYPES = _mod_uyszlp.WEB_ASSET_MIME_TYPES;
44
- exports.DATA_MIME_TYPES = _mod_uyszlp.DATA_MIME_TYPES;
45
- exports.SAFE_WEB_PRESET = _mod_uyszlp.SAFE_WEB_PRESET;
46
- exports.DOCUMENTS_ONLY_PRESET = _mod_uyszlp.DOCUMENTS_ONLY_PRESET;
47
- exports.NO_MEDIA_PRESET = _mod_uyszlp.NO_MEDIA_PRESET;
48
- exports.MINIMAL_MIRROR_PRESET = _mod_uyszlp.MINIMAL_MIRROR_PRESET;
49
- exports.TEXT_ONLY_PRESET = _mod_uyszlp.TEXT_ONLY_PRESET;;
1
+ const _mod_3nz39m = require('./types.cjs');
2
+ exports.WgetError = _mod_3nz39m.WgetError;;
3
+ const _mod_ku99f8 = require('./asset-extractor.cjs');
4
+ exports.AssetExtractor = _mod_ku99f8.AssetExtractor;;
5
+ const _mod_97zske = require('./url-filter.cjs');
6
+ exports.UrlFilter = _mod_97zske.UrlFilter;;
7
+ const _mod_f7re5i = require('./file-writer.cjs');
8
+ exports.FileWriter = _mod_f7re5i.FileWriter;;
9
+ const _mod_p49jy5 = require('./robots.cjs');
10
+ exports.RobotsHandler = _mod_p49jy5.RobotsHandler;;
11
+ const _mod_r5ydtz = require('./resume.cjs');
12
+ exports.ResumeHandler = _mod_r5ydtz.ResumeHandler;;
13
+ const _mod_99nwh2 = require('./progress.cjs');
14
+ exports.ProgressReporter = _mod_99nwh2.ProgressReporter;
15
+ exports.ProgressTracker = _mod_99nwh2.ProgressTracker;
16
+ exports.parseSize = _mod_99nwh2.parseSize;;
17
+ const _mod_ce9x1s = require('./link-converter.cjs');
18
+ exports.LinkConverter = _mod_ce9x1s.LinkConverter;;
19
+ const _mod_ukwaji = require('./downloader.cjs');
20
+ exports.Downloader = _mod_ukwaji.Downloader;;
21
+ const _mod_43jecd = require('./asset-organizer.cjs');
22
+ exports.AssetOrganizer = _mod_43jecd.AssetOrganizer;
23
+ exports.DEFAULT_ASSET_FOLDERS = _mod_43jecd.DEFAULT_ASSET_FOLDERS;;
24
+ const _mod_suisa3 = require('./download-cache.cjs');
25
+ exports.DownloadCache = _mod_suisa3.DownloadCache;;
26
+ const _mod_4559sr = require('./filter-lists.cjs');
27
+ exports.EXECUTABLE_EXTENSIONS = _mod_4559sr.EXECUTABLE_EXTENSIONS;
28
+ exports.ARCHIVE_EXTENSIONS = _mod_4559sr.ARCHIVE_EXTENSIONS;
29
+ exports.DOCUMENT_EXTENSIONS = _mod_4559sr.DOCUMENT_EXTENSIONS;
30
+ exports.IMAGE_EXTENSIONS = _mod_4559sr.IMAGE_EXTENSIONS;
31
+ exports.VIDEO_EXTENSIONS = _mod_4559sr.VIDEO_EXTENSIONS;
32
+ exports.AUDIO_EXTENSIONS = _mod_4559sr.AUDIO_EXTENSIONS;
33
+ exports.FONT_EXTENSIONS = _mod_4559sr.FONT_EXTENSIONS;
34
+ exports.WEB_ASSET_EXTENSIONS = _mod_4559sr.WEB_ASSET_EXTENSIONS;
35
+ exports.DATA_EXTENSIONS = _mod_4559sr.DATA_EXTENSIONS;
36
+ exports.EXECUTABLE_MIME_TYPES = _mod_4559sr.EXECUTABLE_MIME_TYPES;
37
+ exports.ARCHIVE_MIME_TYPES = _mod_4559sr.ARCHIVE_MIME_TYPES;
38
+ exports.DOCUMENT_MIME_TYPES = _mod_4559sr.DOCUMENT_MIME_TYPES;
39
+ exports.IMAGE_MIME_TYPES = _mod_4559sr.IMAGE_MIME_TYPES;
40
+ exports.VIDEO_MIME_TYPES = _mod_4559sr.VIDEO_MIME_TYPES;
41
+ exports.AUDIO_MIME_TYPES = _mod_4559sr.AUDIO_MIME_TYPES;
42
+ exports.FONT_MIME_TYPES = _mod_4559sr.FONT_MIME_TYPES;
43
+ exports.WEB_ASSET_MIME_TYPES = _mod_4559sr.WEB_ASSET_MIME_TYPES;
44
+ exports.DATA_MIME_TYPES = _mod_4559sr.DATA_MIME_TYPES;
45
+ exports.SAFE_WEB_PRESET = _mod_4559sr.SAFE_WEB_PRESET;
46
+ exports.DOCUMENTS_ONLY_PRESET = _mod_4559sr.DOCUMENTS_ONLY_PRESET;
47
+ exports.NO_MEDIA_PRESET = _mod_4559sr.NO_MEDIA_PRESET;
48
+ exports.MINIMAL_MIRROR_PRESET = _mod_4559sr.MINIMAL_MIRROR_PRESET;
49
+ exports.TEXT_ONLY_PRESET = _mod_4559sr.TEXT_ONLY_PRESET;;
50
50
  const { Downloader } = require('./downloader.cjs');
51
51
  const rezo = require('../index.cjs');
52
52
  const { promises: fs } = require("node:fs");
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "rezo",
3
- "version": "1.0.82",
3
+ "version": "1.0.84",
4
4
  "description": "Lightning-fast, enterprise-grade HTTP client for modern JavaScript. Full HTTP/2 support, intelligent cookie management, multiple adapters (HTTP, Fetch, cURL, XHR), streaming, proxy support (HTTP/HTTPS/SOCKS), and cross-environment compatibility.",
5
5
  "main": "dist/index.cjs",
6
6
  "module": "dist/index.js",