rezo 1.0.49 → 1.0.51
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/adapters/http.cjs +1 -1
- package/dist/adapters/http.js +1 -1
- package/dist/adapters/http2.cjs +10 -2
- package/dist/adapters/http2.js +10 -2
- package/dist/adapters/index.cjs +6 -6
- package/dist/cache/index.cjs +9 -9
- package/dist/crawler/crawler.cjs +32 -9
- package/dist/crawler/crawler.js +32 -9
- package/dist/crawler/index.cjs +40 -40
- package/dist/entries/crawler.cjs +5 -5
- package/dist/index.cjs +27 -27
- package/dist/internal/agents/index.cjs +10 -10
- package/dist/proxy/index.cjs +4 -4
- package/dist/queue/index.cjs +8 -8
- package/dist/responses/universal/index.cjs +11 -11
- package/dist/utils/compression.cjs +2 -33
- package/dist/utils/compression.js +2 -33
- package/dist/utils/http-config.cjs +1 -1
- package/dist/utils/http-config.js +1 -1
- package/package.json +1 -1
package/dist/adapters/http.cjs
CHANGED
|
@@ -776,7 +776,7 @@ async function request(config, fetchOptions, requestCount, timing, _stats, respo
|
|
|
776
776
|
if (isRedirected)
|
|
777
777
|
_stats.statusOnNext = "redirect";
|
|
778
778
|
if (isRedirected && location) {
|
|
779
|
-
const redirectUrlObj = new URL(location, url);
|
|
779
|
+
const redirectUrlObj = new URL(typeof location === "string" ? location : location.toString(), url);
|
|
780
780
|
if (!redirectUrlObj.hash && url.hash) {
|
|
781
781
|
redirectUrlObj.hash = url.hash;
|
|
782
782
|
}
|
package/dist/adapters/http.js
CHANGED
|
@@ -776,7 +776,7 @@ async function request(config, fetchOptions, requestCount, timing, _stats, respo
|
|
|
776
776
|
if (isRedirected)
|
|
777
777
|
_stats.statusOnNext = "redirect";
|
|
778
778
|
if (isRedirected && location) {
|
|
779
|
-
const redirectUrlObj = new URL(location, url);
|
|
779
|
+
const redirectUrlObj = new URL(typeof location === "string" ? location : location.toString(), url);
|
|
780
780
|
if (!redirectUrlObj.hash && url.hash) {
|
|
781
781
|
redirectUrlObj.hash = url.hash;
|
|
782
782
|
}
|
package/dist/adapters/http2.cjs
CHANGED
|
@@ -1253,12 +1253,15 @@ async function executeHttp2Stream(config, fetchOptions, requestCount, timing, _s
|
|
|
1253
1253
|
resolve(error);
|
|
1254
1254
|
}
|
|
1255
1255
|
};
|
|
1256
|
-
session.
|
|
1257
|
-
session.
|
|
1256
|
+
session.once("error", sessionErrorHandler);
|
|
1257
|
+
session.once("goaway", (errorCode, lastStreamID) => {
|
|
1258
1258
|
if (config.debug) {
|
|
1259
1259
|
console.log(`[Rezo Debug] HTTP/2: Session GOAWAY received (errorCode: ${errorCode}, lastStreamID: ${lastStreamID})`);
|
|
1260
1260
|
}
|
|
1261
1261
|
});
|
|
1262
|
+
const cleanupSessionListeners = () => {
|
|
1263
|
+
session.removeListener("error", sessionErrorHandler);
|
|
1264
|
+
};
|
|
1262
1265
|
if (config.debug) {
|
|
1263
1266
|
console.log(`[Rezo Debug] HTTP/2: Creating request stream...`);
|
|
1264
1267
|
}
|
|
@@ -1282,6 +1285,7 @@ async function executeHttp2Stream(config, fetchOptions, requestCount, timing, _s
|
|
|
1282
1285
|
timeoutId = setTimeout(() => {
|
|
1283
1286
|
if (!resolved) {
|
|
1284
1287
|
resolved = true;
|
|
1288
|
+
cleanupSessionListeners();
|
|
1285
1289
|
if (config.debug) {
|
|
1286
1290
|
console.log(`[Rezo Debug] HTTP/2: Request timeout after ${requestTimeout}ms (no response received)`);
|
|
1287
1291
|
}
|
|
@@ -1302,6 +1306,7 @@ async function executeHttp2Stream(config, fetchOptions, requestCount, timing, _s
|
|
|
1302
1306
|
if (!resolved && status === 0) {
|
|
1303
1307
|
resolved = true;
|
|
1304
1308
|
clearTimeout(timeoutId);
|
|
1309
|
+
cleanupSessionListeners();
|
|
1305
1310
|
if (config.debug) {
|
|
1306
1311
|
console.log(`[Rezo Debug] HTTP/2: Stream closed without response - retrying with new session`);
|
|
1307
1312
|
}
|
|
@@ -1317,6 +1322,7 @@ async function executeHttp2Stream(config, fetchOptions, requestCount, timing, _s
|
|
|
1317
1322
|
if (!resolved) {
|
|
1318
1323
|
resolved = true;
|
|
1319
1324
|
clearTimeout(timeoutId);
|
|
1325
|
+
cleanupSessionListeners();
|
|
1320
1326
|
const error = buildSmartError(config, fetchOptions, new Error("HTTP/2 stream aborted"));
|
|
1321
1327
|
_stats.statusOnNext = "error";
|
|
1322
1328
|
resolve(error);
|
|
@@ -1329,6 +1335,7 @@ async function executeHttp2Stream(config, fetchOptions, requestCount, timing, _s
|
|
|
1329
1335
|
if (!resolved) {
|
|
1330
1336
|
resolved = true;
|
|
1331
1337
|
clearTimeout(timeoutId);
|
|
1338
|
+
cleanupSessionListeners();
|
|
1332
1339
|
const error = buildSmartError(config, fetchOptions, err);
|
|
1333
1340
|
_stats.statusOnNext = "error";
|
|
1334
1341
|
resolve(error);
|
|
@@ -1427,6 +1434,7 @@ async function executeHttp2Stream(config, fetchOptions, requestCount, timing, _s
|
|
|
1427
1434
|
}
|
|
1428
1435
|
resolved = true;
|
|
1429
1436
|
clearTimeout(timeoutId);
|
|
1437
|
+
cleanupSessionListeners();
|
|
1430
1438
|
try {
|
|
1431
1439
|
updateTiming(config, timing, contentLengthCounter);
|
|
1432
1440
|
if (!config.transfer) {
|
package/dist/adapters/http2.js
CHANGED
|
@@ -1253,12 +1253,15 @@ async function executeHttp2Stream(config, fetchOptions, requestCount, timing, _s
|
|
|
1253
1253
|
resolve(error);
|
|
1254
1254
|
}
|
|
1255
1255
|
};
|
|
1256
|
-
session.
|
|
1257
|
-
session.
|
|
1256
|
+
session.once("error", sessionErrorHandler);
|
|
1257
|
+
session.once("goaway", (errorCode, lastStreamID) => {
|
|
1258
1258
|
if (config.debug) {
|
|
1259
1259
|
console.log(`[Rezo Debug] HTTP/2: Session GOAWAY received (errorCode: ${errorCode}, lastStreamID: ${lastStreamID})`);
|
|
1260
1260
|
}
|
|
1261
1261
|
});
|
|
1262
|
+
const cleanupSessionListeners = () => {
|
|
1263
|
+
session.removeListener("error", sessionErrorHandler);
|
|
1264
|
+
};
|
|
1262
1265
|
if (config.debug) {
|
|
1263
1266
|
console.log(`[Rezo Debug] HTTP/2: Creating request stream...`);
|
|
1264
1267
|
}
|
|
@@ -1282,6 +1285,7 @@ async function executeHttp2Stream(config, fetchOptions, requestCount, timing, _s
|
|
|
1282
1285
|
timeoutId = setTimeout(() => {
|
|
1283
1286
|
if (!resolved) {
|
|
1284
1287
|
resolved = true;
|
|
1288
|
+
cleanupSessionListeners();
|
|
1285
1289
|
if (config.debug) {
|
|
1286
1290
|
console.log(`[Rezo Debug] HTTP/2: Request timeout after ${requestTimeout}ms (no response received)`);
|
|
1287
1291
|
}
|
|
@@ -1302,6 +1306,7 @@ async function executeHttp2Stream(config, fetchOptions, requestCount, timing, _s
|
|
|
1302
1306
|
if (!resolved && status === 0) {
|
|
1303
1307
|
resolved = true;
|
|
1304
1308
|
clearTimeout(timeoutId);
|
|
1309
|
+
cleanupSessionListeners();
|
|
1305
1310
|
if (config.debug) {
|
|
1306
1311
|
console.log(`[Rezo Debug] HTTP/2: Stream closed without response - retrying with new session`);
|
|
1307
1312
|
}
|
|
@@ -1317,6 +1322,7 @@ async function executeHttp2Stream(config, fetchOptions, requestCount, timing, _s
|
|
|
1317
1322
|
if (!resolved) {
|
|
1318
1323
|
resolved = true;
|
|
1319
1324
|
clearTimeout(timeoutId);
|
|
1325
|
+
cleanupSessionListeners();
|
|
1320
1326
|
const error = buildSmartError(config, fetchOptions, new Error("HTTP/2 stream aborted"));
|
|
1321
1327
|
_stats.statusOnNext = "error";
|
|
1322
1328
|
resolve(error);
|
|
@@ -1329,6 +1335,7 @@ async function executeHttp2Stream(config, fetchOptions, requestCount, timing, _s
|
|
|
1329
1335
|
if (!resolved) {
|
|
1330
1336
|
resolved = true;
|
|
1331
1337
|
clearTimeout(timeoutId);
|
|
1338
|
+
cleanupSessionListeners();
|
|
1332
1339
|
const error = buildSmartError(config, fetchOptions, err);
|
|
1333
1340
|
_stats.statusOnNext = "error";
|
|
1334
1341
|
resolve(error);
|
|
@@ -1427,6 +1434,7 @@ async function executeHttp2Stream(config, fetchOptions, requestCount, timing, _s
|
|
|
1427
1434
|
}
|
|
1428
1435
|
resolved = true;
|
|
1429
1436
|
clearTimeout(timeoutId);
|
|
1437
|
+
cleanupSessionListeners();
|
|
1430
1438
|
try {
|
|
1431
1439
|
updateTiming(config, timing, contentLengthCounter);
|
|
1432
1440
|
if (!config.transfer) {
|
package/dist/adapters/index.cjs
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
|
-
const
|
|
2
|
-
exports.detectRuntime =
|
|
3
|
-
exports.getAdapterCapabilities =
|
|
4
|
-
exports.buildAdapterContext =
|
|
5
|
-
exports.getAvailableAdapters =
|
|
6
|
-
exports.selectAdapter =
|
|
1
|
+
const _mod_an14fc = require('./picker.cjs');
|
|
2
|
+
exports.detectRuntime = _mod_an14fc.detectRuntime;
|
|
3
|
+
exports.getAdapterCapabilities = _mod_an14fc.getAdapterCapabilities;
|
|
4
|
+
exports.buildAdapterContext = _mod_an14fc.buildAdapterContext;
|
|
5
|
+
exports.getAvailableAdapters = _mod_an14fc.getAvailableAdapters;
|
|
6
|
+
exports.selectAdapter = _mod_an14fc.selectAdapter;;
|
package/dist/cache/index.cjs
CHANGED
|
@@ -1,9 +1,9 @@
|
|
|
1
|
-
const
|
|
2
|
-
exports.LRUCache =
|
|
3
|
-
const
|
|
4
|
-
exports.DNSCache =
|
|
5
|
-
exports.getGlobalDNSCache =
|
|
6
|
-
exports.resetGlobalDNSCache =
|
|
7
|
-
const
|
|
8
|
-
exports.ResponseCache =
|
|
9
|
-
exports.normalizeResponseCacheConfig =
|
|
1
|
+
const _mod_c06h7j = require('./lru-cache.cjs');
|
|
2
|
+
exports.LRUCache = _mod_c06h7j.LRUCache;;
|
|
3
|
+
const _mod_7lzpm2 = require('./dns-cache.cjs');
|
|
4
|
+
exports.DNSCache = _mod_7lzpm2.DNSCache;
|
|
5
|
+
exports.getGlobalDNSCache = _mod_7lzpm2.getGlobalDNSCache;
|
|
6
|
+
exports.resetGlobalDNSCache = _mod_7lzpm2.resetGlobalDNSCache;;
|
|
7
|
+
const _mod_2v893e = require('./response-cache.cjs');
|
|
8
|
+
exports.ResponseCache = _mod_2v893e.ResponseCache;
|
|
9
|
+
exports.normalizeResponseCacheConfig = _mod_2v893e.normalizeResponseCacheConfig;;
|
package/dist/crawler/crawler.cjs
CHANGED
|
@@ -5,11 +5,12 @@ const { NavigationHistory } = require('./plugin/navigation-history.cjs');
|
|
|
5
5
|
const { RobotsTxt } = require('./plugin/robots-txt.cjs');
|
|
6
6
|
const { parseHTML } = require("linkedom");
|
|
7
7
|
const path = require("node:path");
|
|
8
|
-
const { Rezo } = require('../
|
|
8
|
+
const { Rezo } = require('../adapters/entries/http.cjs');
|
|
9
9
|
const { RezoQueue } = require('../queue/queue.cjs');
|
|
10
10
|
const { Scraper } = require('./scraper.cjs');
|
|
11
11
|
const { CrawlerOptions } = require('./crawler-options.cjs');
|
|
12
12
|
const { loadAdapter } = require('../adapters/picker.cjs');
|
|
13
|
+
const { resetGlobalAgentPool } = require('../utils/agent-pool.cjs');
|
|
13
14
|
String.prototype.addBaseUrl = function(url) {
|
|
14
15
|
url = url instanceof URL ? url.href : url;
|
|
15
16
|
const html = this.replace(/<base\b[^>]*?>/gi, "");
|
|
@@ -94,6 +95,10 @@ class Crawler {
|
|
|
94
95
|
}).then((storage) => {
|
|
95
96
|
this.cacher = storage;
|
|
96
97
|
this.isCacheReady = true;
|
|
98
|
+
}).catch((err) => {
|
|
99
|
+
if (this.config.debug)
|
|
100
|
+
console.warn("[Crawler] Failed to initialize cache:", err);
|
|
101
|
+
this.isCacheReady = true;
|
|
97
102
|
});
|
|
98
103
|
const dit = path.resolve(cacheDir, "urls");
|
|
99
104
|
if (!fs.existsSync(dit))
|
|
@@ -105,6 +110,10 @@ class Crawler {
|
|
|
105
110
|
}).then((storage) => {
|
|
106
111
|
this.urlStorage = storage;
|
|
107
112
|
this.isStorageReady = true;
|
|
113
|
+
}).catch((err) => {
|
|
114
|
+
if (this.config.debug)
|
|
115
|
+
console.warn("[Crawler] Failed to initialize URL storage:", err);
|
|
116
|
+
this.isStorageReady = true;
|
|
108
117
|
});
|
|
109
118
|
} else {
|
|
110
119
|
const dit = path.resolve(this.config.cacheDir, "./cache/urls");
|
|
@@ -117,6 +126,10 @@ class Crawler {
|
|
|
117
126
|
}).then((storage) => {
|
|
118
127
|
this.urlStorage = storage;
|
|
119
128
|
this.isStorageReady = true;
|
|
129
|
+
}).catch((err) => {
|
|
130
|
+
if (this.config.debug)
|
|
131
|
+
console.warn("[Crawler] Failed to initialize URL storage:", err);
|
|
132
|
+
this.isStorageReady = true;
|
|
120
133
|
});
|
|
121
134
|
}
|
|
122
135
|
if (this.config.enableNavigationHistory) {
|
|
@@ -856,12 +869,15 @@ class Crawler {
|
|
|
856
869
|
}
|
|
857
870
|
async execute(method, url, body, options = {}, isEmail, forceRevisit, oxylabsOptions, oxylabsInstanse, decodoInstanse, decodoOptions) {
|
|
858
871
|
await this.waitForStorage();
|
|
872
|
+
console.log("Waiting for storage...");
|
|
859
873
|
if (this.isCacheEnabled) {
|
|
860
874
|
await this.waitForCache();
|
|
861
875
|
}
|
|
876
|
+
console.log("Waiting for cache...");
|
|
862
877
|
if (this.config.enableNavigationHistory) {
|
|
863
878
|
await this.waitForNavigationHistory();
|
|
864
879
|
}
|
|
880
|
+
console.log("Waiting for navigation history...");
|
|
865
881
|
const task = this.queue.add(() => this.executeHttp(method, url, body, options, isEmail, forceRevisit, oxylabsOptions, oxylabsInstanse, decodoInstanse, decodoOptions));
|
|
866
882
|
task.finally(() => this.pendingExecutions.delete(task));
|
|
867
883
|
}
|
|
@@ -888,7 +904,9 @@ class Crawler {
|
|
|
888
904
|
}
|
|
889
905
|
async executeHttp(method, url, body, options = {}, isEmail, forceRevisit, oxylabsOptions, oxylabsInstanse, decodoInstanse, decodoOptions, retryCount = 0, parentUrl) {
|
|
890
906
|
try {
|
|
907
|
+
console.log("Triggering start handlers...");
|
|
891
908
|
await this.triggerStartHandlers();
|
|
909
|
+
console.log("Checking crawl limits...");
|
|
892
910
|
const limitCheck = await this.checkCrawlLimits(url, parentUrl);
|
|
893
911
|
if (!limitCheck.allowed) {
|
|
894
912
|
if (this.config.debug) {
|
|
@@ -903,13 +921,16 @@ class Crawler {
|
|
|
903
921
|
await new Promise((resolve) => setTimeout(resolve, delay));
|
|
904
922
|
}
|
|
905
923
|
const isVisited = forceRevisit ? false : await this.hasUrlInCache(url);
|
|
906
|
-
const cache = await this.getCache(url);
|
|
907
|
-
if (
|
|
908
|
-
|
|
909
|
-
|
|
910
|
-
|
|
924
|
+
const cache = method.toLowerCase() === "get" ? await this.getCache(url) : undefined;
|
|
925
|
+
if (method.toLowerCase() === "get") {
|
|
926
|
+
if (isVisited && !cache)
|
|
927
|
+
return;
|
|
928
|
+
if (isVisited)
|
|
929
|
+
return;
|
|
930
|
+
}
|
|
911
931
|
const requestStartTime = Date.now();
|
|
912
932
|
const response = cache && method === "GET" ? cache : oxylabsInstanse && oxylabsOptions ? await oxylabsInstanse.scrape(url) : decodoInstanse && decodoOptions ? await decodoInstanse.scrape(url) : await (method === "GET" ? this.http.get(url, options) : method === "PATCH" ? this.http.patch(url, body, options) : method === "POST" ? this.http.post(url, body, options) : this.http.put(url, body, options));
|
|
933
|
+
console.log("Response received...");
|
|
913
934
|
if (!cache) {
|
|
914
935
|
const responseTime = Date.now() - requestStartTime;
|
|
915
936
|
this.calculateAutoThrottleDelay(domain, responseTime);
|
|
@@ -943,9 +964,9 @@ class Crawler {
|
|
|
943
964
|
statusCode: res.status
|
|
944
965
|
});
|
|
945
966
|
}
|
|
946
|
-
if (!cache)
|
|
967
|
+
if (!cache && method === "GET")
|
|
947
968
|
await this.saveCache(url, res);
|
|
948
|
-
if (!isVisited)
|
|
969
|
+
if (!isVisited && method === "GET")
|
|
949
970
|
await this.saveUrl(url);
|
|
950
971
|
await this.markUrlVisited(url, {
|
|
951
972
|
status: res.status,
|
|
@@ -1039,6 +1060,7 @@ class Crawler {
|
|
|
1039
1060
|
}
|
|
1040
1061
|
await this.queue.onIdle();
|
|
1041
1062
|
await this.triggerFinishHandlers();
|
|
1063
|
+
await this.destroy();
|
|
1042
1064
|
}
|
|
1043
1065
|
async done() {
|
|
1044
1066
|
return this.waitForAll();
|
|
@@ -1055,7 +1077,7 @@ class Crawler {
|
|
|
1055
1077
|
} catch {}
|
|
1056
1078
|
}
|
|
1057
1079
|
async destroy() {
|
|
1058
|
-
this.queue.
|
|
1080
|
+
this.queue.destroy();
|
|
1059
1081
|
this.events.length = 0;
|
|
1060
1082
|
this.jsonEvents.length = 0;
|
|
1061
1083
|
this.errorEvents.length = 0;
|
|
@@ -1064,6 +1086,7 @@ class Crawler {
|
|
|
1064
1086
|
this.emailDiscoveredEvents.length = 0;
|
|
1065
1087
|
this.emailLeadsEvents.length = 0;
|
|
1066
1088
|
await this.close();
|
|
1089
|
+
resetGlobalAgentPool();
|
|
1067
1090
|
}
|
|
1068
1091
|
}
|
|
1069
1092
|
|
package/dist/crawler/crawler.js
CHANGED
|
@@ -5,11 +5,12 @@ import { NavigationHistory } from './plugin/navigation-history.js';
|
|
|
5
5
|
import { RobotsTxt } from './plugin/robots-txt.js';
|
|
6
6
|
import { parseHTML } from "linkedom";
|
|
7
7
|
import path from "node:path";
|
|
8
|
-
import { Rezo } from '../
|
|
8
|
+
import { Rezo } from '../adapters/entries/http.js';
|
|
9
9
|
import { RezoQueue } from '../queue/queue.js';
|
|
10
10
|
import { Scraper } from './scraper.js';
|
|
11
11
|
import { CrawlerOptions } from './crawler-options.js';
|
|
12
12
|
import { loadAdapter } from '../adapters/picker.js';
|
|
13
|
+
import { resetGlobalAgentPool } from '../utils/agent-pool.js';
|
|
13
14
|
String.prototype.addBaseUrl = function(url) {
|
|
14
15
|
url = url instanceof URL ? url.href : url;
|
|
15
16
|
const html = this.replace(/<base\b[^>]*?>/gi, "");
|
|
@@ -94,6 +95,10 @@ export class Crawler {
|
|
|
94
95
|
}).then((storage) => {
|
|
95
96
|
this.cacher = storage;
|
|
96
97
|
this.isCacheReady = true;
|
|
98
|
+
}).catch((err) => {
|
|
99
|
+
if (this.config.debug)
|
|
100
|
+
console.warn("[Crawler] Failed to initialize cache:", err);
|
|
101
|
+
this.isCacheReady = true;
|
|
97
102
|
});
|
|
98
103
|
const dit = path.resolve(cacheDir, "urls");
|
|
99
104
|
if (!fs.existsSync(dit))
|
|
@@ -105,6 +110,10 @@ export class Crawler {
|
|
|
105
110
|
}).then((storage) => {
|
|
106
111
|
this.urlStorage = storage;
|
|
107
112
|
this.isStorageReady = true;
|
|
113
|
+
}).catch((err) => {
|
|
114
|
+
if (this.config.debug)
|
|
115
|
+
console.warn("[Crawler] Failed to initialize URL storage:", err);
|
|
116
|
+
this.isStorageReady = true;
|
|
108
117
|
});
|
|
109
118
|
} else {
|
|
110
119
|
const dit = path.resolve(this.config.cacheDir, "./cache/urls");
|
|
@@ -117,6 +126,10 @@ export class Crawler {
|
|
|
117
126
|
}).then((storage) => {
|
|
118
127
|
this.urlStorage = storage;
|
|
119
128
|
this.isStorageReady = true;
|
|
129
|
+
}).catch((err) => {
|
|
130
|
+
if (this.config.debug)
|
|
131
|
+
console.warn("[Crawler] Failed to initialize URL storage:", err);
|
|
132
|
+
this.isStorageReady = true;
|
|
120
133
|
});
|
|
121
134
|
}
|
|
122
135
|
if (this.config.enableNavigationHistory) {
|
|
@@ -856,12 +869,15 @@ export class Crawler {
|
|
|
856
869
|
}
|
|
857
870
|
async execute(method, url, body, options = {}, isEmail, forceRevisit, oxylabsOptions, oxylabsInstanse, decodoInstanse, decodoOptions) {
|
|
858
871
|
await this.waitForStorage();
|
|
872
|
+
console.log("Waiting for storage...");
|
|
859
873
|
if (this.isCacheEnabled) {
|
|
860
874
|
await this.waitForCache();
|
|
861
875
|
}
|
|
876
|
+
console.log("Waiting for cache...");
|
|
862
877
|
if (this.config.enableNavigationHistory) {
|
|
863
878
|
await this.waitForNavigationHistory();
|
|
864
879
|
}
|
|
880
|
+
console.log("Waiting for navigation history...");
|
|
865
881
|
const task = this.queue.add(() => this.executeHttp(method, url, body, options, isEmail, forceRevisit, oxylabsOptions, oxylabsInstanse, decodoInstanse, decodoOptions));
|
|
866
882
|
task.finally(() => this.pendingExecutions.delete(task));
|
|
867
883
|
}
|
|
@@ -888,7 +904,9 @@ export class Crawler {
|
|
|
888
904
|
}
|
|
889
905
|
async executeHttp(method, url, body, options = {}, isEmail, forceRevisit, oxylabsOptions, oxylabsInstanse, decodoInstanse, decodoOptions, retryCount = 0, parentUrl) {
|
|
890
906
|
try {
|
|
907
|
+
console.log("Triggering start handlers...");
|
|
891
908
|
await this.triggerStartHandlers();
|
|
909
|
+
console.log("Checking crawl limits...");
|
|
892
910
|
const limitCheck = await this.checkCrawlLimits(url, parentUrl);
|
|
893
911
|
if (!limitCheck.allowed) {
|
|
894
912
|
if (this.config.debug) {
|
|
@@ -903,13 +921,16 @@ export class Crawler {
|
|
|
903
921
|
await new Promise((resolve) => setTimeout(resolve, delay));
|
|
904
922
|
}
|
|
905
923
|
const isVisited = forceRevisit ? false : await this.hasUrlInCache(url);
|
|
906
|
-
const cache = await this.getCache(url);
|
|
907
|
-
if (
|
|
908
|
-
|
|
909
|
-
|
|
910
|
-
|
|
924
|
+
const cache = method.toLowerCase() === "get" ? await this.getCache(url) : undefined;
|
|
925
|
+
if (method.toLowerCase() === "get") {
|
|
926
|
+
if (isVisited && !cache)
|
|
927
|
+
return;
|
|
928
|
+
if (isVisited)
|
|
929
|
+
return;
|
|
930
|
+
}
|
|
911
931
|
const requestStartTime = Date.now();
|
|
912
932
|
const response = cache && method === "GET" ? cache : oxylabsInstanse && oxylabsOptions ? await oxylabsInstanse.scrape(url) : decodoInstanse && decodoOptions ? await decodoInstanse.scrape(url) : await (method === "GET" ? this.http.get(url, options) : method === "PATCH" ? this.http.patch(url, body, options) : method === "POST" ? this.http.post(url, body, options) : this.http.put(url, body, options));
|
|
933
|
+
console.log("Response received...");
|
|
913
934
|
if (!cache) {
|
|
914
935
|
const responseTime = Date.now() - requestStartTime;
|
|
915
936
|
this.calculateAutoThrottleDelay(domain, responseTime);
|
|
@@ -943,9 +964,9 @@ export class Crawler {
|
|
|
943
964
|
statusCode: res.status
|
|
944
965
|
});
|
|
945
966
|
}
|
|
946
|
-
if (!cache)
|
|
967
|
+
if (!cache && method === "GET")
|
|
947
968
|
await this.saveCache(url, res);
|
|
948
|
-
if (!isVisited)
|
|
969
|
+
if (!isVisited && method === "GET")
|
|
949
970
|
await this.saveUrl(url);
|
|
950
971
|
await this.markUrlVisited(url, {
|
|
951
972
|
status: res.status,
|
|
@@ -1039,6 +1060,7 @@ export class Crawler {
|
|
|
1039
1060
|
}
|
|
1040
1061
|
await this.queue.onIdle();
|
|
1041
1062
|
await this.triggerFinishHandlers();
|
|
1063
|
+
await this.destroy();
|
|
1042
1064
|
}
|
|
1043
1065
|
async done() {
|
|
1044
1066
|
return this.waitForAll();
|
|
@@ -1055,7 +1077,7 @@ export class Crawler {
|
|
|
1055
1077
|
} catch {}
|
|
1056
1078
|
}
|
|
1057
1079
|
async destroy() {
|
|
1058
|
-
this.queue.
|
|
1080
|
+
this.queue.destroy();
|
|
1059
1081
|
this.events.length = 0;
|
|
1060
1082
|
this.jsonEvents.length = 0;
|
|
1061
1083
|
this.errorEvents.length = 0;
|
|
@@ -1064,5 +1086,6 @@ export class Crawler {
|
|
|
1064
1086
|
this.emailDiscoveredEvents.length = 0;
|
|
1065
1087
|
this.emailLeadsEvents.length = 0;
|
|
1066
1088
|
await this.close();
|
|
1089
|
+
resetGlobalAgentPool();
|
|
1067
1090
|
}
|
|
1068
1091
|
}
|
package/dist/crawler/index.cjs
CHANGED
|
@@ -1,40 +1,40 @@
|
|
|
1
|
-
const
|
|
2
|
-
exports.Crawler =
|
|
3
|
-
const
|
|
4
|
-
exports.CrawlerOptions =
|
|
5
|
-
const
|
|
6
|
-
exports.RobotsTxt =
|
|
7
|
-
const
|
|
8
|
-
exports.FileCacher =
|
|
9
|
-
const
|
|
10
|
-
exports.UrlStore =
|
|
11
|
-
const
|
|
12
|
-
exports.NavigationHistory =
|
|
13
|
-
const
|
|
14
|
-
exports.Oxylabs =
|
|
15
|
-
const
|
|
16
|
-
exports.OXYLABS_BROWSER_TYPES =
|
|
17
|
-
exports.OXYLABS_COMMON_LOCALES =
|
|
18
|
-
exports.OXYLABS_COMMON_GEO_LOCATIONS =
|
|
19
|
-
exports.OXYLABS_US_STATES =
|
|
20
|
-
exports.OXYLABS_EUROPEAN_COUNTRIES =
|
|
21
|
-
exports.OXYLABS_ASIAN_COUNTRIES =
|
|
22
|
-
exports.getRandomOxylabsBrowserType =
|
|
23
|
-
exports.getRandomOxylabsLocale =
|
|
24
|
-
exports.getRandomOxylabsGeoLocation =
|
|
25
|
-
const
|
|
26
|
-
exports.Decodo =
|
|
27
|
-
const
|
|
28
|
-
exports.DECODO_DEVICE_TYPES =
|
|
29
|
-
exports.DECODO_HEADLESS_MODES =
|
|
30
|
-
exports.DECODO_COMMON_LOCALES =
|
|
31
|
-
exports.DECODO_COMMON_COUNTRIES =
|
|
32
|
-
exports.DECODO_EUROPEAN_COUNTRIES =
|
|
33
|
-
exports.DECODO_ASIAN_COUNTRIES =
|
|
34
|
-
exports.DECODO_US_STATES =
|
|
35
|
-
exports.DECODO_COMMON_CITIES =
|
|
36
|
-
exports.getRandomDecodoDeviceType =
|
|
37
|
-
exports.getRandomDecodoLocale =
|
|
38
|
-
exports.getRandomDecodoCountry =
|
|
39
|
-
exports.getRandomDecodoCity =
|
|
40
|
-
exports.generateDecodoSessionId =
|
|
1
|
+
const _mod_oe2h32 = require('./crawler.cjs');
|
|
2
|
+
exports.Crawler = _mod_oe2h32.Crawler;;
|
|
3
|
+
const _mod_ask6qa = require('./crawler-options.cjs');
|
|
4
|
+
exports.CrawlerOptions = _mod_ask6qa.CrawlerOptions;;
|
|
5
|
+
const _mod_olcke0 = require('./plugin/robots-txt.cjs');
|
|
6
|
+
exports.RobotsTxt = _mod_olcke0.RobotsTxt;;
|
|
7
|
+
const _mod_4422cp = require('./plugin/file-cacher.cjs');
|
|
8
|
+
exports.FileCacher = _mod_4422cp.FileCacher;;
|
|
9
|
+
const _mod_vrdxs5 = require('./plugin/url-store.cjs');
|
|
10
|
+
exports.UrlStore = _mod_vrdxs5.UrlStore;;
|
|
11
|
+
const _mod_0n4r2a = require('./plugin/navigation-history.cjs');
|
|
12
|
+
exports.NavigationHistory = _mod_0n4r2a.NavigationHistory;;
|
|
13
|
+
const _mod_ttzg9i = require('./addon/oxylabs/index.cjs');
|
|
14
|
+
exports.Oxylabs = _mod_ttzg9i.Oxylabs;;
|
|
15
|
+
const _mod_qztowj = require('./addon/oxylabs/options.cjs');
|
|
16
|
+
exports.OXYLABS_BROWSER_TYPES = _mod_qztowj.OXYLABS_BROWSER_TYPES;
|
|
17
|
+
exports.OXYLABS_COMMON_LOCALES = _mod_qztowj.OXYLABS_COMMON_LOCALES;
|
|
18
|
+
exports.OXYLABS_COMMON_GEO_LOCATIONS = _mod_qztowj.OXYLABS_COMMON_GEO_LOCATIONS;
|
|
19
|
+
exports.OXYLABS_US_STATES = _mod_qztowj.OXYLABS_US_STATES;
|
|
20
|
+
exports.OXYLABS_EUROPEAN_COUNTRIES = _mod_qztowj.OXYLABS_EUROPEAN_COUNTRIES;
|
|
21
|
+
exports.OXYLABS_ASIAN_COUNTRIES = _mod_qztowj.OXYLABS_ASIAN_COUNTRIES;
|
|
22
|
+
exports.getRandomOxylabsBrowserType = _mod_qztowj.getRandomBrowserType;
|
|
23
|
+
exports.getRandomOxylabsLocale = _mod_qztowj.getRandomLocale;
|
|
24
|
+
exports.getRandomOxylabsGeoLocation = _mod_qztowj.getRandomGeoLocation;;
|
|
25
|
+
const _mod_vf4tt7 = require('./addon/decodo/index.cjs');
|
|
26
|
+
exports.Decodo = _mod_vf4tt7.Decodo;;
|
|
27
|
+
const _mod_5ztbfj = require('./addon/decodo/options.cjs');
|
|
28
|
+
exports.DECODO_DEVICE_TYPES = _mod_5ztbfj.DECODO_DEVICE_TYPES;
|
|
29
|
+
exports.DECODO_HEADLESS_MODES = _mod_5ztbfj.DECODO_HEADLESS_MODES;
|
|
30
|
+
exports.DECODO_COMMON_LOCALES = _mod_5ztbfj.DECODO_COMMON_LOCALES;
|
|
31
|
+
exports.DECODO_COMMON_COUNTRIES = _mod_5ztbfj.DECODO_COMMON_COUNTRIES;
|
|
32
|
+
exports.DECODO_EUROPEAN_COUNTRIES = _mod_5ztbfj.DECODO_EUROPEAN_COUNTRIES;
|
|
33
|
+
exports.DECODO_ASIAN_COUNTRIES = _mod_5ztbfj.DECODO_ASIAN_COUNTRIES;
|
|
34
|
+
exports.DECODO_US_STATES = _mod_5ztbfj.DECODO_US_STATES;
|
|
35
|
+
exports.DECODO_COMMON_CITIES = _mod_5ztbfj.DECODO_COMMON_CITIES;
|
|
36
|
+
exports.getRandomDecodoDeviceType = _mod_5ztbfj.getRandomDeviceType;
|
|
37
|
+
exports.getRandomDecodoLocale = _mod_5ztbfj.getRandomLocale;
|
|
38
|
+
exports.getRandomDecodoCountry = _mod_5ztbfj.getRandomCountry;
|
|
39
|
+
exports.getRandomDecodoCity = _mod_5ztbfj.getRandomCity;
|
|
40
|
+
exports.generateDecodoSessionId = _mod_5ztbfj.generateSessionId;;
|
package/dist/entries/crawler.cjs
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
const
|
|
2
|
-
exports.Crawler =
|
|
3
|
-
const
|
|
4
|
-
exports.CrawlerOptions =
|
|
5
|
-
exports.Domain =
|
|
1
|
+
const _mod_anwt38 = require('../crawler/crawler.cjs');
|
|
2
|
+
exports.Crawler = _mod_anwt38.Crawler;;
|
|
3
|
+
const _mod_uvsh93 = require('../crawler/crawler-options.cjs');
|
|
4
|
+
exports.CrawlerOptions = _mod_uvsh93.CrawlerOptions;
|
|
5
|
+
exports.Domain = _mod_uvsh93.Domain;;
|
package/dist/index.cjs
CHANGED
|
@@ -1,30 +1,30 @@
|
|
|
1
|
-
const
|
|
2
|
-
exports.Rezo =
|
|
3
|
-
exports.createRezoInstance =
|
|
4
|
-
exports.createDefaultInstance =
|
|
5
|
-
const
|
|
6
|
-
exports.RezoError =
|
|
7
|
-
exports.RezoErrorCode =
|
|
8
|
-
const
|
|
9
|
-
exports.RezoHeaders =
|
|
10
|
-
const
|
|
11
|
-
exports.RezoFormData =
|
|
12
|
-
const
|
|
13
|
-
exports.RezoCookieJar =
|
|
14
|
-
exports.Cookie =
|
|
15
|
-
const
|
|
16
|
-
exports.toCurl =
|
|
17
|
-
exports.fromCurl =
|
|
18
|
-
const
|
|
19
|
-
exports.createDefaultHooks =
|
|
20
|
-
exports.mergeHooks =
|
|
21
|
-
const
|
|
22
|
-
exports.ProxyManager =
|
|
23
|
-
const
|
|
24
|
-
exports.RezoQueue =
|
|
25
|
-
exports.HttpQueue =
|
|
26
|
-
exports.Priority =
|
|
27
|
-
exports.HttpMethodPriority =
|
|
1
|
+
const _mod_piprwa = require('./core/rezo.cjs');
|
|
2
|
+
exports.Rezo = _mod_piprwa.Rezo;
|
|
3
|
+
exports.createRezoInstance = _mod_piprwa.createRezoInstance;
|
|
4
|
+
exports.createDefaultInstance = _mod_piprwa.createDefaultInstance;;
|
|
5
|
+
const _mod_ily76y = require('./errors/rezo-error.cjs');
|
|
6
|
+
exports.RezoError = _mod_ily76y.RezoError;
|
|
7
|
+
exports.RezoErrorCode = _mod_ily76y.RezoErrorCode;;
|
|
8
|
+
const _mod_q4s6h5 = require('./utils/headers.cjs');
|
|
9
|
+
exports.RezoHeaders = _mod_q4s6h5.RezoHeaders;;
|
|
10
|
+
const _mod_lhxcz7 = require('./utils/form-data.cjs');
|
|
11
|
+
exports.RezoFormData = _mod_lhxcz7.RezoFormData;;
|
|
12
|
+
const _mod_ae3n62 = require('./utils/cookies.cjs');
|
|
13
|
+
exports.RezoCookieJar = _mod_ae3n62.RezoCookieJar;
|
|
14
|
+
exports.Cookie = _mod_ae3n62.Cookie;;
|
|
15
|
+
const _mod_c0ipkb = require('./utils/curl.cjs');
|
|
16
|
+
exports.toCurl = _mod_c0ipkb.toCurl;
|
|
17
|
+
exports.fromCurl = _mod_c0ipkb.fromCurl;;
|
|
18
|
+
const _mod_b2lvov = require('./core/hooks.cjs');
|
|
19
|
+
exports.createDefaultHooks = _mod_b2lvov.createDefaultHooks;
|
|
20
|
+
exports.mergeHooks = _mod_b2lvov.mergeHooks;;
|
|
21
|
+
const _mod_7znktr = require('./proxy/manager.cjs');
|
|
22
|
+
exports.ProxyManager = _mod_7znktr.ProxyManager;;
|
|
23
|
+
const _mod_qpvzjb = require('./queue/index.cjs');
|
|
24
|
+
exports.RezoQueue = _mod_qpvzjb.RezoQueue;
|
|
25
|
+
exports.HttpQueue = _mod_qpvzjb.HttpQueue;
|
|
26
|
+
exports.Priority = _mod_qpvzjb.Priority;
|
|
27
|
+
exports.HttpMethodPriority = _mod_qpvzjb.HttpMethodPriority;;
|
|
28
28
|
const { RezoError } = require('./errors/rezo-error.cjs');
|
|
29
29
|
const isRezoError = exports.isRezoError = RezoError.isRezoError;
|
|
30
30
|
const Cancel = exports.Cancel = RezoError;
|
|
@@ -1,10 +1,10 @@
|
|
|
1
|
-
const
|
|
2
|
-
exports.Agent =
|
|
3
|
-
const
|
|
4
|
-
exports.HttpProxyAgent =
|
|
5
|
-
const
|
|
6
|
-
exports.HttpsProxyAgent =
|
|
7
|
-
const
|
|
8
|
-
exports.SocksProxyAgent =
|
|
9
|
-
const
|
|
10
|
-
exports.SocksClient =
|
|
1
|
+
const _mod_hw46n8 = require('./base.cjs');
|
|
2
|
+
exports.Agent = _mod_hw46n8.Agent;;
|
|
3
|
+
const _mod_an9r9f = require('./http-proxy.cjs');
|
|
4
|
+
exports.HttpProxyAgent = _mod_an9r9f.HttpProxyAgent;;
|
|
5
|
+
const _mod_6u3c2s = require('./https-proxy.cjs');
|
|
6
|
+
exports.HttpsProxyAgent = _mod_6u3c2s.HttpsProxyAgent;;
|
|
7
|
+
const _mod_06xizn = require('./socks-proxy.cjs');
|
|
8
|
+
exports.SocksProxyAgent = _mod_06xizn.SocksProxyAgent;;
|
|
9
|
+
const _mod_ap8s94 = require('./socks-client.cjs');
|
|
10
|
+
exports.SocksClient = _mod_ap8s94.SocksClient;;
|
package/dist/proxy/index.cjs
CHANGED
|
@@ -1,9 +1,9 @@
|
|
|
1
1
|
const { Agent, HttpProxyAgent, HttpsProxyAgent, SocksProxyAgent } = require('../internal/agents/index.cjs');
|
|
2
2
|
const { parseProxyString } = require('./parse.cjs');
|
|
3
|
-
const
|
|
4
|
-
exports.ProxyManager =
|
|
5
|
-
const
|
|
6
|
-
exports.parseProxyString =
|
|
3
|
+
const _mod_pfdaeo = require('./manager.cjs');
|
|
4
|
+
exports.ProxyManager = _mod_pfdaeo.ProxyManager;;
|
|
5
|
+
const _mod_9wdrd3 = require('./parse.cjs');
|
|
6
|
+
exports.parseProxyString = _mod_9wdrd3.parseProxyString;;
|
|
7
7
|
function createOptions(uri, opts) {
|
|
8
8
|
if (uri instanceof URL || typeof uri === "string") {
|
|
9
9
|
return {
|
package/dist/queue/index.cjs
CHANGED
|
@@ -1,8 +1,8 @@
|
|
|
1
|
-
const
|
|
2
|
-
exports.RezoQueue =
|
|
3
|
-
const
|
|
4
|
-
exports.HttpQueue =
|
|
5
|
-
exports.extractDomain =
|
|
6
|
-
const
|
|
7
|
-
exports.Priority =
|
|
8
|
-
exports.HttpMethodPriority =
|
|
1
|
+
const _mod_9zllky = require('./queue.cjs');
|
|
2
|
+
exports.RezoQueue = _mod_9zllky.RezoQueue;;
|
|
3
|
+
const _mod_acgz0f = require('./http-queue.cjs');
|
|
4
|
+
exports.HttpQueue = _mod_acgz0f.HttpQueue;
|
|
5
|
+
exports.extractDomain = _mod_acgz0f.extractDomain;;
|
|
6
|
+
const _mod_a6xcwz = require('./types.cjs');
|
|
7
|
+
exports.Priority = _mod_a6xcwz.Priority;
|
|
8
|
+
exports.HttpMethodPriority = _mod_a6xcwz.HttpMethodPriority;;
|
|
@@ -1,11 +1,11 @@
|
|
|
1
|
-
const
|
|
2
|
-
exports.UniversalEventEmitter =
|
|
3
|
-
const
|
|
4
|
-
exports.UniversalStreamResponse =
|
|
5
|
-
exports.StreamResponse =
|
|
6
|
-
const
|
|
7
|
-
exports.UniversalDownloadResponse =
|
|
8
|
-
exports.DownloadResponse =
|
|
9
|
-
const
|
|
10
|
-
exports.UniversalUploadResponse =
|
|
11
|
-
exports.UploadResponse =
|
|
1
|
+
const _mod_ylkelc = require('./event-emitter.cjs');
|
|
2
|
+
exports.UniversalEventEmitter = _mod_ylkelc.UniversalEventEmitter;;
|
|
3
|
+
const _mod_ngrtq8 = require('./stream.cjs');
|
|
4
|
+
exports.UniversalStreamResponse = _mod_ngrtq8.UniversalStreamResponse;
|
|
5
|
+
exports.StreamResponse = _mod_ngrtq8.StreamResponse;;
|
|
6
|
+
const _mod_4er8ge = require('./download.cjs');
|
|
7
|
+
exports.UniversalDownloadResponse = _mod_4er8ge.UniversalDownloadResponse;
|
|
8
|
+
exports.DownloadResponse = _mod_4er8ge.DownloadResponse;;
|
|
9
|
+
const _mod_mxbnd3 = require('./upload.cjs');
|
|
10
|
+
exports.UniversalUploadResponse = _mod_mxbnd3.UniversalUploadResponse;
|
|
11
|
+
exports.UploadResponse = _mod_mxbnd3.UploadResponse;;
|
|
@@ -1,23 +1,4 @@
|
|
|
1
|
-
const { createGunzip, createInflate, createBrotliDecompress } = require("node:zlib");
|
|
2
|
-
const { Transform } = require("node:stream");
|
|
3
|
-
let createZstdDecompress = null;
|
|
4
|
-
let zstdAvailable = null;
|
|
5
|
-
function getZstdDecompressor() {
|
|
6
|
-
if (zstdAvailable === null) {
|
|
7
|
-
try {
|
|
8
|
-
const zlib = require("node:zlib");
|
|
9
|
-
if (typeof zlib.createZstdDecompress === "function") {
|
|
10
|
-
createZstdDecompress = zlib.createZstdDecompress;
|
|
11
|
-
zstdAvailable = true;
|
|
12
|
-
} else {
|
|
13
|
-
zstdAvailable = false;
|
|
14
|
-
}
|
|
15
|
-
} catch {
|
|
16
|
-
zstdAvailable = false;
|
|
17
|
-
}
|
|
18
|
-
}
|
|
19
|
-
return createZstdDecompress;
|
|
20
|
-
}
|
|
1
|
+
const { createGunzip, createInflate, createBrotliDecompress, createZstdDecompress } = require("node:zlib");
|
|
21
2
|
|
|
22
3
|
class CompressionUtil {
|
|
23
4
|
static decompressStream(response, contentEncoding, config) {
|
|
@@ -41,7 +22,7 @@ class CompressionUtil {
|
|
|
41
22
|
case "brotli":
|
|
42
23
|
return response.pipe(createBrotliDecompress());
|
|
43
24
|
case "zstd":
|
|
44
|
-
return
|
|
25
|
+
return response.pipe(createZstdDecompress());
|
|
45
26
|
default:
|
|
46
27
|
return response;
|
|
47
28
|
}
|
|
@@ -61,18 +42,6 @@ class CompressionUtil {
|
|
|
61
42
|
}
|
|
62
43
|
return true;
|
|
63
44
|
}
|
|
64
|
-
static createZstdDecompressStream(response) {
|
|
65
|
-
const decompressor = getZstdDecompressor();
|
|
66
|
-
if (!decompressor) {
|
|
67
|
-
const passthrough = new Transform({
|
|
68
|
-
transform(chunk, encoding, callback) {
|
|
69
|
-
callback(new Error("zstd decompression not available: requires Node.js 22.15+ with native zstd support"));
|
|
70
|
-
}
|
|
71
|
-
});
|
|
72
|
-
return response.pipe(passthrough);
|
|
73
|
-
}
|
|
74
|
-
return response.pipe(decompressor());
|
|
75
|
-
}
|
|
76
45
|
static getSupportedAlgorithms() {
|
|
77
46
|
return ["gzip", "x-gzip", "deflate", "x-deflate", "gzip-raw", "br", "brotli", "zstd"];
|
|
78
47
|
}
|
|
@@ -1,23 +1,4 @@
|
|
|
1
|
-
import { createGunzip, createInflate, createBrotliDecompress } from "node:zlib";
|
|
2
|
-
import { Transform } from "node:stream";
|
|
3
|
-
let createZstdDecompress = null;
|
|
4
|
-
let zstdAvailable = null;
|
|
5
|
-
function getZstdDecompressor() {
|
|
6
|
-
if (zstdAvailable === null) {
|
|
7
|
-
try {
|
|
8
|
-
const zlib = require("node:zlib");
|
|
9
|
-
if (typeof zlib.createZstdDecompress === "function") {
|
|
10
|
-
createZstdDecompress = zlib.createZstdDecompress;
|
|
11
|
-
zstdAvailable = true;
|
|
12
|
-
} else {
|
|
13
|
-
zstdAvailable = false;
|
|
14
|
-
}
|
|
15
|
-
} catch {
|
|
16
|
-
zstdAvailable = false;
|
|
17
|
-
}
|
|
18
|
-
}
|
|
19
|
-
return createZstdDecompress;
|
|
20
|
-
}
|
|
1
|
+
import { createGunzip, createInflate, createBrotliDecompress, createZstdDecompress } from "node:zlib";
|
|
21
2
|
|
|
22
3
|
export class CompressionUtil {
|
|
23
4
|
static decompressStream(response, contentEncoding, config) {
|
|
@@ -41,7 +22,7 @@ export class CompressionUtil {
|
|
|
41
22
|
case "brotli":
|
|
42
23
|
return response.pipe(createBrotliDecompress());
|
|
43
24
|
case "zstd":
|
|
44
|
-
return
|
|
25
|
+
return response.pipe(createZstdDecompress());
|
|
45
26
|
default:
|
|
46
27
|
return response;
|
|
47
28
|
}
|
|
@@ -61,18 +42,6 @@ export class CompressionUtil {
|
|
|
61
42
|
}
|
|
62
43
|
return true;
|
|
63
44
|
}
|
|
64
|
-
static createZstdDecompressStream(response) {
|
|
65
|
-
const decompressor = getZstdDecompressor();
|
|
66
|
-
if (!decompressor) {
|
|
67
|
-
const passthrough = new Transform({
|
|
68
|
-
transform(chunk, encoding, callback) {
|
|
69
|
-
callback(new Error("zstd decompression not available: requires Node.js 22.15+ with native zstd support"));
|
|
70
|
-
}
|
|
71
|
-
});
|
|
72
|
-
return response.pipe(passthrough);
|
|
73
|
-
}
|
|
74
|
-
return response.pipe(decompressor());
|
|
75
|
-
}
|
|
76
45
|
static getSupportedAlgorithms() {
|
|
77
46
|
return ["gzip", "x-gzip", "deflate", "x-deflate", "gzip-raw", "br", "brotli", "zstd"];
|
|
78
47
|
}
|
|
@@ -173,7 +173,7 @@ function setSignal() {
|
|
|
173
173
|
clearTimeout(this.timeoutClearInstanse);
|
|
174
174
|
if (this.timeout && typeof this.timeout === "number" && this.timeout > 100) {
|
|
175
175
|
const controller = new AbortController;
|
|
176
|
-
const timer = setTimeout(() => controller.abort(), this.timeout);
|
|
176
|
+
const timer = setTimeout(() => controller.abort(), this.timeout).unref();
|
|
177
177
|
this.timeoutClearInstanse = timer;
|
|
178
178
|
this.signal = controller.signal;
|
|
179
179
|
}
|
|
@@ -173,7 +173,7 @@ function setSignal() {
|
|
|
173
173
|
clearTimeout(this.timeoutClearInstanse);
|
|
174
174
|
if (this.timeout && typeof this.timeout === "number" && this.timeout > 100) {
|
|
175
175
|
const controller = new AbortController;
|
|
176
|
-
const timer = setTimeout(() => controller.abort(), this.timeout);
|
|
176
|
+
const timer = setTimeout(() => controller.abort(), this.timeout).unref();
|
|
177
177
|
this.timeoutClearInstanse = timer;
|
|
178
178
|
this.signal = controller.signal;
|
|
179
179
|
}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "rezo",
|
|
3
|
-
"version": "1.0.
|
|
3
|
+
"version": "1.0.51",
|
|
4
4
|
"description": "Lightning-fast, enterprise-grade HTTP client for modern JavaScript. Full HTTP/2 support, intelligent cookie management, multiple adapters (HTTP, Fetch, cURL, XHR), streaming, proxy support (HTTP/HTTPS/SOCKS), and cross-environment compatibility.",
|
|
5
5
|
"main": "dist/index.cjs",
|
|
6
6
|
"module": "dist/index.js",
|