rezo 1.0.72 → 1.0.74
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/adapters/entries/curl.d.ts +13 -2
- package/dist/adapters/entries/fetch.d.ts +13 -2
- package/dist/adapters/entries/http.d.ts +13 -2
- package/dist/adapters/entries/http2.d.ts +13 -2
- package/dist/adapters/entries/react-native.d.ts +13 -2
- package/dist/adapters/entries/xhr.d.ts +13 -2
- package/dist/adapters/index.cjs +6 -6
- package/dist/cache/index.cjs +9 -9
- package/dist/crawler/crawler-options.cjs +1 -1
- package/dist/crawler/crawler-options.js +1 -1
- package/dist/crawler/crawler.cjs +320 -89
- package/dist/crawler/crawler.js +320 -89
- package/dist/crawler/index.cjs +40 -40
- package/dist/crawler/plugin/capped-array.cjs +1 -0
- package/dist/crawler/plugin/capped-array.js +1 -0
- package/dist/crawler/plugin/capped-map.cjs +1 -0
- package/dist/crawler/plugin/capped-map.js +1 -0
- package/dist/crawler/plugin/file-cacher.cjs +20 -18
- package/dist/crawler/plugin/file-cacher.js +20 -18
- package/dist/crawler/plugin/health-metrics.cjs +2 -0
- package/dist/crawler/plugin/health-metrics.js +2 -0
- package/dist/crawler/plugin/index.cjs +1 -1
- package/dist/crawler/plugin/index.js +1 -1
- package/dist/crawler/plugin/memory-monitor.cjs +1 -0
- package/dist/crawler/plugin/memory-monitor.js +1 -0
- package/dist/crawler/plugin/navigation-history.cjs +5 -5
- package/dist/crawler/plugin/navigation-history.js +3 -3
- package/dist/crawler/plugin/result-stream.cjs +5 -0
- package/dist/crawler/plugin/result-stream.js +5 -0
- package/dist/crawler/plugin/sqlite-utils.cjs +1 -0
- package/dist/crawler/plugin/sqlite-utils.js +1 -0
- package/dist/crawler/plugin/url-store.cjs +5 -5
- package/dist/crawler/plugin/url-store.js +5 -5
- package/dist/crawler/scraper.cjs +1 -1
- package/dist/crawler/scraper.js +1 -1
- package/dist/crawler.d.ts +152 -25
- package/dist/entries/crawler.cjs +4 -4
- package/dist/errors/rezo-error.cjs +3 -72
- package/dist/errors/rezo-error.js +3 -72
- package/dist/index.cjs +30 -30
- package/dist/index.d.ts +13 -2
- package/dist/internal/agents/index.cjs +10 -10
- package/dist/platform/browser.d.ts +13 -2
- package/dist/platform/bun.d.ts +13 -2
- package/dist/platform/deno.d.ts +13 -2
- package/dist/platform/node.d.ts +13 -2
- package/dist/platform/react-native.d.ts +13 -2
- package/dist/platform/worker.d.ts +13 -2
- package/dist/proxy/index.cjs +4 -4
- package/dist/queue/index.cjs +8 -8
- package/dist/queue/queue.cjs +58 -13
- package/dist/queue/queue.js +58 -13
- package/dist/responses/universal/index.cjs +11 -11
- package/dist/utils/agent-pool.cjs +37 -0
- package/dist/utils/agent-pool.js +37 -0
- package/dist/version.cjs +1 -1
- package/dist/version.js +1 -1
- package/dist/wget/index.cjs +49 -49
- package/dist/wget/index.d.ts +12 -1
- package/package.json +1 -1
package/dist/utils/agent-pool.js
CHANGED
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import * as http from "node:http";
|
|
2
2
|
import * as https from "node:https";
|
|
3
|
+
import * as tls from "node:tls";
|
|
3
4
|
import { getGlobalDNSCache } from '../cache/dns-cache.js';
|
|
4
5
|
const DEFAULT_CONFIG = {
|
|
5
6
|
keepAlive: true,
|
|
@@ -79,6 +80,39 @@ class AgentPool {
|
|
|
79
80
|
});
|
|
80
81
|
}
|
|
81
82
|
createHttpsAgent(key, tlsOptions) {
|
|
83
|
+
const secureContext = tls.createSecureContext({
|
|
84
|
+
ecdhCurve: "X25519:prime256v1:secp384r1",
|
|
85
|
+
ciphers: [
|
|
86
|
+
"TLS_AES_128_GCM_SHA256",
|
|
87
|
+
"TLS_AES_256_GCM_SHA384",
|
|
88
|
+
"TLS_CHACHA20_POLY1305_SHA256",
|
|
89
|
+
"ECDHE-ECDSA-AES128-GCM-SHA256",
|
|
90
|
+
"ECDHE-RSA-AES128-GCM-SHA256",
|
|
91
|
+
"ECDHE-ECDSA-AES256-GCM-SHA384",
|
|
92
|
+
"ECDHE-RSA-AES256-GCM-SHA384",
|
|
93
|
+
"ECDHE-ECDSA-CHACHA20-POLY1305",
|
|
94
|
+
"ECDHE-RSA-CHACHA20-POLY1305",
|
|
95
|
+
"ECDHE-RSA-AES128-SHA",
|
|
96
|
+
"ECDHE-RSA-AES256-SHA",
|
|
97
|
+
"AES128-GCM-SHA256",
|
|
98
|
+
"AES256-GCM-SHA384",
|
|
99
|
+
"AES128-SHA",
|
|
100
|
+
"AES256-SHA"
|
|
101
|
+
].join(":"),
|
|
102
|
+
sigalgs: [
|
|
103
|
+
"ecdsa_secp256r1_sha256",
|
|
104
|
+
"ecdsa_secp384r1_sha384",
|
|
105
|
+
"rsa_pss_rsae_sha256",
|
|
106
|
+
"rsa_pss_rsae_sha384",
|
|
107
|
+
"rsa_pss_rsae_sha512",
|
|
108
|
+
"rsa_pkcs1_sha256",
|
|
109
|
+
"rsa_pkcs1_sha384",
|
|
110
|
+
"rsa_pkcs1_sha512"
|
|
111
|
+
].join(":"),
|
|
112
|
+
minVersion: "TLSv1.2",
|
|
113
|
+
maxVersion: "TLSv1.3",
|
|
114
|
+
sessionTimeout: 3600
|
|
115
|
+
});
|
|
82
116
|
const agentOptions = {
|
|
83
117
|
keepAlive: this.config.keepAlive,
|
|
84
118
|
keepAliveMsecs: this.config.keepAliveMsecs,
|
|
@@ -86,6 +120,7 @@ class AgentPool {
|
|
|
86
120
|
maxFreeSockets: this.config.maxFreeSockets,
|
|
87
121
|
timeout: this.config.timeout,
|
|
88
122
|
scheduling: this.config.scheduling,
|
|
123
|
+
secureContext,
|
|
89
124
|
...tlsOptions
|
|
90
125
|
};
|
|
91
126
|
const lookup = this.createLookupFunction();
|
|
@@ -159,10 +194,12 @@ class AgentPool {
|
|
|
159
194
|
this.evictionTimer = null;
|
|
160
195
|
}
|
|
161
196
|
for (const pooled of this.httpAgents.values()) {
|
|
197
|
+
this.destroyAgentSockets(pooled.agent);
|
|
162
198
|
pooled.agent.destroy();
|
|
163
199
|
}
|
|
164
200
|
this.httpAgents.clear();
|
|
165
201
|
for (const pooled of this.httpsAgents.values()) {
|
|
202
|
+
this.destroyAgentSockets(pooled.agent);
|
|
166
203
|
pooled.agent.destroy();
|
|
167
204
|
}
|
|
168
205
|
this.httpsAgents.clear();
|
package/dist/version.cjs
CHANGED
|
@@ -1,2 +1,2 @@
|
|
|
1
|
-
const VERSION = exports.VERSION = "1.0.
|
|
1
|
+
const VERSION = exports.VERSION = "1.0.74";
|
|
2
2
|
const PACKAGE_NAME = exports.PACKAGE_NAME = "rezo";
|
package/dist/version.js
CHANGED
|
@@ -1,2 +1,2 @@
|
|
|
1
|
-
export const VERSION = "1.0.
|
|
1
|
+
export const VERSION = "1.0.74";
|
|
2
2
|
export const PACKAGE_NAME = "rezo";
|
package/dist/wget/index.cjs
CHANGED
|
@@ -1,52 +1,52 @@
|
|
|
1
|
-
const
|
|
2
|
-
exports.WgetError =
|
|
3
|
-
const
|
|
4
|
-
exports.AssetExtractor =
|
|
5
|
-
const
|
|
6
|
-
exports.UrlFilter =
|
|
7
|
-
const
|
|
8
|
-
exports.FileWriter =
|
|
9
|
-
const
|
|
10
|
-
exports.RobotsHandler =
|
|
11
|
-
const
|
|
12
|
-
exports.ResumeHandler =
|
|
13
|
-
const
|
|
14
|
-
exports.ProgressReporter =
|
|
15
|
-
exports.ProgressTracker =
|
|
16
|
-
exports.parseSize =
|
|
17
|
-
const
|
|
18
|
-
exports.LinkConverter =
|
|
19
|
-
const
|
|
20
|
-
exports.Downloader =
|
|
21
|
-
const
|
|
22
|
-
exports.AssetOrganizer =
|
|
23
|
-
exports.DEFAULT_ASSET_FOLDERS =
|
|
24
|
-
const
|
|
25
|
-
exports.DownloadCache =
|
|
26
|
-
const
|
|
27
|
-
exports.EXECUTABLE_EXTENSIONS =
|
|
28
|
-
exports.ARCHIVE_EXTENSIONS =
|
|
29
|
-
exports.DOCUMENT_EXTENSIONS =
|
|
30
|
-
exports.IMAGE_EXTENSIONS =
|
|
31
|
-
exports.VIDEO_EXTENSIONS =
|
|
32
|
-
exports.AUDIO_EXTENSIONS =
|
|
33
|
-
exports.FONT_EXTENSIONS =
|
|
34
|
-
exports.WEB_ASSET_EXTENSIONS =
|
|
35
|
-
exports.DATA_EXTENSIONS =
|
|
36
|
-
exports.EXECUTABLE_MIME_TYPES =
|
|
37
|
-
exports.ARCHIVE_MIME_TYPES =
|
|
38
|
-
exports.DOCUMENT_MIME_TYPES =
|
|
39
|
-
exports.IMAGE_MIME_TYPES =
|
|
40
|
-
exports.VIDEO_MIME_TYPES =
|
|
41
|
-
exports.AUDIO_MIME_TYPES =
|
|
42
|
-
exports.FONT_MIME_TYPES =
|
|
43
|
-
exports.WEB_ASSET_MIME_TYPES =
|
|
44
|
-
exports.DATA_MIME_TYPES =
|
|
45
|
-
exports.SAFE_WEB_PRESET =
|
|
46
|
-
exports.DOCUMENTS_ONLY_PRESET =
|
|
47
|
-
exports.NO_MEDIA_PRESET =
|
|
48
|
-
exports.MINIMAL_MIRROR_PRESET =
|
|
49
|
-
exports.TEXT_ONLY_PRESET =
|
|
1
|
+
const _mod_8lpt1q = require('./types.cjs');
|
|
2
|
+
exports.WgetError = _mod_8lpt1q.WgetError;;
|
|
3
|
+
const _mod_3knsix = require('./asset-extractor.cjs');
|
|
4
|
+
exports.AssetExtractor = _mod_3knsix.AssetExtractor;;
|
|
5
|
+
const _mod_hmciaa = require('./url-filter.cjs');
|
|
6
|
+
exports.UrlFilter = _mod_hmciaa.UrlFilter;;
|
|
7
|
+
const _mod_25damj = require('./file-writer.cjs');
|
|
8
|
+
exports.FileWriter = _mod_25damj.FileWriter;;
|
|
9
|
+
const _mod_cp8z2y = require('./robots.cjs');
|
|
10
|
+
exports.RobotsHandler = _mod_cp8z2y.RobotsHandler;;
|
|
11
|
+
const _mod_xx31k2 = require('./resume.cjs');
|
|
12
|
+
exports.ResumeHandler = _mod_xx31k2.ResumeHandler;;
|
|
13
|
+
const _mod_xe7f9y = require('./progress.cjs');
|
|
14
|
+
exports.ProgressReporter = _mod_xe7f9y.ProgressReporter;
|
|
15
|
+
exports.ProgressTracker = _mod_xe7f9y.ProgressTracker;
|
|
16
|
+
exports.parseSize = _mod_xe7f9y.parseSize;;
|
|
17
|
+
const _mod_ju89ay = require('./link-converter.cjs');
|
|
18
|
+
exports.LinkConverter = _mod_ju89ay.LinkConverter;;
|
|
19
|
+
const _mod_hypiea = require('./downloader.cjs');
|
|
20
|
+
exports.Downloader = _mod_hypiea.Downloader;;
|
|
21
|
+
const _mod_haffqa = require('./asset-organizer.cjs');
|
|
22
|
+
exports.AssetOrganizer = _mod_haffqa.AssetOrganizer;
|
|
23
|
+
exports.DEFAULT_ASSET_FOLDERS = _mod_haffqa.DEFAULT_ASSET_FOLDERS;;
|
|
24
|
+
const _mod_ej979a = require('./download-cache.cjs');
|
|
25
|
+
exports.DownloadCache = _mod_ej979a.DownloadCache;;
|
|
26
|
+
const _mod_4n02yz = require('./filter-lists.cjs');
|
|
27
|
+
exports.EXECUTABLE_EXTENSIONS = _mod_4n02yz.EXECUTABLE_EXTENSIONS;
|
|
28
|
+
exports.ARCHIVE_EXTENSIONS = _mod_4n02yz.ARCHIVE_EXTENSIONS;
|
|
29
|
+
exports.DOCUMENT_EXTENSIONS = _mod_4n02yz.DOCUMENT_EXTENSIONS;
|
|
30
|
+
exports.IMAGE_EXTENSIONS = _mod_4n02yz.IMAGE_EXTENSIONS;
|
|
31
|
+
exports.VIDEO_EXTENSIONS = _mod_4n02yz.VIDEO_EXTENSIONS;
|
|
32
|
+
exports.AUDIO_EXTENSIONS = _mod_4n02yz.AUDIO_EXTENSIONS;
|
|
33
|
+
exports.FONT_EXTENSIONS = _mod_4n02yz.FONT_EXTENSIONS;
|
|
34
|
+
exports.WEB_ASSET_EXTENSIONS = _mod_4n02yz.WEB_ASSET_EXTENSIONS;
|
|
35
|
+
exports.DATA_EXTENSIONS = _mod_4n02yz.DATA_EXTENSIONS;
|
|
36
|
+
exports.EXECUTABLE_MIME_TYPES = _mod_4n02yz.EXECUTABLE_MIME_TYPES;
|
|
37
|
+
exports.ARCHIVE_MIME_TYPES = _mod_4n02yz.ARCHIVE_MIME_TYPES;
|
|
38
|
+
exports.DOCUMENT_MIME_TYPES = _mod_4n02yz.DOCUMENT_MIME_TYPES;
|
|
39
|
+
exports.IMAGE_MIME_TYPES = _mod_4n02yz.IMAGE_MIME_TYPES;
|
|
40
|
+
exports.VIDEO_MIME_TYPES = _mod_4n02yz.VIDEO_MIME_TYPES;
|
|
41
|
+
exports.AUDIO_MIME_TYPES = _mod_4n02yz.AUDIO_MIME_TYPES;
|
|
42
|
+
exports.FONT_MIME_TYPES = _mod_4n02yz.FONT_MIME_TYPES;
|
|
43
|
+
exports.WEB_ASSET_MIME_TYPES = _mod_4n02yz.WEB_ASSET_MIME_TYPES;
|
|
44
|
+
exports.DATA_MIME_TYPES = _mod_4n02yz.DATA_MIME_TYPES;
|
|
45
|
+
exports.SAFE_WEB_PRESET = _mod_4n02yz.SAFE_WEB_PRESET;
|
|
46
|
+
exports.DOCUMENTS_ONLY_PRESET = _mod_4n02yz.DOCUMENTS_ONLY_PRESET;
|
|
47
|
+
exports.NO_MEDIA_PRESET = _mod_4n02yz.NO_MEDIA_PRESET;
|
|
48
|
+
exports.MINIMAL_MIRROR_PRESET = _mod_4n02yz.MINIMAL_MIRROR_PRESET;
|
|
49
|
+
exports.TEXT_ONLY_PRESET = _mod_4n02yz.TEXT_ONLY_PRESET;;
|
|
50
50
|
const { Downloader } = require('./downloader.cjs');
|
|
51
51
|
const rezo = require('../index.cjs');
|
|
52
52
|
const { promises: fs } = require("node:fs");
|
package/dist/wget/index.d.ts
CHANGED
|
@@ -4718,6 +4718,13 @@ export interface QueueConfig {
|
|
|
4718
4718
|
intervalCap?: number;
|
|
4719
4719
|
/** Carry over unused interval capacity to next interval */
|
|
4720
4720
|
carryoverConcurrencyCount?: boolean;
|
|
4721
|
+
/**
|
|
4722
|
+
* Reject the task promise when an error occurs (default: false)
|
|
4723
|
+
* When false, errors are swallowed and task.resolve(undefined) is called.
|
|
4724
|
+
* This prevents unhandled promise rejections but makes error handling harder.
|
|
4725
|
+
* When true, task.reject(error) is called, allowing proper try/catch handling.
|
|
4726
|
+
*/
|
|
4727
|
+
rejectOnError?: boolean;
|
|
4721
4728
|
}
|
|
4722
4729
|
/**
|
|
4723
4730
|
* Task options when adding to queue
|
|
@@ -4923,8 +4930,12 @@ declare class RezoQueue<T = any> {
|
|
|
4923
4930
|
/**
|
|
4924
4931
|
* Wait for queue size to be less than limit
|
|
4925
4932
|
* @param limit - Size threshold
|
|
4933
|
+
* @param timeoutMs - Optional timeout in milliseconds (default: 0 = no timeout)
|
|
4934
|
+
* If timeout occurs, promise resolves (not rejects) to prevent blocking
|
|
4926
4935
|
*/
|
|
4927
|
-
onSizeLessThan(limit: number): Promise<void>;
|
|
4936
|
+
onSizeLessThan(limit: number, timeoutMs?: number): Promise<void>;
|
|
4937
|
+
/** Maximum recommended handlers per event before warning */
|
|
4938
|
+
private static readonly MAX_HANDLERS_WARNING;
|
|
4928
4939
|
/**
|
|
4929
4940
|
* Register an event handler
|
|
4930
4941
|
* @param event - Event name
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "rezo",
|
|
3
|
-
"version": "1.0.
|
|
3
|
+
"version": "1.0.74",
|
|
4
4
|
"description": "Lightning-fast, enterprise-grade HTTP client for modern JavaScript. Full HTTP/2 support, intelligent cookie management, multiple adapters (HTTP, Fetch, cURL, XHR), streaming, proxy support (HTTP/HTTPS/SOCKS), and cross-environment compatibility.",
|
|
5
5
|
"main": "dist/index.cjs",
|
|
6
6
|
"module": "dist/index.js",
|