rezo 1.0.72 → 1.0.74
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/adapters/entries/curl.d.ts +13 -2
- package/dist/adapters/entries/fetch.d.ts +13 -2
- package/dist/adapters/entries/http.d.ts +13 -2
- package/dist/adapters/entries/http2.d.ts +13 -2
- package/dist/adapters/entries/react-native.d.ts +13 -2
- package/dist/adapters/entries/xhr.d.ts +13 -2
- package/dist/adapters/index.cjs +6 -6
- package/dist/cache/index.cjs +9 -9
- package/dist/crawler/crawler-options.cjs +1 -1
- package/dist/crawler/crawler-options.js +1 -1
- package/dist/crawler/crawler.cjs +320 -89
- package/dist/crawler/crawler.js +320 -89
- package/dist/crawler/index.cjs +40 -40
- package/dist/crawler/plugin/capped-array.cjs +1 -0
- package/dist/crawler/plugin/capped-array.js +1 -0
- package/dist/crawler/plugin/capped-map.cjs +1 -0
- package/dist/crawler/plugin/capped-map.js +1 -0
- package/dist/crawler/plugin/file-cacher.cjs +20 -18
- package/dist/crawler/plugin/file-cacher.js +20 -18
- package/dist/crawler/plugin/health-metrics.cjs +2 -0
- package/dist/crawler/plugin/health-metrics.js +2 -0
- package/dist/crawler/plugin/index.cjs +1 -1
- package/dist/crawler/plugin/index.js +1 -1
- package/dist/crawler/plugin/memory-monitor.cjs +1 -0
- package/dist/crawler/plugin/memory-monitor.js +1 -0
- package/dist/crawler/plugin/navigation-history.cjs +5 -5
- package/dist/crawler/plugin/navigation-history.js +3 -3
- package/dist/crawler/plugin/result-stream.cjs +5 -0
- package/dist/crawler/plugin/result-stream.js +5 -0
- package/dist/crawler/plugin/sqlite-utils.cjs +1 -0
- package/dist/crawler/plugin/sqlite-utils.js +1 -0
- package/dist/crawler/plugin/url-store.cjs +5 -5
- package/dist/crawler/plugin/url-store.js +5 -5
- package/dist/crawler/scraper.cjs +1 -1
- package/dist/crawler/scraper.js +1 -1
- package/dist/crawler.d.ts +152 -25
- package/dist/entries/crawler.cjs +4 -4
- package/dist/errors/rezo-error.cjs +3 -72
- package/dist/errors/rezo-error.js +3 -72
- package/dist/index.cjs +30 -30
- package/dist/index.d.ts +13 -2
- package/dist/internal/agents/index.cjs +10 -10
- package/dist/platform/browser.d.ts +13 -2
- package/dist/platform/bun.d.ts +13 -2
- package/dist/platform/deno.d.ts +13 -2
- package/dist/platform/node.d.ts +13 -2
- package/dist/platform/react-native.d.ts +13 -2
- package/dist/platform/worker.d.ts +13 -2
- package/dist/proxy/index.cjs +4 -4
- package/dist/queue/index.cjs +8 -8
- package/dist/queue/queue.cjs +58 -13
- package/dist/queue/queue.js +58 -13
- package/dist/responses/universal/index.cjs +11 -11
- package/dist/utils/agent-pool.cjs +37 -0
- package/dist/utils/agent-pool.js +37 -0
- package/dist/version.cjs +1 -1
- package/dist/version.js +1 -1
- package/dist/wget/index.cjs +49 -49
- package/dist/wget/index.d.ts +12 -1
- package/package.json +1 -1
|
@@ -2011,6 +2011,13 @@ export interface QueueConfig {
|
|
|
2011
2011
|
intervalCap?: number;
|
|
2012
2012
|
/** Carry over unused interval capacity to next interval */
|
|
2013
2013
|
carryoverConcurrencyCount?: boolean;
|
|
2014
|
+
/**
|
|
2015
|
+
* Reject the task promise when an error occurs (default: false)
|
|
2016
|
+
* When false, errors are swallowed and task.resolve(undefined) is called.
|
|
2017
|
+
* This prevents unhandled promise rejections but makes error handling harder.
|
|
2018
|
+
* When true, task.reject(error) is called, allowing proper try/catch handling.
|
|
2019
|
+
*/
|
|
2020
|
+
rejectOnError?: boolean;
|
|
2014
2021
|
}
|
|
2015
2022
|
/**
|
|
2016
2023
|
* Task options when adding to queue
|
|
@@ -2216,8 +2223,12 @@ declare class RezoQueue<T = any> {
|
|
|
2216
2223
|
/**
|
|
2217
2224
|
* Wait for queue size to be less than limit
|
|
2218
2225
|
* @param limit - Size threshold
|
|
2226
|
+
* @param timeoutMs - Optional timeout in milliseconds (default: 0 = no timeout)
|
|
2227
|
+
* If timeout occurs, promise resolves (not rejects) to prevent blocking
|
|
2219
2228
|
*/
|
|
2220
|
-
onSizeLessThan(limit: number): Promise<void>;
|
|
2229
|
+
onSizeLessThan(limit: number, timeoutMs?: number): Promise<void>;
|
|
2230
|
+
/** Maximum recommended handlers per event before warning */
|
|
2231
|
+
private static readonly MAX_HANDLERS_WARNING;
|
|
2221
2232
|
/**
|
|
2222
2233
|
* Register an event handler
|
|
2223
2234
|
* @param event - Event name
|
|
@@ -4568,7 +4579,7 @@ export interface RezoInstance extends Rezo {
|
|
|
4568
4579
|
*
|
|
4569
4580
|
* IMPORTANT: Update these values when bumping package version.
|
|
4570
4581
|
*/
|
|
4571
|
-
export declare const VERSION = "1.0.
|
|
4582
|
+
export declare const VERSION = "1.0.74";
|
|
4572
4583
|
export declare const isRezoError: typeof RezoError.isRezoError;
|
|
4573
4584
|
export declare const Cancel: typeof RezoError;
|
|
4574
4585
|
export declare const CancelToken: {
|
package/dist/platform/bun.d.ts
CHANGED
|
@@ -2011,6 +2011,13 @@ export interface QueueConfig {
|
|
|
2011
2011
|
intervalCap?: number;
|
|
2012
2012
|
/** Carry over unused interval capacity to next interval */
|
|
2013
2013
|
carryoverConcurrencyCount?: boolean;
|
|
2014
|
+
/**
|
|
2015
|
+
* Reject the task promise when an error occurs (default: false)
|
|
2016
|
+
* When false, errors are swallowed and task.resolve(undefined) is called.
|
|
2017
|
+
* This prevents unhandled promise rejections but makes error handling harder.
|
|
2018
|
+
* When true, task.reject(error) is called, allowing proper try/catch handling.
|
|
2019
|
+
*/
|
|
2020
|
+
rejectOnError?: boolean;
|
|
2014
2021
|
}
|
|
2015
2022
|
/**
|
|
2016
2023
|
* Task options when adding to queue
|
|
@@ -2216,8 +2223,12 @@ declare class RezoQueue<T = any> {
|
|
|
2216
2223
|
/**
|
|
2217
2224
|
* Wait for queue size to be less than limit
|
|
2218
2225
|
* @param limit - Size threshold
|
|
2226
|
+
* @param timeoutMs - Optional timeout in milliseconds (default: 0 = no timeout)
|
|
2227
|
+
* If timeout occurs, promise resolves (not rejects) to prevent blocking
|
|
2219
2228
|
*/
|
|
2220
|
-
onSizeLessThan(limit: number): Promise<void>;
|
|
2229
|
+
onSizeLessThan(limit: number, timeoutMs?: number): Promise<void>;
|
|
2230
|
+
/** Maximum recommended handlers per event before warning */
|
|
2231
|
+
private static readonly MAX_HANDLERS_WARNING;
|
|
2221
2232
|
/**
|
|
2222
2233
|
* Register an event handler
|
|
2223
2234
|
* @param event - Event name
|
|
@@ -4568,7 +4579,7 @@ export interface RezoInstance extends Rezo {
|
|
|
4568
4579
|
*
|
|
4569
4580
|
* IMPORTANT: Update these values when bumping package version.
|
|
4570
4581
|
*/
|
|
4571
|
-
export declare const VERSION = "1.0.
|
|
4582
|
+
export declare const VERSION = "1.0.74";
|
|
4572
4583
|
export declare const isRezoError: typeof RezoError.isRezoError;
|
|
4573
4584
|
export declare const Cancel: typeof RezoError;
|
|
4574
4585
|
export declare const CancelToken: {
|
package/dist/platform/deno.d.ts
CHANGED
|
@@ -2011,6 +2011,13 @@ export interface QueueConfig {
|
|
|
2011
2011
|
intervalCap?: number;
|
|
2012
2012
|
/** Carry over unused interval capacity to next interval */
|
|
2013
2013
|
carryoverConcurrencyCount?: boolean;
|
|
2014
|
+
/**
|
|
2015
|
+
* Reject the task promise when an error occurs (default: false)
|
|
2016
|
+
* When false, errors are swallowed and task.resolve(undefined) is called.
|
|
2017
|
+
* This prevents unhandled promise rejections but makes error handling harder.
|
|
2018
|
+
* When true, task.reject(error) is called, allowing proper try/catch handling.
|
|
2019
|
+
*/
|
|
2020
|
+
rejectOnError?: boolean;
|
|
2014
2021
|
}
|
|
2015
2022
|
/**
|
|
2016
2023
|
* Task options when adding to queue
|
|
@@ -2216,8 +2223,12 @@ declare class RezoQueue<T = any> {
|
|
|
2216
2223
|
/**
|
|
2217
2224
|
* Wait for queue size to be less than limit
|
|
2218
2225
|
* @param limit - Size threshold
|
|
2226
|
+
* @param timeoutMs - Optional timeout in milliseconds (default: 0 = no timeout)
|
|
2227
|
+
* If timeout occurs, promise resolves (not rejects) to prevent blocking
|
|
2219
2228
|
*/
|
|
2220
|
-
onSizeLessThan(limit: number): Promise<void>;
|
|
2229
|
+
onSizeLessThan(limit: number, timeoutMs?: number): Promise<void>;
|
|
2230
|
+
/** Maximum recommended handlers per event before warning */
|
|
2231
|
+
private static readonly MAX_HANDLERS_WARNING;
|
|
2221
2232
|
/**
|
|
2222
2233
|
* Register an event handler
|
|
2223
2234
|
* @param event - Event name
|
|
@@ -4568,7 +4579,7 @@ export interface RezoInstance extends Rezo {
|
|
|
4568
4579
|
*
|
|
4569
4580
|
* IMPORTANT: Update these values when bumping package version.
|
|
4570
4581
|
*/
|
|
4571
|
-
export declare const VERSION = "1.0.
|
|
4582
|
+
export declare const VERSION = "1.0.74";
|
|
4572
4583
|
export declare const isRezoError: typeof RezoError.isRezoError;
|
|
4573
4584
|
export declare const Cancel: typeof RezoError;
|
|
4574
4585
|
export declare const CancelToken: {
|
package/dist/platform/node.d.ts
CHANGED
|
@@ -2011,6 +2011,13 @@ export interface QueueConfig {
|
|
|
2011
2011
|
intervalCap?: number;
|
|
2012
2012
|
/** Carry over unused interval capacity to next interval */
|
|
2013
2013
|
carryoverConcurrencyCount?: boolean;
|
|
2014
|
+
/**
|
|
2015
|
+
* Reject the task promise when an error occurs (default: false)
|
|
2016
|
+
* When false, errors are swallowed and task.resolve(undefined) is called.
|
|
2017
|
+
* This prevents unhandled promise rejections but makes error handling harder.
|
|
2018
|
+
* When true, task.reject(error) is called, allowing proper try/catch handling.
|
|
2019
|
+
*/
|
|
2020
|
+
rejectOnError?: boolean;
|
|
2014
2021
|
}
|
|
2015
2022
|
/**
|
|
2016
2023
|
* Task options when adding to queue
|
|
@@ -2216,8 +2223,12 @@ declare class RezoQueue<T = any> {
|
|
|
2216
2223
|
/**
|
|
2217
2224
|
* Wait for queue size to be less than limit
|
|
2218
2225
|
* @param limit - Size threshold
|
|
2226
|
+
* @param timeoutMs - Optional timeout in milliseconds (default: 0 = no timeout)
|
|
2227
|
+
* If timeout occurs, promise resolves (not rejects) to prevent blocking
|
|
2219
2228
|
*/
|
|
2220
|
-
onSizeLessThan(limit: number): Promise<void>;
|
|
2229
|
+
onSizeLessThan(limit: number, timeoutMs?: number): Promise<void>;
|
|
2230
|
+
/** Maximum recommended handlers per event before warning */
|
|
2231
|
+
private static readonly MAX_HANDLERS_WARNING;
|
|
2221
2232
|
/**
|
|
2222
2233
|
* Register an event handler
|
|
2223
2234
|
* @param event - Event name
|
|
@@ -4568,7 +4579,7 @@ export interface RezoInstance extends Rezo {
|
|
|
4568
4579
|
*
|
|
4569
4580
|
* IMPORTANT: Update these values when bumping package version.
|
|
4570
4581
|
*/
|
|
4571
|
-
export declare const VERSION = "1.0.
|
|
4582
|
+
export declare const VERSION = "1.0.74";
|
|
4572
4583
|
export declare const isRezoError: typeof RezoError.isRezoError;
|
|
4573
4584
|
export declare const Cancel: typeof RezoError;
|
|
4574
4585
|
export declare const CancelToken: {
|
|
@@ -2011,6 +2011,13 @@ export interface QueueConfig {
|
|
|
2011
2011
|
intervalCap?: number;
|
|
2012
2012
|
/** Carry over unused interval capacity to next interval */
|
|
2013
2013
|
carryoverConcurrencyCount?: boolean;
|
|
2014
|
+
/**
|
|
2015
|
+
* Reject the task promise when an error occurs (default: false)
|
|
2016
|
+
* When false, errors are swallowed and task.resolve(undefined) is called.
|
|
2017
|
+
* This prevents unhandled promise rejections but makes error handling harder.
|
|
2018
|
+
* When true, task.reject(error) is called, allowing proper try/catch handling.
|
|
2019
|
+
*/
|
|
2020
|
+
rejectOnError?: boolean;
|
|
2014
2021
|
}
|
|
2015
2022
|
/**
|
|
2016
2023
|
* Task options when adding to queue
|
|
@@ -2216,8 +2223,12 @@ declare class RezoQueue<T = any> {
|
|
|
2216
2223
|
/**
|
|
2217
2224
|
* Wait for queue size to be less than limit
|
|
2218
2225
|
* @param limit - Size threshold
|
|
2226
|
+
* @param timeoutMs - Optional timeout in milliseconds (default: 0 = no timeout)
|
|
2227
|
+
* If timeout occurs, promise resolves (not rejects) to prevent blocking
|
|
2219
2228
|
*/
|
|
2220
|
-
onSizeLessThan(limit: number): Promise<void>;
|
|
2229
|
+
onSizeLessThan(limit: number, timeoutMs?: number): Promise<void>;
|
|
2230
|
+
/** Maximum recommended handlers per event before warning */
|
|
2231
|
+
private static readonly MAX_HANDLERS_WARNING;
|
|
2221
2232
|
/**
|
|
2222
2233
|
* Register an event handler
|
|
2223
2234
|
* @param event - Event name
|
|
@@ -4568,7 +4579,7 @@ export interface RezoInstance extends Rezo {
|
|
|
4568
4579
|
*
|
|
4569
4580
|
* IMPORTANT: Update these values when bumping package version.
|
|
4570
4581
|
*/
|
|
4571
|
-
export declare const VERSION = "1.0.
|
|
4582
|
+
export declare const VERSION = "1.0.74";
|
|
4572
4583
|
export declare const isRezoError: typeof RezoError.isRezoError;
|
|
4573
4584
|
export declare const Cancel: typeof RezoError;
|
|
4574
4585
|
export declare const CancelToken: {
|
|
@@ -2011,6 +2011,13 @@ export interface QueueConfig {
|
|
|
2011
2011
|
intervalCap?: number;
|
|
2012
2012
|
/** Carry over unused interval capacity to next interval */
|
|
2013
2013
|
carryoverConcurrencyCount?: boolean;
|
|
2014
|
+
/**
|
|
2015
|
+
* Reject the task promise when an error occurs (default: false)
|
|
2016
|
+
* When false, errors are swallowed and task.resolve(undefined) is called.
|
|
2017
|
+
* This prevents unhandled promise rejections but makes error handling harder.
|
|
2018
|
+
* When true, task.reject(error) is called, allowing proper try/catch handling.
|
|
2019
|
+
*/
|
|
2020
|
+
rejectOnError?: boolean;
|
|
2014
2021
|
}
|
|
2015
2022
|
/**
|
|
2016
2023
|
* Task options when adding to queue
|
|
@@ -2216,8 +2223,12 @@ declare class RezoQueue<T = any> {
|
|
|
2216
2223
|
/**
|
|
2217
2224
|
* Wait for queue size to be less than limit
|
|
2218
2225
|
* @param limit - Size threshold
|
|
2226
|
+
* @param timeoutMs - Optional timeout in milliseconds (default: 0 = no timeout)
|
|
2227
|
+
* If timeout occurs, promise resolves (not rejects) to prevent blocking
|
|
2219
2228
|
*/
|
|
2220
|
-
onSizeLessThan(limit: number): Promise<void>;
|
|
2229
|
+
onSizeLessThan(limit: number, timeoutMs?: number): Promise<void>;
|
|
2230
|
+
/** Maximum recommended handlers per event before warning */
|
|
2231
|
+
private static readonly MAX_HANDLERS_WARNING;
|
|
2221
2232
|
/**
|
|
2222
2233
|
* Register an event handler
|
|
2223
2234
|
* @param event - Event name
|
|
@@ -4568,7 +4579,7 @@ export interface RezoInstance extends Rezo {
|
|
|
4568
4579
|
*
|
|
4569
4580
|
* IMPORTANT: Update these values when bumping package version.
|
|
4570
4581
|
*/
|
|
4571
|
-
export declare const VERSION = "1.0.
|
|
4582
|
+
export declare const VERSION = "1.0.74";
|
|
4572
4583
|
export declare const isRezoError: typeof RezoError.isRezoError;
|
|
4573
4584
|
export declare const Cancel: typeof RezoError;
|
|
4574
4585
|
export declare const CancelToken: {
|
package/dist/proxy/index.cjs
CHANGED
|
@@ -1,9 +1,9 @@
|
|
|
1
1
|
const { Agent, HttpProxyAgent, HttpsProxyAgent, SocksProxyAgent } = require('../internal/agents/index.cjs');
|
|
2
2
|
const { parseProxyString } = require('./parse.cjs');
|
|
3
|
-
const
|
|
4
|
-
exports.ProxyManager =
|
|
5
|
-
const
|
|
6
|
-
exports.parseProxyString =
|
|
3
|
+
const _mod_dkmwz9 = require('./manager.cjs');
|
|
4
|
+
exports.ProxyManager = _mod_dkmwz9.ProxyManager;;
|
|
5
|
+
const _mod_0ibmjj = require('./parse.cjs');
|
|
6
|
+
exports.parseProxyString = _mod_0ibmjj.parseProxyString;;
|
|
7
7
|
function createOptions(uri, opts) {
|
|
8
8
|
if (uri instanceof URL || typeof uri === "string") {
|
|
9
9
|
return {
|
package/dist/queue/index.cjs
CHANGED
|
@@ -1,8 +1,8 @@
|
|
|
1
|
-
const
|
|
2
|
-
exports.RezoQueue =
|
|
3
|
-
const
|
|
4
|
-
exports.HttpQueue =
|
|
5
|
-
exports.extractDomain =
|
|
6
|
-
const
|
|
7
|
-
exports.Priority =
|
|
8
|
-
exports.HttpMethodPriority =
|
|
1
|
+
const _mod_rllkog = require('./queue.cjs');
|
|
2
|
+
exports.RezoQueue = _mod_rllkog.RezoQueue;;
|
|
3
|
+
const _mod_2glpb0 = require('./http-queue.cjs');
|
|
4
|
+
exports.HttpQueue = _mod_2glpb0.HttpQueue;
|
|
5
|
+
exports.extractDomain = _mod_2glpb0.extractDomain;;
|
|
6
|
+
const _mod_rpdaro = require('./types.cjs');
|
|
7
|
+
exports.Priority = _mod_rpdaro.Priority;
|
|
8
|
+
exports.HttpMethodPriority = _mod_rpdaro.HttpMethodPriority;;
|
package/dist/queue/queue.cjs
CHANGED
|
@@ -35,7 +35,8 @@ class RezoQueue {
|
|
|
35
35
|
throwOnTimeout: config.throwOnTimeout ?? true,
|
|
36
36
|
interval: config.interval ?? 0,
|
|
37
37
|
intervalCap: config.intervalCap ?? 1 / 0,
|
|
38
|
-
carryoverConcurrencyCount: config.carryoverConcurrencyCount ?? false
|
|
38
|
+
carryoverConcurrencyCount: config.carryoverConcurrencyCount ?? false,
|
|
39
|
+
rejectOnError: config.rejectOnError ?? true
|
|
39
40
|
};
|
|
40
41
|
if (!this.config.autoStart) {
|
|
41
42
|
this.isPausedFlag = true;
|
|
@@ -88,9 +89,13 @@ class RezoQueue {
|
|
|
88
89
|
reject(new Error("Task was cancelled before starting"));
|
|
89
90
|
return;
|
|
90
91
|
}
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
92
|
+
let abortHandler;
|
|
93
|
+
if (options.signal) {
|
|
94
|
+
abortHandler = () => this.cancel(task.id);
|
|
95
|
+
options.signal.addEventListener("abort", abortHandler, { once: true });
|
|
96
|
+
task._abortHandler = abortHandler;
|
|
97
|
+
task._signal = options.signal;
|
|
98
|
+
}
|
|
94
99
|
this.insertByPriority(task);
|
|
95
100
|
this.statsData.added++;
|
|
96
101
|
this.hasEverBeenActive = true;
|
|
@@ -120,7 +125,11 @@ class RezoQueue {
|
|
|
120
125
|
const tasks = [...this.queue];
|
|
121
126
|
this.queue = [];
|
|
122
127
|
for (const task of tasks) {
|
|
123
|
-
|
|
128
|
+
if (this.config.rejectOnError) {
|
|
129
|
+
task.reject(new Error("Queue was cleared"));
|
|
130
|
+
} else {
|
|
131
|
+
task.resolve(undefined);
|
|
132
|
+
}
|
|
124
133
|
this.statsData.cancelled++;
|
|
125
134
|
this.emit("cancelled", { id: task.id });
|
|
126
135
|
}
|
|
@@ -130,7 +139,11 @@ class RezoQueue {
|
|
|
130
139
|
const index = this.queue.findIndex((t) => t.id === id);
|
|
131
140
|
if (index !== -1) {
|
|
132
141
|
const [task] = this.queue.splice(index, 1);
|
|
133
|
-
|
|
142
|
+
if (this.config.rejectOnError) {
|
|
143
|
+
task.reject(new Error("Task was cancelled"));
|
|
144
|
+
} else {
|
|
145
|
+
task.resolve(undefined);
|
|
146
|
+
}
|
|
134
147
|
this.statsData.cancelled++;
|
|
135
148
|
this.emit("cancelled", { id });
|
|
136
149
|
this.checkEmpty();
|
|
@@ -143,7 +156,11 @@ class RezoQueue {
|
|
|
143
156
|
const remaining = [];
|
|
144
157
|
for (const task of this.queue) {
|
|
145
158
|
if (predicate({ id: task.id, priority: task.priority })) {
|
|
146
|
-
|
|
159
|
+
if (this.config.rejectOnError) {
|
|
160
|
+
task.reject(new Error("Task was cancelled"));
|
|
161
|
+
} else {
|
|
162
|
+
task.resolve(undefined);
|
|
163
|
+
}
|
|
147
164
|
this.statsData.cancelled++;
|
|
148
165
|
this.emit("cancelled", { id: task.id });
|
|
149
166
|
count++;
|
|
@@ -183,29 +200,48 @@ class RezoQueue {
|
|
|
183
200
|
}
|
|
184
201
|
return this.emptyPromise.promise;
|
|
185
202
|
}
|
|
186
|
-
onSizeLessThan(limit) {
|
|
203
|
+
onSizeLessThan(limit, timeoutMs = 0) {
|
|
187
204
|
if (this.queue.length < limit) {
|
|
188
205
|
return Promise.resolve();
|
|
189
206
|
}
|
|
190
207
|
return new Promise((resolve) => {
|
|
208
|
+
let timeoutId;
|
|
209
|
+
const cleanup = () => {
|
|
210
|
+
this.off("completed", check);
|
|
211
|
+
this.off("cancelled", check);
|
|
212
|
+
this.off("error", check);
|
|
213
|
+
this.off("empty", check);
|
|
214
|
+
if (timeoutId)
|
|
215
|
+
clearTimeout(timeoutId);
|
|
216
|
+
};
|
|
191
217
|
const check = () => {
|
|
192
218
|
if (this.queue.length < limit) {
|
|
193
|
-
|
|
194
|
-
this.off("cancelled", check);
|
|
195
|
-
this.off("error", check);
|
|
219
|
+
cleanup();
|
|
196
220
|
resolve();
|
|
197
221
|
}
|
|
198
222
|
};
|
|
199
223
|
this.on("completed", check);
|
|
200
224
|
this.on("cancelled", check);
|
|
201
225
|
this.on("error", check);
|
|
226
|
+
this.on("empty", check);
|
|
227
|
+
if (timeoutMs > 0) {
|
|
228
|
+
timeoutId = setTimeout(() => {
|
|
229
|
+
cleanup();
|
|
230
|
+
resolve();
|
|
231
|
+
}, timeoutMs);
|
|
232
|
+
}
|
|
202
233
|
});
|
|
203
234
|
}
|
|
235
|
+
static MAX_HANDLERS_WARNING = 100;
|
|
204
236
|
on(event, handler) {
|
|
205
237
|
if (!this.eventHandlers.has(event)) {
|
|
206
238
|
this.eventHandlers.set(event, new Set);
|
|
207
239
|
}
|
|
208
|
-
this.eventHandlers.get(event)
|
|
240
|
+
const handlers = this.eventHandlers.get(event);
|
|
241
|
+
handlers.add(handler);
|
|
242
|
+
if (handlers.size === RezoQueue.MAX_HANDLERS_WARNING) {
|
|
243
|
+
console.warn(`[RezoQueue] Warning: ${handlers.size} handlers registered for '${String(event)}' event. ` + `This may indicate a memory leak. Consider using off() to remove handlers when done.`);
|
|
244
|
+
}
|
|
209
245
|
}
|
|
210
246
|
off(event, handler) {
|
|
211
247
|
this.eventHandlers.get(event)?.delete(handler);
|
|
@@ -285,8 +321,17 @@ class RezoQueue {
|
|
|
285
321
|
clearTimeout(timeoutId);
|
|
286
322
|
this.statsData.failed++;
|
|
287
323
|
this.emit("error", { id: task.id, error });
|
|
288
|
-
|
|
324
|
+
if (this.config.rejectOnError) {
|
|
325
|
+
task.reject(error);
|
|
326
|
+
} else {
|
|
327
|
+
task.resolve(undefined);
|
|
328
|
+
}
|
|
289
329
|
} finally {
|
|
330
|
+
if (task._abortHandler && task._signal) {
|
|
331
|
+
task._signal.removeEventListener("abort", task._abortHandler);
|
|
332
|
+
delete task._abortHandler;
|
|
333
|
+
delete task._signal;
|
|
334
|
+
}
|
|
290
335
|
this.pendingCount--;
|
|
291
336
|
this.checkEmpty();
|
|
292
337
|
this.checkIdle();
|
package/dist/queue/queue.js
CHANGED
|
@@ -35,7 +35,8 @@ export class RezoQueue {
|
|
|
35
35
|
throwOnTimeout: config.throwOnTimeout ?? true,
|
|
36
36
|
interval: config.interval ?? 0,
|
|
37
37
|
intervalCap: config.intervalCap ?? 1 / 0,
|
|
38
|
-
carryoverConcurrencyCount: config.carryoverConcurrencyCount ?? false
|
|
38
|
+
carryoverConcurrencyCount: config.carryoverConcurrencyCount ?? false,
|
|
39
|
+
rejectOnError: config.rejectOnError ?? true
|
|
39
40
|
};
|
|
40
41
|
if (!this.config.autoStart) {
|
|
41
42
|
this.isPausedFlag = true;
|
|
@@ -88,9 +89,13 @@ export class RezoQueue {
|
|
|
88
89
|
reject(new Error("Task was cancelled before starting"));
|
|
89
90
|
return;
|
|
90
91
|
}
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
92
|
+
let abortHandler;
|
|
93
|
+
if (options.signal) {
|
|
94
|
+
abortHandler = () => this.cancel(task.id);
|
|
95
|
+
options.signal.addEventListener("abort", abortHandler, { once: true });
|
|
96
|
+
task._abortHandler = abortHandler;
|
|
97
|
+
task._signal = options.signal;
|
|
98
|
+
}
|
|
94
99
|
this.insertByPriority(task);
|
|
95
100
|
this.statsData.added++;
|
|
96
101
|
this.hasEverBeenActive = true;
|
|
@@ -120,7 +125,11 @@ export class RezoQueue {
|
|
|
120
125
|
const tasks = [...this.queue];
|
|
121
126
|
this.queue = [];
|
|
122
127
|
for (const task of tasks) {
|
|
123
|
-
|
|
128
|
+
if (this.config.rejectOnError) {
|
|
129
|
+
task.reject(new Error("Queue was cleared"));
|
|
130
|
+
} else {
|
|
131
|
+
task.resolve(undefined);
|
|
132
|
+
}
|
|
124
133
|
this.statsData.cancelled++;
|
|
125
134
|
this.emit("cancelled", { id: task.id });
|
|
126
135
|
}
|
|
@@ -130,7 +139,11 @@ export class RezoQueue {
|
|
|
130
139
|
const index = this.queue.findIndex((t) => t.id === id);
|
|
131
140
|
if (index !== -1) {
|
|
132
141
|
const [task] = this.queue.splice(index, 1);
|
|
133
|
-
|
|
142
|
+
if (this.config.rejectOnError) {
|
|
143
|
+
task.reject(new Error("Task was cancelled"));
|
|
144
|
+
} else {
|
|
145
|
+
task.resolve(undefined);
|
|
146
|
+
}
|
|
134
147
|
this.statsData.cancelled++;
|
|
135
148
|
this.emit("cancelled", { id });
|
|
136
149
|
this.checkEmpty();
|
|
@@ -143,7 +156,11 @@ export class RezoQueue {
|
|
|
143
156
|
const remaining = [];
|
|
144
157
|
for (const task of this.queue) {
|
|
145
158
|
if (predicate({ id: task.id, priority: task.priority })) {
|
|
146
|
-
|
|
159
|
+
if (this.config.rejectOnError) {
|
|
160
|
+
task.reject(new Error("Task was cancelled"));
|
|
161
|
+
} else {
|
|
162
|
+
task.resolve(undefined);
|
|
163
|
+
}
|
|
147
164
|
this.statsData.cancelled++;
|
|
148
165
|
this.emit("cancelled", { id: task.id });
|
|
149
166
|
count++;
|
|
@@ -183,29 +200,48 @@ export class RezoQueue {
|
|
|
183
200
|
}
|
|
184
201
|
return this.emptyPromise.promise;
|
|
185
202
|
}
|
|
186
|
-
onSizeLessThan(limit) {
|
|
203
|
+
onSizeLessThan(limit, timeoutMs = 0) {
|
|
187
204
|
if (this.queue.length < limit) {
|
|
188
205
|
return Promise.resolve();
|
|
189
206
|
}
|
|
190
207
|
return new Promise((resolve) => {
|
|
208
|
+
let timeoutId;
|
|
209
|
+
const cleanup = () => {
|
|
210
|
+
this.off("completed", check);
|
|
211
|
+
this.off("cancelled", check);
|
|
212
|
+
this.off("error", check);
|
|
213
|
+
this.off("empty", check);
|
|
214
|
+
if (timeoutId)
|
|
215
|
+
clearTimeout(timeoutId);
|
|
216
|
+
};
|
|
191
217
|
const check = () => {
|
|
192
218
|
if (this.queue.length < limit) {
|
|
193
|
-
|
|
194
|
-
this.off("cancelled", check);
|
|
195
|
-
this.off("error", check);
|
|
219
|
+
cleanup();
|
|
196
220
|
resolve();
|
|
197
221
|
}
|
|
198
222
|
};
|
|
199
223
|
this.on("completed", check);
|
|
200
224
|
this.on("cancelled", check);
|
|
201
225
|
this.on("error", check);
|
|
226
|
+
this.on("empty", check);
|
|
227
|
+
if (timeoutMs > 0) {
|
|
228
|
+
timeoutId = setTimeout(() => {
|
|
229
|
+
cleanup();
|
|
230
|
+
resolve();
|
|
231
|
+
}, timeoutMs);
|
|
232
|
+
}
|
|
202
233
|
});
|
|
203
234
|
}
|
|
235
|
+
static MAX_HANDLERS_WARNING = 100;
|
|
204
236
|
on(event, handler) {
|
|
205
237
|
if (!this.eventHandlers.has(event)) {
|
|
206
238
|
this.eventHandlers.set(event, new Set);
|
|
207
239
|
}
|
|
208
|
-
this.eventHandlers.get(event)
|
|
240
|
+
const handlers = this.eventHandlers.get(event);
|
|
241
|
+
handlers.add(handler);
|
|
242
|
+
if (handlers.size === RezoQueue.MAX_HANDLERS_WARNING) {
|
|
243
|
+
console.warn(`[RezoQueue] Warning: ${handlers.size} handlers registered for '${String(event)}' event. ` + `This may indicate a memory leak. Consider using off() to remove handlers when done.`);
|
|
244
|
+
}
|
|
209
245
|
}
|
|
210
246
|
off(event, handler) {
|
|
211
247
|
this.eventHandlers.get(event)?.delete(handler);
|
|
@@ -285,8 +321,17 @@ export class RezoQueue {
|
|
|
285
321
|
clearTimeout(timeoutId);
|
|
286
322
|
this.statsData.failed++;
|
|
287
323
|
this.emit("error", { id: task.id, error });
|
|
288
|
-
|
|
324
|
+
if (this.config.rejectOnError) {
|
|
325
|
+
task.reject(error);
|
|
326
|
+
} else {
|
|
327
|
+
task.resolve(undefined);
|
|
328
|
+
}
|
|
289
329
|
} finally {
|
|
330
|
+
if (task._abortHandler && task._signal) {
|
|
331
|
+
task._signal.removeEventListener("abort", task._abortHandler);
|
|
332
|
+
delete task._abortHandler;
|
|
333
|
+
delete task._signal;
|
|
334
|
+
}
|
|
290
335
|
this.pendingCount--;
|
|
291
336
|
this.checkEmpty();
|
|
292
337
|
this.checkIdle();
|
|
@@ -1,11 +1,11 @@
|
|
|
1
|
-
const
|
|
2
|
-
exports.UniversalEventEmitter =
|
|
3
|
-
const
|
|
4
|
-
exports.UniversalStreamResponse =
|
|
5
|
-
exports.StreamResponse =
|
|
6
|
-
const
|
|
7
|
-
exports.UniversalDownloadResponse =
|
|
8
|
-
exports.DownloadResponse =
|
|
9
|
-
const
|
|
10
|
-
exports.UniversalUploadResponse =
|
|
11
|
-
exports.UploadResponse =
|
|
1
|
+
const _mod_1hy9oe = require('./event-emitter.cjs');
|
|
2
|
+
exports.UniversalEventEmitter = _mod_1hy9oe.UniversalEventEmitter;;
|
|
3
|
+
const _mod_zi59hf = require('./stream.cjs');
|
|
4
|
+
exports.UniversalStreamResponse = _mod_zi59hf.UniversalStreamResponse;
|
|
5
|
+
exports.StreamResponse = _mod_zi59hf.StreamResponse;;
|
|
6
|
+
const _mod_vm1o39 = require('./download.cjs');
|
|
7
|
+
exports.UniversalDownloadResponse = _mod_vm1o39.UniversalDownloadResponse;
|
|
8
|
+
exports.DownloadResponse = _mod_vm1o39.DownloadResponse;;
|
|
9
|
+
const _mod_zh9h9e = require('./upload.cjs');
|
|
10
|
+
exports.UniversalUploadResponse = _mod_zh9h9e.UniversalUploadResponse;
|
|
11
|
+
exports.UploadResponse = _mod_zh9h9e.UploadResponse;;
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
const http = require("node:http");
|
|
2
2
|
const https = require("node:https");
|
|
3
|
+
const tls = require("node:tls");
|
|
3
4
|
const { getGlobalDNSCache } = require('../cache/dns-cache.cjs');
|
|
4
5
|
const DEFAULT_CONFIG = {
|
|
5
6
|
keepAlive: true,
|
|
@@ -79,6 +80,39 @@ class AgentPool {
|
|
|
79
80
|
});
|
|
80
81
|
}
|
|
81
82
|
createHttpsAgent(key, tlsOptions) {
|
|
83
|
+
const secureContext = tls.createSecureContext({
|
|
84
|
+
ecdhCurve: "X25519:prime256v1:secp384r1",
|
|
85
|
+
ciphers: [
|
|
86
|
+
"TLS_AES_128_GCM_SHA256",
|
|
87
|
+
"TLS_AES_256_GCM_SHA384",
|
|
88
|
+
"TLS_CHACHA20_POLY1305_SHA256",
|
|
89
|
+
"ECDHE-ECDSA-AES128-GCM-SHA256",
|
|
90
|
+
"ECDHE-RSA-AES128-GCM-SHA256",
|
|
91
|
+
"ECDHE-ECDSA-AES256-GCM-SHA384",
|
|
92
|
+
"ECDHE-RSA-AES256-GCM-SHA384",
|
|
93
|
+
"ECDHE-ECDSA-CHACHA20-POLY1305",
|
|
94
|
+
"ECDHE-RSA-CHACHA20-POLY1305",
|
|
95
|
+
"ECDHE-RSA-AES128-SHA",
|
|
96
|
+
"ECDHE-RSA-AES256-SHA",
|
|
97
|
+
"AES128-GCM-SHA256",
|
|
98
|
+
"AES256-GCM-SHA384",
|
|
99
|
+
"AES128-SHA",
|
|
100
|
+
"AES256-SHA"
|
|
101
|
+
].join(":"),
|
|
102
|
+
sigalgs: [
|
|
103
|
+
"ecdsa_secp256r1_sha256",
|
|
104
|
+
"ecdsa_secp384r1_sha384",
|
|
105
|
+
"rsa_pss_rsae_sha256",
|
|
106
|
+
"rsa_pss_rsae_sha384",
|
|
107
|
+
"rsa_pss_rsae_sha512",
|
|
108
|
+
"rsa_pkcs1_sha256",
|
|
109
|
+
"rsa_pkcs1_sha384",
|
|
110
|
+
"rsa_pkcs1_sha512"
|
|
111
|
+
].join(":"),
|
|
112
|
+
minVersion: "TLSv1.2",
|
|
113
|
+
maxVersion: "TLSv1.3",
|
|
114
|
+
sessionTimeout: 3600
|
|
115
|
+
});
|
|
82
116
|
const agentOptions = {
|
|
83
117
|
keepAlive: this.config.keepAlive,
|
|
84
118
|
keepAliveMsecs: this.config.keepAliveMsecs,
|
|
@@ -86,6 +120,7 @@ class AgentPool {
|
|
|
86
120
|
maxFreeSockets: this.config.maxFreeSockets,
|
|
87
121
|
timeout: this.config.timeout,
|
|
88
122
|
scheduling: this.config.scheduling,
|
|
123
|
+
secureContext,
|
|
89
124
|
...tlsOptions
|
|
90
125
|
};
|
|
91
126
|
const lookup = this.createLookupFunction();
|
|
@@ -159,10 +194,12 @@ class AgentPool {
|
|
|
159
194
|
this.evictionTimer = null;
|
|
160
195
|
}
|
|
161
196
|
for (const pooled of this.httpAgents.values()) {
|
|
197
|
+
this.destroyAgentSockets(pooled.agent);
|
|
162
198
|
pooled.agent.destroy();
|
|
163
199
|
}
|
|
164
200
|
this.httpAgents.clear();
|
|
165
201
|
for (const pooled of this.httpsAgents.values()) {
|
|
202
|
+
this.destroyAgentSockets(pooled.agent);
|
|
166
203
|
pooled.agent.destroy();
|
|
167
204
|
}
|
|
168
205
|
this.httpsAgents.clear();
|