rezo 1.0.4 → 1.0.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +352 -9
- package/dist/adapters/curl.cjs +796 -0
- package/dist/adapters/curl.js +796 -0
- package/dist/adapters/entries/curl.d.ts +2332 -20
- package/dist/adapters/entries/fetch.d.ts +289 -20
- package/dist/adapters/entries/http.d.ts +289 -20
- package/dist/adapters/entries/http2.d.ts +289 -20
- package/dist/adapters/entries/react-native.d.ts +289 -20
- package/dist/adapters/entries/xhr.d.ts +289 -20
- package/dist/adapters/index.cjs +6 -6
- package/dist/adapters/picker.cjs +2 -2
- package/dist/adapters/picker.js +2 -2
- package/dist/cache/index.cjs +13 -13
- package/dist/core/rezo.cjs +2 -2
- package/dist/core/rezo.js +2 -2
- package/dist/crawler.d.ts +291 -22
- package/dist/entries/crawler.cjs +5 -5
- package/dist/index.cjs +23 -18
- package/dist/index.d.ts +556 -20
- package/dist/index.js +1 -0
- package/dist/platform/browser.d.ts +289 -20
- package/dist/platform/bun.d.ts +289 -20
- package/dist/platform/deno.d.ts +289 -20
- package/dist/platform/node.d.ts +289 -20
- package/dist/platform/react-native.d.ts +289 -20
- package/dist/platform/worker.d.ts +289 -20
- package/dist/plugin/crawler-options.cjs +1 -1
- package/dist/plugin/crawler-options.js +1 -1
- package/dist/plugin/crawler.cjs +2 -2
- package/dist/plugin/crawler.js +2 -2
- package/dist/plugin/index.cjs +36 -36
- package/dist/proxy/index.cjs +2 -2
- package/dist/proxy/manager.cjs +14 -1
- package/dist/proxy/manager.js +14 -1
- package/dist/queue/http-queue.cjs +313 -0
- package/dist/queue/http-queue.js +312 -0
- package/dist/queue/index.cjs +8 -0
- package/dist/queue/index.js +6 -0
- package/dist/queue/queue.cjs +346 -0
- package/dist/queue/queue.js +344 -0
- package/dist/queue/types.cjs +17 -0
- package/dist/queue/types.js +17 -0
- package/dist/types/curl-options.cjs +25 -0
- package/dist/types/curl-options.js +25 -0
- package/dist/utils/http-config.cjs +0 -15
- package/dist/utils/http-config.js +0 -15
- package/package.json +1 -2
|
@@ -0,0 +1,346 @@
|
|
|
1
|
+
function generateId() {
|
|
2
|
+
return `${Date.now().toString(36)}-${Math.random().toString(36).slice(2, 9)}`;
|
|
3
|
+
}
|
|
4
|
+
|
|
5
|
+
class RezoQueue {
|
|
6
|
+
queue = [];
|
|
7
|
+
pendingCount = 0;
|
|
8
|
+
isPausedFlag = false;
|
|
9
|
+
intervalId;
|
|
10
|
+
intervalCount = 0;
|
|
11
|
+
intervalStart = 0;
|
|
12
|
+
eventHandlers = new Map;
|
|
13
|
+
statsData = {
|
|
14
|
+
added: 0,
|
|
15
|
+
processed: 0,
|
|
16
|
+
completed: 0,
|
|
17
|
+
failed: 0,
|
|
18
|
+
timedOut: 0,
|
|
19
|
+
cancelled: 0,
|
|
20
|
+
averageDuration: 0,
|
|
21
|
+
throughput: 0
|
|
22
|
+
};
|
|
23
|
+
totalDuration = 0;
|
|
24
|
+
throughputWindow = [];
|
|
25
|
+
throughputWindowSize = 60;
|
|
26
|
+
idlePromise;
|
|
27
|
+
emptyPromise;
|
|
28
|
+
config;
|
|
29
|
+
constructor(config = {}) {
|
|
30
|
+
this.config = {
|
|
31
|
+
concurrency: config.concurrency ?? 1 / 0,
|
|
32
|
+
autoStart: config.autoStart ?? true,
|
|
33
|
+
timeout: config.timeout ?? 0,
|
|
34
|
+
throwOnTimeout: config.throwOnTimeout ?? true,
|
|
35
|
+
interval: config.interval ?? 0,
|
|
36
|
+
intervalCap: config.intervalCap ?? 1 / 0,
|
|
37
|
+
carryoverConcurrencyCount: config.carryoverConcurrencyCount ?? false
|
|
38
|
+
};
|
|
39
|
+
if (!this.config.autoStart) {
|
|
40
|
+
this.isPausedFlag = true;
|
|
41
|
+
}
|
|
42
|
+
if (this.config.interval > 0) {
|
|
43
|
+
this.startInterval();
|
|
44
|
+
}
|
|
45
|
+
}
|
|
46
|
+
get state() {
|
|
47
|
+
return {
|
|
48
|
+
pending: this.pendingCount,
|
|
49
|
+
size: this.queue.length,
|
|
50
|
+
total: this.pendingCount + this.queue.length,
|
|
51
|
+
isPaused: this.isPausedFlag,
|
|
52
|
+
isIdle: this.pendingCount === 0 && this.queue.length === 0
|
|
53
|
+
};
|
|
54
|
+
}
|
|
55
|
+
get stats() {
|
|
56
|
+
return { ...this.statsData };
|
|
57
|
+
}
|
|
58
|
+
get concurrency() {
|
|
59
|
+
return this.config.concurrency;
|
|
60
|
+
}
|
|
61
|
+
set concurrency(value) {
|
|
62
|
+
this.config.concurrency = value;
|
|
63
|
+
this.tryRunNext();
|
|
64
|
+
}
|
|
65
|
+
get pending() {
|
|
66
|
+
return this.pendingCount;
|
|
67
|
+
}
|
|
68
|
+
get size() {
|
|
69
|
+
return this.queue.length;
|
|
70
|
+
}
|
|
71
|
+
get isPaused() {
|
|
72
|
+
return this.isPausedFlag;
|
|
73
|
+
}
|
|
74
|
+
add(fn, options = {}) {
|
|
75
|
+
return new Promise((resolve, reject) => {
|
|
76
|
+
const task = {
|
|
77
|
+
id: options.id ?? generateId(),
|
|
78
|
+
fn,
|
|
79
|
+
priority: options.priority ?? 0,
|
|
80
|
+
timeout: options.timeout ?? this.config.timeout,
|
|
81
|
+
signal: options.signal,
|
|
82
|
+
resolve,
|
|
83
|
+
reject,
|
|
84
|
+
addedAt: Date.now()
|
|
85
|
+
};
|
|
86
|
+
if (options.signal?.aborted) {
|
|
87
|
+
reject(new Error("Task was cancelled before starting"));
|
|
88
|
+
return;
|
|
89
|
+
}
|
|
90
|
+
options.signal?.addEventListener("abort", () => {
|
|
91
|
+
this.cancel(task.id);
|
|
92
|
+
});
|
|
93
|
+
this.insertByPriority(task);
|
|
94
|
+
this.statsData.added++;
|
|
95
|
+
this.emit("add", { id: task.id, priority: task.priority });
|
|
96
|
+
if (this.config.autoStart && !this.isPausedFlag) {
|
|
97
|
+
this.tryRunNext();
|
|
98
|
+
}
|
|
99
|
+
});
|
|
100
|
+
}
|
|
101
|
+
addAll(fns, options = {}) {
|
|
102
|
+
return Promise.all(fns.map((fn, i) => this.add(fn, { ...options, id: options.id ? `${options.id}-${i}` : undefined })));
|
|
103
|
+
}
|
|
104
|
+
pause() {
|
|
105
|
+
if (!this.isPausedFlag) {
|
|
106
|
+
this.isPausedFlag = true;
|
|
107
|
+
this.emit("paused", undefined);
|
|
108
|
+
}
|
|
109
|
+
}
|
|
110
|
+
start() {
|
|
111
|
+
if (this.isPausedFlag) {
|
|
112
|
+
this.isPausedFlag = false;
|
|
113
|
+
this.emit("resumed", undefined);
|
|
114
|
+
this.tryRunNext();
|
|
115
|
+
}
|
|
116
|
+
}
|
|
117
|
+
clear() {
|
|
118
|
+
const tasks = [...this.queue];
|
|
119
|
+
this.queue = [];
|
|
120
|
+
for (const task of tasks) {
|
|
121
|
+
task.reject(new Error("Queue was cleared"));
|
|
122
|
+
this.statsData.cancelled++;
|
|
123
|
+
this.emit("cancelled", { id: task.id });
|
|
124
|
+
}
|
|
125
|
+
this.checkEmpty();
|
|
126
|
+
}
|
|
127
|
+
cancel(id) {
|
|
128
|
+
const index = this.queue.findIndex((t) => t.id === id);
|
|
129
|
+
if (index !== -1) {
|
|
130
|
+
const [task] = this.queue.splice(index, 1);
|
|
131
|
+
task.reject(new Error("Task was cancelled"));
|
|
132
|
+
this.statsData.cancelled++;
|
|
133
|
+
this.emit("cancelled", { id });
|
|
134
|
+
this.checkEmpty();
|
|
135
|
+
return true;
|
|
136
|
+
}
|
|
137
|
+
return false;
|
|
138
|
+
}
|
|
139
|
+
cancelBy(predicate) {
|
|
140
|
+
let count = 0;
|
|
141
|
+
const remaining = [];
|
|
142
|
+
for (const task of this.queue) {
|
|
143
|
+
if (predicate({ id: task.id, priority: task.priority })) {
|
|
144
|
+
task.reject(new Error("Task was cancelled"));
|
|
145
|
+
this.statsData.cancelled++;
|
|
146
|
+
this.emit("cancelled", { id: task.id });
|
|
147
|
+
count++;
|
|
148
|
+
} else {
|
|
149
|
+
remaining.push(task);
|
|
150
|
+
}
|
|
151
|
+
}
|
|
152
|
+
this.queue = remaining;
|
|
153
|
+
if (count > 0) {
|
|
154
|
+
this.checkEmpty();
|
|
155
|
+
}
|
|
156
|
+
return count;
|
|
157
|
+
}
|
|
158
|
+
onIdle() {
|
|
159
|
+
if (this.state.isIdle) {
|
|
160
|
+
return Promise.resolve();
|
|
161
|
+
}
|
|
162
|
+
if (!this.idlePromise) {
|
|
163
|
+
let resolvePromise;
|
|
164
|
+
const promise = new Promise((resolve) => {
|
|
165
|
+
resolvePromise = resolve;
|
|
166
|
+
});
|
|
167
|
+
this.idlePromise = { promise, resolve: resolvePromise };
|
|
168
|
+
}
|
|
169
|
+
return this.idlePromise.promise;
|
|
170
|
+
}
|
|
171
|
+
onEmpty() {
|
|
172
|
+
if (this.queue.length === 0) {
|
|
173
|
+
return Promise.resolve();
|
|
174
|
+
}
|
|
175
|
+
if (!this.emptyPromise) {
|
|
176
|
+
let resolvePromise;
|
|
177
|
+
const promise = new Promise((resolve) => {
|
|
178
|
+
resolvePromise = resolve;
|
|
179
|
+
});
|
|
180
|
+
this.emptyPromise = { promise, resolve: resolvePromise };
|
|
181
|
+
}
|
|
182
|
+
return this.emptyPromise.promise;
|
|
183
|
+
}
|
|
184
|
+
onSizeLessThan(limit) {
|
|
185
|
+
if (this.queue.length < limit) {
|
|
186
|
+
return Promise.resolve();
|
|
187
|
+
}
|
|
188
|
+
return new Promise((resolve) => {
|
|
189
|
+
const check = () => {
|
|
190
|
+
if (this.queue.length < limit) {
|
|
191
|
+
this.off("completed", check);
|
|
192
|
+
this.off("cancelled", check);
|
|
193
|
+
this.off("error", check);
|
|
194
|
+
resolve();
|
|
195
|
+
}
|
|
196
|
+
};
|
|
197
|
+
this.on("completed", check);
|
|
198
|
+
this.on("cancelled", check);
|
|
199
|
+
this.on("error", check);
|
|
200
|
+
});
|
|
201
|
+
}
|
|
202
|
+
on(event, handler) {
|
|
203
|
+
if (!this.eventHandlers.has(event)) {
|
|
204
|
+
this.eventHandlers.set(event, new Set);
|
|
205
|
+
}
|
|
206
|
+
this.eventHandlers.get(event).add(handler);
|
|
207
|
+
}
|
|
208
|
+
off(event, handler) {
|
|
209
|
+
this.eventHandlers.get(event)?.delete(handler);
|
|
210
|
+
}
|
|
211
|
+
destroy() {
|
|
212
|
+
this.clear();
|
|
213
|
+
if (this.intervalId) {
|
|
214
|
+
clearInterval(this.intervalId);
|
|
215
|
+
this.intervalId = undefined;
|
|
216
|
+
}
|
|
217
|
+
this.eventHandlers.clear();
|
|
218
|
+
}
|
|
219
|
+
insertByPriority(task) {
|
|
220
|
+
let insertIndex = this.queue.length;
|
|
221
|
+
for (let i = 0;i < this.queue.length; i++) {
|
|
222
|
+
if (task.priority > this.queue[i].priority) {
|
|
223
|
+
insertIndex = i;
|
|
224
|
+
break;
|
|
225
|
+
}
|
|
226
|
+
}
|
|
227
|
+
this.queue.splice(insertIndex, 0, task);
|
|
228
|
+
}
|
|
229
|
+
tryRunNext() {
|
|
230
|
+
if (this.isPausedFlag)
|
|
231
|
+
return;
|
|
232
|
+
if (this.queue.length === 0)
|
|
233
|
+
return;
|
|
234
|
+
if (this.pendingCount >= this.config.concurrency)
|
|
235
|
+
return;
|
|
236
|
+
if (this.config.interval > 0) {
|
|
237
|
+
if (this.intervalCount >= this.config.intervalCap)
|
|
238
|
+
return;
|
|
239
|
+
this.intervalCount++;
|
|
240
|
+
}
|
|
241
|
+
const task = this.queue.shift();
|
|
242
|
+
this.runTask(task);
|
|
243
|
+
this.tryRunNext();
|
|
244
|
+
}
|
|
245
|
+
async runTask(task) {
|
|
246
|
+
this.pendingCount++;
|
|
247
|
+
this.statsData.processed++;
|
|
248
|
+
const wasIdle = this.pendingCount === 1 && this.queue.length === 0;
|
|
249
|
+
if (wasIdle) {
|
|
250
|
+
this.emit("active", undefined);
|
|
251
|
+
}
|
|
252
|
+
this.emit("start", { id: task.id });
|
|
253
|
+
this.emit("next", undefined);
|
|
254
|
+
const startTime = Date.now();
|
|
255
|
+
let timeoutId;
|
|
256
|
+
try {
|
|
257
|
+
let result;
|
|
258
|
+
if (task.timeout && task.timeout > 0) {
|
|
259
|
+
result = await Promise.race([
|
|
260
|
+
task.fn(),
|
|
261
|
+
new Promise((_, reject) => {
|
|
262
|
+
timeoutId = setTimeout(() => {
|
|
263
|
+
this.statsData.timedOut++;
|
|
264
|
+
this.emit("timeout", { id: task.id });
|
|
265
|
+
if (this.config.throwOnTimeout) {
|
|
266
|
+
reject(new Error(`Task ${task.id} timed out after ${task.timeout}ms`));
|
|
267
|
+
}
|
|
268
|
+
}, task.timeout);
|
|
269
|
+
})
|
|
270
|
+
]);
|
|
271
|
+
} else {
|
|
272
|
+
result = await task.fn();
|
|
273
|
+
}
|
|
274
|
+
if (timeoutId)
|
|
275
|
+
clearTimeout(timeoutId);
|
|
276
|
+
const duration = Date.now() - startTime;
|
|
277
|
+
this.recordDuration(duration);
|
|
278
|
+
this.statsData.completed++;
|
|
279
|
+
this.emit("completed", { id: task.id, result, duration });
|
|
280
|
+
task.resolve(result);
|
|
281
|
+
} catch (error) {
|
|
282
|
+
if (timeoutId)
|
|
283
|
+
clearTimeout(timeoutId);
|
|
284
|
+
this.statsData.failed++;
|
|
285
|
+
this.emit("error", { id: task.id, error });
|
|
286
|
+
task.reject(error);
|
|
287
|
+
} finally {
|
|
288
|
+
this.pendingCount--;
|
|
289
|
+
this.checkEmpty();
|
|
290
|
+
this.checkIdle();
|
|
291
|
+
this.tryRunNext();
|
|
292
|
+
}
|
|
293
|
+
}
|
|
294
|
+
recordDuration(duration) {
|
|
295
|
+
this.totalDuration += duration;
|
|
296
|
+
this.statsData.averageDuration = this.totalDuration / this.statsData.completed;
|
|
297
|
+
const now = Date.now();
|
|
298
|
+
this.throughputWindow.push(now);
|
|
299
|
+
const windowStart = now - this.throughputWindowSize * 1000;
|
|
300
|
+
this.throughputWindow = this.throughputWindow.filter((t) => t > windowStart);
|
|
301
|
+
this.statsData.throughput = this.throughputWindow.length / this.throughputWindowSize;
|
|
302
|
+
}
|
|
303
|
+
startInterval() {
|
|
304
|
+
this.intervalStart = Date.now();
|
|
305
|
+
this.intervalCount = 0;
|
|
306
|
+
this.intervalId = setInterval(() => {
|
|
307
|
+
if (!this.config.carryoverConcurrencyCount) {
|
|
308
|
+
this.intervalCount = 0;
|
|
309
|
+
} else {
|
|
310
|
+
this.intervalCount = Math.max(0, this.intervalCount - this.config.intervalCap);
|
|
311
|
+
}
|
|
312
|
+
this.intervalStart = Date.now();
|
|
313
|
+
this.tryRunNext();
|
|
314
|
+
}, this.config.interval);
|
|
315
|
+
}
|
|
316
|
+
emit(event, data) {
|
|
317
|
+
const handlers = this.eventHandlers.get(event);
|
|
318
|
+
if (handlers) {
|
|
319
|
+
for (const handler of handlers) {
|
|
320
|
+
try {
|
|
321
|
+
handler(data);
|
|
322
|
+
} catch {}
|
|
323
|
+
}
|
|
324
|
+
}
|
|
325
|
+
}
|
|
326
|
+
checkEmpty() {
|
|
327
|
+
if (this.queue.length === 0) {
|
|
328
|
+
this.emit("empty", undefined);
|
|
329
|
+
if (this.emptyPromise) {
|
|
330
|
+
this.emptyPromise.resolve();
|
|
331
|
+
this.emptyPromise = undefined;
|
|
332
|
+
}
|
|
333
|
+
}
|
|
334
|
+
}
|
|
335
|
+
checkIdle() {
|
|
336
|
+
if (this.state.isIdle) {
|
|
337
|
+
this.emit("idle", undefined);
|
|
338
|
+
if (this.idlePromise) {
|
|
339
|
+
this.idlePromise.resolve();
|
|
340
|
+
this.idlePromise = undefined;
|
|
341
|
+
}
|
|
342
|
+
}
|
|
343
|
+
}
|
|
344
|
+
}
|
|
345
|
+
|
|
346
|
+
exports.RezoQueue = RezoQueue;
|
|
@@ -0,0 +1,344 @@
|
|
|
1
|
+
function generateId() {
|
|
2
|
+
return `${Date.now().toString(36)}-${Math.random().toString(36).slice(2, 9)}`;
|
|
3
|
+
}
|
|
4
|
+
|
|
5
|
+
export class RezoQueue {
|
|
6
|
+
queue = [];
|
|
7
|
+
pendingCount = 0;
|
|
8
|
+
isPausedFlag = false;
|
|
9
|
+
intervalId;
|
|
10
|
+
intervalCount = 0;
|
|
11
|
+
intervalStart = 0;
|
|
12
|
+
eventHandlers = new Map;
|
|
13
|
+
statsData = {
|
|
14
|
+
added: 0,
|
|
15
|
+
processed: 0,
|
|
16
|
+
completed: 0,
|
|
17
|
+
failed: 0,
|
|
18
|
+
timedOut: 0,
|
|
19
|
+
cancelled: 0,
|
|
20
|
+
averageDuration: 0,
|
|
21
|
+
throughput: 0
|
|
22
|
+
};
|
|
23
|
+
totalDuration = 0;
|
|
24
|
+
throughputWindow = [];
|
|
25
|
+
throughputWindowSize = 60;
|
|
26
|
+
idlePromise;
|
|
27
|
+
emptyPromise;
|
|
28
|
+
config;
|
|
29
|
+
constructor(config = {}) {
|
|
30
|
+
this.config = {
|
|
31
|
+
concurrency: config.concurrency ?? 1 / 0,
|
|
32
|
+
autoStart: config.autoStart ?? true,
|
|
33
|
+
timeout: config.timeout ?? 0,
|
|
34
|
+
throwOnTimeout: config.throwOnTimeout ?? true,
|
|
35
|
+
interval: config.interval ?? 0,
|
|
36
|
+
intervalCap: config.intervalCap ?? 1 / 0,
|
|
37
|
+
carryoverConcurrencyCount: config.carryoverConcurrencyCount ?? false
|
|
38
|
+
};
|
|
39
|
+
if (!this.config.autoStart) {
|
|
40
|
+
this.isPausedFlag = true;
|
|
41
|
+
}
|
|
42
|
+
if (this.config.interval > 0) {
|
|
43
|
+
this.startInterval();
|
|
44
|
+
}
|
|
45
|
+
}
|
|
46
|
+
get state() {
|
|
47
|
+
return {
|
|
48
|
+
pending: this.pendingCount,
|
|
49
|
+
size: this.queue.length,
|
|
50
|
+
total: this.pendingCount + this.queue.length,
|
|
51
|
+
isPaused: this.isPausedFlag,
|
|
52
|
+
isIdle: this.pendingCount === 0 && this.queue.length === 0
|
|
53
|
+
};
|
|
54
|
+
}
|
|
55
|
+
get stats() {
|
|
56
|
+
return { ...this.statsData };
|
|
57
|
+
}
|
|
58
|
+
get concurrency() {
|
|
59
|
+
return this.config.concurrency;
|
|
60
|
+
}
|
|
61
|
+
set concurrency(value) {
|
|
62
|
+
this.config.concurrency = value;
|
|
63
|
+
this.tryRunNext();
|
|
64
|
+
}
|
|
65
|
+
get pending() {
|
|
66
|
+
return this.pendingCount;
|
|
67
|
+
}
|
|
68
|
+
get size() {
|
|
69
|
+
return this.queue.length;
|
|
70
|
+
}
|
|
71
|
+
get isPaused() {
|
|
72
|
+
return this.isPausedFlag;
|
|
73
|
+
}
|
|
74
|
+
add(fn, options = {}) {
|
|
75
|
+
return new Promise((resolve, reject) => {
|
|
76
|
+
const task = {
|
|
77
|
+
id: options.id ?? generateId(),
|
|
78
|
+
fn,
|
|
79
|
+
priority: options.priority ?? 0,
|
|
80
|
+
timeout: options.timeout ?? this.config.timeout,
|
|
81
|
+
signal: options.signal,
|
|
82
|
+
resolve,
|
|
83
|
+
reject,
|
|
84
|
+
addedAt: Date.now()
|
|
85
|
+
};
|
|
86
|
+
if (options.signal?.aborted) {
|
|
87
|
+
reject(new Error("Task was cancelled before starting"));
|
|
88
|
+
return;
|
|
89
|
+
}
|
|
90
|
+
options.signal?.addEventListener("abort", () => {
|
|
91
|
+
this.cancel(task.id);
|
|
92
|
+
});
|
|
93
|
+
this.insertByPriority(task);
|
|
94
|
+
this.statsData.added++;
|
|
95
|
+
this.emit("add", { id: task.id, priority: task.priority });
|
|
96
|
+
if (this.config.autoStart && !this.isPausedFlag) {
|
|
97
|
+
this.tryRunNext();
|
|
98
|
+
}
|
|
99
|
+
});
|
|
100
|
+
}
|
|
101
|
+
addAll(fns, options = {}) {
|
|
102
|
+
return Promise.all(fns.map((fn, i) => this.add(fn, { ...options, id: options.id ? `${options.id}-${i}` : undefined })));
|
|
103
|
+
}
|
|
104
|
+
pause() {
|
|
105
|
+
if (!this.isPausedFlag) {
|
|
106
|
+
this.isPausedFlag = true;
|
|
107
|
+
this.emit("paused", undefined);
|
|
108
|
+
}
|
|
109
|
+
}
|
|
110
|
+
start() {
|
|
111
|
+
if (this.isPausedFlag) {
|
|
112
|
+
this.isPausedFlag = false;
|
|
113
|
+
this.emit("resumed", undefined);
|
|
114
|
+
this.tryRunNext();
|
|
115
|
+
}
|
|
116
|
+
}
|
|
117
|
+
clear() {
|
|
118
|
+
const tasks = [...this.queue];
|
|
119
|
+
this.queue = [];
|
|
120
|
+
for (const task of tasks) {
|
|
121
|
+
task.reject(new Error("Queue was cleared"));
|
|
122
|
+
this.statsData.cancelled++;
|
|
123
|
+
this.emit("cancelled", { id: task.id });
|
|
124
|
+
}
|
|
125
|
+
this.checkEmpty();
|
|
126
|
+
}
|
|
127
|
+
cancel(id) {
|
|
128
|
+
const index = this.queue.findIndex((t) => t.id === id);
|
|
129
|
+
if (index !== -1) {
|
|
130
|
+
const [task] = this.queue.splice(index, 1);
|
|
131
|
+
task.reject(new Error("Task was cancelled"));
|
|
132
|
+
this.statsData.cancelled++;
|
|
133
|
+
this.emit("cancelled", { id });
|
|
134
|
+
this.checkEmpty();
|
|
135
|
+
return true;
|
|
136
|
+
}
|
|
137
|
+
return false;
|
|
138
|
+
}
|
|
139
|
+
cancelBy(predicate) {
|
|
140
|
+
let count = 0;
|
|
141
|
+
const remaining = [];
|
|
142
|
+
for (const task of this.queue) {
|
|
143
|
+
if (predicate({ id: task.id, priority: task.priority })) {
|
|
144
|
+
task.reject(new Error("Task was cancelled"));
|
|
145
|
+
this.statsData.cancelled++;
|
|
146
|
+
this.emit("cancelled", { id: task.id });
|
|
147
|
+
count++;
|
|
148
|
+
} else {
|
|
149
|
+
remaining.push(task);
|
|
150
|
+
}
|
|
151
|
+
}
|
|
152
|
+
this.queue = remaining;
|
|
153
|
+
if (count > 0) {
|
|
154
|
+
this.checkEmpty();
|
|
155
|
+
}
|
|
156
|
+
return count;
|
|
157
|
+
}
|
|
158
|
+
onIdle() {
|
|
159
|
+
if (this.state.isIdle) {
|
|
160
|
+
return Promise.resolve();
|
|
161
|
+
}
|
|
162
|
+
if (!this.idlePromise) {
|
|
163
|
+
let resolvePromise;
|
|
164
|
+
const promise = new Promise((resolve) => {
|
|
165
|
+
resolvePromise = resolve;
|
|
166
|
+
});
|
|
167
|
+
this.idlePromise = { promise, resolve: resolvePromise };
|
|
168
|
+
}
|
|
169
|
+
return this.idlePromise.promise;
|
|
170
|
+
}
|
|
171
|
+
onEmpty() {
|
|
172
|
+
if (this.queue.length === 0) {
|
|
173
|
+
return Promise.resolve();
|
|
174
|
+
}
|
|
175
|
+
if (!this.emptyPromise) {
|
|
176
|
+
let resolvePromise;
|
|
177
|
+
const promise = new Promise((resolve) => {
|
|
178
|
+
resolvePromise = resolve;
|
|
179
|
+
});
|
|
180
|
+
this.emptyPromise = { promise, resolve: resolvePromise };
|
|
181
|
+
}
|
|
182
|
+
return this.emptyPromise.promise;
|
|
183
|
+
}
|
|
184
|
+
onSizeLessThan(limit) {
|
|
185
|
+
if (this.queue.length < limit) {
|
|
186
|
+
return Promise.resolve();
|
|
187
|
+
}
|
|
188
|
+
return new Promise((resolve) => {
|
|
189
|
+
const check = () => {
|
|
190
|
+
if (this.queue.length < limit) {
|
|
191
|
+
this.off("completed", check);
|
|
192
|
+
this.off("cancelled", check);
|
|
193
|
+
this.off("error", check);
|
|
194
|
+
resolve();
|
|
195
|
+
}
|
|
196
|
+
};
|
|
197
|
+
this.on("completed", check);
|
|
198
|
+
this.on("cancelled", check);
|
|
199
|
+
this.on("error", check);
|
|
200
|
+
});
|
|
201
|
+
}
|
|
202
|
+
on(event, handler) {
|
|
203
|
+
if (!this.eventHandlers.has(event)) {
|
|
204
|
+
this.eventHandlers.set(event, new Set);
|
|
205
|
+
}
|
|
206
|
+
this.eventHandlers.get(event).add(handler);
|
|
207
|
+
}
|
|
208
|
+
off(event, handler) {
|
|
209
|
+
this.eventHandlers.get(event)?.delete(handler);
|
|
210
|
+
}
|
|
211
|
+
destroy() {
|
|
212
|
+
this.clear();
|
|
213
|
+
if (this.intervalId) {
|
|
214
|
+
clearInterval(this.intervalId);
|
|
215
|
+
this.intervalId = undefined;
|
|
216
|
+
}
|
|
217
|
+
this.eventHandlers.clear();
|
|
218
|
+
}
|
|
219
|
+
insertByPriority(task) {
|
|
220
|
+
let insertIndex = this.queue.length;
|
|
221
|
+
for (let i = 0;i < this.queue.length; i++) {
|
|
222
|
+
if (task.priority > this.queue[i].priority) {
|
|
223
|
+
insertIndex = i;
|
|
224
|
+
break;
|
|
225
|
+
}
|
|
226
|
+
}
|
|
227
|
+
this.queue.splice(insertIndex, 0, task);
|
|
228
|
+
}
|
|
229
|
+
tryRunNext() {
|
|
230
|
+
if (this.isPausedFlag)
|
|
231
|
+
return;
|
|
232
|
+
if (this.queue.length === 0)
|
|
233
|
+
return;
|
|
234
|
+
if (this.pendingCount >= this.config.concurrency)
|
|
235
|
+
return;
|
|
236
|
+
if (this.config.interval > 0) {
|
|
237
|
+
if (this.intervalCount >= this.config.intervalCap)
|
|
238
|
+
return;
|
|
239
|
+
this.intervalCount++;
|
|
240
|
+
}
|
|
241
|
+
const task = this.queue.shift();
|
|
242
|
+
this.runTask(task);
|
|
243
|
+
this.tryRunNext();
|
|
244
|
+
}
|
|
245
|
+
async runTask(task) {
|
|
246
|
+
this.pendingCount++;
|
|
247
|
+
this.statsData.processed++;
|
|
248
|
+
const wasIdle = this.pendingCount === 1 && this.queue.length === 0;
|
|
249
|
+
if (wasIdle) {
|
|
250
|
+
this.emit("active", undefined);
|
|
251
|
+
}
|
|
252
|
+
this.emit("start", { id: task.id });
|
|
253
|
+
this.emit("next", undefined);
|
|
254
|
+
const startTime = Date.now();
|
|
255
|
+
let timeoutId;
|
|
256
|
+
try {
|
|
257
|
+
let result;
|
|
258
|
+
if (task.timeout && task.timeout > 0) {
|
|
259
|
+
result = await Promise.race([
|
|
260
|
+
task.fn(),
|
|
261
|
+
new Promise((_, reject) => {
|
|
262
|
+
timeoutId = setTimeout(() => {
|
|
263
|
+
this.statsData.timedOut++;
|
|
264
|
+
this.emit("timeout", { id: task.id });
|
|
265
|
+
if (this.config.throwOnTimeout) {
|
|
266
|
+
reject(new Error(`Task ${task.id} timed out after ${task.timeout}ms`));
|
|
267
|
+
}
|
|
268
|
+
}, task.timeout);
|
|
269
|
+
})
|
|
270
|
+
]);
|
|
271
|
+
} else {
|
|
272
|
+
result = await task.fn();
|
|
273
|
+
}
|
|
274
|
+
if (timeoutId)
|
|
275
|
+
clearTimeout(timeoutId);
|
|
276
|
+
const duration = Date.now() - startTime;
|
|
277
|
+
this.recordDuration(duration);
|
|
278
|
+
this.statsData.completed++;
|
|
279
|
+
this.emit("completed", { id: task.id, result, duration });
|
|
280
|
+
task.resolve(result);
|
|
281
|
+
} catch (error) {
|
|
282
|
+
if (timeoutId)
|
|
283
|
+
clearTimeout(timeoutId);
|
|
284
|
+
this.statsData.failed++;
|
|
285
|
+
this.emit("error", { id: task.id, error });
|
|
286
|
+
task.reject(error);
|
|
287
|
+
} finally {
|
|
288
|
+
this.pendingCount--;
|
|
289
|
+
this.checkEmpty();
|
|
290
|
+
this.checkIdle();
|
|
291
|
+
this.tryRunNext();
|
|
292
|
+
}
|
|
293
|
+
}
|
|
294
|
+
recordDuration(duration) {
|
|
295
|
+
this.totalDuration += duration;
|
|
296
|
+
this.statsData.averageDuration = this.totalDuration / this.statsData.completed;
|
|
297
|
+
const now = Date.now();
|
|
298
|
+
this.throughputWindow.push(now);
|
|
299
|
+
const windowStart = now - this.throughputWindowSize * 1000;
|
|
300
|
+
this.throughputWindow = this.throughputWindow.filter((t) => t > windowStart);
|
|
301
|
+
this.statsData.throughput = this.throughputWindow.length / this.throughputWindowSize;
|
|
302
|
+
}
|
|
303
|
+
startInterval() {
|
|
304
|
+
this.intervalStart = Date.now();
|
|
305
|
+
this.intervalCount = 0;
|
|
306
|
+
this.intervalId = setInterval(() => {
|
|
307
|
+
if (!this.config.carryoverConcurrencyCount) {
|
|
308
|
+
this.intervalCount = 0;
|
|
309
|
+
} else {
|
|
310
|
+
this.intervalCount = Math.max(0, this.intervalCount - this.config.intervalCap);
|
|
311
|
+
}
|
|
312
|
+
this.intervalStart = Date.now();
|
|
313
|
+
this.tryRunNext();
|
|
314
|
+
}, this.config.interval);
|
|
315
|
+
}
|
|
316
|
+
emit(event, data) {
|
|
317
|
+
const handlers = this.eventHandlers.get(event);
|
|
318
|
+
if (handlers) {
|
|
319
|
+
for (const handler of handlers) {
|
|
320
|
+
try {
|
|
321
|
+
handler(data);
|
|
322
|
+
} catch {}
|
|
323
|
+
}
|
|
324
|
+
}
|
|
325
|
+
}
|
|
326
|
+
checkEmpty() {
|
|
327
|
+
if (this.queue.length === 0) {
|
|
328
|
+
this.emit("empty", undefined);
|
|
329
|
+
if (this.emptyPromise) {
|
|
330
|
+
this.emptyPromise.resolve();
|
|
331
|
+
this.emptyPromise = undefined;
|
|
332
|
+
}
|
|
333
|
+
}
|
|
334
|
+
}
|
|
335
|
+
checkIdle() {
|
|
336
|
+
if (this.state.isIdle) {
|
|
337
|
+
this.emit("idle", undefined);
|
|
338
|
+
if (this.idlePromise) {
|
|
339
|
+
this.idlePromise.resolve();
|
|
340
|
+
this.idlePromise = undefined;
|
|
341
|
+
}
|
|
342
|
+
}
|
|
343
|
+
}
|
|
344
|
+
}
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
const Priority = exports.Priority = {
|
|
2
|
+
LOWEST: 0,
|
|
3
|
+
LOW: 25,
|
|
4
|
+
NORMAL: 50,
|
|
5
|
+
HIGH: 75,
|
|
6
|
+
HIGHEST: 100,
|
|
7
|
+
CRITICAL: 1000
|
|
8
|
+
};
|
|
9
|
+
const HttpMethodPriority = exports.HttpMethodPriority = {
|
|
10
|
+
HEAD: 100,
|
|
11
|
+
GET: 75,
|
|
12
|
+
OPTIONS: 50,
|
|
13
|
+
POST: 50,
|
|
14
|
+
PUT: 50,
|
|
15
|
+
PATCH: 50,
|
|
16
|
+
DELETE: 25
|
|
17
|
+
};
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
export const Priority = {
|
|
2
|
+
LOWEST: 0,
|
|
3
|
+
LOW: 25,
|
|
4
|
+
NORMAL: 50,
|
|
5
|
+
HIGH: 75,
|
|
6
|
+
HIGHEST: 100,
|
|
7
|
+
CRITICAL: 1000
|
|
8
|
+
};
|
|
9
|
+
export const HttpMethodPriority = {
|
|
10
|
+
HEAD: 100,
|
|
11
|
+
GET: 75,
|
|
12
|
+
OPTIONS: 50,
|
|
13
|
+
POST: 50,
|
|
14
|
+
PUT: 50,
|
|
15
|
+
PATCH: 50,
|
|
16
|
+
DELETE: 25
|
|
17
|
+
};
|