power-queues 2.1.9 → 2.1.11
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +61 -36
- package/dist/index.cjs +53 -35
- package/dist/index.d.cts +13 -2
- package/dist/index.d.ts +13 -2
- package/dist/index.js +39 -38
- package/package.json +6 -4
package/README.md
CHANGED
|
@@ -41,15 +41,18 @@ const queue = new PowerQueues({
|
|
|
41
41
|
|
|
42
42
|
await queue.loadScripts(true);
|
|
43
43
|
|
|
44
|
-
await queue.addTasks('
|
|
45
|
-
{
|
|
46
|
-
{
|
|
44
|
+
await queue.addTasks('ws', [
|
|
45
|
+
{ body: 'welcome', userId: 42 },
|
|
46
|
+
{ body: 'hello', userId: 51 }
|
|
47
47
|
]);
|
|
48
48
|
```
|
|
49
49
|
|
|
50
|
-
Example of
|
|
50
|
+
Example of queue worker for sending message to client via WebSocket and executing a MySQL insert transaction:
|
|
51
51
|
|
|
52
52
|
``` ts
|
|
53
|
+
import express from 'express';
|
|
54
|
+
import http from 'http';
|
|
55
|
+
import { Server } from 'socket.io';
|
|
53
56
|
import mysql from 'mysql2/promise';
|
|
54
57
|
import Redis from 'ioredis';
|
|
55
58
|
import type { IORedisLike } from 'power-redis';
|
|
@@ -70,7 +73,11 @@ const pool = mysql.createPool({
|
|
|
70
73
|
});
|
|
71
74
|
const redis = new Redis('redis://127.0.0.1:6379');
|
|
72
75
|
|
|
73
|
-
|
|
76
|
+
const app = express();
|
|
77
|
+
const server = http.createServer(app);
|
|
78
|
+
const io = new Server(server);
|
|
79
|
+
|
|
80
|
+
export class WebSocketAndMysqlCreateQueue extends PowerQueues {
|
|
74
81
|
public readonly selectStuckCount: number = 256;
|
|
75
82
|
public readonly selectCount: number = 256;
|
|
76
83
|
public readonly retryCount: number = 3;
|
|
@@ -84,50 +91,68 @@ export class ExampleQueue extends PowerQueues {
|
|
|
84
91
|
this.redis = redis;
|
|
85
92
|
}
|
|
86
93
|
|
|
87
|
-
async
|
|
88
|
-
const
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
94
|
+
async onExecute(queueName: string, task: Task) {
|
|
95
|
+
const id = uuid();
|
|
96
|
+
|
|
97
|
+
io.to(`user:${task.payload.userId}`).emit('alerts', {
|
|
98
|
+
body: task.payload.body,
|
|
99
|
+
id,
|
|
100
|
+
});
|
|
101
|
+
return {
|
|
102
|
+
...task,
|
|
103
|
+
payload: {
|
|
104
|
+
...task.payload,
|
|
105
|
+
id,
|
|
106
|
+
},
|
|
107
|
+
};
|
|
108
|
+
}
|
|
102
109
|
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
110
|
+
async onBatchReady(queueName: string, tasks: Task[]) {
|
|
111
|
+
const values = tasks.map((task) => task.payload);
|
|
112
|
+
const conn = await pool.getConnection();
|
|
113
|
+
|
|
114
|
+
try {
|
|
115
|
+
await conn.beginTransaction();
|
|
116
|
+
|
|
117
|
+
const cols = Object.keys(values[0]);
|
|
118
|
+
const placeholder = `(${cols.map(() => '?').join(',')})`;
|
|
119
|
+
const sql = `INSERT INTO \`alerts\` (${cols.map((c) => `\`${c}\``).join(',')}) VALUES ${values.map(() => placeholder).join(',')}`;
|
|
120
|
+
const params = [];
|
|
121
|
+
|
|
122
|
+
for (const row of values) {
|
|
123
|
+
for (const c of cols) {
|
|
124
|
+
params.push(row[c]);
|
|
107
125
|
}
|
|
108
|
-
await conn.execute(sql, params);
|
|
109
|
-
await conn.commit();
|
|
110
|
-
}
|
|
111
|
-
catch (err) {
|
|
112
|
-
await conn.rollback();
|
|
113
|
-
throw err;
|
|
114
|
-
}
|
|
115
|
-
finally {
|
|
116
|
-
conn.release();
|
|
117
126
|
}
|
|
127
|
+
await conn.execute(sql, params);
|
|
128
|
+
await conn.commit();
|
|
129
|
+
}
|
|
130
|
+
catch (err) {
|
|
131
|
+
await queryRunner.rollbackTransaction();
|
|
132
|
+
throw err;
|
|
133
|
+
}
|
|
134
|
+
finally {
|
|
135
|
+
await queryRunner.release();
|
|
118
136
|
}
|
|
119
137
|
}
|
|
120
138
|
|
|
139
|
+
async onError(err: any, queueName: string, task: Task): Promise<Task> {
|
|
140
|
+
console.error('Alert error', queueName, task, (process.env.NODE_ENV === 'production')
|
|
141
|
+
? err.message
|
|
142
|
+
: err);
|
|
143
|
+
return task;
|
|
144
|
+
}
|
|
145
|
+
|
|
121
146
|
async onBatchError(err: any, queueName: string, tasks: Array<[ string, any, number, string, string, number ]>) {
|
|
122
|
-
|
|
147
|
+
console.error('Batch error', queueName, tasks.length, (process.env.NODE_ENV === 'production')
|
|
123
148
|
? err.message
|
|
124
|
-
: err
|
|
149
|
+
: err);
|
|
125
150
|
}
|
|
126
151
|
}
|
|
127
152
|
|
|
128
153
|
const exampleQueue = new ExampleQueue();
|
|
129
154
|
|
|
130
|
-
exampleQueue.runQueue('
|
|
155
|
+
exampleQueue.runQueue('ws');
|
|
131
156
|
```
|
|
132
157
|
|
|
133
158
|
## ⚖️ power-queues vs Existing Solutions
|
package/dist/index.cjs
CHANGED
|
@@ -1,7 +1,9 @@
|
|
|
1
1
|
"use strict";
|
|
2
|
+
var __create = Object.create;
|
|
2
3
|
var __defProp = Object.defineProperty;
|
|
3
4
|
var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
|
|
4
5
|
var __getOwnPropNames = Object.getOwnPropertyNames;
|
|
6
|
+
var __getProtoOf = Object.getPrototypeOf;
|
|
5
7
|
var __hasOwnProp = Object.prototype.hasOwnProperty;
|
|
6
8
|
var __export = (target, all) => {
|
|
7
9
|
for (var name in all)
|
|
@@ -15,20 +17,29 @@ var __copyProps = (to, from, except, desc) => {
|
|
|
15
17
|
}
|
|
16
18
|
return to;
|
|
17
19
|
};
|
|
20
|
+
var __toESM = (mod, isNodeMode, target) => (target = mod != null ? __create(__getProtoOf(mod)) : {}, __copyProps(
|
|
21
|
+
// If the importer is in node compatibility mode or this is not an ESM
|
|
22
|
+
// file that has been converted to a CommonJS file using a Babel-
|
|
23
|
+
// compatible transform (i.e. "__esModule" has not been set), then set
|
|
24
|
+
// "default" to the CommonJS "module.exports" for node compatibility.
|
|
25
|
+
isNodeMode || !mod || !mod.__esModule ? __defProp(target, "default", { value: mod, enumerable: true }) : target,
|
|
26
|
+
mod
|
|
27
|
+
));
|
|
18
28
|
var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
|
|
19
29
|
|
|
20
30
|
// src/index.ts
|
|
21
31
|
var index_exports = {};
|
|
22
32
|
__export(index_exports, {
|
|
23
|
-
PowerQueues: () => PowerQueues
|
|
33
|
+
PowerQueues: () => PowerQueues,
|
|
34
|
+
wait: () => wait
|
|
24
35
|
});
|
|
25
36
|
module.exports = __toCommonJS(index_exports);
|
|
26
37
|
|
|
27
38
|
// src/PowerQueues.ts
|
|
39
|
+
var import_p_limit = __toESM(require("p-limit"), 1);
|
|
40
|
+
var import_node_crypto = require("crypto");
|
|
28
41
|
var import_node_events = require("events");
|
|
29
42
|
var import_power_redis = require("power-redis");
|
|
30
|
-
var import_full_utils = require("full-utils");
|
|
31
|
-
var import_uuid = require("uuid");
|
|
32
43
|
|
|
33
44
|
// src/scripts.ts
|
|
34
45
|
var XAddBulk = `
|
|
@@ -265,6 +276,9 @@ var SelectStuck = `
|
|
|
265
276
|
`;
|
|
266
277
|
|
|
267
278
|
// src/PowerQueues.ts
|
|
279
|
+
async function wait(timeout = 0) {
|
|
280
|
+
await new Promise((resolve) => setTimeout(() => resolve(true), timeout));
|
|
281
|
+
}
|
|
268
282
|
var PowerQueues = class extends import_power_redis.PowerRedis {
|
|
269
283
|
constructor() {
|
|
270
284
|
super(...arguments);
|
|
@@ -272,21 +286,23 @@ var PowerQueues = class extends import_power_redis.PowerRedis {
|
|
|
272
286
|
this.scripts = {};
|
|
273
287
|
this.host = "host";
|
|
274
288
|
this.group = "gr1";
|
|
275
|
-
this.selectStuckCount =
|
|
276
|
-
this.selectStuckTimeout =
|
|
277
|
-
this.selectStuckMaxTimeout =
|
|
278
|
-
this.selectCount =
|
|
279
|
-
this.selectTimeout =
|
|
280
|
-
this.buildBatchCount =
|
|
281
|
-
this.buildBatchMaxCount =
|
|
289
|
+
this.selectStuckCount = 256;
|
|
290
|
+
this.selectStuckTimeout = 2e4;
|
|
291
|
+
this.selectStuckMaxTimeout = 40;
|
|
292
|
+
this.selectCount = 1024;
|
|
293
|
+
this.selectTimeout = 100;
|
|
294
|
+
this.buildBatchCount = 1e3;
|
|
295
|
+
this.buildBatchMaxCount = 2e4;
|
|
282
296
|
this.retryCount = 1;
|
|
283
297
|
this.executeSync = false;
|
|
284
|
-
this.idemLockTimeout =
|
|
285
|
-
this.idemDoneTimeout =
|
|
298
|
+
this.idemLockTimeout = 15e3;
|
|
299
|
+
this.idemDoneTimeout = 1e4;
|
|
286
300
|
this.logStatus = false;
|
|
287
|
-
this.logStatusTimeout =
|
|
288
|
-
this.approveCount =
|
|
301
|
+
this.logStatusTimeout = 12e4;
|
|
302
|
+
this.approveCount = 4e3;
|
|
289
303
|
this.removeOnExecuted = true;
|
|
304
|
+
this.concurrency = 256;
|
|
305
|
+
this.limit = (0, import_p_limit.default)(this.concurrency);
|
|
290
306
|
}
|
|
291
307
|
signal() {
|
|
292
308
|
return this.abort.signal;
|
|
@@ -296,6 +312,7 @@ var PowerQueues = class extends import_power_redis.PowerRedis {
|
|
|
296
312
|
}
|
|
297
313
|
async runQueue(queueName, from = "0-0") {
|
|
298
314
|
(0, import_node_events.setMaxListeners)(0, this.abort.signal);
|
|
315
|
+
this.limit = (0, import_p_limit.default)(this.concurrency);
|
|
299
316
|
await this.createGroup(queueName, from);
|
|
300
317
|
await this.consumerLoop(queueName, from);
|
|
301
318
|
}
|
|
@@ -315,10 +332,10 @@ var PowerQueues = class extends import_power_redis.PowerRedis {
|
|
|
315
332
|
let tasks = [];
|
|
316
333
|
try {
|
|
317
334
|
tasks = await this.select(queueName, from);
|
|
318
|
-
} catch
|
|
335
|
+
} catch {
|
|
319
336
|
}
|
|
320
|
-
if (!(
|
|
321
|
-
await
|
|
337
|
+
if (!Array.isArray(tasks) || !(tasks.length > 0)) {
|
|
338
|
+
await wait(300);
|
|
322
339
|
continue;
|
|
323
340
|
}
|
|
324
341
|
try {
|
|
@@ -336,7 +353,7 @@ var PowerQueues = class extends import_power_redis.PowerRedis {
|
|
|
336
353
|
})));
|
|
337
354
|
} catch {
|
|
338
355
|
}
|
|
339
|
-
await
|
|
356
|
+
await wait(300);
|
|
340
357
|
}
|
|
341
358
|
}
|
|
342
359
|
}
|
|
@@ -377,7 +394,7 @@ var PowerQueues = class extends import_power_redis.PowerRedis {
|
|
|
377
394
|
}
|
|
378
395
|
}
|
|
379
396
|
async approve(queueName, tasks) {
|
|
380
|
-
if (!(
|
|
397
|
+
if (!Array.isArray(tasks) || !(tasks.length > 0)) {
|
|
381
398
|
return 0;
|
|
382
399
|
}
|
|
383
400
|
const approveCount = Math.max(500, Math.min(4e3, this.approveCount));
|
|
@@ -393,7 +410,7 @@ var PowerQueues = class extends import_power_redis.PowerRedis {
|
|
|
393
410
|
}
|
|
394
411
|
async select(queueName, from = "0-0") {
|
|
395
412
|
let selected = await this.selectS(queueName, from);
|
|
396
|
-
if (!(
|
|
413
|
+
if (!Array.isArray(selected) || !(selected.length > 0)) {
|
|
397
414
|
selected = await this.selectF(queueName, from);
|
|
398
415
|
}
|
|
399
416
|
return this.selectP(selected);
|
|
@@ -401,7 +418,7 @@ var PowerQueues = class extends import_power_redis.PowerRedis {
|
|
|
401
418
|
async selectS(queueName, from = "0-0") {
|
|
402
419
|
try {
|
|
403
420
|
const res = await this.runScript("SelectStuck", [queueName], [this.group, this.consumer(), String(this.selectStuckTimeout), String(this.selectStuckCount), String(this.selectStuckMaxTimeout)], SelectStuck);
|
|
404
|
-
return
|
|
421
|
+
return Array.isArray(res) ? res : [];
|
|
405
422
|
} catch (err) {
|
|
406
423
|
if (String(err?.message || "").includes("NOGROUP")) {
|
|
407
424
|
await this.createGroup(queueName, from);
|
|
@@ -425,7 +442,7 @@ var PowerQueues = class extends import_power_redis.PowerRedis {
|
|
|
425
442
|
">"
|
|
426
443
|
);
|
|
427
444
|
rows = res?.[0]?.[1] ?? [];
|
|
428
|
-
if (!(
|
|
445
|
+
if (!Array.isArray(rows) || !(rows.length > 0)) {
|
|
429
446
|
return [];
|
|
430
447
|
}
|
|
431
448
|
} catch (err) {
|
|
@@ -442,9 +459,9 @@ var PowerQueues = class extends import_power_redis.PowerRedis {
|
|
|
442
459
|
return Array.from(raw || []).map((e) => {
|
|
443
460
|
const id = Buffer.isBuffer(e?.[0]) ? e[0].toString() : e?.[0];
|
|
444
461
|
const kvRaw = e?.[1] ?? [];
|
|
445
|
-
const kv =
|
|
462
|
+
const kv = Array.isArray(kvRaw) ? kvRaw.map((x) => Buffer.isBuffer(x) ? x.toString() : x) : [];
|
|
446
463
|
return [id, kv];
|
|
447
|
-
}).filter(([id, kv]) =>
|
|
464
|
+
}).filter(([id, kv]) => typeof id === "string" && id.length > 0 && Array.isArray(kv) && (kv.length & 1) === 0).map(([id, kv]) => {
|
|
448
465
|
const { payload, createdAt, job, idemKey, attempt } = this.values(kv);
|
|
449
466
|
return [id, this.payload(payload), createdAt, job, idemKey, Number(attempt)];
|
|
450
467
|
});
|
|
@@ -487,10 +504,10 @@ var PowerQueues = class extends import_power_redis.PowerRedis {
|
|
|
487
504
|
}
|
|
488
505
|
let start = Date.now();
|
|
489
506
|
if (!this.executeSync && promises.length > 0) {
|
|
490
|
-
await Promise.all(promises.map((item) => item()));
|
|
507
|
+
await Promise.all(promises.map((item) => this.limit(() => item())));
|
|
491
508
|
}
|
|
492
509
|
await this.onBatchReady(queueName, result);
|
|
493
|
-
if (!(
|
|
510
|
+
if ((!Array.isArray(result) || !(result.length > 0)) && contended > tasks.length >> 1) {
|
|
494
511
|
await this.waitAbortable(15 + Math.floor(Math.random() * 35) + Math.min(250, 15 * contended + Math.floor(Math.random() * 40)));
|
|
495
512
|
}
|
|
496
513
|
return result;
|
|
@@ -668,7 +685,7 @@ var PowerQueues = class extends import_power_redis.PowerRedis {
|
|
|
668
685
|
}
|
|
669
686
|
async runScript(name, keys, args, defaultCode) {
|
|
670
687
|
if (!this.scripts[name]) {
|
|
671
|
-
if (!(
|
|
688
|
+
if (!(typeof defaultCode === "string" && defaultCode.length > 0)) {
|
|
672
689
|
throw new Error(`Undefined script "${name}". Save it before executing.`);
|
|
673
690
|
}
|
|
674
691
|
this.saveScript(name, defaultCode);
|
|
@@ -716,20 +733,20 @@ var PowerQueues = class extends import_power_redis.PowerRedis {
|
|
|
716
733
|
throw new Error("Load lua script failed.");
|
|
717
734
|
}
|
|
718
735
|
saveScript(name, codeBody) {
|
|
719
|
-
if (!(
|
|
736
|
+
if (!(typeof codeBody === "string" && codeBody.length > 0)) {
|
|
720
737
|
throw new Error("Script body is empty.");
|
|
721
738
|
}
|
|
722
739
|
this.scripts[name] = { codeBody };
|
|
723
740
|
return codeBody;
|
|
724
741
|
}
|
|
725
742
|
async addTasks(queueName, data, opts = {}) {
|
|
726
|
-
if (!(
|
|
743
|
+
if (!Array.isArray(data) || !(data.length > 0)) {
|
|
727
744
|
throw new Error("Tasks is not filled.");
|
|
728
745
|
}
|
|
729
|
-
if (!(
|
|
746
|
+
if (!(typeof queueName === "string" && queueName.length > 0)) {
|
|
730
747
|
throw new Error("Queue name is required.");
|
|
731
748
|
}
|
|
732
|
-
opts.job = opts.job ?? (0,
|
|
749
|
+
opts.job = opts.job ?? (0, import_node_crypto.randomUUID)();
|
|
733
750
|
const batches = this.buildBatches(data, opts);
|
|
734
751
|
const result = new Array(data.length);
|
|
735
752
|
const promises = [];
|
|
@@ -770,8 +787,8 @@ var PowerQueues = class extends import_power_redis.PowerRedis {
|
|
|
770
787
|
const entry = {
|
|
771
788
|
payload: JSON.stringify(taskP),
|
|
772
789
|
attempt: Number(opts.attempt || 0),
|
|
773
|
-
job: opts.job ?? (0,
|
|
774
|
-
idemKey: String(idemKey || (0,
|
|
790
|
+
job: opts.job ?? (0, import_node_crypto.randomUUID)(),
|
|
791
|
+
idemKey: String(idemKey || (0, import_node_crypto.randomUUID)()),
|
|
775
792
|
createdAt
|
|
776
793
|
};
|
|
777
794
|
const reqKeysLength = this.keysLength(entry);
|
|
@@ -827,7 +844,7 @@ var PowerQueues = class extends import_power_redis.PowerRedis {
|
|
|
827
844
|
argv.push(String(id));
|
|
828
845
|
argv.push(String(pairs));
|
|
829
846
|
for (const token of flat) {
|
|
830
|
-
argv.push(
|
|
847
|
+
argv.push(token === null || token === void 0 ? "" : typeof token === "string" && token.length > 0 ? token : String(token));
|
|
831
848
|
}
|
|
832
849
|
}
|
|
833
850
|
return argv;
|
|
@@ -853,5 +870,6 @@ var PowerQueues = class extends import_power_redis.PowerRedis {
|
|
|
853
870
|
};
|
|
854
871
|
// Annotate the CommonJS export names for ESM import in node:
|
|
855
872
|
0 && (module.exports = {
|
|
856
|
-
PowerQueues
|
|
873
|
+
PowerQueues,
|
|
874
|
+
wait
|
|
857
875
|
});
|
package/dist/index.d.cts
CHANGED
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import { PowerRedis, IORedisLike } from 'power-redis';
|
|
2
2
|
|
|
3
|
+
type Constructor<T = {}> = new (...args: any[]) => T;
|
|
3
4
|
type SavedScript = {
|
|
4
5
|
codeReady?: string;
|
|
5
6
|
codeBody: string;
|
|
@@ -18,6 +19,13 @@ type AddTasksOptions = {
|
|
|
18
19
|
attempt?: number;
|
|
19
20
|
createdAt?: number;
|
|
20
21
|
};
|
|
22
|
+
type IdempotencyKeys = {
|
|
23
|
+
prefix: string;
|
|
24
|
+
doneKey: string;
|
|
25
|
+
lockKey: string;
|
|
26
|
+
startKey: string;
|
|
27
|
+
token: string;
|
|
28
|
+
};
|
|
21
29
|
type Task = {
|
|
22
30
|
job: string;
|
|
23
31
|
id?: string;
|
|
@@ -27,8 +35,9 @@ type Task = {
|
|
|
27
35
|
attempt: number;
|
|
28
36
|
};
|
|
29
37
|
|
|
38
|
+
declare function wait(timeout?: number): Promise<void>;
|
|
30
39
|
declare class PowerQueues extends PowerRedis {
|
|
31
|
-
abort
|
|
40
|
+
private abort;
|
|
32
41
|
redis: IORedisLike;
|
|
33
42
|
readonly scripts: Record<string, SavedScript>;
|
|
34
43
|
readonly host: string;
|
|
@@ -48,6 +57,8 @@ declare class PowerQueues extends PowerRedis {
|
|
|
48
57
|
readonly logStatusTimeout: number;
|
|
49
58
|
readonly approveCount: number;
|
|
50
59
|
readonly removeOnExecuted: boolean;
|
|
60
|
+
readonly concurrency: number;
|
|
61
|
+
private limit;
|
|
51
62
|
private signal;
|
|
52
63
|
private consumer;
|
|
53
64
|
runQueue(queueName: string, from?: '$' | '0-0'): Promise<void>;
|
|
@@ -90,4 +101,4 @@ declare class PowerQueues extends PowerRedis {
|
|
|
90
101
|
onRetry(err: any, queueName: string, task: Task): Promise<void>;
|
|
91
102
|
}
|
|
92
103
|
|
|
93
|
-
export { type AddTasksOptions, PowerQueues, type Task };
|
|
104
|
+
export { type AddTasksOptions, type Constructor, type IdempotencyKeys, PowerQueues, type SavedScript, type Task, wait };
|
package/dist/index.d.ts
CHANGED
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import { PowerRedis, IORedisLike } from 'power-redis';
|
|
2
2
|
|
|
3
|
+
type Constructor<T = {}> = new (...args: any[]) => T;
|
|
3
4
|
type SavedScript = {
|
|
4
5
|
codeReady?: string;
|
|
5
6
|
codeBody: string;
|
|
@@ -18,6 +19,13 @@ type AddTasksOptions = {
|
|
|
18
19
|
attempt?: number;
|
|
19
20
|
createdAt?: number;
|
|
20
21
|
};
|
|
22
|
+
type IdempotencyKeys = {
|
|
23
|
+
prefix: string;
|
|
24
|
+
doneKey: string;
|
|
25
|
+
lockKey: string;
|
|
26
|
+
startKey: string;
|
|
27
|
+
token: string;
|
|
28
|
+
};
|
|
21
29
|
type Task = {
|
|
22
30
|
job: string;
|
|
23
31
|
id?: string;
|
|
@@ -27,8 +35,9 @@ type Task = {
|
|
|
27
35
|
attempt: number;
|
|
28
36
|
};
|
|
29
37
|
|
|
38
|
+
declare function wait(timeout?: number): Promise<void>;
|
|
30
39
|
declare class PowerQueues extends PowerRedis {
|
|
31
|
-
abort
|
|
40
|
+
private abort;
|
|
32
41
|
redis: IORedisLike;
|
|
33
42
|
readonly scripts: Record<string, SavedScript>;
|
|
34
43
|
readonly host: string;
|
|
@@ -48,6 +57,8 @@ declare class PowerQueues extends PowerRedis {
|
|
|
48
57
|
readonly logStatusTimeout: number;
|
|
49
58
|
readonly approveCount: number;
|
|
50
59
|
readonly removeOnExecuted: boolean;
|
|
60
|
+
readonly concurrency: number;
|
|
61
|
+
private limit;
|
|
51
62
|
private signal;
|
|
52
63
|
private consumer;
|
|
53
64
|
runQueue(queueName: string, from?: '$' | '0-0'): Promise<void>;
|
|
@@ -90,4 +101,4 @@ declare class PowerQueues extends PowerRedis {
|
|
|
90
101
|
onRetry(err: any, queueName: string, task: Task): Promise<void>;
|
|
91
102
|
}
|
|
92
103
|
|
|
93
|
-
export { type AddTasksOptions, PowerQueues, type Task };
|
|
104
|
+
export { type AddTasksOptions, type Constructor, type IdempotencyKeys, PowerQueues, type SavedScript, type Task, wait };
|
package/dist/index.js
CHANGED
|
@@ -1,14 +1,8 @@
|
|
|
1
1
|
// src/PowerQueues.ts
|
|
2
|
+
import pLimit from "p-limit";
|
|
3
|
+
import { randomUUID } from "crypto";
|
|
2
4
|
import { setMaxListeners } from "events";
|
|
3
5
|
import { PowerRedis } from "power-redis";
|
|
4
|
-
import {
|
|
5
|
-
isArrFilled,
|
|
6
|
-
isArr,
|
|
7
|
-
isStrFilled,
|
|
8
|
-
isExists,
|
|
9
|
-
wait
|
|
10
|
-
} from "full-utils";
|
|
11
|
-
import { v4 as uuid } from "uuid";
|
|
12
6
|
|
|
13
7
|
// src/scripts.ts
|
|
14
8
|
var XAddBulk = `
|
|
@@ -245,6 +239,9 @@ var SelectStuck = `
|
|
|
245
239
|
`;
|
|
246
240
|
|
|
247
241
|
// src/PowerQueues.ts
|
|
242
|
+
async function wait(timeout = 0) {
|
|
243
|
+
await new Promise((resolve) => setTimeout(() => resolve(true), timeout));
|
|
244
|
+
}
|
|
248
245
|
var PowerQueues = class extends PowerRedis {
|
|
249
246
|
constructor() {
|
|
250
247
|
super(...arguments);
|
|
@@ -252,21 +249,23 @@ var PowerQueues = class extends PowerRedis {
|
|
|
252
249
|
this.scripts = {};
|
|
253
250
|
this.host = "host";
|
|
254
251
|
this.group = "gr1";
|
|
255
|
-
this.selectStuckCount =
|
|
256
|
-
this.selectStuckTimeout =
|
|
257
|
-
this.selectStuckMaxTimeout =
|
|
258
|
-
this.selectCount =
|
|
259
|
-
this.selectTimeout =
|
|
260
|
-
this.buildBatchCount =
|
|
261
|
-
this.buildBatchMaxCount =
|
|
252
|
+
this.selectStuckCount = 256;
|
|
253
|
+
this.selectStuckTimeout = 2e4;
|
|
254
|
+
this.selectStuckMaxTimeout = 40;
|
|
255
|
+
this.selectCount = 1024;
|
|
256
|
+
this.selectTimeout = 100;
|
|
257
|
+
this.buildBatchCount = 1e3;
|
|
258
|
+
this.buildBatchMaxCount = 2e4;
|
|
262
259
|
this.retryCount = 1;
|
|
263
260
|
this.executeSync = false;
|
|
264
|
-
this.idemLockTimeout =
|
|
265
|
-
this.idemDoneTimeout =
|
|
261
|
+
this.idemLockTimeout = 15e3;
|
|
262
|
+
this.idemDoneTimeout = 1e4;
|
|
266
263
|
this.logStatus = false;
|
|
267
|
-
this.logStatusTimeout =
|
|
268
|
-
this.approveCount =
|
|
264
|
+
this.logStatusTimeout = 12e4;
|
|
265
|
+
this.approveCount = 4e3;
|
|
269
266
|
this.removeOnExecuted = true;
|
|
267
|
+
this.concurrency = 256;
|
|
268
|
+
this.limit = pLimit(this.concurrency);
|
|
270
269
|
}
|
|
271
270
|
signal() {
|
|
272
271
|
return this.abort.signal;
|
|
@@ -276,6 +275,7 @@ var PowerQueues = class extends PowerRedis {
|
|
|
276
275
|
}
|
|
277
276
|
async runQueue(queueName, from = "0-0") {
|
|
278
277
|
setMaxListeners(0, this.abort.signal);
|
|
278
|
+
this.limit = pLimit(this.concurrency);
|
|
279
279
|
await this.createGroup(queueName, from);
|
|
280
280
|
await this.consumerLoop(queueName, from);
|
|
281
281
|
}
|
|
@@ -295,9 +295,9 @@ var PowerQueues = class extends PowerRedis {
|
|
|
295
295
|
let tasks = [];
|
|
296
296
|
try {
|
|
297
297
|
tasks = await this.select(queueName, from);
|
|
298
|
-
} catch
|
|
298
|
+
} catch {
|
|
299
299
|
}
|
|
300
|
-
if (!
|
|
300
|
+
if (!Array.isArray(tasks) || !(tasks.length > 0)) {
|
|
301
301
|
await wait(300);
|
|
302
302
|
continue;
|
|
303
303
|
}
|
|
@@ -357,7 +357,7 @@ var PowerQueues = class extends PowerRedis {
|
|
|
357
357
|
}
|
|
358
358
|
}
|
|
359
359
|
async approve(queueName, tasks) {
|
|
360
|
-
if (!
|
|
360
|
+
if (!Array.isArray(tasks) || !(tasks.length > 0)) {
|
|
361
361
|
return 0;
|
|
362
362
|
}
|
|
363
363
|
const approveCount = Math.max(500, Math.min(4e3, this.approveCount));
|
|
@@ -373,7 +373,7 @@ var PowerQueues = class extends PowerRedis {
|
|
|
373
373
|
}
|
|
374
374
|
async select(queueName, from = "0-0") {
|
|
375
375
|
let selected = await this.selectS(queueName, from);
|
|
376
|
-
if (!
|
|
376
|
+
if (!Array.isArray(selected) || !(selected.length > 0)) {
|
|
377
377
|
selected = await this.selectF(queueName, from);
|
|
378
378
|
}
|
|
379
379
|
return this.selectP(selected);
|
|
@@ -381,7 +381,7 @@ var PowerQueues = class extends PowerRedis {
|
|
|
381
381
|
async selectS(queueName, from = "0-0") {
|
|
382
382
|
try {
|
|
383
383
|
const res = await this.runScript("SelectStuck", [queueName], [this.group, this.consumer(), String(this.selectStuckTimeout), String(this.selectStuckCount), String(this.selectStuckMaxTimeout)], SelectStuck);
|
|
384
|
-
return
|
|
384
|
+
return Array.isArray(res) ? res : [];
|
|
385
385
|
} catch (err) {
|
|
386
386
|
if (String(err?.message || "").includes("NOGROUP")) {
|
|
387
387
|
await this.createGroup(queueName, from);
|
|
@@ -405,7 +405,7 @@ var PowerQueues = class extends PowerRedis {
|
|
|
405
405
|
">"
|
|
406
406
|
);
|
|
407
407
|
rows = res?.[0]?.[1] ?? [];
|
|
408
|
-
if (!
|
|
408
|
+
if (!Array.isArray(rows) || !(rows.length > 0)) {
|
|
409
409
|
return [];
|
|
410
410
|
}
|
|
411
411
|
} catch (err) {
|
|
@@ -422,9 +422,9 @@ var PowerQueues = class extends PowerRedis {
|
|
|
422
422
|
return Array.from(raw || []).map((e) => {
|
|
423
423
|
const id = Buffer.isBuffer(e?.[0]) ? e[0].toString() : e?.[0];
|
|
424
424
|
const kvRaw = e?.[1] ?? [];
|
|
425
|
-
const kv =
|
|
425
|
+
const kv = Array.isArray(kvRaw) ? kvRaw.map((x) => Buffer.isBuffer(x) ? x.toString() : x) : [];
|
|
426
426
|
return [id, kv];
|
|
427
|
-
}).filter(([id, kv]) =>
|
|
427
|
+
}).filter(([id, kv]) => typeof id === "string" && id.length > 0 && Array.isArray(kv) && (kv.length & 1) === 0).map(([id, kv]) => {
|
|
428
428
|
const { payload, createdAt, job, idemKey, attempt } = this.values(kv);
|
|
429
429
|
return [id, this.payload(payload), createdAt, job, idemKey, Number(attempt)];
|
|
430
430
|
});
|
|
@@ -467,10 +467,10 @@ var PowerQueues = class extends PowerRedis {
|
|
|
467
467
|
}
|
|
468
468
|
let start = Date.now();
|
|
469
469
|
if (!this.executeSync && promises.length > 0) {
|
|
470
|
-
await Promise.all(promises.map((item) => item()));
|
|
470
|
+
await Promise.all(promises.map((item) => this.limit(() => item())));
|
|
471
471
|
}
|
|
472
472
|
await this.onBatchReady(queueName, result);
|
|
473
|
-
if (!
|
|
473
|
+
if ((!Array.isArray(result) || !(result.length > 0)) && contended > tasks.length >> 1) {
|
|
474
474
|
await this.waitAbortable(15 + Math.floor(Math.random() * 35) + Math.min(250, 15 * contended + Math.floor(Math.random() * 40)));
|
|
475
475
|
}
|
|
476
476
|
return result;
|
|
@@ -648,7 +648,7 @@ var PowerQueues = class extends PowerRedis {
|
|
|
648
648
|
}
|
|
649
649
|
async runScript(name, keys, args, defaultCode) {
|
|
650
650
|
if (!this.scripts[name]) {
|
|
651
|
-
if (!
|
|
651
|
+
if (!(typeof defaultCode === "string" && defaultCode.length > 0)) {
|
|
652
652
|
throw new Error(`Undefined script "${name}". Save it before executing.`);
|
|
653
653
|
}
|
|
654
654
|
this.saveScript(name, defaultCode);
|
|
@@ -696,20 +696,20 @@ var PowerQueues = class extends PowerRedis {
|
|
|
696
696
|
throw new Error("Load lua script failed.");
|
|
697
697
|
}
|
|
698
698
|
saveScript(name, codeBody) {
|
|
699
|
-
if (!
|
|
699
|
+
if (!(typeof codeBody === "string" && codeBody.length > 0)) {
|
|
700
700
|
throw new Error("Script body is empty.");
|
|
701
701
|
}
|
|
702
702
|
this.scripts[name] = { codeBody };
|
|
703
703
|
return codeBody;
|
|
704
704
|
}
|
|
705
705
|
async addTasks(queueName, data, opts = {}) {
|
|
706
|
-
if (!
|
|
706
|
+
if (!Array.isArray(data) || !(data.length > 0)) {
|
|
707
707
|
throw new Error("Tasks is not filled.");
|
|
708
708
|
}
|
|
709
|
-
if (!
|
|
709
|
+
if (!(typeof queueName === "string" && queueName.length > 0)) {
|
|
710
710
|
throw new Error("Queue name is required.");
|
|
711
711
|
}
|
|
712
|
-
opts.job = opts.job ??
|
|
712
|
+
opts.job = opts.job ?? randomUUID();
|
|
713
713
|
const batches = this.buildBatches(data, opts);
|
|
714
714
|
const result = new Array(data.length);
|
|
715
715
|
const promises = [];
|
|
@@ -750,8 +750,8 @@ var PowerQueues = class extends PowerRedis {
|
|
|
750
750
|
const entry = {
|
|
751
751
|
payload: JSON.stringify(taskP),
|
|
752
752
|
attempt: Number(opts.attempt || 0),
|
|
753
|
-
job: opts.job ??
|
|
754
|
-
idemKey: String(idemKey ||
|
|
753
|
+
job: opts.job ?? randomUUID(),
|
|
754
|
+
idemKey: String(idemKey || randomUUID()),
|
|
755
755
|
createdAt
|
|
756
756
|
};
|
|
757
757
|
const reqKeysLength = this.keysLength(entry);
|
|
@@ -807,7 +807,7 @@ var PowerQueues = class extends PowerRedis {
|
|
|
807
807
|
argv.push(String(id));
|
|
808
808
|
argv.push(String(pairs));
|
|
809
809
|
for (const token of flat) {
|
|
810
|
-
argv.push(
|
|
810
|
+
argv.push(token === null || token === void 0 ? "" : typeof token === "string" && token.length > 0 ? token : String(token));
|
|
811
811
|
}
|
|
812
812
|
}
|
|
813
813
|
return argv;
|
|
@@ -832,5 +832,6 @@ var PowerQueues = class extends PowerRedis {
|
|
|
832
832
|
}
|
|
833
833
|
};
|
|
834
834
|
export {
|
|
835
|
-
PowerQueues
|
|
835
|
+
PowerQueues,
|
|
836
|
+
wait
|
|
836
837
|
};
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "power-queues",
|
|
3
|
-
"version": "2.1.
|
|
3
|
+
"version": "2.1.11",
|
|
4
4
|
"description": "High-performance Redis Streams queue for Node.js with Lua-powered bulk XADD, idempotent workers, heartbeat locks, stuck-task recovery, retries, DLQ, and distributed processing.",
|
|
5
5
|
"author": "ihor-bielchenko",
|
|
6
6
|
"license": "MIT",
|
|
@@ -49,6 +49,9 @@
|
|
|
49
49
|
"redis-pipeline",
|
|
50
50
|
"redis-multi",
|
|
51
51
|
"redis-atomic",
|
|
52
|
+
"p-limit",
|
|
53
|
+
"limit",
|
|
54
|
+
"concurrency",
|
|
52
55
|
"json-serialization",
|
|
53
56
|
"safe-serialization",
|
|
54
57
|
"key-schema",
|
|
@@ -81,8 +84,7 @@
|
|
|
81
84
|
"power-redis"
|
|
82
85
|
],
|
|
83
86
|
"dependencies": {
|
|
84
|
-
"
|
|
85
|
-
"power-redis": "^2.0.23"
|
|
86
|
-
"uuid": "^13.0.0"
|
|
87
|
+
"p-limit": "^7.3.0",
|
|
88
|
+
"power-redis": "^2.0.23"
|
|
87
89
|
}
|
|
88
90
|
}
|