power-queues 2.1.6 → 2.1.8

package/README.md

@@ -35,26 +35,99 @@ yarn add power-queues
 
 ``` ts
 const queue = new PowerQueues({
-	stream: 'email',
+	stream: 'mysql',
 	group: 'workers',
 });
 
 await queue.loadScripts(true);
 
-await queue.addTasks('email', [
-	{ payload: { type: 'welcome', userId: 42 } },
-	{ payload: { type: 'hello', userId: 51 } }
+await queue.addTasks('mysql_create:example:table_name', [
+	{ type: 'welcome', userId: 42 },
+	{ type: 'hello', userId: 51 }
 ]);
 ```
 
-Worker:
+Example of a worker for executing a MySQL insert transaction:
 
 ``` ts
-class EmailWorker extends PowerQueues {
-	async onExecute(id, payload) {
-		await sendEmail(payload);
+import mysql from 'mysql2/promise';
+import Redis from 'ioredis';
+import type { IORedisLike } from 'power-redis';
+import { type Task, PowerQueues, } from 'power-queues';
+import { 
+	isArrFilled,
+	isObjFilled,
+} from 'full-utils';
+
+const pool = mysql.createPool({
+	host: '127.0.0.1',
+	port: 3306,
+	user: 'user',
+	password: 'password',
+	database: 'example',
+	waitForConnections: true,
+	connectionLimit: 10,
+});
+const redis = new Redis('redis://127.0.0.1:6379');
+
+export class ExampleQueue extends PowerQueues {
+	public readonly selectStuckCount: number = 256;
+	public readonly selectCount: number = 256;
+	public readonly retryCount: number = 3;
+	public readonly executeSync: boolean = true;
+	public readonly removeOnExecuted: boolean = true;
+	public redis!: IORedisLike;
+
+	constructor() {
+		super();
+
+		this.redis = redis;
+	}
+
+	async onBatchReady(queueName: string, tasks: Task[]) {
+		const values = tasks
+			.filter((task) => isObjFilled(task.payload))
+			.map((task) => task.payload);
+
+		if (isArrFilled(values)) {
+			const conn = await pool.getConnection();
+
+			try {
+				await conn.beginTransaction();
+
+				const cols = Object.keys(values[0]);
+				const placeholder = `(${cols.map(() => '?').join(',')})`;
+				const sql = `INSERT INTO \`alerts\` (${cols.map((c) => `\`${c}\``).join(',')}) VALUES ${values.map(() => placeholder).join(',')}`;
+				const params = [];
+
+				for (const row of values) {
+					for (const c of cols) {
+						params.push(row[c]);
+					}
+				}
+				await conn.execute(sql, params);
+				await conn.commit();
+			}
+			catch (err) {
+				await conn.rollback();
+				throw err;
+			}
+			finally {
+				conn.release();
+			}
+		}
+	}
+
+	async onBatchError(err: any, queueName: string, tasks: Array<[ string, any, number, string, string, number ]>) {
+		this.logger.error('Transaction error', queueName, tasks.length, (process.env.NODE_ENV === 'production')
+			? err.message
+			: err, tasks.map((task) => task[1]));
 	}
 }
+
+const exampleQueue = new ExampleQueue();
+
+exampleQueue.runQueue('mysql_create:example:table_name');
 ```
 
 ## ⚖️ power-queues vs Existing Solutions
@@ -124,27 +197,24 @@ and efficiently.
 When retries reach the configured limit:
 - the task is moved into `${stream}:dlq`
 - includes: payload, attempt count, job, timestamp, error text
-- fully JSON‑safe
 
 Perfect for monitoring or later re‑processing.
 
 ### ✔ Zero‑Overhead Serialization
 **power-queues** uses:
 - safe JSON encoding
-- optional "flat" key/value task format
 - predictable and optimized payload transformation
 
 This keeps Redis memory layout clean and eliminates overhead.
 
 ### ✔ Complete Set of Lifecycle Hooks
 You can extend any part of the execution flow:
-- `onSelected`
 - `onExecute`
+- `onBatchReady`
 - `onSuccess`
 - `onError`
-- `onRetry`
 - `onBatchError`
-- `onReady`
+- `onRetry`
 
 This allows full integration with:
 - monitoring systems
@@ -163,6 +233,7 @@ Ensures resilience in failover scenarios.
 
 ### ✔ Job Progress Tracking
 Optional per‑job counters:
+- `job:total`
 - `job:ok`
 - `job:err`
 - `job:ready`

package/dist/index.cjs

@@ -326,7 +326,14 @@ var PowerQueues = class extends import_power_redis.PowerRedis {
       } catch (err) {
         await this.batchError(err, queueName, tasks);
         try {
-          await this.approve(queueName, tasks.map((task) => task[0]));
+          await this.approve(queueName, tasks.map((task) => ({
+            id: task[0],
+            createdAt: Number(task[2]),
+            payload: task[1],
+            job: task[3],
+            idemKey: task[4],
+            attempt: Number(task[5] || 0)
+          })));
         } catch {
         }
         await (0, import_full_utils.wait)(300);
@@ -337,17 +344,17 @@ var PowerQueues = class extends import_power_redis.PowerRedis {
     try {
       const filtered = {};
       tasks.forEach((task, index) => {
-        const key = String(task[5] + ":" + task[2] + ":" + task[3]);
+        const key = JSON.stringify([task[5] || "0", task[2], task[3]]);
         if (!filtered[key]) {
           filtered[key] = [];
         }
-        filtered[key].push(tasks[index][1]);
+        filtered[key].push({ ...tasks[index][1], idemKey: tasks[index][4] });
       });
       for (let key in filtered) {
         const filteredTasks = filtered[key];
-        const keySplit = key.split(":");
-        const attempt = Number(keySplit[0] || 0);
-        const job = String(keySplit[2]);
+        const keyP = JSON.parse(key);
+        const attempt = Number(keyP[0] || 0);
+        const job = String(keyP[2]);
         if (!(attempt >= this.retryCount - 1)) {
           await this.addTasks(queueName, filteredTasks, {
             job,
@@ -363,12 +370,10 @@ var PowerQueues = class extends import_power_redis.PowerRedis {
         }
       }
     } catch (err2) {
-      throw new Error(`Batch error. ${err2.message}`);
     }
     try {
       await this.onBatchError(err, queueName, tasks);
     } catch (err2) {
-      throw new Error(`Batch error. ${err2.message}`);
     }
   }
   async approve(queueName, tasks) {
@@ -440,7 +445,7 @@ var PowerQueues = class extends import_power_redis.PowerRedis {
       const kv = (0, import_full_utils.isArr)(kvRaw) ? kvRaw.map((x) => Buffer.isBuffer(x) ? x.toString() : x) : [];
       return [id, kv];
     }).filter(([id, kv]) => (0, import_full_utils.isStrFilled)(id) && (0, import_full_utils.isArr)(kv) && (kv.length & 1) === 0).map(([id, kv]) => {
-      const { payload, createdAt, job, idemKey = "", attempt } = this.values(kv);
+      const { payload, createdAt, job, idemKey, attempt } = this.values(kv);
       return [id, this.payload(payload), createdAt, job, idemKey, Number(attempt)];
     });
   }
@@ -567,7 +572,8 @@ var PowerQueues = class extends import_power_redis.PowerRedis {
       await this.addTasks(queueName, [{ ...taskP.payload }], {
         createdAt: taskP.createdAt,
         job: taskP.job,
-        attempt: (taskP.attempt || 0) + 1
+        attempt: (taskP.attempt || 0) + 1,
+        idemKey: taskP.idemKey
       });
     } else if (this.logStatus) {
       const dlqKey = queueName + ":dlq";
@@ -577,7 +583,8 @@ var PowerQueues = class extends import_power_redis.PowerRedis {
       await this.addTasks(dlqKey, [{ ...taskP.payload }], {
         createdAt: taskP.createdAt,
         job: taskP.job,
-        attempt: taskP.attempt
+        attempt: taskP.attempt,
+        idemKey: taskP.idemKey
       });
     }
     return await this.onError(err, queueName, { ...taskP, attempt: (taskP.attempt || 0) + 1 });
@@ -758,19 +765,17 @@ var PowerQueues = class extends import_power_redis.PowerRedis {
   }
   buildBatches(tasks, opts = {}) {
     const batches = [];
-    let batch = [], realKeysLength = 0;
+    let batch = [], realKeysLength = 8;
     for (let task of tasks) {
       const createdAt = opts?.createdAt || Date.now();
-      let entry = {};
-      if ((0, import_full_utils.isObj)(entry)) {
-        entry = {
-          payload: JSON.stringify(task),
-          attempt: Number(opts.attempt || 0),
-          job: opts.job ?? (0, import_uuid.v4)(),
-          idemKey: (0, import_uuid.v4)(),
-          createdAt
-        };
-      }
+      const { idemKey, ...taskP } = task;
+      const entry = {
+        payload: JSON.stringify(taskP),
+        attempt: Number(opts.attempt || 0),
+        job: opts.job ?? (0, import_uuid.v4)(),
+        idemKey: String((idemKey ?? opts?.idemKey) || (0, import_uuid.v4)()),
+        createdAt
+      };
       const reqKeysLength = this.keysLength(entry);
       if (batch.length && (batch.length >= this.buildBatchCount || realKeysLength + reqKeysLength > this.buildBatchMaxCount)) {
         batches.push(batch);
@@ -786,9 +791,6 @@ var PowerQueues = class extends import_power_redis.PowerRedis {
     return batches;
   }
   keysLength(task) {
-    if ("payload" in task && (0, import_full_utils.isObj)(task.payload)) {
-      return 2 + Object.keys(task.payload).length * 2;
-    }
     return 2 + Object.keys(task).length * 2;
   }
   payloadBatch(data, opts) {
@@ -813,7 +815,7 @@ var PowerQueues = class extends import_power_redis.PowerRedis {
       const entry = item;
       const id = entry.id ?? "*";
       let flat = [];
-      if ("payload" in entry && (0, import_full_utils.isObjFilled)(entry)) {
+      if ("payload" in entry) {
         for (const [k, v] of Object.entries(entry)) {
           flat.push(k, v);
         }
@@ -821,7 +823,7 @@ var PowerQueues = class extends import_power_redis.PowerRedis {
         throw new Error('Task must have "payload" or "flat".');
       }
       const pairs = flat.length / 2;
-      if ((0, import_full_utils.isNumNZ)(pairs)) {
+      if (pairs <= 0) {
         throw new Error('Task "flat" must contain at least one field/value pair.');
       }
       argv.push(String(id));

package/dist/index.d.cts

@@ -12,6 +12,7 @@ type AddTasksOptions = {
     approx?: boolean;
     exact?: boolean;
     trimLimit?: number;
+    idemKey?: string;
     job?: string;
     status?: boolean;
     statusTimeoutMs?: number;

package/dist/index.d.ts

@@ -12,6 +12,7 @@ type AddTasksOptions = {
     approx?: boolean;
     exact?: boolean;
     trimLimit?: number;
+    idemKey?: string;
     job?: string;
     status?: boolean;
     statusTimeoutMs?: number;

package/dist/index.js

@@ -2,13 +2,10 @@
 import { setMaxListeners } from "events";
 import { PowerRedis } from "power-redis";
 import {
-  isObjFilled,
-  isObj,
   isArrFilled,
   isArr,
   isStrFilled,
   isExists,
-  isNumNZ,
   wait
 } from "full-utils";
 import { v4 as uuid } from "uuid";
@@ -309,7 +306,14 @@ var PowerQueues = class extends PowerRedis {
       } catch (err) {
         await this.batchError(err, queueName, tasks);
         try {
-          await this.approve(queueName, tasks.map((task) => task[0]));
+          await this.approve(queueName, tasks.map((task) => ({
+            id: task[0],
+            createdAt: Number(task[2]),
+            payload: task[1],
+            job: task[3],
+            idemKey: task[4],
+            attempt: Number(task[5] || 0)
+          })));
         } catch {
         }
         await wait(300);
@@ -320,17 +324,17 @@ var PowerQueues = class extends PowerRedis {
     try {
       const filtered = {};
       tasks.forEach((task, index) => {
-        const key = String(task[5] + ":" + task[2] + ":" + task[3]);
+        const key = JSON.stringify([task[5] || "0", task[2], task[3]]);
         if (!filtered[key]) {
           filtered[key] = [];
         }
-        filtered[key].push(tasks[index][1]);
+        filtered[key].push({ ...tasks[index][1], idemKey: tasks[index][4] });
       });
       for (let key in filtered) {
         const filteredTasks = filtered[key];
-        const keySplit = key.split(":");
-        const attempt = Number(keySplit[0] || 0);
-        const job = String(keySplit[2]);
+        const keyP = JSON.parse(key);
+        const attempt = Number(keyP[0] || 0);
+        const job = String(keyP[2]);
         if (!(attempt >= this.retryCount - 1)) {
           await this.addTasks(queueName, filteredTasks, {
             job,
@@ -346,12 +350,10 @@ var PowerQueues = class extends PowerRedis {
         }
       }
     } catch (err2) {
-      throw new Error(`Batch error. ${err2.message}`);
     }
     try {
       await this.onBatchError(err, queueName, tasks);
     } catch (err2) {
-      throw new Error(`Batch error. ${err2.message}`);
     }
   }
   async approve(queueName, tasks) {
@@ -423,7 +425,7 @@ var PowerQueues = class extends PowerRedis {
       const kv = isArr(kvRaw) ? kvRaw.map((x) => Buffer.isBuffer(x) ? x.toString() : x) : [];
       return [id, kv];
     }).filter(([id, kv]) => isStrFilled(id) && isArr(kv) && (kv.length & 1) === 0).map(([id, kv]) => {
-      const { payload, createdAt, job, idemKey = "", attempt } = this.values(kv);
+      const { payload, createdAt, job, idemKey, attempt } = this.values(kv);
       return [id, this.payload(payload), createdAt, job, idemKey, Number(attempt)];
     });
   }
@@ -550,7 +552,8 @@ var PowerQueues = class extends PowerRedis {
       await this.addTasks(queueName, [{ ...taskP.payload }], {
         createdAt: taskP.createdAt,
         job: taskP.job,
-        attempt: (taskP.attempt || 0) + 1
+        attempt: (taskP.attempt || 0) + 1,
+        idemKey: taskP.idemKey
       });
     } else if (this.logStatus) {
       const dlqKey = queueName + ":dlq";
@@ -560,7 +563,8 @@ var PowerQueues = class extends PowerRedis {
       await this.addTasks(dlqKey, [{ ...taskP.payload }], {
         createdAt: taskP.createdAt,
         job: taskP.job,
-        attempt: taskP.attempt
+        attempt: taskP.attempt,
+        idemKey: taskP.idemKey
       });
     }
     return await this.onError(err, queueName, { ...taskP, attempt: (taskP.attempt || 0) + 1 });
@@ -741,19 +745,17 @@ var PowerQueues = class extends PowerRedis {
   }
   buildBatches(tasks, opts = {}) {
     const batches = [];
-    let batch = [], realKeysLength = 0;
+    let batch = [], realKeysLength = 8;
     for (let task of tasks) {
       const createdAt = opts?.createdAt || Date.now();
-      let entry = {};
-      if (isObj(entry)) {
-        entry = {
-          payload: JSON.stringify(task),
-          attempt: Number(opts.attempt || 0),
-          job: opts.job ?? uuid(),
-          idemKey: uuid(),
-          createdAt
-        };
-      }
+      const { idemKey, ...taskP } = task;
+      const entry = {
+        payload: JSON.stringify(taskP),
+        attempt: Number(opts.attempt || 0),
+        job: opts.job ?? uuid(),
+        idemKey: String((idemKey ?? opts?.idemKey) || uuid()),
+        createdAt
+      };
       const reqKeysLength = this.keysLength(entry);
       if (batch.length && (batch.length >= this.buildBatchCount || realKeysLength + reqKeysLength > this.buildBatchMaxCount)) {
         batches.push(batch);
@@ -769,9 +771,6 @@ var PowerQueues = class extends PowerRedis {
     return batches;
   }
   keysLength(task) {
-    if ("payload" in task && isObj(task.payload)) {
-      return 2 + Object.keys(task.payload).length * 2;
-    }
     return 2 + Object.keys(task).length * 2;
   }
   payloadBatch(data, opts) {
@@ -796,7 +795,7 @@ var PowerQueues = class extends PowerRedis {
       const entry = item;
       const id = entry.id ?? "*";
       let flat = [];
-      if ("payload" in entry && isObjFilled(entry)) {
+      if ("payload" in entry) {
         for (const [k, v] of Object.entries(entry)) {
           flat.push(k, v);
         }
@@ -804,7 +803,7 @@ var PowerQueues = class extends PowerRedis {
         throw new Error('Task must have "payload" or "flat".');
       }
       const pairs = flat.length / 2;
-      if (isNumNZ(pairs)) {
+      if (pairs <= 0) {
         throw new Error('Task "flat" must contain at least one field/value pair.');
       }
       argv.push(String(id));

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "power-queues",
-	"version": "2.1.6",
+	"version": "2.1.8",
 	"description": "High-performance Redis Streams queue for Node.js with Lua-powered bulk XADD, idempotent workers, heartbeat locks, stuck-task recovery, retries, DLQ, and distributed processing.",
 	"author": "ihor-bielchenko",
 	"license": "MIT",