power-queues 2.1.3 → 2.1.5

package/dist/index.cjs

@@ -25,6 +25,7 @@ __export(index_exports, {
 module.exports = __toCommonJS(index_exports);
 
 // src/PowerQueues.ts
+var import_node_events = require("events");
 var import_power_redis = require("power-redis");
 var import_full_utils = require("full-utils");
 var import_uuid = require("uuid");
@@ -275,7 +276,7 @@ var PowerQueues = class extends import_power_redis.PowerRedis {
     this.selectStuckTimeout = 6e4;
     this.selectStuckMaxTimeout = 80;
     this.selectCount = 200;
-    this.selectTimeout = 5e3;
+    this.selectTimeout = 3e3;
     this.buildBatchCount = 800;
     this.buildBatchMaxCount = 1e4;
     this.retryCount = 1;
@@ -283,7 +284,7 @@ var PowerQueues = class extends import_power_redis.PowerRedis {
     this.idemLockTimeout = 18e4;
     this.idemDoneTimeout = 6e4;
     this.logStatus = false;
-    this.logStatusTimeout = 6e5;
+    this.logStatusTimeout = 18e5;
     this.approveCount = 2e3;
     this.removeOnExecuted = true;
   }
@@ -294,6 +295,7 @@ var PowerQueues = class extends import_power_redis.PowerRedis {
     return this.host + ":" + process.pid;
   }
   async runQueue(queueName, from = "0-0") {
+    (0, import_node_events.setMaxListeners)(0, this.abort.signal);
     await this.createGroup(queueName, from);
     await this.consumerLoop(queueName, from);
   }
@@ -339,17 +341,26 @@ var PowerQueues = class extends import_power_redis.PowerRedis {
         if (!filtered[key]) {
           filtered[key] = [];
         }
-        filtered[key].push(tasks[index]);
+        filtered[key].push(tasks[index][1]);
       });
       for (let key in filtered) {
         const filteredTasks = filtered[key];
         const keySplit = key.split(":");
         const attempt = Number(keySplit[0]);
-        await this.addTasks(queueName + (attempt >= this.retryCount - 1 ? ":dlq" : ""), filteredTasks.map((task) => ({ ...task[1] })), {
-          createdAt: Number(keySplit[1]),
-          job: keySplit[2],
-          attempt: attempt + 1
-        });
+        const job = String(keySplit[2]);
+        if (!(attempt >= this.retryCount - 1)) {
+          await this.addTasks(queueName, filteredTasks, {
+            job,
+            attempt: attempt + 1
+          });
+        } else if (this.logStatus) {
+          const statusKey = `${queueName}:${job}:`;
+          await this.setOne(statusKey + "err", Number(await this.getOne(statusKey + "err") || 0) + filteredTasks.length, this.logStatusTimeout);
+          await this.setOne(statusKey + "ready", Number(await this.getOne(statusKey + "ready") || 0) + filteredTasks.length, this.logStatusTimeout);
+          await this.addTasks(queueName + ":dlq", filteredTasks, {
+            job
+          });
+        }
       }
     } catch (err2) {
       throw new Error(`Batch error. ${err2.message}`);
@@ -469,8 +480,9 @@ var PowerQueues = class extends import_power_redis.PowerRedis {
         }
       }
     }
+    let start = Date.now();
     if (!this.executeSync && promises.length > 0) {
-      await Promise.all(promises.map((item) => item));
+      await Promise.all(promises.map((item) => item()));
     }
     await this.onBatchReady(queueName, result);
     if (!(0, import_full_utils.isArrFilled)(result) && contended > tasks.length >> 1) {
@@ -549,26 +561,24 @@ var PowerQueues = class extends import_power_redis.PowerRedis {
     return await this.onSuccess(queueName, task);
   }
   async error(err, queueName, task) {
-    const dlqKey = queueName + ":dlq";
     const taskP = { ...task };
-    if (taskP.attempt >= this.retryCount - 1) {
-      const statusKey = `${queueName}:${taskP.job}:`;
-      if (this.logStatus) {
-        await this.incr(statusKey + "err", this.logStatusTimeout);
-        await this.incr(statusKey + "ready", this.logStatusTimeout);
-      }
-      await this.addTasks(dlqKey, [{ ...taskP.payload }], {
-        createdAt: taskP.createdAt,
-        job: taskP.job,
-        attempt: taskP.attempt
-      });
-    } else {
+    if (!(taskP.attempt >= this.retryCount - 1)) {
       await this.onRetry(err, queueName, taskP);
       await this.addTasks(queueName, [{ ...taskP.payload }], {
         createdAt: taskP.createdAt,
         job: taskP.job,
         attempt: (taskP.attempt || 0) + 1
       });
+    } else if (this.logStatus) {
+      const dlqKey = queueName + ":dlq";
+      const statusKey = `${queueName}:${taskP.job}:`;
+      await this.incr(statusKey + "err", this.logStatusTimeout);
+      await this.incr(statusKey + "ready", this.logStatusTimeout);
+      await this.addTasks(dlqKey, [{ ...taskP.payload }], {
+        createdAt: taskP.createdAt,
+        job: taskP.job,
+        attempt: taskP.attempt
+      });
     }
     return await this.onError(err, queueName, { ...taskP, attempt: (taskP.attempt || 0) + 1 });
   }
@@ -616,22 +626,21 @@ var PowerQueues = class extends import_power_redis.PowerRedis {
       return;
     }
     const workerHeartbeatTimeoutMs = Math.max(1e3, Math.floor(Math.max(5e3, this.idemLockTimeout | 0) / 4));
-    let timer;
-    let alive = true;
-    let hbFails = 0;
+    let timer, alive = true, hbFails = 0;
     const stop = () => {
       alive = false;
       if (timer) {
         clearTimeout(timer);
       }
     };
-    const signal = this.signal();
-    const onAbort = () => stop();
-    signal?.addEventListener?.("abort", onAbort, { once: true });
     const tick = async () => {
       if (!alive) {
         return;
       }
+      if (this.signal()?.aborted) {
+        stop();
+        return;
+      }
       try {
         const ok = await this.sendHeartbeat(keys);
         hbFails = ok ? 0 : hbFails + 1;
@@ -650,10 +659,7 @@ var PowerQueues = class extends import_power_redis.PowerRedis {
     };
     timer = setTimeout(tick, workerHeartbeatTimeoutMs);
     timer.unref?.();
-    return () => {
-      signal?.removeEventListener?.("abort", onAbort);
-      stop();
-    };
+    return () => stop();
   }
   async runScript(name, keys, args, defaultCode) {
     if (!this.scripts[name]) {

package/dist/index.d.cts

@@ -83,7 +83,7 @@ declare class PowerQueues extends PowerRedis {
     private payloadBatch;
     beforeExecute(queueName: string, tasks: Array<[string, any, number, string, string, number]>): Promise<[string, any, number, string, string, number][]>;
     onExecute(queueName: string, task: Task): Promise<Task>;
-    onBatchReady(queueName: string, tasks: Array<Task>): Promise<void>;
+    onBatchReady(queueName: string, tasks: Task[]): Promise<void>;
     onSuccess(queueName: string, task: Task): Promise<Task>;
     onError(err: any, queueName: string, task: Task): Promise<Task>;
     onBatchError(err: any, queueName: string, tasks: Array<[string, any, number, string, string, number]>): Promise<void>;

package/dist/index.d.ts

@@ -83,7 +83,7 @@ declare class PowerQueues extends PowerRedis {
     private payloadBatch;
     beforeExecute(queueName: string, tasks: Array<[string, any, number, string, string, number]>): Promise<[string, any, number, string, string, number][]>;
     onExecute(queueName: string, task: Task): Promise<Task>;
-    onBatchReady(queueName: string, tasks: Array<Task>): Promise<void>;
+    onBatchReady(queueName: string, tasks: Task[]): Promise<void>;
     onSuccess(queueName: string, task: Task): Promise<Task>;
     onError(err: any, queueName: string, task: Task): Promise<Task>;
     onBatchError(err: any, queueName: string, tasks: Array<[string, any, number, string, string, number]>): Promise<void>;

package/dist/index.js

@@ -1,4 +1,5 @@
 // src/PowerQueues.ts
+import { setMaxListeners } from "events";
 import { PowerRedis } from "power-redis";
 import {
   isObjFilled,
@@ -258,7 +259,7 @@ var PowerQueues = class extends PowerRedis {
     this.selectStuckTimeout = 6e4;
     this.selectStuckMaxTimeout = 80;
     this.selectCount = 200;
-    this.selectTimeout = 5e3;
+    this.selectTimeout = 3e3;
     this.buildBatchCount = 800;
     this.buildBatchMaxCount = 1e4;
     this.retryCount = 1;
@@ -266,7 +267,7 @@ var PowerQueues = class extends PowerRedis {
     this.idemLockTimeout = 18e4;
     this.idemDoneTimeout = 6e4;
     this.logStatus = false;
-    this.logStatusTimeout = 6e5;
+    this.logStatusTimeout = 18e5;
     this.approveCount = 2e3;
     this.removeOnExecuted = true;
   }
@@ -277,6 +278,7 @@ var PowerQueues = class extends PowerRedis {
     return this.host + ":" + process.pid;
   }
   async runQueue(queueName, from = "0-0") {
+    setMaxListeners(0, this.abort.signal);
     await this.createGroup(queueName, from);
     await this.consumerLoop(queueName, from);
   }
@@ -322,17 +324,26 @@ var PowerQueues = class extends PowerRedis {
         if (!filtered[key]) {
           filtered[key] = [];
         }
-        filtered[key].push(tasks[index]);
+        filtered[key].push(tasks[index][1]);
       });
       for (let key in filtered) {
         const filteredTasks = filtered[key];
         const keySplit = key.split(":");
         const attempt = Number(keySplit[0]);
-        await this.addTasks(queueName + (attempt >= this.retryCount - 1 ? ":dlq" : ""), filteredTasks.map((task) => ({ ...task[1] })), {
-          createdAt: Number(keySplit[1]),
-          job: keySplit[2],
-          attempt: attempt + 1
-        });
+        const job = String(keySplit[2]);
+        if (!(attempt >= this.retryCount - 1)) {
+          await this.addTasks(queueName, filteredTasks, {
+            job,
+            attempt: attempt + 1
+          });
+        } else if (this.logStatus) {
+          const statusKey = `${queueName}:${job}:`;
+          await this.setOne(statusKey + "err", Number(await this.getOne(statusKey + "err") || 0) + filteredTasks.length, this.logStatusTimeout);
+          await this.setOne(statusKey + "ready", Number(await this.getOne(statusKey + "ready") || 0) + filteredTasks.length, this.logStatusTimeout);
+          await this.addTasks(queueName + ":dlq", filteredTasks, {
+            job
+          });
+        }
       }
     } catch (err2) {
       throw new Error(`Batch error. ${err2.message}`);
@@ -452,8 +463,9 @@ var PowerQueues = class extends PowerRedis {
         }
       }
     }
+    let start = Date.now();
     if (!this.executeSync && promises.length > 0) {
-      await Promise.all(promises.map((item) => item));
+      await Promise.all(promises.map((item) => item()));
     }
     await this.onBatchReady(queueName, result);
     if (!isArrFilled(result) && contended > tasks.length >> 1) {
@@ -532,26 +544,24 @@ var PowerQueues = class extends PowerRedis {
     return await this.onSuccess(queueName, task);
   }
   async error(err, queueName, task) {
-    const dlqKey = queueName + ":dlq";
     const taskP = { ...task };
-    if (taskP.attempt >= this.retryCount - 1) {
-      const statusKey = `${queueName}:${taskP.job}:`;
-      if (this.logStatus) {
-        await this.incr(statusKey + "err", this.logStatusTimeout);
-        await this.incr(statusKey + "ready", this.logStatusTimeout);
-      }
-      await this.addTasks(dlqKey, [{ ...taskP.payload }], {
-        createdAt: taskP.createdAt,
-        job: taskP.job,
-        attempt: taskP.attempt
-      });
-    } else {
+    if (!(taskP.attempt >= this.retryCount - 1)) {
       await this.onRetry(err, queueName, taskP);
       await this.addTasks(queueName, [{ ...taskP.payload }], {
         createdAt: taskP.createdAt,
         job: taskP.job,
         attempt: (taskP.attempt || 0) + 1
       });
+    } else if (this.logStatus) {
+      const dlqKey = queueName + ":dlq";
+      const statusKey = `${queueName}:${taskP.job}:`;
+      await this.incr(statusKey + "err", this.logStatusTimeout);
+      await this.incr(statusKey + "ready", this.logStatusTimeout);
+      await this.addTasks(dlqKey, [{ ...taskP.payload }], {
+        createdAt: taskP.createdAt,
+        job: taskP.job,
+        attempt: taskP.attempt
+      });
     }
     return await this.onError(err, queueName, { ...taskP, attempt: (taskP.attempt || 0) + 1 });
   }
@@ -599,22 +609,21 @@ var PowerQueues = class extends PowerRedis {
       return;
     }
     const workerHeartbeatTimeoutMs = Math.max(1e3, Math.floor(Math.max(5e3, this.idemLockTimeout | 0) / 4));
-    let timer;
-    let alive = true;
-    let hbFails = 0;
+    let timer, alive = true, hbFails = 0;
     const stop = () => {
       alive = false;
       if (timer) {
         clearTimeout(timer);
       }
     };
-    const signal = this.signal();
-    const onAbort = () => stop();
-    signal?.addEventListener?.("abort", onAbort, { once: true });
     const tick = async () => {
       if (!alive) {
         return;
       }
+      if (this.signal()?.aborted) {
+        stop();
+        return;
+      }
       try {
         const ok = await this.sendHeartbeat(keys);
         hbFails = ok ? 0 : hbFails + 1;
@@ -633,10 +642,7 @@ var PowerQueues = class extends PowerRedis {
     };
     timer = setTimeout(tick, workerHeartbeatTimeoutMs);
     timer.unref?.();
-    return () => {
-      signal?.removeEventListener?.("abort", onAbort);
-      stop();
-    };
+    return () => stop();
   }
   async runScript(name, keys, args, defaultCode) {
     if (!this.scripts[name]) {

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "power-queues",
-	"version": "2.1.3",
+	"version": "2.1.5",
 	"description": "High-performance Redis Streams queue for Node.js with Lua-powered bulk XADD, idempotent workers, heartbeat locks, stuck-task recovery, retries, DLQ, and distributed processing.",
 	"author": "ihor-bielchenko",
 	"license": "MIT",