power-queues 2.1.1 → 2.1.3

package/dist/index.cjs

@@ -283,9 +283,9 @@ var PowerQueues = class extends import_power_redis.PowerRedis {
     this.idemLockTimeout = 18e4;
     this.idemDoneTimeout = 6e4;
     this.logStatus = false;
-    this.logStatusTimeout = 3e5;
+    this.logStatusTimeout = 6e5;
     this.approveCount = 2e3;
-    this.removeOnExecuted = false;
+    this.removeOnExecuted = true;
   }
   signal() {
     return this.abort.signal;
@@ -316,7 +316,7 @@ var PowerQueues = class extends import_power_redis.PowerRedis {
       } catch (err) {
       }
       if (!(0, import_full_utils.isArrFilled)(tasks)) {
-        await (0, import_full_utils.wait)(400);
+        await (0, import_full_utils.wait)(300);
         continue;
       }
       try {
@@ -327,7 +327,7 @@ var PowerQueues = class extends import_power_redis.PowerRedis {
           await this.approve(queueName, tasks.map((task) => task[0]));
         } catch {
         }
-        await (0, import_full_utils.wait)(400);
+        await (0, import_full_utils.wait)(300);
       }
     }
   }
@@ -345,7 +345,7 @@ var PowerQueues = class extends import_power_redis.PowerRedis {
         const filteredTasks = filtered[key];
         const keySplit = key.split(":");
         const attempt = Number(keySplit[0]);
-        await this.addTasks(queueName + (attempt + 1 >= this.retryCount ? ":dlq" : ""), filteredTasks.map((task) => ({ ...task[1] })), {
+        await this.addTasks(queueName + (attempt >= this.retryCount - 1 ? ":dlq" : ""), filteredTasks.map((task) => ({ ...task[1] })), {
           createdAt: Number(keySplit[1]),
           job: keySplit[2],
           attempt: attempt + 1
@@ -360,15 +360,15 @@ var PowerQueues = class extends import_power_redis.PowerRedis {
       throw new Error(`Batch error. ${err2.message}`);
     }
   }
-  async approve(queueName, ids) {
-    if (!(0, import_full_utils.isArrFilled)(ids)) {
+  async approve(queueName, tasks) {
+    if (!(0, import_full_utils.isArrFilled)(tasks)) {
       return 0;
     }
     const approveCount = Math.max(500, Math.min(4e3, this.approveCount));
     let total = 0, i = 0;
-    while (i < ids.length) {
-      const room = Math.min(approveCount, ids.length - i);
-      const part = ids.slice(i, i + room);
+    while (i < tasks.length) {
+      const room = Math.min(approveCount, tasks.length - i);
+      const part = tasks.slice(i, i + room).map((item) => String(item.id || ""));
       const approved = await this.runScript("Approve", [queueName], [this.group, this.removeOnExecuted ? "1" : "0", ...part], Approve);
       total += Number(approved || 0);
       i += room;
@@ -452,27 +452,27 @@ var PowerQueues = class extends import_power_redis.PowerRedis {
     let contended = 0, promises = [];
     for (const [id, payload, createdAt, job, idemKey, attempt] of tasks) {
       if (!this.executeSync) {
-        promises.push((async () => {
+        promises.push(async () => {
           const r = await this.executeProcess(queueName, { id, payload, createdAt, job, idemKey, attempt });
           if (r.id) {
-            result.push(id);
+            result.push(r);
           } else if (r.contended) {
             contended++;
           }
-        })());
+        });
       } else {
         const r = await this.executeProcess(queueName, { id, payload, createdAt, job, idemKey, attempt });
         if (r.id) {
-          result.push(id);
+          result.push(r);
         } else if (r.contended) {
           contended++;
         }
       }
     }
     if (!this.executeSync && promises.length > 0) {
-      await Promise.all(promises);
+      await Promise.all(promises.map((item) => item));
     }
-    await this.onBatchSuccess(queueName, tasks);
+    await this.onBatchReady(queueName, result);
     if (!(0, import_full_utils.isArrFilled)(result) && contended > tasks.length >> 1) {
       await this.waitAbortable(15 + Math.floor(Math.random() * 35) + Math.min(250, 15 * contended + Math.floor(Math.random() * 40)));
     }
@@ -482,7 +482,7 @@ var PowerQueues = class extends import_power_redis.PowerRedis {
     const keys = this.idempotencyKeys(queueName, String(task.idemKey));
     const allow = await this.idempotencyAllow(keys);
     if (allow === 1) {
-      return { id: task.id };
+      return { contended: true };
     } else if (allow === 0) {
       let ttl = -2;
       try {
@@ -498,19 +498,13 @@ var PowerQueues = class extends import_power_redis.PowerRedis {
     const heartbeat = this.heartbeat(keys) || (() => {
     });
     try {
-      await this.onExecute(queueName, task);
+      const processed = await this.onExecute(queueName, task);
       await this.idempotencyDone(keys);
-      await this.success(queueName, task);
-      return { id: task.id };
+      return await this.success(queueName, processed);
     } catch (err) {
       try {
-        task.attempt = task.attempt + 1;
-        await this.error(err, queueName, task);
-        if (task.attempt >= this.retryCount) {
-          await this.idempotencyFree(keys);
-          return { id: task.id };
-        }
         await this.idempotencyFree(keys);
+        return await this.error(err, queueName, task);
       } catch (err2) {
       }
     } finally {
@@ -552,31 +546,31 @@ var PowerQueues = class extends import_power_redis.PowerRedis {
       await this.incr(statusKey + "ok", this.logStatusTimeout);
       await this.incr(statusKey + "ready", this.logStatusTimeout);
     }
-    await this.onSuccess(queueName, task);
+    return await this.onSuccess(queueName, task);
   }
   async error(err, queueName, task) {
     const dlqKey = queueName + ":dlq";
     const taskP = { ...task };
-    if (task.attempt >= this.retryCount) {
-      const statusKey = `${queueName}:${task.job}:`;
-      await this.addTasks(dlqKey, [{ ...taskP.payload }], {
-        createdAt: task.createdAt,
-        job: task.job,
-        attempt: task.attempt
-      });
+    if (taskP.attempt >= this.retryCount - 1) {
+      const statusKey = `${queueName}:${taskP.job}:`;
       if (this.logStatus) {
         await this.incr(statusKey + "err", this.logStatusTimeout);
         await this.incr(statusKey + "ready", this.logStatusTimeout);
       }
+      await this.addTasks(dlqKey, [{ ...taskP.payload }], {
+        createdAt: taskP.createdAt,
+        job: taskP.job,
+        attempt: taskP.attempt
+      });
     } else {
-      await this.onRetry(err, queueName, task);
+      await this.onRetry(err, queueName, taskP);
       await this.addTasks(queueName, [{ ...taskP.payload }], {
-        createdAt: task.createdAt,
-        job: task.job,
-        attempt: task.attempt
+        createdAt: taskP.createdAt,
+        job: taskP.job,
+        attempt: (taskP.attempt || 0) + 1
       });
     }
-    await this.onError(err, queueName, task);
+    return await this.onError(err, queueName, { ...taskP, attempt: (taskP.attempt || 0) + 1 });
   }
   async waitAbortable(ttl) {
     return new Promise((resolve) => {
@@ -586,7 +580,7 @@ var PowerQueues = class extends import_power_redis.PowerRedis {
       }
       let delay;
       if (ttl > 0) {
-        const base = Math.max(25, Math.min(ttl, 5e3));
+        const base = Math.max(25, Math.min(ttl, 4e3));
         const jitter = Math.floor(Math.min(base, 200) * Math.random());
         delay = base + jitter;
       } else {
@@ -836,12 +830,15 @@ var PowerQueues = class extends import_power_redis.PowerRedis {
     return tasks;
   }
   async onExecute(queueName, task) {
+    return task;
   }
-  async onBatchSuccess(queueName, tasks) {
+  async onBatchReady(queueName, tasks) {
   }
   async onSuccess(queueName, task) {
+    return task;
   }
   async onError(err, queueName, task) {
+    return task;
   }
   async onBatchError(err, queueName, tasks) {
   }

package/dist/index.d.cts

@@ -81,12 +81,12 @@ declare class PowerQueues extends PowerRedis {
     private buildBatches;
     private keysLength;
     private payloadBatch;
-    beforeExecute(queueName: string, tasks: Array<[string, any[], number, string, string, number]>): Promise<[string, any[], number, string, string, number][]>;
-    onExecute(queueName: string, task: Task): Promise<void>;
-    onBatchSuccess(queueName: string, tasks: Array<[string, any[], number, string, string, number]>): Promise<void>;
-    onSuccess(queueName: string, task: Task): Promise<void>;
-    onError(err: any, queueName: string, task: Task): Promise<void>;
-    onBatchError(err: any, queueName: string, tasks: Array<[string, any[], number, string, string, number]>): Promise<void>;
+    beforeExecute(queueName: string, tasks: Array<[string, any, number, string, string, number]>): Promise<[string, any, number, string, string, number][]>;
+    onExecute(queueName: string, task: Task): Promise<Task>;
+    onBatchReady(queueName: string, tasks: Array<Task>): Promise<void>;
+    onSuccess(queueName: string, task: Task): Promise<Task>;
+    onError(err: any, queueName: string, task: Task): Promise<Task>;
+    onBatchError(err: any, queueName: string, tasks: Array<[string, any, number, string, string, number]>): Promise<void>;
     onRetry(err: any, queueName: string, task: Task): Promise<void>;
 }
 

package/dist/index.d.ts

@@ -81,12 +81,12 @@ declare class PowerQueues extends PowerRedis {
     private buildBatches;
     private keysLength;
     private payloadBatch;
-    beforeExecute(queueName: string, tasks: Array<[string, any[], number, string, string, number]>): Promise<[string, any[], number, string, string, number][]>;
-    onExecute(queueName: string, task: Task): Promise<void>;
-    onBatchSuccess(queueName: string, tasks: Array<[string, any[], number, string, string, number]>): Promise<void>;
-    onSuccess(queueName: string, task: Task): Promise<void>;
-    onError(err: any, queueName: string, task: Task): Promise<void>;
-    onBatchError(err: any, queueName: string, tasks: Array<[string, any[], number, string, string, number]>): Promise<void>;
+    beforeExecute(queueName: string, tasks: Array<[string, any, number, string, string, number]>): Promise<[string, any, number, string, string, number][]>;
+    onExecute(queueName: string, task: Task): Promise<Task>;
+    onBatchReady(queueName: string, tasks: Array<Task>): Promise<void>;
+    onSuccess(queueName: string, task: Task): Promise<Task>;
+    onError(err: any, queueName: string, task: Task): Promise<Task>;
+    onBatchError(err: any, queueName: string, tasks: Array<[string, any, number, string, string, number]>): Promise<void>;
     onRetry(err: any, queueName: string, task: Task): Promise<void>;
 }
 

package/dist/index.js

@@ -266,9 +266,9 @@ var PowerQueues = class extends PowerRedis {
     this.idemLockTimeout = 18e4;
     this.idemDoneTimeout = 6e4;
     this.logStatus = false;
-    this.logStatusTimeout = 3e5;
+    this.logStatusTimeout = 6e5;
     this.approveCount = 2e3;
-    this.removeOnExecuted = false;
+    this.removeOnExecuted = true;
   }
   signal() {
     return this.abort.signal;
@@ -299,7 +299,7 @@ var PowerQueues = class extends PowerRedis {
       } catch (err) {
       }
       if (!isArrFilled(tasks)) {
-        await wait(400);
+        await wait(300);
         continue;
       }
       try {
@@ -310,7 +310,7 @@ var PowerQueues = class extends PowerRedis {
           await this.approve(queueName, tasks.map((task) => task[0]));
         } catch {
         }
-        await wait(400);
+        await wait(300);
       }
     }
   }
@@ -328,7 +328,7 @@ var PowerQueues = class extends PowerRedis {
         const filteredTasks = filtered[key];
         const keySplit = key.split(":");
         const attempt = Number(keySplit[0]);
-        await this.addTasks(queueName + (attempt + 1 >= this.retryCount ? ":dlq" : ""), filteredTasks.map((task) => ({ ...task[1] })), {
+        await this.addTasks(queueName + (attempt >= this.retryCount - 1 ? ":dlq" : ""), filteredTasks.map((task) => ({ ...task[1] })), {
           createdAt: Number(keySplit[1]),
           job: keySplit[2],
           attempt: attempt + 1
@@ -343,15 +343,15 @@ var PowerQueues = class extends PowerRedis {
       throw new Error(`Batch error. ${err2.message}`);
     }
   }
-  async approve(queueName, ids) {
-    if (!isArrFilled(ids)) {
+  async approve(queueName, tasks) {
+    if (!isArrFilled(tasks)) {
       return 0;
     }
     const approveCount = Math.max(500, Math.min(4e3, this.approveCount));
     let total = 0, i = 0;
-    while (i < ids.length) {
-      const room = Math.min(approveCount, ids.length - i);
-      const part = ids.slice(i, i + room);
+    while (i < tasks.length) {
+      const room = Math.min(approveCount, tasks.length - i);
+      const part = tasks.slice(i, i + room).map((item) => String(item.id || ""));
       const approved = await this.runScript("Approve", [queueName], [this.group, this.removeOnExecuted ? "1" : "0", ...part], Approve);
       total += Number(approved || 0);
       i += room;
@@ -435,27 +435,27 @@ var PowerQueues = class extends PowerRedis {
     let contended = 0, promises = [];
     for (const [id, payload, createdAt, job, idemKey, attempt] of tasks) {
       if (!this.executeSync) {
-        promises.push((async () => {
+        promises.push(async () => {
           const r = await this.executeProcess(queueName, { id, payload, createdAt, job, idemKey, attempt });
           if (r.id) {
-            result.push(id);
+            result.push(r);
           } else if (r.contended) {
             contended++;
           }
-        })());
+        });
       } else {
         const r = await this.executeProcess(queueName, { id, payload, createdAt, job, idemKey, attempt });
         if (r.id) {
-          result.push(id);
+          result.push(r);
         } else if (r.contended) {
           contended++;
         }
       }
     }
     if (!this.executeSync && promises.length > 0) {
-      await Promise.all(promises);
+      await Promise.all(promises.map((item) => item));
     }
-    await this.onBatchSuccess(queueName, tasks);
+    await this.onBatchReady(queueName, result);
     if (!isArrFilled(result) && contended > tasks.length >> 1) {
       await this.waitAbortable(15 + Math.floor(Math.random() * 35) + Math.min(250, 15 * contended + Math.floor(Math.random() * 40)));
     }
@@ -465,7 +465,7 @@ var PowerQueues = class extends PowerRedis {
     const keys = this.idempotencyKeys(queueName, String(task.idemKey));
     const allow = await this.idempotencyAllow(keys);
     if (allow === 1) {
-      return { id: task.id };
+      return { contended: true };
     } else if (allow === 0) {
       let ttl = -2;
       try {
@@ -481,19 +481,13 @@ var PowerQueues = class extends PowerRedis {
     const heartbeat = this.heartbeat(keys) || (() => {
     });
     try {
-      await this.onExecute(queueName, task);
+      const processed = await this.onExecute(queueName, task);
       await this.idempotencyDone(keys);
-      await this.success(queueName, task);
-      return { id: task.id };
+      return await this.success(queueName, processed);
     } catch (err) {
       try {
-        task.attempt = task.attempt + 1;
-        await this.error(err, queueName, task);
-        if (task.attempt >= this.retryCount) {
-          await this.idempotencyFree(keys);
-          return { id: task.id };
-        }
         await this.idempotencyFree(keys);
+        return await this.error(err, queueName, task);
       } catch (err2) {
       }
     } finally {
@@ -535,31 +529,31 @@ var PowerQueues = class extends PowerRedis {
       await this.incr(statusKey + "ok", this.logStatusTimeout);
       await this.incr(statusKey + "ready", this.logStatusTimeout);
     }
-    await this.onSuccess(queueName, task);
+    return await this.onSuccess(queueName, task);
   }
   async error(err, queueName, task) {
     const dlqKey = queueName + ":dlq";
     const taskP = { ...task };
-    if (task.attempt >= this.retryCount) {
-      const statusKey = `${queueName}:${task.job}:`;
-      await this.addTasks(dlqKey, [{ ...taskP.payload }], {
-        createdAt: task.createdAt,
-        job: task.job,
-        attempt: task.attempt
-      });
+    if (taskP.attempt >= this.retryCount - 1) {
+      const statusKey = `${queueName}:${taskP.job}:`;
       if (this.logStatus) {
         await this.incr(statusKey + "err", this.logStatusTimeout);
         await this.incr(statusKey + "ready", this.logStatusTimeout);
       }
+      await this.addTasks(dlqKey, [{ ...taskP.payload }], {
+        createdAt: taskP.createdAt,
+        job: taskP.job,
+        attempt: taskP.attempt
+      });
     } else {
-      await this.onRetry(err, queueName, task);
+      await this.onRetry(err, queueName, taskP);
       await this.addTasks(queueName, [{ ...taskP.payload }], {
-        createdAt: task.createdAt,
-        job: task.job,
-        attempt: task.attempt
+        createdAt: taskP.createdAt,
+        job: taskP.job,
+        attempt: (taskP.attempt || 0) + 1
       });
     }
-    await this.onError(err, queueName, task);
+    return await this.onError(err, queueName, { ...taskP, attempt: (taskP.attempt || 0) + 1 });
   }
   async waitAbortable(ttl) {
     return new Promise((resolve) => {
@@ -569,7 +563,7 @@ var PowerQueues = class extends PowerRedis {
       }
       let delay;
       if (ttl > 0) {
-        const base = Math.max(25, Math.min(ttl, 5e3));
+        const base = Math.max(25, Math.min(ttl, 4e3));
         const jitter = Math.floor(Math.min(base, 200) * Math.random());
         delay = base + jitter;
       } else {
@@ -819,12 +813,15 @@ var PowerQueues = class extends PowerRedis {
     return tasks;
   }
   async onExecute(queueName, task) {
+    return task;
   }
-  async onBatchSuccess(queueName, tasks) {
+  async onBatchReady(queueName, tasks) {
   }
   async onSuccess(queueName, task) {
+    return task;
   }
   async onError(err, queueName, task) {
+    return task;
   }
   async onBatchError(err, queueName, tasks) {
   }

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "power-queues",
-	"version": "2.1.1",
+	"version": "2.1.3",
 	"description": "High-performance Redis Streams queue for Node.js with Lua-powered bulk XADD, idempotent workers, heartbeat locks, stuck-task recovery, retries, DLQ, and distributed processing.",
 	"author": "ihor-bielchenko",
 	"license": "MIT",