power-queues 2.1.2 → 2.1.4

package/dist/index.cjs

@@ -275,7 +275,7 @@ var PowerQueues = class extends import_power_redis.PowerRedis {
     this.selectStuckTimeout = 6e4;
     this.selectStuckMaxTimeout = 80;
     this.selectCount = 200;
-    this.selectTimeout = 5e3;
+    this.selectTimeout = 3e3;
     this.buildBatchCount = 800;
     this.buildBatchMaxCount = 1e4;
     this.retryCount = 1;
@@ -283,7 +283,7 @@ var PowerQueues = class extends import_power_redis.PowerRedis {
     this.idemLockTimeout = 18e4;
     this.idemDoneTimeout = 6e4;
     this.logStatus = false;
-    this.logStatusTimeout = 3e5;
+    this.logStatusTimeout = 18e5;
     this.approveCount = 2e3;
     this.removeOnExecuted = true;
   }
@@ -316,7 +316,7 @@ var PowerQueues = class extends import_power_redis.PowerRedis {
       } catch (err) {
       }
       if (!(0, import_full_utils.isArrFilled)(tasks)) {
-        await (0, import_full_utils.wait)(400);
+        await (0, import_full_utils.wait)(300);
         continue;
       }
       try {
@@ -327,7 +327,7 @@ var PowerQueues = class extends import_power_redis.PowerRedis {
           await this.approve(queueName, tasks.map((task) => task[0]));
         } catch {
         }
-        await (0, import_full_utils.wait)(400);
+        await (0, import_full_utils.wait)(300);
       }
     }
   }
@@ -339,17 +339,26 @@ var PowerQueues = class extends import_power_redis.PowerRedis {
         if (!filtered[key]) {
           filtered[key] = [];
         }
-        filtered[key].push(tasks[index]);
+        filtered[key].push(tasks[index][1]);
       });
       for (let key in filtered) {
         const filteredTasks = filtered[key];
         const keySplit = key.split(":");
         const attempt = Number(keySplit[0]);
-        await this.addTasks(queueName + (attempt + 1 >= this.retryCount ? ":dlq" : ""), filteredTasks.map((task) => ({ ...task[1] })), {
-          createdAt: Number(keySplit[1]),
-          job: keySplit[2],
-          attempt: attempt + 1
-        });
+        const job = String(keySplit[2]);
+        if (!(attempt >= this.retryCount - 1)) {
+          await this.addTasks(queueName, filteredTasks, {
+            job,
+            attempt: attempt + 1
+          });
+        } else if (this.logStatus) {
+          const statusKey = `${queueName}:${job}:`;
+          await this.setOne(statusKey + "err", Number(await this.getOne(statusKey + "err") || 0) + filteredTasks.length, this.logStatusTimeout);
+          await this.setOne(statusKey + "ready", Number(await this.getOne(statusKey + "ready") || 0) + filteredTasks.length, this.logStatusTimeout);
+          await this.addTasks(queueName + ":dlq", filteredTasks, {
+            job
+          });
+        }
       }
     } catch (err2) {
       throw new Error(`Batch error. ${err2.message}`);
@@ -452,14 +461,14 @@ var PowerQueues = class extends import_power_redis.PowerRedis {
     let contended = 0, promises = [];
     for (const [id, payload, createdAt, job, idemKey, attempt] of tasks) {
       if (!this.executeSync) {
-        promises.push((async () => {
+        promises.push(async () => {
           const r = await this.executeProcess(queueName, { id, payload, createdAt, job, idemKey, attempt });
           if (r.id) {
             result.push(r);
           } else if (r.contended) {
             contended++;
           }
-        })());
+        });
       } else {
         const r = await this.executeProcess(queueName, { id, payload, createdAt, job, idemKey, attempt });
         if (r.id) {
@@ -469,10 +478,11 @@ var PowerQueues = class extends import_power_redis.PowerRedis {
         }
       }
     }
+    let start = Date.now();
     if (!this.executeSync && promises.length > 0) {
-      await Promise.all(promises);
+      await Promise.all(promises.map((item) => item()));
     }
-    await this.onBatchSuccess(queueName, result);
+    await this.onBatchReady(queueName, result);
     if (!(0, import_full_utils.isArrFilled)(result) && contended > tasks.length >> 1) {
       await this.waitAbortable(15 + Math.floor(Math.random() * 35) + Math.min(250, 15 * contended + Math.floor(Math.random() * 40)));
     }
@@ -482,7 +492,7 @@ var PowerQueues = class extends import_power_redis.PowerRedis {
     const keys = this.idempotencyKeys(queueName, String(task.idemKey));
     const allow = await this.idempotencyAllow(keys);
     if (allow === 1) {
-      return { id: task.id };
+      return { contended: true };
     } else if (allow === 0) {
       let ttl = -2;
       try {
@@ -503,13 +513,8 @@ var PowerQueues = class extends import_power_redis.PowerRedis {
       return await this.success(queueName, processed);
     } catch (err) {
       try {
-        task.attempt = task.attempt + 1;
-        await this.error(err, queueName, task);
-        if (task.attempt >= this.retryCount) {
-          await this.idempotencyFree(keys);
-          return task;
-        }
         await this.idempotencyFree(keys);
+        return await this.error(err, queueName, task);
       } catch (err2) {
       }
     } finally {
@@ -554,28 +559,26 @@ var PowerQueues = class extends import_power_redis.PowerRedis {
     return await this.onSuccess(queueName, task);
   }
   async error(err, queueName, task) {
-    const dlqKey = queueName + ":dlq";
     const taskP = { ...task };
-    if (task.attempt >= this.retryCount) {
-      const statusKey = `${queueName}:${task.job}:`;
-      await this.addTasks(dlqKey, [{ ...taskP.payload }], {
-        createdAt: task.createdAt,
-        job: task.job,
-        attempt: task.attempt
-      });
-      if (this.logStatus) {
-        await this.incr(statusKey + "err", this.logStatusTimeout);
-        await this.incr(statusKey + "ready", this.logStatusTimeout);
-      }
-    } else {
-      await this.onRetry(err, queueName, task);
+    if (!(taskP.attempt >= this.retryCount - 1)) {
+      await this.onRetry(err, queueName, taskP);
       await this.addTasks(queueName, [{ ...taskP.payload }], {
-        createdAt: task.createdAt,
-        job: task.job,
-        attempt: task.attempt
+        createdAt: taskP.createdAt,
+        job: taskP.job,
+        attempt: (taskP.attempt || 0) + 1
+      });
+    } else if (this.logStatus) {
+      const dlqKey = queueName + ":dlq";
+      const statusKey = `${queueName}:${taskP.job}:`;
+      await this.incr(statusKey + "err", this.logStatusTimeout);
+      await this.incr(statusKey + "ready", this.logStatusTimeout);
+      await this.addTasks(dlqKey, [{ ...taskP.payload }], {
+        createdAt: taskP.createdAt,
+        job: taskP.job,
+        attempt: taskP.attempt
       });
     }
-    await this.onError(err, queueName, task);
+    return await this.onError(err, queueName, { ...taskP, attempt: (taskP.attempt || 0) + 1 });
   }
   async waitAbortable(ttl) {
     return new Promise((resolve) => {
@@ -585,7 +588,7 @@ var PowerQueues = class extends import_power_redis.PowerRedis {
       }
       let delay;
       if (ttl > 0) {
-        const base = Math.max(25, Math.min(ttl, 5e3));
+        const base = Math.max(25, Math.min(ttl, 4e3));
         const jitter = Math.floor(Math.min(base, 200) * Math.random());
         delay = base + jitter;
       } else {
@@ -837,12 +840,13 @@ var PowerQueues = class extends import_power_redis.PowerRedis {
   async onExecute(queueName, task) {
     return task;
   }
-  async onBatchSuccess(queueName, tasks) {
+  async onBatchReady(queueName, tasks) {
   }
   async onSuccess(queueName, task) {
     return task;
   }
   async onError(err, queueName, task) {
+    return task;
   }
   async onBatchError(err, queueName, tasks) {
   }

package/dist/index.d.cts

@@ -81,12 +81,12 @@ declare class PowerQueues extends PowerRedis {
     private buildBatches;
     private keysLength;
     private payloadBatch;
-    beforeExecute(queueName: string, tasks: Array<[string, any[], number, string, string, number]>): Promise<[string, any[], number, string, string, number][]>;
+    beforeExecute(queueName: string, tasks: Array<[string, any, number, string, string, number]>): Promise<[string, any, number, string, string, number][]>;
     onExecute(queueName: string, task: Task): Promise<Task>;
-    onBatchSuccess(queueName: string, tasks: Array<Task>): Promise<void>;
+    onBatchReady(queueName: string, tasks: Task[]): Promise<void>;
     onSuccess(queueName: string, task: Task): Promise<Task>;
-    onError(err: any, queueName: string, task: Task): Promise<void>;
-    onBatchError(err: any, queueName: string, tasks: Array<[string, any[], number, string, string, number]>): Promise<void>;
+    onError(err: any, queueName: string, task: Task): Promise<Task>;
+    onBatchError(err: any, queueName: string, tasks: Array<[string, any, number, string, string, number]>): Promise<void>;
     onRetry(err: any, queueName: string, task: Task): Promise<void>;
 }
 

package/dist/index.d.ts

@@ -81,12 +81,12 @@ declare class PowerQueues extends PowerRedis {
     private buildBatches;
     private keysLength;
     private payloadBatch;
-    beforeExecute(queueName: string, tasks: Array<[string, any[], number, string, string, number]>): Promise<[string, any[], number, string, string, number][]>;
+    beforeExecute(queueName: string, tasks: Array<[string, any, number, string, string, number]>): Promise<[string, any, number, string, string, number][]>;
     onExecute(queueName: string, task: Task): Promise<Task>;
-    onBatchSuccess(queueName: string, tasks: Array<Task>): Promise<void>;
+    onBatchReady(queueName: string, tasks: Task[]): Promise<void>;
     onSuccess(queueName: string, task: Task): Promise<Task>;
-    onError(err: any, queueName: string, task: Task): Promise<void>;
-    onBatchError(err: any, queueName: string, tasks: Array<[string, any[], number, string, string, number]>): Promise<void>;
+    onError(err: any, queueName: string, task: Task): Promise<Task>;
+    onBatchError(err: any, queueName: string, tasks: Array<[string, any, number, string, string, number]>): Promise<void>;
     onRetry(err: any, queueName: string, task: Task): Promise<void>;
 }
 

package/dist/index.js

@@ -258,7 +258,7 @@ var PowerQueues = class extends PowerRedis {
     this.selectStuckTimeout = 6e4;
     this.selectStuckMaxTimeout = 80;
     this.selectCount = 200;
-    this.selectTimeout = 5e3;
+    this.selectTimeout = 3e3;
     this.buildBatchCount = 800;
     this.buildBatchMaxCount = 1e4;
     this.retryCount = 1;
@@ -266,7 +266,7 @@ var PowerQueues = class extends PowerRedis {
     this.idemLockTimeout = 18e4;
     this.idemDoneTimeout = 6e4;
     this.logStatus = false;
-    this.logStatusTimeout = 3e5;
+    this.logStatusTimeout = 18e5;
     this.approveCount = 2e3;
     this.removeOnExecuted = true;
   }
@@ -299,7 +299,7 @@ var PowerQueues = class extends PowerRedis {
       } catch (err) {
       }
       if (!isArrFilled(tasks)) {
-        await wait(400);
+        await wait(300);
         continue;
       }
       try {
@@ -310,7 +310,7 @@ var PowerQueues = class extends PowerRedis {
           await this.approve(queueName, tasks.map((task) => task[0]));
         } catch {
         }
-        await wait(400);
+        await wait(300);
       }
     }
   }
@@ -322,17 +322,26 @@ var PowerQueues = class extends PowerRedis {
         if (!filtered[key]) {
           filtered[key] = [];
         }
-        filtered[key].push(tasks[index]);
+        filtered[key].push(tasks[index][1]);
       });
       for (let key in filtered) {
         const filteredTasks = filtered[key];
         const keySplit = key.split(":");
         const attempt = Number(keySplit[0]);
-        await this.addTasks(queueName + (attempt + 1 >= this.retryCount ? ":dlq" : ""), filteredTasks.map((task) => ({ ...task[1] })), {
-          createdAt: Number(keySplit[1]),
-          job: keySplit[2],
-          attempt: attempt + 1
-        });
+        const job = String(keySplit[2]);
+        if (!(attempt >= this.retryCount - 1)) {
+          await this.addTasks(queueName, filteredTasks, {
+            job,
+            attempt: attempt + 1
+          });
+        } else if (this.logStatus) {
+          const statusKey = `${queueName}:${job}:`;
+          await this.setOne(statusKey + "err", Number(await this.getOne(statusKey + "err") || 0) + filteredTasks.length, this.logStatusTimeout);
+          await this.setOne(statusKey + "ready", Number(await this.getOne(statusKey + "ready") || 0) + filteredTasks.length, this.logStatusTimeout);
+          await this.addTasks(queueName + ":dlq", filteredTasks, {
+            job
+          });
+        }
       }
     } catch (err2) {
       throw new Error(`Batch error. ${err2.message}`);
@@ -435,14 +444,14 @@ var PowerQueues = class extends PowerRedis {
     let contended = 0, promises = [];
     for (const [id, payload, createdAt, job, idemKey, attempt] of tasks) {
       if (!this.executeSync) {
-        promises.push((async () => {
+        promises.push(async () => {
           const r = await this.executeProcess(queueName, { id, payload, createdAt, job, idemKey, attempt });
           if (r.id) {
             result.push(r);
           } else if (r.contended) {
             contended++;
           }
-        })());
+        });
       } else {
         const r = await this.executeProcess(queueName, { id, payload, createdAt, job, idemKey, attempt });
         if (r.id) {
@@ -452,10 +461,11 @@ var PowerQueues = class extends PowerRedis {
         }
       }
     }
+    let start = Date.now();
     if (!this.executeSync && promises.length > 0) {
-      await Promise.all(promises);
+      await Promise.all(promises.map((item) => item()));
     }
-    await this.onBatchSuccess(queueName, result);
+    await this.onBatchReady(queueName, result);
     if (!isArrFilled(result) && contended > tasks.length >> 1) {
       await this.waitAbortable(15 + Math.floor(Math.random() * 35) + Math.min(250, 15 * contended + Math.floor(Math.random() * 40)));
     }
@@ -465,7 +475,7 @@ var PowerQueues = class extends PowerRedis {
     const keys = this.idempotencyKeys(queueName, String(task.idemKey));
     const allow = await this.idempotencyAllow(keys);
     if (allow === 1) {
-      return { id: task.id };
+      return { contended: true };
     } else if (allow === 0) {
       let ttl = -2;
       try {
@@ -486,13 +496,8 @@ var PowerQueues = class extends PowerRedis {
       return await this.success(queueName, processed);
     } catch (err) {
       try {
-        task.attempt = task.attempt + 1;
-        await this.error(err, queueName, task);
-        if (task.attempt >= this.retryCount) {
-          await this.idempotencyFree(keys);
-          return task;
-        }
         await this.idempotencyFree(keys);
+        return await this.error(err, queueName, task);
       } catch (err2) {
       }
     } finally {
@@ -537,28 +542,26 @@ var PowerQueues = class extends PowerRedis {
     return await this.onSuccess(queueName, task);
   }
   async error(err, queueName, task) {
-    const dlqKey = queueName + ":dlq";
     const taskP = { ...task };
-    if (task.attempt >= this.retryCount) {
-      const statusKey = `${queueName}:${task.job}:`;
-      await this.addTasks(dlqKey, [{ ...taskP.payload }], {
-        createdAt: task.createdAt,
-        job: task.job,
-        attempt: task.attempt
-      });
-      if (this.logStatus) {
-        await this.incr(statusKey + "err", this.logStatusTimeout);
-        await this.incr(statusKey + "ready", this.logStatusTimeout);
-      }
-    } else {
-      await this.onRetry(err, queueName, task);
+    if (!(taskP.attempt >= this.retryCount - 1)) {
+      await this.onRetry(err, queueName, taskP);
       await this.addTasks(queueName, [{ ...taskP.payload }], {
-        createdAt: task.createdAt,
-        job: task.job,
-        attempt: task.attempt
+        createdAt: taskP.createdAt,
+        job: taskP.job,
+        attempt: (taskP.attempt || 0) + 1
+      });
+    } else if (this.logStatus) {
+      const dlqKey = queueName + ":dlq";
+      const statusKey = `${queueName}:${taskP.job}:`;
+      await this.incr(statusKey + "err", this.logStatusTimeout);
+      await this.incr(statusKey + "ready", this.logStatusTimeout);
+      await this.addTasks(dlqKey, [{ ...taskP.payload }], {
+        createdAt: taskP.createdAt,
+        job: taskP.job,
+        attempt: taskP.attempt
       });
     }
-    await this.onError(err, queueName, task);
+    return await this.onError(err, queueName, { ...taskP, attempt: (taskP.attempt || 0) + 1 });
   }
   async waitAbortable(ttl) {
     return new Promise((resolve) => {
@@ -568,7 +571,7 @@ var PowerQueues = class extends PowerRedis {
       }
       let delay;
       if (ttl > 0) {
-        const base = Math.max(25, Math.min(ttl, 5e3));
+        const base = Math.max(25, Math.min(ttl, 4e3));
         const jitter = Math.floor(Math.min(base, 200) * Math.random());
         delay = base + jitter;
       } else {
@@ -820,12 +823,13 @@ var PowerQueues = class extends PowerRedis {
   async onExecute(queueName, task) {
     return task;
   }
-  async onBatchSuccess(queueName, tasks) {
+  async onBatchReady(queueName, tasks) {
   }
   async onSuccess(queueName, task) {
     return task;
   }
   async onError(err, queueName, task) {
+    return task;
   }
   async onBatchError(err, queueName, tasks) {
   }

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "power-queues",
-	"version": "2.1.2",
+	"version": "2.1.4",
 	"description": "High-performance Redis Streams queue for Node.js with Lua-powered bulk XADD, idempotent workers, heartbeat locks, stuck-task recovery, retries, DLQ, and distributed processing.",
 	"author": "ihor-bielchenko",
 	"license": "MIT",