power-queues 2.0.5 → 2.0.6

package/dist/index.cjs

@@ -264,8 +264,6 @@ var PowerQueues = class extends import_power_redis.PowerRedis {
     this.scripts = {};
     this.addingBatchTasksCount = 800;
     this.addingBatchKeysLimit = 1e4;
-    this.idemOn = true;
-    this.idemKey = "";
     this.workerExecuteLockTimeoutMs = 18e4;
     this.workerCacheTaskTimeoutMs = 60;
     this.approveBatchTasksCount = 2e3;
@@ -280,11 +278,14 @@ var PowerQueues = class extends import_power_redis.PowerRedis {
     this.recoveryStuckTasksTimeoutMs = 6e4;
     this.workerLoopIntervalMs = 5e3;
     this.workerSelectionTimeoutMs = 80;
+    this.workerMaxRetries = 5;
+    this.workerClearAttemptsTimeoutMs = 864e5;
+    this.workerStatusTimeoutMs = 864e5;
   }
   async onSelected(data) {
     return data;
   }
-  async onExecute(id, payload, createdAt, job, key) {
+  async onExecute(id, payload, createdAt, job, key, attempt) {
   }
   async onExecuted(data) {
   }
@@ -309,6 +310,7 @@ var PowerQueues = class extends import_power_redis.PowerRedis {
           await this.approve(ids);
         }
       } catch (err) {
+        await this.batchError(err);
         await (0, import_full_utils.wait)(600);
       }
     }
@@ -320,7 +322,8 @@ var PowerQueues = class extends import_power_redis.PowerRedis {
     if (typeof queueName !== "string" || !(queueName.length > 0)) {
       throw new Error("Queue name is required.");
     }
-    const batches = this.buildBatches(data);
+    const job = (0, import_uuid.v4)();
+    const batches = this.buildBatches(data, job, opts.idem);
     const result = new Array(data.length);
     const promises = [];
     let cursor = 0;
@@ -343,6 +346,16 @@ var PowerQueues = class extends import_power_redis.PowerRedis {
         }
       }
     });
+    if (opts.status) {
+      await this.redis.set(`${queueName}:${job}:total`, data.length);
+      await this.redis.set(`${queueName}:${job}:ready`, 0);
+      await this.redis.set(`${queueName}:${job}:err`, 0);
+      await this.redis.set(`${queueName}:${job}:ok`, 0);
+      await this.redis.pexpire(`${queueName}:${job}:total`, this.workerStatusTimeoutMs);
+      await this.redis.pexpire(`${queueName}:${job}:ready`, this.workerStatusTimeoutMs);
+      await this.redis.pexpire(`${queueName}:${job}:err`, this.workerStatusTimeoutMs);
+      await this.redis.pexpire(`${queueName}:${job}:ok`, this.workerStatusTimeoutMs);
+    }
     await Promise.all(runners);
     return result;
   }
@@ -452,35 +465,32 @@ var PowerQueues = class extends import_power_redis.PowerRedis {
     }
     return argv;
   }
-  buildBatches(tasks) {
-    const job = (0, import_uuid.v4)();
+  buildBatches(tasks, job, idem) {
     const batches = [];
     let batch = [], realKeysLength = 0;
     for (let task of tasks) {
+      const createdAt = task?.createdAt || Date.now();
       let entry = task;
-      if (this.idemOn) {
-        const createdAt = entry?.createdAt || Date.now();
-        let idemKey = entry?.idemKey || (0, import_uuid.v4)();
-        if (typeof entry.payload === "object") {
-          if (this.idemKey && typeof entry.payload[this.idemKey] === "string" && entry.payload[this.idemKey].length > 0) {
-            idemKey = entry.payload[this.idemKey];
+      if (typeof entry.payload === "object") {
+        entry = {
+          ...entry,
+          payload: {
+            payload: JSON.stringify(entry.payload),
+            createdAt,
+            job
           }
-          entry = {
-            ...entry,
-            payload: {
-              payload: JSON.stringify(entry.payload),
-              createdAt,
-              job,
-              idemKey
-            }
-          };
-        } else if (Array.isArray(entry.flat)) {
-          entry.flat.push("createdAt");
-          entry.flat.push(String(createdAt));
-          entry.flat.push("job");
-          entry.flat.push(job);
+        };
+        if (idem) {
+          entry.payload["idemKey"] = entry?.idemKey || (0, import_uuid.v4)();
+        }
+      } else if (Array.isArray(entry.flat)) {
+        entry.flat.push("createdAt");
+        entry.flat.push(String(createdAt));
+        entry.flat.push("job");
+        entry.flat.push(job);
+        if (idem) {
           entry.flat.push("idemKey");
-          entry.flat.push(idemKey);
+          entry.flat.push(entry?.idemKey || (0, import_uuid.v4)());
         }
       }
       const reqKeysLength = this.keysLength(entry);
@@ -500,6 +510,33 @@ var PowerQueues = class extends import_power_redis.PowerRedis {
   keysLength(task) {
     return 2 + ("flat" in task && Array.isArray(task.flat) && task.flat.length ? task.flat.length : Object.keys(task).length * 2);
   }
+  attemptsKey(id) {
+    const safeStream = this.stream.replace(/[^\w:\-]/g, "_");
+    const safeId = id.replace(/[^\w:\-]/g, "_");
+    return `q:${safeStream}:attempts:${safeId}`;
+  }
+  async incrAttempts(id) {
+    try {
+      const key = this.attemptsKey(id);
+      const attempts = await this.redis.incr(key);
+      await this.redis.pexpire(key, this.workerClearAttemptsTimeoutMs);
+      return attempts;
+    } catch (err) {
+    }
+    return 0;
+  }
+  async getAttempts(id) {
+    const key = this.attemptsKey(id);
+    const v = await this.redis.get(key);
+    return Number(v || 0);
+  }
+  async clearAttempts(id) {
+    const key = this.attemptsKey(id);
+    try {
+      await this.redis.del(key);
+    } catch (e) {
+    }
+  }
   async success(id, payload, createdAt, job, key) {
     if (this.executeJobStatus) {
       await this.status(id, payload, createdAt, job, key);
@@ -511,6 +548,18 @@ var PowerQueues = class extends import_power_redis.PowerRedis {
     const { ready = 0, ok = 0 } = await this.getMany(prefix);
     await this.setMany([{ key: `${prefix}ready`, value: ready + 1 }, { key: `${prefix}ok`, value: ok + 1 }], this.executeJobStatusTtlSec);
   }
+  async batchError(err, tasks) {
+  }
+  async error(err, id, payload, createdAt, job, key) {
+    await this.onError(err, id, payload, createdAt, job, key);
+  }
+  async onError(err, id, payload, createdAt, job, key) {
+  }
+  async attempt(err, id, payload, createdAt, job, key, attempts) {
+    await this.onRetry(err, id, payload, createdAt, job, key, attempts);
+  }
+  async onRetry(err, id, payload, createdAt, job, key, attempts) {
+  }
   async execute(tasks) {
     const result = [];
     let contended = 0, promises = [];
@@ -542,6 +591,7 @@ var PowerQueues = class extends import_power_redis.PowerRedis {
         await this.waitAbortable(15 + Math.floor(Math.random() * 35) + Math.min(250, 15 * contended + Math.floor(Math.random() * 40)));
       }
     } catch (err) {
+      await this.batchError(err, tasks);
     }
     return result;
   }
@@ -550,10 +600,27 @@ var PowerQueues = class extends import_power_redis.PowerRedis {
       return await this.idempotency(id, payload, createdAt, job, key);
     } else {
       try {
-        await this.onExecute(id, payload, createdAt, job, key);
+        await this.onExecute(id, payload, createdAt, job, key, await this.getAttempts(id));
         await this.success(id, payload, createdAt, job, key);
         return { id };
       } catch (err) {
+        const attempts = await this.incrAttempts(id);
+        await this.attempt(err, id, payload, createdAt, job, key, attempts);
+        await this.error(err, id, payload, createdAt, job, key);
+        if (attempts >= this.workerMaxRetries) {
+          await this.addTasks(`${this.stream}:dlq`, [{
+            payload: {
+              ...payload,
+              error: String(err?.message || err),
+              createdAt,
+              job,
+              id,
+              attempts
+            }
+          }]);
+          await this.clearAttempts(id);
+          return { id };
+        }
       }
     }
     return {};
@@ -593,12 +660,29 @@ var PowerQueues = class extends import_power_redis.PowerRedis {
     const heartbeat = this.heartbeat(keys) || (() => {
     });
     try {
-      await this.onExecute(id, payload, createdAt, job, key);
+      await this.onExecute(id, payload, createdAt, job, key, await this.getAttempts(id));
       await this.idempotencyDone(keys);
       await this.success(id, payload, createdAt, job, key);
       return { id };
     } catch (err) {
+      const attempts = await this.incrAttempts(id);
       try {
+        await this.attempt(err, id, payload, createdAt, job, key, attempts);
+        await this.error(err, id, payload, createdAt, job, key);
+        if (attempts >= this.workerMaxRetries) {
+          await this.addTasks(`${this.stream}:dlq`, [{
+            payload: {
+              ...payload,
+              error: String(err?.message || err),
+              createdAt,
+              job,
+              id
+            }
+          }]);
+          await this.clearAttempts(id);
+          await this.idempotencyFree(keys);
+          return { id };
+        }
         await this.idempotencyFree(keys);
       } catch (err2) {
       }
@@ -760,11 +844,10 @@ var PowerQueues = class extends import_power_redis.PowerRedis {
       const id = Buffer.isBuffer(e?.[0]) ? e[0].toString() : e?.[0];
       const kvRaw = e?.[1] ?? [];
       const kv = Array.isArray(kvRaw) ? kvRaw.map((x) => Buffer.isBuffer(x) ? x.toString() : x) : [];
-      return [id, kv, 0, "", ""];
+      return [id, kv];
     }).filter(([id, kv]) => typeof id === "string" && id.length > 0 && Array.isArray(kv) && (kv.length & 1) === 0).map(([id, kv]) => {
-      const values = this.values(kv);
-      const { idemKey = "", createdAt, job, ...data } = this.payload(values);
-      return [id, data, createdAt, job, idemKey];
+      const { idemKey = "", job, createdAt, payload } = this.values(kv);
+      return [id, this.payload(payload), createdAt, job, idemKey];
     });
   }
   values(value) {

package/dist/index.d.cts

@@ -13,6 +13,8 @@ type AddTasksOptions = {
     exact?: boolean;
     trimLimit?: number;
     id?: string;
+    status?: boolean;
+    idem?: boolean;
 };
 type Task = {
     job: string;
@@ -35,8 +37,6 @@ declare class PowerQueues extends PowerRedis {
     readonly scripts: Record<string, SavedScript>;
     readonly addingBatchTasksCount: number;
     readonly addingBatchKeysLimit: number;
-    readonly idemOn: boolean;
-    readonly idemKey: string;
     readonly workerExecuteLockTimeoutMs: number;
     readonly workerCacheTaskTimeoutMs: number;
     readonly approveBatchTasksCount: number;
@@ -51,8 +51,11 @@ declare class PowerQueues extends PowerRedis {
     readonly recoveryStuckTasksTimeoutMs: number;
     readonly workerLoopIntervalMs: number;
     readonly workerSelectionTimeoutMs: number;
+    readonly workerMaxRetries: number;
+    readonly workerClearAttemptsTimeoutMs: number;
+    readonly workerStatusTimeoutMs: number;
     onSelected(data: Array<[string, any[], number, string, string]>): Promise<[string, any[], number, string, string][]>;
-    onExecute(id: string, payload: any, createdAt: number, job: string, key: string): Promise<void>;
+    onExecute(id: string, payload: any, createdAt: number, job: string, key: string, attempt: number): Promise<void>;
     onExecuted(data: Array<[string, any[], number, string, string]>): Promise<void>;
     onSuccess(id: string, payload: any, createdAt: number, job: string, key: string): Promise<void>;
     runQueue(): Promise<void>;
@@ -66,8 +69,17 @@ declare class PowerQueues extends PowerRedis {
     private payloadBatch;
     private buildBatches;
     private keysLength;
+    private attemptsKey;
+    private incrAttempts;
+    private getAttempts;
+    private clearAttempts;
     private success;
     private status;
+    private batchError;
+    private error;
+    onError(err: any, id: string, payload: any, createdAt: number, job: string, key: string): Promise<void>;
+    private attempt;
+    onRetry(err: any, id: string, payload: any, createdAt: number, job: string, key: string, attempts: number): Promise<void>;
     private execute;
     private executeProcess;
     private approve;

package/dist/index.d.ts

@@ -13,6 +13,8 @@ type AddTasksOptions = {
     exact?: boolean;
     trimLimit?: number;
     id?: string;
+    status?: boolean;
+    idem?: boolean;
 };
 type Task = {
     job: string;
@@ -35,8 +37,6 @@ declare class PowerQueues extends PowerRedis {
     readonly scripts: Record<string, SavedScript>;
     readonly addingBatchTasksCount: number;
     readonly addingBatchKeysLimit: number;
-    readonly idemOn: boolean;
-    readonly idemKey: string;
     readonly workerExecuteLockTimeoutMs: number;
     readonly workerCacheTaskTimeoutMs: number;
     readonly approveBatchTasksCount: number;
@@ -51,8 +51,11 @@ declare class PowerQueues extends PowerRedis {
     readonly recoveryStuckTasksTimeoutMs: number;
     readonly workerLoopIntervalMs: number;
     readonly workerSelectionTimeoutMs: number;
+    readonly workerMaxRetries: number;
+    readonly workerClearAttemptsTimeoutMs: number;
+    readonly workerStatusTimeoutMs: number;
     onSelected(data: Array<[string, any[], number, string, string]>): Promise<[string, any[], number, string, string][]>;
-    onExecute(id: string, payload: any, createdAt: number, job: string, key: string): Promise<void>;
+    onExecute(id: string, payload: any, createdAt: number, job: string, key: string, attempt: number): Promise<void>;
     onExecuted(data: Array<[string, any[], number, string, string]>): Promise<void>;
     onSuccess(id: string, payload: any, createdAt: number, job: string, key: string): Promise<void>;
     runQueue(): Promise<void>;
@@ -66,8 +69,17 @@ declare class PowerQueues extends PowerRedis {
     private payloadBatch;
     private buildBatches;
     private keysLength;
+    private attemptsKey;
+    private incrAttempts;
+    private getAttempts;
+    private clearAttempts;
     private success;
     private status;
+    private batchError;
+    private error;
+    onError(err: any, id: string, payload: any, createdAt: number, job: string, key: string): Promise<void>;
+    private attempt;
+    onRetry(err: any, id: string, payload: any, createdAt: number, job: string, key: string, attempts: number): Promise<void>;
     private execute;
     private executeProcess;
     private approve;

package/dist/index.js

@@ -238,8 +238,6 @@ var PowerQueues = class extends PowerRedis {
     this.scripts = {};
     this.addingBatchTasksCount = 800;
     this.addingBatchKeysLimit = 1e4;
-    this.idemOn = true;
-    this.idemKey = "";
     this.workerExecuteLockTimeoutMs = 18e4;
     this.workerCacheTaskTimeoutMs = 60;
     this.approveBatchTasksCount = 2e3;
@@ -254,11 +252,14 @@ var PowerQueues = class extends PowerRedis {
     this.recoveryStuckTasksTimeoutMs = 6e4;
     this.workerLoopIntervalMs = 5e3;
     this.workerSelectionTimeoutMs = 80;
+    this.workerMaxRetries = 5;
+    this.workerClearAttemptsTimeoutMs = 864e5;
+    this.workerStatusTimeoutMs = 864e5;
   }
   async onSelected(data) {
     return data;
   }
-  async onExecute(id, payload, createdAt, job, key) {
+  async onExecute(id, payload, createdAt, job, key, attempt) {
   }
   async onExecuted(data) {
   }
@@ -283,6 +284,7 @@ var PowerQueues = class extends PowerRedis {
           await this.approve(ids);
         }
       } catch (err) {
+        await this.batchError(err);
         await wait(600);
       }
     }
@@ -294,7 +296,8 @@ var PowerQueues = class extends PowerRedis {
     if (typeof queueName !== "string" || !(queueName.length > 0)) {
       throw new Error("Queue name is required.");
     }
-    const batches = this.buildBatches(data);
+    const job = uuid();
+    const batches = this.buildBatches(data, job, opts.idem);
     const result = new Array(data.length);
     const promises = [];
     let cursor = 0;
@@ -317,6 +320,16 @@ var PowerQueues = class extends PowerRedis {
         }
       }
     });
+    if (opts.status) {
+      await this.redis.set(`${queueName}:${job}:total`, data.length);
+      await this.redis.set(`${queueName}:${job}:ready`, 0);
+      await this.redis.set(`${queueName}:${job}:err`, 0);
+      await this.redis.set(`${queueName}:${job}:ok`, 0);
+      await this.redis.pexpire(`${queueName}:${job}:total`, this.workerStatusTimeoutMs);
+      await this.redis.pexpire(`${queueName}:${job}:ready`, this.workerStatusTimeoutMs);
+      await this.redis.pexpire(`${queueName}:${job}:err`, this.workerStatusTimeoutMs);
+      await this.redis.pexpire(`${queueName}:${job}:ok`, this.workerStatusTimeoutMs);
+    }
     await Promise.all(runners);
     return result;
   }
@@ -426,35 +439,32 @@ var PowerQueues = class extends PowerRedis {
     }
     return argv;
   }
-  buildBatches(tasks) {
-    const job = uuid();
+  buildBatches(tasks, job, idem) {
     const batches = [];
     let batch = [], realKeysLength = 0;
     for (let task of tasks) {
+      const createdAt = task?.createdAt || Date.now();
       let entry = task;
-      if (this.idemOn) {
-        const createdAt = entry?.createdAt || Date.now();
-        let idemKey = entry?.idemKey || uuid();
-        if (typeof entry.payload === "object") {
-          if (this.idemKey && typeof entry.payload[this.idemKey] === "string" && entry.payload[this.idemKey].length > 0) {
-            idemKey = entry.payload[this.idemKey];
+      if (typeof entry.payload === "object") {
+        entry = {
+          ...entry,
+          payload: {
+            payload: JSON.stringify(entry.payload),
+            createdAt,
+            job
           }
-          entry = {
-            ...entry,
-            payload: {
-              payload: JSON.stringify(entry.payload),
-              createdAt,
-              job,
-              idemKey
-            }
-          };
-        } else if (Array.isArray(entry.flat)) {
-          entry.flat.push("createdAt");
-          entry.flat.push(String(createdAt));
-          entry.flat.push("job");
-          entry.flat.push(job);
+        };
+        if (idem) {
+          entry.payload["idemKey"] = entry?.idemKey || uuid();
+        }
+      } else if (Array.isArray(entry.flat)) {
+        entry.flat.push("createdAt");
+        entry.flat.push(String(createdAt));
+        entry.flat.push("job");
+        entry.flat.push(job);
+        if (idem) {
           entry.flat.push("idemKey");
-          entry.flat.push(idemKey);
+          entry.flat.push(entry?.idemKey || uuid());
         }
       }
       const reqKeysLength = this.keysLength(entry);
@@ -474,6 +484,33 @@ var PowerQueues = class extends PowerRedis {
   keysLength(task) {
     return 2 + ("flat" in task && Array.isArray(task.flat) && task.flat.length ? task.flat.length : Object.keys(task).length * 2);
   }
+  attemptsKey(id) {
+    const safeStream = this.stream.replace(/[^\w:\-]/g, "_");
+    const safeId = id.replace(/[^\w:\-]/g, "_");
+    return `q:${safeStream}:attempts:${safeId}`;
+  }
+  async incrAttempts(id) {
+    try {
+      const key = this.attemptsKey(id);
+      const attempts = await this.redis.incr(key);
+      await this.redis.pexpire(key, this.workerClearAttemptsTimeoutMs);
+      return attempts;
+    } catch (err) {
+    }
+    return 0;
+  }
+  async getAttempts(id) {
+    const key = this.attemptsKey(id);
+    const v = await this.redis.get(key);
+    return Number(v || 0);
+  }
+  async clearAttempts(id) {
+    const key = this.attemptsKey(id);
+    try {
+      await this.redis.del(key);
+    } catch (e) {
+    }
+  }
   async success(id, payload, createdAt, job, key) {
     if (this.executeJobStatus) {
       await this.status(id, payload, createdAt, job, key);
@@ -485,6 +522,18 @@ var PowerQueues = class extends PowerRedis {
     const { ready = 0, ok = 0 } = await this.getMany(prefix);
     await this.setMany([{ key: `${prefix}ready`, value: ready + 1 }, { key: `${prefix}ok`, value: ok + 1 }], this.executeJobStatusTtlSec);
   }
+  async batchError(err, tasks) {
+  }
+  async error(err, id, payload, createdAt, job, key) {
+    await this.onError(err, id, payload, createdAt, job, key);
+  }
+  async onError(err, id, payload, createdAt, job, key) {
+  }
+  async attempt(err, id, payload, createdAt, job, key, attempts) {
+    await this.onRetry(err, id, payload, createdAt, job, key, attempts);
+  }
+  async onRetry(err, id, payload, createdAt, job, key, attempts) {
+  }
   async execute(tasks) {
     const result = [];
     let contended = 0, promises = [];
@@ -516,6 +565,7 @@ var PowerQueues = class extends PowerRedis {
         await this.waitAbortable(15 + Math.floor(Math.random() * 35) + Math.min(250, 15 * contended + Math.floor(Math.random() * 40)));
       }
     } catch (err) {
+      await this.batchError(err, tasks);
     }
     return result;
   }
@@ -524,10 +574,27 @@ var PowerQueues = class extends PowerRedis {
       return await this.idempotency(id, payload, createdAt, job, key);
     } else {
       try {
-        await this.onExecute(id, payload, createdAt, job, key);
+        await this.onExecute(id, payload, createdAt, job, key, await this.getAttempts(id));
         await this.success(id, payload, createdAt, job, key);
         return { id };
       } catch (err) {
+        const attempts = await this.incrAttempts(id);
+        await this.attempt(err, id, payload, createdAt, job, key, attempts);
+        await this.error(err, id, payload, createdAt, job, key);
+        if (attempts >= this.workerMaxRetries) {
+          await this.addTasks(`${this.stream}:dlq`, [{
+            payload: {
+              ...payload,
+              error: String(err?.message || err),
+              createdAt,
+              job,
+              id,
+              attempts
+            }
+          }]);
+          await this.clearAttempts(id);
+          return { id };
+        }
       }
     }
     return {};
@@ -567,12 +634,29 @@ var PowerQueues = class extends PowerRedis {
     const heartbeat = this.heartbeat(keys) || (() => {
     });
     try {
-      await this.onExecute(id, payload, createdAt, job, key);
+      await this.onExecute(id, payload, createdAt, job, key, await this.getAttempts(id));
       await this.idempotencyDone(keys);
       await this.success(id, payload, createdAt, job, key);
       return { id };
     } catch (err) {
+      const attempts = await this.incrAttempts(id);
       try {
+        await this.attempt(err, id, payload, createdAt, job, key, attempts);
+        await this.error(err, id, payload, createdAt, job, key);
+        if (attempts >= this.workerMaxRetries) {
+          await this.addTasks(`${this.stream}:dlq`, [{
+            payload: {
+              ...payload,
+              error: String(err?.message || err),
+              createdAt,
+              job,
+              id
+            }
+          }]);
+          await this.clearAttempts(id);
+          await this.idempotencyFree(keys);
+          return { id };
+        }
         await this.idempotencyFree(keys);
       } catch (err2) {
       }
@@ -734,11 +818,10 @@ var PowerQueues = class extends PowerRedis {
       const id = Buffer.isBuffer(e?.[0]) ? e[0].toString() : e?.[0];
       const kvRaw = e?.[1] ?? [];
       const kv = Array.isArray(kvRaw) ? kvRaw.map((x) => Buffer.isBuffer(x) ? x.toString() : x) : [];
-      return [id, kv, 0, "", ""];
+      return [id, kv];
     }).filter(([id, kv]) => typeof id === "string" && id.length > 0 && Array.isArray(kv) && (kv.length & 1) === 0).map(([id, kv]) => {
-      const values = this.values(kv);
-      const { idemKey = "", createdAt, job, ...data } = this.payload(values);
-      return [id, data, createdAt, job, idemKey];
+      const { idemKey = "", job, createdAt, payload } = this.values(kv);
+      return [id, this.payload(payload), createdAt, job, idemKey];
     });
   }
   values(value) {

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "power-queues",
-	"version": "2.0.5",
+	"version": "2.0.6",
 	"description": "Base classes for implementing custom queues in redis under high load conditions based on nestjs.",
 	"author": "ihor-bielchenko",
 	"license": "MIT",