power-queues 2.1.10 → 2.1.12

package/dist/index.cjs

@@ -1,7 +1,9 @@
 "use strict";
+var __create = Object.create;
 var __defProp = Object.defineProperty;
 var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
 var __getOwnPropNames = Object.getOwnPropertyNames;
+var __getProtoOf = Object.getPrototypeOf;
 var __hasOwnProp = Object.prototype.hasOwnProperty;
 var __export = (target, all) => {
   for (var name in all)
@@ -15,20 +17,29 @@ var __copyProps = (to, from, except, desc) => {
   }
   return to;
 };
+var __toESM = (mod, isNodeMode, target) => (target = mod != null ? __create(__getProtoOf(mod)) : {}, __copyProps(
+  // If the importer is in node compatibility mode or this is not an ESM
+  // file that has been converted to a CommonJS file using a Babel-
+  // compatible transform (i.e. "__esModule" has not been set), then set
+  // "default" to the CommonJS "module.exports" for node compatibility.
+  isNodeMode || !mod || !mod.__esModule ? __defProp(target, "default", { value: mod, enumerable: true }) : target,
+  mod
+));
 var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
 
 // src/index.ts
 var index_exports = {};
 __export(index_exports, {
-  PowerQueues: () => PowerQueues
+  PowerQueues: () => PowerQueues,
+  wait: () => wait
 });
 module.exports = __toCommonJS(index_exports);
 
 // src/PowerQueues.ts
+var import_p_limit = __toESM(require("p-limit"), 1);
+var import_node_crypto = require("crypto");
 var import_node_events = require("events");
 var import_power_redis = require("power-redis");
-var import_full_utils = require("full-utils");
-var import_uuid = require("uuid");
 
 // src/scripts.ts
 var XAddBulk = `
@@ -265,6 +276,9 @@ var SelectStuck = `
 `;
 
 // src/PowerQueues.ts
+async function wait(timeout = 0) {
+  await new Promise((resolve) => setTimeout(() => resolve(true), timeout));
+}
 var PowerQueues = class extends import_power_redis.PowerRedis {
   constructor() {
     super(...arguments);
@@ -272,21 +286,23 @@ var PowerQueues = class extends import_power_redis.PowerRedis {
     this.scripts = {};
     this.host = "host";
     this.group = "gr1";
-    this.selectStuckCount = 200;
-    this.selectStuckTimeout = 6e4;
-    this.selectStuckMaxTimeout = 80;
-    this.selectCount = 200;
-    this.selectTimeout = 3e3;
-    this.buildBatchCount = 800;
-    this.buildBatchMaxCount = 1e4;
+    this.selectStuckCount = 256;
+    this.selectStuckTimeout = 2e4;
+    this.selectStuckMaxTimeout = 40;
+    this.selectCount = 1024;
+    this.selectTimeout = 100;
+    this.buildBatchCount = 1e3;
+    this.buildBatchMaxCount = 2e4;
     this.retryCount = 1;
     this.executeSync = false;
-    this.idemLockTimeout = 18e4;
-    this.idemDoneTimeout = 6e4;
+    this.idemLockTimeout = 15e3;
+    this.idemDoneTimeout = 1e4;
     this.logStatus = false;
-    this.logStatusTimeout = 18e5;
-    this.approveCount = 2e3;
+    this.logStatusTimeout = 12e4;
+    this.approveCount = 4e3;
     this.removeOnExecuted = true;
+    this.concurrency = 256;
+    this.limit = (0, import_p_limit.default)(this.concurrency);
   }
   signal() {
     return this.abort.signal;
@@ -296,6 +312,7 @@ var PowerQueues = class extends import_power_redis.PowerRedis {
   }
   async runQueue(queueName, from = "0-0") {
     (0, import_node_events.setMaxListeners)(0, this.abort.signal);
+    this.limit = (0, import_p_limit.default)(this.concurrency);
     await this.createGroup(queueName, from);
     await this.consumerLoop(queueName, from);
   }
@@ -315,10 +332,10 @@ var PowerQueues = class extends import_power_redis.PowerRedis {
       let tasks = [];
       try {
         tasks = await this.select(queueName, from);
-      } catch (err) {
+      } catch {
       }
-      if (!(0, import_full_utils.isArrFilled)(tasks)) {
-        await (0, import_full_utils.wait)(300);
+      if (!Array.isArray(tasks) || !(tasks.length > 0)) {
+        await wait(300);
         continue;
       }
       try {
@@ -336,7 +353,7 @@ var PowerQueues = class extends import_power_redis.PowerRedis {
           })));
         } catch {
         }
-        await (0, import_full_utils.wait)(300);
+        await wait(300);
       }
     }
   }
@@ -377,7 +394,7 @@ var PowerQueues = class extends import_power_redis.PowerRedis {
     }
   }
   async approve(queueName, tasks) {
-    if (!(0, import_full_utils.isArrFilled)(tasks)) {
+    if (!Array.isArray(tasks) || !(tasks.length > 0)) {
       return 0;
     }
     const approveCount = Math.max(500, Math.min(4e3, this.approveCount));
@@ -393,7 +410,7 @@ var PowerQueues = class extends import_power_redis.PowerRedis {
   }
   async select(queueName, from = "0-0") {
     let selected = await this.selectS(queueName, from);
-    if (!(0, import_full_utils.isArrFilled)(selected)) {
+    if (!Array.isArray(selected) || !(selected.length > 0)) {
       selected = await this.selectF(queueName, from);
     }
     return this.selectP(selected);
@@ -401,7 +418,7 @@ var PowerQueues = class extends import_power_redis.PowerRedis {
   async selectS(queueName, from = "0-0") {
     try {
       const res = await this.runScript("SelectStuck", [queueName], [this.group, this.consumer(), String(this.selectStuckTimeout), String(this.selectStuckCount), String(this.selectStuckMaxTimeout)], SelectStuck);
-      return (0, import_full_utils.isArr)(res) ? res : [];
+      return Array.isArray(res) ? res : [];
     } catch (err) {
       if (String(err?.message || "").includes("NOGROUP")) {
         await this.createGroup(queueName, from);
@@ -425,7 +442,7 @@ var PowerQueues = class extends import_power_redis.PowerRedis {
         ">"
       );
       rows = res?.[0]?.[1] ?? [];
-      if (!(0, import_full_utils.isArrFilled)(rows)) {
+      if (!Array.isArray(rows) || !(rows.length > 0)) {
         return [];
       }
     } catch (err) {
@@ -442,9 +459,9 @@ var PowerQueues = class extends import_power_redis.PowerRedis {
     return Array.from(raw || []).map((e) => {
       const id = Buffer.isBuffer(e?.[0]) ? e[0].toString() : e?.[0];
       const kvRaw = e?.[1] ?? [];
-      const kv = (0, import_full_utils.isArr)(kvRaw) ? kvRaw.map((x) => Buffer.isBuffer(x) ? x.toString() : x) : [];
+      const kv = Array.isArray(kvRaw) ? kvRaw.map((x) => Buffer.isBuffer(x) ? x.toString() : x) : [];
       return [id, kv];
-    }).filter(([id, kv]) => (0, import_full_utils.isStrFilled)(id) && (0, import_full_utils.isArr)(kv) && (kv.length & 1) === 0).map(([id, kv]) => {
+    }).filter(([id, kv]) => typeof id === "string" && id.length > 0 && Array.isArray(kv) && (kv.length & 1) === 0).map(([id, kv]) => {
       const { payload, createdAt, job, idemKey, attempt } = this.values(kv);
       return [id, this.payload(payload), createdAt, job, idemKey, Number(attempt)];
     });
@@ -487,10 +504,10 @@ var PowerQueues = class extends import_power_redis.PowerRedis {
     }
     let start = Date.now();
     if (!this.executeSync && promises.length > 0) {
-      await Promise.all(promises.map((item) => item()));
+      await Promise.all(promises.map((item) => this.limit(() => item())));
     }
     await this.onBatchReady(queueName, result);
-    if (!(0, import_full_utils.isArrFilled)(result) && contended > tasks.length >> 1) {
+    if ((!Array.isArray(result) || !(result.length > 0)) && contended > tasks.length >> 1) {
       await this.waitAbortable(15 + Math.floor(Math.random() * 35) + Math.min(250, 15 * contended + Math.floor(Math.random() * 40)));
     }
     return result;
@@ -668,7 +685,7 @@ var PowerQueues = class extends import_power_redis.PowerRedis {
   }
   async runScript(name, keys, args, defaultCode) {
     if (!this.scripts[name]) {
-      if (!(0, import_full_utils.isStrFilled)(defaultCode)) {
+      if (!(typeof defaultCode === "string" && defaultCode.length > 0)) {
         throw new Error(`Undefined script "${name}". Save it before executing.`);
       }
       this.saveScript(name, defaultCode);
@@ -716,20 +733,20 @@ var PowerQueues = class extends import_power_redis.PowerRedis {
     throw new Error("Load lua script failed.");
   }
   saveScript(name, codeBody) {
-    if (!(0, import_full_utils.isStrFilled)(codeBody)) {
+    if (!(typeof codeBody === "string" && codeBody.length > 0)) {
       throw new Error("Script body is empty.");
     }
     this.scripts[name] = { codeBody };
     return codeBody;
   }
   async addTasks(queueName, data, opts = {}) {
-    if (!(0, import_full_utils.isArrFilled)(data)) {
+    if (!Array.isArray(data) || !(data.length > 0)) {
       throw new Error("Tasks is not filled.");
     }
-    if (!(0, import_full_utils.isStrFilled)(queueName)) {
+    if (!(typeof queueName === "string" && queueName.length > 0)) {
       throw new Error("Queue name is required.");
     }
-    opts.job = opts.job ?? (0, import_uuid.v4)();
+    opts.job = opts.job ?? (0, import_node_crypto.randomUUID)();
     const batches = this.buildBatches(data, opts);
     const result = new Array(data.length);
     const promises = [];
@@ -770,8 +787,8 @@ var PowerQueues = class extends import_power_redis.PowerRedis {
       const entry = {
         payload: JSON.stringify(taskP),
         attempt: Number(opts.attempt || 0),
-        job: opts.job ?? (0, import_uuid.v4)(),
-        idemKey: String(idemKey || (0, import_uuid.v4)()),
+        job: opts.job ?? (0, import_node_crypto.randomUUID)(),
+        idemKey: String(idemKey || (0, import_node_crypto.randomUUID)()),
         createdAt
       };
       const reqKeysLength = this.keysLength(entry);
@@ -827,7 +844,7 @@ var PowerQueues = class extends import_power_redis.PowerRedis {
       argv.push(String(id));
       argv.push(String(pairs));
       for (const token of flat) {
-        argv.push(!(0, import_full_utils.isExists)(token) ? "" : (0, import_full_utils.isStrFilled)(token) ? token : String(token));
+        argv.push(token === null || token === void 0 ? "" : typeof token === "string" && token.length > 0 ? token : String(token));
       }
     }
     return argv;
@@ -853,5 +870,6 @@ var PowerQueues = class extends import_power_redis.PowerRedis {
 };
 // Annotate the CommonJS export names for ESM import in node:
 0 && (module.exports = {
-  PowerQueues
+  PowerQueues,
+  wait
 });

package/dist/index.d.cts

@@ -1,5 +1,6 @@
 import { PowerRedis, IORedisLike } from 'power-redis';
 
+type Constructor<T = {}> = new (...args: any[]) => T;
 type SavedScript = {
     codeReady?: string;
     codeBody: string;
@@ -18,6 +19,13 @@ type AddTasksOptions = {
     attempt?: number;
     createdAt?: number;
 };
+type IdempotencyKeys = {
+    prefix: string;
+    doneKey: string;
+    lockKey: string;
+    startKey: string;
+    token: string;
+};
 type Task = {
     job: string;
     id?: string;
@@ -27,6 +35,7 @@ type Task = {
     attempt: number;
 };
 
+declare function wait(timeout?: number): Promise<void>;
 declare class PowerQueues extends PowerRedis {
     abort: AbortController;
     redis: IORedisLike;
@@ -48,6 +57,8 @@ declare class PowerQueues extends PowerRedis {
     readonly logStatusTimeout: number;
     readonly approveCount: number;
     readonly removeOnExecuted: boolean;
+    readonly concurrency: number;
+    private limit;
     private signal;
     private consumer;
     runQueue(queueName: string, from?: '$' | '0-0'): Promise<void>;
@@ -90,4 +101,4 @@ declare class PowerQueues extends PowerRedis {
     onRetry(err: any, queueName: string, task: Task): Promise<void>;
 }
 
-export { type AddTasksOptions, PowerQueues, type Task };
+export { type AddTasksOptions, type Constructor, type IdempotencyKeys, PowerQueues, type SavedScript, type Task, wait };

package/dist/index.d.ts

@@ -1,5 +1,6 @@
 import { PowerRedis, IORedisLike } from 'power-redis';
 
+type Constructor<T = {}> = new (...args: any[]) => T;
 type SavedScript = {
     codeReady?: string;
     codeBody: string;
@@ -18,6 +19,13 @@ type AddTasksOptions = {
     attempt?: number;
     createdAt?: number;
 };
+type IdempotencyKeys = {
+    prefix: string;
+    doneKey: string;
+    lockKey: string;
+    startKey: string;
+    token: string;
+};
 type Task = {
     job: string;
     id?: string;
@@ -27,6 +35,7 @@ type Task = {
     attempt: number;
 };
 
+declare function wait(timeout?: number): Promise<void>;
 declare class PowerQueues extends PowerRedis {
     abort: AbortController;
     redis: IORedisLike;
@@ -48,6 +57,8 @@ declare class PowerQueues extends PowerRedis {
     readonly logStatusTimeout: number;
     readonly approveCount: number;
     readonly removeOnExecuted: boolean;
+    readonly concurrency: number;
+    private limit;
     private signal;
     private consumer;
     runQueue(queueName: string, from?: '$' | '0-0'): Promise<void>;
@@ -90,4 +101,4 @@ declare class PowerQueues extends PowerRedis {
     onRetry(err: any, queueName: string, task: Task): Promise<void>;
 }
 
-export { type AddTasksOptions, PowerQueues, type Task };
+export { type AddTasksOptions, type Constructor, type IdempotencyKeys, PowerQueues, type SavedScript, type Task, wait };

package/dist/index.js

@@ -1,14 +1,8 @@
 // src/PowerQueues.ts
+import pLimit from "p-limit";
+import { randomUUID } from "crypto";
 import { setMaxListeners } from "events";
 import { PowerRedis } from "power-redis";
-import {
-  isArrFilled,
-  isArr,
-  isStrFilled,
-  isExists,
-  wait
-} from "full-utils";
-import { v4 as uuid } from "uuid";
 
 // src/scripts.ts
 var XAddBulk = `
@@ -245,6 +239,9 @@ var SelectStuck = `
 `;
 
 // src/PowerQueues.ts
+async function wait(timeout = 0) {
+  await new Promise((resolve) => setTimeout(() => resolve(true), timeout));
+}
 var PowerQueues = class extends PowerRedis {
   constructor() {
     super(...arguments);
@@ -252,21 +249,23 @@ var PowerQueues = class extends PowerRedis {
     this.scripts = {};
     this.host = "host";
     this.group = "gr1";
-    this.selectStuckCount = 200;
-    this.selectStuckTimeout = 6e4;
-    this.selectStuckMaxTimeout = 80;
-    this.selectCount = 200;
-    this.selectTimeout = 3e3;
-    this.buildBatchCount = 800;
-    this.buildBatchMaxCount = 1e4;
+    this.selectStuckCount = 256;
+    this.selectStuckTimeout = 2e4;
+    this.selectStuckMaxTimeout = 40;
+    this.selectCount = 1024;
+    this.selectTimeout = 100;
+    this.buildBatchCount = 1e3;
+    this.buildBatchMaxCount = 2e4;
     this.retryCount = 1;
     this.executeSync = false;
-    this.idemLockTimeout = 18e4;
-    this.idemDoneTimeout = 6e4;
+    this.idemLockTimeout = 15e3;
+    this.idemDoneTimeout = 1e4;
     this.logStatus = false;
-    this.logStatusTimeout = 18e5;
-    this.approveCount = 2e3;
+    this.logStatusTimeout = 12e4;
+    this.approveCount = 4e3;
     this.removeOnExecuted = true;
+    this.concurrency = 256;
+    this.limit = pLimit(this.concurrency);
   }
   signal() {
     return this.abort.signal;
@@ -276,6 +275,7 @@ var PowerQueues = class extends PowerRedis {
   }
   async runQueue(queueName, from = "0-0") {
     setMaxListeners(0, this.abort.signal);
+    this.limit = pLimit(this.concurrency);
     await this.createGroup(queueName, from);
     await this.consumerLoop(queueName, from);
   }
@@ -295,9 +295,9 @@ var PowerQueues = class extends PowerRedis {
       let tasks = [];
       try {
         tasks = await this.select(queueName, from);
-      } catch (err) {
+      } catch {
       }
-      if (!isArrFilled(tasks)) {
+      if (!Array.isArray(tasks) || !(tasks.length > 0)) {
         await wait(300);
         continue;
       }
@@ -357,7 +357,7 @@ var PowerQueues = class extends PowerRedis {
     }
   }
   async approve(queueName, tasks) {
-    if (!isArrFilled(tasks)) {
+    if (!Array.isArray(tasks) || !(tasks.length > 0)) {
       return 0;
     }
     const approveCount = Math.max(500, Math.min(4e3, this.approveCount));
@@ -373,7 +373,7 @@ var PowerQueues = class extends PowerRedis {
   }
   async select(queueName, from = "0-0") {
     let selected = await this.selectS(queueName, from);
-    if (!isArrFilled(selected)) {
+    if (!Array.isArray(selected) || !(selected.length > 0)) {
       selected = await this.selectF(queueName, from);
     }
     return this.selectP(selected);
@@ -381,7 +381,7 @@ var PowerQueues = class extends PowerRedis {
   async selectS(queueName, from = "0-0") {
     try {
       const res = await this.runScript("SelectStuck", [queueName], [this.group, this.consumer(), String(this.selectStuckTimeout), String(this.selectStuckCount), String(this.selectStuckMaxTimeout)], SelectStuck);
-      return isArr(res) ? res : [];
+      return Array.isArray(res) ? res : [];
     } catch (err) {
       if (String(err?.message || "").includes("NOGROUP")) {
         await this.createGroup(queueName, from);
@@ -405,7 +405,7 @@ var PowerQueues = class extends PowerRedis {
         ">"
       );
       rows = res?.[0]?.[1] ?? [];
-      if (!isArrFilled(rows)) {
+      if (!Array.isArray(rows) || !(rows.length > 0)) {
         return [];
       }
     } catch (err) {
@@ -422,9 +422,9 @@ var PowerQueues = class extends PowerRedis {
     return Array.from(raw || []).map((e) => {
       const id = Buffer.isBuffer(e?.[0]) ? e[0].toString() : e?.[0];
       const kvRaw = e?.[1] ?? [];
-      const kv = isArr(kvRaw) ? kvRaw.map((x) => Buffer.isBuffer(x) ? x.toString() : x) : [];
+      const kv = Array.isArray(kvRaw) ? kvRaw.map((x) => Buffer.isBuffer(x) ? x.toString() : x) : [];
       return [id, kv];
-    }).filter(([id, kv]) => isStrFilled(id) && isArr(kv) && (kv.length & 1) === 0).map(([id, kv]) => {
+    }).filter(([id, kv]) => typeof id === "string" && id.length > 0 && Array.isArray(kv) && (kv.length & 1) === 0).map(([id, kv]) => {
       const { payload, createdAt, job, idemKey, attempt } = this.values(kv);
       return [id, this.payload(payload), createdAt, job, idemKey, Number(attempt)];
     });
@@ -467,10 +467,10 @@ var PowerQueues = class extends PowerRedis {
     }
     let start = Date.now();
     if (!this.executeSync && promises.length > 0) {
-      await Promise.all(promises.map((item) => item()));
+      await Promise.all(promises.map((item) => this.limit(() => item())));
     }
     await this.onBatchReady(queueName, result);
-    if (!isArrFilled(result) && contended > tasks.length >> 1) {
+    if ((!Array.isArray(result) || !(result.length > 0)) && contended > tasks.length >> 1) {
       await this.waitAbortable(15 + Math.floor(Math.random() * 35) + Math.min(250, 15 * contended + Math.floor(Math.random() * 40)));
     }
     return result;
@@ -648,7 +648,7 @@ var PowerQueues = class extends PowerRedis {
   }
   async runScript(name, keys, args, defaultCode) {
     if (!this.scripts[name]) {
-      if (!isStrFilled(defaultCode)) {
+      if (!(typeof defaultCode === "string" && defaultCode.length > 0)) {
         throw new Error(`Undefined script "${name}". Save it before executing.`);
       }
       this.saveScript(name, defaultCode);
@@ -696,20 +696,20 @@ var PowerQueues = class extends PowerRedis {
     throw new Error("Load lua script failed.");
   }
   saveScript(name, codeBody) {
-    if (!isStrFilled(codeBody)) {
+    if (!(typeof codeBody === "string" && codeBody.length > 0)) {
       throw new Error("Script body is empty.");
     }
     this.scripts[name] = { codeBody };
     return codeBody;
   }
   async addTasks(queueName, data, opts = {}) {
-    if (!isArrFilled(data)) {
+    if (!Array.isArray(data) || !(data.length > 0)) {
       throw new Error("Tasks is not filled.");
     }
-    if (!isStrFilled(queueName)) {
+    if (!(typeof queueName === "string" && queueName.length > 0)) {
       throw new Error("Queue name is required.");
     }
-    opts.job = opts.job ?? uuid();
+    opts.job = opts.job ?? randomUUID();
     const batches = this.buildBatches(data, opts);
     const result = new Array(data.length);
     const promises = [];
@@ -750,8 +750,8 @@ var PowerQueues = class extends PowerRedis {
       const entry = {
         payload: JSON.stringify(taskP),
         attempt: Number(opts.attempt || 0),
-        job: opts.job ?? uuid(),
-        idemKey: String(idemKey || uuid()),
+        job: opts.job ?? randomUUID(),
+        idemKey: String(idemKey || randomUUID()),
         createdAt
       };
       const reqKeysLength = this.keysLength(entry);
@@ -807,7 +807,7 @@ var PowerQueues = class extends PowerRedis {
       argv.push(String(id));
       argv.push(String(pairs));
       for (const token of flat) {
-        argv.push(!isExists(token) ? "" : isStrFilled(token) ? token : String(token));
+        argv.push(token === null || token === void 0 ? "" : typeof token === "string" && token.length > 0 ? token : String(token));
       }
     }
     return argv;
@@ -832,5 +832,6 @@ var PowerQueues = class extends PowerRedis {
   }
 };
 export {
-  PowerQueues
+  PowerQueues,
+  wait
 };

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "power-queues",
-	"version": "2.1.10",
+	"version": "2.1.12",
 	"description": "High-performance Redis Streams queue for Node.js with Lua-powered bulk XADD, idempotent workers, heartbeat locks, stuck-task recovery, retries, DLQ, and distributed processing.",
 	"author": "ihor-bielchenko",
 	"license": "MIT",
@@ -49,6 +49,9 @@
 		"redis-pipeline",
 		"redis-multi",
 		"redis-atomic",
+		"p-limit",
+		"limit",
+		"concurrency",
 		"json-serialization",
 		"safe-serialization",
 		"key-schema",
@@ -81,8 +84,7 @@
 		"power-redis"
 	],
 	"dependencies": {
-		"full-utils": "^3.0.9",
-		"power-redis": "^2.0.23",
-		"uuid": "^13.0.0"
+		"p-limit": "^7.3.0",
+		"power-redis": "^2.0.23"
 	}
 }