redis-smq 8.0.0-rc.15 → 8.0.0-rc.16

package/CHANGELOG.md

@@ -1,5 +1,34 @@
 
 
+## [8.0.0-rc.16](https://github.com/weyoss/redis-smq/compare/v8.0.0-rc.15...v8.0.0-rc.16) (2024-01-28)
+
+
+### Features
+
+* allow running/sandboxing message handlers using worker threads ([53095bd](https://github.com/weyoss/redis-smq/commit/53095bd2689ccb2f701365bafb4ef8e550da82b3))
+
+
+### Bug Fixes
+
+* export message handler errors ([26d2689](https://github.com/weyoss/redis-smq/commit/26d26890da139d88c46ce220e3dda3d9c8e03aee))
+
+
+### Documentation
+
+* **ConsumeMessageWorker:** update docs and clean up ([140322e](https://github.com/weyoss/redis-smq/commit/140322e3520ee3db905ea977a159064f339aa7d2))
+* fix broken links ([aad9010](https://github.com/weyoss/redis-smq/commit/aad9010e70ed332b154f4fbdcb8ac657a14fe6b9))
+* update messages and queues documentation ([7c3d06e](https://github.com/weyoss/redis-smq/commit/7c3d06ed3765ffc14dfece4f725241aa4535e1e7))
+
+
+### Codebase Refactoring
+
+* **ConsumeMessageWorker:** improve typings ([66c6ca9](https://github.com/weyoss/redis-smq/commit/66c6ca948677f76d496fe0b988bd195e1a7108a2))
+
+
+### Tests
+
+* allow running/sandboxing message handlers using worker threads ([79e6886](https://github.com/weyoss/redis-smq/commit/79e688674ed11ec34ce2f74d5f15c8eefa129767))
+
 ## [8.0.0-rc.15](https://github.com/weyoss/redis-smq/compare/v8.0.0-rc.14...v8.0.0-rc.15) (2024-01-25)
 
 

package/README.md

@@ -27,6 +27,7 @@ RedisSMQ is a Node.js library for queuing messages (aka jobs) and processing the
 * Supports [Point-2-Point](docs/queue-delivery-models.md#point-2-point-delivery-model) and [Pub/Sub](docs/queue-delivery-models.md#pubsub-delivery-model) [delivery models](docs/queue-delivery-models.md).
 * Both [delivery models](docs/queue-delivery-models.md) are reliable. For cases of failure, while delivering/consuming messages, [at-least-once](docs/api/classes/ProducibleMessage.md#setretrythreshold) and [at-most-once](docs/api/classes/ProducibleMessage.md#setretrythreshold) modes may be configured.
 * [3 queuing strategies](docs/queues.md): [FIFO queues](docs/queues.md#fifo-first-in-first-out-queues), [LIFO queues](docs/queues.md#lifo-last-in-first-out-queues), and [Priority Queues](docs/queues.md#priority-queues).
+* [Message Handler Worker Threads](docs/message-handler-worker-threads.md) which allow sandboxing and running your message handler from a separate isolated thread and without affecting the performance of other message handlers from the same consumer.
 * Messages can be [set to expire](docs/api/classes/ProducibleMessage.md#setttl) when not delivered within a given amount of time or to have a [consumption timeout](docs/api/classes/ProducibleMessage.md#setconsumetimeout) while being in process.
 * Queues may be [rate Limited](docs/queue-rate-limiting.md) to control the rate at which the messages are consumed.
 * Has a builtin [scheduler](docs/scheduling-messages.md) allowing messages [to be delayed](docs/api/classes/ProducibleMessage.md#setscheduleddelay), [to be delivered for N times](docs/api/classes/ProducibleMessage.md#setscheduledrepeat) with an optional [period between deliveries](docs/api/classes/ProducibleMessage.md#setscheduledrepeatperiod), or simply [to be scheduled using CRON expressions](docs/api/classes/ProducibleMessage.md#setscheduledcron).
@@ -66,42 +67,44 @@ A queue is responsible for holding messages which are produced by producers and
 ### Creating a queue
 
 ```javascript
-const { Queue, EQueueType, EQueueDeliveryModel } = require('redis-smq');
-
 const queue = new Queue();
-
-// Creating a LIFO queue
-queue.save('my_queue', EQueueType.LIFO_QUEUE, EQueueDeliveryModel.POINT_TO_POINT, (err) => console.log(err));
+queue.save('my_queue', EQueueType.LIFO_QUEUE, EQueueDeliveryModel.POINT_TO_POINT, (err) => {
+  if (err) console.error(err)
+});
 ```
 
 In the example above we are defining a [LIFO queue](docs/queues.md#lifo-last-in-first-out-queues) with a [POINT-2-POINT delivery model](docs/queue-delivery-models.md#point-2-point-delivery-model).
 
+See [Queues](docs/queues.md) for more details.
+
 ### Producing a message
 
 ```javascript
-const { Producer, ProducibleMessage } = require('redis-smq');
-
-const producer = new Producer();
-
 const msg = new ProducibleMessage();
 msg.setQueue('my_queue').setBody('Hello Word!')
-
-producer.produce(msg, (err) => console.log(err));
+producer.produce(msg, (err, ids) => {
+  if (err) console.error(err);
+  else console.log(`Produced message IDs are: ${ids.join(', ')}`)
+});
 ```
 
+See [Producing Messages](docs/producing-messages.md) for more details.
+
 ### Consuming a message
 
 ```javascript
-const { Consumer } = require('redis-smq');
-
 const consumer = new Consumer();
 const messageHandler = (msg, cb) => {
-  console.log(msg.getBody());
+  console.log(msg.body);
   cb();
 }
-consumer.consume('my_queue', messageHandler, (err) => console.log(err));
+consumer.consume('my_queue', messageHandler, (err) => {
+  if (err) console.error(err);
+});
 ```
 
+See [Consuming Messages](docs/consuming-messages.md) for more details.
+
 ## Documentation
 
 See [RedisSMQ Docs](docs/README.md) for more details.

package/dist/index.cjs

@@ -69,6 +69,8 @@ __export(redis_smq_exports, {
   ConsumerGroups: () => ConsumerGroups,
   ConsumerInvalidGroupIdError: () => ConsumerInvalidGroupIdError,
   ConsumerMessageHandlerAlreadyExistsError: () => ConsumerMessageHandlerAlreadyExistsError,
+  ConsumerMessageHandlerError: () => ConsumerMessageHandlerError,
+  ConsumerMessageHandlerWorkerError: () => ConsumerMessageHandlerWorkerError,
   EConsumeMessageDeadLetterCause: () => EConsumeMessageDeadLetterCause,
   EConsumeMessageUnacknowledgedCause: () => EConsumeMessageUnacknowledgedCause,
   EExchangeType: () => EExchangeType,
@@ -78,6 +80,8 @@ __export(redis_smq_exports, {
   EQueueDeliveryModel: () => EQueueDeliveryModel,
   EQueueProperty: () => EQueueProperty,
   EQueueType: () => EQueueType,
+  EWorkerThreadMessageCodeConsume: () => EWorkerThreadMessageCodeConsume,
+  EWorkerThreadMessageCodeExit: () => EWorkerThreadMessageCodeExit,
   ExchangeDirect: () => ExchangeDirect,
   ExchangeError: () => ExchangeError,
   ExchangeFanOut: () => ExchangeFanOut,
@@ -437,6 +441,22 @@ var EConsumeMessageUnacknowledgedCause = /* @__PURE__ */ ((EConsumeMessageUnackn
   return EConsumeMessageUnacknowledgedCause3;
 })(EConsumeMessageUnacknowledgedCause || {});
 
+// types/consumer/consume-message-worker.ts
+var EWorkerThreadMessageCodeExit = /* @__PURE__ */ ((EWorkerThreadMessageCodeExit2) => {
+  EWorkerThreadMessageCodeExit2[EWorkerThreadMessageCodeExit2["WORKER_DATA_REQUIRED"] = 100] = "WORKER_DATA_REQUIRED";
+  EWorkerThreadMessageCodeExit2[EWorkerThreadMessageCodeExit2["INVALID_HANDLER_TYPE"] = 101] = "INVALID_HANDLER_TYPE";
+  EWorkerThreadMessageCodeExit2[EWorkerThreadMessageCodeExit2["HANDLER_IMPORT_ERROR"] = 102] = "HANDLER_IMPORT_ERROR";
+  EWorkerThreadMessageCodeExit2[EWorkerThreadMessageCodeExit2["UNCAUGHT_EXCEPTION"] = 103] = "UNCAUGHT_EXCEPTION";
+  EWorkerThreadMessageCodeExit2[EWorkerThreadMessageCodeExit2["TERMINATED"] = 104] = "TERMINATED";
+  return EWorkerThreadMessageCodeExit2;
+})(EWorkerThreadMessageCodeExit || {});
+var EWorkerThreadMessageCodeConsume = /* @__PURE__ */ ((EWorkerThreadMessageCodeConsume2) => {
+  EWorkerThreadMessageCodeConsume2[EWorkerThreadMessageCodeConsume2["OK"] = 200] = "OK";
+  EWorkerThreadMessageCodeConsume2[EWorkerThreadMessageCodeConsume2["MESSAGE_PROCESSING_ERROR"] = 201] = "MESSAGE_PROCESSING_ERROR";
+  EWorkerThreadMessageCodeConsume2[EWorkerThreadMessageCodeConsume2["MESSAGE_PROCESSING_CAUGHT_ERROR"] = 202] = "MESSAGE_PROCESSING_CAUGHT_ERROR";
+  return EWorkerThreadMessageCodeConsume2;
+})(EWorkerThreadMessageCodeConsume || {});
+
 // types/exchange/common.ts
 var EExchangeType = /* @__PURE__ */ ((EExchangeType2) => {
   EExchangeType2[EExchangeType2["DIRECT"] = 0] = "DIRECT";
@@ -674,6 +694,23 @@ var MessageDeleteError = class extends MessageError {
   }
 };
 
+// src/lib/consumer/message-handler/errors/consumer-message-handler.error.ts
+var ConsumerMessageHandlerError = class extends ConsumerError {
+  constructor(msg) {
+    super(msg);
+  }
+};
+
+// src/lib/consumer/message-handler/errors/consumer-message-handler-worker.error.ts
+var ConsumerMessageHandlerWorkerError = class extends ConsumerMessageHandlerError {
+  constructor(msg) {
+    const { code, error } = msg;
+    super(
+      `Error code: ${code}.${error ? `Message: ${error.name}: ${error.message}` : ""}`
+    );
+  }
+};
+
 // src/lib/consumer/consumer-heartbeat.ts
 var os = __toESM(require("os"), 1);
 var import_redis_smq_common10 = require("redis-smq-common");
@@ -981,6 +1018,8 @@ var Base = class extends import_redis_smq_common11.EventEmitter {
 };
 
 // src/lib/consumer/message-handler/message-handler.ts
+var import_fs = require("fs");
+var import_path3 = __toESM(require("path"), 1);
 var import_uuid4 = require("uuid");
 
 // src/lib/consumer/message-handler/processing-queue.ts
@@ -1419,6 +1458,13 @@ var MessageEnvelope = class {
       consumerGroupId: this.getConsumerGroupId()
     };
   }
+  transfer() {
+    return __spreadProps(__spreadValues({}, this.toJSON()), {
+      id: this.getId(),
+      messageState: this.getMessageState().toJSON(),
+      status: this.getStatus()
+    });
+  }
   hasRetryThresholdExceeded() {
     const threshold = this.producibleMessage.getRetryThreshold();
     return this.messageState.getAttempts() + 1 >= threshold;
@@ -2679,87 +2725,76 @@ var DequeueMessage = class {
 // src/lib/consumer/message-handler/consume-message.ts
 var import_redis_smq_common24 = require("redis-smq-common");
 
-// src/lib/message/_create-consumable-message.ts
-function _createConsumableMessage(msg) {
-  return {
-    getPublishedAt() {
-      return msg.getPublishedAt();
-    },
-    getScheduledAt() {
-      return msg.getScheduledAt();
-    },
-    getScheduledMessageId() {
-      return msg.getScheduledMessageId();
-    },
-    getId() {
-      return msg.getId();
-    },
-    getStatus() {
-      return msg.getStatus();
-    },
-    hasPriority() {
-      return msg.producibleMessage.hasPriority();
-    },
-    getQueue() {
-      return msg.producibleMessage.getQueue();
-    },
-    getDestinationQueue() {
-      return msg.getDestinationQueue();
-    },
-    getPriority() {
-      return msg.producibleMessage.getPriority();
-    },
-    getBody() {
-      return msg.producibleMessage.getBody();
-    },
-    getTTL() {
-      return msg.producibleMessage.getTTL();
-    },
-    getRetryThreshold() {
-      return msg.producibleMessage.getRetryThreshold();
-    },
-    getRetryDelay() {
-      return msg.producibleMessage.getRetryDelay();
-    },
-    getConsumeTimeout() {
-      return msg.producibleMessage.getConsumeTimeout();
-    },
-    getCreatedAt() {
-      return msg.producibleMessage.getCreatedAt();
-    },
-    getScheduledRepeat() {
-      return msg.producibleMessage.getScheduledRepeat();
-    },
-    getScheduledRepeatPeriod() {
-      return msg.producibleMessage.getScheduledRepeatPeriod();
-    },
-    getScheduledCRON() {
-      return msg.producibleMessage.getScheduledCRON();
-    },
-    getScheduledDelay() {
-      return msg.producibleMessage.getScheduledDelay();
-    },
-    getFanOut() {
-      return msg.producibleMessage.getFanOut();
-    },
-    getTopic() {
-      return msg.producibleMessage.getTopic();
-    },
-    toJSON() {
-      return msg.toJSON();
-    },
-    getExchange() {
-      return msg.getExchange();
-    },
-    getConsumerGroupId() {
-      return msg.getConsumerGroupId();
+// src/lib/consumer/message-handler/consume-message-worker.ts
+var import_path2 = __toESM(require("path"), 1);
+var import_worker_threads = require("worker_threads");
+var ConsumeMessageWorker = class {
+  constructor(messageHandlerFilename) {
+    this.messageHandlerThread = null;
+    this.messageHandlerFilename = messageHandlerFilename;
+  }
+  getMessageHandlerThread() {
+    if (!this.messageHandlerThread) {
+      this.messageHandlerThread = new import_worker_threads.Worker(
+        import_path2.default.resolve(__dirname, "./consume-message-worker-thread.js"),
+        {
+          workerData: this.messageHandlerFilename
+        }
+      );
+      this.messageHandlerThread.on("messageerror", (err) => {
+        console.error(err);
+      });
+      this.messageHandlerThread.on("error", (err) => {
+        console.error(err);
+      });
+      this.messageHandlerThread.on("exit", () => {
+        this.messageHandlerThread = null;
+      });
     }
-  };
-}
+    return this.messageHandlerThread;
+  }
+  consume(message, cb) {
+    const worker = this.getMessageHandlerThread();
+    const cleanUp = () => {
+      worker.removeListener("message", onMessage).removeListener("exit", onExit);
+    };
+    const onMessage = (msg) => {
+      cleanUp();
+      if (msg.code !== 200 /* OK */) {
+        console.error(`ConsumerMessageHandlerWorkerError`, msg);
+        cb(new ConsumerMessageHandlerWorkerError(msg));
+      } else
+        cb();
+    };
+    const onExit = () => {
+      cleanUp();
+      const msg = {
+        code: 104 /* TERMINATED */,
+        error: null
+      };
+      console.error("ConsumerMessageHandlerWorkerError", msg);
+      cb(new ConsumerMessageHandlerWorkerError(msg));
+    };
+    worker.once("message", onMessage);
+    worker.once("exit", onExit);
+    worker.postMessage(message);
+  }
+  quit(cb) {
+    const callback = () => {
+      this.messageHandlerThread = null;
+      cb();
+    };
+    if (this.messageHandlerThread) {
+      this.messageHandlerThread.terminate().then(callback).catch(callback);
+    } else
+      cb();
+  }
+};
 
 // src/lib/consumer/message-handler/consume-message.ts
 var ConsumeMessage = class {
   constructor(messageHandler, redisClient2, logger3) {
+    this.consumeMessageWorker = null;
     this.redisClient = redisClient2;
     this.messageHandler = messageHandler;
     this.queue = messageHandler.getQueue();
@@ -2843,6 +2878,20 @@ var ConsumeMessage = class {
       }
     );
   }
+  getConsumeMessageWorker(messageHandlerFilename) {
+    if (!this.consumeMessageWorker) {
+      this.consumeMessageWorker = new ConsumeMessageWorker(
+        messageHandlerFilename
+      );
+    }
+    return this.consumeMessageWorker;
+  }
+  invokeMessageHandler(messageHandler, msg, cb) {
+    if (typeof messageHandler === "string") {
+      this.getConsumeMessageWorker(messageHandler).consume(msg, cb);
+    } else
+      messageHandler(msg, cb);
+  }
   consumeMessage(msg) {
     let isTimeout = false;
     let timer = null;
@@ -2884,8 +2933,9 @@ var ConsumeMessage = class {
           }
         }
       };
-      this.messageHandler.getHandler()(
-        _createConsumableMessage(msg),
+      this.invokeMessageHandler(
+        this.messageHandler.getHandler(),
+        msg.transfer(),
         onConsumed
       );
     } catch (error) {
@@ -2905,6 +2955,12 @@ var ConsumeMessage = class {
     } else
       this.consumeMessage(message);
   }
+  quit(cb) {
+    if (this.consumeMessageWorker) {
+      this.consumeMessageWorker.quit(cb);
+    } else
+      cb();
+  }
 };
 
 // src/lib/consumer/message-handler/message-handler.ts
@@ -2920,7 +2976,7 @@ var MessageHandler = class extends import_redis_smq_common25.EventEmitter {
     this.dequeueRedisClient = dequeueRedisClient;
     this.sharedRedisClient = sharedRedisClient;
     this.handler = messageHandler;
-    this.powerSwitch = new import_redis_smq_common25.PowerSwitch();
+    this.powerSwitch = new import_redis_smq_common25.PowerSwitch(false);
     this.logger = logger3;
     this.dequeueMessage = new DequeueMessage(this, dequeueRedisClient);
     this.consumeMessage = new ConsumeMessage(this, dequeueRedisClient, logger3);
@@ -3010,6 +3066,17 @@ var MessageHandler = class extends import_redis_smq_common25.EventEmitter {
   shutdownDequeueClient(cb) {
     this.dequeueRedisClient.halt(cb);
   }
+  forceShutDown(cb) {
+    this.powerSwitch.rollback();
+    import_redis_smq_common25.async.waterfall(
+      [
+        (cb2) => this.dequeueMessage.quit(cb2),
+        (cb2) => this.consumeMessage.quit(cb2),
+        (cb2) => this.shutdownDequeueClient(cb2)
+      ],
+      cb
+    );
+  }
   handleError(err) {
     if (this.powerSwitch.isRunning() || this.powerSwitch.isGoingUp()) {
       this.emit("error", err);
@@ -3020,35 +3087,63 @@ var MessageHandler = class extends import_redis_smq_common25.EventEmitter {
   }
   run(cb) {
     this.powerSwitch.goingUp();
-    this.dequeueMessage.run((err) => {
-      if (err)
-        cb(err);
-      else {
-        this.powerSwitch.commit();
-        this.emit("up");
-        cb();
+    import_redis_smq_common25.async.waterfall(
+      [
+        (cb2) => {
+          if (typeof this.handler === "string") {
+            if (![".js", ".cjs"].includes(import_path3.default.extname(this.handler))) {
+              cb2(new ConsumerMessageHandlerError("Invalid file extension"));
+            } else
+              (0, import_fs.stat)(this.handler, (err) => {
+                if (err)
+                  cb2(new ConsumerMessageHandlerError(err.message));
+                else
+                  cb2();
+              });
+          } else
+            cb2();
+        },
+        (cb2) => this.dequeueMessage.run((err) => {
+          if (err)
+            cb2(err);
+          else {
+            this.powerSwitch.commit();
+            this.emit("up");
+            cb2();
+          }
+        })
+      ],
+      (err) => {
+        if (err)
+          this.forceShutDown(() => cb(err));
+        else
+          cb();
       }
-    });
+    );
   }
   shutdown(messageUnacknowledgedCause, cb) {
     const goDown = () => {
-      this.powerSwitch.goingDown();
-      import_redis_smq_common25.async.waterfall(
-        [
-          (cb2) => this.dequeueMessage.quit(cb2),
-          (cb2) => this.cleanUp(messageUnacknowledgedCause, cb2),
-          (cb2) => this.shutdownDequeueClient(cb2)
-        ],
-        (err) => {
-          if (err)
-            cb(err);
-          else {
-            this.powerSwitch.commit();
-            this.emit("down");
-            cb();
+      const r = this.powerSwitch.goingDown();
+      if (r) {
+        import_redis_smq_common25.async.waterfall(
+          [
+            (cb2) => this.dequeueMessage.quit(cb2),
+            (cb2) => this.consumeMessage.quit(cb2),
+            (cb2) => this.cleanUp(messageUnacknowledgedCause, cb2),
+            (cb2) => this.shutdownDequeueClient(cb2)
+          ],
+          (err) => {
+            if (err)
+              cb(err);
+            else {
+              this.powerSwitch.commit();
+              this.emit("down");
+              cb();
+            }
           }
-        }
-      );
+        );
+      } else
+        cb();
     };
     if (this.powerSwitch.isGoingUp())
       this.once("up", goDown);
@@ -3147,7 +3242,12 @@ var MessageHandlerRunner = class {
           client,
           handlerParams
         );
-        handler.run(cb);
+        handler.run((err2) => {
+          if (err2)
+            this.removeMessageHandler(handlerParams.queue, () => cb(err2));
+          else
+            cb();
+        });
       }
     });
   }
@@ -4488,7 +4588,7 @@ var Message = class {
           else {
             cb(
               null,
-              reply.map((i) => _createConsumableMessage(i))
+              reply.map((i) => i.transfer())
             );
           }
         });
@@ -4507,7 +4607,7 @@ var Message = class {
           else if (!reply)
             cb(new import_redis_smq_common36.CallbackEmptyReplyError());
           else
-            cb(null, _createConsumableMessage(reply));
+            cb(null, reply.transfer());
         });
     });
   }
@@ -5727,6 +5827,8 @@ function disconnect(cb) {
   ConsumerGroups,
   ConsumerInvalidGroupIdError,
   ConsumerMessageHandlerAlreadyExistsError,
+  ConsumerMessageHandlerError,
+  ConsumerMessageHandlerWorkerError,
   EConsumeMessageDeadLetterCause,
   EConsumeMessageUnacknowledgedCause,
   EExchangeType,
@@ -5736,6 +5838,8 @@ function disconnect(cb) {
   EQueueDeliveryModel,
   EQueueProperty,
   EQueueType,
+  EWorkerThreadMessageCodeConsume,
+  EWorkerThreadMessageCodeExit,
   ExchangeDirect,
   ExchangeError,
   ExchangeFanOut,