rascal 15.0.1 → 16.1.0

package/CHANGELOG.md

@@ -1,5 +1,15 @@
 # Change Log
 
+## 16.1.0
+
+- Added concurrency option for managing RabbitMQ topology. Rascal will create and use upto this number of channels when asserting/checking/deleting/purging queues, exchanges and bindings.
+
+## 16.0.0
+
+- Automatically set replyTo message property when a publication references a replyTo queue
+- Stop prefixing binding keys with replyTo uuid
+- Drop support for Node 12
+
 ## 15.0.1
 
 - Fix MaxListenersExceeded warning when there are more than 10 vhosts - See https://github.com/guidesmiths/rascal/issues/206

package/README.md

@@ -30,6 +30,7 @@ Rascal seeks to either solve these problems, make them easier to deal with or br
 - Transparent encryption / decryption
 - Automatic reconnection and resubscription
 - Advanced error handling including delayed, limited retries
+- RPC Support
 - Redelivery protection
 - Channel pooling
 - Flow control
@@ -679,6 +680,26 @@ Enable to purge the queue during initialisation. Useful when running automated t
 }
 ```
 
+##### replyTo
+
+Sometimes you want to publish a message, and have the consumer of the message send a reply to the same application instance that published the original message. This can be difficult if you application is deployed using multiple instances which share a common configuration. Quite often the solution is to make your application stateless so it doesn't matter which instance receives the reply. An altnernative is to mark the queue as a reply queue using the replyTo.
+
+```json
+{
+  "queues": {
+    "q1": {
+      "replyTo": true
+    }
+  },
+  "publications": {
+    "exchange": "e1",
+    "replyTo": "q1"
+  }
+}
+```
+
+When true, Rascal will append a uuid to the queue name so that it is unique for each instance of the application. Use this conjunction with the publication replyTo property, to automaticlaly set the replyTo property on outbound messages to the unique queue name. You may also want to make make the queue non durable and exclusive too (see below).
+
 ##### options
 
 Define any further configuration in an options block
@@ -1691,6 +1712,10 @@ is equivalent to...
 }
 ```
 
+### Concurrency
+
+If you have a high number of exchanges, queues and bindings you may wish to initialise Rascal using multiple channels to improve startup time. Do this per vhost by setting the `concurrency` option to the number of channels you want to create and use.
+
 ### Connect
 
 Rascal is a rich pub/sub wrapper and as such hides much of the amqplib [channel api](https://www.squaremobius.net/amqp.node/channel_api.html#channel). If you need to access this you can programmatically establish a connection to a vhost as follows.

package/lib/amqp/Publication.js

@@ -37,6 +37,7 @@ function Publication(vhost, borrowChannelFn, returnChannelFn, destroyChannelFn,
     const content = getContent(payload);
     publishConfig.options.contentType = publishConfig.options.contentType || content.type;
     publishConfig.options.messageId = publishConfig.options.messageId || uuid();
+    publishConfig.options.replyTo = publishConfig.options.replyTo || publishConfig.replyTo;
 
     publishConfig.encryption ? _publishEncrypted(content.buffer, publishConfig, next) : _publish(content.buffer, publishConfig, next);
   };

package/lib/amqp/Vhost.js

@@ -26,10 +26,10 @@ function Vhost(vhostConfig, components) {
   let confirmChannelPool;
   const channelCreator = async.queue(createChannel, 1);
 
-  const init = async.compose(tasks.closeChannel, tasks.applyBindings, tasks.purgeQueues, tasks.checkQueues, tasks.assertQueues, tasks.checkExchanges, tasks.assertExchanges, tasks.createChannel, tasks.createConnection, tasks.checkVhost, tasks.assertVhost);
+  const init = async.compose(tasks.closeChannels, tasks.applyBindings, tasks.purgeQueues, tasks.checkQueues, tasks.assertQueues, tasks.checkExchanges, tasks.assertExchanges, tasks.createChannels, tasks.createConnection, tasks.checkVhost, tasks.assertVhost);
   const connect = async.compose(tasks.createConnection);
-  const purge = async.compose(tasks.closeConnection, tasks.closeChannel, tasks.purgeQueues, tasks.createChannel, tasks.createConnection);
-  const nuke = async.compose(tasks.closeConnection, tasks.closeChannel, tasks.deleteQueues, tasks.deleteExchanges, tasks.createChannel, tasks.createConnection);
+  const purge = async.compose(tasks.closeConnection, tasks.closeChannels, tasks.purgeQueues, tasks.createChannels, tasks.createConnection);
+  const nuke = async.compose(tasks.closeConnection, tasks.closeChannels, tasks.deleteQueues, tasks.deleteExchanges, tasks.createChannels, tasks.createConnection);
   let timer = backoff({});
   let paused = true;
   let shuttingDown = false;

package/lib/amqp/tasks/applyBindings.js

@@ -9,10 +9,12 @@ module.exports = _.curry((config, ctx, next) => {
     exchange: bindExchange,
   };
 
-  async.eachSeries(
+  async.eachOfLimit(
     _.values(config.bindings),
-    (binding, callback) => {
-      bind[binding.destinationType](config, ctx.channel, binding, callback);
+    config.concurrency,
+    (binding, index, cb) => {
+      const channel = ctx.channels[index % config.concurrency];
+      bind[binding.destinationType](config, channel, binding, cb);
     },
     (err) => {
       next(err, config, ctx);

package/lib/amqp/tasks/assertExchanges.js

@@ -3,10 +3,12 @@ const _ = require('lodash');
 const async = require('async');
 
 module.exports = _.curry((config, ctx, next) => {
-  async.eachSeries(
+  async.eachOfLimit(
     _.keys(config.exchanges),
-    (name, callback) => {
-      assertExchange(ctx.channel, config.exchanges[name], callback);
+    config.concurrency,
+    (name, index, cb) => {
+      const channel = ctx.channels[index % config.concurrency];
+      assertExchange(channel, config.exchanges[name], cb);
     },
     (err) => {
       next(err, config, ctx);
@@ -18,5 +20,9 @@ function assertExchange(channel, config, next) {
   if (!config.assert) return next();
   if (config.fullyQualifiedName === '') return next();
   debug('Asserting exchange: %s', config.fullyQualifiedName);
-  channel.assertExchange(config.fullyQualifiedName, config.type, config.options, next);
+  channel.assertExchange(config.fullyQualifiedName, config.type, config.options, (err) => {
+    if (err) return next(err);
+    debug('Asserted exchange: %s', config.fullyQualifiedName);
+    next();
+  });
 }

package/lib/amqp/tasks/assertQueues.js

@@ -3,10 +3,12 @@ const _ = require('lodash');
 const async = require('async');
 
 module.exports = _.curry((config, ctx, next) => {
-  async.eachSeries(
+  async.eachOfLimit(
     _.keys(config.queues),
-    (name, callback) => {
-      assertQueue(ctx.channel, config.queues[name], callback);
+    config.concurrency,
+    (name, index, cb) => {
+      const channel = ctx.channels[index % config.concurrency];
+      assertQueue(channel, config.queues[name], cb);
     },
     (err) => {
       next(err, config, ctx);

package/lib/amqp/tasks/checkExchanges.js

@@ -3,10 +3,12 @@ const _ = require('lodash');
 const async = require('async');
 
 module.exports = _.curry((config, ctx, next) => {
-  async.eachSeries(
+  async.eachOfLimit(
     _.keys(config.exchanges),
-    (name, callback) => {
-      checkExchange(ctx.channel, config.exchanges[name], callback);
+    config.concurrency,
+    (name, index, cb) => {
+      const channel = ctx.channels[index % config.concurrency];
+      checkExchange(channel, config.exchanges[name], cb);
     },
     (err) => {
       next(err, config, ctx);

package/lib/amqp/tasks/checkQueues.js

@@ -3,10 +3,12 @@ const _ = require('lodash');
 const async = require('async');
 
 module.exports = _.curry((config, ctx, next) => {
-  async.eachSeries(
+  async.eachOfLimit(
     _.keys(config.queues),
-    (name, callback) => {
-      checkQueue(ctx.channel, config.queues[name], callback);
+    config.concurrency,
+    (name, index, cb) => {
+      const channel = ctx.channels[index % config.concurrency];
+      checkQueue(channel, config.queues[name], cb);
     },
     (err) => {
       next(err, config, ctx);

package/lib/amqp/tasks/closeChannels.js

@@ -0,0 +1,18 @@
+const debug = require('debug')('rascal:tasks:closeChannels');
+const async = require('async');
+const _ = require('lodash');
+
+module.exports = _.curry((config, ctx, next) => {
+  debug('Closing %d channels', ctx.channels.length);
+
+  async.each(
+    ctx.channels,
+    (channel, cb) => {
+      channel.close(cb);
+    },
+    (err) => {
+      delete ctx.channels;
+      return next(err, config, ctx);
+    }
+  );
+});

package/lib/amqp/tasks/createChannels.js

@@ -0,0 +1,19 @@
+const debug = require('debug')('rascal:tasks:createChannel');
+const async = require('async');
+const _ = require('lodash');
+
+module.exports = _.curry((config, ctx, next) => {
+  debug('Creating %d channels', config.concurrency);
+
+  async.times(
+    config.concurrency,
+    (index, cb) => {
+      ctx.connection.createChannel(cb);
+    },
+    (err, channels) => {
+      if (err) return next(err, config, ctx);
+      ctx.channels = channels;
+      next(null, config, ctx);
+    }
+  );
+});

package/lib/amqp/tasks/deleteExchanges.js

@@ -3,10 +3,12 @@ const _ = require('lodash');
 const async = require('async');
 
 module.exports = _.curry((config, ctx, next) => {
-  async.eachSeries(
+  async.eachOfLimit(
     _.keys(config.exchanges),
-    (name, callback) => {
-      deleteExchange(ctx.channel, config.exchanges[name], callback);
+    config.concurrency,
+    (name, index, cb) => {
+      const channel = ctx.channels[index % config.concurrency];
+      deleteExchange(channel, config.exchanges[name], cb);
     },
     (err) => {
       next(err, config, ctx);

package/lib/amqp/tasks/deleteQueues.js

@@ -3,10 +3,12 @@ const _ = require('lodash');
 const async = require('async');
 
 module.exports = _.curry((config, ctx, next) => {
-  async.eachSeries(
+  async.eachOfLimit(
     _.keys(config.queues),
-    (name, callback) => {
-      deleteQueue(ctx.channel, config.queues[name], callback);
+    config.concurrency,
+    (name, index, cb) => {
+      const channel = ctx.channels[index % config.concurrency];
+      deleteQueue(channel, config.queues[name], cb);
     },
     (err) => {
       next(err, config, ctx);

package/lib/amqp/tasks/index.js

@@ -6,9 +6,9 @@ exports.bounceVhost = require('./bounceVhost');
 exports.checkExchanges = require('./checkExchanges');
 exports.checkQueues = require('./checkQueues');
 exports.checkVhost = require('./checkVhost');
-exports.closeChannel = require('./closeChannel');
+exports.closeChannels = require('./closeChannels');
 exports.closeConnection = require('./closeConnection');
-exports.createChannel = require('./createChannel');
+exports.createChannels = require('./createChannels');
 exports.createConnection = require('./createConnection');
 exports.deleteExchanges = require('./deleteExchanges');
 exports.deleteQueues = require('./deleteQueues');

package/lib/amqp/tasks/purgeQueues.js

@@ -3,10 +3,12 @@ const _ = require('lodash');
 const async = require('async');
 
 module.exports = _.curry((config, ctx, next) => {
-  async.eachSeries(
+  async.eachOfLimit(
     _.keys(config.queues),
-    (name, callback) => {
-      purgeQueue(ctx.channel, config.queues[name], ctx, callback);
+    config.concurrency,
+    (name, index, cb) => {
+      const channel = ctx.channels[index % config.concurrency];
+      purgeQueue(channel, config.queues[name], ctx, cb);
     },
     (err) => {
       next(err, config, ctx);

package/lib/config/baseline.js

@@ -1,6 +1,7 @@
 module.exports = {
   defaults: {
     vhosts: {
+      concurrency: 1,
       publicationChannelPools: {
         regularPool: {
           autostart: false,

package/lib/config/configure.js

@@ -43,6 +43,7 @@ module.exports = _.curry((rascalConfig, next) => {
       {
         name,
         namespace: rascalConfig.defaults.vhosts.namespace,
+        concurrency: rascalConfig.defaults.vhosts.concurrency,
         connectionStrategy: rascalConfig.defaults.vhosts.connectionStrategy,
         publicationChannelPools: rascalConfig.defaults.vhosts.publicationChannelPools,
       },
@@ -133,11 +134,11 @@ module.exports = _.curry((rascalConfig, next) => {
       },
       {}
     );
-
     _.each(_.defaults({}, publications, defaultPublications), configurePublication);
   }
 
   function configurePublication(publicationConfig, name) {
+    if (publicationConfig.destination) return;
     debug('Configuring publication: %s', name);
     if (rascalConfig.publications[name] && rascalConfig.publications[name].vhost !== publicationConfig.vhost) throw new Error(format('Duplicate publication: %s', name));
     rascalConfig.publications[name] = _.defaultsDeep(publicationConfig, { name }, rascalConfig.defaults.publications);
@@ -145,6 +146,13 @@ module.exports = _.curry((rascalConfig, next) => {
     const destination = Object.prototype.hasOwnProperty.call(publicationConfig, 'exchange') ? rascalConfig.vhosts[publicationConfig.vhost].exchanges[publicationConfig.exchange] : rascalConfig.vhosts[publicationConfig.vhost].queues[publicationConfig.queue];
     rascalConfig.publications[name].destination = destination.fullyQualifiedName;
 
+    if (publicationConfig.replyTo) {
+      const replyToQueue = rascalConfig.vhosts[publicationConfig.vhost].queues[publicationConfig.replyTo];
+
+      if (!replyToQueue) throw new Error(`Publication: ${name} refers to an unknown reply queue: ${publicationConfig.replyTo}`);
+      publicationConfig.replyTo = replyToQueue.fullyQualifiedName;
+    }
+
     if (publicationConfig.encryption && _.isString(publicationConfig.encryption)) {
       rascalConfig.publications[name].encryption = _.defaultsDeep({ name: publicationConfig.encryption }, rascalConfig.encryption[publicationConfig.encryption]);
     }
@@ -260,10 +268,6 @@ module.exports = _.curry((rascalConfig, next) => {
       if (bindingConfig.qualifyBindingKeys) {
         config.bindings[name].bindingKey = fqn.qualify(bindingConfig.bindingKey, config.namespace);
       }
-      if (bindingConfig.destinationType === 'queue') {
-        const queue = config.queues[bindingConfig.destination];
-        config.bindings[name].bindingKey = fqn.prefix(queue && queue.replyTo, bindingConfig.bindingKey, '.');
-      }
     });
   }
 

package/lib/config/schema.json

@@ -57,6 +57,10 @@
             "$ref": "#/definitions/connection"
           }
         },
+        "concurrency": {
+          "type": "integer",
+          "minimum": 1
+        },
         "exchanges": {
           "oneOf": [
             {

package/lib/config/validate.js

@@ -20,7 +20,7 @@ module.exports = _.curry((config, next) => {
   }
 
   function validateVhost(vhost, vhostName) {
-    validateAttributes('Vhost', vhost, vhostName, ['defaults', 'namespace', 'name', 'publicationChannelPools', 'connection', 'connections', 'connectionStrategy', 'exchanges', 'queues', 'bindings', 'check', 'assert']);
+    validateAttributes('Vhost', vhost, vhostName, ['defaults', 'namespace', 'name', 'concurrency', 'publicationChannelPools', 'connection', 'connections', 'connectionStrategy', 'exchanges', 'queues', 'bindings', 'check', 'assert']);
     validateConnectionStrategy(vhost.connectionStrategy, vhostName);
     validateConnectionAttributes(vhost.connection, vhostName, ['slashes', 'protocol', 'hostname', 'user', 'password', 'port', 'vhost', 'options', 'retry', 'auth', 'pathname', 'query', 'url', 'loggableUrl', 'management']);
     validateManagementConnectionAttributes(_.get(vhost), 'connection.management', vhostName, ['slashes', 'protocol', 'hostname', 'user', 'password', 'port', 'vhost', 'options', 'auth', 'pathname', 'query', 'url', 'loggableUrl']);
@@ -101,7 +101,7 @@ module.exports = _.curry((config, next) => {
   }
 
   function validatePublication(publication, publicationName) {
-    validateAttributes('Publication', publication, publicationName, ['name', 'vhost', 'exchange', 'queue', 'routingKey', 'confirm', 'options', 'destination', 'autoCreated', 'deprecated', 'encryption', 'timeout']);
+    validateAttributes('Publication', publication, publicationName, ['name', 'vhost', 'exchange', 'queue', 'routingKey', 'confirm', 'options', 'destination', 'autoCreated', 'deprecated', 'encryption', 'replyTo', 'timeout']);
     if (!publication.vhost) throw new Error(format('Publication: %s is missing a vhost', publicationName));
     if (!(Object.prototype.hasOwnProperty.call(publication, 'exchange') || publication.queue)) throw new Error(format('Publication: %s is missing an exchange or a queue', publicationName));
     if (Object.prototype.hasOwnProperty.call(publication, 'exchange') && publication.queue) throw new Error(format('Publication: %s has an exchange and a queue', publicationName));

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "rascal",
-  "version": "15.0.1",
+  "version": "16.1.0",
   "description": "A config driven wrapper for amqplib supporting multi-host connections, automatic error recovery, redelivery flood protection, transparent encryption / decryption,  channel pooling and publication timeouts",
   "main": "index.js",
   "dependencies": {
@@ -30,13 +30,13 @@
     "prettier": "^2.4.1",
     "random-readable": "^1.0.1",
     "superagent-defaults": "^0.1.14",
-    "zunit": "^3.2.1"
+    "zunit": "^4.0.0"
   },
   "peerDependencies": {
     "amqplib": ">=0.5.5"
   },
   "engines": {
-    "node": ">=12.0.0"
+    "node": ">=14"
   },
   "scripts": {
     "test": "zUnit",

package/lib/amqp/tasks/closeChannel.js

@@ -1,12 +0,0 @@
-const debug = require('debug')('rascal:tasks:closeChannel');
-const _ = require('lodash');
-
-module.exports = _.curry((config, ctx, next) => {
-  debug('Closing channel');
-
-  ctx.channel.close((err) => {
-    if (err) return next(err, config, ctx);
-    delete ctx.channel;
-    next(null, config, ctx);
-  });
-});

package/lib/amqp/tasks/createChannel.js

@@ -1,12 +0,0 @@
-const debug = require('debug')('rascal:tasks:createChannel');
-const _ = require('lodash');
-
-module.exports = _.curry((config, ctx, next) => {
-  debug('Creating channel');
-
-  ctx.connection.createChannel((err, channel) => {
-    if (err) return next(err, config, ctx);
-    ctx.channel = channel;
-    next(null, config, ctx);
-  });
-});