rascal 16.0.0 → 16.1.0

package/CHANGELOG.md

@@ -1,5 +1,9 @@
 # Change Log
 
+## 16.1.0
+
+- Added concurrency option for managing RabbitMQ topology. Rascal will create and use upto this number of channels when asserting/checking/deleting/purging queues, exchanges and bindings.
+
 ## 16.0.0
 
 - Automatically set replyTo message property when a publication references a replyTo queue

package/README.md

@@ -692,6 +692,7 @@ Sometimes you want to publish a message, and have the consumer of the message se
     }
   },
   "publications": {
+    "exchange": "e1",
     "replyTo": "q1"
   }
 }
@@ -1711,6 +1712,10 @@ is equivalent to...
 }
 ```
 
+### Concurrency
+
+If you have a high number of exchanges, queues and bindings you may wish to initialise Rascal using multiple channels to improve startup time. Do this per vhost by setting the `concurrency` option to the number of channels you want to create and use.
+
 ### Connect
 
 Rascal is a rich pub/sub wrapper and as such hides much of the amqplib [channel api](https://www.squaremobius.net/amqp.node/channel_api.html#channel). If you need to access this you can programmatically establish a connection to a vhost as follows.

package/lib/amqp/Vhost.js

@@ -26,10 +26,10 @@ function Vhost(vhostConfig, components) {
   let confirmChannelPool;
   const channelCreator = async.queue(createChannel, 1);
 
-  const init = async.compose(tasks.closeChannel, tasks.applyBindings, tasks.purgeQueues, tasks.checkQueues, tasks.assertQueues, tasks.checkExchanges, tasks.assertExchanges, tasks.createChannel, tasks.createConnection, tasks.checkVhost, tasks.assertVhost);
+  const init = async.compose(tasks.closeChannels, tasks.applyBindings, tasks.purgeQueues, tasks.checkQueues, tasks.assertQueues, tasks.checkExchanges, tasks.assertExchanges, tasks.createChannels, tasks.createConnection, tasks.checkVhost, tasks.assertVhost);
   const connect = async.compose(tasks.createConnection);
-  const purge = async.compose(tasks.closeConnection, tasks.closeChannel, tasks.purgeQueues, tasks.createChannel, tasks.createConnection);
-  const nuke = async.compose(tasks.closeConnection, tasks.closeChannel, tasks.deleteQueues, tasks.deleteExchanges, tasks.createChannel, tasks.createConnection);
+  const purge = async.compose(tasks.closeConnection, tasks.closeChannels, tasks.purgeQueues, tasks.createChannels, tasks.createConnection);
+  const nuke = async.compose(tasks.closeConnection, tasks.closeChannels, tasks.deleteQueues, tasks.deleteExchanges, tasks.createChannels, tasks.createConnection);
   let timer = backoff({});
   let paused = true;
   let shuttingDown = false;

package/lib/amqp/tasks/applyBindings.js

@@ -9,10 +9,12 @@ module.exports = _.curry((config, ctx, next) => {
     exchange: bindExchange,
   };
 
-  async.eachSeries(
+  async.eachOfLimit(
     _.values(config.bindings),
-    (binding, callback) => {
-      bind[binding.destinationType](config, ctx.channel, binding, callback);
+    config.concurrency,
+    (binding, index, cb) => {
+      const channel = ctx.channels[index % config.concurrency];
+      bind[binding.destinationType](config, channel, binding, cb);
     },
     (err) => {
       next(err, config, ctx);

package/lib/amqp/tasks/assertExchanges.js

@@ -3,10 +3,12 @@ const _ = require('lodash');
 const async = require('async');
 
 module.exports = _.curry((config, ctx, next) => {
-  async.eachSeries(
+  async.eachOfLimit(
     _.keys(config.exchanges),
-    (name, callback) => {
-      assertExchange(ctx.channel, config.exchanges[name], callback);
+    config.concurrency,
+    (name, index, cb) => {
+      const channel = ctx.channels[index % config.concurrency];
+      assertExchange(channel, config.exchanges[name], cb);
     },
     (err) => {
       next(err, config, ctx);
@@ -18,5 +20,9 @@ function assertExchange(channel, config, next) {
   if (!config.assert) return next();
   if (config.fullyQualifiedName === '') return next();
   debug('Asserting exchange: %s', config.fullyQualifiedName);
-  channel.assertExchange(config.fullyQualifiedName, config.type, config.options, next);
+  channel.assertExchange(config.fullyQualifiedName, config.type, config.options, (err) => {
+    if (err) return next(err);
+    debug('Asserted exchange: %s', config.fullyQualifiedName);
+    next();
+  });
 }

package/lib/amqp/tasks/assertQueues.js

@@ -3,10 +3,12 @@ const _ = require('lodash');
 const async = require('async');
 
 module.exports = _.curry((config, ctx, next) => {
-  async.eachSeries(
+  async.eachOfLimit(
     _.keys(config.queues),
-    (name, callback) => {
-      assertQueue(ctx.channel, config.queues[name], callback);
+    config.concurrency,
+    (name, index, cb) => {
+      const channel = ctx.channels[index % config.concurrency];
+      assertQueue(channel, config.queues[name], cb);
     },
     (err) => {
       next(err, config, ctx);

package/lib/amqp/tasks/checkExchanges.js

@@ -3,10 +3,12 @@ const _ = require('lodash');
 const async = require('async');
 
 module.exports = _.curry((config, ctx, next) => {
-  async.eachSeries(
+  async.eachOfLimit(
     _.keys(config.exchanges),
-    (name, callback) => {
-      checkExchange(ctx.channel, config.exchanges[name], callback);
+    config.concurrency,
+    (name, index, cb) => {
+      const channel = ctx.channels[index % config.concurrency];
+      checkExchange(channel, config.exchanges[name], cb);
     },
     (err) => {
       next(err, config, ctx);

package/lib/amqp/tasks/checkQueues.js

@@ -3,10 +3,12 @@ const _ = require('lodash');
 const async = require('async');
 
 module.exports = _.curry((config, ctx, next) => {
-  async.eachSeries(
+  async.eachOfLimit(
     _.keys(config.queues),
-    (name, callback) => {
-      checkQueue(ctx.channel, config.queues[name], callback);
+    config.concurrency,
+    (name, index, cb) => {
+      const channel = ctx.channels[index % config.concurrency];
+      checkQueue(channel, config.queues[name], cb);
     },
     (err) => {
       next(err, config, ctx);

package/lib/amqp/tasks/closeChannels.js

@@ -0,0 +1,18 @@
+const debug = require('debug')('rascal:tasks:closeChannels');
+const async = require('async');
+const _ = require('lodash');
+
+module.exports = _.curry((config, ctx, next) => {
+  debug('Closing %d channels', ctx.channels.length);
+
+  async.each(
+    ctx.channels,
+    (channel, cb) => {
+      channel.close(cb);
+    },
+    (err) => {
+      delete ctx.channels;
+      return next(err, config, ctx);
+    }
+  );
+});

package/lib/amqp/tasks/createChannels.js

@@ -0,0 +1,19 @@
+const debug = require('debug')('rascal:tasks:createChannel');
+const async = require('async');
+const _ = require('lodash');
+
+module.exports = _.curry((config, ctx, next) => {
+  debug('Creating %d channels', config.concurrency);
+
+  async.times(
+    config.concurrency,
+    (index, cb) => {
+      ctx.connection.createChannel(cb);
+    },
+    (err, channels) => {
+      if (err) return next(err, config, ctx);
+      ctx.channels = channels;
+      next(null, config, ctx);
+    }
+  );
+});

package/lib/amqp/tasks/deleteExchanges.js

@@ -3,10 +3,12 @@ const _ = require('lodash');
 const async = require('async');
 
 module.exports = _.curry((config, ctx, next) => {
-  async.eachSeries(
+  async.eachOfLimit(
     _.keys(config.exchanges),
-    (name, callback) => {
-      deleteExchange(ctx.channel, config.exchanges[name], callback);
+    config.concurrency,
+    (name, index, cb) => {
+      const channel = ctx.channels[index % config.concurrency];
+      deleteExchange(channel, config.exchanges[name], cb);
     },
     (err) => {
       next(err, config, ctx);

package/lib/amqp/tasks/deleteQueues.js

@@ -3,10 +3,12 @@ const _ = require('lodash');
 const async = require('async');
 
 module.exports = _.curry((config, ctx, next) => {
-  async.eachSeries(
+  async.eachOfLimit(
     _.keys(config.queues),
-    (name, callback) => {
-      deleteQueue(ctx.channel, config.queues[name], callback);
+    config.concurrency,
+    (name, index, cb) => {
+      const channel = ctx.channels[index % config.concurrency];
+      deleteQueue(channel, config.queues[name], cb);
     },
     (err) => {
       next(err, config, ctx);

package/lib/amqp/tasks/index.js

@@ -6,9 +6,9 @@ exports.bounceVhost = require('./bounceVhost');
 exports.checkExchanges = require('./checkExchanges');
 exports.checkQueues = require('./checkQueues');
 exports.checkVhost = require('./checkVhost');
-exports.closeChannel = require('./closeChannel');
+exports.closeChannels = require('./closeChannels');
 exports.closeConnection = require('./closeConnection');
-exports.createChannel = require('./createChannel');
+exports.createChannels = require('./createChannels');
 exports.createConnection = require('./createConnection');
 exports.deleteExchanges = require('./deleteExchanges');
 exports.deleteQueues = require('./deleteQueues');

package/lib/amqp/tasks/purgeQueues.js

@@ -3,10 +3,12 @@ const _ = require('lodash');
 const async = require('async');
 
 module.exports = _.curry((config, ctx, next) => {
-  async.eachSeries(
+  async.eachOfLimit(
     _.keys(config.queues),
-    (name, callback) => {
-      purgeQueue(ctx.channel, config.queues[name], ctx, callback);
+    config.concurrency,
+    (name, index, cb) => {
+      const channel = ctx.channels[index % config.concurrency];
+      purgeQueue(channel, config.queues[name], ctx, cb);
     },
     (err) => {
       next(err, config, ctx);

package/lib/config/baseline.js

@@ -1,6 +1,7 @@
 module.exports = {
   defaults: {
     vhosts: {
+      concurrency: 1,
       publicationChannelPools: {
         regularPool: {
           autostart: false,

package/lib/config/configure.js

@@ -43,6 +43,7 @@ module.exports = _.curry((rascalConfig, next) => {
       {
         name,
         namespace: rascalConfig.defaults.vhosts.namespace,
+        concurrency: rascalConfig.defaults.vhosts.concurrency,
         connectionStrategy: rascalConfig.defaults.vhosts.connectionStrategy,
         publicationChannelPools: rascalConfig.defaults.vhosts.publicationChannelPools,
       },

package/lib/config/schema.json

@@ -57,6 +57,10 @@
             "$ref": "#/definitions/connection"
           }
         },
+        "concurrency": {
+          "type": "integer",
+          "minimum": 1
+        },
         "exchanges": {
           "oneOf": [
             {

package/lib/config/validate.js

@@ -20,7 +20,7 @@ module.exports = _.curry((config, next) => {
   }
 
   function validateVhost(vhost, vhostName) {
-    validateAttributes('Vhost', vhost, vhostName, ['defaults', 'namespace', 'name', 'publicationChannelPools', 'connection', 'connections', 'connectionStrategy', 'exchanges', 'queues', 'bindings', 'check', 'assert']);
+    validateAttributes('Vhost', vhost, vhostName, ['defaults', 'namespace', 'name', 'concurrency', 'publicationChannelPools', 'connection', 'connections', 'connectionStrategy', 'exchanges', 'queues', 'bindings', 'check', 'assert']);
     validateConnectionStrategy(vhost.connectionStrategy, vhostName);
     validateConnectionAttributes(vhost.connection, vhostName, ['slashes', 'protocol', 'hostname', 'user', 'password', 'port', 'vhost', 'options', 'retry', 'auth', 'pathname', 'query', 'url', 'loggableUrl', 'management']);
     validateManagementConnectionAttributes(_.get(vhost), 'connection.management', vhostName, ['slashes', 'protocol', 'hostname', 'user', 'password', 'port', 'vhost', 'options', 'auth', 'pathname', 'query', 'url', 'loggableUrl']);

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "rascal",
-  "version": "16.0.0",
+  "version": "16.1.0",
   "description": "A config driven wrapper for amqplib supporting multi-host connections, automatic error recovery, redelivery flood protection, transparent encryption / decryption,  channel pooling and publication timeouts",
   "main": "index.js",
   "dependencies": {

package/lib/amqp/tasks/closeChannel.js

@@ -1,12 +0,0 @@
-const debug = require('debug')('rascal:tasks:closeChannel');
-const _ = require('lodash');
-
-module.exports = _.curry((config, ctx, next) => {
-  debug('Closing channel');
-
-  ctx.channel.close((err) => {
-    if (err) return next(err, config, ctx);
-    delete ctx.channel;
-    next(null, config, ctx);
-  });
-});

package/lib/amqp/tasks/createChannel.js

@@ -1,12 +0,0 @@
-const debug = require('debug')('rascal:tasks:createChannel');
-const _ = require('lodash');
-
-module.exports = _.curry((config, ctx, next) => {
-  debug('Creating channel');
-
-  ctx.connection.createChannel((err, channel) => {
-    if (err) return next(err, config, ctx);
-    ctx.channel = channel;
-    next(null, config, ctx);
-  });
-});