qdone 2.0.7-alpha → 2.0.9-alpha

package/commonjs/src/consumer.js

@@ -83,19 +83,20 @@ var DoNotProcess = /** @class */ (function (_super) {
     return DoNotProcess;
 }(Error));
 exports.DoNotProcess = DoNotProcess;
-var debug = (0, debug_1.default)('qdone:worker');
+var debug = (0, debug_1.default)('qdone:consumer');
 // Global flag for shutdown request
 var shutdownRequested = false;
 var shutdownCallbacks = [];
 function requestShutdown() {
+    debug('requestShutdown');
     shutdownRequested = true;
     for (var _i = 0, shutdownCallbacks_1 = shutdownCallbacks; _i < shutdownCallbacks_1.length; _i++) {
         var callback = shutdownCallbacks_1[_i];
-        try {
-            callback();
-        }
-        catch (e) { }
+        debug('callback', callback);
+        callback();
+        // try { callback() } catch (e) { }
     }
+    debug('requestShutdown done');
 }
 exports.requestShutdown = requestShutdown;
 function processMessage(message, callback, qname, qrl, opt) {
@@ -167,7 +168,7 @@ function processMessage(message, callback, qname, qrl, opt) {
             switch (_a.label) {
                 case 0:
                     debug('processMessage', message, qname, qrl);
-                    payload = JSON.parse(message.Body);
+                    payload = opt.json ? JSON.parse(message.Body) : message.Body;
                     if (opt.verbose) {
                         console.error(chalk_1.default.blue('  Processing payload:'), payload);
                     }
@@ -366,16 +367,19 @@ function processMessages(queues, callback, options) {
         // Callback to help facilitate better UX at shutdown
         function shutdownCallback() {
             if (opt.verbose) {
-                debug({ activeLoops: activeLoops });
-                var activeQueues = Object.keys(activeLoops).filter(function (q) { return activeLoops[q]; }).map(function (q) { return q.slice(opt.prefix.length); });
-                if (activeQueues.length) {
-                    console.error(chalk_1.default.blue('Waiting for work to finish on the following queues: ') + activeQueues.join(chalk_1.default.blue(', ')));
+                debug({ activeLoops: activeLoops, completedLoops: completedLoops });
+                if (activeLoops.length) {
+                    console.error(chalk_1.default.blue('Waiting for work to finish on the following queues: '));
+                    for (var _i = 0, activeLoops_2 = activeLoops; _i < activeLoops_2.length; _i++) {
+                        var _a = activeLoops_2[_i], id = _a[0], _b = _a[1], qname = _b.qname, qrl = _b.qrl, promise = _b.promise;
+                        console.error('    ' + qname + "job ".concat(id));
+                    }
                 }
-                clearTimeout(delayTimeout);
+                // clearTimeout(delayTimeout)
             }
         }
         // Listen to a queue until it is out of messages
-        function listenLoop(qname, qrl) {
+        function listenLoop(qname, qrl, loopId) {
             return __awaiter(this, void 0, void 0, function () {
                 var _a, noJobs, jobsSucceeded, jobsFailed, err_3;
                 return __generator(this, function (_b) {
@@ -392,11 +396,14 @@ function processMessages(queues, callback, options) {
                             return [4 /*yield*/, pollSingleQueue(qname, qrl, callback, opt)];
                         case 1:
                             _a = _b.sent(), noJobs = _a.noJobs, jobsSucceeded = _a.jobsSucceeded, jobsFailed = _a.jobsFailed;
+                            debug('pollSingleQueue return');
                             stats.noJobs += noJobs;
                             stats.jobsFailed += jobsFailed;
                             stats.jobsSucceeded += jobsSucceeded;
-                            // No work? return to outer loop
-                            if (noJobs)
+                            debug({ stats: stats, noJobs: noJobs });
+                            // No work? Shutdown requested? Return to outer loop
+                            debug({ noJobs: noJobs, shutdownRequested: shutdownRequested });
+                            if (noJobs || shutdownRequested)
                                 return [2 /*return*/];
                             // Otherwise keep going
                             return [2 /*return*/, listenLoop(qname, qrl)];
@@ -407,34 +414,38 @@ function processMessages(queues, callback, options) {
                             console.error(chalk_1.default.blue('  error : ') + err_3);
                             return [3 /*break*/, 4];
                         case 3:
-                            delete activeLoops[qname];
+                            completedLoops.set(loopId, activeLoops.get(loopId));
                             return [7 /*endfinally*/];
                         case 4: return [2 /*return*/];
                     }
                 });
             });
         }
-        var opt, stats, activeLoops, delayTimeout, delay, start, selectedPairs, _i, selectedPairs_1, _a, qname, qrl, msSoFar, msUntilNextResolve;
-        return __generator(this, function (_b) {
-            switch (_b.label) {
+        var opt, loopCounter, stats, maxActiveLoops, activeLoops, completedLoops, delayTimeout, delay, start, selectedPairs, _i, selectedPairs_1, _a, qname, qrl, loopId, msSoFar, msUntilNextResolve, _b, completedLoops_1, _c, id, _d, qname, qrl, promise, _e, activeLoops_1, _f, id, _g, qname, qrl, promise;
+        return __generator(this, function (_h) {
+            switch (_h.label) {
                 case 0:
                     opt = (0, defaults_js_1.getOptionsWithDefaults)(options);
                     debug('processMessages', { queues: queues, callback: callback, options: options, opt: opt });
+                    loopCounter = 0;
                     stats = { noJobs: 0, jobsSucceeded: 0, jobsFailed: 0 };
-                    activeLoops = {};
+                    maxActiveLoops = 10;
+                    activeLoops = new Map();
+                    completedLoops = new Map();
                     delay = function (ms) { return new Promise(function (resolve) {
                         delayTimeout = setTimeout(resolve, ms);
                     }); };
                     shutdownCallbacks.push(shutdownCallback);
-                    _b.label = 1;
+                    _h.label = 1;
                 case 1:
-                    if (!!shutdownRequested) return [3 /*break*/, 5];
+                    if (!!shutdownRequested) return [3 /*break*/, 9];
                     start = new Date();
                     return [4 /*yield*/, resolveQueues(queues, opt)];
                 case 2:
-                    selectedPairs = _b.sent();
+                    selectedPairs = _h.sent();
+                    debug({ selectedPairs: selectedPairs });
                     if (shutdownRequested)
-                        return [3 /*break*/, 5];
+                        return [3 /*break*/, 9];
                     // But only if we have queues to listen on
                     if (selectedPairs.length) {
                         // Randomize order
@@ -450,30 +461,60 @@ function processMessages(queues, callback, options) {
                         // Launch listen loop for each queue
                         for (_i = 0, selectedPairs_1 = selectedPairs; _i < selectedPairs_1.length; _i++) {
                             _a = selectedPairs_1[_i], qname = _a.qname, qrl = _a.qrl;
-                            if (!activeLoops[qname])
-                                activeLoops[qname] = listenLoop(qname, qrl);
+                            // Bail if we already have too many
+                            if (activeLoops.size >= maxActiveLoops) {
+                                if (opt.verbose)
+                                    console.error(chalk_1.default.yellow('Hit active worker limit of ') + maxActiveLoops);
+                                break;
+                            }
+                            loopId = loopCounter++;
+                            activeLoops.set(loopId, { qname: qname, qrl: qrl, promise: listenLoop(qname, qrl, loopId) });
                         }
                     }
                     if (!!shutdownRequested) return [3 /*break*/, 4];
                     msSoFar = Math.max(0, new Date() - start);
-                    msUntilNextResolve = Math.max(0, opt.waitTime * 1000 - msSoFar);
+                    msUntilNextResolve = Math.max(0, /*opt.waitTime **/ 1000 - msSoFar);
                     debug({ msSoFar: msSoFar, msUntilNextResolve: msUntilNextResolve });
                     if (!msUntilNextResolve) return [3 /*break*/, 4];
                     if (opt.verbose)
                         console.error(chalk_1.default.blue('Will resolve queues again in ' + Math.round(msUntilNextResolve / 1000) + ' seconds'));
                     return [4 /*yield*/, delay(msUntilNextResolve)];
                 case 3:
-                    _b.sent();
-                    _b.label = 4;
-                case 4: return [3 /*break*/, 1];
-                case 5: 
-                // Wait on all work to finish
-                // shutdownCallback()
-                return [4 /*yield*/, Promise.all(Object.values(activeLoops))];
+                    _h.sent();
+                    _h.label = 4;
+                case 4:
+                    _b = 0, completedLoops_1 = completedLoops;
+                    _h.label = 5;
+                case 5:
+                    if (!(_b < completedLoops_1.length)) return [3 /*break*/, 8];
+                    _c = completedLoops_1[_b], id = _c[0], _d = _c[1], qname = _d.qname, qrl = _d.qrl, promise = _d.promise;
+                    return [4 /*yield*/, promise]; // make sure the promise resolves
                 case 6:
-                    // Wait on all work to finish
-                    // shutdownCallback()
-                    _b.sent();
+                    _h.sent(); // make sure the promise resolves
+                    debug('Cleaning up', { id: id, qname: qname, qrl: qrl, promise: promise });
+                    activeLoops.delete(id);
+                    _h.label = 7;
+                case 7:
+                    _b++;
+                    return [3 /*break*/, 5];
+                case 8: return [3 /*break*/, 1];
+                case 9:
+                    debug('out here', { activeLoops: activeLoops });
+                    _e = 0, activeLoops_1 = activeLoops;
+                    _h.label = 10;
+                case 10:
+                    if (!(_e < activeLoops_1.length)) return [3 /*break*/, 13];
+                    _f = activeLoops_1[_e], id = _f[0], _g = _f[1], qname = _g.qname, qrl = _g.qrl, promise = _g.promise;
+                    debug('Waiting on active loop', id);
+                    return [4 /*yield*/, promise]; // make sure the promise resolves
+                case 11:
+                    _h.sent(); // make sure the promise resolves
+                    _h.label = 12;
+                case 12:
+                    _e++;
+                    return [3 /*break*/, 10];
+                case 13:
+                    debug('after all');
                     return [2 /*return*/];
             }
         });

package/commonjs/src/enqueue.js

@@ -328,7 +328,7 @@ function sendMessageBatch(qrl, messages, opt) {
             if (opt.fifo) {
                 params.Entries = params.Entries.map(function (message) { return Object.assign({
                     MessageGroupId: opt.groupIdPerMessage ? uuidFunction() : opt.groupId,
-                    MessageDeduplicationId: uuidFunction()
+                    MessageDeduplicationId: opt.deduplicationId || uuidFunction()
                 }, message); });
             }
             if (opt.delay) {
@@ -400,14 +400,13 @@ function sendMessageBatch(qrl, messages, opt) {
     });
 }
 exports.sendMessageBatch = sendMessageBatch;
-var messages = {};
 var requestCount = 0;
 //
 // Flushes the internal message buffer for qrl.
 // If the message is too large, batch is retried with half the messages.
 // Returns number of messages flushed.
 //
-function flushMessages(qrl, opt) {
+function flushMessages(qrl, opt, sendBuffer) {
     return __awaiter(this, void 0, void 0, function () {
         function whileNotEmpty() {
             return __awaiter(this, void 0, void 0, function () {
@@ -415,18 +414,18 @@ function flushMessages(qrl, opt) {
                 return __generator(this, function (_a) {
                     switch (_a.label) {
                         case 0:
-                            if (!(messages[qrl] && messages[qrl].length))
+                            if (!(sendBuffer[qrl] && sendBuffer[qrl].length))
                                 return [2 /*return*/, numFlushed
                                     // Construct batch until full
                                 ];
                             batch = [];
-                            nextSize = JSON.stringify(messages[qrl][0]).length;
+                            nextSize = JSON.stringify(sendBuffer[qrl][0]).length;
                             totalSize = 0;
-                            while ((totalSize + nextSize) < 262144 && messages[qrl].length && batch.length < 10) {
-                                batch.push(messages[qrl].shift());
+                            while ((totalSize + nextSize) < 262144 && sendBuffer[qrl].length && batch.length < 10) {
+                                batch.push(sendBuffer[qrl].shift());
                                 totalSize += nextSize;
-                                if (messages[qrl].length)
-                                    nextSize = JSON.stringify(messages[qrl][0]).length;
+                                if (sendBuffer[qrl].length)
+                                    nextSize = JSON.stringify(sendBuffer[qrl][0]).length;
                                 else
                                     nextSize = 0;
                             }
@@ -468,17 +467,16 @@ exports.flushMessages = flushMessages;
 // Automaticaly flushes if queue has >= 10 messages.
 // Returns number of messages flushed.
 //
-var messageIndex = 0;
-function addMessage(qrl, command, opt) {
+function addMessage(qrl, command, messageIndex, opt, sendBuffer) {
     return __awaiter(this, void 0, void 0, function () {
         var message;
         return __generator(this, function (_a) {
-            message = formatMessage(command, messageIndex++);
-            messages[qrl] = messages[qrl] || [];
-            messages[qrl].push(message);
-            debug({ location: 'addMessage', messages: messages });
-            if (messages[qrl].length >= 10) {
-                return [2 /*return*/, flushMessages(qrl, opt)];
+            message = formatMessage(command, messageIndex);
+            sendBuffer[qrl] = sendBuffer[qrl] || [];
+            sendBuffer[qrl].push(message);
+            debug({ location: 'addMessage', sendBuffer: sendBuffer });
+            if (sendBuffer[qrl].length >= 10) {
+                return [2 /*return*/, flushMessages(qrl, opt, sendBuffer)];
             }
             return [2 /*return*/, 0];
         });
@@ -512,7 +510,7 @@ exports.enqueue = enqueue;
 //
 function enqueueBatch(pairs, options) {
     return __awaiter(this, void 0, void 0, function () {
-        var opt, normalizedPairs, uniqueQnames, createPromises, _i, uniqueQnames_1, qname, initialFlushTotal, _a, normalizedPairs_1, _b, qname, command, qrl, _c, extraFlushPromises, qrl, extraFlushCounts, extraFlushTotal, totalFlushed;
+        var opt, normalizedPairs, uniqueQnames, createPromises, _i, uniqueQnames_1, qname, sendBuffer, messageIndex, initialFlushTotal, _a, normalizedPairs_1, _b, qname, command, qrl, _c, extraFlushPromises, qrl, extraFlushCounts, extraFlushTotal, totalFlushed;
         return __generator(this, function (_d) {
             switch (_d.label) {
                 case 0:
@@ -540,6 +538,8 @@ function enqueueBatch(pairs, options) {
                     // After we've prefetched, all qrls are in cache
                     // so go back through the list of pairs and fire off messages
                     requestCount = 0;
+                    sendBuffer = {};
+                    messageIndex = 0;
                     initialFlushTotal = 0;
                     _a = 0, normalizedPairs_1 = normalizedPairs;
                     _d.label = 2;
@@ -550,7 +550,7 @@ function enqueueBatch(pairs, options) {
                 case 3:
                     qrl = _d.sent();
                     _c = initialFlushTotal;
-                    return [4 /*yield*/, addMessage(qrl, command, opt)];
+                    return [4 /*yield*/, addMessage(qrl, command, messageIndex++, opt, sendBuffer)];
                 case 4:
                     initialFlushTotal = _c + _d.sent();
                     _d.label = 5;
@@ -559,8 +559,8 @@ function enqueueBatch(pairs, options) {
                     return [3 /*break*/, 2];
                 case 6:
                     extraFlushPromises = [];
-                    for (qrl in messages) {
-                        extraFlushPromises.push(flushMessages(qrl, opt));
+                    for (qrl in sendBuffer) {
+                        extraFlushPromises.push(flushMessages(qrl, opt, sendBuffer));
                     }
                     return [4 /*yield*/, Promise.all(extraFlushPromises)];
                 case 7:

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "qdone",
-  "version": "2.0.7-alpha",
+  "version": "2.0.9-alpha",
   "description": "Language agnostic job queue for SQS",
   "type": "module",
   "main": "./index.js",

package/src/consumer.js

@@ -27,22 +27,26 @@ import { getSQSClient } from './sqs.js'
 //
 export class DoNotProcess extends Error {}
 
-const debug = Debug('qdone:worker')
+const debug = Debug('qdone:consumer')
 
 // Global flag for shutdown request
 let shutdownRequested = false
 const shutdownCallbacks = []
 
 export function requestShutdown () {
+  debug('requestShutdown')
   shutdownRequested = true
   for (const callback of shutdownCallbacks) {
-    try { callback() } catch (e) { }
+    debug('callback', callback)
+    callback()
+    // try { callback() } catch (e) { }
   }
+  debug('requestShutdown done')
 }
 
 export async function processMessage (message, callback, qname, qrl, opt) {
   debug('processMessage', message, qname, qrl)
-  const payload = JSON.parse(message.Body)
+  const payload = opt.json ? JSON.parse(message.Body) : message.Body
   if (opt.verbose) {
     console.error(chalk.blue('  Processing payload:'), payload)
   } else if (!opt.disableLog) {
@@ -248,8 +252,11 @@ export async function processMessages (queues, callback, options) {
   const opt = getOptionsWithDefaults(options)
   debug('processMessages', { queues, callback, options, opt })
 
+  let loopCounter = 0
   const stats = { noJobs: 0, jobsSucceeded: 0, jobsFailed: 0 }
-  const activeLoops = {}
+  const maxActiveLoops = 10
+  const activeLoops = new Map()
+  const completedLoops = new Map()
 
   // This delay function keeps a timeout reference around so it can be
   // cancelled at shutdown
@@ -261,18 +268,20 @@ export async function processMessages (queues, callback, options) {
   // Callback to help facilitate better UX at shutdown
   function shutdownCallback () {
     if (opt.verbose) {
-      debug({ activeLoops })
-      const activeQueues = Object.keys(activeLoops).filter(q => activeLoops[q]).map(q => q.slice(opt.prefix.length))
-      if (activeQueues.length) {
-        console.error(chalk.blue('Waiting for work to finish on the following queues: ') + activeQueues.join(chalk.blue(', ')))
+      debug({ activeLoops, completedLoops })
+      if (activeLoops.length) {
+        console.error(chalk.blue('Waiting for work to finish on the following queues: '))
+        for (const [id, { qname, qrl, promise }] of activeLoops) {
+          console.error('    ' + qname + `job ${id}`)
+        }
       }
-      clearTimeout(delayTimeout)
+      // clearTimeout(delayTimeout)
     }
   }
   shutdownCallbacks.push(shutdownCallback)
 
   // Listen to a queue until it is out of messages
-  async function listenLoop (qname, qrl) {
+  async function listenLoop (qname, qrl, loopId) {
     try {
       if (shutdownRequested) return
       if (opt.verbose) {
@@ -284,12 +293,15 @@ export async function processMessages (queues, callback, options) {
       }
       // Aggregate the results
       const { noJobs, jobsSucceeded, jobsFailed } = await pollSingleQueue(qname, qrl, callback, opt)
+      debug('pollSingleQueue return')
       stats.noJobs += noJobs
       stats.jobsFailed += jobsFailed
       stats.jobsSucceeded += jobsSucceeded
+      debug({ stats, noJobs })
 
-      // No work? return to outer loop
-      if (noJobs) return
+      // No work? Shutdown requested? Return to outer loop
+      debug({ noJobs, shutdownRequested })
+      if (noJobs || shutdownRequested) return
 
       // Otherwise keep going
       return listenLoop(qname, qrl)
@@ -298,7 +310,7 @@ export async function processMessages (queues, callback, options) {
       console.error(chalk.red('  ERROR in listenLoop'))
       console.error(chalk.blue('  error : ') + err)
     } finally {
-      delete activeLoops[qname]
+      completedLoops.set(loopId, activeLoops.get(loopId))
     }
   }
 
@@ -306,6 +318,7 @@ export async function processMessages (queues, callback, options) {
   while (!shutdownRequested) { // eslint-disable-line
     const start = new Date()
     const selectedPairs = await resolveQueues(queues, opt)
+    debug({ selectedPairs })
     if (shutdownRequested) break
 
     // But only if we have queues to listen on
@@ -323,24 +336,43 @@ export async function processMessages (queues, callback, options) {
 
       // Launch listen loop for each queue
       for (const { qname, qrl } of selectedPairs) {
-        if (!activeLoops[qname]) activeLoops[qname] = listenLoop(qname, qrl)
+        // Bail if we already have too many
+        if (activeLoops.size >= maxActiveLoops) {
+          if (opt.verbose) console.error(chalk.yellow('Hit active worker limit of ') + maxActiveLoops)
+          break
+        }
+        const loopId = loopCounter++
+        activeLoops.set(loopId, { qname, qrl, promise: listenLoop(qname, qrl, loopId) })
       }
     }
+
     // Wait until the next time we need to resolve
     if (!shutdownRequested) {
       const msSoFar = Math.max(0, new Date() - start)
-      const msUntilNextResolve = Math.max(0, opt.waitTime * 1000 - msSoFar)
+      const msUntilNextResolve = Math.max(0, /*opt.waitTime **/ 1000 - msSoFar)
       debug({ msSoFar, msUntilNextResolve })
       if (msUntilNextResolve) {
         if (opt.verbose) console.error(chalk.blue('Will resolve queues again in ' + Math.round(msUntilNextResolve / 1000) + ' seconds'))
         await delay(msUntilNextResolve)
       }
     }
+
+    // Cleanup completed loops
+    for (const [id, { qname, qrl, promise }] of completedLoops) {
+      await promise // make sure the promise resolves
+      debug('Cleaning up', { id, qname, qrl, promise })
+      activeLoops.delete(id)
+    }
   }
+  debug('out here', { activeLoops })
 
   // Wait on all work to finish
   // shutdownCallback()
-  await Promise.all(Object.values(activeLoops))
+  for (const [id, { qname, qrl, promise }] of activeLoops) {
+    debug('Waiting on active loop', id)
+    await promise // make sure the promise resolves
+  }
+  debug('after all')
 }
 
 debug('loaded')

package/src/enqueue.js

@@ -209,7 +209,7 @@ export async function sendMessageBatch (qrl, messages, opt) {
     params.Entries = params.Entries.map(
       message => Object.assign({
         MessageGroupId: opt.groupIdPerMessage ? uuidFunction() : opt.groupId,
-        MessageDeduplicationId: uuidFunction()
+        MessageDeduplicationId: opt.deduplicationId || uuidFunction()
       }, message)
     )
   }
@@ -267,7 +267,6 @@ export async function sendMessageBatch (qrl, messages, opt) {
   return backoff.run(send, shouldRetry)
 }
 
-const messages = {}
 let requestCount = 0
 
 //
@@ -275,20 +274,20 @@ let requestCount = 0
 // If the message is too large, batch is retried with half the messages.
 // Returns number of messages flushed.
 //
-export async function flushMessages (qrl, opt) {
+export async function flushMessages (qrl, opt, sendBuffer) {
   debug('flushMessages', qrl)
   // Flush until empty
   let numFlushed = 0
   async function whileNotEmpty () {
-    if (!(messages[qrl] && messages[qrl].length)) return numFlushed
+    if (!(sendBuffer[qrl] && sendBuffer[qrl].length)) return numFlushed
     // Construct batch until full
     const batch = []
-    let nextSize = JSON.stringify(messages[qrl][0]).length
+    let nextSize = JSON.stringify(sendBuffer[qrl][0]).length
     let totalSize = 0
-    while ((totalSize + nextSize) < 262144 && messages[qrl].length && batch.length < 10) {
-      batch.push(messages[qrl].shift())
+    while ((totalSize + nextSize) < 262144 && sendBuffer[qrl].length && batch.length < 10) {
+      batch.push(sendBuffer[qrl].shift())
       totalSize += nextSize
-      if (messages[qrl].length) nextSize = JSON.stringify(messages[qrl][0]).length
+      if (sendBuffer[qrl].length) nextSize = JSON.stringify(sendBuffer[qrl][0]).length
       else nextSize = 0
     }
 
@@ -321,14 +320,13 @@ export async function flushMessages (qrl, opt) {
 // Automaticaly flushes if queue has >= 10 messages.
 // Returns number of messages flushed.
 //
-let messageIndex = 0
-export async function addMessage (qrl, command, opt) {
-  const message = formatMessage(command, messageIndex++)
-  messages[qrl] = messages[qrl] || []
-  messages[qrl].push(message)
-  debug({ location: 'addMessage', messages })
-  if (messages[qrl].length >= 10) {
-    return flushMessages(qrl, opt)
+export async function addMessage (qrl, command, messageIndex, opt, sendBuffer) {
+  const message = formatMessage(command, messageIndex)
+  sendBuffer[qrl] = sendBuffer[qrl] || []
+  sendBuffer[qrl].push(message)
+  debug({ location: 'addMessage', sendBuffer })
+  if (sendBuffer[qrl].length >= 10) {
+    return flushMessages(qrl, opt, sendBuffer)
   }
   return 0
 }
@@ -370,16 +368,18 @@ export async function enqueueBatch (pairs, options) {
   // After we've prefetched, all qrls are in cache
   // so go back through the list of pairs and fire off messages
   requestCount = 0
+  const sendBuffer = {}
+  let messageIndex = 0
   let initialFlushTotal = 0
   for (const { qname, command } of normalizedPairs) {
     const qrl = await getOrCreateQueue(qname, opt)
-    initialFlushTotal += await addMessage(qrl, command, opt)
+    initialFlushTotal += await addMessage(qrl, command, messageIndex++, opt, sendBuffer)
   }
 
   // And flush any remaining messages
   const extraFlushPromises = []
-  for (const qrl in messages) {
-    extraFlushPromises.push(flushMessages(qrl, opt))
+  for (const qrl in sendBuffer) {
+    extraFlushPromises.push(flushMessages(qrl, opt, sendBuffer))
   }
   const extraFlushCounts = await Promise.all(extraFlushPromises)
   const extraFlushTotal = extraFlushCounts.reduce((a, b) => a + b, 0)