qdone 2.0.8-alpha → 2.0.9-alpha

package/commonjs/src/consumer.js

@@ -83,19 +83,20 @@ var DoNotProcess = /** @class */ (function (_super) {
     return DoNotProcess;
 }(Error));
 exports.DoNotProcess = DoNotProcess;
-var debug = (0, debug_1.default)('qdone:worker');
+var debug = (0, debug_1.default)('qdone:consumer');
 // Global flag for shutdown request
 var shutdownRequested = false;
 var shutdownCallbacks = [];
 function requestShutdown() {
+    debug('requestShutdown');
     shutdownRequested = true;
     for (var _i = 0, shutdownCallbacks_1 = shutdownCallbacks; _i < shutdownCallbacks_1.length; _i++) {
         var callback = shutdownCallbacks_1[_i];
-        try {
-            callback();
-        }
-        catch (e) { }
+        debug('callback', callback);
+        callback();
+        // try { callback() } catch (e) { }
     }
+    debug('requestShutdown done');
 }
 exports.requestShutdown = requestShutdown;
 function processMessage(message, callback, qname, qrl, opt) {
@@ -167,7 +168,7 @@ function processMessage(message, callback, qname, qrl, opt) {
             switch (_a.label) {
                 case 0:
                     debug('processMessage', message, qname, qrl);
-                    payload = JSON.parse(message.Body);
+                    payload = opt.json ? JSON.parse(message.Body) : message.Body;
                     if (opt.verbose) {
                         console.error(chalk_1.default.blue('  Processing payload:'), payload);
                     }
@@ -366,16 +367,19 @@ function processMessages(queues, callback, options) {
         // Callback to help facilitate better UX at shutdown
         function shutdownCallback() {
             if (opt.verbose) {
-                debug({ activeLoops: activeLoops });
-                var activeQueues = Object.keys(activeLoops).filter(function (q) { return activeLoops[q]; }).map(function (q) { return q.slice(opt.prefix.length); });
-                if (activeQueues.length) {
-                    console.error(chalk_1.default.blue('Waiting for work to finish on the following queues: ') + activeQueues.join(chalk_1.default.blue(', ')));
+                debug({ activeLoops: activeLoops, completedLoops: completedLoops });
+                if (activeLoops.length) {
+                    console.error(chalk_1.default.blue('Waiting for work to finish on the following queues: '));
+                    for (var _i = 0, activeLoops_2 = activeLoops; _i < activeLoops_2.length; _i++) {
+                        var _a = activeLoops_2[_i], id = _a[0], _b = _a[1], qname = _b.qname, qrl = _b.qrl, promise = _b.promise;
+                        console.error('    ' + qname + "job ".concat(id));
+                    }
                 }
-                clearTimeout(delayTimeout);
+                // clearTimeout(delayTimeout)
             }
         }
         // Listen to a queue until it is out of messages
-        function listenLoop(qname, qrl) {
+        function listenLoop(qname, qrl, loopId) {
             return __awaiter(this, void 0, void 0, function () {
                 var _a, noJobs, jobsSucceeded, jobsFailed, err_3;
                 return __generator(this, function (_b) {
@@ -392,11 +396,14 @@ function processMessages(queues, callback, options) {
                             return [4 /*yield*/, pollSingleQueue(qname, qrl, callback, opt)];
                         case 1:
                             _a = _b.sent(), noJobs = _a.noJobs, jobsSucceeded = _a.jobsSucceeded, jobsFailed = _a.jobsFailed;
+                            debug('pollSingleQueue return');
                             stats.noJobs += noJobs;
                             stats.jobsFailed += jobsFailed;
                             stats.jobsSucceeded += jobsSucceeded;
-                            // No work? return to outer loop
-                            if (noJobs)
+                            debug({ stats: stats, noJobs: noJobs });
+                            // No work? Shutdown requested? Return to outer loop
+                            debug({ noJobs: noJobs, shutdownRequested: shutdownRequested });
+                            if (noJobs || shutdownRequested)
                                 return [2 /*return*/];
                             // Otherwise keep going
                             return [2 /*return*/, listenLoop(qname, qrl)];
@@ -407,34 +414,38 @@ function processMessages(queues, callback, options) {
                             console.error(chalk_1.default.blue('  error : ') + err_3);
                             return [3 /*break*/, 4];
                         case 3:
-                            delete activeLoops[qname];
+                            completedLoops.set(loopId, activeLoops.get(loopId));
                             return [7 /*endfinally*/];
                         case 4: return [2 /*return*/];
                     }
                 });
             });
         }
-        var opt, stats, activeLoops, delayTimeout, delay, start, selectedPairs, _i, selectedPairs_1, _a, qname, qrl, msSoFar, msUntilNextResolve;
-        return __generator(this, function (_b) {
-            switch (_b.label) {
+        var opt, loopCounter, stats, maxActiveLoops, activeLoops, completedLoops, delayTimeout, delay, start, selectedPairs, _i, selectedPairs_1, _a, qname, qrl, loopId, msSoFar, msUntilNextResolve, _b, completedLoops_1, _c, id, _d, qname, qrl, promise, _e, activeLoops_1, _f, id, _g, qname, qrl, promise;
+        return __generator(this, function (_h) {
+            switch (_h.label) {
                 case 0:
                     opt = (0, defaults_js_1.getOptionsWithDefaults)(options);
                     debug('processMessages', { queues: queues, callback: callback, options: options, opt: opt });
+                    loopCounter = 0;
                     stats = { noJobs: 0, jobsSucceeded: 0, jobsFailed: 0 };
-                    activeLoops = {};
+                    maxActiveLoops = 10;
+                    activeLoops = new Map();
+                    completedLoops = new Map();
                     delay = function (ms) { return new Promise(function (resolve) {
                         delayTimeout = setTimeout(resolve, ms);
                     }); };
                     shutdownCallbacks.push(shutdownCallback);
-                    _b.label = 1;
+                    _h.label = 1;
                 case 1:
-                    if (!!shutdownRequested) return [3 /*break*/, 5];
+                    if (!!shutdownRequested) return [3 /*break*/, 9];
                     start = new Date();
                     return [4 /*yield*/, resolveQueues(queues, opt)];
                 case 2:
-                    selectedPairs = _b.sent();
+                    selectedPairs = _h.sent();
+                    debug({ selectedPairs: selectedPairs });
                     if (shutdownRequested)
-                        return [3 /*break*/, 5];
+                        return [3 /*break*/, 9];
                     // But only if we have queues to listen on
                     if (selectedPairs.length) {
                         // Randomize order
@@ -450,30 +461,60 @@ function processMessages(queues, callback, options) {
                         // Launch listen loop for each queue
                         for (_i = 0, selectedPairs_1 = selectedPairs; _i < selectedPairs_1.length; _i++) {
                             _a = selectedPairs_1[_i], qname = _a.qname, qrl = _a.qrl;
-                            if (!activeLoops[qname])
-                                activeLoops[qname] = listenLoop(qname, qrl);
+                            // Bail if we already have too many
+                            if (activeLoops.size >= maxActiveLoops) {
+                                if (opt.verbose)
+                                    console.error(chalk_1.default.yellow('Hit active worker limit of ') + maxActiveLoops);
+                                break;
+                            }
+                            loopId = loopCounter++;
+                            activeLoops.set(loopId, { qname: qname, qrl: qrl, promise: listenLoop(qname, qrl, loopId) });
                         }
                     }
                     if (!!shutdownRequested) return [3 /*break*/, 4];
                     msSoFar = Math.max(0, new Date() - start);
-                    msUntilNextResolve = Math.max(0, opt.waitTime * 1000 - msSoFar);
+                    msUntilNextResolve = Math.max(0, /*opt.waitTime **/ 1000 - msSoFar);
                     debug({ msSoFar: msSoFar, msUntilNextResolve: msUntilNextResolve });
                     if (!msUntilNextResolve) return [3 /*break*/, 4];
                     if (opt.verbose)
                         console.error(chalk_1.default.blue('Will resolve queues again in ' + Math.round(msUntilNextResolve / 1000) + ' seconds'));
                     return [4 /*yield*/, delay(msUntilNextResolve)];
                 case 3:
-                    _b.sent();
-                    _b.label = 4;
-                case 4: return [3 /*break*/, 1];
-                case 5: 
-                // Wait on all work to finish
-                // shutdownCallback()
-                return [4 /*yield*/, Promise.all(Object.values(activeLoops))];
+                    _h.sent();
+                    _h.label = 4;
+                case 4:
+                    _b = 0, completedLoops_1 = completedLoops;
+                    _h.label = 5;
+                case 5:
+                    if (!(_b < completedLoops_1.length)) return [3 /*break*/, 8];
+                    _c = completedLoops_1[_b], id = _c[0], _d = _c[1], qname = _d.qname, qrl = _d.qrl, promise = _d.promise;
+                    return [4 /*yield*/, promise]; // make sure the promise resolves
                 case 6:
-                    // Wait on all work to finish
-                    // shutdownCallback()
-                    _b.sent();
+                    _h.sent(); // make sure the promise resolves
+                    debug('Cleaning up', { id: id, qname: qname, qrl: qrl, promise: promise });
+                    activeLoops.delete(id);
+                    _h.label = 7;
+                case 7:
+                    _b++;
+                    return [3 /*break*/, 5];
+                case 8: return [3 /*break*/, 1];
+                case 9:
+                    debug('out here', { activeLoops: activeLoops });
+                    _e = 0, activeLoops_1 = activeLoops;
+                    _h.label = 10;
+                case 10:
+                    if (!(_e < activeLoops_1.length)) return [3 /*break*/, 13];
+                    _f = activeLoops_1[_e], id = _f[0], _g = _f[1], qname = _g.qname, qrl = _g.qrl, promise = _g.promise;
+                    debug('Waiting on active loop', id);
+                    return [4 /*yield*/, promise]; // make sure the promise resolves
+                case 11:
+                    _h.sent(); // make sure the promise resolves
+                    _h.label = 12;
+                case 12:
+                    _e++;
+                    return [3 /*break*/, 10];
+                case 13:
+                    debug('after all');
                     return [2 /*return*/];
             }
         });

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "qdone",
-  "version": "2.0.8-alpha",
+  "version": "2.0.9-alpha",
   "description": "Language agnostic job queue for SQS",
   "type": "module",
   "main": "./index.js",

package/src/consumer.js

@@ -27,22 +27,26 @@ import { getSQSClient } from './sqs.js'
 //
 export class DoNotProcess extends Error {}
 
-const debug = Debug('qdone:worker')
+const debug = Debug('qdone:consumer')
 
 // Global flag for shutdown request
 let shutdownRequested = false
 const shutdownCallbacks = []
 
 export function requestShutdown () {
+  debug('requestShutdown')
   shutdownRequested = true
   for (const callback of shutdownCallbacks) {
-    try { callback() } catch (e) { }
+    debug('callback', callback)
+    callback()
+    // try { callback() } catch (e) { }
   }
+  debug('requestShutdown done')
 }
 
 export async function processMessage (message, callback, qname, qrl, opt) {
   debug('processMessage', message, qname, qrl)
-  const payload = JSON.parse(message.Body)
+  const payload = opt.json ? JSON.parse(message.Body) : message.Body
   if (opt.verbose) {
     console.error(chalk.blue('  Processing payload:'), payload)
   } else if (!opt.disableLog) {
@@ -248,8 +252,11 @@ export async function processMessages (queues, callback, options) {
   const opt = getOptionsWithDefaults(options)
   debug('processMessages', { queues, callback, options, opt })
 
+  let loopCounter = 0
   const stats = { noJobs: 0, jobsSucceeded: 0, jobsFailed: 0 }
-  const activeLoops = {}
+  const maxActiveLoops = 10
+  const activeLoops = new Map()
+  const completedLoops = new Map()
 
   // This delay function keeps a timeout reference around so it can be
   // cancelled at shutdown
@@ -261,18 +268,20 @@ export async function processMessages (queues, callback, options) {
   // Callback to help facilitate better UX at shutdown
   function shutdownCallback () {
     if (opt.verbose) {
-      debug({ activeLoops })
-      const activeQueues = Object.keys(activeLoops).filter(q => activeLoops[q]).map(q => q.slice(opt.prefix.length))
-      if (activeQueues.length) {
-        console.error(chalk.blue('Waiting for work to finish on the following queues: ') + activeQueues.join(chalk.blue(', ')))
+      debug({ activeLoops, completedLoops })
+      if (activeLoops.length) {
+        console.error(chalk.blue('Waiting for work to finish on the following queues: '))
+        for (const [id, { qname, qrl, promise }] of activeLoops) {
+          console.error('    ' + qname + `job ${id}`)
+        }
       }
-      clearTimeout(delayTimeout)
+      // clearTimeout(delayTimeout)
     }
   }
   shutdownCallbacks.push(shutdownCallback)
 
   // Listen to a queue until it is out of messages
-  async function listenLoop (qname, qrl) {
+  async function listenLoop (qname, qrl, loopId) {
     try {
       if (shutdownRequested) return
       if (opt.verbose) {
@@ -284,12 +293,15 @@ export async function processMessages (queues, callback, options) {
       }
       // Aggregate the results
       const { noJobs, jobsSucceeded, jobsFailed } = await pollSingleQueue(qname, qrl, callback, opt)
+      debug('pollSingleQueue return')
       stats.noJobs += noJobs
       stats.jobsFailed += jobsFailed
       stats.jobsSucceeded += jobsSucceeded
+      debug({ stats, noJobs })
 
-      // No work? return to outer loop
-      if (noJobs) return
+      // No work? Shutdown requested? Return to outer loop
+      debug({ noJobs, shutdownRequested })
+      if (noJobs || shutdownRequested) return
 
       // Otherwise keep going
       return listenLoop(qname, qrl)
@@ -298,7 +310,7 @@ export async function processMessages (queues, callback, options) {
       console.error(chalk.red('  ERROR in listenLoop'))
       console.error(chalk.blue('  error : ') + err)
     } finally {
-      delete activeLoops[qname]
+      completedLoops.set(loopId, activeLoops.get(loopId))
     }
   }
 
@@ -306,6 +318,7 @@ export async function processMessages (queues, callback, options) {
   while (!shutdownRequested) { // eslint-disable-line
     const start = new Date()
     const selectedPairs = await resolveQueues(queues, opt)
+    debug({ selectedPairs })
     if (shutdownRequested) break
 
     // But only if we have queues to listen on
@@ -323,24 +336,43 @@ export async function processMessages (queues, callback, options) {
 
       // Launch listen loop for each queue
       for (const { qname, qrl } of selectedPairs) {
-        if (!activeLoops[qname]) activeLoops[qname] = listenLoop(qname, qrl)
+        // Bail if we already have too many
+        if (activeLoops.size >= maxActiveLoops) {
+          if (opt.verbose) console.error(chalk.yellow('Hit active worker limit of ') + maxActiveLoops)
+          break
+        }
+        const loopId = loopCounter++
+        activeLoops.set(loopId, { qname, qrl, promise: listenLoop(qname, qrl, loopId) })
       }
     }
+
     // Wait until the next time we need to resolve
     if (!shutdownRequested) {
       const msSoFar = Math.max(0, new Date() - start)
-      const msUntilNextResolve = Math.max(0, opt.waitTime * 1000 - msSoFar)
+      const msUntilNextResolve = Math.max(0, /*opt.waitTime **/ 1000 - msSoFar)
       debug({ msSoFar, msUntilNextResolve })
       if (msUntilNextResolve) {
         if (opt.verbose) console.error(chalk.blue('Will resolve queues again in ' + Math.round(msUntilNextResolve / 1000) + ' seconds'))
         await delay(msUntilNextResolve)
       }
     }
+
+    // Cleanup completed loops
+    for (const [id, { qname, qrl, promise }] of completedLoops) {
+      await promise // make sure the promise resolves
+      debug('Cleaning up', { id, qname, qrl, promise })
+      activeLoops.delete(id)
+    }
   }
+  debug('out here', { activeLoops })
 
   // Wait on all work to finish
   // shutdownCallback()
-  await Promise.all(Object.values(activeLoops))
+  for (const [id, { qname, qrl, promise }] of activeLoops) {
+    debug('Waiting on active loop', id)
+    await promise // make sure the promise resolves
+  }
+  debug('after all')
 }
 
 debug('loaded')