qdone 2.0.12-alpha → 2.0.14-alpha

package/commonjs/src/scheduler/queueManager.js

@@ -47,7 +47,6 @@ var chalk_1 = __importDefault(require("chalk"));
 var debug_1 = __importDefault(require("debug"));
 var qrlCache_js_1 = require("../qrlCache.js");
 var idleQueues_js_1 = require("../idleQueues.js");
-var sqs_js_1 = require("../sqs.js");
 var debug = (0, debug_1.default)('qdone:queueManager');
 var QueueManager = /** @class */ (function () {
     function QueueManager(opt, queues, resolveSeconds) {
@@ -56,72 +55,143 @@ var QueueManager = /** @class */ (function () {
         this.queues = queues;
         this.resolveSeconds = resolveSeconds;
         this.selectedPairs = [];
+        this.icehouse = {};
         this.resolveTimeout = undefined;
         this.shutdownRequested = false;
-        this.resolveQueues();
+        this.resolvePromise = this.resolveQueues();
     }
+    // Sends a queue to the icehouse, where it waits for a while before being
+    // checked again
+    QueueManager.prototype.updateIcehouse = function (qrl, emptyReceive) {
+        var foundEntry = !!this.icehouse[qrl];
+        var _a = this.icehouse[qrl] || {
+            lastCheck: new Date(),
+            secondsToWait: Math.round(20 + 10 * Math.random()),
+            numEmptyReceives: 0 + emptyReceive
+        }, lastCheck = _a.lastCheck, secondsToWait = _a.secondsToWait, numEmptyReceives = _a.numEmptyReceives;
+        if (emptyReceive) {
+            var now = new Date();
+            var secondsElapsed = lastCheck - now;
+            var minWait = 10;
+            var maxWait = 600;
+            var baseSeconds = Math.pow(numEmptyReceives, 2) * 20;
+            var jitterSeconds = Math.round((Math.random() - 0.5) * baseSeconds);
+            var newSecondsToWait = Math.max(minWait, Math.min(maxWait, baseSeconds + jitterSeconds));
+            var newEntry = { lastCheck: now, secondsToWait: newSecondsToWait, numEmptyReceives: numEmptyReceives + 1 };
+            this.icehouse[qrl] = newEntry;
+            if (this.opt.verbose) {
+                console.error(chalk_1.default.blue('Sending queue to icehouse'), qrl, chalk_1.default.blue('for'), newSecondsToWait, chalk_1.default.blue('seconds'));
+            }
+            debug({ foundEntry: foundEntry, newEntry: newEntry, lastCheck: lastCheck, secondsToWait: secondsToWait, now: now, secondsElapsed: secondsElapsed, maxWait: maxWait, minWait: minWait, baseSeconds: baseSeconds, jitterSeconds: jitterSeconds });
+        }
+        else {
+            delete this.icehouse[qrl];
+        }
+    };
+    // Returns true if the queue should be kept in the icehouse
+    QueueManager.prototype.keepInIcehouse = function (qrl, now) {
+        if (this.icehouse[qrl]) {
+            var _a = this.icehouse[qrl], lastCheck = _a.lastCheck, secondsToWait = _a.secondsToWait;
+            var secondsElapsed = Math.round((now - lastCheck) / 1000);
+            debug({ icehouseCheck: { qrl: qrl, lastCheck: lastCheck, secondsToWait: secondsToWait, secondsElapsed: secondsElapsed } });
+            var letOut = secondsElapsed > secondsToWait;
+            if (letOut && this.opt.verbose) {
+                console.error(chalk_1.default.blue('Coming out of icehouse:'), qrl);
+            }
+            return !letOut;
+        }
+        else {
+            return false;
+        }
+    };
     QueueManager.prototype.resolveQueues = function () {
         return __awaiter(this, void 0, void 0, function () {
-            var qnames, pairs, activePairs;
+            var qnames, pairs, now, filteredPairs, activePairs;
             var _this = this;
             return __generator(this, function (_a) {
                 switch (_a.label) {
                     case 0:
+                        clearTimeout(this.resolveTimeout);
                         if (this.shutdownRequested)
                             return [2 /*return*/];
-                        // Start processing
-                        if (this.opt.verbose)
-                            console.error(chalk_1.default.blue('Resolving queues: ') + this.queues.join(' '));
                         qnames = this.queues.map(function (queue) { return (0, qrlCache_js_1.normalizeQueueName)(queue, _this.opt); });
                         return [4 /*yield*/, (0, qrlCache_js_1.getQnameUrlPairs)(qnames, this.opt)];
                     case 1:
                         pairs = _a.sent();
+                        if (this.opt.verbose)
+                            console.error(chalk_1.default.blue('Resolving queues:'));
                         if (this.shutdownRequested)
                             return [2 /*return*/];
+                        now = new Date();
+                        filteredPairs = pairs
+                            // first failed
+                            .filter(function (_a) {
+                            var qname = _a.qname, qrl = _a.qrl;
+                            var suf = _this.opt.failSuffix + (_this.opt.fifo ? '.fifo' : '');
+                            var isFailQueue = qname.slice(-suf.length) === suf;
+                            return _this.opt.includeFailed ? true : !isFailQueue;
+                        })
+                            // first fifo
+                            .filter(function (_a) {
+                            var qname = _a.qname, qrl = _a.qrl;
+                            var isFifo = qname.endsWith('.fifo');
+                            return _this.opt.fifo ? isFifo : !isFifo;
+                        })
+                            // then icehouse
+                            .filter(function (_a) {
+                            var qname = _a.qname, qrl = _a.qrl;
+                            return !_this.keepInIcehouse(qrl, now);
+                        });
                         activePairs = [];
                         if (!this.opt.activeOnly) return [3 /*break*/, 3];
-                        debug({ pairsBeforeCheck: pairs });
-                        return [4 /*yield*/, Promise.all(pairs.map(function (pair) { return __awaiter(_this, void 0, void 0, function () {
-                                var idle;
-                                return __generator(this, function (_a) {
-                                    switch (_a.label) {
-                                        case 0: return [4 /*yield*/, (0, idleQueues_js_1.cheapIdleCheck)(pair.qname, pair.qrl, this.opt)];
-                                        case 1:
-                                            idle = (_a.sent()).idle;
-                                            if (!idle)
-                                                activePairs.push(pair);
-                                            return [2 /*return*/];
-                                    }
+                        if (this.opt.verbose) {
+                            console.error(chalk_1.default.blue('  checking active only'));
+                        }
+                        return [4 /*yield*/, Promise.all(filteredPairs.map(function (_a) {
+                                var qname = _a.qname, qrl = _a.qrl;
+                                return __awaiter(_this, void 0, void 0, function () {
+                                    var idle;
+                                    return __generator(this, function (_b) {
+                                        switch (_b.label) {
+                                            case 0: return [4 /*yield*/, (0, idleQueues_js_1.cheapIdleCheck)(qname, qrl, this.opt)];
+                                            case 1:
+                                                idle = (_b.sent()).result.idle;
+                                                debug({ idle: idle, qname: qname });
+                                                if (!idle)
+                                                    activePairs.push({ qname: qname, qrl: qrl });
+                                                return [2 /*return*/];
+                                        }
+                                    });
                                 });
-                            }); }))];
+                            }))];
                     case 2:
                         _a.sent();
                         _a.label = 3;
                     case 3:
                         if (this.shutdownRequested)
                             return [2 /*return*/];
-                        // Finished resolving
-                        if (this.opt.verbose) {
-                            console.error(chalk_1.default.blue('  done'));
-                            console.error();
-                        }
                         // Figure out which queues we want to listen on, choosing between active and
                         // all, filtering out failed queues if the user wants that
-                        this.selectedPairs = (this.opt.activeOnly ? activePairs : pairs)
-                            .filter(function (_a) {
-                            var qname = _a.qname;
-                            var suf = _this.opt.failSuffix + (_this.opt.fifo ? '.fifo' : '');
-                            var isFailQueue = qname.slice(-suf.length) === suf;
-                            var shouldInclude = _this.opt.includeFailed ? true : !isFailQueue;
-                            return shouldInclude;
-                        });
+                        this.selectedPairs = (this.opt.activeOnly ? activePairs : filteredPairs);
                         // Randomize order
                         this.selectedPairs.sort(function () { return 0.5 - Math.random(); });
+                        if (this.opt.verbose)
+                            console.error(chalk_1.default.blue('  selected:\n  ') + this.selectedPairs.map(function (_a) {
+                                var qname = _a.qname;
+                                return qname;
+                            }).join('\n  '));
                         debug('selectedPairs', this.selectedPairs);
+                        // Finished resolving
+                        if (this.opt.verbose) {
+                            console.error(chalk_1.default.blue('  done'));
+                            console.error();
+                        }
                         if (this.opt.verbose) {
                             console.error(chalk_1.default.blue('Will resolve queues again in ' + this.resolveSeconds + ' seconds'));
                         }
-                        this.resolveTimeout = setTimeout(function () { return _this.resolveQueues(); }, this.resolveSeconds * 1000);
+                        this.resolveTimeout = setTimeout(function () {
+                            _this.resolvePromise = _this.resolveQueues();
+                        }, this.resolveSeconds * 1000);
                         return [2 /*return*/];
                 }
             });
@@ -137,8 +207,19 @@ var QueueManager = /** @class */ (function () {
         return this.selectedPairs;
     };
     QueueManager.prototype.shutdown = function () {
-        clearTimeout(this.resolveTimeout);
-        this.shutdownRequested = true;
+        return __awaiter(this, void 0, void 0, function () {
+            return __generator(this, function (_a) {
+                switch (_a.label) {
+                    case 0:
+                        this.shutdownRequested = true;
+                        clearTimeout(this.resolveTimeout);
+                        return [4 /*yield*/, this.resolvePromise];
+                    case 1:
+                        _a.sent();
+                        return [2 /*return*/];
+                }
+            });
+        });
     };
     return QueueManager;
 }());

package/commonjs/src/scheduler/systemMonitor.js

@@ -6,59 +6,43 @@
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.SystemMonitor = void 0;
 var SystemMonitor = /** @class */ (function () {
-    function SystemMonitor(opt, smoothingFactor, reportSeconds) {
-        if (smoothingFactor === void 0) { smoothingFactor = 0.5; }
-        if (reportSeconds === void 0) { reportSeconds = 5; }
-        this.opt = opt;
-        this.smoothingFactor = smoothingFactor;
+    function SystemMonitor(reportCallback, reportSeconds) {
+        if (reportSeconds === void 0) { reportSeconds = 1; }
+        this.reportCallback = reportCallback || console.log;
         this.reportSeconds = reportSeconds;
-        this.measurements = {
-            setTimeout: [],
-            setImmediate: []
-        };
-        this.timeouts = {
-            setTimeout: undefined,
-            setImmediate: undefined,
-            reportLatency: undefined
-        };
-        this.measureLatencySetTimeout();
+        this.measurements = [];
+        this.measure();
         this.reportLatency();
     }
-    SystemMonitor.prototype.measureLatencySetTimeout = function () {
+    SystemMonitor.prototype.measure = function () {
         var _this = this;
+        clearTimeout(this.measureTimeout);
         var start = new Date();
-        this.timeouts.setTimeout = setTimeout(function () {
+        this.measureTimeout = setTimeout(function () {
             var latency = new Date() - start;
-            _this.measurements.setTimeout.push(latency);
-            if (_this.measurements.setTimeout.length > 1000)
-                _this.measurements.setTimeout.shift();
-            _this.measureLatencySetTimeout();
+            _this.measurements.push(latency);
+            if (_this.measurements.length > 1000)
+                _this.measurements.shift();
+            _this.measure();
         });
     };
     SystemMonitor.prototype.getLatency = function () {
-        var results = {};
-        for (var k in this.measurements) {
-            var values = this.measurements[k];
-            results[k] = values.length ? values.reduce(function (a, b) { return a + b; }, 0) / values.length : 0;
-        }
-        return results;
+        return this.measurements.length ? this.measurements.reduce(function (a, b) { return a + b; }, 0) / this.measurements.length : 0;
     };
     SystemMonitor.prototype.reportLatency = function () {
         var _this = this;
-        this.timeouts.reportLatency = setTimeout(function () {
-            var _a;
-            for (var k in _this.measurements) {
-                var values = _this.measurements[k];
-                var mean = values.length ? values.reduce(function (a, b) { return a + b; }, 0) / values.length : 0;
-                console.log((_a = {}, _a[k] = mean, _a));
-            }
+        clearTimeout(this.reportTimeout);
+        this.reportTimeout = setTimeout(function () {
+            var latency = _this.getLatency();
+            // console.log({ latency })
+            if (_this.reportCallback)
+                _this.reportCallback(latency);
             _this.reportLatency();
         }, this.reportSeconds * 1000);
     };
     SystemMonitor.prototype.shutdown = function () {
-        console.log(this.measurements);
-        for (var k in this.timeouts)
-            clearTimeout(this.timeouts[k]);
+        clearTimeout(this.measureTimeout);
+        clearTimeout(this.reportTimeout);
     };
     return SystemMonitor;
 }());

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "qdone",
-  "version": "2.0.12-alpha",
+  "version": "2.0.14-alpha",
   "description": "Language agnostic job queue for SQS",
   "type": "module",
   "main": "./index.js",

package/src/consumer.js

@@ -2,7 +2,7 @@
  * Consumer implementation.
  */
 
-import { ReceiveMessageCommand } from '@aws-sdk/client-sqs'
+import { ReceiveMessageCommand, QueueDoesNotExist } from '@aws-sdk/client-sqs'
 import chalk from 'chalk'
 import Debug from 'debug'
 
@@ -18,12 +18,12 @@ const debug = Debug('qdone:consumer')
 let shutdownRequested = false
 const shutdownCallbacks = []
 
-export function requestShutdown () {
+export async function requestShutdown () {
   debug('requestShutdown')
   shutdownRequested = true
   for (const callback of shutdownCallbacks) {
     debug('callback', callback)
-    callback()
+    await callback()
     // try { callback() } catch (e) { }
   }
   debug('requestShutdown done')
@@ -35,7 +35,7 @@ export async function getMessages (qrl, opt, maxMessages) {
     MaxNumberOfMessages: maxMessages,
     MessageAttributeNames: ['All'],
     QueueUrl: qrl,
-    VisibilityTimeout: 30,
+    VisibilityTimeout: 60,
     WaitTimeSeconds: opt.waitTime
   }
   const response = await getSQSClient().send(new ReceiveMessageCommand(params))
@@ -47,20 +47,21 @@ export async function getMessages (qrl, opt, maxMessages) {
 // Consumer
 //
 export async function processMessages (queues, callback, options) {
+  debug({ options })
   const opt = getOptionsWithDefaults(options)
-  debug('processMessages', { queues, callback, options, opt })
+  debug('processMessages', { queues, callback, options, opt, argv: process.argv })
 
-  const systemMonitor = new SystemMonitor(opt)
+  let lastLatency = 10
+  const systemMonitor = new SystemMonitor(latency => {
+    const percentDifference = 100 * Math.abs(lastLatency - latency) / lastLatency
+    if (percentDifference > 10 && opt.verbose) {
+      console.error(chalk.blue('Latency:', Math.round(latency), 'ms'))
+    }
+    lastLatency = latency
+  })
   const jobExecutor = new JobExecutor(opt)
   const queueManager = new QueueManager(opt, queues)
-  const maxActiveJobs = 100
-  debug({ systemMonitor, jobExecutor, queueManager })
-
-  shutdownCallbacks.push(() => {
-    systemMonitor.shutdown()
-    queueManager.shutdown()
-    jobExecutor.shutdown()
-  })
+  // debug({ systemMonitor, jobExecutor, queueManager })
 
   // This delay function keeps a timeout reference around so it can be
   // cancelled at shutdown
@@ -69,28 +70,55 @@ export async function processMessages (queues, callback, options) {
     delayTimeout = setTimeout(resolve, ms)
   })
 
+  shutdownCallbacks.push(async () => {
+    clearTimeout(delayTimeout)
+    await queueManager.shutdown()
+    debug({ queueManager: 'done' })
+    await jobExecutor.shutdown()
+    debug({ jobExecutor: 'done' })
+    await systemMonitor.shutdown()
+    debug({ systemMonitor: 'done' })
+  })
+
   // Keep track of how many messages could be returned from each queue
   const activeQrls = new Set()
   let maxReturnCount = 0
   const listen = async (qname, qrl, maxMessages) => {
     activeQrls.add(qrl)
     maxReturnCount += maxMessages
-    const messages = await getMessages(qrl, opt, maxMessages)
-    for (const message of messages) {
-      jobExecutor.executeJob(message, callback, qname, qrl)
+    try {
+      const messages = await getMessages(qrl, opt, maxMessages)
+      if (messages.length) {
+        for (const message of messages) {
+          jobExecutor.executeJob(message, callback, qname, qrl)
+        }
+        queueManager.updateIcehouse(qrl, false)
+      } else {
+        // If we didn't get any, update the icehouse so we can back off
+        queueManager.updateIcehouse(qrl, true)
+      }
+
+      // Max job accounting
+      maxReturnCount -= maxMessages
+      activeQrls.delete(qrl)
+    } catch (e) {
+      // If the queue has been cleaned up, we should back off anyway
+      if (e instanceof QueueDoesNotExist) {
+        queueManager.updateIcehouse(qrl, true)
+      } else {
+        throw e
+      }
     }
-    maxReturnCount -= maxMessages
-    activeQrls.delete(qrl)
   }
 
   while (!shutdownRequested) { // eslint-disable-line
     // Figure out how we are running
-    const allowedJobs = maxActiveJobs - jobExecutor.activeJobCount() - maxReturnCount
+    const allowedJobs = opt.maxConcurrentJobs - jobExecutor.activeJobCount() - maxReturnCount
     const maxLatency = 100
     const latency = systemMonitor.getLatency().setTimeout
     const latencyFactor = 1 - Math.abs(Math.min(latency / maxLatency, 1)) // 0 if latency is at max, 1 if latency 0
     const targetJobs = Math.round(allowedJobs * latencyFactor)
-    debug({ allowedJobs, maxLatency, latency, latencyFactor, targetJobs, activeQrls })
+    // debug({ allowedJobs, maxLatency, latency, latencyFactor, targetJobs, activeQrls })
 
     let jobsLeft = targetJobs
     for (const { qname, qrl } of queueManager.getPairs()) {
@@ -101,7 +129,7 @@ export async function processMessages (queues, callback, options) {
       if (opt.verbose) {
         console.error(chalk.blue('Listening on: '), qname)
       }
-      debug({ listenedTo: { qname, maxMessages, jobsLeft } })
+      // debug({ listenedTo: { qname, maxMessages, jobsLeft } })
     }
     await delay(1000)
   }

package/src/defaults.js

@@ -36,6 +36,7 @@ export const defaults = Object.freeze({
   killAfter: 30,
   archive: false,
   activeOnly: false,
+  maxConcurrentJobs: 100,
 
   // Idle Queues
   idleFor: 60,
@@ -43,6 +44,12 @@ export const defaults = Object.freeze({
   unpair: false
 })
 
+function validateInteger (opt, name) {
+  const parsed = parseInt(opt[name], 10)
+  if (isNaN(parsed)) throw new Error(`${name} needs to be an integer.`)
+  return parsed
+}
+
 /**
  * This function should be called by each exposed API entry point on the
  * options passed in from the caller. It supports options named in camelCase
@@ -93,15 +100,29 @@ export function getOptionsWithDefaults (options) {
     archive: options.archive || defaults.archive,
     activeOnly: options.activeOnly || options['active-only'] || defaults.activeOnly,
     includeFailed: options.includeFailed || options['include-failed'] || defaults.includeFailed,
+    maxConcurrentJobs: options.maxConcurrentJobs || defaults.maxConcurrentJobs,
 
     // Idle Queues
     idleFor: options.idleFor || options['idle-for'] || defaults.idleFor,
     delete: options.delete || defaults.delete,
     unpair: options.delete || defaults.unpair
   }
+
+  // Setting this env here means we don't have to in AWS SDK constructors
   process.env.AWS_REGION = opt.region
 
-  // TODO: validate options
+  // Validation
+  opt.cacheTtlSeconds = validateInteger(opt, 'cacheTtlSeconds')
+  opt.messageRetentionPeriod = validateInteger(opt, 'messageRetentionPeriod')
+  opt.delay = validateInteger(opt, 'delay')
+  opt.sendRetries = validateInteger(opt, 'sendRetries')
+  opt.failDelay = validateInteger(opt, 'failDelay')
+  opt.dlqAfter = validateInteger(opt, 'dlqAfter')
+  opt.waitTime = validateInteger(opt, 'waitTime')
+  opt.killAfter = validateInteger(opt, 'killAfter')
+  opt.maxConcurrentJobs = validateInteger(opt, 'maxConcurrentJobs')
+  opt.idleFor = validateInteger(opt, 'idleFor')
+
   return opt
 }
 

package/src/idleQueues.js

@@ -40,7 +40,7 @@ export async function _cheapIdleCheck (qname, qrl, opt) {
   const client = getSQSClient()
   const cmd = new GetQueueAttributesCommand({ AttributeNames: attributeNames, QueueUrl: qrl })
   const data = await client.send(cmd)
-  // debug('data', data)
+  debug('data', data)
   const result = data.Attributes
   result.queue = qname.slice(opt.prefix.length)
   // We are idle if all the messages attributes are zero