qdone 2.0.25-alpha → 2.0.27-alpha

package/commonjs/src/consumer.js

@@ -118,7 +118,7 @@ exports.getMessages = getMessages;
 //
 function processMessages(queues, callback, options) {
     return __awaiter(this, void 0, void 0, function () {
-        var opt, lastLatency, systemMonitor, jobExecutor, queueManager, delayTimeout, delay, activeQrls, maxReturnCount, listen, allowedJobs, maxLatency, freeMemory, totalMemory, memoryThreshold, freememThreshold, remainingMemory, latency, latencyFactor, freememFactor, targetJobs, jobsLeft, _i, _a, _b, qname, qrl, maxMessages;
+        var opt, lastLatency, systemMonitor, jobExecutor, queueManager, cores, delayTimeout, delay, activeQrls, maxReturnCount, listen, allowedJobs, maxLatency, latency, latencyFactor, freeMemory, totalMemory, memoryThreshold, freememThreshold, remainingMemory, freememFactor, oneMinuteLoad, loadPerCore, loadFactor, overallFactor, targetJobs, jobsLeft, _i, _a, _b, qname, qrl, maxMessages;
         var _this = this;
         return __generator(this, function (_c) {
             switch (_c.label) {
@@ -136,6 +136,7 @@ function processMessages(queues, callback, options) {
                     });
                     jobExecutor = new jobExecutor_js_1.JobExecutor(opt);
                     queueManager = new queueManager_js_1.QueueManager(opt, queues, 60);
+                    cores = (0, os_1.cpus)().length;
                     delay = function (ms) { return new Promise(function (resolve) {
                         delayTimeout = setTimeout(resolve, ms);
                     }); };
@@ -167,6 +168,9 @@ function processMessages(queues, callback, options) {
                         return __generator(this, function (_a) {
                             switch (_a.label) {
                                 case 0:
+                                    if (opt.verbose) {
+                                        console.error(chalk_1.default.blue('Listening on: '), qname);
+                                    }
                                     activeQrls.add(qrl);
                                     maxReturnCount += maxMessages;
                                     _a.label = 1;
@@ -175,16 +179,18 @@ function processMessages(queues, callback, options) {
                                     return [4 /*yield*/, getMessages(qrl, opt, maxMessages)];
                                 case 2:
                                     messages = _a.sent();
-                                    if (messages.length) {
-                                        for (_i = 0, messages_1 = messages; _i < messages_1.length; _i++) {
-                                            message = messages_1[_i];
-                                            jobExecutor.executeJob(message, callback, qname, qrl, function () { return queueManager.updateIcehouse(qrl, true); });
+                                    if (!shutdownRequested) {
+                                        if (messages.length) {
+                                            for (_i = 0, messages_1 = messages; _i < messages_1.length; _i++) {
+                                                message = messages_1[_i];
+                                                jobExecutor.executeJob(message, callback, qname, qrl, function () { return queueManager.updateIcehouse(qrl, true); });
+                                            }
+                                            queueManager.updateIcehouse(qrl, false);
+                                        }
+                                        else {
+                                            // If we didn't get any, update the icehouse so we can back off
+                                            queueManager.updateIcehouse(qrl, true);
                                         }
-                                        queueManager.updateIcehouse(qrl, false);
-                                    }
-                                    else {
-                                        // If we didn't get any, update the icehouse so we can back off
-                                        queueManager.updateIcehouse(qrl, true);
                                     }
                                     // Max job accounting
                                     maxReturnCount -= maxMessages;
@@ -209,18 +215,24 @@ function processMessages(queues, callback, options) {
                     if (!!shutdownRequested) return [3 /*break*/, 3];
                     allowedJobs = Math.max(0, opt.maxConcurrentJobs - jobExecutor.activeJobCount() - maxReturnCount);
                     maxLatency = 100;
+                    latency = systemMonitor.getLatency() || 10;
+                    latencyFactor = 1 - Math.abs(Math.min(latency / maxLatency, 1)) // 0 if latency is at max, 1 if latency 0
+                    ;
                     freeMemory = (0, os_1.freemem)();
                     totalMemory = (0, os_1.totalmem)();
                     memoryThreshold = totalMemory * opt.maxMemoryPercent / 100;
                     freememThreshold = totalMemory - memoryThreshold;
                     remainingMemory = Math.max(0, freeMemory - freememThreshold);
-                    latency = systemMonitor.getLatency() || 10;
-                    latencyFactor = 1 - Math.abs(Math.min(latency / maxLatency, 1)) // 0 if latency is at max, 1 if latency 0
-                    ;
                     freememFactor = Math.min(1, Math.max(0, remainingMemory / memoryThreshold));
-                    targetJobs = Math.round(allowedJobs * latencyFactor * freememFactor);
+                    oneMinuteLoad = (0, os_1.loadavg)()[0];
+                    loadPerCore = oneMinuteLoad / cores;
+                    loadFactor = 1 - Math.min(1, Math.max(0, loadPerCore / 3));
+                    overallFactor = Math.min(latencyFactor, freememFactor, loadFactor);
+                    targetJobs = Math.round(allowedJobs * overallFactor);
                     jobsLeft = targetJobs;
-                    debug({ jobCount: jobExecutor.activeJobCount(), freeMemory: freeMemory, totalMemory: totalMemory, freememThreshold: freememThreshold, remainingMemory: remainingMemory, memoryThreshold: memoryThreshold, maxReturnCount: maxReturnCount, allowedJobs: allowedJobs, maxLatency: maxLatency, latency: latency, latencyFactor: latencyFactor, freememFactor: freememFactor, targetJobs: targetJobs, activeQrls: activeQrls });
+                    if (opt.verbose) {
+                        console.error({ jobCount: jobExecutor.activeJobCount(), freeMemory: freeMemory, totalMemory: totalMemory, freememThreshold: freememThreshold, remainingMemory: remainingMemory, memoryThreshold: memoryThreshold, maxReturnCount: maxReturnCount, allowedJobs: allowedJobs, maxLatency: maxLatency, latency: latency, latencyFactor: latencyFactor, freememFactor: freememFactor, oneMinuteLoad: oneMinuteLoad, loadPerCore: loadPerCore, loadFactor: loadFactor, overallFactor: overallFactor, targetJobs: targetJobs, activeQrls: activeQrls });
+                    }
                     for (_i = 0, _a = queueManager.getPairs(); _i < _a.length; _i++) {
                         _b = _a[_i], qname = _b.qname, qrl = _b.qrl;
                         // debug({ evaluating: { qname, qrl, jobsLeft, activeQrlsHasQrl: activeQrls.has(qrl) } })
@@ -229,9 +241,6 @@ function processMessages(queues, callback, options) {
                         maxMessages = Math.min(10, jobsLeft);
                         listen(qname, qrl, maxMessages);
                         jobsLeft -= maxMessages;
-                        if (opt.verbose) {
-                            console.error(chalk_1.default.blue('Listening on: '), qname);
-                        }
                         // debug({ listenedTo: { qname, maxMessages, jobsLeft } })
                     }
                     return [4 /*yield*/, delay(1000)];

package/commonjs/src/scheduler/jobExecutor.js

@@ -74,6 +74,9 @@ var JobExecutor = /** @class */ (function () {
                         this.shutdownRequested = true;
                         // Trigger a maintenance run right away in case it speeds us up
                         clearTimeout(this.maintainVisibilityTimeout);
+                        if (this.opt.verbose) {
+                            console.error(chalk_1.default.blue('Shutting down jobExecutor'));
+                        }
                         return [4 /*yield*/, this.maintainPromise];
                     case 1:
                         _a.sent();
@@ -93,12 +96,24 @@ var JobExecutor = /** @class */ (function () {
      */
     JobExecutor.prototype.maintainVisibility = function () {
         return __awaiter(this, void 0, void 0, function () {
-            var start, jobsToExtendByQrl, jobsToDeleteByQrl, jobsToCleanup, i, job, jobRunTime, jobsToDelete, jobsToExtend, doubled, secondsUntilMax, _a, _b, _c, _i, qrl, jobsToExtend, entries, messageId, job, entry, input, result, count, _d, _e, _f, _g, qrl, jobsToDelete, entries, messageId, job, entry, input, result, count, msElapsed, msPeriod, msLeft, msMin, nextCheckInMs;
+            var nextCheckInMs, start, jobsToExtendByQrl, jobsToDeleteByQrl, jobsToCleanup, i, job, jobRunTime, jobsToDelete, jobsToExtend, doubled, secondsUntilMax, _a, _b, _c, _i, qrl, jobsToExtend, entries, messageId, job, entry, input, result, count, _d, _e, _f, _g, qrl, jobsToDelete, entries, messageId, job, entry, input, result, count;
             var _this = this;
             return __generator(this, function (_h) {
                 switch (_h.label) {
                     case 0:
+                        // Bail if we are shutting down
+                        if (this.shutdownRequested && this.stats.activeJobs === 0 && this.jobs.length === 0) {
+                            if (this.opt.verbose) {
+                                console.error(chalk_1.default.blue('All workers done, finishing shutdown of jobExecutor'));
+                            }
+                            return [2 /*return*/];
+                        }
+                        // Reset our timeout
                         clearTimeout(this.maintainVisibilityTimeout);
+                        nextCheckInMs = this.shutdownRequested ? 1000 : 10 * 1000;
+                        this.maintainVisibilityTimeout = setTimeout(function () {
+                            _this.maintainPromise = _this.maintainVisibility();
+                        }, nextCheckInMs);
                         start = new Date();
                         jobsToExtendByQrl = {};
                         jobsToDeleteByQrl = {};
@@ -249,18 +264,6 @@ var JobExecutor = /** @class */ (function () {
                                 return true;
                             }
                         });
-                        // Bail if we are shutting down
-                        if (this.shutdownRequested && this.stats.activeJobs === 0 && this.jobs.length === 0)
-                            return [2 /*return*/];
-                        msElapsed = new Date() - start;
-                        msPeriod = this.shutdownRequested ? 1 * 1000 : 10 * 1000;
-                        msLeft = Math.max(0, msPeriod - msElapsed);
-                        msMin = this.shutdownRequested ? 1000 : 0;
-                        nextCheckInMs = Math.max(msMin, msLeft);
-                        debug({ msElapsed: msElapsed, msPeriod: msPeriod, msLeft: msLeft, msMin: msMin, nextCheckInMs: nextCheckInMs });
-                        this.maintainVisibilityTimeout = setTimeout(function () {
-                            _this.maintainPromise = _this.maintainVisibility();
-                        }, nextCheckInMs);
                         return [2 /*return*/];
                 }
             });
@@ -272,6 +275,8 @@ var JobExecutor = /** @class */ (function () {
             return __generator(this, function (_a) {
                 switch (_a.label) {
                     case 0:
+                        if (this.shutdownRequested)
+                            throw new Error('jobExecutor is shutting down so cannot execute new job');
                         payload = this.opt.json ? JSON.parse(message.Body) : message.Body;
                         visibilityTimeout = 60;
                         job = {

package/commonjs/src/scheduler/queueManager.js

@@ -114,6 +114,12 @@ var QueueManager = /** @class */ (function () {
                         clearTimeout(this.resolveTimeout);
                         if (this.shutdownRequested)
                             return [2 /*return*/];
+                        this.resolveTimeout = setTimeout(function () {
+                            _this.resolvePromise = _this.resolveQueues();
+                        }, this.resolveSeconds * 1000);
+                        if (this.opt.verbose) {
+                            console.error(chalk_1.default.blue('Will resolve queues again in ' + this.resolveSeconds + ' seconds'));
+                        }
                         qnames = this.queues.map(function (queue) { return (0, qrlCache_js_1.normalizeQueueName)(queue, _this.opt); });
                         return [4 /*yield*/, (0, qrlCache_js_1.getQnameUrlPairs)(qnames, this.opt)];
                     case 1:
@@ -183,15 +189,9 @@ var QueueManager = /** @class */ (function () {
                         debug('selectedPairs', this.selectedPairs);
                         // Finished resolving
                         if (this.opt.verbose) {
-                            console.error(chalk_1.default.blue('  done'));
+                            console.error(chalk_1.default.blue('Done resolving'));
                             console.error();
                         }
-                        if (this.opt.verbose) {
-                            console.error(chalk_1.default.blue('Will resolve queues again in ' + this.resolveSeconds + ' seconds'));
-                        }
-                        this.resolveTimeout = setTimeout(function () {
-                            _this.resolvePromise = _this.resolveQueues();
-                        }, this.resolveSeconds * 1000);
                         return [2 /*return*/];
                 }
             });
@@ -218,6 +218,9 @@ var QueueManager = /** @class */ (function () {
                     case 0:
                         this.shutdownRequested = true;
                         clearTimeout(this.resolveTimeout);
+                        if (this.opt.verbose) {
+                            console.error(chalk_1.default.blue('Waiting for queues to resolve'));
+                        }
                         return [4 /*yield*/, this.resolvePromise];
                     case 1:
                         _a.sent();

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "qdone",
-  "version": "2.0.25-alpha",
+  "version": "2.0.27-alpha",
   "description": "Language agnostic job queue for SQS",
   "type": "module",
   "main": "./index.js",

package/src/consumer.js

@@ -2,7 +2,7 @@
  * Consumer implementation.
  */
 
-import { freemem, totalmem } from 'os'
+import { freemem, totalmem, loadavg, cpus } from 'os'
 import { ReceiveMessageCommand, QueueDoesNotExist } from '@aws-sdk/client-sqs'
 import chalk from 'chalk'
 import Debug from 'debug'
@@ -62,6 +62,7 @@ export async function processMessages (queues, callback, options) {
   })
   const jobExecutor = new JobExecutor(opt)
   const queueManager = new QueueManager(opt, queues, 60)
+  const cores = cpus().length
   // debug({ systemMonitor, jobExecutor, queueManager })
 
   // This delay function keeps a timeout reference around so it can be
@@ -85,18 +86,24 @@ export async function processMessages (queues, callback, options) {
   const activeQrls = new Set()
   let maxReturnCount = 0
   const listen = async (qname, qrl, maxMessages) => {
+    if (opt.verbose) {
+      console.error(chalk.blue('Listening on: '), qname)
+    }
     activeQrls.add(qrl)
     maxReturnCount += maxMessages
     try {
       const messages = await getMessages(qrl, opt, maxMessages)
-      if (messages.length) {
-        for (const message of messages) {
-          jobExecutor.executeJob(message, callback, qname, qrl, () => queueManager.updateIcehouse(qrl, true))
+
+      if (!shutdownRequested) {
+        if (messages.length) {
+          for (const message of messages) {
+            jobExecutor.executeJob(message, callback, qname, qrl, () => queueManager.updateIcehouse(qrl, true))
+          }
+          queueManager.updateIcehouse(qrl, false)
+        } else {
+          // If we didn't get any, update the icehouse so we can back off
+          queueManager.updateIcehouse(qrl, true)
         }
-        queueManager.updateIcehouse(qrl, false)
-      } else {
-        // If we didn't get any, update the icehouse so we can back off
-        queueManager.updateIcehouse(qrl, true)
       }
 
       // Max job accounting
@@ -115,27 +122,38 @@ export async function processMessages (queues, callback, options) {
   while (!shutdownRequested) { // eslint-disable-line
     // Figure out how we are running
     const allowedJobs = Math.max(0, opt.maxConcurrentJobs - jobExecutor.activeJobCount() - maxReturnCount)
+
+    // Latency
     const maxLatency = 100
+    const latency = systemMonitor.getLatency() || 10
+    const latencyFactor = 1 - Math.abs(Math.min(latency / maxLatency, 1)) // 0 if latency is at max, 1 if latency 0
+
+    // Memory
     const freeMemory = freemem()
     const totalMemory = totalmem()
     const memoryThreshold = totalMemory * opt.maxMemoryPercent / 100
     const freememThreshold = totalMemory - memoryThreshold
     const remainingMemory = Math.max(0, freeMemory - freememThreshold)
-    const latency = systemMonitor.getLatency() || 10
-    const latencyFactor = 1 - Math.abs(Math.min(latency / maxLatency, 1)) // 0 if latency is at max, 1 if latency 0
     const freememFactor = Math.min(1, Math.max(0, remainingMemory / memoryThreshold))
-    const targetJobs = Math.round(allowedJobs * latencyFactor * freememFactor)
+
+    // Load
+    const oneMinuteLoad = loadavg()[0]
+    const loadPerCore = oneMinuteLoad / cores
+    const loadFactor = 1 - Math.min(1, Math.max(0, loadPerCore / 3))
+
+    const overallFactor = Math.min(latencyFactor, freememFactor, loadFactor)
+    const targetJobs = Math.round(allowedJobs * overallFactor)
     let jobsLeft = targetJobs
-    debug({ jobCount: jobExecutor.activeJobCount(), freeMemory, totalMemory, freememThreshold, remainingMemory, memoryThreshold, maxReturnCount, allowedJobs, maxLatency, latency, latencyFactor, freememFactor, targetJobs, activeQrls })
+
+    if (opt.verbose) {
+      console.error({ jobCount: jobExecutor.activeJobCount(), freeMemory, totalMemory, freememThreshold, remainingMemory, memoryThreshold, maxReturnCount, allowedJobs, maxLatency, latency, latencyFactor, freememFactor, oneMinuteLoad, loadPerCore, loadFactor, overallFactor, targetJobs, activeQrls })
+    }
     for (const { qname, qrl } of queueManager.getPairs()) {
       // debug({ evaluating: { qname, qrl, jobsLeft, activeQrlsHasQrl: activeQrls.has(qrl) } })
       if (jobsLeft <= 0 || activeQrls.has(qrl)) continue
       const maxMessages = Math.min(10, jobsLeft)
       listen(qname, qrl, maxMessages)
       jobsLeft -= maxMessages
-      if (opt.verbose) {
-        console.error(chalk.blue('Listening on: '), qname)
-      }
       // debug({ listenedTo: { qname, maxMessages, jobsLeft } })
     }
     await delay(1000)

package/src/scheduler/jobExecutor.js

@@ -35,6 +35,9 @@ export class JobExecutor {
     this.shutdownRequested = true
     // Trigger a maintenance run right away in case it speeds us up
     clearTimeout(this.maintainVisibilityTimeout)
+    if (this.opt.verbose) {
+      console.error(chalk.blue('Shutting down jobExecutor'))
+    }
     await this.maintainPromise
     await this.maintainVisibility()
   }
@@ -47,7 +50,21 @@ export class JobExecutor {
    * Changes message visibility on all running jobs using as few calls as possible.
    */
   async maintainVisibility () {
+    // Bail if we are shutting down
+    if (this.shutdownRequested && this.stats.activeJobs === 0 && this.jobs.length === 0) {
+      if (this.opt.verbose) {
+        console.error(chalk.blue('All workers done, finishing shutdown of jobExecutor'))
+      }
+      return
+    }
+
+    // Reset our timeout
     clearTimeout(this.maintainVisibilityTimeout)
+    const nextCheckInMs = this.shutdownRequested ? 1000 : 10 * 1000
+    this.maintainVisibilityTimeout = setTimeout(() => {
+      this.maintainPromise = this.maintainVisibility()
+    }, nextCheckInMs)
+
     // debug('maintainVisibility', this.jobs)
     const start = new Date()
     const jobsToExtendByQrl = {}
@@ -176,24 +193,10 @@ export class JobExecutor {
         return true
       }
     })
-
-    // Bail if we are shutting down
-    if (this.shutdownRequested && this.stats.activeJobs === 0 && this.jobs.length === 0) return
-
-    // Check later, but count the time we spent. Make sure we check at least
-    // every period seconds.
-    const msElapsed = new Date() - start
-    const msPeriod = this.shutdownRequested ? 1 * 1000 : 10 * 1000
-    const msLeft = Math.max(0, msPeriod - msElapsed)
-    const msMin = this.shutdownRequested ? 1000 : 0
-    const nextCheckInMs = Math.max(msMin, msLeft)
-    debug({ msElapsed, msPeriod, msLeft, msMin, nextCheckInMs })
-    this.maintainVisibilityTimeout = setTimeout(() => {
-      this.maintainPromise = this.maintainVisibility()
-    }, nextCheckInMs)
   }
 
   async executeJob (message, callback, qname, qrl, failedCallback) {
+    if (this.shutdownRequested) throw new Error('jobExecutor is shutting down so cannot execute new job')
     // Create job entry and track it
     const payload = this.opt.json ? JSON.parse(message.Body) : message.Body
     const visibilityTimeout = 60

package/src/scheduler/queueManager.js

@@ -69,6 +69,12 @@ export class QueueManager {
   async resolveQueues () {
     clearTimeout(this.resolveTimeout)
     if (this.shutdownRequested) return
+    this.resolveTimeout = setTimeout(() => {
+      this.resolvePromise = this.resolveQueues()
+    }, this.resolveSeconds * 1000)
+    if (this.opt.verbose) {
+      console.error(chalk.blue('Will resolve queues again in ' + this.resolveSeconds + ' seconds'))
+    }
 
     // Start processing
     const qnames = this.queues.map(queue => normalizeQueueName(queue, this.opt))
@@ -120,16 +126,9 @@ export class QueueManager {
 
     // Finished resolving
     if (this.opt.verbose) {
-      console.error(chalk.blue('  done'))
+      console.error(chalk.blue('Done resolving'))
       console.error()
     }
-
-    if (this.opt.verbose) {
-      console.error(chalk.blue('Will resolve queues again in ' + this.resolveSeconds + ' seconds'))
-    }
-    this.resolveTimeout = setTimeout(() => {
-      this.resolvePromise = this.resolveQueues()
-    }, this.resolveSeconds * 1000)
   }
 
   // Return the next queue in the lineup
@@ -147,6 +146,9 @@ export class QueueManager {
   async shutdown () {
     this.shutdownRequested = true
     clearTimeout(this.resolveTimeout)
+    if (this.opt.verbose) {
+      console.error(chalk.blue('Waiting for queues to resolve'))
+    }
     await this.resolvePromise
   }
 }