qdone 2.0.24-alpha → 2.0.26-alpha

package/commonjs/src/consumer.js

@@ -118,7 +118,7 @@ exports.getMessages = getMessages;
 //
 function processMessages(queues, callback, options) {
     return __awaiter(this, void 0, void 0, function () {
-        var opt, lastLatency, systemMonitor, jobExecutor, queueManager, delayTimeout, delay, activeQrls, maxReturnCount, listen, allowedJobs, maxLatency, freeMemory, totalMemory, remainingMemory, memoryThreshold, latency, latencyFactor, freememFactor, targetJobs, jobsLeft, _i, _a, _b, qname, qrl, maxMessages;
+        var opt, lastLatency, systemMonitor, jobExecutor, queueManager, cores, delayTimeout, delay, activeQrls, maxReturnCount, listen, allowedJobs, maxLatency, latency, latencyFactor, freeMemory, totalMemory, memoryThreshold, freememThreshold, remainingMemory, freememFactor, oneMinuteLoad, loadPerCore, loadFactor, overallFactor, targetJobs, jobsLeft, _i, _a, _b, qname, qrl, maxMessages;
         var _this = this;
         return __generator(this, function (_c) {
             switch (_c.label) {
@@ -136,6 +136,7 @@ function processMessages(queues, callback, options) {
                     });
                     jobExecutor = new jobExecutor_js_1.JobExecutor(opt);
                     queueManager = new queueManager_js_1.QueueManager(opt, queues, 60);
+                    cores = (0, os_1.cpus)().length;
                     delay = function (ms) { return new Promise(function (resolve) {
                         delayTimeout = setTimeout(resolve, ms);
                     }); };
@@ -209,17 +210,22 @@ function processMessages(queues, callback, options) {
                     if (!!shutdownRequested) return [3 /*break*/, 3];
                     allowedJobs = Math.max(0, opt.maxConcurrentJobs - jobExecutor.activeJobCount() - maxReturnCount);
                     maxLatency = 100;
-                    freeMemory = (0, os_1.freemem)();
-                    totalMemory = (0, os_1.totalmem)();
-                    remainingMemory = totalMemory - freeMemory;
-                    memoryThreshold = totalMemory * opt.maxMemoryPercent / 100;
                     latency = systemMonitor.getLatency() || 10;
                     latencyFactor = 1 - Math.abs(Math.min(latency / maxLatency, 1)) // 0 if latency is at max, 1 if latency 0
                     ;
+                    freeMemory = (0, os_1.freemem)();
+                    totalMemory = (0, os_1.totalmem)();
+                    memoryThreshold = totalMemory * opt.maxMemoryPercent / 100;
+                    freememThreshold = totalMemory - memoryThreshold;
+                    remainingMemory = Math.max(0, freeMemory - freememThreshold);
                     freememFactor = Math.min(1, Math.max(0, remainingMemory / memoryThreshold));
-                    targetJobs = Math.round(allowedJobs * latencyFactor * freememFactor);
+                    oneMinuteLoad = (0, os_1.loadavg)()[0];
+                    loadPerCore = oneMinuteLoad / cores;
+                    loadFactor = 1 - Math.min(1, Math.max(0, loadPerCore / 3));
+                    overallFactor = Math.min(latencyFactor, freememFactor, loadFactor);
+                    targetJobs = Math.round(allowedJobs * overallFactor);
                     jobsLeft = targetJobs;
-                    debug({ jobCount: jobExecutor.activeJobCount(), freeMemory: freeMemory, totalMemory: totalMemory, remainingMemory: remainingMemory, memoryThreshold: memoryThreshold, maxReturnCount: maxReturnCount, allowedJobs: allowedJobs, maxLatency: maxLatency, latency: latency, latencyFactor: latencyFactor, freememFactor: freememFactor, targetJobs: targetJobs, activeQrls: activeQrls });
+                    debug({ jobCount: jobExecutor.activeJobCount(), freeMemory: freeMemory, totalMemory: totalMemory, freememThreshold: freememThreshold, remainingMemory: remainingMemory, memoryThreshold: memoryThreshold, maxReturnCount: maxReturnCount, allowedJobs: allowedJobs, maxLatency: maxLatency, latency: latency, latencyFactor: latencyFactor, freememFactor: freememFactor, oneMinuteLoad: oneMinuteLoad, loadPerCore: loadPerCore, loadFactor: loadFactor, overallFactor: overallFactor, targetJobs: targetJobs, activeQrls: activeQrls });
                     for (_i = 0, _a = queueManager.getPairs(); _i < _a.length; _i++) {
                         _b = _a[_i], qname = _b.qname, qrl = _b.qrl;
                         // debug({ evaluating: { qname, qrl, jobsLeft, activeQrlsHasQrl: activeQrls.has(qrl) } })

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "qdone",
-  "version": "2.0.24-alpha",
+  "version": "2.0.26-alpha",
   "description": "Language agnostic job queue for SQS",
   "type": "module",
   "main": "./index.js",

package/src/consumer.js

@@ -2,7 +2,7 @@
  * Consumer implementation.
  */
 
-import { freemem, totalmem } from 'os'
+import { freemem, totalmem, loadavg, cpus } from 'os'
 import { ReceiveMessageCommand, QueueDoesNotExist } from '@aws-sdk/client-sqs'
 import chalk from 'chalk'
 import Debug from 'debug'
@@ -62,6 +62,7 @@ export async function processMessages (queues, callback, options) {
   })
   const jobExecutor = new JobExecutor(opt)
   const queueManager = new QueueManager(opt, queues, 60)
+  const cores = cpus().length
   // debug({ systemMonitor, jobExecutor, queueManager })
 
   // This delay function keeps a timeout reference around so it can be
@@ -115,17 +116,29 @@ export async function processMessages (queues, callback, options) {
   while (!shutdownRequested) { // eslint-disable-line
     // Figure out how we are running
     const allowedJobs = Math.max(0, opt.maxConcurrentJobs - jobExecutor.activeJobCount() - maxReturnCount)
+
+    // Latency
     const maxLatency = 100
+    const latency = systemMonitor.getLatency() || 10
+    const latencyFactor = 1 - Math.abs(Math.min(latency / maxLatency, 1)) // 0 if latency is at max, 1 if latency 0
+
+    // Memory
     const freeMemory = freemem()
     const totalMemory = totalmem()
-    const remainingMemory = totalMemory - freeMemory
     const memoryThreshold = totalMemory * opt.maxMemoryPercent / 100
-    const latency = systemMonitor.getLatency() || 10
-    const latencyFactor = 1 - Math.abs(Math.min(latency / maxLatency, 1)) // 0 if latency is at max, 1 if latency 0
+    const freememThreshold = totalMemory - memoryThreshold
+    const remainingMemory = Math.max(0, freeMemory - freememThreshold)
     const freememFactor = Math.min(1, Math.max(0, remainingMemory / memoryThreshold))
-    const targetJobs = Math.round(allowedJobs * latencyFactor * freememFactor)
+
+    // Load
+    const oneMinuteLoad = loadavg()[0]
+    const loadPerCore = oneMinuteLoad / cores
+    const loadFactor = 1 - Math.min(1, Math.max(0, loadPerCore / 3))
+
+    const overallFactor = Math.min(latencyFactor, freememFactor, loadFactor)
+    const targetJobs = Math.round(allowedJobs * overallFactor)
     let jobsLeft = targetJobs
-    debug({ jobCount: jobExecutor.activeJobCount(), freeMemory, totalMemory, remainingMemory, memoryThreshold, maxReturnCount, allowedJobs, maxLatency, latency, latencyFactor, freememFactor, targetJobs, activeQrls })
+    debug({ jobCount: jobExecutor.activeJobCount(), freeMemory, totalMemory, freememThreshold, remainingMemory, memoryThreshold, maxReturnCount, allowedJobs, maxLatency, latency, latencyFactor, freememFactor, oneMinuteLoad, loadPerCore, loadFactor, overallFactor, targetJobs, activeQrls })
     for (const { qname, qrl } of queueManager.getPairs()) {
       // debug({ evaluating: { qname, qrl, jobsLeft, activeQrlsHasQrl: activeQrls.has(qrl) } })
       if (jobsLeft <= 0 || activeQrls.has(qrl)) continue