qdone 2.0.33-alpha → 2.0.34-alpha

package/commonjs/src/consumer.js

@@ -61,7 +61,7 @@ async function processMessages(queues, callback, options) {
         lastLatency = latency;
     });
     const jobExecutor = new jobExecutor_js_1.JobExecutor(opt);
-    const queueManager = new queueManager_js_1.QueueManager(opt, queues, 60);
+    const queueManager = new queueManager_js_1.QueueManager(opt, queues, 10);
     const cores = (0, os_1.cpus)().length;
     // debug({ systemMonitor, jobExecutor, queueManager })
     // This delay function keeps a timeout reference around so it can be
@@ -101,8 +101,7 @@ async function processMessages(queues, callback, options) {
                 }
             }
             // Max job accounting
-            if (messages.length)
-                maxReturnCount -= messages.length;
+            maxReturnCount -= maxMessages;
             activeQrls.delete(qrl);
         }
         catch (e) {
@@ -117,7 +116,8 @@ async function processMessages(queues, callback, options) {
     };
     while (!shutdownRequested) { // eslint-disable-line
         // Figure out how we are running
-        const allowedJobs = Math.max(0, opt.maxConcurrentJobs - jobExecutor.runningJobCount() - maxReturnCount);
+        const runningJobs = jobExecutor.runningJobCount();
+        const allowedJobs = Math.max(0, opt.maxConcurrentJobs - maxReturnCount - runningJobs);
         // Latency
         const maxLatency = 100;
         const latency = systemMonitor.getLatency() || 10;
@@ -137,12 +137,14 @@ async function processMessages(queues, callback, options) {
         const targetJobs = Math.round(allowedJobs * overallFactor);
         let jobsLeft = targetJobs;
         if (opt.verbose) {
-            // console.error({ maxConcurrentJobs: opt.maxConcurrentJobs, jobCount: jobExecutor.activeJobCount(), allowedJobs, maxLatency, latencyFactor, freememFactor, loadFactor, overallFactor, targetJobs, activeQrls })
+            console.error({ maxConcurrentJobs: opt.maxConcurrentJobs, maxReturnCount, runningJobs, allowedJobs, maxLatency, latencyFactor, freememFactor, loadFactor, overallFactor, targetJobs, activeQrls });
         }
         for (const { qname, qrl } of queueManager.getPairs()) {
             // const qcount = jobExecutor.runningJobCountForQueue(qname)
             // console.log({ evaluating: { qname, qrl, qcount, jobsLeft, activeQrlsHasQrl: activeQrls.has(qrl) } })
-            if (jobsLeft <= 0 || activeQrls.has(qrl))
+            if (jobsLeft <= 0)
+                break;
+            if (activeQrls.has(qrl))
                 continue;
             const maxMessages = Math.min(10, jobsLeft);
             listen(qname, qrl, maxMessages);

package/commonjs/src/scheduler/queueManager.js

@@ -146,6 +146,7 @@ class QueueManager {
     }
     getPairs() {
         const now = new Date();
+        this.selectedPairs.sort(() => 0.5 - Math.random());
         return this.selectedPairs.filter(({ qname, qrl }) => !this.keepInIcehouse(qrl, now));
     }
     async shutdown() {

package/npm-shrinkwrap.json

@@ -1,12 +1,12 @@
 {
   "name": "qdone",
-  "version": "2.0.33-alpha",
+  "version": "2.0.34-alpha",
   "lockfileVersion": 2,
   "requires": true,
   "packages": {
     "": {
       "name": "qdone",
-      "version": "2.0.33-alpha",
+      "version": "2.0.34-alpha",
       "license": "MIT",
       "dependencies": {
         "@aws-sdk/client-cloudwatch": "3.465.0",

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "qdone",
-  "version": "2.0.33-alpha",
+  "version": "2.0.34-alpha",
   "description": "Language agnostic job queue for SQS",
   "type": "module",
   "main": "./index.js",

package/src/consumer.js

@@ -61,7 +61,7 @@ export async function processMessages (queues, callback, options) {
     lastLatency = latency
   })
   const jobExecutor = new JobExecutor(opt)
-  const queueManager = new QueueManager(opt, queues, 60)
+  const queueManager = new QueueManager(opt, queues, 10)
   const cores = cpus().length
   // debug({ systemMonitor, jobExecutor, queueManager })
 
@@ -105,7 +105,7 @@ export async function processMessages (queues, callback, options) {
       }
 
       // Max job accounting
-      if (messages.length) maxReturnCount -= messages.length
+      maxReturnCount -= maxMessages
       activeQrls.delete(qrl)
     } catch (e) {
       // If the queue has been cleaned up, we should back off anyway
@@ -119,7 +119,8 @@ export async function processMessages (queues, callback, options) {
 
   while (!shutdownRequested) { // eslint-disable-line
     // Figure out how we are running
-    const allowedJobs = Math.max(0, opt.maxConcurrentJobs - jobExecutor.runningJobCount() - maxReturnCount)
+    const runningJobs = jobExecutor.runningJobCount()
+    const allowedJobs = Math.max(0, opt.maxConcurrentJobs - maxReturnCount - runningJobs)
 
     // Latency
     const maxLatency = 100
@@ -144,12 +145,13 @@ export async function processMessages (queues, callback, options) {
     let jobsLeft = targetJobs
 
     if (opt.verbose) {
-      // console.error({ maxConcurrentJobs: opt.maxConcurrentJobs, jobCount: jobExecutor.activeJobCount(), allowedJobs, maxLatency, latencyFactor, freememFactor, loadFactor, overallFactor, targetJobs, activeQrls })
+      console.error({ maxConcurrentJobs: opt.maxConcurrentJobs, maxReturnCount, runningJobs, allowedJobs, maxLatency, latencyFactor, freememFactor, loadFactor, overallFactor, targetJobs, activeQrls })
     }
     for (const { qname, qrl } of queueManager.getPairs()) {
       // const qcount = jobExecutor.runningJobCountForQueue(qname)
       // console.log({ evaluating: { qname, qrl, qcount, jobsLeft, activeQrlsHasQrl: activeQrls.has(qrl) } })
-      if (jobsLeft <= 0 || activeQrls.has(qrl)) continue
+      if (jobsLeft <= 0) break
+      if (activeQrls.has(qrl)) continue
       const maxMessages = Math.min(10, jobsLeft)
       listen(qname, qrl, maxMessages)
       jobsLeft -= maxMessages

package/src/scheduler/queueManager.js

@@ -149,6 +149,7 @@ export class QueueManager {
 
   getPairs () {
     const now = new Date()
+    this.selectedPairs.sort(() => 0.5 - Math.random())
     return this.selectedPairs.filter(({ qname, qrl }) => !this.keepInIcehouse(qrl, now))
   }