qdone 2.0.32-alpha → 2.0.34-alpha

package/commonjs/src/consumer.js

@@ -61,7 +61,7 @@ async function processMessages(queues, callback, options) {
         lastLatency = latency;
     });
     const jobExecutor = new jobExecutor_js_1.JobExecutor(opt);
-    const queueManager = new queueManager_js_1.QueueManager(opt, queues, 60);
+    const queueManager = new queueManager_js_1.QueueManager(opt, queues, 10);
     const cores = (0, os_1.cpus)().length;
     // debug({ systemMonitor, jobExecutor, queueManager })
     // This delay function keeps a timeout reference around so it can be
@@ -101,8 +101,7 @@ async function processMessages(queues, callback, options) {
                 }
             }
             // Max job accounting
-            if (messages.length)
-                maxReturnCount -= messages.length;
+            maxReturnCount -= maxMessages;
             activeQrls.delete(qrl);
         }
         catch (e) {
@@ -117,7 +116,8 @@ async function processMessages(queues, callback, options) {
     };
     while (!shutdownRequested) { // eslint-disable-line
         // Figure out how we are running
-        const allowedJobs = Math.max(0, opt.maxConcurrentJobs - jobExecutor.runningJobCount() - maxReturnCount);
+        const runningJobs = jobExecutor.runningJobCount();
+        const allowedJobs = Math.max(0, opt.maxConcurrentJobs - maxReturnCount - runningJobs);
         // Latency
         const maxLatency = 100;
         const latency = systemMonitor.getLatency() || 10;
@@ -137,12 +137,14 @@ async function processMessages(queues, callback, options) {
         const targetJobs = Math.round(allowedJobs * overallFactor);
         let jobsLeft = targetJobs;
         if (opt.verbose) {
-            // console.error({ maxConcurrentJobs: opt.maxConcurrentJobs, jobCount: jobExecutor.activeJobCount(), allowedJobs, maxLatency, latencyFactor, freememFactor, loadFactor, overallFactor, targetJobs, activeQrls })
+            console.error({ maxConcurrentJobs: opt.maxConcurrentJobs, maxReturnCount, runningJobs, allowedJobs, maxLatency, latencyFactor, freememFactor, loadFactor, overallFactor, targetJobs, activeQrls });
         }
         for (const { qname, qrl } of queueManager.getPairs()) {
             // const qcount = jobExecutor.runningJobCountForQueue(qname)
             // console.log({ evaluating: { qname, qrl, qcount, jobsLeft, activeQrlsHasQrl: activeQrls.has(qrl) } })
-            if (jobsLeft <= 0 || activeQrls.has(qrl))
+            if (jobsLeft <= 0)
+                break;
+            if (activeQrls.has(qrl))
                 continue;
             const maxMessages = Math.min(10, jobsLeft);
             listen(qname, qrl, maxMessages);

package/commonjs/src/scheduler/jobExecutor.js

@@ -40,10 +40,8 @@ class JobExecutor {
         if (this.opt.verbose) {
             console.error(chalk_1.default.blue('Shutting down jobExecutor'));
         }
-        while (this.stats.activeJobs > 0) {
-            await this.maintainPromise;
-            await this.maintainVisibility();
-        }
+        await this.maintainPromise;
+        await this.maintainVisibility();
     }
     activeJobCount() {
         return this.stats.activeJobs;
@@ -122,19 +120,18 @@ class JobExecutor {
             console.error(chalk_1.default.blue('Jobs: '));
             for (const [qname, jobs] of this.jobsByQueue.entries()) {
                 if (jobs.size) {
-                    console.error(chalk_1.default.blue('  queue:'), qname);
-                    for (const job of jobs)
-                        if (job.status === 'running')
-                            console.error(chalk_1.default.green('    running: '), job.payload);
+                    const stats = {};
                     for (const job of jobs)
-                        if (job.status === 'waiting')
-                            console.error(chalk_1.default.gold('    waiting:'), job.payload);
-                    for (const job of jobs)
-                        if (job.status === 'complete' || job.status === 'deleting')
-                            console.error(chalk_1.default.blue('    complete:'), job.payload);
-                    for (const job of jobs)
-                        if (job.status === 'failed')
-                            console.error(chalk_1.default.red('    failed:'), job.payload);
+                        stats[job.status] = (stats[job.status] || 0) + 1;
+                    console.error(chalk_1.default.blue('  queue:'), qname);
+                    if (stats.running)
+                        console.error(chalk_1.default.green('    running: '), stats.running);
+                    if (stats.waiting)
+                        console.error(chalk_1.default.yellow('    waiting: '), stats.waiting);
+                    if (stats.deleting)
+                        console.error(chalk_1.default.blue('    complete: '), stats.deleting);
+                    if (stats.failed)
+                        console.error(chalk_1.default.red('    failed: '), stats.failed);
                 }
             }
         }

package/commonjs/src/scheduler/queueManager.js

@@ -146,6 +146,7 @@ class QueueManager {
     }
     getPairs() {
         const now = new Date();
+        this.selectedPairs.sort(() => 0.5 - Math.random());
         return this.selectedPairs.filter(({ qname, qrl }) => !this.keepInIcehouse(qrl, now));
     }
     async shutdown() {

package/npm-shrinkwrap.json

@@ -1,12 +1,12 @@
 {
   "name": "qdone",
-  "version": "2.0.32-alpha",
+  "version": "2.0.34-alpha",
   "lockfileVersion": 2,
   "requires": true,
   "packages": {
     "": {
       "name": "qdone",
-      "version": "2.0.32-alpha",
+      "version": "2.0.34-alpha",
       "license": "MIT",
       "dependencies": {
         "@aws-sdk/client-cloudwatch": "3.465.0",

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "qdone",
-  "version": "2.0.32-alpha",
+  "version": "2.0.34-alpha",
   "description": "Language agnostic job queue for SQS",
   "type": "module",
   "main": "./index.js",

package/src/consumer.js

@@ -61,7 +61,7 @@ export async function processMessages (queues, callback, options) {
     lastLatency = latency
   })
   const jobExecutor = new JobExecutor(opt)
-  const queueManager = new QueueManager(opt, queues, 60)
+  const queueManager = new QueueManager(opt, queues, 10)
   const cores = cpus().length
   // debug({ systemMonitor, jobExecutor, queueManager })
 
@@ -105,7 +105,7 @@ export async function processMessages (queues, callback, options) {
       }
 
       // Max job accounting
-      if (messages.length) maxReturnCount -= messages.length
+      maxReturnCount -= maxMessages
       activeQrls.delete(qrl)
     } catch (e) {
       // If the queue has been cleaned up, we should back off anyway
@@ -119,7 +119,8 @@ export async function processMessages (queues, callback, options) {
 
   while (!shutdownRequested) { // eslint-disable-line
     // Figure out how we are running
-    const allowedJobs = Math.max(0, opt.maxConcurrentJobs - jobExecutor.runningJobCount() - maxReturnCount)
+    const runningJobs = jobExecutor.runningJobCount()
+    const allowedJobs = Math.max(0, opt.maxConcurrentJobs - maxReturnCount - runningJobs)
 
     // Latency
     const maxLatency = 100
@@ -144,12 +145,13 @@ export async function processMessages (queues, callback, options) {
     let jobsLeft = targetJobs
 
     if (opt.verbose) {
-      // console.error({ maxConcurrentJobs: opt.maxConcurrentJobs, jobCount: jobExecutor.activeJobCount(), allowedJobs, maxLatency, latencyFactor, freememFactor, loadFactor, overallFactor, targetJobs, activeQrls })
+      console.error({ maxConcurrentJobs: opt.maxConcurrentJobs, maxReturnCount, runningJobs, allowedJobs, maxLatency, latencyFactor, freememFactor, loadFactor, overallFactor, targetJobs, activeQrls })
     }
     for (const { qname, qrl } of queueManager.getPairs()) {
       // const qcount = jobExecutor.runningJobCountForQueue(qname)
       // console.log({ evaluating: { qname, qrl, qcount, jobsLeft, activeQrlsHasQrl: activeQrls.has(qrl) } })
-      if (jobsLeft <= 0 || activeQrls.has(qrl)) continue
+      if (jobsLeft <= 0) break
+      if (activeQrls.has(qrl)) continue
       const maxMessages = Math.min(10, jobsLeft)
       listen(qname, qrl, maxMessages)
       jobsLeft -= maxMessages

package/src/scheduler/jobExecutor.js

@@ -41,10 +41,8 @@ export class JobExecutor {
     if (this.opt.verbose) {
       console.error(chalk.blue('Shutting down jobExecutor'))
     }
-    while (this.stats.activeJobs > 0) {
-      await this.maintainPromise
-      await this.maintainVisibility()
-    }
+    await this.maintainPromise
+    await this.maintainVisibility()
   }
 
   activeJobCount () {
@@ -129,11 +127,13 @@ export class JobExecutor {
       console.error(chalk.blue('Jobs: '))
       for (const [qname, jobs] of this.jobsByQueue.entries()) {
         if (jobs.size) {
+          const stats = {}
+          for (const job of jobs) stats[job.status] = (stats[job.status] || 0) + 1
           console.error(chalk.blue('  queue:'), qname)
-          for (const job of jobs) if (job.status === 'running') console.error(chalk.green('    running: '), job.payload)
-          for (const job of jobs) if (job.status === 'waiting') console.error(chalk.gold('    waiting:'), job.payload)
-          for (const job of jobs) if (job.status === 'complete' || job.status === 'deleting') console.error(chalk.blue('    complete:'), job.payload)
-          for (const job of jobs) if (job.status === 'failed') console.error(chalk.red('    failed:'), job.payload)
+          if (stats.running) console.error(chalk.green('    running: '), stats.running)
+          if (stats.waiting) console.error(chalk.yellow('    waiting: '), stats.waiting)
+          if (stats.deleting) console.error(chalk.blue('    complete: '), stats.deleting)
+          if (stats.failed) console.error(chalk.red('    failed: '), stats.failed)
         }
       }
     }

package/src/scheduler/queueManager.js

@@ -149,6 +149,7 @@ export class QueueManager {
 
   getPairs () {
     const now = new Date()
+    this.selectedPairs.sort(() => 0.5 - Math.random())
     return this.selectedPairs.filter(({ qname, qrl }) => !this.keepInIcehouse(qrl, now))
   }