qdone 2.0.53-alpha → 2.0.55-alpha

package/commonjs/src/consumer.js

@@ -80,19 +80,27 @@ async function processMessages(queues, callback, options) {
         debug({ systemMonitor: 'done' });
     });
     // Keep track of how many messages could be returned from each queue
-    const activeQrls = new Set();
+    const activeQrls = new Map();
+    const listeningQrls = new Set();
     let maxReturnCount = 0;
     const listen = async (qname, qrl, maxMessages) => {
         if (opt.verbose) {
             console.error(chalk_1.default.blue('Listening on: '), qname);
         }
-        activeQrls.add(qrl);
         maxReturnCount += maxMessages;
         try {
+            listeningQrls.add(qrl);
             const messages = await getMessages(qrl, opt, maxMessages);
+            listeningQrls.delete(qrl);
             if (!shutdownRequested) {
                 if (messages.length) {
-                    jobExecutor.executeJobs(messages, callback, qname, qrl);
+                    activeQrls.set(qrl, (activeQrls.get(qrl) || 0) + 1);
+                    await jobExecutor.executeJobs(messages, callback, qname, qrl);
+                    const count = activeQrls.get(qrl) - 1;
+                    if (count)
+                        activeQrls.set(qrl, count);
+                    else
+                        activeQrls.delete(qrl);
                     queueManager.updateIcehouse(qrl, false);
                 }
                 else {
@@ -102,7 +110,6 @@ async function processMessages(queues, callback, options) {
             }
             // Max job accounting
             maxReturnCount -= maxMessages;
-            activeQrls.delete(qrl);
         }
         catch (e) {
             // If the queue has been cleaned up, we should back off anyway
@@ -114,6 +121,14 @@ async function processMessages(queues, callback, options) {
             }
         }
     };
+    if (opt.verbose) {
+        function printUrls() {
+            console.error({ activeQrls, listeningQrls });
+            if (!shutdownRequested)
+                setTimeout(printUrls, 2000);
+        }
+        printUrls();
+    }
     while (!shutdownRequested) { // eslint-disable-line
         // Figure out how we are running
         const runningJobs = jobExecutor.runningJobCount();
@@ -137,21 +152,21 @@ async function processMessages(queues, callback, options) {
         const targetJobs = Math.round(allowedJobs * overallFactor);
         let jobsLeft = targetJobs;
         if (opt.verbose) {
-            console.error({ maxConcurrentJobs: opt.maxConcurrentJobs, maxReturnCount, runningJobs, allowedJobs, maxLatency, latencyFactor, freememFactor, loadFactor, overallFactor, targetJobs, activeQrls });
+            console.error({ maxConcurrentJobs: opt.maxConcurrentJobs, maxReturnCount, runningJobs, allowedJobs, maxLatency, latencyFactor, freememFactor, loadFactor, overallFactor, targetJobs });
         }
         for (const { qname, qrl } of queueManager.getPairs()) {
             // const qcount = jobExecutor.runningJobCountForQueue(qname)
             // console.log({ evaluating: { qname, qrl, qcount, jobsLeft, activeQrlsHasQrl: activeQrls.has(qrl) } })
             if (jobsLeft <= 0)
                 break;
-            if (activeQrls.has(qrl))
+            if (listeningQrls.has(qrl))
                 continue;
             const maxMessages = Math.min(10, jobsLeft);
             listen(qname, qrl, maxMessages);
             jobsLeft -= maxMessages;
             // debug({ listenedTo: { qname, maxMessages, jobsLeft } })
         }
-        await delay(1000);
+        await delay(300);
     }
     debug('after all');
 }

package/commonjs/src/idleQueues.js

@@ -137,7 +137,7 @@ async function checkIdle(qname, qrl, opt) {
     if (cheapResult.idle === false || cheapResult.exists === false) {
         return {
             queue: qname.slice(opt.prefix.length),
-            cheap: cheapResult,
+            cheap: { SQS, result: cheapResult },
             idle: cheapResult.idle,
             exists: cheapResult.exists,
             apiCalls: { SQS, CloudWatch: 0 }

package/commonjs/src/scheduler/jobExecutor.js

@@ -363,6 +363,7 @@ class JobExecutor {
         // Begin tracking jobs
         const jobs = messages.map(message => this.addJob(message, callback, qname, qrl));
         const isFifo = qrl.endsWith('.fifo');
+        const runningJobs = [];
         // console.log(jobs)
         // Begin executing
         for (const [job, i] of jobs.map((job, i) => [job, i])) {
@@ -374,8 +375,9 @@ class JobExecutor {
             if (nextJobIsSerial)
                 await this.runJob(job);
             else
-                this.runJob(job);
+                runningJobs.push(this.runJob(job));
         }
+        await Promise.all(runningJobs);
     }
 }
 exports.JobExecutor = JobExecutor;

package/commonjs/src/scheduler/systemMonitor.js

@@ -71,7 +71,7 @@ class SystemMonitor {
      * second is better to reduce latency of detecting the change.
      */
     measureLoad() {
-        const [newLoad,] = os_1.default.loadavg();
+        const [newLoad] = os_1.default.loadavg();
         const previousLoad = this.oneMinuteLoad;
         if (previousLoad !== newLoad) {
             const e = 1884 / 2048; // see include/linux/sched/loadavg.h

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "qdone",
-  "version": "2.0.53-alpha",
+  "version": "2.0.55-alpha",
   "description": "A distributed scheduler for SQS",
   "type": "module",
   "main": "./index.js",

package/src/cli.js

@@ -451,19 +451,14 @@ export async function idleQueues (argv, testHook) {
     {
       content: [
         { count: '1 + q + i', desc: 'q: number of queues in pattern\ni: number of idle queues' },
-        { context: 'with --delete options', count: '1 + q + 3i', desc: 'q: number of queues in pattern\ni: number of idle queues' },
-        { context: 'with --unpair option', count: '1 + q', desc: 'q: number of queues in pattern' },
-        { context: 'with --unpair and --delete options', count: '1 + q + i', desc: 'q: number of queues in pattern\ni: number of idle queues' },
-        { desc: 'NOTE: the --unpair option not cheaper if you include fail queues, because it doubles q.' }
+        { context: 'with --delete options', count: '1 + q + 3i', desc: 'q: number of queues in pattern\ni: number of idle queues' }
       ],
       long: true
     },
     { content: 'CloudWatch API Call Complexity', raw: true, long: true },
     {
       content: [
-        { count: 'min: 0 (if queue and fail queue have waiting messages)\nmax: 12q\nexpected (approximate observed): 0.5q + 12i', desc: 'q: number of queues in pattern\ni: number of idle queues' },
-        { context: 'with --unpair option', count: 'min: 0 (if queue has waiting messages)\nmax: 6q\nexpected (approximate observed): q + 6i', desc: 'q: number of queues in pattern\ni: number of idle queues' },
-        { desc: 'NOTE: the --unpair option not cheaper if you include fail queues, because it doubles q.' }
+        { count: 'min: 0 (if queue and fail queue have waiting messages)\nmax: 12q\nexpected (approximate observed): 0.5q + 12i', desc: 'q: number of queues in pattern\ni: number of idle queues' }
       ],
       long: true
     },
@@ -479,7 +474,6 @@ export async function idleQueues (argv, testHook) {
     debug('idleQueues options', options)
     if (options.help) return Promise.resolve(console.log(getUsage(usageSections)))
     if (!options._unknown || options._unknown.length === 0) throw new UsageError('idle-queues requres one or more <queue> arguments')
-    if (options['include-failed'] && !options.unpair) throw new UsageError('--include-failed should be used with --unpair')
     if (options['idle-for'] < 5) throw new UsageError('--idle-for must be at least 5 minutes (CloudWatch limitation)')
     queues = options._unknown
     debug('queues', queues)

package/src/consumer.js

@@ -2,7 +2,7 @@
  * Consumer implementation.
  */
 
-import { freemem, totalmem, loadavg, cpus } from 'os'
+import { freemem, totalmem, cpus } from 'os'
 import { ReceiveMessageCommand, QueueDoesNotExist } from '@aws-sdk/client-sqs'
 import chalk from 'chalk'
 import Debug from 'debug'
@@ -83,20 +83,26 @@ export async function processMessages (queues, callback, options) {
   })
 
   // Keep track of how many messages could be returned from each queue
-  const activeQrls = new Set()
+  const activeQrls = new Map()
+  const listeningQrls = new Set()
   let maxReturnCount = 0
   const listen = async (qname, qrl, maxMessages) => {
     if (opt.verbose) {
       console.error(chalk.blue('Listening on: '), qname)
     }
-    activeQrls.add(qrl)
     maxReturnCount += maxMessages
     try {
+      listeningQrls.add(qrl)
       const messages = await getMessages(qrl, opt, maxMessages)
+      listeningQrls.delete(qrl)
 
       if (!shutdownRequested) {
         if (messages.length) {
-          jobExecutor.executeJobs(messages, callback, qname, qrl)
+          activeQrls.set(qrl, (activeQrls.get(qrl) || 0) + 1)
+          await jobExecutor.executeJobs(messages, callback, qname, qrl)
+          const count = activeQrls.get(qrl) - 1
+          if (count) activeQrls.set(qrl, count)
+          else activeQrls.delete(qrl)
           queueManager.updateIcehouse(qrl, false)
         } else {
           // If we didn't get any, update the icehouse so we can back off
@@ -106,7 +112,6 @@ export async function processMessages (queues, callback, options) {
 
       // Max job accounting
       maxReturnCount -= maxMessages
-      activeQrls.delete(qrl)
     } catch (e) {
       // If the queue has been cleaned up, we should back off anyway
       if (e instanceof QueueDoesNotExist) {
@@ -117,6 +122,14 @@ export async function processMessages (queues, callback, options) {
     }
   }
 
+  if (opt.verbose) {
+    function printUrls () {
+      console.error({ activeQrls, listeningQrls })
+      if (!shutdownRequested) setTimeout(printUrls, 2000)
+    }
+    printUrls()
+  }
+
   while (!shutdownRequested) { // eslint-disable-line
     // Figure out how we are running
     const runningJobs = jobExecutor.runningJobCount()
@@ -145,19 +158,19 @@ export async function processMessages (queues, callback, options) {
     let jobsLeft = targetJobs
 
     if (opt.verbose) {
-      console.error({ maxConcurrentJobs: opt.maxConcurrentJobs, maxReturnCount, runningJobs, allowedJobs, maxLatency, latencyFactor, freememFactor, loadFactor, overallFactor, targetJobs, activeQrls })
+      console.error({ maxConcurrentJobs: opt.maxConcurrentJobs, maxReturnCount, runningJobs, allowedJobs, maxLatency, latencyFactor, freememFactor, loadFactor, overallFactor, targetJobs })
     }
     for (const { qname, qrl } of queueManager.getPairs()) {
       // const qcount = jobExecutor.runningJobCountForQueue(qname)
       // console.log({ evaluating: { qname, qrl, qcount, jobsLeft, activeQrlsHasQrl: activeQrls.has(qrl) } })
       if (jobsLeft <= 0) break
-      if (activeQrls.has(qrl)) continue
+      if (listeningQrls.has(qrl)) continue
       const maxMessages = Math.min(10, jobsLeft)
       listen(qname, qrl, maxMessages)
       jobsLeft -= maxMessages
       // debug({ listenedTo: { qname, maxMessages, jobsLeft } })
     }
-    await delay(1000)
+    await delay(300)
   }
   debug('after all')
 }

package/src/idleQueues.js

@@ -7,7 +7,7 @@ import { getCloudWatchClient } from './cloudWatch.js'
 import { getOptionsWithDefaults } from './defaults.js'
 import { GetQueueAttributesCommand, DeleteQueueCommand, QueueDoesNotExist } from '@aws-sdk/client-sqs'
 import { GetMetricStatisticsCommand } from '@aws-sdk/client-cloudwatch'
-import { normalizeFailQueueName, normalizeDLQName, getQnameUrlPairs, fifoSuffix } from './qrlCache.js'
+import { normalizeFailQueueName, normalizeDLQName, getQnameUrlPairs, fifoSuffix, qrlCacheSet } from './qrlCache.js'
 import { getCache, setCache } from './cache.js'
 // const AWS = require('aws-sdk')
 
@@ -133,7 +133,7 @@ export async function checkIdle (qname, qrl, opt) {
   if (cheapResult.idle === false || cheapResult.exists === false) {
     return {
       queue: qname.slice(opt.prefix.length),
-      cheap: cheapResult,
+      cheap: { SQS, result: cheapResult },
       idle: cheapResult.idle,
       exists: cheapResult.exists,
       apiCalls: { SQS, CloudWatch: 0 }

package/src/scheduler/jobExecutor.js

@@ -373,6 +373,7 @@ export class JobExecutor {
     // Begin tracking jobs
     const jobs = messages.map(message => this.addJob(message, callback, qname, qrl))
     const isFifo = qrl.endsWith('.fifo')
+    const runningJobs = []
 
     // console.log(jobs)
 
@@ -385,7 +386,8 @@ export class JobExecutor {
       // console.log({ i, nextJobAtt: nextJob?.message?.Attributes, nextJobIsSerial })
       // Execute serial or parallel
       if (nextJobIsSerial) await this.runJob(job)
-      else this.runJob(job)
+      else runningJobs.push(this.runJob(job))
     }
+    await Promise.all(runningJobs)
   }
 }

package/src/scheduler/systemMonitor.js

@@ -71,7 +71,7 @@ export class SystemMonitor {
    */
 
   measureLoad () {
-    const [newLoad, ] = os.loadavg()
+    const [newLoad] = os.loadavg()
     const previousLoad = this.oneMinuteLoad
     if (previousLoad !== newLoad) {
       const e = 1884 / 2048 // see include/linux/sched/loadavg.h
@@ -84,7 +84,7 @@ export class SystemMonitor {
     }
   }
 
-  getLoad() {
+  getLoad () {
     return this.instantaneousLoad
   }