qdone 2.0.26-alpha → 2.0.28-alpha

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "qdone",
-  "version": "2.0.26-alpha",
+  "version": "2.0.28-alpha",
   "description": "Language agnostic job queue for SQS",
   "type": "module",
   "main": "./index.js",

package/src/consumer.js

@@ -86,18 +86,24 @@ export async function processMessages (queues, callback, options) {
   const activeQrls = new Set()
   let maxReturnCount = 0
   const listen = async (qname, qrl, maxMessages) => {
+    if (opt.verbose) {
+      console.error(chalk.blue('Listening on: '), qname)
+    }
     activeQrls.add(qrl)
     maxReturnCount += maxMessages
     try {
       const messages = await getMessages(qrl, opt, maxMessages)
-      if (messages.length) {
-        for (const message of messages) {
-          jobExecutor.executeJob(message, callback, qname, qrl, () => queueManager.updateIcehouse(qrl, true))
+
+      if (!shutdownRequested) {
+        if (messages.length) {
+          for (const message of messages) {
+            jobExecutor.executeJob(message, callback, qname, qrl, () => queueManager.updateIcehouse(qrl, true))
+          }
+          queueManager.updateIcehouse(qrl, false)
+        } else {
+          // If we didn't get any, update the icehouse so we can back off
+          queueManager.updateIcehouse(qrl, true)
         }
-        queueManager.updateIcehouse(qrl, false)
-      } else {
-        // If we didn't get any, update the icehouse so we can back off
-        queueManager.updateIcehouse(qrl, true)
       }
 
       // Max job accounting
@@ -138,16 +144,16 @@ export async function processMessages (queues, callback, options) {
     const overallFactor = Math.min(latencyFactor, freememFactor, loadFactor)
     const targetJobs = Math.round(allowedJobs * overallFactor)
     let jobsLeft = targetJobs
-    debug({ jobCount: jobExecutor.activeJobCount(), freeMemory, totalMemory, freememThreshold, remainingMemory, memoryThreshold, maxReturnCount, allowedJobs, maxLatency, latency, latencyFactor, freememFactor, oneMinuteLoad, loadPerCore, loadFactor, overallFactor, targetJobs, activeQrls })
+
+    if (opt.verbose) {
+      console.error({ jobCount: jobExecutor.activeJobCount(), freeMemory, totalMemory, freememThreshold, remainingMemory, memoryThreshold, maxReturnCount, allowedJobs, maxLatency, latency, latencyFactor, freememFactor, oneMinuteLoad, loadPerCore, loadFactor, overallFactor, targetJobs, activeQrls })
+    }
     for (const { qname, qrl } of queueManager.getPairs()) {
       // debug({ evaluating: { qname, qrl, jobsLeft, activeQrlsHasQrl: activeQrls.has(qrl) } })
       if (jobsLeft <= 0 || activeQrls.has(qrl)) continue
       const maxMessages = Math.min(10, jobsLeft)
       listen(qname, qrl, maxMessages)
       jobsLeft -= maxMessages
-      if (opt.verbose) {
-        console.error(chalk.blue('Listening on: '), qname)
-      }
       // debug({ listenedTo: { qname, maxMessages, jobsLeft } })
     }
     await delay(1000)

package/src/scheduler/jobExecutor.js

@@ -35,6 +35,9 @@ export class JobExecutor {
     this.shutdownRequested = true
     // Trigger a maintenance run right away in case it speeds us up
     clearTimeout(this.maintainVisibilityTimeout)
+    if (this.opt.verbose) {
+      console.error(chalk.blue('Shutting down jobExecutor'))
+    }
     await this.maintainPromise
     await this.maintainVisibility()
   }
@@ -47,7 +50,21 @@ export class JobExecutor {
    * Changes message visibility on all running jobs using as few calls as possible.
    */
   async maintainVisibility () {
+    // Bail if we are shutting down
+    if (this.shutdownRequested && this.stats.activeJobs === 0 && this.jobs.length === 0) {
+      if (this.opt.verbose) {
+        console.error(chalk.blue('All workers done, finishing shutdown of jobExecutor'))
+      }
+      return
+    }
+
+    // Reset our timeout
     clearTimeout(this.maintainVisibilityTimeout)
+    const nextCheckInMs = this.shutdownRequested ? 1000 : 10 * 1000
+    this.maintainVisibilityTimeout = setTimeout(() => {
+      this.maintainPromise = this.maintainVisibility()
+    }, nextCheckInMs)
+
     // debug('maintainVisibility', this.jobs)
     const start = new Date()
     const jobsToExtendByQrl = {}
@@ -84,7 +101,7 @@ export class JobExecutor {
           const secondsUntilMax = Math.max(1, maxJobSeconds - jobRunTime)
           // const secondsUntilKill = Math.max(1, this.opt.killAfter - jobRunTime)
           job.visibilityTimeout = Math.min(doubled, secondsUntilMax) //, secondsUntilKill)
-          job.extendAtSecond = Math.round(jobRunTime + job.visibilityTimeout) // this is what we use next time
+          job.extendAtSecond = Math.round(jobRunTime + job.visibilityTimeout / 2) // this is what we use next time
           debug({ doubled, secondsUntilMax, job })
         }
       }
@@ -102,7 +119,7 @@ export class JobExecutor {
         while (messageId++ < 10 && jobsToExtend.length) {
           const job = jobsToExtend.shift()
           const entry = {
-            Id: '' + messageId,
+            Id: job.message.MessageId,
             ReceiptHandle: job.message.ReceiptHandle,
             VisibilityTimeout: job.visibilityTimeout
           }
@@ -116,6 +133,12 @@ export class JobExecutor {
         const result = await getSQSClient().send(new ChangeMessageVisibilityBatchCommand(input))
         debug('ChangeMessageVisibilityBatch returned', result)
         this.stats.sqsCalls++
+        if (result.Failed) {
+          console.error('FAILED_MESSAGES', result.Failed)
+          for (const failed of result.Failed) {
+            console.error('FAILED_TO_EXTEND_JOB', this.jobsByMessageId[failed.Id])
+          }
+        }
         if (result.Successful) {
           const count = result.Successful.length || 0
           this.stats.timeoutsExtended += count
@@ -139,7 +162,7 @@ export class JobExecutor {
         while (messageId++ < 10 && jobsToDelete.length) {
           const job = jobsToDelete.shift()
           const entry = {
-            Id: '' + messageId,
+            Id: job.message.MessageId,
             ReceiptHandle: job.message.ReceiptHandle,
             VisibilityTimeout: job.visibilityTimeout
           }
@@ -152,6 +175,12 @@ export class JobExecutor {
         debug({ DeleteMessageBatch: input })
         const result = await getSQSClient().send(new DeleteMessageBatchCommand(input))
         this.stats.sqsCalls++
+        if (result.Failed) {
+          console.error('FAILED_MESSAGES', result.Failed)
+          for (const failed of result.Failed) {
+            console.error('FAILED_TO_DELETE_JOB', this.jobsByMessageId[failed.Id])
+          }
+        }
         if (result.Successful) {
           const count = result.Successful.length || 0
           this.stats.jobsDeleted += count
@@ -176,24 +205,10 @@ export class JobExecutor {
         return true
       }
     })
-
-    // Bail if we are shutting down
-    if (this.shutdownRequested && this.stats.activeJobs === 0 && this.jobs.length === 0) return
-
-    // Check later, but count the time we spent. Make sure we check at least
-    // every period seconds.
-    const msElapsed = new Date() - start
-    const msPeriod = this.shutdownRequested ? 1 * 1000 : 10 * 1000
-    const msLeft = Math.max(0, msPeriod - msElapsed)
-    const msMin = this.shutdownRequested ? 1000 : 0
-    const nextCheckInMs = Math.max(msMin, msLeft)
-    debug({ msElapsed, msPeriod, msLeft, msMin, nextCheckInMs })
-    this.maintainVisibilityTimeout = setTimeout(() => {
-      this.maintainPromise = this.maintainVisibility()
-    }, nextCheckInMs)
   }
 
   async executeJob (message, callback, qname, qrl, failedCallback) {
+    if (this.shutdownRequested) throw new Error('jobExecutor is shutting down so cannot execute new job')
     // Create job entry and track it
     const payload = this.opt.json ? JSON.parse(message.Body) : message.Body
     const visibilityTimeout = 60

package/src/scheduler/queueManager.js

@@ -69,6 +69,12 @@ export class QueueManager {
   async resolveQueues () {
     clearTimeout(this.resolveTimeout)
     if (this.shutdownRequested) return
+    this.resolveTimeout = setTimeout(() => {
+      this.resolvePromise = this.resolveQueues()
+    }, this.resolveSeconds * 1000)
+    if (this.opt.verbose) {
+      console.error(chalk.blue('Will resolve queues again in ' + this.resolveSeconds + ' seconds'))
+    }
 
     // Start processing
     const qnames = this.queues.map(queue => normalizeQueueName(queue, this.opt))
@@ -120,16 +126,9 @@ export class QueueManager {
 
     // Finished resolving
     if (this.opt.verbose) {
-      console.error(chalk.blue('  done'))
+      console.error(chalk.blue('Done resolving'))
       console.error()
     }
-
-    if (this.opt.verbose) {
-      console.error(chalk.blue('Will resolve queues again in ' + this.resolveSeconds + ' seconds'))
-    }
-    this.resolveTimeout = setTimeout(() => {
-      this.resolvePromise = this.resolveQueues()
-    }, this.resolveSeconds * 1000)
   }
 
   // Return the next queue in the lineup
@@ -147,6 +146,9 @@ export class QueueManager {
   async shutdown () {
     this.shutdownRequested = true
     clearTimeout(this.resolveTimeout)
+    if (this.opt.verbose) {
+      console.error(chalk.blue('Waiting for queues to resolve'))
+    }
     await this.resolvePromise
   }
 }