qdone 2.0.29-alpha → 2.0.31-alpha

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "qdone",
-  "version": "2.0.29-alpha",
+  "version": "2.0.31-alpha",
   "description": "Language agnostic job queue for SQS",
   "type": "module",
   "main": "./index.js",
@@ -47,7 +47,7 @@
   "scripts": {
     "start": "./src/bin.js",
     "test": "NODE_OPTIONS='--experimental-json-modules --experimental-vm-modules --no-warnings' jest",
-    "build": "tsc --allowJs index.js --outdir commonjs --esModuleInterop --module commonjs",
+    "build": "tsc --allowJs index.js --outdir commonjs --esModuleInterop --module commonjs  --target es2021 --strict --skipLibCheck --forceConsistentCasingInFileNames",
     "clean": "rm -rf commonjs/src commonjs/*.js coverage",
     "lint": "standard",
     "coverage": "nyc report --reporter=text-lcov | coveralls",

package/src/consumer.js

@@ -96,9 +96,7 @@ export async function processMessages (queues, callback, options) {
 
       if (!shutdownRequested) {
         if (messages.length) {
-          for (const message of messages) {
-            jobExecutor.executeJob(message, callback, qname, qrl)
-          }
+          jobExecutor.executeJobs(messages, callback, qname, qrl)
           queueManager.updateIcehouse(qrl, false)
         } else {
           // If we didn't get any, update the icehouse so we can back off
@@ -107,7 +105,7 @@ export async function processMessages (queues, callback, options) {
       }
 
       // Max job accounting
-      maxReturnCount -= maxMessages
+      if (messages.length) maxReturnCount -= messages.length
       activeQrls.delete(qrl)
     } catch (e) {
       // If the queue has been cleaned up, we should back off anyway
@@ -121,7 +119,7 @@ export async function processMessages (queues, callback, options) {
 
   while (!shutdownRequested) { // eslint-disable-line
     // Figure out how we are running
-    const allowedJobs = Math.max(0, opt.maxConcurrentJobs - jobExecutor.activeJobCount() - maxReturnCount)
+    const allowedJobs = Math.max(0, opt.maxConcurrentJobs - jobExecutor.runningJobCount() - maxReturnCount)
 
     // Latency
     const maxLatency = 100
@@ -146,10 +144,11 @@ export async function processMessages (queues, callback, options) {
     let jobsLeft = targetJobs
 
     if (opt.verbose) {
-      console.error({ maxConcurrentJobs: opt.maxConcurrentJobs, jobCount: jobExecutor.activeJobCount(), allowedJobs, maxLatency, latencyFactor, freememFactor, loadFactor, overallFactor, targetJobs, activeQrls })
+      // console.error({ maxConcurrentJobs: opt.maxConcurrentJobs, jobCount: jobExecutor.activeJobCount(), allowedJobs, maxLatency, latencyFactor, freememFactor, loadFactor, overallFactor, targetJobs, activeQrls })
     }
     for (const { qname, qrl } of queueManager.getPairs()) {
-      // debug({ evaluating: { qname, qrl, jobsLeft, activeQrlsHasQrl: activeQrls.has(qrl) } })
+      // const qcount = jobExecutor.runningJobCountForQueue(qname)
+      // console.log({ evaluating: { qname, qrl, qcount, jobsLeft, activeQrlsHasQrl: activeQrls.has(qrl) } })
       if (jobsLeft <= 0 || activeQrls.has(qrl)) continue
       const maxMessages = Math.min(10, jobsLeft)
       listen(qname, qrl, maxMessages)

package/src/defaults.js

@@ -26,8 +26,8 @@ export const defaults = Object.freeze({
   messageRetentionPeriod: 1209600,
   delay: 0,
   sendRetries: 6,
-  failDelay: 0,
-  dlq: false,
+  failDelay: 120,
+  dlq: true,
   dlqSuffix: '_dead',
   dlqAfter: 3,
 
@@ -90,7 +90,7 @@ export function getOptionsWithDefaults (options) {
     delay: options.delay || defaults.delay,
     sendRetries: options['send-retries'] || defaults.sendRetries,
     failDelay: options.failDelay || options['fail-delay'] || defaults.failDelay,
-    dlq: dlq || defaults.dlq,
+    dlq: dlq === false ? false : (dlq || defaults.dlq),
     dlqSuffix: options.dlqSuffix || options['dlq-suffix'] || defaults.dlqSuffix,
     dlqAfter: options.dlqAfter || options['dlq-after'] || defaults.dlqAfter,
     tags: options.tags || undefined,

package/src/enqueue.js

@@ -79,7 +79,7 @@ export async function getOrCreateFailQueue (queue, opt) {
         maxReceiveCount: opt.dlqAfter + ''
       })
     }
-    if (opt.failDelay) params.Attributes.DelaySeconds = opt.failDelay
+    if (opt.failDelay) params.Attributes.DelaySeconds = opt.failDelay + ''
     if (opt.tags) params.tags = opt.tags
     if (opt.fifo) params.Attributes.FifoQueue = 'true'
     const cmd = new CreateQueueCommand(params)

package/src/scheduler/jobExecutor.js

@@ -19,8 +19,11 @@ export class JobExecutor {
     this.opt = opt
     this.jobs = []
     this.jobsByMessageId = {}
+    this.jobsByQueue = new Map()
     this.stats = {
       activeJobs: 0,
+      waitingJobs: 0,
+      runningJobs: 0,
       sqsCalls: 0,
       timeoutsExtended: 0,
       jobsSucceeded: 0,
@@ -46,6 +49,20 @@ export class JobExecutor {
     return this.stats.activeJobs
   }
 
+  runningJobCount () {
+    return this.stats.runningJobs
+  }
+
+  /**
+   * Returns the number of jobs running in a queue.
+   */
+  runningJobCountForQueue (qname) {
+    const jobs = this.jobsByQueue.get(qname) || new Set()
+    let runningCount = 0
+    for (const job of jobs.values()) runningCount += job.status === 'running'
+    return runningCount
+  }
+
   /**
    * Changes message visibility on all running jobs using as few calls as possible.
    */
@@ -71,15 +88,12 @@ export class JobExecutor {
     const jobsToDeleteByQrl = {}
     const jobsToCleanup = new Set()
 
-    if (this.opt.verbose) {
-      console.error(chalk.blue('Stats: '), this.stats)
-      console.error(chalk.blue('Running: '), this.jobs.filter(j => j.status === 'processing').map(({ qname, message }) => ({ qname, payload: message.Body })))
-    }
-
     // Build list of jobs we need to deal with
+    const jobStatuses = {}
     for (let i = 0; i < this.jobs.length; i++) {
       const job = this.jobs[i]
       const jobRunTime = Math.round((start - job.start) / 1000)
+      jobStatuses[job.status] = (jobStatuses[job.status] || 0) + 1
       // debug('considering job', job)
       if (job.status === 'complete') {
         const jobsToDelete = jobsToDeleteByQrl[job.qrl] || []
@@ -88,7 +102,8 @@ export class JobExecutor {
         jobsToDeleteByQrl[job.qrl] = jobsToDelete
       } else if (job.status === 'failed') {
         jobsToCleanup.add(job)
-      } else if (job.status === 'processing') {
+      } else if (job.status !== 'deleting') {
+        // Any other job state gets visibility accounting
         debug('processing', { job, jobRunTime })
         if (jobRunTime >= job.extendAtSecond) {
           // Add it to our organized list of jobs
@@ -106,7 +121,11 @@ export class JobExecutor {
         }
       }
     }
-    // debug('maintainVisibility', { jobsToDeleteByQrl, jobsToExtendByQrl })
+
+    if (this.opt.verbose) {
+      console.error(chalk.blue('Stats: '), { stats: this.stats, jobStatuses })
+      console.error(chalk.blue('Running: '), this.jobs.filter(j => j.status === 'processing').map(({ qname, message }) => ({ qname, payload: message.Body })))
+    }
 
     // Extend in batches for each queue
     for (const qrl in jobsToExtendByQrl) {
@@ -199,7 +218,9 @@ export class JobExecutor {
     this.jobs = this.jobs.filter(job => {
       if (job.status === 'deleting' || job.status === 'failed') {
         debug('removed', job.message.MessageId)
+        // Accounting
         delete this.jobsByMessageId[job.message.MessageId]
+        this.jobsByQueue.get(job.qname).delete(job)
         return false
       } else {
         return true
@@ -207,20 +228,19 @@ export class JobExecutor {
     })
   }
 
-  async executeJob (message, callback, qname, qrl, failedCallback) {
-    if (this.shutdownRequested) throw new Error('jobExecutor is shutting down so cannot execute new job')
+  addJob (message, callback, qname, qrl) {
     // Create job entry and track it
-    const payload = this.opt.json ? JSON.parse(message.Body) : message.Body
-    const visibilityTimeout = 60
+    const defaultVisibilityTimeout = 60
     const job = {
-      status: 'processing',
+      status: 'waiting',
       start: new Date(),
-      visibilityTimeout,
-      extendAtSecond: visibilityTimeout / 2,
+      visibilityTimeout: defaultVisibilityTimeout,
+      extendAtSecond: defaultVisibilityTimeout / 2,
       payload: this.opt.json ? JSON.parse(message.Body) : message.Body,
       message,
       callback,
       qname,
+      prettyQname: qname.slice(this.opt.prefix.length),
       qrl
     }
 
@@ -237,29 +257,51 @@ export class JobExecutor {
       throw e
     }
 
-    // debug('executeJob', job)
+    // Accounting
     this.jobs.push(job)
     this.jobsByMessageId[job.message.MessageId] = job
+
+    // Track all jobs for each queue
+    if (!this.jobsByQueue.has(job.qname)) this.jobsByQueue.set(job.qname, new Set())
+    this.jobsByQueue.get(job.qname).add(job)
+
     this.stats.activeJobs++
+    this.stats.waitingJobs++
     if (this.opt.verbose) {
-      console.error(chalk.blue('Executing:'), qname, chalk.blue('-->'), job.payload)
+      console.error(chalk.blue('Got message:'), job.prettyQname, chalk.blue('-->'), job.payload, job.message.MessageId)
     } else if (!this.opt.disableLog) {
       console.log(JSON.stringify({
-        event: 'MESSAGE_PROCESSING_START',
+        event: 'MESSAGE_RECEIVED',
         timestamp: new Date(),
-        qrl,
+        queue: job.qname,
         messageId: message.MessageId,
         payload: job.payload
       }))
     }
+    return job
+  }
 
-    // Execute job
+  async runJob (job) {
     try {
-      const queue = qname.slice(this.opt.prefix.length)
-      const result = await callback(queue, payload)
-      debug('executeJob callback finished', { payload, result })
       if (this.opt.verbose) {
-        console.error(chalk.green('SUCCESS'), message.Body)
+        console.error(chalk.blue('Running:'), job.prettyQname, chalk.blue('-->'), job.payload, job.message.MessageId)
+      } else if (!this.opt.disableLog) {
+        console.log(JSON.stringify({
+          event: 'MESSAGE_PROCESSING_START',
+          timestamp: new Date(),
+          queue: job.qname,
+          messageId: job.message.MessageId,
+          payload: job.payload
+        }))
+      }
+      job.status = 'running'
+      this.stats.runningJobs++
+      this.stats.waitingJobs--
+      const queue = job.qname.slice(this.opt.prefix.length)
+      const result = await job.callback(queue, job.payload)
+      debug('executeJob callback finished', { payload: job.payload, result })
+      if (this.opt.verbose) {
+        console.error(chalk.green('SUCCESS'), job.payload)
       }
       job.status = 'complete'
 
@@ -269,35 +311,57 @@ export class JobExecutor {
       } else if (!this.opt.disableLog) {
         console.log(JSON.stringify({
           event: 'MESSAGE_PROCESSING_COMPLETE',
+          queue: job.qname,
           timestamp: new Date(),
-          messageId: message.MessageId,
-          payload
+          messageId: job.message.MessageId,
+          payload: job.payload
         }))
       }
       this.stats.jobsSucceeded++
     } catch (err) {
-      // Notify caller that we failed
-      if (failedCallback) failedCallback(message, qname, qrl)
+      job.status = 'failed'
+      this.stats.jobsFailed++
       // Fail path for job execution
       if (this.opt.verbose) {
-        console.error(chalk.red('FAILED'), message.Body)
+        console.error(chalk.red('FAILED'), job.payload)
         console.error(chalk.blue('  error : ') + err)
       } else if (!this.opt.disableLog) {
         // Production error logging
         console.log(JSON.stringify({
           event: 'MESSAGE_PROCESSING_FAILED',
           reason: 'exception thrown',
-          qrl,
+          queue: job.qname,
           timestamp: new Date(),
-          messageId: message.MessageId,
-          payload,
+          messageId: job.message.MessageId,
+          payload: job.payload,
           errorMessage: err.toString().split('\n').slice(1).join('\n').trim() || undefined,
           err
         }))
       }
-      job.status = 'failed'
-      this.stats.jobsFailed++
     }
     this.stats.activeJobs--
+    this.stats.runningJobs--
+  }
+
+  async executeJobs (messages, callback, qname, qrl) {
+    if (this.shutdownRequested) throw new Error('jobExecutor is shutting down so cannot execute new jobs')
+
+    // Begin tracking jobs
+    const jobs = messages.map(message => this.addJob(message, callback, qname, qrl))
+    const isFifo = qrl.endsWith('.fifo')
+
+    // console.log(jobs)
+
+    // Begin executing
+    for (const [job, i] of jobs.map((job, i) => [job, i])) {
+      // Figure out if the next job needs to happen in serial, otherwise we can parallel execute
+      const nextJob = jobs[i + 1]
+      const nextJobIsSerial = isFifo && nextJob && job.message?.Attributes?.GroupId === nextJob.message?.Attributes?.GroupId
+
+      // console.log({ i, nextJobAtt: nextJob?.message?.Attributes, nextJobIsSerial })
+      // Execute serial or parallel
+      if (nextJobIsSerial) await this.runJob(job)
+      else this.runJob(job)
+    }
   }
 }

package/src/scheduler/queueManager.js

@@ -35,7 +35,7 @@ export class QueueManager {
       const now = new Date()
       const secondsElapsed = lastCheck - now
       const minWait = 10
-      const maxWait = 600
+      const maxWait = 120
       const baseSeconds = numEmptyReceives ** 2 * 20
       const jitterSeconds = Math.round((Math.random() - 0.5) * baseSeconds)
       const newSecondsToWait = Math.max(minWait, Math.min(maxWait, baseSeconds + jitterSeconds))