qdone 2.0.9-alpha → 2.0.10-alpha

package/src/consumer.js

@@ -2,31 +2,16 @@
  * Consumer implementation.
  */
 
-import {
-  ChangeMessageVisibilityCommand,
-  ReceiveMessageCommand,
-  DeleteMessageCommand
-} from '@aws-sdk/client-sqs'
+import { ReceiveMessageCommand } from '@aws-sdk/client-sqs'
 import chalk from 'chalk'
 import Debug from 'debug'
 
-import { normalizeQueueName, getQnameUrlPairs } from './qrlCache.js'
-import { cheapIdleCheck } from './idleQueues.js'
+import { SystemMonitor } from './scheduler/systemMonitor.js'
+import { QueueManager } from './scheduler/queueManager.js'
+import { JobExecutor } from './scheduler/jobExecutor.js'
 import { getOptionsWithDefaults } from './defaults.js'
 import { getSQSClient } from './sqs.js'
 
-//
-// Throwing an instance of this Error allows the processMessages callback to
-// refuse a message which then gets immediately returned to the queue.
-//
-// This has the side effect of throtting the queue since it stops polling on
-// the queue until the next queue resolution in processMessages.
-//
-// This is useful for implementing schedulers on top of qdone, for example, to
-// look at the queue name and decide whether to take on a new message.
-//
-export class DoNotProcess extends Error {}
-
 const debug = Debug('qdone:consumer')
 
 // Global flag for shutdown request
@@ -44,205 +29,18 @@ export function requestShutdown () {
   debug('requestShutdown done')
 }
 
-export async function processMessage (message, callback, qname, qrl, opt) {
-  debug('processMessage', message, qname, qrl)
-  const payload = opt.json ? JSON.parse(message.Body) : message.Body
-  if (opt.verbose) {
-    console.error(chalk.blue('  Processing payload:'), payload)
-  } else if (!opt.disableLog) {
-    console.log(JSON.stringify({
-      event: 'MESSAGE_PROCESSING_START',
-      timestamp: new Date(),
-      messageId: message.MessageId,
-      payload
-    }))
-  }
-
-  const jobStart = new Date()
-  let visibilityTimeout = 30 // this should be the queue timeout
-  let timeoutExtender
-
-  async function extendTimeout () {
-    debug('extendTimeout')
-    const maxJobRun = 12 * 60 * 60
-    const jobRunTime = ((new Date()) - jobStart) / 1000
-    // Double every time, up to max
-    visibilityTimeout = Math.min(visibilityTimeout * 2, maxJobRun - jobRunTime, opt.killAfter - jobRunTime)
-    if (opt.verbose) {
-      console.error(
-        chalk.blue('  Ran for ') + jobRunTime +
-        chalk.blue(' seconds, requesting another ') + visibilityTimeout +
-        chalk.blue(' seconds')
-      )
-    }
-
-    try {
-      const result = await getSQSClient().send(new ChangeMessageVisibilityCommand({
-        QueueUrl: qrl,
-        ReceiptHandle: message.ReceiptHandle,
-        VisibilityTimeout: visibilityTimeout
-      }))
-      debug('ChangeMessageVisibility returned', result)
-      if (
-        jobRunTime + visibilityTimeout >= maxJobRun ||
-        jobRunTime + visibilityTimeout >= opt.killAfter
-      ) {
-        if (opt.verbose) console.error(chalk.yellow('  warning: this is our last time extension'))
-      } else {
-        // Extend when we get 50% of the way to timeout
-        timeoutExtender = setTimeout(extendTimeout, visibilityTimeout * 1000 * 0.5)
-      }
-    } catch (err) {
-      debug('ChangeMessageVisibility threw', err)
-      // Rejection means we're ouuta time, whatever, let the job die
-      if (opt.verbose) {
-        console.error(chalk.red('  failed to extend job: ') + err)
-      } else if (!opt.disableLog) {
-        // Production error logging
-        console.log(JSON.stringify({
-          event: 'MESSAGE_PROCESSING_FAILED',
-          reason: 'ran longer than --kill-after',
-          timestamp: new Date(),
-          messageId: message.MessageId,
-          payload,
-          errorMessage: err.toString().split('\n').slice(1).join('\n').trim() || undefined,
-          err
-        }))
-      }
-    }
-  }
-
-  // Extend when we get 50% of the way to timeout
-  timeoutExtender = setTimeout(extendTimeout, visibilityTimeout * 1000 * 0.5)
-  debug('timeout', visibilityTimeout * 1000 * 0.5)
-
-  try {
-    // Process message
-    const queue = qname.slice(opt.prefix.length)
-    const result = await callback(queue, payload)
-    debug('processMessage callback finished', { payload, result })
-    clearTimeout(timeoutExtender)
-    if (opt.verbose) {
-      console.error(chalk.green('  SUCCESS'))
-      console.error(chalk.blue('  cleaning up (removing message) ...'))
-    }
-    await getSQSClient().send(new DeleteMessageCommand({
-      QueueUrl: qrl,
-      ReceiptHandle: message.ReceiptHandle
-    }))
-    if (opt.verbose) {
-      console.error(chalk.blue('  done'))
-      console.error()
-    } else if (!opt.disableLog) {
-      console.log(JSON.stringify({
-        event: 'MESSAGE_PROCESSING_COMPLETE',
-        timestamp: new Date(),
-        messageId: message.MessageId,
-        payload
-      }))
-    }
-    return { noJobs: 0, jobsSucceeded: 1, jobsFailed: 0 }
-  } catch (err) {
-    debug('exec.catch')
-    clearTimeout(timeoutExtender)
-
-    // If the callback does not want to process this message, return to queue
-    if (err instanceof DoNotProcess) {
-      if (opt.verbose) {
-        console.error(chalk.blue('  callback ') + chalk.yellow('REFUSED'))
-        console.error(chalk.blue('  cleaning up (removing message) ...'))
-      }
-      const result = await getSQSClient().send(new ChangeMessageVisibilityCommand({
-        QueueUrl: qrl,
-        ReceiptHandle: message.ReceiptHandle,
-        VisibilityTimeout: 0
-      }))
-      debug('ChangeMessageVisibility returned', result)
-      return { noJobs: 1, jobsSucceeded: 0, jobsFailed: 0 }
-    }
-
-    // Fail path for job execution
-    if (opt.verbose) {
-      console.error(chalk.red('  FAILED'))
-      console.error(chalk.blue('  error : ') + err)
-    } else if (!opt.disableLog) {
-      // Production error logging
-      console.log(JSON.stringify({
-        event: 'MESSAGE_PROCESSING_FAILED',
-        reason: 'exception thrown',
-        timestamp: new Date(),
-        messageId: message.MessageId,
-        payload,
-        errorMessage: err.toString().split('\n').slice(1).join('\n').trim() || undefined,
-        err
-      }))
-    }
-    return { noJobs: 0, jobsSucceeded: 0, jobsFailed: 1 }
-  }
-}
-
-//
-// Pull work off of a single queue
-//
-export async function pollSingleQueue (qname, qrl, callback, opt) {
-  debug('pollSingleQueue', { qname, qrl, callback, opt })
+export async function getMessages (qrl, opt, maxMessages) {
   const params = {
     AttributeNames: ['All'],
-    MaxNumberOfMessages: 1,
+    MaxNumberOfMessages: maxMessages,
     MessageAttributeNames: ['All'],
     QueueUrl: qrl,
     VisibilityTimeout: 30,
     WaitTimeSeconds: opt.waitTime
   }
   const response = await getSQSClient().send(new ReceiveMessageCommand(params))
-  debug('ReceiveMessage response', response)
-  if (shutdownRequested) return { noJobs: 0, jobsSucceeded: 0, jobsFailed: 0 }
-  if (response.Messages) {
-    const message = response.Messages[0]
-    if (opt.verbose) console.error(chalk.blue('  Found message ' + message.MessageId))
-    return processMessage(message, callback, qname, qrl, opt)
-  } else {
-    return { noJobs: 1, jobsSucceeded: 0, jobsFailed: 0 }
-  }
-}
-
-//
-// Resolve a set of queues
-//
-export async function resolveQueues (queues, opt) {
-  // Start processing
-  if (opt.verbose) console.error(chalk.blue('Resolving queues: ') + queues.join(' '))
-  const qnames = queues.map(queue => normalizeQueueName(queue, opt))
-  const pairs = await getQnameUrlPairs(qnames, opt)
-
-  // Figure out which pairs are active
-  const activePairs = []
-  if (opt.activeOnly) {
-    debug({ pairsBeforeCheck: pairs })
-    await Promise.all(pairs.map(async pair => {
-      const { idle } = await cheapIdleCheck(pair.qname, pair.qrl, opt)
-      if (!idle) activePairs.push(pair)
-    }))
-  }
-
-  // Finished resolving
-  debug('getQnameUrlPairs.then')
-  if (opt.verbose) {
-    console.error(chalk.blue('  done'))
-    console.error()
-  }
-
-  // Figure out which queues we want to listen on, choosing between active and
-  // all, filtering out failed queues if the user wants that
-  const selectedPairs = (opt.activeOnly ? activePairs : pairs)
-    .filter(({ qname }) => {
-      const suf = opt.failSuffix + (opt.fifo ? '.fifo' : '')
-      const isFailQueue = qname.slice(-suf.length) === suf
-      const shouldInclude = opt.includeFailed ? true : !isFailQueue
-      return shouldInclude
-    })
-
-  return selectedPairs
+  // debug('ReceiveMessage response', response)
+  return response.Messages || []
 }
 
 //
@@ -252,11 +50,17 @@ export async function processMessages (queues, callback, options) {
   const opt = getOptionsWithDefaults(options)
   debug('processMessages', { queues, callback, options, opt })
 
-  let loopCounter = 0
-  const stats = { noJobs: 0, jobsSucceeded: 0, jobsFailed: 0 }
-  const maxActiveLoops = 10
-  const activeLoops = new Map()
-  const completedLoops = new Map()
+  const systemMonitor = new SystemMonitor(opt)
+  const jobExecutor = new JobExecutor(opt)
+  const queueManager = new QueueManager(opt, queues)
+  const maxActiveJobs = 100
+  debug({ systemMonitor, jobExecutor, queueManager })
+
+  shutdownCallbacks.push(() => {
+    systemMonitor.shutdown()
+    queueManager.shutdown()
+    jobExecutor.shutdown()
+  })
 
   // This delay function keeps a timeout reference around so it can be
   // cancelled at shutdown
@@ -265,114 +69,41 @@ export async function processMessages (queues, callback, options) {
     delayTimeout = setTimeout(resolve, ms)
   })
 
-  // Callback to help facilitate better UX at shutdown
-  function shutdownCallback () {
-    if (opt.verbose) {
-      debug({ activeLoops, completedLoops })
-      if (activeLoops.length) {
-        console.error(chalk.blue('Waiting for work to finish on the following queues: '))
-        for (const [id, { qname, qrl, promise }] of activeLoops) {
-          console.error('    ' + qname + `job ${id}`)
-        }
-      }
-      // clearTimeout(delayTimeout)
-    }
-  }
-  shutdownCallbacks.push(shutdownCallback)
-
-  // Listen to a queue until it is out of messages
-  async function listenLoop (qname, qrl, loopId) {
-    try {
-      if (shutdownRequested) return
-      if (opt.verbose) {
-        console.error(
-          chalk.blue('Looking for work on ') +
-          qname.slice(opt.prefix.length) +
-          chalk.blue(' (' + qrl + ')')
-        )
-      }
-      // Aggregate the results
-      const { noJobs, jobsSucceeded, jobsFailed } = await pollSingleQueue(qname, qrl, callback, opt)
-      debug('pollSingleQueue return')
-      stats.noJobs += noJobs
-      stats.jobsFailed += jobsFailed
-      stats.jobsSucceeded += jobsSucceeded
-      debug({ stats, noJobs })
-
-      // No work? Shutdown requested? Return to outer loop
-      debug({ noJobs, shutdownRequested })
-      if (noJobs || shutdownRequested) return
-
-      // Otherwise keep going
-      return listenLoop(qname, qrl)
-    } catch (err) {
-      // TODO: Sentry
-      console.error(chalk.red('  ERROR in listenLoop'))
-      console.error(chalk.blue('  error : ') + err)
-    } finally {
-      completedLoops.set(loopId, activeLoops.get(loopId))
+  // Keep track of how many messages could be returned from each queue
+  const activeQrls = new Set()
+  let maxReturnCount = 0
+  const listen = async (qname, qrl, maxMessages) => {
+    activeQrls.add(qrl)
+    maxReturnCount += maxMessages
+    const messages = await getMessages(qrl, opt, maxMessages)
+    for (const message of messages) {
+      jobExecutor.executeJob(message, callback, qname, qrl)
     }
+    maxReturnCount -= maxMessages
+    activeQrls.delete(qrl)
   }
 
-  // Resolve loop
   while (!shutdownRequested) { // eslint-disable-line
-    const start = new Date()
-    const selectedPairs = await resolveQueues(queues, opt)
-    debug({ selectedPairs })
-    if (shutdownRequested) break
-
-    // But only if we have queues to listen on
-    if (selectedPairs.length) {
-      // Randomize order
-      selectedPairs.sort(() => 0.5 - Math.random())
-
-      if (opt.verbose) {
-        console.error(chalk.blue('Listening to queues (in this order):'))
-        console.error(selectedPairs.map(({ qname, qrl }) =>
-          '  ' + qname.slice(opt.prefix.length) + chalk.blue(' - ' + qrl)
-        ).join('\n'))
-        console.error()
-      }
-
-      // Launch listen loop for each queue
-      for (const { qname, qrl } of selectedPairs) {
-        // Bail if we already have too many
-        if (activeLoops.size >= maxActiveLoops) {
-          if (opt.verbose) console.error(chalk.yellow('Hit active worker limit of ') + maxActiveLoops)
-          break
-        }
-        const loopId = loopCounter++
-        activeLoops.set(loopId, { qname, qrl, promise: listenLoop(qname, qrl, loopId) })
+    // Figure out how we are running
+    const allowedJobs = maxActiveJobs - jobExecutor.activeJobCount() - maxReturnCount
+    const maxLatency = 100
+    const latency = systemMonitor.getLatency().setTimeout
+    const latencyFactor = 1 - Math.abs(Math.min(latency / maxLatency, 1)) // 0 if latency is at max, 1 if latency 0
+    const targetJobs = Math.round(allowedJobs * latencyFactor)
+    debug({ allowedJobs, maxLatency, latency, latencyFactor, targetJobs, activeQrls })
+
+    let jobsLeft = targetJobs
+    for (const { qname, qrl } of queueManager.getPairs()) {
+      if (jobsLeft <= 0 || activeQrls.has(qrl)) continue
+      const maxMessages = Math.min(10, jobsLeft)
+      listen(qname, qrl, maxMessages)
+      jobsLeft -= maxMessages
+      if (this.opt.verbose) {
+        console.error(chalk.blue('Listening on: '), qname)
       }
+      debug({ listenedTo: { qname, maxMessages, jobsLeft } })
     }
-
-    // Wait until the next time we need to resolve
-    if (!shutdownRequested) {
-      const msSoFar = Math.max(0, new Date() - start)
-      const msUntilNextResolve = Math.max(0, /*opt.waitTime **/ 1000 - msSoFar)
-      debug({ msSoFar, msUntilNextResolve })
-      if (msUntilNextResolve) {
-        if (opt.verbose) console.error(chalk.blue('Will resolve queues again in ' + Math.round(msUntilNextResolve / 1000) + ' seconds'))
-        await delay(msUntilNextResolve)
-      }
-    }
-
-    // Cleanup completed loops
-    for (const [id, { qname, qrl, promise }] of completedLoops) {
-      await promise // make sure the promise resolves
-      debug('Cleaning up', { id, qname, qrl, promise })
-      activeLoops.delete(id)
-    }
-  }
-  debug('out here', { activeLoops })
-
-  // Wait on all work to finish
-  // shutdownCallback()
-  for (const [id, { qname, qrl, promise }] of activeLoops) {
-    debug('Waiting on active loop', id)
-    await promise // make sure the promise resolves
+    await delay(1000)
   }
   debug('after all')
 }
-
-debug('loaded')

package/src/scheduler/jobExecutor.js

@@ -0,0 +1,242 @@
+/**
+ * Component to manage all the currently executing jobs, including extending
+ * their visibility timeouts and deleting them when they are successful.
+ */
+
+import {
+  ChangeMessageVisibilityBatchCommand,
+  DeleteMessageBatchCommand
+} from '@aws-sdk/client-sqs'
+import chalk from 'chalk'
+import Debug from 'debug'
+
+import { getSQSClient } from '../sqs.js'
+
+const debug = Debug('qdone:jobExecutor')
+
+const maxJobSeconds = 12 * 60 * 60
+
+export class JobExecutor {
+  constructor (opt) {
+    this.opt = opt
+    this.jobs = []
+    this.stats = {
+      activeJobs: 0,
+      sqsCalls: 0,
+      timeoutsExtended: 0,
+      jobsSucceeded: 0,
+      jobsFailed: 0,
+      jobsDeleted: 0
+    }
+    this.maintainVisibility()
+    debug({ this: this })
+  }
+
+  shutdown () {
+    this.shutdownRequested = true
+    if (this.stats.activeJobs === 0 && this.jobs.length === 0) {
+      clearTimeout(this.maintainVisibilityTimeout)
+    }
+  }
+
+  activeJobCount () {
+    return this.stats.activeJobs
+  }
+
+  /**
+   * Changes message visibility on all running jobs using as few calls as possible.
+   */
+  async maintainVisibility () {
+    debug('maintainVisibility', this.jobs)
+    const now = new Date()
+    const jobsToExtendByQrl = new Map()
+    const jobsToDeleteByQrl = new Map()
+    const jobsToCleanup = new Set()
+
+    if (this.opt.verbose) {
+      console.error(chalk.blue('Stats: '), this.stats)
+      console.error(chalk.blue('Running: '), this.jobs.map(({ qname, message }) => ({ qname, payload: message.Body })))
+    }
+
+    // Build list of jobs we need to deal with
+    for (const job of this.jobs) {
+      const jobRunTime = (now - job.start) / 1000
+      if (job.status === 'complete') {
+        const jobsToDelete = jobsToDeleteByQrl.get(job.qrl) || []
+        jobsToDelete.push(job)
+        jobsToDeleteByQrl.set(job.qrl, jobsToDelete)
+      } else if (job.status === 'failed') {
+        jobsToCleanup.add(job)
+      } else if (jobRunTime >= job.exendAtSecond) {
+        // Add it to our organized list of jobs
+        const jobsToExtend = jobsToExtendByQrl.get(job.qrl) || []
+        jobsToExtend.push(job)
+        jobsToExtendByQrl.set(job.qrl, jobsToExtend)
+
+        // Update the visibility timeout, double every time, up to max
+        const doubled = job.visibilityTimeout * 2
+        const secondsUntilMax = maxJobSeconds - jobRunTime
+        const secondsUntilKill = this.opt.killAfter - jobRunTime
+        job.visibilityTimeout = Math.min(double, secondsUntilMax, secondsUntilKill)
+        job.extendAtSecond = jobRunTime + job.visibilityTimeout // this is what we use next time
+      }
+    }
+    debug('maintainVisibility', { jobsToDeleteByQrl, jobsToExtendByQrl })
+
+    // Extend in batches for each queue
+    for (const [qrl, jobsToExtend] of jobsToExtendByQrl) {
+      while (jobsToExtend.length) {
+        // Build list of messages to go in this batch
+        const entries = []
+        let messageId = 0
+        while (messageId++ < 10 && jobsToExtend.length) {
+          const job = jobsToExtend.shift()
+          const entry = {
+            Id: '' + messageId,
+            ReceiptHandle: job.message.ReceiptHandle,
+            VisibilityTimeout: job.visibilityTimeout
+          }
+          entries.push(entry)
+        }
+
+        // Change batch
+        const input = { QueueUrl: qrl, Entries: entries }
+        debug({ ChangeMessageVisibilityBatch: input })
+        const result = await getSQSClient().send(new ChangeMessageVisibilityBatchCommand(input))
+        debug('ChangeMessageVisibilityBatch returned', result)
+        this.stats.sqsCalls++
+        this.stats.timeoutsExtended += 10
+        // TODO Sentry
+      }
+      if (this.opt.verbose) {
+        console.error(chalk.blue('Extended these jobs: '), jobsToExtend)
+      } else if (!this.opt.disableLog) {
+        console.log(JSON.stringify({
+          event: 'EXTEND_VISIBILITY_TIMEOUTS',
+          timestamp: now,
+          messageIds: jobsToExtend.map(({ message }) => message.MessageId)
+        }))
+      }
+    }
+
+    // Delete in batches for each queue
+    for (const [qrl, jobsToDelete] of jobsToDeleteByQrl) {
+      while (jobsToDelete.length) {
+        // Build list of messages to go in this batch
+        const entries = []
+        let messageId = 0
+        while (messageId++ < 10 && jobsToDelete.length) {
+          const job = jobsToDelete.shift()
+          const entry = {
+            Id: '' + messageId,
+            ReceiptHandle: job.message.ReceiptHandle,
+            VisibilityTimeout: job.visibilityTimeout
+          }
+          entries.push(entry)
+        }
+
+        // Delete batch
+        const input = { QueueUrl: qrl, Entries: entries }
+        debug({ DeleteMessageBatch: input })
+        const result = await getSQSClient().send(new DeleteMessageBatchCommand(input))
+        this.stats.sqsCalls++
+        this.stats.jobsDeleted += 10
+        debug('DeleteMessageBatch returned', result)
+        // TODO Sentry
+      }
+      if (this.opt.verbose) {
+        console.error(chalk.blue('Deleted these finished jobs: '), jobsToDelete)
+      } else if (!this.opt.disableLog) {
+        console.log(JSON.stringify({
+          event: 'DELETE_MESSAGES',
+          timestamp: now,
+          messageIds: jobsToDelete.map(({ message }) => message.MessageId)
+        }))
+      }
+    }
+
+    // Get rid of deleted and failed jobs
+    this.jobs = this.jobs.filter(j => j.status === 'processing')
+
+    // Check again later, unless we are shutting down and nothing's left
+    if (this.shutdownRequested && this.stats.activeJobs === 0 && this.jobs.length === 0) return
+    this.maintainVisibilityTimeout = setTimeout(() => this.maintainVisibility(), 10 * 1000)
+  }
+
+  async executeJob (message, callback, qname, qrl) {
+    // Create job entry and track it
+    const payload = this.opt.json ? JSON.parse(message.Body) : message.Body
+    const visibilityTimeout = 30
+    const job = {
+      status: 'processing',
+      start: new Date(),
+      visibilityTimeout: 30,
+      extendAtSecond: visibilityTimeout / 2,
+      payload: this.opt.json ? JSON.parse(message.Body) : message.Body,
+      message,
+      callback,
+      qname,
+      qrl
+    }
+    debug('executeJob', job)
+    this.jobs.push(job)
+    this.stats.activeJobs++
+    if (this.opt.verbose) {
+      console.error(chalk.blue('Executing:'), qname, chalk.blue('-->'), job.payload)
+    } else if (!this.opt.disableLog) {
+      console.log(JSON.stringify({
+        event: 'MESSAGE_PROCESSING_START',
+        timestamp: new Date(),
+        qname,
+        messageId: message.MessageId,
+        payload: job.payload
+      }))
+    }
+
+    // Execute job
+    try {
+      const queue = qname.slice(this.opt.prefix.length)
+      const result = await callback(queue, payload)
+      debug('executeJob callback finished', { payload, result })
+      if (this.opt.verbose) {
+        console.error(chalk.green('SUCCESS'), message.Body)
+      }
+      job.status = 'complete'
+
+      if (this.opt.verbose) {
+        console.error(chalk.blue('  done'))
+        console.error()
+      } else if (!this.opt.disableLog) {
+        console.log(JSON.stringify({
+          event: 'MESSAGE_PROCESSING_COMPLETE',
+          timestamp: new Date(),
+          messageId: message.MessageId,
+          payload
+        }))
+      }
+      this.stats.jobsSucceeded++
+    } catch (err) {
+      debug('exec.catch')
+      // Fail path for job execution
+      if (this.opt.verbose) {
+        console.error(chalk.red('FAILED'), message.Body)
+        console.error(chalk.blue('  error : ') + err)
+      } else if (!this.opt.disableLog) {
+        // Production error logging
+        console.log(JSON.stringify({
+          event: 'MESSAGE_PROCESSING_FAILED',
+          reason: 'exception thrown',
+          qname,
+          timestamp: new Date(),
+          messageId: message.MessageId,
+          payload,
+          errorMessage: err.toString().split('\n').slice(1).join('\n').trim() || undefined,
+          err
+        }))
+      }
+      job.status = 'failed'
+      this.stats.jobsFailed++
+    }
+    this.stats.activeJobs--
+  }
+}