qdone 2.0.12-alpha → 2.0.14-alpha

package/src/scheduler/jobExecutor.js

@@ -20,6 +20,7 @@ export class JobExecutor {
   constructor (opt) {
     this.opt = opt
     this.jobs = []
+    this.jobsByMessageId = {}
     this.stats = {
       activeJobs: 0,
       sqsCalls: 0,
@@ -28,15 +29,16 @@ export class JobExecutor {
       jobsFailed: 0,
       jobsDeleted: 0
     }
-    this.maintainVisibility()
+    this.maintainPromise = this.maintainVisibility()
     debug({ this: this })
   }
 
-  shutdown () {
+  async shutdown () {
     this.shutdownRequested = true
-    if (this.stats.activeJobs === 0 && this.jobs.length === 0) {
-      clearTimeout(this.maintainVisibilityTimeout)
-    }
+    // Trigger a maintenance run right away in case it speeds us up
+    clearTimeout(this.maintainVisibilityTimeout)
+    await this.maintainPromise
+    await this.maintainVisibility()
   }
 
   activeJobCount () {
@@ -47,8 +49,9 @@ export class JobExecutor {
    * Changes message visibility on all running jobs using as few calls as possible.
    */
   async maintainVisibility () {
-    debug('maintainVisibility', this.jobs)
-    const now = new Date()
+    clearTimeout(this.maintainVisibilityTimeout)
+    // debug('maintainVisibility', this.jobs)
+    const start = new Date()
     const jobsToExtendByQrl = {}
     const jobsToDeleteByQrl = {}
     const jobsToCleanup = new Set()
@@ -59,29 +62,36 @@ export class JobExecutor {
     }
 
     // Build list of jobs we need to deal with
-    for (const job of this.jobs) {
-      const jobRunTime = (now - job.start) / 1000
+    for (let i = 0; i < this.jobs.length; i++) {
+      const job = this.jobs[i]
+      const jobRunTime = Math.round((start - job.start) / 1000)
+      // debug('considering job', job)
       if (job.status === 'complete') {
         const jobsToDelete = jobsToDeleteByQrl[job.qrl] || []
+        job.status = 'deleting'
         jobsToDelete.push(job)
         jobsToDeleteByQrl[job.qrl] = jobsToDelete
       } else if (job.status === 'failed') {
         jobsToCleanup.add(job)
-      } else if (jobRunTime >= job.exendAtSecond) {
-        // Add it to our organized list of jobs
-        const jobsToExtend = jobsToExtendByQrl[job.qrl] || []
-        jobsToExtend.push(job)
-        jobsToExtendByQrl[job.qrl] = jobsToExtend
-
-        // Update the visibility timeout, double every time, up to max
-        const doubled = job.visibilityTimeout * 2
-        const secondsUntilMax = maxJobSeconds - jobRunTime
-        const secondsUntilKill = this.opt.killAfter - jobRunTime
-        job.visibilityTimeout = Math.min(double, secondsUntilMax, secondsUntilKill)
-        job.extendAtSecond = jobRunTime + job.visibilityTimeout // this is what we use next time
+      } else if (job.status === 'processing') {
+        debug('processing', { job, jobRunTime })
+        if (jobRunTime >= job.extendAtSecond) {
+          // Add it to our organized list of jobs
+          const jobsToExtend = jobsToExtendByQrl[job.qrl] || []
+          jobsToExtend.push(job)
+          jobsToExtendByQrl[job.qrl] = jobsToExtend
+
+          // Update the visibility timeout, double every time, up to max
+          const doubled = job.visibilityTimeout * 2
+          const secondsUntilMax = Math.max(1, maxJobSeconds - jobRunTime)
+          // const secondsUntilKill = Math.max(1, this.opt.killAfter - jobRunTime)
+          job.visibilityTimeout = Math.min(doubled, secondsUntilMax) //, secondsUntilKill)
+          job.extendAtSecond = Math.round(jobRunTime + job.visibilityTimeout) // this is what we use next time
+          debug({ doubled, secondsUntilMax, job })
+        }
       }
     }
-    debug('maintainVisibility', { jobsToDeleteByQrl, jobsToExtendByQrl })
+    // debug('maintainVisibility', { jobsToDeleteByQrl, jobsToExtendByQrl })
 
     // Extend in batches for each queue
     for (const qrl in jobsToExtendByQrl) {
@@ -100,6 +110,7 @@ export class JobExecutor {
           }
           entries.push(entry)
         }
+        debug({ entries })
 
         // Change batch
         const input = { QueueUrl: qrl, Entries: entries }
@@ -107,18 +118,17 @@ export class JobExecutor {
         const result = await getSQSClient().send(new ChangeMessageVisibilityBatchCommand(input))
         debug('ChangeMessageVisibilityBatch returned', result)
         this.stats.sqsCalls++
-        if (result.Successful) this.stats.timeoutsExtended += result.Successful.length || 0
+        if (result.Successful) {
+          const count = result.Successful.length || 0
+          this.stats.timeoutsExtended += count
+          if (this.opt.verbose) {
+            console.error(chalk.blue('Extended'), count, chalk.blue('jobs'))
+          } else if (!this.opt.disableLog) {
+            console.log(JSON.stringify({ event: 'EXTEND_VISIBILITY_TIMEOUTS', timestamp: start, count, qrl }))
+          }
+        }
         // TODO Sentry
       }
-      if (this.opt.verbose) {
-        console.error(chalk.blue('Extended these jobs: '), jobsToExtend)
-      } else if (!this.opt.disableLog) {
-        console.log(JSON.stringify({
-          event: 'EXTEND_VISIBILITY_TIMEOUTS',
-          timestamp: now,
-          messageIds: jobsToExtend.map(({ message }) => message.MessageId)
-        }))
-      }
     }
 
     // Delete in batches for each queue
@@ -137,43 +147,62 @@ export class JobExecutor {
           }
           entries.push(entry)
         }
+        debug({ entries })
 
         // Delete batch
         const input = { QueueUrl: qrl, Entries: entries }
         debug({ DeleteMessageBatch: input })
         const result = await getSQSClient().send(new DeleteMessageBatchCommand(input))
         this.stats.sqsCalls++
-        if (result.Successful) this.stats.jobsDeleted += result.Successful.length || 0
+        if (result.Successful) {
+          const count = result.Successful.length || 0
+          this.stats.jobsDeleted += count
+          if (this.opt.verbose) {
+            console.error(chalk.blue('Deleted'), count, chalk.blue('jobs'))
+          } else if (!this.opt.disableLog) {
+            console.log(JSON.stringify({ event: 'DELETE_MESSAGES', timestamp: start, count, qrl }))
+          }
+        }
         debug('DeleteMessageBatch returned', result)
         // TODO Sentry
       }
-      if (this.opt.verbose) {
-        console.error(chalk.blue('Deleted these finished jobs: '), jobsToDelete)
-      } else if (!this.opt.disableLog) {
-        console.log(JSON.stringify({
-          event: 'DELETE_MESSAGES',
-          timestamp: now,
-          messageIds: jobsToDelete.map(({ message }) => message.MessageId)
-        }))
-      }
     }
 
     // Get rid of deleted and failed jobs
-    this.jobs = this.jobs.filter(j => j.status === 'processing')
+    this.jobs = this.jobs.filter(job => {
+      if (job.status === 'deleting' || job.status === 'failed') {
+        debug('removed', job.message.MessageId)
+        delete this.jobsByMessageId[job.message.MessageId]
+        return false
+      } else {
+        return true
+      }
+    })
 
-    // Check again later, unless we are shutting down and nothing's left
+    // Bail if we are shutting down
     if (this.shutdownRequested && this.stats.activeJobs === 0 && this.jobs.length === 0) return
-    this.maintainVisibilityTimeout = setTimeout(() => this.maintainVisibility(), 10 * 1000)
+
+    // Check later, but count the time we spent. Make sure we check at least
+    // every period seconds.
+    const msElapsed = new Date() - start
+    const msPeriod = this.shutdownRequested ? 1 * 1000 : 10 * 1000
+    const msLeft = Math.max(0, msPeriod - msElapsed)
+    const msMin = this.shutdownRequested ? 1000 : 0
+    const nextCheckInMs = Math.max(msMin, msLeft)
+    debug({ msElapsed, msPeriod, msLeft, msMin, nextCheckInMs })
+    this.maintainVisibilityTimeout = setTimeout(() => {
+      this.maintainPromise = this.maintainVisibility()
+    }, nextCheckInMs)
   }
 
   async executeJob (message, callback, qname, qrl) {
     // Create job entry and track it
     const payload = this.opt.json ? JSON.parse(message.Body) : message.Body
-    const visibilityTimeout = 30
+    const visibilityTimeout = 60
     const job = {
       status: 'processing',
       start: new Date(),
-      visibilityTimeout: 30,
+      visibilityTimeout,
       extendAtSecond: visibilityTimeout / 2,
       payload: this.opt.json ? JSON.parse(message.Body) : message.Body,
       message,
@@ -181,8 +210,23 @@ export class JobExecutor {
       qname,
       qrl
     }
-    debug('executeJob', job)
+
+    // See if we are already executing this job
+    const oldJob = this.jobsByMessageId[job.message.MessageId]
+    if (oldJob) {
+      // If we actually see the same job again, we fucked up, probably due to
+      // the system being overloaded and us missing our extension call. So
+      // we'll celebrate this occasion by throwing a big fat error.
+      debug({ oldJob })
+      const e = new Error(`Saw job ${oldJob.message.MessageId} twice`)
+      e.job = oldJob
+      // TODO: sentry breadcrumb
+      throw e
+    }
+
+    // debug('executeJob', job)
     this.jobs.push(job)
+    this.jobsByMessageId[job.message.MessageId] = job
     this.stats.activeJobs++
     if (this.opt.verbose) {
       console.error(chalk.blue('Executing:'), qname, chalk.blue('-->'), job.payload)
@@ -190,7 +234,7 @@ export class JobExecutor {
       console.log(JSON.stringify({
         event: 'MESSAGE_PROCESSING_START',
         timestamp: new Date(),
-        qname,
+        qrl,
         messageId: message.MessageId,
         payload: job.payload
       }))
@@ -219,7 +263,7 @@ export class JobExecutor {
       }
       this.stats.jobsSucceeded++
     } catch (err) {
-      debug('exec.catch')
+      // debug('exec.catch', err)
       // Fail path for job execution
       if (this.opt.verbose) {
         console.error(chalk.red('FAILED'), message.Body)
@@ -229,7 +273,7 @@ export class JobExecutor {
         console.log(JSON.stringify({
           event: 'MESSAGE_PROCESSING_FAILED',
           reason: 'exception thrown',
-          qname,
+          qrl,
           timestamp: new Date(),
           messageId: message.MessageId,
           payload,

package/src/scheduler/queueManager.js

@@ -7,7 +7,6 @@ import Debug from 'debug'
 
 import { normalizeQueueName, getQnameUrlPairs } from '../qrlCache.js'
 import { cheapIdleCheck } from '../idleQueues.js'
-import { getSQSClient } from '../sqs.js'
 
 const debug = Debug('qdone:queueManager')
 
@@ -17,57 +16,120 @@ export class QueueManager {
     this.queues = queues
     this.resolveSeconds = resolveSeconds
     this.selectedPairs = []
+    this.icehouse = {}
     this.resolveTimeout = undefined
     this.shutdownRequested = false
-    this.resolveQueues()
+    this.resolvePromise = this.resolveQueues()
+  }
+
+  // Sends a queue to the icehouse, where it waits for a while before being
+  // checked again
+  updateIcehouse (qrl, emptyReceive) {
+    const foundEntry = !!this.icehouse[qrl]
+    const { lastCheck, secondsToWait, numEmptyReceives } = this.icehouse[qrl] || {
+      lastCheck: new Date(),
+      secondsToWait: Math.round(20 + 10 * Math.random()),
+      numEmptyReceives: 0 + emptyReceive
+    }
+    if (emptyReceive) {
+      const now = new Date()
+      const secondsElapsed = lastCheck - now
+      const minWait = 10
+      const maxWait = 600
+      const baseSeconds = numEmptyReceives ** 2 * 20
+      const jitterSeconds = Math.round((Math.random() - 0.5) * baseSeconds)
+      const newSecondsToWait = Math.max(minWait, Math.min(maxWait, baseSeconds + jitterSeconds))
+      const newEntry = { lastCheck: now, secondsToWait: newSecondsToWait, numEmptyReceives: numEmptyReceives + 1 }
+      this.icehouse[qrl] = newEntry
+      if (this.opt.verbose) {
+        console.error(chalk.blue('Sending queue to icehouse'), qrl, chalk.blue('for'), newSecondsToWait, chalk.blue('seconds'))
+      }
+      debug({ foundEntry, newEntry, lastCheck, secondsToWait, now, secondsElapsed, maxWait, minWait, baseSeconds, jitterSeconds })
+    } else {
+      delete this.icehouse[qrl]
+    }
+  }
+
+  // Returns true if the queue should be kept in the icehouse
+  keepInIcehouse (qrl, now) {
+    if (this.icehouse[qrl]) {
+      const { lastCheck, secondsToWait } = this.icehouse[qrl]
+      const secondsElapsed = Math.round((now - lastCheck) / 1000)
+      debug({ icehouseCheck: { qrl, lastCheck, secondsToWait, secondsElapsed } })
+      const letOut = secondsElapsed > secondsToWait
+      if (letOut && this.opt.verbose) {
+        console.error(chalk.blue('Coming out of icehouse:'), qrl)
+      }
+      return !letOut
+    } else {
+      return false
+    }
   }
 
   async resolveQueues () {
+    clearTimeout(this.resolveTimeout)
     if (this.shutdownRequested) return
 
     // Start processing
-    if (this.opt.verbose) console.error(chalk.blue('Resolving queues: ') + this.queues.join(' '))
     const qnames = this.queues.map(queue => normalizeQueueName(queue, this.opt))
     const pairs = await getQnameUrlPairs(qnames, this.opt)
+    if (this.opt.verbose) console.error(chalk.blue('Resolving queues:'))
 
     if (this.shutdownRequested) return
 
+    // Filter out queues
+    const now = new Date()
+    const filteredPairs = pairs
+      // first failed
+      .filter(({ qname, qrl }) => {
+        const suf = this.opt.failSuffix + (this.opt.fifo ? '.fifo' : '')
+        const isFailQueue = qname.slice(-suf.length) === suf
+        return this.opt.includeFailed ? true : !isFailQueue
+      })
+      // first fifo
+      .filter(({ qname, qrl }) => {
+        const isFifo = qname.endsWith('.fifo')
+        return this.opt.fifo ? isFifo : !isFifo
+      })
+      // then icehouse
+      .filter(({ qname, qrl }) => !this.keepInIcehouse(qrl, now))
+
     // Figure out which pairs are active
     const activePairs = []
     if (this.opt.activeOnly) {
-      debug({ pairsBeforeCheck: pairs })
-      await Promise.all(pairs.map(async pair => {
-        const { idle } = await cheapIdleCheck(pair.qname, pair.qrl, this.opt)
-        if (!idle) activePairs.push(pair)
+      if (this.opt.verbose) {
+        console.error(chalk.blue('  checking active only'))
+      }
+      await Promise.all(filteredPairs.map(async ({ qname, qrl }) => {
+        const { result: { idle } } = await cheapIdleCheck(qname, qrl, this.opt)
+        debug({ idle, qname })
+        if (!idle) activePairs.push({ qname, qrl })
       }))
     }
 
     if (this.shutdownRequested) return
 
-    // Finished resolving
-    if (this.opt.verbose) {
-      console.error(chalk.blue('  done'))
-      console.error()
-    }
-
     // Figure out which queues we want to listen on, choosing between active and
     // all, filtering out failed queues if the user wants that
-    this.selectedPairs = (this.opt.activeOnly ? activePairs : pairs)
-      .filter(({ qname }) => {
-        const suf = this.opt.failSuffix + (this.opt.fifo ? '.fifo' : '')
-        const isFailQueue = qname.slice(-suf.length) === suf
-        const shouldInclude = this.opt.includeFailed ? true : !isFailQueue
-        return shouldInclude
-      })
+    this.selectedPairs = (this.opt.activeOnly ? activePairs : filteredPairs)
 
     // Randomize order
     this.selectedPairs.sort(() => 0.5 - Math.random())
+    if (this.opt.verbose) console.error(chalk.blue('  selected:\n  ') + this.selectedPairs.map(({ qname }) => qname).join('\n  '))
     debug('selectedPairs', this.selectedPairs)
 
+    // Finished resolving
+    if (this.opt.verbose) {
+      console.error(chalk.blue('  done'))
+      console.error()
+    }
+
     if (this.opt.verbose) {
       console.error(chalk.blue('Will resolve queues again in ' + this.resolveSeconds + ' seconds'))
     }
-    this.resolveTimeout = setTimeout(() => this.resolveQueues(), this.resolveSeconds * 1000)
+    this.resolveTimeout = setTimeout(() => {
+      this.resolvePromise = this.resolveQueues()
+    }, this.resolveSeconds * 1000)
   }
 
   // Return the next queue in the lineup
@@ -81,8 +143,9 @@ export class QueueManager {
     return this.selectedPairs
   }
 
-  shutdown () {
-    clearTimeout(this.resolveTimeout)
+  async shutdown () {
     this.shutdownRequested = true
+    clearTimeout(this.resolveTimeout)
+    await this.resolvePromise
   }
 }

package/src/scheduler/systemMonitor.js

@@ -4,55 +4,41 @@
  */
 
 export class SystemMonitor {
-  constructor (opt, smoothingFactor = 0.5, reportSeconds = 5) {
-    this.opt = opt
-    this.smoothingFactor = smoothingFactor
+  constructor (reportCallback, reportSeconds = 1) {
+    this.reportCallback = reportCallback || console.log
     this.reportSeconds = reportSeconds
-    this.measurements = {
-      setTimeout: [],
-      setImmediate: []
-    }
-    this.timeouts = {
-      setTimeout: undefined,
-      setImmediate: undefined,
-      reportLatency: undefined
-    }
-    this.measureLatencySetTimeout()
+    this.measurements = []
+    this.measure()
     this.reportLatency()
   }
 
-  measureLatencySetTimeout () {
+  measure () {
+    clearTimeout(this.measureTimeout)
     const start = new Date()
-    this.timeouts.setTimeout = setTimeout(() => {
+    this.measureTimeout = setTimeout(() => {
       const latency = new Date() - start
-      this.measurements.setTimeout.push(latency)
-      if (this.measurements.setTimeout.length > 1000) this.measurements.setTimeout.shift()
-      this.measureLatencySetTimeout()
+      this.measurements.push(latency)
+      if (this.measurements.length > 1000) this.measurements.shift()
+      this.measure()
     })
   }
 
   getLatency () {
-    const results = {}
-    for (const k in this.measurements) {
-      const values = this.measurements[k]
-      results[k] = values.length ? values.reduce((a, b) => a + b, 0) / values.length : 0
-    }
-    return results
+    return this.measurements.length ? this.measurements.reduce((a, b) => a + b, 0) / this.measurements.length : 0
   }
 
   reportLatency () {
-    this.timeouts.reportLatency = setTimeout(() => {
-      for (const k in this.measurements) {
-        const values = this.measurements[k]
-        const mean = values.length ? values.reduce((a, b) => a + b, 0) / values.length : 0
-        console.log({ [k]: mean })
-      }
+    clearTimeout(this.reportTimeout)
+    this.reportTimeout = setTimeout(() => {
+      const latency = this.getLatency()
+      // console.log({ latency })
+      if (this.reportCallback) this.reportCallback(latency)
       this.reportLatency()
     }, this.reportSeconds * 1000)
   }
 
   shutdown () {
-    console.log(this.measurements)
-    for (const k in this.timeouts) clearTimeout(this.timeouts[k])
+    clearTimeout(this.measureTimeout)
+    clearTimeout(this.reportTimeout)
   }
 }