qdone 1.6.0 → 2.0.0-alpha

package/src/cli.js

@@ -1,15 +1,23 @@
-
-const debug = require('debug')('qdone:cli')
-const Q = require('q')
-const fs = require('fs')
-const readline = require('readline')
-const chalk = require('chalk')
-const commandLineCommands = require('command-line-commands')
-const commandLineArgs = require('command-line-args')
-const getUsage = require('command-line-usage')
-const uuid = require('uuid')
+/**
+ * Command line interface implementation
+ */
+import { createReadStream, openSync } from 'node:fs'
+import { createInterface } from 'node:readline'
+import { createRequire } from 'module'
+import getUsage from 'command-line-usage'
+import commandLineCommands from 'command-line-commands'
+import commandLineArgs from 'command-line-args'
+import Debug from 'debug'
+import chalk from 'chalk'
+
+import { QueueDoesNotExist } from '@aws-sdk/client-sqs'
+import { defaults, setupAWS, setupVerbose, getOptionsWithDefaults } from './defaults.js'
+import { shutdownCache } from './cache.js'
+import { withSentry } from './sentry.js'
+
+const debug = Debug('qdone:cli')
+const require = createRequire(import.meta.url)
 const packageJson = require('../package.json')
-
 class UsageError extends Error {}
 
 const awsUsageHeader = { content: 'AWS SQS Authentication', raw: true, long: true }
@@ -24,40 +32,33 @@ const awsUsageBody = {
 }
 
 const globalOptionDefinitions = [
-  { name: 'prefix', type: String, defaultValue: 'qdone_', description: 'Prefix to place at the front of each SQS queue name [default: qdone_]' },
-  { name: 'fail-suffix', type: String, defaultValue: '_failed', description: 'Suffix to append to each queue to generate fail queue name [default: _failed]' },
-  { name: 'region', type: String, defaultValue: 'us-east-1', description: 'AWS region for Queues [default: us-east-1]' },
-  { name: 'quiet', alias: 'q', type: Boolean, defaultValue: false, description: 'Turn on production logging. Automatically set if stderr is not a tty.' },
-  { name: 'verbose', alias: 'v', type: Boolean, defaultValue: false, description: 'Turn on verbose output. Automatically set if stderr is a tty.' },
+  { name: 'prefix', type: String, description: `Prefix to place at the front of each SQS queue name [default: ${defaults.prefix}]` },
+  { name: 'fail-suffix', type: String, description: `Suffix to append to each queue to generate fail queue name [default: ${defaults.failSuffix}]` },
+  { name: 'region', type: String, description: `AWS region for Queues [default: ${defaults.region}]` },
+  { name: 'quiet', alias: 'q', type: Boolean, description: 'Turn on production logging. Automatically set if stderr is not a tty.' },
+  { name: 'verbose', alias: 'v', type: Boolean, description: 'Turn on verbose output. Automatically set if stderr is a tty.' },
   { name: 'version', alias: 'V', type: Boolean, description: 'Show version number' },
   { name: 'cache-uri', type: String, description: 'URL to caching cluster. Only redis://... currently supported.' },
-  { name: 'cache-prefix', type: String, defaultValue: 'qdone:', description: 'Prefix for all keys in cache.' },
-  { name: 'cache-ttl-seconds', type: Number, defaultValue: 10, description: 'Number of seconds to cache GetQueueAttributes calls.' },
-  { name: 'help', type: Boolean, description: 'Print full help message.' }
+  { name: 'cache-prefix', type: String, description: `Prefix for all keys in cache. [default: ${defaults.cachePrefix}]` },
+  { name: 'cache-ttl-seconds', type: Number, description: `Number of seconds to cache GetQueueAttributes calls. [default: ${defaults.cacheTtlSeconds}]` },
+  { name: 'help', type: Boolean, description: 'Print full help message.' },
+  { name: 'sentry-dsn', type: String, description: 'Optional Sentry DSN to track unhandled errors.' }
 ]
 
-function setupAWS (options) {
-  debug('loading aws-sdk')
-  const AWS = require('aws-sdk')
-  AWS.config.setPromisesDependency(Q.Promise)
-  AWS.config.update({ region: options.region })
-  debug('loaded')
-}
-
-function setupVerbose (options) {
-  const verbose = options.verbose || (process.stderr.isTTY && !options.quiet)
-  const quiet = options.quiet || (!process.stderr.isTTY && !options.verbose)
-  options.verbose = verbose
-  options.quiet = quiet
-}
-
 const enqueueOptionDefinitions = [
   { name: 'fifo', alias: 'f', type: Boolean, description: 'Create new queues as FIFOs' },
-  { name: 'group-id', alias: 'g', type: String, defaultValue: uuid.v1(), description: 'FIFO Group ID to use for all messages enqueued in current command. Defaults to an string unique to this invocation.' },
-  { name: 'group-id-per-message', type: Boolean, description: 'Use a unique Group ID for every message, even messages in the same batch.' }
+  { name: 'group-id', alias: 'g', type: String, description: 'FIFO Group ID to use for all messages enqueued in current command. Defaults to a string unique to this invocation.' },
+  { name: 'group-id-per-message', type: Boolean, description: 'Use a unique Group ID for every message, even messages in the same batch.' },
+  { name: 'deduplication-id', type: String, description: 'A Message Deduplication ID to give SQS when sending a message. Use this option if you are managing retries outside of qdone, and make sure the ID is the same for each retry in the deduplication window. Defaults to a string unique to this invocation.' },
+  { name: 'message-retention-period', type: Number, description: `Number of seconds to retain jobs (up to 14 days). [default: ${defaults.messageRetentionPeriod}]` },
+  { name: 'delay', alias: 'd', type: Number, description: 'Delays delivery of each message by the given number of seconds (up to 900 seconds, or 15 minutes). Defaults to immediate delivery (no delay).' },
+  { name: 'dlq', type: Boolean, description: 'Send messages from the failed queue to a DLQ.' },
+  { name: 'dql-suffix', type: String, description: `Suffix to append to each queue to generate DLQ name [default: ${defaults.dlqSuffix}]` },
+  { name: 'dql-after', type: String, description: `Drives message to the DLQ after this many failures in the failed queue. [default: ${defaults.dlqAfter}]` },
+  { name: 'tag', type: String, multiple: true, description: 'Adds an AWS tag to queue creation. Use the format Key=Value. Can specify multiple times.' }
 ]
 
-exports.enqueue = function enqueue (argv) {
+export async function enqueue (argv, testHook) {
   const optionDefinitions = [].concat(enqueueOptionDefinitions, globalOptionDefinitions)
   const usageSections = [
     { content: 'usage: qdone enqueue [options] <queue> <command>', raw: true },
@@ -75,14 +76,80 @@ exports.enqueue = function enqueue (argv) {
   debug('enqueue argv', argv)
 
   // Parse command and options
+  let options, queue, command
   try {
-    var options = commandLineArgs(optionDefinitions, { argv, partial: true })
+    options = commandLineArgs(optionDefinitions, { argv, partial: true })
     setupVerbose(options)
     debug('enqueue options', options)
     if (options.help) return Promise.resolve(console.log(getUsage(usageSections)))
     if (!options._unknown || options._unknown.length !== 2) throw new UsageError('enqueue requires both <queue> and <command> arguments')
-    var [queue, command] = options._unknown
+    queue = options._unknown[0]
+    command = options._unknown[1]
     debug('queue', queue, 'command', command)
+  } catch (err) {
+    console.log(getUsage(usageSections.filter(s => !s.long)))
+    throw err
+  }
+
+  // Process tags
+  if (options.tag && options.tag.length) {
+    options.tags = {}
+    for (const input of options.tag) {
+      debug({ input })
+      if (input.indexOf('=') === -1) throw new UsageError('Tags must be separated with the "=" character.')
+      const [key, ...rest] = input.split('=')
+      const value = rest.join('=')
+      debug({ input, key, rest, value, tags: options.tags })
+      options.tags[key] = value
+    }
+  }
+
+  // Load module after AWS global load
+  setupAWS(options)
+  const { enqueue: enqueueOriginal } = await import('./enqueue.js')
+  const enqueue = testHook || enqueueOriginal
+
+  // Normal (non batch) enqueue
+  const opt = getOptionsWithDefaults(options)
+  const result = (
+    await withSentry(async () => enqueue(queue, command, opt), opt)
+  )
+  debug('enqueue returned', result)
+  if (options.verbose) console.error(chalk.blue('Enqueued job ') + result.MessageId)
+  return result
+}
+
+const monitorOptionDefinitions = [
+  { name: 'save', alias: 's', type: Boolean, description: 'Saves data to CloudWatch' }
+]
+
+export async function monitor (argv) {
+  const optionDefinitions = [].concat(monitorOptionDefinitions, globalOptionDefinitions)
+  const usageSections = [
+    { content: 'usage: qdone monitor <queuePattern> ', raw: true },
+    { content: 'Options', raw: true },
+    { optionList: optionDefinitions },
+    { content: 'SQS API Call Complexity', raw: true, long: true },
+    {
+      content: [
+        { count: '1 + N', summary: 'one call to resolve the queue names (potentially more calls if there are pages)\none call per queue to get attributes' }
+      ],
+      long: true
+    },
+    awsUsageHeader, awsUsageBody
+  ]
+  debug('monitor argv', argv)
+
+  // Parse command and options
+  let options, queue
+  try {
+    options = commandLineArgs(optionDefinitions, { argv, partial: true })
+    setupVerbose(options)
+    debug('enqueue options', options)
+    if (options.help) return Promise.resolve(console.log(getUsage(usageSections)))
+    if (!options._unknown || options._unknown.length !== 1) throw new UsageError('monitor requires the <queuePattern> argument')
+    queue = options._unknown[0]
+    debug('queue', queue)
   } catch (e) {
     console.log(getUsage(usageSections.filter(s => !s.long)))
     return Promise.reject(e)
@@ -90,19 +157,44 @@ exports.enqueue = function enqueue (argv) {
 
   // Load module after AWS global load
   setupAWS(options)
-  const enqueue = require('./enqueue')
+  const { getAggregateData } = await import('./monitor.js')
+  const { putAggregateData } = await import('./cloudWatch.js')
+  const data = await getAggregateData(queue)
+  console.log(data)
+  if (options.save) {
+    process.stderr.write('Saving to CloudWatch...')
+    await putAggregateData(data)
+    process.stderr.write('done\n')
+  }
+  return data
+}
 
-  // Normal (non batch) enqueue
-  return enqueue
-    .enqueue(queue, command, options)
-    .then(function (result) {
-      debug('enqueue returned', result)
-      if (options.verbose) console.error(chalk.blue('Enqueued job ') + result.MessageId)
-      return result
+export async function loadBatchFile (filename) {
+  const file = filename === '-' ? process.stdin : createReadStream(filename, { fd: openSync(filename, 'r') })
+  const pairs = []
+  await new Promise((resolve, reject) => {
+    debug('file', file.name || 'stdin')
+    // Construct (queue, command) pairs from input
+    const input = createInterface({ input: file })
+    input.on('line', line => {
+      const parts = line.split(/\s+/)
+      const queue = parts[0]
+      const command = line.slice(queue.length).trim()
+      pairs.push({ queue, command })
     })
+    input.on('error', reject)
+    input.on('close', resolve)
+  })
+  return pairs
+}
+
+export async function loadBatchFiles (filenames) {
+  const results = await Promise.all(filenames.map(loadBatchFile))
+  const pairs = results.flat()
+  return pairs
 }
 
-exports.enqueueBatch = function enqueueBatch (argv) {
+export async function enqueueBatch (argv, testHook) {
   const optionDefinitions = [].concat(enqueueOptionDefinitions, globalOptionDefinitions)
   const usageSections = [
     { content: 'usage: qdone enqueue-batch [options] <file...>', raw: true },
@@ -121,61 +213,46 @@ exports.enqueueBatch = function enqueueBatch (argv) {
   debug('enqueue-batch argv', argv)
 
   // Parse command and options
-  let files
+  let filenames, options
   try {
-    var options = commandLineArgs(optionDefinitions, { argv, partial: true })
+    options = commandLineArgs(optionDefinitions, { argv, partial: true })
     setupVerbose(options)
     debug('enqueue-batch options', options)
     if (options.help) return Promise.resolve(console.log(getUsage(usageSections)))
     if (!options._unknown || options._unknown.length === 0) throw new UsageError('enqueue-batch requres one or more <file> arguments')
     debug('filenames', options._unknown)
-    files = options._unknown.map(f => f === '-' ? process.stdin : fs.createReadStream(f, { fd: fs.openSync(f, 'r') }))
+    filenames = options._unknown
   } catch (err) {
     console.log(getUsage(usageSections.filter(s => !s.long)))
-    return Promise.reject(err)
+    throw err
   }
 
   // Load module after AWS global load
   setupAWS(options)
-  const enqueue = require('./enqueue')
-  const pairs = []
+  const { enqueueBatch: enqueueBatchOriginal } = await import('./enqueue.js')
+  const enqueueBatch = testHook || enqueueBatchOriginal
 
   // Load data and enqueue it
-  return Promise.all(
-    files.map(function (file) {
-      // Construct (queue, command) pairs from input
-      debug('file', file.name || 'stdin')
-      const input = readline.createInterface({ input: file })
-      const deferred = Q.defer()
-      input.on('line', line => {
-        const parts = line.split(/\s+/)
-        const queue = parts[0]
-        const command = line.slice(queue.length).trim()
-        pairs.push({ queue, command })
-      })
-      input.on('error', deferred.reject)
-      input.on('close', deferred.resolve)
-      return deferred.promise
-    })
+  const pairs = await loadBatchFiles(filenames)
+  debug('pairs', pairs)
+
+  // Normal (non batch) enqueue
+  const opt = getOptionsWithDefaults(options)
+  const result = (
+    await withSentry(async () => enqueueBatch(pairs, opt), opt)
   )
-    .then(function () {
-      debug('pairs', pairs)
-      return enqueue
-        .enqueueBatch(pairs, options)
-        .then(function (result) {
-          debug('enqueueBatch returned', result)
-          if (options.verbose) console.error(chalk.blue('Enqueued ') + result + chalk.blue(' jobs'))
-        })
-    })
+  debug('enqueueBatch returned', result)
+  if (options.verbose) console.error(chalk.blue('Enqueued ') + result + chalk.blue(' jobs'))
 }
 
-exports.worker = function worker (argv) {
+export async function worker (argv, testHook) {
   const optionDefinitions = [
     { name: 'kill-after', alias: 'k', type: Number, defaultValue: 30, description: 'Kill job after this many seconds [default: 30]' },
     { name: 'wait-time', alias: 'w', type: Number, defaultValue: 20, description: 'Listen at most this long on each queue [default: 20]' },
     { name: 'include-failed', type: Boolean, description: 'When using \'*\' do not ignore fail queues.' },
     { name: 'active-only', type: Boolean, description: 'Listen only to queues with pending messages.' },
     { name: 'drain', type: Boolean, description: 'Run until no more work is found and quit. NOTE: if used with  --wait-time 0, this option will not drain queues.' },
+    { name: 'archive', type: Boolean, description: 'Does not run jobs, just prints commands to stdout. Use this flag for draining a queue and recording the commands that were in it.' },
     { name: 'fifo', alias: 'f', type: Boolean, description: 'Automatically adds .fifo to queue names. Only listens to fifo queues when using \'*\'.' }
   ].concat(globalOptionDefinitions)
 
@@ -198,9 +275,9 @@ exports.worker = function worker (argv) {
   debug('enqueue-batch argv', argv)
 
   // Parse command and options
-  let queues
+  let queues, options
   try {
-    var options = commandLineArgs(optionDefinitions, { argv, partial: true })
+    options = commandLineArgs(optionDefinitions, { argv, partial: true })
     setupVerbose(options)
     debug('worker options', options)
     if (options.help) return Promise.resolve(console.log(getUsage(usageSections)))
@@ -210,17 +287,18 @@ exports.worker = function worker (argv) {
     debug('queues', queues)
   } catch (err) {
     console.log(getUsage(usageSections.filter(s => !s.long)))
-    return Promise.reject(err)
+    throw err
   }
 
   // Load module after AWS global load
   setupAWS(options)
-  const worker = require('./worker')
+  const { listen: originalListen, requestShutdown } = await import('./worker.js')
+  const listen = testHook || originalListen
 
-  var jobCount = 0
-  var jobsSucceeded = 0
-  var jobsFailed = 0
-  var shutdownRequested = false
+  let jobCount = 0
+  let jobsSucceeded = 0
+  let jobsFailed = 0
+  let shutdownRequested = false
 
   function handleShutdown () {
     // Second signal forces shutdown
@@ -229,7 +307,7 @@ exports.worker = function worker (argv) {
       process.kill(-process.pid, 'SIGKILL')
     }
     shutdownRequested = true
-    worker.requestShutdown()
+    requestShutdown()
     if (options.verbose) {
       console.error(chalk.yellow('Shutdown requested. Will stop when current job is done or a second signal is recieved.'))
       if (process.stdout.isTTY) {
@@ -240,56 +318,59 @@ exports.worker = function worker (argv) {
   process.on('SIGINT', handleShutdown)
   process.on('SIGTERM', handleShutdown)
 
-  function workLoop () {
+  async function workLoop () {
     if (shutdownRequested) {
       if (options.verbose) console.error(chalk.blue('Shutting down as requested.'))
       return Promise.resolve()
     }
-    return worker
-      .listen(queues, options)
-      .then(function (result) {
-        debug('listen returned', result)
-
-        // Handle delay in the case we don't have any queues
-        if (result === 'noQueues') {
-          const roundDelay = Math.max(1000, options['wait-time'] * 1000)
-          if (options.verbose) console.error(chalk.yellow('No queues to listen on!'))
-          if (options.drain) {
-            console.error(chalk.blue('Shutting down because we are in drain mode and no work is available.'))
-            return Promise.resolve()
-          }
-          console.error(chalk.yellow('Retrying in ' + (roundDelay / 1000) + 's'))
-          return Q.delay(roundDelay).then(workLoop)
-        }
-
-        const ranJob = (result.jobsSucceeded + result.jobsFailed) > 0
-        jobCount += result.jobsSucceeded + result.jobsFailed
-        jobsFailed += result.jobsFailed
-        jobsSucceeded += result.jobsSucceeded
-        // Draining continues to listen as long as there is work
-        if (options.drain) {
-          if (ranJob) return workLoop()
-          if (options.verbose) {
-            console.error(chalk.blue('Ran ') + jobCount + chalk.blue(' jobs: ') + jobsSucceeded + chalk.blue(' succeeded ') + jobsFailed + chalk.blue(' failed'))
-          }
-          // return Promise.resolve(jobCount)
-        } else {
-          // If we're not draining, loop forever
-          // We can go immediately if we just ran a job
-          if (ranJob) return workLoop()
-          // Otherwise, we could do backoff logic here to slow down requests when
-          // work is not happening (at the expense of latency)
-          // But we won't do that now.
-          return workLoop()
-        }
-      })
+    // const result = await listen(queues, options)
+    const opt = getOptionsWithDefaults(options)
+    const result = (
+      await withSentry(async () => listen(queues, opt), opt)
+    )
+    debug('listen returned', result)
+
+    // Handle delay in the case we don't have any queues
+    if (result === 'noQueues') {
+      const roundDelay = Math.max(1000, options['wait-time'] * 1000)
+      if (options.verbose) console.error(chalk.yellow('No queues to listen on!'))
+      if (options.drain) {
+        console.error(chalk.blue('Shutting down because we are in drain mode and no work is available.'))
+        return Promise.resolve()
+      }
+      console.error(chalk.yellow('Retrying in ' + (roundDelay / 1000) + 's'))
+      const delay = (ms) => new Promise(resolve => setTimeout(resolve, ms))
+      return delay(roundDelay).then(workLoop)
+    }
+
+    const ranJob = (result.jobsSucceeded + result.jobsFailed) > 0
+    jobCount += result.jobsSucceeded + result.jobsFailed
+    jobsFailed += result.jobsFailed
+    jobsSucceeded += result.jobsSucceeded
+    // Draining continues to listen as long as there is work
+    if (options.drain) {
+      if (ranJob) return workLoop()
+      if (options.verbose) {
+        console.error(chalk.blue('Ran ') + jobCount + chalk.blue(' jobs: ') + jobsSucceeded + chalk.blue(' succeeded ') + jobsFailed + chalk.blue(' failed'))
+      }
+      // return Promise.resolve(jobCount)
+    } else {
+      // If we're not draining, loop forever
+      // We can go immediately if we just ran a job
+      if (ranJob) return workLoop()
+      // Otherwise, we could do backoff logic here to slow down requests when
+      // work is not happening (at the expense of latency)
+      // But we won't do that now.
+      return workLoop()
+    }
   }
+
   return workLoop()
 }
 
-exports.idleQueues = function idleQueues (argv) {
+export async function idleQueues (argv, testHook) {
   const optionDefinitions = [
-    { name: 'idle-for', alias: 'o', type: Number, defaultValue: 60, description: 'Minutes of inactivity after which a queue is considered idle. [default: 60]' },
+    { name: 'idle-for', alias: 'o', type: Number, defaultValue: defaults.idleFor, description: `Minutes of inactivity after which a queue is considered idle. [default: ${defaults.idleFor}]` },
     { name: 'delete', type: Boolean, description: 'Delete the queue if it is idle. The fail queue also must be idle unless you use --unpair.' },
     { name: 'unpair', type: Boolean, description: 'Treat queues and their fail queues as independent. By default they are treated as a unit.' },
     { name: 'include-failed', type: Boolean, description: 'When using \'*\' do not ignore fail queues. This option only applies if you use --unpair. Otherwise, queues and fail queues are treated as a unit.' }
@@ -324,9 +405,9 @@ exports.idleQueues = function idleQueues (argv) {
   debug('idleQueues argv', argv)
 
   // Parse command and options
-  let queues
+  let queues, options
   try {
-    var options = commandLineArgs(optionDefinitions, { argv, partial: true })
+    options = commandLineArgs(optionDefinitions, { argv, partial: true })
     setupVerbose(options)
     debug('idleQueues options', options)
     if (options.help) return Promise.resolve(console.log(getUsage(usageSections)))
@@ -342,48 +423,56 @@ exports.idleQueues = function idleQueues (argv) {
 
   // Load module after AWS global load
   setupAWS(options)
-  const idleQueues = require('./idleQueues')
-
-  return idleQueues
-    .idleQueues(queues, options)
-    .then(function (result) {
-      debug('idleQueues returned', result)
-      if (result === 'noQueues') return Promise.resolve()
-      const callsSQS = result.map(a => a.apiCalls.SQS).reduce((a, b) => a + b, 0)
-      const callsCloudWatch = result.map(a => a.apiCalls.CloudWatch).reduce((a, b) => a + b, 0)
-      if (options.verbose) console.error(chalk.blue('Used ') + callsSQS + chalk.blue(' SQS and ') + callsCloudWatch + chalk.blue(' CloudWatch API calls.'))
-      // Print idle queues to stdout
-      result.filter(a => a.idle).map(a => a.queue).forEach(q => console.log(q))
-      return result
-    })
-    .catch(err => {
-      if (err.code === 'AWS.SimpleQueueService.NonExistentQueue') {
-        console.error(chalk.yellow('This error can occur when you run this command immediately after deleting a queue. Wait 60 seconds and try again.'))
-        return Promise.reject(err)
-      }
-    })
+  const { idleQueues: idleQueuesOriginal } = await import('./idleQueues.js')
+  const idleQueues = testHook || idleQueuesOriginal
+  const opt = getOptionsWithDefaults(options)
+  try {
+    const result = (
+      await withSentry(async () => idleQueues(queues, opt), opt)
+    )
+    debug('idleQueues returned', result)
+    if (result === 'noQueues') return Promise.resolve()
+    const callsSQS = result.map(a => a.apiCalls.SQS).reduce((a, b) => a + b, 0)
+    const callsCloudWatch = result.map(a => a.apiCalls.CloudWatch).reduce((a, b) => a + b, 0)
+    if (options.verbose) console.error(chalk.blue('Used ') + callsSQS + chalk.blue(' SQS and ') + callsCloudWatch + chalk.blue(' CloudWatch API calls.'))
+
+    // Print idle queues to stdout
+    result.filter(a => a.idle).map(a => a.queue).forEach(q => console.log(q))
+    return result
+  } catch (err) {
+    if (err instanceof QueueDoesNotExist) {
+      console.error(chalk.yellow('This error can occur when you run this command immediately after deleting a queue. Wait 60 seconds and try again.'))
+    }
+    throw err
+  }
 }
 
-exports.root = function root (originalArgv) {
-  const validCommands = [null, 'enqueue', 'enqueue-batch', 'worker', 'idle-queues']
+export async function root (originalArgv, testHook) {
+  const validCommands = [null, 'enqueue', 'enqueue-batch', 'worker', 'idle-queues', 'monitor']
   const usageSections = [
     { content: 'qdone - Command line job queue for SQS', raw: true, long: true },
     { content: 'usage: qdone [options] <command>', raw: true },
     { content: 'Commands', raw: true },
-    { content: [
-      { name: 'enqueue', summary: 'Enqueue a single command' },
-      { name: 'enqueue-batch', summary: 'Enqueue multiple commands from stdin or a file' },
-      { name: 'worker', summary: 'Execute work on one or more queues' },
-      { name: 'idle-queues', summary: 'Write a list of idle queues to stdout' }
-    ] },
+    {
+      content: [
+        { name: 'enqueue', summary: 'Enqueue a single command' },
+        { name: 'enqueue-batch', summary: 'Enqueue multiple commands from stdin or a file' },
+        { name: 'worker', summary: 'Execute work on one or more queues' },
+        { name: 'idle-queues', summary: 'Write a list of idle queues to stdout' },
+        { name: 'monitor', summary: 'Monitor multiple queues at once' }
+      ]
+    },
     { content: 'Global Options', raw: true },
     { optionList: globalOptionDefinitions },
     awsUsageHeader, awsUsageBody
   ]
 
   // Parse command and options
+  let command, argv
   try {
-    var { command, argv } = commandLineCommands(validCommands, originalArgv)
+    const parsed = commandLineCommands(validCommands, originalArgv)
+    command = parsed.command
+    argv = parsed.argv
     debug('command', command)
 
     // Root command
@@ -391,7 +480,7 @@ exports.root = function root (originalArgv) {
       const options = commandLineArgs(globalOptionDefinitions, { argv: originalArgv })
       setupVerbose(options)
       debug('options', options)
-      if (options.version) return Promise.resolve(console.log(packageJson.version))
+      if (options.version) return console.log(packageJson.version)
       else if (options.help) return Promise.resolve(console.log(getUsage(usageSections)))
       else console.log(getUsage(usageSections.filter(s => !s.long)))
       return Promise.resolve()
@@ -403,32 +492,31 @@ exports.root = function root (originalArgv) {
 
   // Run child commands
   if (command === 'enqueue') {
-    return exports.enqueue(argv)
+    return enqueue(argv, testHook)
   } else if (command === 'enqueue-batch') {
-    return exports.enqueueBatch(argv)
+    return enqueueBatch(argv, testHook)
   } else if (command === 'worker') {
-    return exports.worker(argv)
+    return worker(argv, testHook)
   } else if (command === 'idle-queues') {
-    return exports.idleQueues(argv)
+    return idleQueues(argv, testHook)
+  } else if (command === 'monitor') {
+    return monitor(argv, testHook)
   }
 }
 
-exports.run = function run (argv) {
+export async function run (argv, testHook) {
   debug('run', argv)
-  return exports
-    .root(argv)
-    .then(() => {
-      // If cache actually is active, it will keep our program from exiting
-      // until we disconnect the cache client
-      const cache = require('./cache')
-      cache.resetClient()
-    })
-    .catch(function (err) {
-      if (err.code === 'AccessDenied') console.log(getUsage([awsUsageHeader, awsUsageBody]))
-      console.error(chalk.red.bold(err))
-      console.error(err.stack.slice(err.stack.indexOf('\n') + 1))
-      throw err
-    })
+  try {
+    await root(argv, testHook)
+    // If cache actually is active, it will keep our program from exiting
+    // until we disconnect the cache client
+    shutdownCache()
+  } catch (err) {
+    if (err.Code === 'AccessDenied') console.log(getUsage([awsUsageHeader, awsUsageBody]))
+    console.error(chalk.red.bold(err))
+    console.error(err.stack.slice(err.stack.indexOf('\n') + 1))
+    throw err
+  }
 }
 
 debug('loaded')