qdone 1.7.0 → 2.0.0-alpha

package/src/qrlCache.js

@@ -1,134 +1,161 @@
+/**
+ * A cache for maintaining queue urls. Avoids having to call the GetQueueUrl
+ * api too often.
+ */
 
-const Q = require('q')
-const debug = require('debug')('qdone:qrlCache')
-const chalk = require('chalk')
-const url = require('url')
-const path = require('path')
-const AWS = require('aws-sdk')
+import { URL } from 'url'
+import { basename } from 'path'
+import { ListQueuesCommand, GetQueueUrlCommand, QueueDoesNotExist } from '@aws-sdk/client-sqs'
+import { getSQSClient } from './sqs.js'
+import Debug from 'debug'
+const debug = Debug('qdone:qrlCache')
 
-const qcache = {}
+// The actual cache is shared across callers.
+const qcache = new Map()
 
-function _get (qname) {
-  return qcache[qname]
+//
+// Normalizes a queue name to end with .fifo if opt.fifo is set
+//
+export const fifoSuffix = '.fifo'
+export function normalizeQueueName (qname, opt) {
+  const hasFifo = qname.endsWith(fifoSuffix)
+  const needsFifo = opt.fifo && qname.slice(-1) !== '*'
+  const needsPrefix = !qname.startsWith(opt.prefix)
+  const base = hasFifo ? qname.slice(0, -fifoSuffix.length) : qname
+  return (needsPrefix ? opt.prefix : '') + base + (needsFifo ? fifoSuffix : '')
 }
 
 //
-// Normalizes a queue name to end with .fifo if options.fifo is set
+// Normalizes fail queue name, appending both --fail-suffix and .fifo depending on opt
 //
-const fifoSuffix = '.fifo'
-exports.fifoSuffix = fifoSuffix
-exports.normalizeQueueName = function normalizeQueueName (qname, options) {
-  const sliced = qname.slice(0, -fifoSuffix.length)
-  const suffix = qname.slice(-fifoSuffix.length)
-  const base = suffix === fifoSuffix ? sliced : qname
-  return base + (options.fifo && qname.slice(-1) !== '*' ? fifoSuffix : '')
+export function normalizeFailQueueName (fqname, opt) {
+  const newopt = Object.assign({}, opt, { fifo: false })
+  const base = normalizeQueueName(fqname, newopt) // strip fifo
+  debug({ fqname, base, opt })
+  const needsFail = !base.endsWith(opt.failSuffix)
+  return base + (needsFail ? opt.failSuffix : '') + (opt.fifo ? fifoSuffix : '')
 }
 
 //
-// Normalizes fail queue name, appending both --fail-suffix and .fifo depending on options
+// Normalizes dlq queue name, appending both --dlq-suffix and .fifo depending on opt
 //
-exports.normalizeFailQueueName = function normalizeFailQueueName (qname, options) {
-  qname = exports.normalizeQueueName(qname, { fifo: false }) // strip .fifo if it is there
-  const sliced = qname.slice(0, -options['fail-suffix'].length)
-  const suffix = qname.slice(-options['fail-suffix'].length)
-  const base = suffix === options['fail-suffix'] ? sliced : qname // strip --fail-suffix if it is there
-  return (base + options['fail-suffix']) + (options.fifo ? fifoSuffix : '')
+export function normalizeDLQName (dqname, opt) {
+  const newopt = Object.assign({}, opt, { fifo: false })
+  const base = normalizeQueueName(dqname, newopt) // strip fifo
+  const needsDead = !base.endsWith(opt.dlqSuffix)
+  return base + (needsDead ? opt.dlqSuffix : '') + (opt.fifo ? fifoSuffix : '')
 }
 
 //
 // Clear cache
 //
-exports.clear = function clear () {
-  Object.keys(qcache).forEach(function (key) {
-    delete qcache[key]
-  })
+export function qrlCacheClear () {
+  qcache.clear()
 }
 
 //
 // Get QRL (Queue URL)
 // Returns a promise for the queue name
 //
-exports.get = function get (qname) {
+export async function qrlCacheGet (qname) {
   debug('get', qname, qcache)
-  const sqs = new AWS.SQS()
-  if (Object.prototype.hasOwnProperty.call(qcache, qname)) return Q.fcall(_get, qname)
-  return sqs
-    .getQueueUrl({ QueueName: qname })
-    .promise()
-    .then(function (data) {
-      debug('getQueueUrl returned', data)
-      qcache[qname] = data.QueueUrl
-      debug('qcache', Object.keys(qcache), 'get', qname, ' => ', qcache[qname])
-      return Q.fcall(_get, qname)
-    })
+  if (qcache.has(qname)) return qcache.get(qname)
+  const client = getSQSClient()
+  const input = { QueueName: qname }
+  const cmd = new GetQueueUrlCommand(input)
+  // debug({ cmd })
+  const result = await client.send(cmd)
+  // debug('result', result)
+  // if (!result) throw new Error(`No such queue ${qname}`)
+  const { QueueUrl: qrl } = result
+  // debug('getQueueUrl returned', data)
+  qcache.set(qname, qrl)
+  // debug('qcache', Object.keys(qcache), 'get', qname, ' => ', qcache[qname])
+  return qrl
 }
 
 //
 // Set QRL (Queue URL)
 // Immediately updates the cache
 //
-exports.set = function set (qname, qrl) {
-  qcache[qname] = qrl
-  debug('qcache', Object.keys(qcache), 'set', qname, ' => ', qcache[qname])
+export function qrlCacheSet (qname, qrl) {
+  qcache.set(qname, qrl)
+  // debug('qcache', Object.keys(qcache), 'set', qname, ' => ', qcache[qname])
 }
 
 //
 // Invalidate cache for given qname
 //
-exports.invalidate = function invalidate (qname) {
-  debug('qcache', Object.keys(qcache), 'delete', qname, ' (was ', qcache[qname], ')')
-  delete qcache[qname]
+export function qrlCacheInvalidate (qname) {
+  // debug('qcache', Object.keys(qcache), 'delete', qname, ' (was ', qcache[qname], ')')
+  qcache.delete(qname)
 }
 
 //
 // Ingets multiple QRLs
 // Extracts queue names from an array of QRLs and immediately updates the cache.
 //
-function ingestQRLs (qrls) {
+export function ingestQRLs (qrls) {
   const pairs = []
   debug('ingestQRLs', qrls)
-  qrls.forEach(function (qrl) {
-    const qname = path.basename((new url.URL(qrl)).pathname)
-    qcache[qname] = qrl
-    pairs.push({ qname: qname, qrl: qrl })
-    debug('qcache', Object.keys(qcache), 'ingestQRLs', qname, ' => ', qcache[qname])
-  })
+  for (const qrl of qrls) {
+    const qname = basename((new URL(qrl)).pathname)
+    qrlCacheSet(qname, qrl)
+    pairs.push({ qname, qrl })
+    // debug('qcache', Object.keys(qcache), 'ingestQRLs', qname, ' => ', qcache[qname])
+  }
   return pairs
 }
 
+/**
+ * Returns qrls for queues matching the given prefix and regex.
+ */
+export async function getMatchingQueues (prefix, nextToken) {
+  debug('getMatchingQueues', prefix, nextToken)
+  const input = { QueueNamePrefix: prefix, MaxResults: 1000 }
+  if (nextToken) input.NextToken = nextToken
+  const client = getSQSClient()
+  const command = new ListQueuesCommand(input)
+  const result = await client.send(command)
+  debug({ result })
+  const { QueueUrls: qrls, NextToken: keepGoing } = result || {}
+  debug({ qrls, keepGoing })
+  if (keepGoing) (qrls || []).push(...await getMatchingQueues(prefix, keepGoing))
+  return qrls || []
+}
+
 //
 // Resolves into a flattened aray of {qname: ..., qrl: ...} objects.
 //
-exports.getQnameUrlPairs = function getQnameUrlPairs (qnames, options) {
-  const sqs = new AWS.SQS()
-  return Q.all(qnames.map(function (qname) {
-    return qname.slice(-1) === '*' // wildcard queues support
-      ? sqs
-        .listQueues({ QueueNamePrefix: qname.slice(0, -1) })
-        .promise()
-        .then(function (data) {
-          debug('listQueues return', data)
-          if (options.fifo) {
-            // Remove non-fifo queues
-            data.QueueUrls = (data.QueueUrls || []).filter(url => url.slice(-fifoSuffix.length) === fifoSuffix)
-          }
-          return ingestQRLs(data.QueueUrls || [])
-        })
-      : exports
-        .get(qname)
-        .then(function (qrl) {
-          debug('qrlCache.get returned', qrl)
-          return { qname: qname, qrl: qrl }
-        })
-        .catch(function (err) {
-          debug('qrlCache.get failed', err)
-          if (options.verbose) console.error('  ' + chalk.red(qname.slice(options.prefix.length) + ' - ' + err))
-        })
-  }))
-    .then(function (results) {
-    // flatten nested results
-      return ([].concat.apply([], results)).filter(r => r)
-    })
+export async function getQnameUrlPairs (qnames, opt) {
+  const promises = qnames.map(
+    async function getQueueUrlOrUrls (qname) {
+      if (qname.slice(-1) === '*') {
+        // Wildcard queues support
+        const prefix = qname.slice(0, -1)
+        const queues = await getMatchingQueues(prefix)
+        debug('listQueues return', queues)
+        if (opt.fifo) {
+          // Remove non-fifo queues
+          const filteredQueues = queues.filter(url => url.slice(-fifoSuffix.length) === fifoSuffix)
+          return ingestQRLs(filteredQueues)
+        } else {
+          return ingestQRLs(queues)
+        }
+      } else {
+        // Normal, non wildcard queue support
+        try {
+          return { qname, qrl: await qrlCacheGet(qname) }
+        } catch (err) {
+          if (!(err instanceof QueueDoesNotExist)) throw err
+        }
+      }
+    }
+  )
+
+  // Flatten nested results
+  const results = await Promise.all(promises)
+  return ([].concat.apply([], results)).filter(r => r)
 }
 
 debug('loaded')

package/src/sentry.js

@@ -0,0 +1,30 @@
+/**
+ * Routines for handling Sentry instrumentation
+ */
+
+import Debug from 'debug'
+import { init, captureException } from '@sentry/node'
+
+const debug = Debug('qdone:sentry')
+
+let sentryWasInit = false
+export async function withSentry (callback, opt) {
+  // Bail if sentry isn't enabled
+  if (!opt.sentryDsn) return callback()
+
+  // Init sentry if it's not already
+  if (!sentryWasInit) {
+    init({ dsn: opt.sentryDsn, traceSampleRate: 0 })
+    sentryWasInit = true
+  }
+  try {
+    const result = await callback()
+    debug({ result })
+    return result
+  } catch (err) {
+    debug({ err })
+    const sentryResult = await captureException(err)
+    debug({ sentryResult })
+    throw err
+  }
+}

package/src/sqs.js

@@ -0,0 +1,73 @@
+/**
+ * Functions that deal with SQS
+ */
+
+import { SQSClient, ListQueuesCommand, GetQueueAttributesCommand } from '@aws-sdk/client-sqs'
+import { basename } from 'path'
+import Debug from 'debug'
+const debug = Debug('qdone:sqs')
+
+/**
+ * Utility function to return an instantiated, shared SQSClient.
+ */
+let client
+export function getSQSClient () {
+  if (client) return client
+  client = new SQSClient()
+  return client
+}
+
+/**
+ * Utility function to set the client explicitly, used in testing.
+ */
+export function setSQSClient (explicitClient) {
+  client = explicitClient
+}
+
+/**
+ * Returns qrls for queues matching the given prefix and regex.
+ */
+export async function getMatchingQueues (prefix, regex) {
+  const input = { QueueNamePrefix: prefix, MaxResults: 1000 }
+  const client = getSQSClient()
+  async function processQueues (nextToken) {
+    if (nextToken) input.NextToken = nextToken
+    const command = new ListQueuesCommand(input)
+    // debug({ nextToken, input, command })
+    const result = await client.send(command)
+    // debug({ result })
+    const { QueueUrls: qrls, NextToken: nextToken2 } = result
+    // debug({ qrls, nextToken2 })
+    return (qrls || []).filter(q => regex.test(q)).concat(nextToken2 ? await processQueues(nextToken2) : [])
+  }
+  return processQueues()
+}
+
+/**
+ * Gets attributes on every queue in parallel.
+ */
+export async function getQueueAttributes (qrls) {
+  const promises = []
+  // debug({ qrls })
+  for (const qrl of qrls) {
+    const input = {
+      QueueUrl: qrl,
+      AttributeNames: [
+        'ApproximateNumberOfMessages',
+        'ApproximateNumberOfMessagesNotVisible',
+        'ApproximateNumberOfMessagesDelayed'
+      ]
+    }
+    const command = new GetQueueAttributesCommand(input)
+    // debug({ input, command })
+    promises.push((async () => {
+      const queue = basename(qrl)
+      const result = await client.send(command)
+      // debug({ queue, result })
+      return { queue, result }
+    })())
+  }
+  return Promise.all(promises)
+}
+
+debug('loaded')