qdone 2.0.34-alpha → 2.0.36-alpha

package/src/check.js

@@ -0,0 +1,205 @@
+import chalk from 'chalk'
+import Debug from 'debug'
+import {
+  GetQueueAttributesCommand,
+  SetQueueAttributesCommand,
+  QueueDoesNotExist
+} from '@aws-sdk/client-sqs'
+
+import {
+  qrlCacheGet,
+  normalizeQueueName,
+  normalizeFailQueueName,
+  normalizeDLQName,
+  getQnameUrlPairs
+} from './qrlCache.js'
+import { getSQSClient } from './sqs.js'
+import {
+  getDLQParams,
+  getFailParams,
+  getQueueParams,
+  getOrCreateFailQueue,
+  getOrCreateDLQ
+} from './enqueue.js'
+import { getOptionsWithDefaults } from './defaults.js'
+
+const debug = Debug('qdone:check')
+
+/**
+ * Loops through attributes, checking each and returning true if they match
+ */
+export function attributesMatch (current, desired, opt, indent = '') {
+  let match = true
+  for (const attribute in desired) {
+    if (current[attribute] !== desired[attribute]) {
+      if (opt.verbose) console.error(chalk.yellow(indent + 'Attribute mismatch: ') + attribute + chalk.yellow(' should be ') + desired[attribute] + chalk.yellow(' but is ') + current[attribute])
+      match = false
+    }
+  }
+  return match
+}
+
+/**
+ * Checks a DLQ, creating if the create option is set and modifying it if the
+ * overwrite option is set.
+ */
+export async function checkDLQ (queue, qrl, opt, indent = '') {
+  debug({ checkDLQ: { queue, qrl } })
+  const dqname = normalizeDLQName(queue, opt)
+  if (opt.verbose) console.error(chalk.blue(indent + 'checking ') + dqname)
+
+  // Check DLQ
+  let dqrl
+  try {
+    dqrl = await qrlCacheGet(dqname)
+  } catch (e) {
+    if (!(e instanceof QueueDoesNotExist)) throw e
+    if (opt.verbose) console.error(chalk.red(indent + '  does not exist'))
+    if (opt.create) {
+      if (opt.verbose) console.error(chalk.green(indent + '  creating'))
+      dqrl = await getOrCreateDLQ(queue, opt)
+    } else {
+      return
+    }
+  }
+
+  // Check attributes
+  const { params: { Attributes: desired } } = getDLQParams(queue, opt)
+  const { Attributes: current } = await getQueueAttributes(dqrl)
+  if (attributesMatch(current, desired, opt, indent + '  ')) {
+    if (opt.verbose) console.error(chalk.green(indent + '  all good'))
+  } else {
+    if (opt.overwrite) {
+      if (opt.verbose) console.error(chalk.green(indent + '  modifying'))
+      return setQueueAttributes(dqrl, desired)
+    }
+  }
+}
+
+/**
+ * Checks a fail queue, creating if the create option is set and modifying it if the
+ * overwrite option is set.
+ */
+export async function checkFailQueue (queue, qrl, opt, indent = '') {
+  // Check dead first
+  await checkDLQ(queue, qrl, opt, indent)
+
+  // Check fail queue
+  const fqname = normalizeFailQueueName(queue, opt)
+  let fqrl
+  try {
+    fqrl = await qrlCacheGet(fqname)
+  } catch (e) {
+    if (!(e instanceof QueueDoesNotExist)) throw e
+    if (opt.verbose) console.error(chalk.red(indent + '  does not exist'))
+    if (opt.create) {
+      if (opt.verbose) console.error(chalk.green(indent + '  creating'))
+      fqrl = await getOrCreateFailQueue(queue, opt)
+    } else {
+      return
+    }
+  }
+
+  try {
+    // Get fail queue params, creating fail queue if it doesn't exist and create flag is set
+    if (opt.verbose) console.error(chalk.blue(indent + 'checking ') + fqname)
+    const { params: { Attributes: desired } } = await getFailParams(queue, opt)
+    const { Attributes: current } = await getQueueAttributes(fqrl)
+    if (attributesMatch(current, desired, opt, indent + '  ')) {
+      if (opt.verbose) console.error(chalk.green(indent + '  all good'))
+    } else {
+      if (opt.overwrite) {
+        if (opt.verbose) console.error(chalk.green(indent + '  modifying'))
+        return setQueueAttributes(fqrl, desired)
+      }
+    }
+  } catch (e) {
+    if (!(e instanceof QueueDoesNotExist)) throw e
+    if (opt.verbose) console.error(chalk.red(indent + '  missing dlq'))
+  }
+}
+
+/**
+ * Checks a queue, creating or modifying it if the create option is set
+ * and it needs it.
+ */
+export async function checkQueue (queue, qrl, opt, indent = '') {
+  const qname = normalizeQueueName(queue, opt)
+  if (opt.verbose) console.error(chalk.blue(indent + 'checking ') + qname)
+  await checkFailQueue(queue, qrl, opt, indent + '  ')
+  try {
+    const { params: { Attributes: desired } } = await getQueueParams(queue, opt)
+    const { Attributes: current, $metadata } = await getQueueAttributes(qrl)
+    debug({ current, $metadata })
+    if (attributesMatch(current, desired, opt, indent + '  ')) {
+      if (opt.verbose) console.error(chalk.green(indent + '  all good'))
+    } else {
+      if (opt.overwrite) {
+        if (opt.verbose) console.error(chalk.green(indent + '  modifying'))
+        return setQueueAttributes(qrl, desired)
+      }
+    }
+  } catch (e) {
+    if (!(e instanceof QueueDoesNotExist)) throw e
+    if (opt.verbose) console.error(chalk.red(indent + '  missing fail queue'))
+  }
+}
+
+export async function getQueueAttributes (qrl) {
+  debug('getQueueAttributes(', qrl, ')')
+  const client = getSQSClient()
+  const params = { AttributeNames: ['All'], QueueUrl: qrl }
+  const cmd = new GetQueueAttributesCommand(params)
+  // debug({ cmd })
+  const data = await client.send(cmd)
+  debug('GetQueueAttributes returned', data)
+  return data
+}
+
+export async function setQueueAttributes (qrl, attributes) {
+  debug('setQueueAttributes(', qrl, attributes, ')')
+  const client = getSQSClient()
+  const params = { Attributes: attributes, QueueUrl: qrl }
+  const cmd = new SetQueueAttributesCommand(params)
+  debug({ cmd })
+  const data = await client.send(cmd)
+  debug('SetQueueAttributes returned', data)
+  return data
+}
+
+//
+// Enqueue a single command
+// Returns a promise for the SQS API response.
+//
+export async function check (queues, options) {
+  debug('check(', { queues }, ')')
+  const opt = getOptionsWithDefaults(options)
+
+  // Start processing
+  if (opt.verbose) console.error(chalk.blue('Resolving queues: ') + queues.join(' '))
+  const qnames = queues.map(queue => normalizeQueueName(queue, opt))
+  const pairs = await getQnameUrlPairs(qnames, opt)
+
+  // Figure out which queues we want to listen on, choosing between active and
+  // all, filtering out failed queues if the user wants that
+  const selectedPairs = pairs
+    .filter(({ qname }) => qname)
+    .filter(({ qname }) => {
+      const suf = opt.failSuffix + (opt.fifo ? '.fifo' : '')
+      const deadSuf = opt.dlqSuffix + (opt.fifo ? '.fifo' : '')
+      const isFailQueue = qname.slice(-suf.length) === suf
+      const isDeadQueue = qname.slice(-deadSuf.length) === deadSuf
+      const isPlain = !isFailQueue && !isDeadQueue
+      const shouldInclude = isPlain || (isFailQueue && opt.includeFailed) || (isDeadQueue && opt.includeDead)
+      return shouldInclude
+    })
+
+  for (const { qname, qrl } of selectedPairs) {
+    debug({ qname, qrl })
+    await checkQueue(qname, qrl, opt)
+  }
+
+  debug({ pairs })
+}
+
+debug('loaded')

package/src/cli.js

@@ -42,6 +42,9 @@ const globalOptionDefinitions = [
   { name: 'cache-prefix', type: String, description: `Prefix for all keys in cache. [default: ${defaults.cachePrefix}]` },
   { name: 'cache-ttl-seconds', type: Number, description: `Number of seconds to cache GetQueueAttributes calls. [default: ${defaults.cacheTtlSeconds}]` },
   { name: 'help', type: Boolean, description: 'Print full help message.' },
+  { name: 'external-dedup', type: Boolean, description: 'Moves deduplication from SQS to qdone external cache, allowing a longer deduplication window and alternate semantics.' },
+  { name: 'dedup-period', type: Number, description: 'Number of seconds (counting from the first enqueue) to prevent a duplicate message from being sent. Resets after a message with that deduplication id has been successfully processed. Minumum 360 seconds.' },
+  { name: 'dedup-stats', type: Boolean, description: 'Keeps statistics on dedup keys. Disabled by default. Increases load on redis.' },
   { name: 'sentry-dsn', type: String, description: 'Optional Sentry DSN to track unhandled errors.' }
 ]
 
@@ -54,8 +57,23 @@ const enqueueOptionDefinitions = [
   { name: 'delay', type: Number, description: 'Delays delivery of the enqueued message by the given number of seconds (up to 900 seconds, or 15 minutes). Defaults to immediate delivery (no delay).' },
   { name: 'fail-delay', type: Number, description: 'Delays delivery of all messages on this queue by the given number of seconds (up to 900 seconds, or 15 minutes). Only takes effect if this queue is created during this enqueue operation. Defaults to immediate delivery (no delay).' },
   { name: 'dlq', type: Boolean, description: 'Send messages from the failed queue to a DLQ.' },
-  { name: 'dql-suffix', type: String, description: `Suffix to append to each queue to generate DLQ name [default: ${defaults.dlqSuffix}]` },
-  { name: 'dql-after', type: String, description: `Drives message to the DLQ after this many failures in the failed queue. [default: ${defaults.dlqAfter}]` },
+  { name: 'dlq-suffix', type: String, description: `Suffix to append to each queue to generate DLQ name [default: ${defaults.dlqSuffix}]` },
+  { name: 'dlq-after', type: String, description: `Drives message to the DLQ after this many failures in the failed queue. [default: ${defaults.dlqAfter}]` },
+  { name: 'tag', type: String, multiple: true, description: 'Adds an AWS tag to queue creation. Use the format Key=Value. Can specify multiple times.' }
+]
+
+const checkOptionDefinitions = [
+  { name: 'create', type: Boolean, description: 'Create queues that do not exist' },
+  { name: 'overwrite', type: Boolean, description: 'Overwrite queue attributes that do not match expected' },
+  { name: 'fifo', alias: 'f', type: Boolean, description: 'Create new queues as FIFOs' },
+  { name: 'include-failed', type: Boolean, description: 'When using \'*\' do not ignore fail queues.' },
+  { name: 'include-dead', type: Boolean, description: 'When using \'*\' do not ignore dead queues.' },
+  { name: 'message-retention-period', type: Number, description: `Number of seconds to retain jobs (up to 14 days). [default: ${defaults.messageRetentionPeriod}]` },
+  { name: 'delay', type: Number, description: 'Delays delivery of the enqueued message by the given number of seconds (up to 900 seconds, or 15 minutes). Defaults to immediate delivery (no delay).' },
+  { name: 'fail-delay', type: Number, description: 'Delays delivery of all messages on this queue by the given number of seconds (up to 900 seconds, or 15 minutes). Only takes effect if this queue is created during this enqueue operation. Defaults to immediate delivery (no delay).' },
+  { name: 'dlq', type: Boolean, description: 'Send messages from the failed queue to a DLQ.' },
+  { name: 'dlq-suffix', type: String, description: `Suffix to append to each queue to generate DLQ name [default: ${defaults.dlqSuffix}]` },
+  { name: 'dlq-after', type: String, description: `Drives message to the DLQ after this many failures in the failed queue. [default: ${defaults.dlqAfter}]` },
   { name: 'tag', type: String, multiple: true, description: 'Adds an AWS tag to queue creation. Use the format Key=Value. Can specify multiple times.' }
 ]
 
@@ -120,6 +138,64 @@ export async function enqueue (argv, testHook) {
   return result
 }
 
+export async function check (argv, testHook) {
+  const optionDefinitions = [].concat(checkOptionDefinitions, globalOptionDefinitions)
+  const usageSections = [
+    { content: 'usage: qdone check [options] <queue>', raw: true },
+    { content: 'Options', raw: true },
+    { optionList: optionDefinitions },
+    { content: 'SQS API Call Complexity', raw: true, long: true },
+    {
+      content: [
+        { count: '2 [ + 3 ]', summary: 'one call to resolve the queue name\none call to check the command\none extra calls if the queue does not match and --modify option is set' }
+      ],
+      long: true
+    },
+    awsUsageHeader, awsUsageBody
+  ]
+  debug('check argv', argv)
+
+  // Parse command and options
+  let queues, options
+  try {
+    options = commandLineArgs(optionDefinitions, { argv, partial: true })
+    setupVerbose(options)
+    debug('check options', options)
+    if (options.help) return Promise.resolve(console.log(getUsage(usageSections)))
+    if (!options._unknown || options._unknown.length === 0) throw new UsageError('check requres one or more <queue> arguments')
+    queues = options._unknown
+    debug('queues', queues)
+  } catch (err) {
+    console.log(getUsage(usageSections.filter(s => !s.long)))
+    throw err
+  }
+
+  // Process tags
+  if (options.tag && options.tag.length) {
+    options.tags = {}
+    for (const input of options.tag) {
+      debug({ input })
+      if (input.indexOf('=') === -1) throw new UsageError('Tags must be separated with the "=" character.')
+      const [key, ...rest] = input.split('=')
+      const value = rest.join('=')
+      debug({ input, key, rest, value, tags: options.tags })
+      options.tags[key] = value
+    }
+  }
+
+  // Load module after AWS global load
+  setupAWS(options)
+  const { check: checkOriginal } = await import('./check.js')
+  const check = testHook || checkOriginal
+
+  // Normal (non batch) enqueue
+  const opt = getOptionsWithDefaults(options)
+  const result = (
+    await withSentry(async () => check(queues, opt), opt)
+  )
+  return result
+}
+
 const monitorOptionDefinitions = [
   { name: 'save', alias: 's', type: Boolean, description: 'Saves data to CloudWatch' }
 ]
@@ -449,7 +525,7 @@ export async function idleQueues (argv, testHook) {
 }
 
 export async function root (originalArgv, testHook) {
-  const validCommands = [null, 'enqueue', 'enqueue-batch', 'worker', 'idle-queues', 'monitor']
+  const validCommands = [null, 'enqueue', 'enqueue-batch', 'worker', 'idle-queues', 'monitor', 'check']
   const usageSections = [
     { content: 'qdone - Command line job queue for SQS', raw: true, long: true },
     { content: 'usage: qdone [options] <command>', raw: true },
@@ -502,6 +578,8 @@ export async function root (originalArgv, testHook) {
     return idleQueues(argv, testHook)
   } else if (command === 'monitor') {
     return monitor(argv, testHook)
+  } else if (command === 'check') {
+    return check(argv, testHook)
   }
 }
 

package/src/dedup.js

@@ -0,0 +1,256 @@
+import { createHash } from 'crypto'
+import { v1 as uuidV1 } from 'uuid'
+import { getCacheClient } from './cache.js'
+import Debug from 'debug'
+const debug = Debug('qdone:dedup')
+
+/**
+ * Returns a MessageDeduplicationId key appropriate for using with Amazon SQS
+ * for the given message. The passed dedupContent will be returned untouched
+ * if it meets all the requirements for SQS's MessageDeduplicationId,
+ * otherwise disallowed characters will be replaced by `_` and content longer
+ * than 128 characters will be truncated and a hash of the content appended.
+ * @param {String} dedupContent - Content used to construct the deduplication id.
+ * @param {Object} opt - Opt object from getOptionsWithDefaults()
+ * @returns {String} the cache key
+ */
+export function getDeduplicationId (dedupContent, opt) {
+  debug({ getDeduplicationId: { dedupContent } })
+  // Don't transmit long keys to redis
+  dedupContent = dedupContent.trim().replace(/[^a-zA-Z0-9!"#$%&'()*+,-./:;<=>?@[\\\]^_`{|}~]/g, '_')
+  const max = 128
+  const sep = '...sha1:'
+  if (dedupContent.length > max) {
+    dedupContent = dedupContent.slice(0, max - sep.length - 40) + '...sha1:' + createHash('sha1').update(dedupContent).digest('hex')
+  }
+  return dedupContent
+}
+
+/**
+ * Returns the cache key given a deduplication id.
+ * @param {String} dedupId - a deduplication id returned from getDeduplicationId
+ * @param opt - Opt object from getOptionsWithDefaults()
+ * @returns the cache key
+ */
+export function getCacheKey (dedupId, opt) {
+  const cacheKey = opt.cachePrefix + 'dedup:' + dedupId
+  debug({ getCacheKey: { cacheKey } })
+  return cacheKey
+}
+
+/**
+ * Modifies a message (parameters to SendMessageCommand) to add the parameters
+ * for whatever deduplication options the caller has set.
+ * @param {String} message - parameters to SendMessageCommand
+ * @param {Object} opt - Opt object from getOptionsWithDefaults()
+ * @param {Object} [messageOptions] - optional per message options. We only care about the key deduplicationId.
+ * @returns {Object} the modified parameters/message object
+ */
+export function addDedupParamsToMessage (message, opt, messageOptions) {
+  // Either of these means we need to calculate an id
+  if (opt.fifo || opt.externalDedup) {
+    const uuidFunction = opt.uuidFunction || uuidV1
+
+    if (opt.deduplicationId) message.MessageDeduplicationId = opt.deduplicationId
+    if (opt.dedupIdPerMessage) message.MessageDeduplicationId = uuidFunction()
+    if (messageOptions?.deduplicationId) message.MessageDeduplicationId = messageOptions.deduplicationId
+
+    // Fallback to using the message body
+    if (!message.MessageDeduplicationId) {
+      message.MessageDeduplicationId = getDeduplicationId(message.MessageBody, opt)
+    }
+
+    // Track our own dedup id so we can look it up upon ReceiveMessage
+    if (opt.externalDedup) {
+      message.MessageAttributes = {
+        QdoneDeduplicationId: {
+          StringValue: message.MessageDeduplicationId,
+          DataType: 'String'
+        }
+      }
+      // If we are using our own dedup, then we must disable the SQS dedup by
+      // providing a different unique ID. Otherwise SQS will interact with us.
+      if (opt.fifo) message.MessageDeduplicationId = uuidFunction()
+    }
+
+    // Non fifo can't have this parameter
+    if (!opt.fifo) delete message.MessageDeduplicationId
+  }
+  return message
+}
+
+/**
+ * Updates statistics in redis, of which there are two:
+ * 1. duplicateSet - a set who's members are cache keys and scores are the number of duplicate
+ *                   runs prevented by dedup.
+ * 2. expirationSet - a set who's members are cache keys and scores are when the cache key expires
+ * @param {String} cacheKey
+ * @param {Number} duplicates - the number of duplicates, must be at least 1 to gather stats
+ * @param {Number} expireAt - timestamp for when this key's dedupPeriod expires
+ * @param {Object} opt - Opt object from getOptionsWithDefaults()
+ * @param {Object} pipeline - (Optional) redis pipeline you will exec() yourself
+ */
+export async function updateStats (cacheKey, duplicates, expireAt, opt, pipeline) {
+  if (duplicates >= 1) {
+    const duplicateSet = opt.cachePrefix + 'dedup-stats:duplicateSet'
+    const expirationSet = opt.cachePrefix + 'dedup-stats:expirationSet'
+    const hadPipeline = !!pipeline
+    if (!hadPipeline) pipeline = getCacheClient(opt).multi()
+    pipeline.zadd(duplicateSet, 'GT', duplicates, cacheKey)
+    pipeline.zadd(expirationSet, 'GT', expireAt, cacheKey)
+    if (!hadPipeline) await pipeline.exec()
+  }
+}
+
+/**
+ * Removes expired items from stats.
+ */
+export async function statMaintenance (opt) {
+  const duplicateSet = opt.cachePrefix + 'dedup-stats:duplicateSet'
+  const expirationSet = opt.cachePrefix + 'dedup-stats:expirationSet'
+  const client = getCacheClient(opt)
+  const now = new Date().getTime()
+
+  // Grab a batch of expired keys
+  debug({ statMaintenance: { aboutToGo: true, expirationSet }})
+  const expiredStats = await client.zrange(expirationSet, '-inf', now, 'BYSCORE')
+  debug({ statMaintenance: { expiredStats }})
+
+  // And remove them from indexes, main storage
+  if (expiredStats.length) {
+    const result = await client.multi()
+      .zrem(expirationSet, expiredStats)
+      .zrem(duplicateSet, expiredStats)
+      .exec()
+    debug({ statMaintenance: { result }})
+  }
+}
+
+/**
+ * Determines whether we should enqueue this message or whether it is a duplicate.
+ * Returns true if enqueuing the message would not result in a duplicate.
+ * @param {Object} message - Parameters to SendMessageCommand
+ * @param {Object} opt - Opt object from getOptionsWithDefaults()
+ * @returns {Boolean} true if the message can be enqueued without duplicate, else false
+ */
+export async function dedupShouldEnqueue (message, opt) {
+  const client = getCacheClient(opt)
+  const dedupId = message?.MessageAttributes?.QdoneDeduplicationId?.StringValue
+  const cacheKey = getCacheKey(dedupId, opt)
+  const expireAt = new Date().getTime() + opt.dedupPeriod
+  const copies = await client.incr(cacheKey)
+  debug({ action: 'shouldEnqueue', cacheKey, copies })
+  if (copies === 1) {
+    await client.expireat(cacheKey, expireAt)
+    return true
+  }
+  if (opt.dedupStats) {
+    const duplicates = copies - 1
+    await updateStats(cacheKey, duplicates, expireAt, opt)
+  }
+  return false
+}
+
+/**
+ * Determines which messages we should enqueue, returning only those that
+ * would not be duplicates.
+ * @param {Array[Object]} messages - Entries array for the SendMessageBatchCommand
+ * @param {Object} opt - Opt object from getOptionsWithDefaults()
+ * @returns {Array[Object]} an array of messages that can be safely enqueued. Could be empty.
+ */
+export async function dedupShouldEnqueueMulti (messages, opt) {
+  debug({ dedupShouldEnqueueMulti: { messages, opt }})
+  const expireAt = new Date().getTime() + opt.dedupPeriod
+  // Increment all
+  const incrPipeline = getCacheClient(opt).pipeline()
+  for (const message of messages) {
+    const dedupId = message?.MessageAttributes?.QdoneDeduplicationId?.StringValue
+    const cacheKey = getCacheKey(dedupId, opt)
+    incrPipeline.incr(cacheKey)
+  }
+  const responses = await incrPipeline.exec()
+  debug({ dedupShouldEnqueueMulti: { messages, responses } })
+
+  // Figure out dedup period
+  const minDedupPeriod = 6 * 60
+  const dedupPeriod = Math.min(opt.dedupPeriod, minDedupPeriod)
+
+  // Interpret responses and expire keys for races we won
+  const expirePipeline = getCacheClient(opt).pipeline()
+  const statsPipeline = opt.dedupStats ? getCacheClient(opt).pipeline() : undefined
+  const messagesToEnqueue = []
+  for (let i = 0; i < messages.length; i++) {
+    const message = messages[i]
+    const [, copies] = responses[i]
+    const dedupId = message?.MessageAttributes?.QdoneDeduplicationId?.StringValue
+    const cacheKey = getCacheKey(dedupId, opt)
+    if (copies === 1) {
+      messagesToEnqueue.push(message)
+      expirePipeline.expireat(cacheKey, expireAt)
+    } else if (opt.dedupStats) {
+      const duplicates = copies - 1
+      updateStats(cacheKey, duplicates, expireAt, opt, statsPipeline)
+    }
+  }
+  await expirePipeline.exec()
+  if (opt.dedupStats) await statsPipeline.exec()
+  return messagesToEnqueue
+}
+
+/**
+ * Marks a message as processed so that subsequent calls to dedupShouldEnqueue
+ * and dedupShouldEnqueueMulti will allow a message to be enqueued again
+ * without waiting for dedupPeriod to expire.
+ * @param {Object} message - Return value from RecieveMessageCommand
+ * @param {Object} opt - Opt object from getOptionsWithDefaults()
+ * @returns {Number} 1 if a cache key was deleted, otherwise 0
+ */
+export async function dedupSuccessfullyProcessed (message, opt) {
+  const client = getCacheClient(opt)
+  const dedupId = message?.MessageAttributes?.QdoneDeduplicationId?.StringValue
+  if (dedupId) {
+    const cacheKey = getCacheKey(dedupId, opt)
+    const count = await client.del(cacheKey)
+    // Probabalistic stat maintenance
+    if (opt.dedupStats) {
+      const chance = 1 / 100.0
+      if (Math.random() < chance) await statMaintenance(opt)
+    }
+    return count
+  }
+  return 0
+}
+
+/**
+ * Marks an array of messages as processed so that subsequent calls to
+ * dedupShouldEnqueue and dedupShouldEnqueueMulti will allow a message to be
+ * enqueued again without waiting for dedupPeriod to expire.
+ * @param {Array[Object]} messages - Return values from RecieveMessageCommand
+ * @param {Object} opt - Opt object from getOptionsWithDefaults()
+ * @returns {Number} number of deleted keys
+ */
+export async function dedupSuccessfullyProcessedMulti (messages, opt) {
+  debug({ messages, dedupSuccessfullyProcessedMulti: { messages, opt }})
+  const cacheKeys = []
+  for (const message of messages) {
+    const dedupId = message?.MessageAttributes?.QdoneDeduplicationId?.StringValue
+    if (dedupId) {
+      const cacheKey = getCacheKey(dedupId, opt)
+      cacheKeys.push(cacheKey)
+    }
+  }
+  debug({ dedupSuccessfullyProcessedMulti: { cacheKeys }})
+  if (cacheKeys.length) {
+    const numDeleted = await getCacheClient(opt).del(cacheKeys)
+    // const numDeleted = results.map(([, val]) => val).reduce((a, b) => a + b, 0)
+    debug({ dedupSuccessfullyProcessedMulti: { cacheKeys, numDeleted } })
+
+    // Probabalistic stat maintenance
+    if (opt.dedupStats) {
+      const chance = numDeleted / 100.0
+      if (Math.random() < chance) await statMaintenance(opt)
+    }
+    return numDeleted
+  }
+  return 0
+}