qdone 2.0.35-alpha → 2.0.37-alpha

package/src/enqueue.js

@@ -1,4 +1,4 @@
-import { addBreadcrumb } from '@sentry/node'
+import { addBreadcrumb, setExtra } from '@sentry/node'
 import { v1 as uuidV1 } from 'uuid'
 import chalk from 'chalk'
 import Debug from 'debug'
@@ -20,14 +20,30 @@ import {
   normalizeDLQName
 } from './qrlCache.js'
 import { getSQSClient } from './sqs.js'
-import { getOptionsWithDefaults } from './defaults.js'
+import {
+  addDedupParamsToMessage,
+  dedupShouldEnqueue,
+  dedupSuccessfullyProcessed
+} from './dedup.js'
+import { getOptionsWithDefaults, validateMessageOptions } from './defaults.js'
 import { ExponentialBackoff } from './exponentialBackoff.js'
 
 const debug = Debug('qdone:enqueue')
 
+export function getDLQParams (queue, opt) {
+  const dqname = normalizeDLQName(queue, opt)
+  const params = {
+    Attributes: { MessageRetentionPeriod: opt.messageRetentionPeriod + '' },
+    QueueName: dqname
+  }
+  if (opt.tags) params.tags = opt.tags
+  if (opt.fifo) params.Attributes.FifoQueue = 'true'
+  return { dqname, params }
+}
+
 export async function getOrCreateDLQ (queue, opt) {
   debug('getOrCreateDLQ(', queue, ')')
-  const dqname = normalizeDLQName(queue, opt)
+  const { dqname, params } = getDLQParams(queue, opt)
   try {
     const dqrl = await qrlCacheGet(dqname)
     return dqrl
@@ -37,12 +53,6 @@ export async function getOrCreateDLQ (queue, opt) {
 
     // Create our DLQ
     const client = getSQSClient()
-    const params = {
-      Attributes: { MessageRetentionPeriod: opt.messageRetentionPeriod + '' },
-      QueueName: dqname
-    }
-    if (opt.tags) params.tags = opt.tags
-    if (opt.fifo) params.Attributes.FifoQueue = 'true'
     const cmd = new CreateQueueCommand(params)
     if (opt.verbose) console.error(chalk.blue('Creating dead letter queue ') + dqname)
     const data = await client.send(cmd)
@@ -53,35 +63,62 @@ export async function getOrCreateDLQ (queue, opt) {
   }
 }
 
-export async function getOrCreateFailQueue (queue, opt) {
+/**
+ * Returns the parameters needed for creating a failed queue. If DLQ options
+ * are set, it makes an API call to get this DLQ's ARN.
+ */
+export async function getFailParams (queue, opt) {
+  const fqname = normalizeFailQueueName(queue, opt)
+  const params = {
+    Attributes: { MessageRetentionPeriod: opt.messageRetentionPeriod + '' },
+    QueueName: fqname
+  }
+  // If we have a dlq, we grab it and set a redrive policy
+  if (opt.dlq) {
+    const dqname = normalizeDLQName(queue, opt)
+    const dqrl = await qrlCacheGet(dqname)
+    const dqa = await getQueueAttributes(dqrl)
+    debug('dqa', dqa)
+    params.Attributes.RedrivePolicy = JSON.stringify({
+      deadLetterTargetArn: dqa.Attributes.QueueArn,
+      maxReceiveCount: opt.dlqAfter
+    })
+  }
+  if (opt.failDelay) params.Attributes.DelaySeconds = opt.failDelay + ''
+  if (opt.tags) params.tags = opt.tags
+  if (opt.fifo) params.Attributes.FifoQueue = 'true'
+  return params
+}
+
+/**
+ * Returns the qrl for the failed queue for the given queue. Creates the queue
+ * if it does not exist.
+ */
+export async function getOrCreateFailQueue (queue, opt, doesNotExist) {
   debug('getOrCreateFailQueue(', queue, ')')
   const fqname = normalizeFailQueueName(queue, opt)
   try {
+    // Bail early if the caller knew we didn't have a queue
+    if (doesNotExist) throw new QueueDoesNotExist(fqname)
     const fqrl = await qrlCacheGet(fqname)
     return fqrl
   } catch (err) {
     // Anything other than queue doesn't exist gets re-thrown
     if (!(err instanceof QueueDoesNotExist)) throw err
 
-    // Crate our fail queue
-    const client = getSQSClient()
-    const params = {
-      Attributes: { MessageRetentionPeriod: opt.messageRetentionPeriod + '' },
-      QueueName: fqname
-    }
-    // If we have a dlq, we grab it and set a redrive policy
-    if (opt.dlq) {
-      const dqrl = await getOrCreateDLQ(queue, opt)
-      const dqa = await getQueueAttributes(dqrl)
-      debug('dqa', dqa)
-      params.Attributes.RedrivePolicy = JSON.stringify({
-        deadLetterTargetArn: dqa.Attributes.QueueArn,
-        maxReceiveCount: opt.dlqAfter + ''
-      })
+    // Grab params, creating DLQ if needed
+    let params
+    try {
+      params = await getFailParams(queue, opt)
+    } catch (e) {
+      // If DLQ doesn't exist, create it
+      if (!(opt.dlq && e instanceof QueueDoesNotExist)) throw e
+      await getOrCreateDLQ(queue, opt)
+      params = await getFailParams(queue, opt)
     }
-    if (opt.failDelay) params.Attributes.DelaySeconds = opt.failDelay + ''
-    if (opt.tags) params.tags = opt.tags
-    if (opt.fifo) params.Attributes.FifoQueue = 'true'
+
+    // Create our fail queue
+    const client = getSQSClient()
     const cmd = new CreateQueueCommand(params)
     if (opt.verbose) console.error(chalk.blue('Creating fail queue ') + fqname)
     const data = await client.send(cmd)
@@ -92,6 +129,30 @@ export async function getOrCreateFailQueue (queue, opt) {
   }
 }
 
+/**
+ * Returns the parameters needed for creating a queue. If fail options
+ * are set, it makes an API call to get the fail queue's ARN.
+ */
+export async function getQueueParams (queue, opt) {
+  const qname = normalizeQueueName(queue, opt)
+  const fqname = normalizeFailQueueName(queue, opt)
+  const fqrl = await qrlCacheGet(fqname, opt)
+  const fqa = await getQueueAttributes(fqrl)
+  const params = {
+    Attributes: {
+      MessageRetentionPeriod: opt.messageRetentionPeriod + '',
+      RedrivePolicy: JSON.stringify({
+        deadLetterTargetArn: fqa.Attributes.QueueArn,
+        maxReceiveCount: 1
+      })
+    },
+    QueueName: qname
+  }
+  if (opt.tags) params.tags = opt.tags
+  if (opt.fifo) params.Attributes.FifoQueue = 'true'
+  return params
+}
+
 /**
  * Returns a qrl for a queue that either exists or does not
  */
@@ -105,29 +166,25 @@ export async function getOrCreateQueue (queue, opt) {
     // Anything other than queue doesn't exist gets re-thrown
     if (!(err instanceof QueueDoesNotExist)) throw err
 
-    // Get our fail queue so we can create our own
-    const fqrl = await getOrCreateFailQueue(qname, opt)
-    const fqa = await getQueueAttributes(fqrl)
+    // Grab params, creating fail queue if needed
+    let params
+    try {
+      params = await getQueueParams(qname, opt)
+    } catch (e) {
+      // If fail queue doesn't exist, create it
+      if (!(e instanceof QueueDoesNotExist)) throw e
+      await getOrCreateFailQueue(qname, opt, true)
+      params = await getQueueParams(qname, opt)
+    }
+
+    debug({ getOrCreateQueue: { qname, params } })
 
     // Create our queue
     const client = getSQSClient()
-    const params = {
-      Attributes: {
-        MessageRetentionPeriod: opt.messageRetentionPeriod + '',
-        RedrivePolicy: JSON.stringify({
-          deadLetterTargetArn: fqa.Attributes.QueueArn,
-          maxReceiveCount: '1'
-        })
-      },
-      QueueName: qname
-    }
-    if (opt.tags) params.tags = opt.tags
-    if (opt.fifo) params.Attributes.FifoQueue = 'true'
     const cmd = new CreateQueueCommand(params)
-    debug({ params })
-    if (opt.verbose) console.error(chalk.blue('Creating queue ') + qname)
+    if (opt.verbose) console.error(chalk.blue('Creating fail queue ') + qname)
     const data = await client.send(cmd)
-    debug('createQueue returned', data)
+    debug('AWS createQueue returned', data)
     const qrl = data.QueueUrl
     qrlCacheSet(qname, qrl)
     return qrl
@@ -145,17 +202,15 @@ export async function getQueueAttributes (qrl) {
   return data
 }
 
-export function formatMessage (command, id) {
-  const message = {
-    /*
-    MessageAttributes: {
-      City: { DataType: 'String', StringValue: 'Any City' },
-      Population: { DataType: 'Number', StringValue: '1250800' }
-    },
-    */
-    MessageBody: command
-  }
+export function formatMessage (body, id, opt, messageOptions) {
+  const message = { MessageBody: body }
   if (typeof id !== 'undefined') message.Id = '' + id
+  if (opt.fifo) {
+    message.MessageGroupId = messageOptions?.groupId || opt?.groupId
+  }
+  addDedupParamsToMessage(message, opt, messageOptions)
+  if (opt.delay) message.DelaySeconds = opt.delay
+  if (messageOptions?.delay) message.DelaySeconds = messageOptions.delay
   return message
 }
 
@@ -166,14 +221,19 @@ const retryableExceptions = [
   QueueDoesNotExist // Queue could temporarily not exist due to eventual consistency, let it retry
 ]
 
-export async function sendMessage (qrl, command, opt) {
+export async function sendMessage (qrl, command, opt, messageOptions) {
   debug('sendMessage(', qrl, command, ')')
-  const params = Object.assign({ QueueUrl: qrl }, formatMessage(command))
-  if (opt.fifo) {
-    params.MessageGroupId = opt.groupId
-    params.MessageDeduplicationId = opt.deduplicationId || uuidV1()
+  const uuidFunction = opt.uuidFunction || uuidV1
+  const params = {
+    QueueUrl: qrl,
+    ...formatMessage(command, null, opt, messageOptions)
+  }
+
+  // See if we even have to send it
+  if (opt.externalDedup) {
+    const shouldEnqueue = await dedupShouldEnqueue(params, opt)
+    if (!shouldEnqueue) return { MessageId: uuidFunction() }
   }
-  if (opt.delay) params.DelaySeconds = opt.delay
 
   // Send it
   const client = getSQSClient()
@@ -187,12 +247,15 @@ export async function sendMessage (qrl, command, opt) {
     return data
   }
   const shouldRetry = async (result, error) => {
+    if (!error) return false
     for (const exceptionClass of retryableExceptions) {
       if (error instanceof exceptionClass) {
         debug({ sendMessageRetryingBecause: { error, result } })
         return true
       }
     }
+    // If we could not send it, we also need to remove our dedup flag
+    await dedupSuccessfullyProcessed(params, opt)
     return false
   }
   const result = await backoff.run(send, shouldRetry)
@@ -203,25 +266,19 @@ export async function sendMessage (qrl, command, opt) {
 export async function sendMessageBatch (qrl, messages, opt) {
   debug('sendMessageBatch(', qrl, messages.map(e => Object.assign(Object.assign({}, e), { MessageBody: e.MessageBody.slice(0, 10) + '...' })), ')')
   const params = { Entries: messages, QueueUrl: qrl }
-  const uuidFunction = opt.uuidFunction || uuidV1
-  // Add in group id if we're using fifo
-  if (opt.fifo) {
-    params.Entries = params.Entries.map(
-      message => Object.assign({
-        MessageGroupId: opt.groupIdPerMessage ? uuidFunction() : opt.groupId,
-        MessageDeduplicationId: opt.deduplicationId || uuidFunction()
-      }, message)
-    )
-  }
-  if (opt.delay) {
-    params.Entries = params.Entries.map(message =>
-      Object.assign({ DelaySeconds: opt.delay }, message))
-  }
   if (opt.sentryDsn) {
     addBreadcrumb({ category: 'sendMessageBatch', message: JSON.stringify({ params }), level: 'debug' })
   }
   debug({ params })
 
+  // See which messages we even have to send
+  if (opt.externalDedup) {
+    const promises = params.Entries.map(async m => ({ m, shouldEnqueue: await dedupShouldEnqueue(m, opt) }))
+    const results = await Promise.all(promises)
+    params.Entries = results.filter(({ shouldEnqueue }) => shouldEnqueue)
+    if (!params.Entries.length) return
+  }
+
   // Send them
   const client = getSQSClient()
   const cmd = new SendMessageBatchCommand(params)
@@ -320,8 +377,8 @@ export async function flushMessages (qrl, opt, sendBuffer) {
 // Automaticaly flushes if queue has >= 10 messages.
 // Returns number of messages flushed.
 //
-export async function addMessage (qrl, command, messageIndex, opt, sendBuffer) {
-  const message = formatMessage(command, messageIndex)
+export async function addMessage (qrl, command, messageIndex, opt, sendBuffer, messageOptions) {
+  const message = formatMessage(command, messageIndex, opt, messageOptions)
   sendBuffer[qrl] = sendBuffer[qrl] || []
   sendBuffer[qrl].push(message)
   debug({ location: 'addMessage', sendBuffer })
@@ -338,8 +395,16 @@ export async function addMessage (qrl, command, messageIndex, opt, sendBuffer) {
 export async function enqueue (queue, command, options) {
   debug('enqueue(', { queue, command }, ')')
   const opt = getOptionsWithDefaults(options)
-  const qrl = await getOrCreateQueue(queue, opt)
-  return sendMessage(qrl, command, opt)
+  if (opt.sentryDsn) {
+    setExtra({ qdoneOperation: 'enqueue', args: { queue, command, opt } })
+  }
+  try {
+    const qrl = await getOrCreateQueue(queue, opt)
+    return sendMessage(qrl, command, opt)
+  } catch (e) {
+    console.log(e)
+    throw e
+  }
 }
 
 //
@@ -349,43 +414,50 @@ export async function enqueue (queue, command, options) {
 export async function enqueueBatch (pairs, options) {
   debug('enqueueBatch(', pairs, ')')
   const opt = getOptionsWithDefaults(options)
-
-  // Find unique queues so we can pre-fetch qrls. We do this so that all
-  // queues are created prior to going through our flush logic
-  const normalizedPairs = pairs.map(({ queue, command }) => ({
-    qname: normalizeQueueName(queue, opt),
-    command
-  }))
-  const uniqueQnames = new Set(normalizedPairs.map(p => p.qname))
-
-  // Prefetch qrls / create queues in parallel
-  const createPromises = []
-  for (const qname of uniqueQnames) {
-    createPromises.push(getOrCreateQueue(qname, opt))
-  }
-  await Promise.all(createPromises)
-
-  // After we've prefetched, all qrls are in cache
-  // so go back through the list of pairs and fire off messages
-  requestCount = 0
-  const sendBuffer = {}
-  let messageIndex = 0
-  let initialFlushTotal = 0
-  for (const { qname, command } of normalizedPairs) {
-    const qrl = await getOrCreateQueue(qname, opt)
-    initialFlushTotal += await addMessage(qrl, command, messageIndex++, opt, sendBuffer)
+  if (opt.sentryDsn) {
+    setExtra({ qdoneOperation: 'enqueueBatch', args: { pairs, opt } })
   }
+  try {
+    // Find unique queues so we can pre-fetch qrls. We do this so that all
+    // queues are created prior to going through our flush logic
+    const normalizedPairs = pairs.map(({ queue, command, messageOptions }) => ({
+      qname: normalizeQueueName(queue, opt),
+      command,
+      messageOptions: validateMessageOptions(messageOptions)
+    }))
+    const uniqueQnames = new Set(normalizedPairs.map(p => p.qname))
 
-  // And flush any remaining messages
-  const extraFlushPromises = []
-  for (const qrl in sendBuffer) {
-    extraFlushPromises.push(flushMessages(qrl, opt, sendBuffer))
+    // Prefetch qrls / create queues in parallel
+    const createPromises = []
+    for (const qname of uniqueQnames) {
+      createPromises.push(getOrCreateQueue(qname, opt))
+    }
+    await Promise.all(createPromises)
+    // After we've prefetched, all qrls are in cache
+    // so go back through the list of pairs and fire off messages
+    requestCount = 0
+    const sendBuffer = {}
+    let messageIndex = 0
+    let initialFlushTotal = 0
+    for (const { qname, command, messageOptions } of normalizedPairs) {
+      const qrl = await getOrCreateQueue(qname, opt)
+      initialFlushTotal += await addMessage(qrl, command, messageIndex++, opt, sendBuffer, messageOptions)
+    }
+
+    // And flush any remaining messages
+    const extraFlushPromises = []
+    for (const qrl in sendBuffer) {
+      extraFlushPromises.push(flushMessages(qrl, opt, sendBuffer))
+    }
+    const extraFlushCounts = await Promise.all(extraFlushPromises)
+    const extraFlushTotal = extraFlushCounts.reduce((a, b) => a + b, 0)
+    const totalFlushed = initialFlushTotal + extraFlushTotal
+    debug({ initialFlushTotal, extraFlushTotal, totalFlushed })
+    return totalFlushed
+  } catch (e) {
+    console.log(e)
+    throw e
   }
-  const extraFlushCounts = await Promise.all(extraFlushPromises)
-  const extraFlushTotal = extraFlushCounts.reduce((a, b) => a + b, 0)
-  const totalFlushed = initialFlushTotal + extraFlushTotal
-  debug({ initialFlushTotal, extraFlushTotal, totalFlushed })
-  return totalFlushed
 }
 
 debug('loaded')

package/src/qrlCache.js

@@ -66,7 +66,7 @@ export async function qrlCacheGet (qname) {
   // debug({ cmd })
   const result = await client.send(cmd)
   // debug('result', result)
-  // if (!result) throw new Error(`No such queue ${qname}`)
+  if (!result) throw new QueueDoesNotExist(qname)
   const { QueueUrl: qrl } = result
   // debug('getQueueUrl returned', data)
   qcache.set(qname, qrl)

package/src/scheduler/jobExecutor.js

@@ -8,6 +8,7 @@ import { ChangeMessageVisibilityBatchCommand, DeleteMessageBatchCommand } from '
 import chalk from 'chalk'
 import Debug from 'debug'
 
+import { dedupSuccessfullyProcessed } from '../dedup.js'
 import { getSQSClient } from '../sqs.js'
 
 const debug = Debug('qdone:jobExecutor')
@@ -221,6 +222,10 @@ export class JobExecutor {
           }
         }
         debug('DeleteMessageBatch returned', result)
+
+        // Mark batch as processed for dedup
+        await Promise.all(entries.map(e => dedupSuccessfullyProcessed(this.jobsByMessageId[e.Id], this.opt)))
+
         // TODO Sentry
       }
     }

package/src/worker.js

@@ -13,6 +13,7 @@ import treeKill from 'tree-kill'
 import chalk from 'chalk'
 import Debug from 'debug'
 
+import { dedupSuccessfullyProcessed } from './dedup.js'
 import { normalizeQueueName, getQnameUrlPairs } from './qrlCache.js'
 import { getOptionsWithDefaults } from './defaults.js'
 import { cheapIdleCheck } from './idleQueues.js'
@@ -129,10 +130,15 @@ export async function executeJob (job, qname, qrl, opt) {
       QueueUrl: qrl,
       ReceiptHandle: job.ReceiptHandle
     }))
+
     if (opt.verbose) {
       console.error(chalk.blue('  done'))
       console.error()
     }
+
+    // Let dedup system know we processed it
+    await dedupSuccessfullyProcessed(job, opt)
+
     return { noJobs: 0, jobsSucceeded: 1, jobsFailed: 0 }
   } catch (err) {
     // Fail path for job execution