qdone 2.0.34-alpha → 2.0.36-alpha

package/src/defaults.js

@@ -18,11 +18,16 @@ export const defaults = Object.freeze({
   fifo: false,
   disableLog: false,
   includeFailed: false,
+  includeDead: false,
+  externalDedup: false,
+  dedupPeriod: 60 * 5,
+  dedupStats: false,
 
   // Enqueue
   groupId: uuidv1(),
   groupIdPerMessage: false,
   deduplicationId: undefined,
+  dedupIdPerMessage: false,
   messageRetentionPeriod: 1209600,
   delay: 0,
   sendRetries: 6,
@@ -42,7 +47,10 @@ export const defaults = Object.freeze({
   // Idle Queues
   idleFor: 60,
   delete: false,
-  unpair: false
+  unpair: false,
+
+  // Check
+  create: false
 })
 
 function validateInteger (opt, name) {
@@ -51,6 +59,19 @@ function validateInteger (opt, name) {
   return parsed
 }
 
+export function validateMessageOptions (messageOptions) {
+  const validKeys = ['deduplicaitonId', 'groupId']
+  if (typeof messageOptions === 'object' &&
+      !Array.isArray(messageOptions) &&
+      messageOptions !== null) {
+    for (const key in messageOptions) {
+      if (!validKeys.includes(key)) throw new Error(`Invalid message option ${key}`)
+    }
+    return messageOptions
+  }
+  return {}
+}
+
 /**
  * This function should be called by each exposed API entry point on the
  * options passed in from the caller. It supports options named in camelCase
@@ -64,7 +85,7 @@ export function getOptionsWithDefaults (options) {
   if (!options) options = {}
 
   // Activate DLQ if any option is set
-  const dlq = options.dlq || !!(options['dlq-suffix'] || options['dlq-after'] || options['dlq-name'])
+  const dlq = options.dlq || !!(options['dlq-suffix'] || options['dlq-after'] || options['dlq-name'] || options.dlqSuffix || options.dlqAfter || options.dlqName)
 
   const opt = {
     // Shared
@@ -76,6 +97,11 @@ export function getOptionsWithDefaults (options) {
     fifo: options.fifo || defaults.fifo,
     sentryDsn: options.sentryDsn || options['sentry-dsn'],
     disableLog: options.disableLog || options['disable-log'] || defaults.disableLog,
+    includeFailed: options.includeFailed || options['include-failed'] || defaults.includeFailed,
+    includeDead: options.includeDead || options['include-dead'] || defaults.includeDead,
+    externalDedup: options.externalDedup || options['external-dedup'] || defaults.externalDedup,
+    dedupPeriod: options.dedupPeriod || options['dedup-period'] || defaults.dedupPeriod,
+    dedupStats: options.dedupStats || options['dedup-stats'] || defaults.dedupStats,
 
     // Cache
     cacheUri: options.cacheUri || options['cache-uri'] || defaults.cacheUri,
@@ -86,6 +112,7 @@ export function getOptionsWithDefaults (options) {
     groupId: options.groupId || options['group-id'] || defaults.groupId,
     groupIdPerMessage: false,
     deduplicationId: options.deduplicationId || options['deduplication-id'] || defaults.deduplicationId,
+    dedupIdPerMessage: options.dedupIdPerMessage || options['dedup-id-per-message'] || defaults.dedupIdPerMessage,
     messageRetentionPeriod: options.messageRetentionPeriod || options['message-retention-period'] || defaults.messageRetentionPeriod,
     delay: options.delay || defaults.delay,
     sendRetries: options['send-retries'] || defaults.sendRetries,
@@ -100,14 +127,17 @@ export function getOptionsWithDefaults (options) {
     killAfter: options.killAfter || options['kill-after'] || defaults.killAfter,
     archive: options.archive || defaults.archive,
     activeOnly: options.activeOnly || options['active-only'] || defaults.activeOnly,
-    includeFailed: options.includeFailed || options['include-failed'] || defaults.includeFailed,
     maxConcurrentJobs: options.maxConcurrentJobs || defaults.maxConcurrentJobs,
     maxMemoryPercent: options.maxMemoryPercent || defaults.maxMemoryPercent,
 
     // Idle Queues
     idleFor: options.idleFor || options['idle-for'] || defaults.idleFor,
     delete: options.delete || defaults.delete,
-    unpair: options.delete || defaults.unpair
+    unpair: options.delete || defaults.unpair,
+
+    // Check
+    create: options.create || defaults.create,
+    overwrite: options.overwrite || defaults.overwrite
   }
 
   // Setting this env here means we don't have to in AWS SDK constructors
@@ -118,6 +148,7 @@ export function getOptionsWithDefaults (options) {
   opt.messageRetentionPeriod = validateInteger(opt, 'messageRetentionPeriod')
   opt.delay = validateInteger(opt, 'delay')
   opt.sendRetries = validateInteger(opt, 'sendRetries')
+  opt.dedupPeriod = validateInteger(opt, 'dedupPeriod')
   opt.failDelay = validateInteger(opt, 'failDelay')
   opt.dlqAfter = validateInteger(opt, 'dlqAfter')
   opt.waitTime = validateInteger(opt, 'waitTime')
@@ -126,6 +157,11 @@ export function getOptionsWithDefaults (options) {
   opt.maxMemoryPercent = validateInteger(opt, 'maxMemoryPercent')
   opt.idleFor = validateInteger(opt, 'idleFor')
 
+  // Validate dedup args
+  if (opt.externalDedup && !opt.cacheUri) throw new Error('--external-dedup requires the --cache-uri argument')
+  if (opt.externalDedup && (!opt.dedupPeriod || opt.dedupPeriod < 1)) throw new Error('--external-dedup of redis requires a --dedup-period > 1 second')
+  if (opt.dedupIdPerMessage && opt.deduplicationId) throw new Error('Use either --deduplication-id or --dedup-id-per-message but not both')
+
   return opt
 }
 

package/src/enqueue.js

@@ -1,4 +1,4 @@
-import { addBreadcrumb } from '@sentry/node'
+import { addBreadcrumb, setExtra } from '@sentry/node'
 import { v1 as uuidV1 } from 'uuid'
 import chalk from 'chalk'
 import Debug from 'debug'
@@ -20,14 +20,31 @@ import {
   normalizeDLQName
 } from './qrlCache.js'
 import { getSQSClient } from './sqs.js'
-import { getOptionsWithDefaults } from './defaults.js'
+import {
+  addDedupParamsToMessage,
+  dedupShouldEnqueue,
+  dedupShouldEnqueueMulti,
+  dedupSuccessfullyProcessed
+} from './dedup.js'
+import { getOptionsWithDefaults, validateMessageOptions } from './defaults.js'
 import { ExponentialBackoff } from './exponentialBackoff.js'
 
 const debug = Debug('qdone:enqueue')
 
+export function getDLQParams (queue, opt) {
+  const dqname = normalizeDLQName(queue, opt)
+  const params = {
+    Attributes: { MessageRetentionPeriod: opt.messageRetentionPeriod + '' },
+    QueueName: dqname
+  }
+  if (opt.tags) params.tags = opt.tags
+  if (opt.fifo) params.Attributes.FifoQueue = 'true'
+  return { dqname, params }
+}
+
 export async function getOrCreateDLQ (queue, opt) {
   debug('getOrCreateDLQ(', queue, ')')
-  const dqname = normalizeDLQName(queue, opt)
+  const { dqname, params } = getDLQParams(queue, opt)
   try {
     const dqrl = await qrlCacheGet(dqname)
     return dqrl
@@ -37,12 +54,6 @@ export async function getOrCreateDLQ (queue, opt) {
 
     // Create our DLQ
     const client = getSQSClient()
-    const params = {
-      Attributes: { MessageRetentionPeriod: opt.messageRetentionPeriod + '' },
-      QueueName: dqname
-    }
-    if (opt.tags) params.tags = opt.tags
-    if (opt.fifo) params.Attributes.FifoQueue = 'true'
     const cmd = new CreateQueueCommand(params)
     if (opt.verbose) console.error(chalk.blue('Creating dead letter queue ') + dqname)
     const data = await client.send(cmd)
@@ -53,35 +64,62 @@ export async function getOrCreateDLQ (queue, opt) {
   }
 }
 
-export async function getOrCreateFailQueue (queue, opt) {
+/**
+ * Returns the parameters needed for creating a failed queue. If DLQ options
+ * are set, it makes an API call to get this DLQ's ARN.
+ */
+export async function getFailParams (queue, opt) {
+  const fqname = normalizeFailQueueName(queue, opt)
+  const params = {
+    Attributes: { MessageRetentionPeriod: opt.messageRetentionPeriod + '' },
+    QueueName: fqname
+  }
+  // If we have a dlq, we grab it and set a redrive policy
+  if (opt.dlq) {
+    const dqname = normalizeDLQName(queue, opt)
+    const dqrl = await qrlCacheGet(dqname)
+    const dqa = await getQueueAttributes(dqrl)
+    debug('dqa', dqa)
+    params.Attributes.RedrivePolicy = JSON.stringify({
+      deadLetterTargetArn: dqa.Attributes.QueueArn,
+      maxReceiveCount: opt.dlqAfter
+    })
+  }
+  if (opt.failDelay) params.Attributes.DelaySeconds = opt.failDelay + ''
+  if (opt.tags) params.tags = opt.tags
+  if (opt.fifo) params.Attributes.FifoQueue = 'true'
+  return params
+}
+
+/**
+ * Returns the qrl for the failed queue for the given queue. Creates the queue
+ * if it does not exist.
+ */
+export async function getOrCreateFailQueue (queue, opt, doesNotExist) {
   debug('getOrCreateFailQueue(', queue, ')')
   const fqname = normalizeFailQueueName(queue, opt)
   try {
+    // Bail early if the caller knew we didn't have a queue
+    if (doesNotExist) throw new QueueDoesNotExist(fqname)
     const fqrl = await qrlCacheGet(fqname)
     return fqrl
   } catch (err) {
     // Anything other than queue doesn't exist gets re-thrown
     if (!(err instanceof QueueDoesNotExist)) throw err
 
-    // Crate our fail queue
-    const client = getSQSClient()
-    const params = {
-      Attributes: { MessageRetentionPeriod: opt.messageRetentionPeriod + '' },
-      QueueName: fqname
-    }
-    // If we have a dlq, we grab it and set a redrive policy
-    if (opt.dlq) {
-      const dqrl = await getOrCreateDLQ(queue, opt)
-      const dqa = await getQueueAttributes(dqrl)
-      debug('dqa', dqa)
-      params.Attributes.RedrivePolicy = JSON.stringify({
-        deadLetterTargetArn: dqa.Attributes.QueueArn,
-        maxReceiveCount: opt.dlqAfter + ''
-      })
+    // Grab params, creating DLQ if needed
+    let params
+    try {
+      params = await getFailParams(queue, opt)
+    } catch (e) {
+      // If DLQ doesn't exist, create it
+      if (!(opt.dlq && e instanceof QueueDoesNotExist)) throw e
+      await getOrCreateDLQ(queue, opt)
+      params = await getFailParams(queue, opt)
     }
-    if (opt.failDelay) params.Attributes.DelaySeconds = opt.failDelay + ''
-    if (opt.tags) params.tags = opt.tags
-    if (opt.fifo) params.Attributes.FifoQueue = 'true'
+
+    // Create our fail queue
+    const client = getSQSClient()
     const cmd = new CreateQueueCommand(params)
     if (opt.verbose) console.error(chalk.blue('Creating fail queue ') + fqname)
     const data = await client.send(cmd)
@@ -92,6 +130,30 @@ export async function getOrCreateFailQueue (queue, opt) {
   }
 }
 
+/**
+ * Returns the parameters needed for creating a queue. If fail options
+ * are set, it makes an API call to get the fail queue's ARN.
+ */
+export async function getQueueParams (queue, opt) {
+  const qname = normalizeQueueName(queue, opt)
+  const fqname = normalizeFailQueueName(queue, opt)
+  const fqrl = await qrlCacheGet(fqname, opt)
+  const fqa = await getQueueAttributes(fqrl)
+  const params = {
+    Attributes: {
+      MessageRetentionPeriod: opt.messageRetentionPeriod + '',
+      RedrivePolicy: JSON.stringify({
+        deadLetterTargetArn: fqa.Attributes.QueueArn,
+        maxReceiveCount: 1
+      })
+    },
+    QueueName: qname
+  }
+  if (opt.tags) params.tags = opt.tags
+  if (opt.fifo) params.Attributes.FifoQueue = 'true'
+  return params
+}
+
 /**
  * Returns a qrl for a queue that either exists or does not
  */
@@ -105,29 +167,25 @@ export async function getOrCreateQueue (queue, opt) {
     // Anything other than queue doesn't exist gets re-thrown
     if (!(err instanceof QueueDoesNotExist)) throw err
 
-    // Get our fail queue so we can create our own
-    const fqrl = await getOrCreateFailQueue(qname, opt)
-    const fqa = await getQueueAttributes(fqrl)
+    // Grab params, creating fail queue if needed
+    let params
+    try {
+      params = await getQueueParams(qname, opt)
+    } catch (e) {
+      // If fail queue doesn't exist, create it
+      if (!(e instanceof QueueDoesNotExist)) throw e
+      await getOrCreateFailQueue(qname, opt, true)
+      params = await getQueueParams(qname, opt)
+    }
+
+    debug({ getOrCreateQueue: { qname, params } })
 
     // Create our queue
     const client = getSQSClient()
-    const params = {
-      Attributes: {
-        MessageRetentionPeriod: opt.messageRetentionPeriod + '',
-        RedrivePolicy: JSON.stringify({
-          deadLetterTargetArn: fqa.Attributes.QueueArn,
-          maxReceiveCount: '1'
-        })
-      },
-      QueueName: qname
-    }
-    if (opt.tags) params.tags = opt.tags
-    if (opt.fifo) params.Attributes.FifoQueue = 'true'
     const cmd = new CreateQueueCommand(params)
-    debug({ params })
-    if (opt.verbose) console.error(chalk.blue('Creating queue ') + qname)
+    if (opt.verbose) console.error(chalk.blue('Creating fail queue ') + qname)
     const data = await client.send(cmd)
-    debug('createQueue returned', data)
+    debug('AWS createQueue returned', data)
     const qrl = data.QueueUrl
     qrlCacheSet(qname, qrl)
     return qrl
@@ -145,17 +203,15 @@ export async function getQueueAttributes (qrl) {
   return data
 }
 
-export function formatMessage (command, id) {
-  const message = {
-    /*
-    MessageAttributes: {
-      City: { DataType: 'String', StringValue: 'Any City' },
-      Population: { DataType: 'Number', StringValue: '1250800' }
-    },
-    */
-    MessageBody: command
-  }
+export function formatMessage (body, id, opt, messageOptions) {
+  const message = { MessageBody: body }
   if (typeof id !== 'undefined') message.Id = '' + id
+  if (opt.fifo) {
+    message.MessageGroupId = messageOptions?.groupId || opt?.groupId
+  }
+  addDedupParamsToMessage(message, opt, messageOptions)
+  if (opt.delay) message.DelaySeconds = opt.delay
+  if (messageOptions?.delay) message.DelaySeconds = messageOptions.delay
   return message
 }
 
@@ -166,14 +222,19 @@ const retryableExceptions = [
   QueueDoesNotExist // Queue could temporarily not exist due to eventual consistency, let it retry
 ]
 
-export async function sendMessage (qrl, command, opt) {
+export async function sendMessage (qrl, command, opt, messageOptions) {
   debug('sendMessage(', qrl, command, ')')
-  const params = Object.assign({ QueueUrl: qrl }, formatMessage(command))
-  if (opt.fifo) {
-    params.MessageGroupId = opt.groupId
-    params.MessageDeduplicationId = opt.deduplicationId || uuidV1()
+  const uuidFunction = opt.uuidFunction || uuidV1
+  const params = {
+    QueueUrl: qrl,
+    ...formatMessage(command, null, opt, messageOptions)
+  }
+
+  // See if we even have to send it
+  if (opt.externalDedup) {
+    const shouldEnqueue = await dedupShouldEnqueue(params, opt)
+    if (!shouldEnqueue) return { MessageId: uuidFunction() }
   }
-  if (opt.delay) params.DelaySeconds = opt.delay
 
   // Send it
   const client = getSQSClient()
@@ -187,12 +248,15 @@ export async function sendMessage (qrl, command, opt) {
     return data
   }
   const shouldRetry = async (result, error) => {
+    if (!error) return false
     for (const exceptionClass of retryableExceptions) {
       if (error instanceof exceptionClass) {
         debug({ sendMessageRetryingBecause: { error, result } })
         return true
       }
     }
+    // If we could not send it, we also need to remove our dedup flag
+    await dedupSuccessfullyProcessed(params, opt)
     return false
   }
   const result = await backoff.run(send, shouldRetry)
@@ -203,25 +267,17 @@ export async function sendMessage (qrl, command, opt) {
 export async function sendMessageBatch (qrl, messages, opt) {
   debug('sendMessageBatch(', qrl, messages.map(e => Object.assign(Object.assign({}, e), { MessageBody: e.MessageBody.slice(0, 10) + '...' })), ')')
   const params = { Entries: messages, QueueUrl: qrl }
-  const uuidFunction = opt.uuidFunction || uuidV1
-  // Add in group id if we're using fifo
-  if (opt.fifo) {
-    params.Entries = params.Entries.map(
-      message => Object.assign({
-        MessageGroupId: opt.groupIdPerMessage ? uuidFunction() : opt.groupId,
-        MessageDeduplicationId: opt.deduplicationId || uuidFunction()
-      }, message)
-    )
-  }
-  if (opt.delay) {
-    params.Entries = params.Entries.map(message =>
-      Object.assign({ DelaySeconds: opt.delay }, message))
-  }
   if (opt.sentryDsn) {
     addBreadcrumb({ category: 'sendMessageBatch', message: JSON.stringify({ params }), level: 'debug' })
   }
   debug({ params })
 
+  // See which messages we even have to send
+  if (opt.externalDedup) {
+    params.Entries = await dedupShouldEnqueueMulti(params.Entries, opt)
+    if (!params.Entries.length) return
+  }
+
   // Send them
   const client = getSQSClient()
   const cmd = new SendMessageBatchCommand(params)
@@ -320,8 +376,8 @@ export async function flushMessages (qrl, opt, sendBuffer) {
 // Automaticaly flushes if queue has >= 10 messages.
 // Returns number of messages flushed.
 //
-export async function addMessage (qrl, command, messageIndex, opt, sendBuffer) {
-  const message = formatMessage(command, messageIndex)
+export async function addMessage (qrl, command, messageIndex, opt, sendBuffer, messageOptions) {
+  const message = formatMessage(command, messageIndex, opt, messageOptions)
   sendBuffer[qrl] = sendBuffer[qrl] || []
   sendBuffer[qrl].push(message)
   debug({ location: 'addMessage', sendBuffer })
@@ -338,8 +394,16 @@ export async function addMessage (qrl, command, messageIndex, opt, sendBuffer) {
 export async function enqueue (queue, command, options) {
   debug('enqueue(', { queue, command }, ')')
   const opt = getOptionsWithDefaults(options)
-  const qrl = await getOrCreateQueue(queue, opt)
-  return sendMessage(qrl, command, opt)
+  if (opt.sentryDsn) {
+    setExtra({ qdoneOperation: 'enqueue', args: { queue, command, opt } })
+  }
+  try {
+    const qrl = await getOrCreateQueue(queue, opt)
+    return sendMessage(qrl, command, opt)
+  } catch (e) {
+    console.log(e)
+    throw e
+  }
 }
 
 //
@@ -349,43 +413,50 @@ export async function enqueue (queue, command, options) {
 export async function enqueueBatch (pairs, options) {
   debug('enqueueBatch(', pairs, ')')
   const opt = getOptionsWithDefaults(options)
-
-  // Find unique queues so we can pre-fetch qrls. We do this so that all
-  // queues are created prior to going through our flush logic
-  const normalizedPairs = pairs.map(({ queue, command }) => ({
-    qname: normalizeQueueName(queue, opt),
-    command
-  }))
-  const uniqueQnames = new Set(normalizedPairs.map(p => p.qname))
-
-  // Prefetch qrls / create queues in parallel
-  const createPromises = []
-  for (const qname of uniqueQnames) {
-    createPromises.push(getOrCreateQueue(qname, opt))
-  }
-  await Promise.all(createPromises)
-
-  // After we've prefetched, all qrls are in cache
-  // so go back through the list of pairs and fire off messages
-  requestCount = 0
-  const sendBuffer = {}
-  let messageIndex = 0
-  let initialFlushTotal = 0
-  for (const { qname, command } of normalizedPairs) {
-    const qrl = await getOrCreateQueue(qname, opt)
-    initialFlushTotal += await addMessage(qrl, command, messageIndex++, opt, sendBuffer)
+  if (opt.sentryDsn) {
+    setExtra({ qdoneOperation: 'enqueueBatch', args: { pairs, opt } })
   }
+  try {
+    // Find unique queues so we can pre-fetch qrls. We do this so that all
+    // queues are created prior to going through our flush logic
+    const normalizedPairs = pairs.map(({ queue, command, messageOptions }) => ({
+      qname: normalizeQueueName(queue, opt),
+      command,
+      messageOptions: validateMessageOptions(messageOptions)
+    }))
+    const uniqueQnames = new Set(normalizedPairs.map(p => p.qname))
 
-  // And flush any remaining messages
-  const extraFlushPromises = []
-  for (const qrl in sendBuffer) {
-    extraFlushPromises.push(flushMessages(qrl, opt, sendBuffer))
+    // Prefetch qrls / create queues in parallel
+    const createPromises = []
+    for (const qname of uniqueQnames) {
+      createPromises.push(getOrCreateQueue(qname, opt))
+    }
+    await Promise.all(createPromises)
+    // After we've prefetched, all qrls are in cache
+    // so go back through the list of pairs and fire off messages
+    requestCount = 0
+    const sendBuffer = {}
+    let messageIndex = 0
+    let initialFlushTotal = 0
+    for (const { qname, command, messageOptions } of normalizedPairs) {
+      const qrl = await getOrCreateQueue(qname, opt)
+      initialFlushTotal += await addMessage(qrl, command, messageIndex++, opt, sendBuffer, messageOptions)
+    }
+
+    // And flush any remaining messages
+    const extraFlushPromises = []
+    for (const qrl in sendBuffer) {
+      extraFlushPromises.push(flushMessages(qrl, opt, sendBuffer))
+    }
+    const extraFlushCounts = await Promise.all(extraFlushPromises)
+    const extraFlushTotal = extraFlushCounts.reduce((a, b) => a + b, 0)
+    const totalFlushed = initialFlushTotal + extraFlushTotal
+    debug({ initialFlushTotal, extraFlushTotal, totalFlushed })
+    return totalFlushed
+  } catch (e) {
+    console.log(e)
+    throw e
   }
-  const extraFlushCounts = await Promise.all(extraFlushPromises)
-  const extraFlushTotal = extraFlushCounts.reduce((a, b) => a + b, 0)
-  const totalFlushed = initialFlushTotal + extraFlushTotal
-  debug({ initialFlushTotal, extraFlushTotal, totalFlushed })
-  return totalFlushed
 }
 
 debug('loaded')

package/src/qrlCache.js

@@ -66,7 +66,7 @@ export async function qrlCacheGet (qname) {
   // debug({ cmd })
   const result = await client.send(cmd)
   // debug('result', result)
-  // if (!result) throw new Error(`No such queue ${qname}`)
+  if (!result) throw new QueueDoesNotExist(qname)
   const { QueueUrl: qrl } = result
   // debug('getQueueUrl returned', data)
   qcache.set(qname, qrl)

package/src/scheduler/jobExecutor.js

@@ -8,6 +8,7 @@ import { ChangeMessageVisibilityBatchCommand, DeleteMessageBatchCommand } from '
 import chalk from 'chalk'
 import Debug from 'debug'
 
+import { dedupSuccessfullyProcessedMulti } from '../dedup.js'
 import { getSQSClient } from '../sqs.js'
 
 const debug = Debug('qdone:jobExecutor')
@@ -221,6 +222,10 @@ export class JobExecutor {
           }
         }
         debug('DeleteMessageBatch returned', result)
+
+        // Mark batch as processed for dedup
+        await dedupSuccessfullyProcessedMulti(entries.map(e => this.jobsByMessageId[e.Id]), this.opt)
+
         // TODO Sentry
       }
     }

package/src/scheduler/queueManager.js

@@ -104,7 +104,7 @@ export class QueueManager {
       .filter(({ qname, qrl }) => {
         const isFifo = qname.endsWith('.fifo')
         const isDead = isFifo ? qname.endsWith('_dead.fifo') : qname.endsWith('_dead')
-        return !isDead
+        return this.opt.includeDead ? true : !isDead
       })
       // then icehouse
       .filter(({ qname, qrl }) => !this.keepInIcehouse(qrl, now))

package/src/worker.js

@@ -13,6 +13,7 @@ import treeKill from 'tree-kill'
 import chalk from 'chalk'
 import Debug from 'debug'
 
+import { dedupSuccessfullyProcessed } from './dedup.js'
 import { normalizeQueueName, getQnameUrlPairs } from './qrlCache.js'
 import { getOptionsWithDefaults } from './defaults.js'
 import { cheapIdleCheck } from './idleQueues.js'
@@ -129,10 +130,15 @@ export async function executeJob (job, qname, qrl, opt) {
       QueueUrl: qrl,
       ReceiptHandle: job.ReceiptHandle
     }))
+
     if (opt.verbose) {
       console.error(chalk.blue('  done'))
       console.error()
     }
+
+    // Let dedup system know we processed it
+    await dedupSuccessfullyProcessed(job, opt)
+
     return { noJobs: 0, jobsSucceeded: 1, jobsFailed: 0 }
   } catch (err) {
     // Fail path for job execution