qdone 2.0.56-alpha → 2.1.1

package/LICENSE

@@ -1,6 +1,6 @@
 ISC License
 
-Copyright (c) 2017, Ryan Witt
+Copyright (c) 2017-2026, SureDone, Inc.
 
 Permission to use, copy, modify, and/or distribute this software for any
 purpose with or without fee is hereby granted, provided that the above
@@ -12,4 +12,4 @@ AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT,
 INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM
 LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE
 OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
-PERFORMANCE OF THIS SOFTWARE.
+PERFORMANCE OF THIS SOFTWARE.

package/README.md

@@ -1,9 +1,6 @@
 [![NPM Package](https://img.shields.io/npm/v/qdone.svg)](https://www.npmjs.com/package/qdone)
-[![Build Status](https://travis-ci.org/suredone/qdone.svg?branch=master)](https://travis-ci.org/suredone/qdone)
-[![Coverage Status](https://coveralls.io/repos/github/suredone/qdone/badge.svg)](https://coveralls.io/github/suredone/qdone)
-[![Dependencies](https://img.shields.io/david/suredone/qdone.svg)](https://david-dm.org/suredone/qdone)
+[![Build](https://github.com/suredone/qdone/actions/workflows/build.yaml/badge.svg)](https://github.com/suredone/qdone/actions/workflows/build.yaml)
 [![Standard - JavaScript Style Guide](https://img.shields.io/badge/code_style-standard-brightgreen.svg)](https://standardjs.com)
-[![Greenkeeper badge](https://badges.greenkeeper.io/suredone/qdone.svg)](https://greenkeeper.io/)
 
 # qdone
 

package/commonjs/src/enqueue.js

@@ -223,7 +223,7 @@ const retryableExceptions = [
     client_sqs_1.KmsThrottled,
     client_sqs_1.QueueDoesNotExist // Queue could temporarily not exist due to eventual consistency, let it retry
 ];
-async function sendMessage(qrl, command, opt, messageOptions) {
+async function sendMessage(qrl, queue, command, opt, messageOptions) {
     debug('sendMessage(', qrl, command, ')');
     const uuidFunction = opt.uuidFunction || uuid_1.v1;
     const params = {
@@ -238,7 +238,7 @@ async function sendMessage(qrl, command, opt, messageOptions) {
     }
     // Send it
     const client = (0, sqs_js_1.getSQSClient)();
-    const cmd = new client_sqs_1.SendMessageCommand(params);
+    let cmd = new client_sqs_1.SendMessageCommand(params);
     debug({ cmd });
     const backoff = new exponentialBackoff_js_1.ExponentialBackoff(opt.sendRetries);
     const send = async (attemptNumber) => {
@@ -250,6 +250,14 @@ async function sendMessage(qrl, command, opt, messageOptions) {
     const shouldRetry = async (result, error) => {
         if (!error)
             return false;
+        if (error instanceof client_sqs_1.QueueDoesNotExist) {
+            const qname = (0, qrlCache_js_1.normalizeQueueName)(queue, opt);
+            (0, qrlCache_js_1.qrlCacheInvalidate)(qname);
+            // clear cache in case cache does not reflect reality, then try recreating the queue before sending message again
+            const newQrl = await getOrCreateQueue(queue, opt);
+            params.QueueUrl = newQrl;
+            cmd = new client_sqs_1.SendMessageCommand(params);
+        }
         for (const exceptionClass of retryableExceptions) {
             if (error instanceof exceptionClass) {
                 debug({ sendMessageRetryingBecause: { error, result } });
@@ -265,7 +273,7 @@ async function sendMessage(qrl, command, opt, messageOptions) {
     return result;
 }
 exports.sendMessage = sendMessage;
-async function sendMessageBatch(qrl, messages, opt) {
+async function sendMessageBatch(qrl, queue, messages, opt) {
     debug('sendMessageBatch(', qrl, messages.map(e => Object.assign(Object.assign({}, e), { MessageBody: e.MessageBody.slice(0, 10) + '...' })), ')');
     const params = { Entries: messages, QueueUrl: qrl };
     if (opt.sentryDsn) {
@@ -291,7 +299,7 @@ async function sendMessageBatch(qrl, messages, opt) {
     }
     // Send them
     const client = (0, sqs_js_1.getSQSClient)();
-    const cmd = new client_sqs_1.SendMessageBatchCommand(params);
+    let cmd = new client_sqs_1.SendMessageBatchCommand(params);
     debug({ cmd });
     const backoff = new exponentialBackoff_js_1.ExponentialBackoff(opt.sendRetries);
     const send = async (attemptNumber) => {
@@ -299,7 +307,7 @@ async function sendMessageBatch(qrl, messages, opt) {
         const data = await client.send(cmd);
         return data;
     };
-    const shouldRetry = (result, error) => {
+    const shouldRetry = async (result, error) => {
         debug({ shouldRetry: { error, result } });
         if (result) {
             // Handle failed result of one or more messages in the batch
@@ -323,6 +331,14 @@ async function sendMessageBatch(qrl, messages, opt) {
             if (opt.sentryDsn) {
                 (0, node_1.addBreadcrumb)({ category: 'sendMessageBatch', message: JSON.stringify({ error }), level: 'error' });
             }
+            if (error instanceof client_sqs_1.QueueDoesNotExist) {
+                const qname = (0, qrlCache_js_1.normalizeQueueName)(queue, opt);
+                (0, qrlCache_js_1.qrlCacheInvalidate)(qname);
+                // Clear stale cache entry and recreate queue before retrying
+                const newQrl = await getOrCreateQueue(queue, opt);
+                params.QueueUrl = newQrl;
+                cmd = new client_sqs_1.SendMessageBatchCommand(params);
+            }
             for (const exceptionClass of retryableExceptions) {
                 debug({ exceptionClass, retryableExceptions });
                 if (error instanceof exceptionClass) {
@@ -341,8 +357,8 @@ let requestCount = 0;
 // If the message is too large, batch is retried with half the messages.
 // Returns number of messages flushed.
 //
-async function flushMessages(qrl, opt, sendBuffer) {
-    debug('flushMessages', { qrl, sendBuffer });
+async function flushMessages(qrl, queue, opt, sendBuffer) {
+    debug('flushMessages', { qrl, queue, sendBuffer });
     // Track our outgoing messages to map with Failed / Successful returns
     const messagesById = new Map();
     const resultsById = new Map();
@@ -376,7 +392,7 @@ async function flushMessages(qrl, opt, sendBuffer) {
                 nextSize = 0;
         }
         // Send batch
-        const data = await sendMessageBatch(qrl, batch, opt);
+        const data = await sendMessageBatch(qrl, queue, batch, opt);
         // Fail if there are any individual message failures
         if (data?.Failed && data?.Failed.length) {
             const err = new Error('One or more message failures: ' + JSON.stringify(data.Failed));
@@ -412,13 +428,13 @@ exports.flushMessages = flushMessages;
 // Returns number of messages flushed.
 //
 const debugAddMessage = (0, debug_1.default)('qdone:enqueue:addMessage');
-async function addMessage(qrl, command, messageIndex, opt, sendBuffer, messageOptions) {
+async function addMessage(qrl, queue, command, messageIndex, opt, sendBuffer, messageOptions) {
     const message = formatMessage(command, messageIndex, opt, messageOptions);
     sendBuffer[qrl] = sendBuffer[qrl] || [];
     sendBuffer[qrl].push(message);
     debugAddMessage({ location: 'addMessage', messageIndex, sendBuffer });
     if (sendBuffer[qrl].length >= 10) {
-        return flushMessages(qrl, opt, sendBuffer);
+        return flushMessages(qrl, queue, opt, sendBuffer);
     }
     return { numFlushed: 0, results: [] };
 }
@@ -435,7 +451,7 @@ async function enqueue(queue, command, options) {
     }
     try {
         const qrl = await getOrCreateQueue(queue, opt);
-        return sendMessage(qrl, command, opt);
+        return sendMessage(qrl, queue, command, opt);
     }
     catch (e) {
         console.log(e);
@@ -463,10 +479,11 @@ async function enqueueBatch(pairs, options) {
             messageOptions: (0, defaults_js_1.validateMessageOptions)(messageOptions)
         }));
         const uniqueQnames = new Set(normalizedPairs.map(p => p.qname));
-        // Prefetch qrls / create queues in parallel
+        // Prefetch qrls / create queues in parallel, building a reverse map for cache invalidation
         const createPromises = [];
+        const qrlToQname = new Map();
         for (const qname of uniqueQnames) {
-            createPromises.push(getOrCreateQueue(qname, opt));
+            createPromises.push(getOrCreateQueue(qname, opt).then(qrl => qrlToQname.set(qrl, qname)));
         }
         await Promise.all(createPromises);
         // After we've prefetched, all qrls are in cache
@@ -477,14 +494,14 @@ async function enqueueBatch(pairs, options) {
         let initialFlushTotal = 0;
         for (const { qname, command, messageOptions } of normalizedPairs) {
             const qrl = await getOrCreateQueue(qname, opt);
-            const { numFlushed, results } = await addMessage(qrl, command, messageIndex++, opt, sendBuffer, messageOptions);
+            const { numFlushed, results } = await addMessage(qrl, qname, command, messageIndex++, opt, sendBuffer, messageOptions);
             initialFlushTotal += numFlushed;
             allResults.push(...results);
         }
         // And flush any remaining messages
         const extraFlushPromises = [];
         for (const qrl in sendBuffer) {
-            extraFlushPromises.push(flushMessages(qrl, opt, sendBuffer));
+            extraFlushPromises.push(flushMessages(qrl, qrlToQname.get(qrl), opt, sendBuffer));
         }
         let extraFlushTotal = 0;
         for (const { numFlushed, results } of await Promise.all(extraFlushPromises)) {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "qdone",
-  "version": "2.0.56-alpha",
+  "version": "2.1.1",
   "description": "A distributed scheduler for SQS",
   "type": "module",
   "main": "./index.js",
@@ -57,8 +57,12 @@
     "publish-latest": "npm publish --tag latest",
     "publish-next": "npm publish --tag next"
   },
-  "author": "Ryan Witt",
-  "license": "MIT",
+  "author": "SureDone, Inc.",
+  "contributors": [
+    "Ryan Witt <onecreativenerd@gmail.com>",
+    "Shane O'Hanlon <shane.c.ohanlon@gmail.com>"
+  ],
+  "license": "ISC",
   "repository": {
     "type": "git",
     "url": "git+https://github.com/suredone/qdone.git"

package/src/check.js

@@ -103,7 +103,7 @@ export async function checkFailQueue (queue, qrl, opt, indent = '') {
   try {
     // Get fail queue params, creating fail queue if it doesn't exist and create flag is set
     if (opt.verbose) console.error(chalk.blue(indent + 'checking ') + fqname)
-    const { params: { Attributes: desired } } = await getFailParams(queue, opt)
+    const { Attributes: desired } = await getFailParams(queue, opt)
     const { Attributes: current } = await getQueueAttributes(fqrl)
     if (attributesMatch(current, desired, opt, indent + '  ')) {
       if (opt.verbose) console.error(chalk.green(indent + '  all good'))
@@ -128,7 +128,7 @@ export async function checkQueue (queue, qrl, opt, indent = '') {
   if (opt.verbose) console.error(chalk.blue(indent + 'checking ') + qname)
   await checkFailQueue(queue, qrl, opt, indent + '  ')
   try {
-    const { params: { Attributes: desired } } = await getQueueParams(queue, opt)
+    const { Attributes: desired } = await getQueueParams(queue, opt)
     const { Attributes: current, $metadata } = await getQueueAttributes(qrl)
     debug({ current, $metadata })
     if (attributesMatch(current, desired, opt, indent + '  ')) {

package/src/cli.js

@@ -520,7 +520,8 @@ export async function root (originalArgv, testHook) {
         { name: 'enqueue-batch', summary: 'Enqueue multiple commands from stdin or a file' },
         { name: 'worker', summary: 'Execute work on one or more queues' },
         { name: 'idle-queues', summary: 'Write a list of idle queues to stdout' },
-        { name: 'monitor', summary: 'Monitor multiple queues at once' }
+        { name: 'monitor', summary: 'Monitor multiple queues at once' },
+        { name: 'check', summary: 'Check or update queue configurations' }
       ]
     },
     { content: 'Global Options', raw: true },

package/src/enqueue.js

@@ -15,6 +15,7 @@ import {
 import {
   qrlCacheGet,
   qrlCacheSet,
+  qrlCacheInvalidate,
   normalizeQueueName,
   normalizeFailQueueName,
   normalizeDLQName
@@ -221,7 +222,7 @@ const retryableExceptions = [
   QueueDoesNotExist // Queue could temporarily not exist due to eventual consistency, let it retry
 ]
 
-export async function sendMessage (qrl, command, opt, messageOptions) {
+export async function sendMessage (qrl, queue, command, opt, messageOptions) {
   debug('sendMessage(', qrl, command, ')')
   const uuidFunction = opt.uuidFunction || uuidV1
   const params = {
@@ -237,7 +238,7 @@ export async function sendMessage (qrl, command, opt, messageOptions) {
 
   // Send it
   const client = getSQSClient()
-  const cmd = new SendMessageCommand(params)
+  let cmd = new SendMessageCommand(params)
   debug({ cmd })
   const backoff = new ExponentialBackoff(opt.sendRetries)
   const send = async (attemptNumber) => {
@@ -248,6 +249,16 @@ export async function sendMessage (qrl, command, opt, messageOptions) {
   }
   const shouldRetry = async (result, error) => {
     if (!error) return false
+
+    if (error instanceof QueueDoesNotExist) {
+      const qname = normalizeQueueName(queue, opt)
+      qrlCacheInvalidate(qname)
+      // clear cache in case cache does not reflect reality, then try recreating the queue before sending message again
+      const newQrl = await getOrCreateQueue(queue, opt)
+      params.QueueUrl = newQrl
+      cmd = new SendMessageCommand(params)
+    }
+
     for (const exceptionClass of retryableExceptions) {
       if (error instanceof exceptionClass) {
         debug({ sendMessageRetryingBecause: { error, result } })
@@ -263,7 +274,7 @@ export async function sendMessage (qrl, command, opt, messageOptions) {
   return result
 }
 
-export async function sendMessageBatch (qrl, messages, opt) {
+export async function sendMessageBatch (qrl, queue, messages, opt) {
   debug('sendMessageBatch(', qrl, messages.map(e => Object.assign(Object.assign({}, e), { MessageBody: e.MessageBody.slice(0, 10) + '...' })), ')')
   const params = { Entries: messages, QueueUrl: qrl }
   if (opt.sentryDsn) {
@@ -292,7 +303,7 @@ export async function sendMessageBatch (qrl, messages, opt) {
 
   // Send them
   const client = getSQSClient()
-  const cmd = new SendMessageBatchCommand(params)
+  let cmd = new SendMessageBatchCommand(params)
   debug({ cmd })
   const backoff = new ExponentialBackoff(opt.sendRetries)
   const send = async (attemptNumber) => {
@@ -300,7 +311,7 @@ export async function sendMessageBatch (qrl, messages, opt) {
     const data = await client.send(cmd)
     return data
   }
-  const shouldRetry = (result, error) => {
+  const shouldRetry = async (result, error) => {
     debug({ shouldRetry: { error, result } })
     if (result) {
       // Handle failed result of one or more messages in the batch
@@ -323,6 +334,16 @@ export async function sendMessageBatch (qrl, messages, opt) {
       if (opt.sentryDsn) {
         addBreadcrumb({ category: 'sendMessageBatch', message: JSON.stringify({ error }), level: 'error' })
       }
+
+      if (error instanceof QueueDoesNotExist) {
+        const qname = normalizeQueueName(queue, opt)
+        qrlCacheInvalidate(qname)
+        // Clear stale cache entry and recreate queue before retrying
+        const newQrl = await getOrCreateQueue(queue, opt)
+        params.QueueUrl = newQrl
+        cmd = new SendMessageBatchCommand(params)
+      }
+
       for (const exceptionClass of retryableExceptions) {
         debug({ exceptionClass, retryableExceptions })
         if (error instanceof exceptionClass) {
@@ -342,8 +363,8 @@ let requestCount = 0
 // If the message is too large, batch is retried with half the messages.
 // Returns number of messages flushed.
 //
-export async function flushMessages (qrl, opt, sendBuffer) {
-  debug('flushMessages', { qrl, sendBuffer })
+export async function flushMessages (qrl, queue, opt, sendBuffer) {
+  debug('flushMessages', { qrl, queue, sendBuffer })
   // Track our outgoing messages to map with Failed / Successful returns
   const messagesById = new Map()
   const resultsById = new Map()
@@ -376,7 +397,7 @@ export async function flushMessages (qrl, opt, sendBuffer) {
     }
 
     // Send batch
-    const data = await sendMessageBatch(qrl, batch, opt)
+    const data = await sendMessageBatch(qrl, queue, batch, opt)
 
     // Fail if there are any individual message failures
     if (data?.Failed && data?.Failed.length) {
@@ -413,13 +434,13 @@ export async function flushMessages (qrl, opt, sendBuffer) {
 // Returns number of messages flushed.
 //
 const debugAddMessage = Debug('qdone:enqueue:addMessage')
-export async function addMessage (qrl, command, messageIndex, opt, sendBuffer, messageOptions) {
+export async function addMessage (qrl, queue, command, messageIndex, opt, sendBuffer, messageOptions) {
   const message = formatMessage(command, messageIndex, opt, messageOptions)
   sendBuffer[qrl] = sendBuffer[qrl] || []
   sendBuffer[qrl].push(message)
   debugAddMessage({ location: 'addMessage', messageIndex, sendBuffer })
   if (sendBuffer[qrl].length >= 10) {
-    return flushMessages(qrl, opt, sendBuffer)
+    return flushMessages(qrl, queue, opt, sendBuffer)
   }
   return { numFlushed: 0, results: [] }
 }
@@ -436,7 +457,7 @@ export async function enqueue (queue, command, options) {
   }
   try {
     const qrl = await getOrCreateQueue(queue, opt)
-    return sendMessage(qrl, command, opt)
+    return sendMessage(qrl, queue, command, opt)
   } catch (e) {
     console.log(e)
     throw e
@@ -464,10 +485,13 @@ export async function enqueueBatch (pairs, options) {
     }))
     const uniqueQnames = new Set(normalizedPairs.map(p => p.qname))
 
-    // Prefetch qrls / create queues in parallel
+    // Prefetch qrls / create queues in parallel, building a reverse map for cache invalidation
     const createPromises = []
+    const qrlToQname = new Map()
     for (const qname of uniqueQnames) {
-      createPromises.push(getOrCreateQueue(qname, opt))
+      createPromises.push(
+        getOrCreateQueue(qname, opt).then(qrl => qrlToQname.set(qrl, qname))
+      )
     }
     await Promise.all(createPromises)
     // After we've prefetched, all qrls are in cache
@@ -478,7 +502,7 @@ export async function enqueueBatch (pairs, options) {
     let initialFlushTotal = 0
     for (const { qname, command, messageOptions } of normalizedPairs) {
       const qrl = await getOrCreateQueue(qname, opt)
-      const { numFlushed, results } = await addMessage(qrl, command, messageIndex++, opt, sendBuffer, messageOptions)
+      const { numFlushed, results } = await addMessage(qrl, qname, command, messageIndex++, opt, sendBuffer, messageOptions)
       initialFlushTotal += numFlushed
       allResults.push(...results)
     }
@@ -486,7 +510,7 @@ export async function enqueueBatch (pairs, options) {
     // And flush any remaining messages
     const extraFlushPromises = []
     for (const qrl in sendBuffer) {
-      extraFlushPromises.push(flushMessages(qrl, opt, sendBuffer))
+      extraFlushPromises.push(flushMessages(qrl, qrlToQname.get(qrl), opt, sendBuffer))
     }
     let extraFlushTotal = 0
     for (const { numFlushed, results } of await Promise.all(extraFlushPromises)) {