qdone 2.0.35-alpha → 2.0.37-alpha

package/commonjs/src/enqueue.js

@@ -3,7 +3,7 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
     return (mod && mod.__esModule) ? mod : { "default": mod };
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.enqueueBatch = exports.enqueue = exports.addMessage = exports.flushMessages = exports.sendMessageBatch = exports.sendMessage = exports.formatMessage = exports.getQueueAttributes = exports.getOrCreateQueue = exports.getOrCreateFailQueue = exports.getOrCreateDLQ = void 0;
+exports.enqueueBatch = exports.enqueue = exports.addMessage = exports.flushMessages = exports.sendMessageBatch = exports.sendMessage = exports.formatMessage = exports.getQueueAttributes = exports.getOrCreateQueue = exports.getQueueParams = exports.getOrCreateFailQueue = exports.getFailParams = exports.getOrCreateDLQ = exports.getDLQParams = void 0;
 const node_1 = require("@sentry/node");
 const uuid_1 = require("uuid");
 const chalk_1 = __importDefault(require("chalk"));
@@ -11,12 +11,26 @@ const debug_1 = __importDefault(require("debug"));
 const client_sqs_1 = require("@aws-sdk/client-sqs");
 const qrlCache_js_1 = require("./qrlCache.js");
 const sqs_js_1 = require("./sqs.js");
+const dedup_js_1 = require("./dedup.js");
 const defaults_js_1 = require("./defaults.js");
 const exponentialBackoff_js_1 = require("./exponentialBackoff.js");
 const debug = (0, debug_1.default)('qdone:enqueue');
+function getDLQParams(queue, opt) {
+    const dqname = (0, qrlCache_js_1.normalizeDLQName)(queue, opt);
+    const params = {
+        Attributes: { MessageRetentionPeriod: opt.messageRetentionPeriod + '' },
+        QueueName: dqname
+    };
+    if (opt.tags)
+        params.tags = opt.tags;
+    if (opt.fifo)
+        params.Attributes.FifoQueue = 'true';
+    return { dqname, params };
+}
+exports.getDLQParams = getDLQParams;
 async function getOrCreateDLQ(queue, opt) {
     debug('getOrCreateDLQ(', queue, ')');
-    const dqname = (0, qrlCache_js_1.normalizeDLQName)(queue, opt);
+    const { dqname, params } = getDLQParams(queue, opt);
     try {
         const dqrl = await (0, qrlCache_js_1.qrlCacheGet)(dqname);
         return dqrl;
@@ -27,14 +41,6 @@ async function getOrCreateDLQ(queue, opt) {
             throw err;
         // Create our DLQ
         const client = (0, sqs_js_1.getSQSClient)();
-        const params = {
-            Attributes: { MessageRetentionPeriod: opt.messageRetentionPeriod + '' },
-            QueueName: dqname
-        };
-        if (opt.tags)
-            params.tags = opt.tags;
-        if (opt.fifo)
-            params.Attributes.FifoQueue = 'true';
         const cmd = new client_sqs_1.CreateQueueCommand(params);
         if (opt.verbose)
             console.error(chalk_1.default.blue('Creating dead letter queue ') + dqname);
@@ -46,10 +52,47 @@ async function getOrCreateDLQ(queue, opt) {
     }
 }
 exports.getOrCreateDLQ = getOrCreateDLQ;
-async function getOrCreateFailQueue(queue, opt) {
+/**
+ * Returns the parameters needed for creating a failed queue. If DLQ options
+ * are set, it makes an API call to get this DLQ's ARN.
+ */
+async function getFailParams(queue, opt) {
+    const fqname = (0, qrlCache_js_1.normalizeFailQueueName)(queue, opt);
+    const params = {
+        Attributes: { MessageRetentionPeriod: opt.messageRetentionPeriod + '' },
+        QueueName: fqname
+    };
+    // If we have a dlq, we grab it and set a redrive policy
+    if (opt.dlq) {
+        const dqname = (0, qrlCache_js_1.normalizeDLQName)(queue, opt);
+        const dqrl = await (0, qrlCache_js_1.qrlCacheGet)(dqname);
+        const dqa = await getQueueAttributes(dqrl);
+        debug('dqa', dqa);
+        params.Attributes.RedrivePolicy = JSON.stringify({
+            deadLetterTargetArn: dqa.Attributes.QueueArn,
+            maxReceiveCount: opt.dlqAfter
+        });
+    }
+    if (opt.failDelay)
+        params.Attributes.DelaySeconds = opt.failDelay + '';
+    if (opt.tags)
+        params.tags = opt.tags;
+    if (opt.fifo)
+        params.Attributes.FifoQueue = 'true';
+    return params;
+}
+exports.getFailParams = getFailParams;
+/**
+ * Returns the qrl for the failed queue for the given queue. Creates the queue
+ * if it does not exist.
+ */
+async function getOrCreateFailQueue(queue, opt, doesNotExist) {
     debug('getOrCreateFailQueue(', queue, ')');
     const fqname = (0, qrlCache_js_1.normalizeFailQueueName)(queue, opt);
     try {
+        // Bail early if the caller knew we didn't have a queue
+        if (doesNotExist)
+            throw new client_sqs_1.QueueDoesNotExist(fqname);
         const fqrl = await (0, qrlCache_js_1.qrlCacheGet)(fqname);
         return fqrl;
     }
@@ -57,28 +100,20 @@ async function getOrCreateFailQueue(queue, opt) {
         // Anything other than queue doesn't exist gets re-thrown
         if (!(err instanceof client_sqs_1.QueueDoesNotExist))
             throw err;
-        // Crate our fail queue
-        const client = (0, sqs_js_1.getSQSClient)();
-        const params = {
-            Attributes: { MessageRetentionPeriod: opt.messageRetentionPeriod + '' },
-            QueueName: fqname
-        };
-        // If we have a dlq, we grab it and set a redrive policy
-        if (opt.dlq) {
-            const dqrl = await getOrCreateDLQ(queue, opt);
-            const dqa = await getQueueAttributes(dqrl);
-            debug('dqa', dqa);
-            params.Attributes.RedrivePolicy = JSON.stringify({
-                deadLetterTargetArn: dqa.Attributes.QueueArn,
-                maxReceiveCount: opt.dlqAfter + ''
-            });
+        // Grab params, creating DLQ if needed
+        let params;
+        try {
+            params = await getFailParams(queue, opt);
         }
-        if (opt.failDelay)
-            params.Attributes.DelaySeconds = opt.failDelay + '';
-        if (opt.tags)
-            params.tags = opt.tags;
-        if (opt.fifo)
-            params.Attributes.FifoQueue = 'true';
+        catch (e) {
+            // If DLQ doesn't exist, create it
+            if (!(opt.dlq && e instanceof client_sqs_1.QueueDoesNotExist))
+                throw e;
+            await getOrCreateDLQ(queue, opt);
+            params = await getFailParams(queue, opt);
+        }
+        // Create our fail queue
+        const client = (0, sqs_js_1.getSQSClient)();
         const cmd = new client_sqs_1.CreateQueueCommand(params);
         if (opt.verbose)
             console.error(chalk_1.default.blue('Creating fail queue ') + fqname);
@@ -90,6 +125,32 @@ async function getOrCreateFailQueue(queue, opt) {
     }
 }
 exports.getOrCreateFailQueue = getOrCreateFailQueue;
+/**
+ * Returns the parameters needed for creating a queue. If fail options
+ * are set, it makes an API call to get the fail queue's ARN.
+ */
+async function getQueueParams(queue, opt) {
+    const qname = (0, qrlCache_js_1.normalizeQueueName)(queue, opt);
+    const fqname = (0, qrlCache_js_1.normalizeFailQueueName)(queue, opt);
+    const fqrl = await (0, qrlCache_js_1.qrlCacheGet)(fqname, opt);
+    const fqa = await getQueueAttributes(fqrl);
+    const params = {
+        Attributes: {
+            MessageRetentionPeriod: opt.messageRetentionPeriod + '',
+            RedrivePolicy: JSON.stringify({
+                deadLetterTargetArn: fqa.Attributes.QueueArn,
+                maxReceiveCount: 1
+            })
+        },
+        QueueName: qname
+    };
+    if (opt.tags)
+        params.tags = opt.tags;
+    if (opt.fifo)
+        params.Attributes.FifoQueue = 'true';
+    return params;
+}
+exports.getQueueParams = getQueueParams;
 /**
  * Returns a qrl for a queue that either exists or does not
  */
@@ -104,31 +165,26 @@ async function getOrCreateQueue(queue, opt) {
         // Anything other than queue doesn't exist gets re-thrown
         if (!(err instanceof client_sqs_1.QueueDoesNotExist))
             throw err;
-        // Get our fail queue so we can create our own
-        const fqrl = await getOrCreateFailQueue(qname, opt);
-        const fqa = await getQueueAttributes(fqrl);
+        // Grab params, creating fail queue if needed
+        let params;
+        try {
+            params = await getQueueParams(qname, opt);
+        }
+        catch (e) {
+            // If fail queue doesn't exist, create it
+            if (!(e instanceof client_sqs_1.QueueDoesNotExist))
+                throw e;
+            await getOrCreateFailQueue(qname, opt, true);
+            params = await getQueueParams(qname, opt);
+        }
+        debug({ getOrCreateQueue: { qname, params } });
         // Create our queue
         const client = (0, sqs_js_1.getSQSClient)();
-        const params = {
-            Attributes: {
-                MessageRetentionPeriod: opt.messageRetentionPeriod + '',
-                RedrivePolicy: JSON.stringify({
-                    deadLetterTargetArn: fqa.Attributes.QueueArn,
-                    maxReceiveCount: '1'
-                })
-            },
-            QueueName: qname
-        };
-        if (opt.tags)
-            params.tags = opt.tags;
-        if (opt.fifo)
-            params.Attributes.FifoQueue = 'true';
         const cmd = new client_sqs_1.CreateQueueCommand(params);
-        debug({ params });
         if (opt.verbose)
-            console.error(chalk_1.default.blue('Creating queue ') + qname);
+            console.error(chalk_1.default.blue('Creating fail queue ') + qname);
         const data = await client.send(cmd);
-        debug('createQueue returned', data);
+        debug('AWS createQueue returned', data);
         const qrl = data.QueueUrl;
         (0, qrlCache_js_1.qrlCacheSet)(qname, qrl);
         return qrl;
@@ -146,18 +202,18 @@ async function getQueueAttributes(qrl) {
     return data;
 }
 exports.getQueueAttributes = getQueueAttributes;
-function formatMessage(command, id) {
-    const message = {
-        /*
-        MessageAttributes: {
-          City: { DataType: 'String', StringValue: 'Any City' },
-          Population: { DataType: 'Number', StringValue: '1250800' }
-        },
-        */
-        MessageBody: command
-    };
+function formatMessage(body, id, opt, messageOptions) {
+    const message = { MessageBody: body };
     if (typeof id !== 'undefined')
         message.Id = '' + id;
+    if (opt.fifo) {
+        message.MessageGroupId = messageOptions?.groupId || opt?.groupId;
+    }
+    (0, dedup_js_1.addDedupParamsToMessage)(message, opt, messageOptions);
+    if (opt.delay)
+        message.DelaySeconds = opt.delay;
+    if (messageOptions?.delay)
+        message.DelaySeconds = messageOptions.delay;
     return message;
 }
 exports.formatMessage = formatMessage;
@@ -167,15 +223,19 @@ const retryableExceptions = [
     client_sqs_1.KmsThrottled,
     client_sqs_1.QueueDoesNotExist // Queue could temporarily not exist due to eventual consistency, let it retry
 ];
-async function sendMessage(qrl, command, opt) {
+async function sendMessage(qrl, command, opt, messageOptions) {
     debug('sendMessage(', qrl, command, ')');
-    const params = Object.assign({ QueueUrl: qrl }, formatMessage(command));
-    if (opt.fifo) {
-        params.MessageGroupId = opt.groupId;
-        params.MessageDeduplicationId = opt.deduplicationId || (0, uuid_1.v1)();
+    const uuidFunction = opt.uuidFunction || uuid_1.v1;
+    const params = {
+        QueueUrl: qrl,
+        ...formatMessage(command, null, opt, messageOptions)
+    };
+    // See if we even have to send it
+    if (opt.externalDedup) {
+        const shouldEnqueue = await (0, dedup_js_1.dedupShouldEnqueue)(params, opt);
+        if (!shouldEnqueue)
+            return { MessageId: uuidFunction() };
     }
-    if (opt.delay)
-        params.DelaySeconds = opt.delay;
     // Send it
     const client = (0, sqs_js_1.getSQSClient)();
     const cmd = new client_sqs_1.SendMessageCommand(params);
@@ -188,12 +248,16 @@ async function sendMessage(qrl, command, opt) {
         return data;
     };
     const shouldRetry = async (result, error) => {
+        if (!error)
+            return false;
         for (const exceptionClass of retryableExceptions) {
             if (error instanceof exceptionClass) {
                 debug({ sendMessageRetryingBecause: { error, result } });
                 return true;
             }
         }
+        // If we could not send it, we also need to remove our dedup flag
+        await (0, dedup_js_1.dedupSuccessfullyProcessed)(params, opt);
         return false;
     };
     const result = await backoff.run(send, shouldRetry);
@@ -204,21 +268,18 @@ exports.sendMessage = sendMessage;
 async function sendMessageBatch(qrl, messages, opt) {
     debug('sendMessageBatch(', qrl, messages.map(e => Object.assign(Object.assign({}, e), { MessageBody: e.MessageBody.slice(0, 10) + '...' })), ')');
     const params = { Entries: messages, QueueUrl: qrl };
-    const uuidFunction = opt.uuidFunction || uuid_1.v1;
-    // Add in group id if we're using fifo
-    if (opt.fifo) {
-        params.Entries = params.Entries.map(message => Object.assign({
-            MessageGroupId: opt.groupIdPerMessage ? uuidFunction() : opt.groupId,
-            MessageDeduplicationId: opt.deduplicationId || uuidFunction()
-        }, message));
-    }
-    if (opt.delay) {
-        params.Entries = params.Entries.map(message => Object.assign({ DelaySeconds: opt.delay }, message));
-    }
     if (opt.sentryDsn) {
         (0, node_1.addBreadcrumb)({ category: 'sendMessageBatch', message: JSON.stringify({ params }), level: 'debug' });
     }
     debug({ params });
+    // See which messages we even have to send
+    if (opt.externalDedup) {
+        const promises = params.Entries.map(async (m) => ({ m, shouldEnqueue: await (0, dedup_js_1.dedupShouldEnqueue)(m, opt) }));
+        const results = await Promise.all(promises);
+        params.Entries = results.filter(({ shouldEnqueue }) => shouldEnqueue);
+        if (!params.Entries.length)
+            return;
+    }
     // Send them
     const client = (0, sqs_js_1.getSQSClient)();
     const cmd = new client_sqs_1.SendMessageBatchCommand(params);
@@ -318,8 +379,8 @@ exports.flushMessages = flushMessages;
 // Automaticaly flushes if queue has >= 10 messages.
 // Returns number of messages flushed.
 //
-async function addMessage(qrl, command, messageIndex, opt, sendBuffer) {
-    const message = formatMessage(command, messageIndex);
+async function addMessage(qrl, command, messageIndex, opt, sendBuffer, messageOptions) {
+    const message = formatMessage(command, messageIndex, opt, messageOptions);
     sendBuffer[qrl] = sendBuffer[qrl] || [];
     sendBuffer[qrl].push(message);
     debug({ location: 'addMessage', sendBuffer });
@@ -336,8 +397,17 @@ exports.addMessage = addMessage;
 async function enqueue(queue, command, options) {
     debug('enqueue(', { queue, command }, ')');
     const opt = (0, defaults_js_1.getOptionsWithDefaults)(options);
-    const qrl = await getOrCreateQueue(queue, opt);
-    return sendMessage(qrl, command, opt);
+    if (opt.sentryDsn) {
+        (0, node_1.setExtra)({ qdoneOperation: 'enqueue', args: { queue, command, opt } });
+    }
+    try {
+        const qrl = await getOrCreateQueue(queue, opt);
+        return sendMessage(qrl, command, opt);
+    }
+    catch (e) {
+        console.log(e);
+        throw e;
+    }
 }
 exports.enqueue = enqueue;
 //
@@ -347,39 +417,49 @@ exports.enqueue = enqueue;
 async function enqueueBatch(pairs, options) {
     debug('enqueueBatch(', pairs, ')');
     const opt = (0, defaults_js_1.getOptionsWithDefaults)(options);
-    // Find unique queues so we can pre-fetch qrls. We do this so that all
-    // queues are created prior to going through our flush logic
-    const normalizedPairs = pairs.map(({ queue, command }) => ({
-        qname: (0, qrlCache_js_1.normalizeQueueName)(queue, opt),
-        command
-    }));
-    const uniqueQnames = new Set(normalizedPairs.map(p => p.qname));
-    // Prefetch qrls / create queues in parallel
-    const createPromises = [];
-    for (const qname of uniqueQnames) {
-        createPromises.push(getOrCreateQueue(qname, opt));
+    if (opt.sentryDsn) {
+        (0, node_1.setExtra)({ qdoneOperation: 'enqueueBatch', args: { pairs, opt } });
     }
-    await Promise.all(createPromises);
-    // After we've prefetched, all qrls are in cache
-    // so go back through the list of pairs and fire off messages
-    requestCount = 0;
-    const sendBuffer = {};
-    let messageIndex = 0;
-    let initialFlushTotal = 0;
-    for (const { qname, command } of normalizedPairs) {
-        const qrl = await getOrCreateQueue(qname, opt);
-        initialFlushTotal += await addMessage(qrl, command, messageIndex++, opt, sendBuffer);
+    try {
+        // Find unique queues so we can pre-fetch qrls. We do this so that all
+        // queues are created prior to going through our flush logic
+        const normalizedPairs = pairs.map(({ queue, command, messageOptions }) => ({
+            qname: (0, qrlCache_js_1.normalizeQueueName)(queue, opt),
+            command,
+            messageOptions: (0, defaults_js_1.validateMessageOptions)(messageOptions)
+        }));
+        const uniqueQnames = new Set(normalizedPairs.map(p => p.qname));
+        // Prefetch qrls / create queues in parallel
+        const createPromises = [];
+        for (const qname of uniqueQnames) {
+            createPromises.push(getOrCreateQueue(qname, opt));
+        }
+        await Promise.all(createPromises);
+        // After we've prefetched, all qrls are in cache
+        // so go back through the list of pairs and fire off messages
+        requestCount = 0;
+        const sendBuffer = {};
+        let messageIndex = 0;
+        let initialFlushTotal = 0;
+        for (const { qname, command, messageOptions } of normalizedPairs) {
+            const qrl = await getOrCreateQueue(qname, opt);
+            initialFlushTotal += await addMessage(qrl, command, messageIndex++, opt, sendBuffer, messageOptions);
+        }
+        // And flush any remaining messages
+        const extraFlushPromises = [];
+        for (const qrl in sendBuffer) {
+            extraFlushPromises.push(flushMessages(qrl, opt, sendBuffer));
+        }
+        const extraFlushCounts = await Promise.all(extraFlushPromises);
+        const extraFlushTotal = extraFlushCounts.reduce((a, b) => a + b, 0);
+        const totalFlushed = initialFlushTotal + extraFlushTotal;
+        debug({ initialFlushTotal, extraFlushTotal, totalFlushed });
+        return totalFlushed;
     }
-    // And flush any remaining messages
-    const extraFlushPromises = [];
-    for (const qrl in sendBuffer) {
-        extraFlushPromises.push(flushMessages(qrl, opt, sendBuffer));
+    catch (e) {
+        console.log(e);
+        throw e;
     }
-    const extraFlushCounts = await Promise.all(extraFlushPromises);
-    const extraFlushTotal = extraFlushCounts.reduce((a, b) => a + b, 0);
-    const totalFlushed = initialFlushTotal + extraFlushTotal;
-    debug({ initialFlushTotal, extraFlushTotal, totalFlushed });
-    return totalFlushed;
 }
 exports.enqueueBatch = enqueueBatch;
 debug('loaded');

package/commonjs/src/qrlCache.js

@@ -70,7 +70,8 @@ async function qrlCacheGet(qname) {
     // debug({ cmd })
     const result = await client.send(cmd);
     // debug('result', result)
-    // if (!result) throw new Error(`No such queue ${qname}`)
+    if (!result)
+        throw new client_sqs_1.QueueDoesNotExist(qname);
     const { QueueUrl: qrl } = result;
     // debug('getQueueUrl returned', data)
     qcache.set(qname, qrl);

package/commonjs/src/scheduler/jobExecutor.js

@@ -11,6 +11,7 @@ exports.JobExecutor = void 0;
 const client_sqs_1 = require("@aws-sdk/client-sqs");
 const chalk_1 = __importDefault(require("chalk"));
 const debug_1 = __importDefault(require("debug"));
+const dedup_js_1 = require("../dedup.js");
 const sqs_js_1 = require("../sqs.js");
 const debug = (0, debug_1.default)('qdone:jobExecutor');
 const maxJobSeconds = 12 * 60 * 60;
@@ -217,6 +218,8 @@ class JobExecutor {
                     }
                 }
                 debug('DeleteMessageBatch returned', result);
+                // Mark batch as processed for dedup
+                await Promise.all(entries.map(e => (0, dedup_js_1.dedupSuccessfullyProcessed)(this.jobsByMessageId[e.Id], this.opt)));
                 // TODO Sentry
             }
         }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "qdone",
-  "version": "2.0.35-alpha",
+  "version": "2.0.37-alpha",
   "description": "A distributed scheduler for SQS",
   "type": "module",
   "main": "./index.js",
@@ -21,6 +21,7 @@
     "command-line-usage": "^7.0.1",
     "debug": "^4.3.4",
     "ioredis": "^5.3.2",
+    "ioredis-mock": "^8.9.0",
     "standard": "^17.1.0",
     "tree-kill": "^1.2.2",
     "uuid": "^9.0.1"
@@ -50,7 +51,7 @@
     "build": "tsc --allowJs index.js --outdir commonjs --esModuleInterop --module commonjs  --target es2021 --strict --skipLibCheck --forceConsistentCasingInFileNames",
     "clean": "rm -rf commonjs/src commonjs/*.js coverage",
     "lint": "standard",
-    "coverage": "nyc report --reporter=text-lcov | coveralls",
+    "coverage": "NODE_OPTIONS='--experimental-json-modules --experimental-vm-modules --no-warnings' jest --coverage | coveralls",
     "standard": "standard",
     "prep-for-publish": "echo YOU MUST USE NPM TO PREP FOR PUBLISH && pnpm run clean && pnpm run build && npm shrinkwrap --production && echo now commit shrinkwrap and use npm run publish-{next,latest}",
     "publish-latest": "npm publish --tag latest",

package/src/cache.js

@@ -12,15 +12,16 @@ let client
  * how to connect.
  */
 export function getCacheClient (opt) {
+  const RedisClass = opt.Redis || Redis
   if (client) {
     return client
   } else if (opt.cacheUri) {
     const url = new URL(opt.cacheUri)
     if (url.protocol === 'redis:') {
-      client = new Redis(url.toString())
+      client = new RedisClass(url.toString())
     } else if (url.protocol === 'redis-cluster:') {
       url.protocol = 'redis:'
-      client = new Redis.Cluster([url.toString()], { slotsRefreshInterval: 60 * 1000 })
+      client = new RedisClass.Cluster([url.toString()], { slotsRefreshInterval: 60 * 1000 })
     } else {
       throw new UsageError(`Only redis:// or redis-cluster:// URLs are currently supported. Got: ${url.protocol}`)
     }