qdone 2.1.0 → 2.2.0

package/LICENSE

@@ -1,6 +1,6 @@
 ISC License
 
-Copyright (c) 2017, Ryan Witt
+Copyright (c) 2017-2026, SureDone, Inc.
 
 Permission to use, copy, modify, and/or distribute this software for any
 purpose with or without fee is hereby granted, provided that the above
@@ -12,4 +12,4 @@ AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT,
 INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM
 LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE
 OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
-PERFORMANCE OF THIS SOFTWARE.
+PERFORMANCE OF THIS SOFTWARE.

package/README.md

@@ -1,9 +1,6 @@
 [![NPM Package](https://img.shields.io/npm/v/qdone.svg)](https://www.npmjs.com/package/qdone)
-[![Build Status](https://travis-ci.org/suredone/qdone.svg?branch=master)](https://travis-ci.org/suredone/qdone)
-[![Coverage Status](https://coveralls.io/repos/github/suredone/qdone/badge.svg)](https://coveralls.io/github/suredone/qdone)
-[![Dependencies](https://img.shields.io/david/suredone/qdone.svg)](https://david-dm.org/suredone/qdone)
+[![Build](https://github.com/suredone/qdone/actions/workflows/build.yaml/badge.svg)](https://github.com/suredone/qdone/actions/workflows/build.yaml)
 [![Standard - JavaScript Style Guide](https://img.shields.io/badge/code_style-standard-brightgreen.svg)](https://standardjs.com)
-[![Greenkeeper badge](https://badges.greenkeeper.io/suredone/qdone.svg)](https://greenkeeper.io/)
 
 # qdone
 

package/commonjs/index.js

@@ -0,0 +1,11 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.monitor = exports.requestShutdown = exports.processMessages = exports.enqueueBatch = exports.enqueue = void 0;
+var enqueue_js_1 = require("./src/enqueue.js");
+Object.defineProperty(exports, "enqueue", { enumerable: true, get: function () { return enqueue_js_1.enqueue; } });
+Object.defineProperty(exports, "enqueueBatch", { enumerable: true, get: function () { return enqueue_js_1.enqueueBatch; } });
+var consumer_js_1 = require("./src/consumer.js");
+Object.defineProperty(exports, "processMessages", { enumerable: true, get: function () { return consumer_js_1.processMessages; } });
+Object.defineProperty(exports, "requestShutdown", { enumerable: true, get: function () { return consumer_js_1.requestShutdown; } });
+var monitor_js_1 = require("./src/monitor.js");
+Object.defineProperty(exports, "monitor", { enumerable: true, get: function () { return monitor_js_1.monitor; } });

package/commonjs/src/cache.js

@@ -0,0 +1,71 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.getCacheClient = getCacheClient;
+exports.shutdownCache = shutdownCache;
+exports.getCache = getCache;
+exports.setCache = setCache;
+const ioredis_1 = __importDefault(require("ioredis"));
+const url_1 = require("url");
+const debug_1 = __importDefault(require("debug"));
+const debug = (0, debug_1.default)('qdone:cache');
+class UsageError extends Error {
+}
+let client;
+/**
+ * Internal function to setup redis client. Parses URI to figure out
+ * how to connect.
+ */
+function getCacheClient(opt) {
+    const RedisClass = opt.Redis || ioredis_1.default;
+    if (client) {
+        return client;
+    }
+    else if (opt.cacheUri) {
+        const url = new url_1.URL(opt.cacheUri);
+        if (url.protocol === 'redis:') {
+            client = new RedisClass(url.toString());
+        }
+        else if (url.protocol === 'redis-cluster:') {
+            url.protocol = 'redis:';
+            client = new RedisClass.Cluster([url.toString()], { slotsRefreshInterval: 60 * 1000 });
+        }
+        else {
+            throw new UsageError(`Only redis:// or redis-cluster:// URLs are currently supported. Got: ${url.protocol}`);
+        }
+        return client;
+    }
+    else {
+        throw new UsageError('Caching requires the --cache-uri option');
+    }
+}
+function shutdownCache() {
+    if (client)
+        client.quit();
+    client = undefined;
+}
+/**
+ * Returns a promise for the item. Resolves to false if cache is empty, object
+ * if it is found.
+ */
+async function getCache(key, opt) {
+    const client = getCacheClient(opt);
+    const cacheKey = opt.cachePrefix + key;
+    debug({ action: 'getCache', cacheKey });
+    const result = await client.get(cacheKey);
+    debug({ action: 'getCache got', cacheKey, result });
+    return result ? JSON.parse(result) : undefined;
+}
+/**
+ * Returns a promise for the status. Encodes object as JSON
+ */
+async function setCache(key, value, opt) {
+    const client = getCacheClient(opt);
+    const encoded = JSON.stringify(value);
+    const cacheKey = opt.cachePrefix + key;
+    debug({ action: 'setCache', cacheKey, value });
+    return client.setex(cacheKey, opt.cacheTtlSeconds, encoded);
+}
+debug('loaded');

package/commonjs/src/cloudWatch.js

@@ -0,0 +1,111 @@
+"use strict";
+/**
+ * Functions that deal with CloudWatch
+ */
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.getCloudWatchClient = getCloudWatchClient;
+exports.setCloudWatchClient = setCloudWatchClient;
+exports.putAggregateData = putAggregateData;
+const client_cloudwatch_1 = require("@aws-sdk/client-cloudwatch");
+const debug_1 = __importDefault(require("debug"));
+const debug = (0, debug_1.default)('qdone:cloudWatch');
+/**
+ * Utility function to return an instantiated, shared CloudWatchClient.
+ */
+let client;
+function getCloudWatchClient() {
+    if (client)
+        return client;
+    client = new client_cloudwatch_1.CloudWatchClient();
+    return client;
+}
+/**
+ * Utility function to set the client explicitly, used in testing.
+ */
+function setCloudWatchClient(explicitClient) {
+    client = explicitClient;
+}
+/**
+ * Takes data in the form returned by getAggregageData() and pushes it to
+ * CloudWatch metrics under the given queueName.
+ *
+ * @param queueName {String} - The name of the wildcard queue these metrics are for.
+ * @param total {Object} - returned object from getAggregateData()
+ */
+async function putAggregateData(total, timestamp) {
+    const client = getCloudWatchClient();
+    const now = timestamp || new Date();
+    const input = {
+        Namespace: 'qmonitor',
+        MetricData: [
+            {
+                MetricName: 'totalQueues',
+                Dimensions: [{
+                        Name: 'queueName',
+                        Value: total.queueName
+                    }],
+                Timestamp: now,
+                Value: total.totalQueues,
+                Unit: 'Count'
+            },
+            {
+                MetricName: 'contributingQueueCount',
+                Dimensions: [{
+                        Name: 'queueName',
+                        Value: total.queueName
+                    }],
+                Timestamp: now,
+                Value: total.contributingQueueNames.length,
+                Unit: 'Count'
+            },
+            {
+                MetricName: 'ApproximateNumberOfMessages',
+                Dimensions: [{
+                        Name: 'queueName',
+                        Value: total.queueName
+                    }],
+                Timestamp: now,
+                Value: total.ApproximateNumberOfMessages || 0,
+                Unit: 'Count'
+            },
+            {
+                MetricName: 'ApproximateNumberOfMessagesDelayed',
+                Dimensions: [{
+                        Name: 'queueName',
+                        Value: total.queueName
+                    }],
+                Timestamp: now,
+                Value: total.ApproximateNumberOfMessagesDelayed || 0,
+                Unit: 'Count'
+            },
+            {
+                MetricName: 'ApproximateNumberOfMessagesNotVisible',
+                Dimensions: [{
+                        Name: 'queueName',
+                        Value: total.queueName
+                    }],
+                Timestamp: now,
+                Value: total.ApproximateNumberOfMessagesNotVisible || 0,
+                Unit: 'Count'
+            },
+            {
+                MetricName: 'ApproximateAgeOfOldestMessage',
+                Dimensions: [{
+                        Name: 'queueName',
+                        Value: total.queueName
+                    }],
+                Timestamp: now,
+                Value: total.ApproximateAgeOfOldestMessage || 0,
+                Unit: 'Seconds'
+            }
+        ]
+    };
+    const command = new client_cloudwatch_1.PutMetricDataCommand(input);
+    // debug({ input, command })
+    const response = await client.send(command);
+    debug({ response });
+}
+debug('loaded');

package/commonjs/src/consumer.js

@@ -0,0 +1,172 @@
+"use strict";
+/**
+ * Consumer implementation.
+ */
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.requestShutdown = requestShutdown;
+exports.getMessages = getMessages;
+exports.processMessages = processMessages;
+const os_1 = require("os");
+const client_sqs_1 = require("@aws-sdk/client-sqs");
+const chalk_1 = __importDefault(require("chalk"));
+const debug_1 = __importDefault(require("debug"));
+const systemMonitor_js_1 = require("./scheduler/systemMonitor.js");
+const queueManager_js_1 = require("./scheduler/queueManager.js");
+const jobExecutor_js_1 = require("./scheduler/jobExecutor.js");
+const defaults_js_1 = require("./defaults.js");
+const sqs_js_1 = require("./sqs.js");
+const debug = (0, debug_1.default)('qdone:consumer');
+// Global flag for shutdown request
+let shutdownRequested = false;
+const shutdownCallbacks = [];
+async function requestShutdown() {
+    debug('requestShutdown');
+    shutdownRequested = true;
+    for (const callback of shutdownCallbacks) {
+        debug('callback', callback);
+        await callback();
+        // try { callback() } catch (e) { }
+    }
+    debug('requestShutdown done');
+}
+async function getMessages(qrl, opt, maxMessages) {
+    const params = {
+        AttributeNames: ['All'],
+        MaxNumberOfMessages: maxMessages,
+        MessageAttributeNames: ['All'],
+        QueueUrl: qrl,
+        VisibilityTimeout: 120,
+        WaitTimeSeconds: opt.waitTime
+    };
+    const response = await (0, sqs_js_1.getSQSClient)().send(new client_sqs_1.ReceiveMessageCommand(params));
+    // debug('ReceiveMessage response', response)
+    return response.Messages || [];
+}
+//
+// Consumer
+//
+async function processMessages(queues, callback, options) {
+    debug({ options });
+    const opt = (0, defaults_js_1.getOptionsWithDefaults)(options);
+    debug('processMessages', { queues, callback, options, opt, argv: process.argv });
+    let lastLatency = 10;
+    const systemMonitor = new systemMonitor_js_1.SystemMonitor(latency => {
+        const percentDifference = 100 * Math.abs(lastLatency - latency) / lastLatency;
+        if (percentDifference > 10 && opt.verbose) {
+            console.error(chalk_1.default.blue('Latency:', Math.round(latency), 'ms'));
+        }
+        lastLatency = latency;
+    });
+    const jobExecutor = new jobExecutor_js_1.JobExecutor(opt);
+    const queueManager = new queueManager_js_1.QueueManager(opt, queues, 10);
+    const cores = (0, os_1.cpus)().length;
+    // debug({ systemMonitor, jobExecutor, queueManager })
+    // This delay function keeps a timeout reference around so it can be
+    // cancelled at shutdown
+    let delayTimeout;
+    const delay = (ms) => new Promise(resolve => {
+        delayTimeout = setTimeout(resolve, ms);
+    });
+    shutdownCallbacks.push(async () => {
+        clearTimeout(delayTimeout);
+        await queueManager.shutdown();
+        debug({ queueManager: 'done' });
+        await jobExecutor.shutdown();
+        debug({ jobExecutor: 'done' });
+        await systemMonitor.shutdown();
+        debug({ systemMonitor: 'done' });
+    });
+    // Keep track of how many messages could be returned from each queue
+    const activeQrls = new Map();
+    const listeningQrls = new Set();
+    let maxReturnCount = 0;
+    const listen = async (qname, qrl, maxMessages) => {
+        if (opt.verbose) {
+            console.error(chalk_1.default.blue('Listening on: '), qname);
+        }
+        maxReturnCount += maxMessages;
+        try {
+            listeningQrls.add(qrl);
+            const messages = await getMessages(qrl, opt, maxMessages);
+            listeningQrls.delete(qrl);
+            if (!shutdownRequested) {
+                if (messages.length) {
+                    activeQrls.set(qrl, (activeQrls.get(qrl) || 0) + 1);
+                    await jobExecutor.executeJobs(messages, callback, qname, qrl);
+                    const count = activeQrls.get(qrl) - 1;
+                    if (count)
+                        activeQrls.set(qrl, count);
+                    else
+                        activeQrls.delete(qrl);
+                    queueManager.updateIcehouse(qrl, false);
+                }
+                else {
+                    // If we didn't get any, update the icehouse so we can back off
+                    queueManager.updateIcehouse(qrl, true);
+                }
+            }
+            // Max job accounting
+            maxReturnCount -= maxMessages;
+        }
+        catch (e) {
+            // If the queue has been cleaned up, we should back off anyway
+            if (e instanceof client_sqs_1.QueueDoesNotExist) {
+                queueManager.updateIcehouse(qrl, true);
+            }
+            else {
+                throw e;
+            }
+        }
+    };
+    if (opt.verbose) {
+        function printUrls() {
+            console.error({ activeQrls, listeningQrls });
+            if (!shutdownRequested)
+                setTimeout(printUrls, 2000);
+        }
+        printUrls();
+    }
+    while (!shutdownRequested) { // eslint-disable-line
+        // Figure out how we are running
+        const runningJobs = jobExecutor.runningJobCount();
+        const allowedJobs = Math.max(0, opt.maxConcurrentJobs - maxReturnCount - runningJobs);
+        // Latency
+        const maxLatency = 100;
+        const latency = systemMonitor.getLatency() || 10;
+        const latencyFactor = 1 - Math.abs(Math.min(latency / maxLatency, 1)); // 0 if latency is at max, 1 if latency 0
+        // Memory
+        const freeMemory = (0, os_1.freemem)();
+        const totalMemory = (0, os_1.totalmem)();
+        const memoryThreshold = totalMemory * opt.maxMemoryPercent / 100;
+        const freememThreshold = totalMemory - memoryThreshold;
+        const remainingMemory = Math.max(0, freeMemory - freememThreshold);
+        const freememFactor = Math.min(1, Math.max(0, remainingMemory / memoryThreshold));
+        // Load
+        const oneMinuteLoad = systemMonitor.getLoad();
+        const loadPerCore = oneMinuteLoad / cores;
+        const loadFactor = 1 - Math.min(1, Math.max(0, loadPerCore / 3));
+        const overallFactor = Math.min(latencyFactor, freememFactor, loadFactor);
+        const targetJobs = Math.round(allowedJobs * overallFactor);
+        let jobsLeft = targetJobs;
+        if (opt.verbose) {
+            console.error({ maxConcurrentJobs: opt.maxConcurrentJobs, maxReturnCount, runningJobs, allowedJobs, maxLatency, latencyFactor, freememFactor, loadFactor, overallFactor, targetJobs });
+        }
+        for (const { qname, qrl } of queueManager.getPairs()) {
+            // const qcount = jobExecutor.runningJobCountForQueue(qname)
+            // console.log({ evaluating: { qname, qrl, qcount, jobsLeft, activeQrlsHasQrl: activeQrls.has(qrl) } })
+            if (jobsLeft <= 0)
+                break;
+            if (listeningQrls.has(qrl))
+                continue;
+            const maxMessages = Math.min(10, jobsLeft);
+            listen(qname, qrl, maxMessages);
+            jobsLeft -= maxMessages;
+            // debug({ listenedTo: { qname, maxMessages, jobsLeft } })
+        }
+        await delay(300);
+    }
+    debug('after all');
+}

package/commonjs/src/dedup.js

@@ -0,0 +1,265 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.getDeduplicationId = getDeduplicationId;
+exports.getCacheKey = getCacheKey;
+exports.addDedupParamsToMessage = addDedupParamsToMessage;
+exports.updateStats = updateStats;
+exports.statMaintenance = statMaintenance;
+exports.dedupShouldEnqueue = dedupShouldEnqueue;
+exports.dedupShouldEnqueueMulti = dedupShouldEnqueueMulti;
+exports.dedupSuccessfullyProcessed = dedupSuccessfullyProcessed;
+exports.dedupSuccessfullyProcessedMulti = dedupSuccessfullyProcessedMulti;
+const crypto_1 = require("crypto");
+const uuid_1 = require("uuid");
+const cache_js_1 = require("./cache.js");
+const debug_1 = __importDefault(require("debug"));
+const debug = (0, debug_1.default)('qdone:dedup');
+/**
+ * Returns a MessageDeduplicationId key appropriate for using with Amazon SQS
+ * for the given message. The passed dedupContent will be returned untouched
+ * if it meets all the requirements for SQS's MessageDeduplicationId,
+ * otherwise disallowed characters will be replaced by `_` and content longer
+ * than 128 characters will be truncated and a hash of the content appended.
+ * @param {String} dedupContent - Content used to construct the deduplication id.
+ * @param {Object} opt - Opt object from getOptionsWithDefaults()
+ * @returns {String} the cache key
+ */
+function getDeduplicationId(dedupContent, opt) {
+    debug({ getDeduplicationId: { dedupContent } });
+    // Don't transmit long keys to redis
+    dedupContent = dedupContent.trim().replace(/[^a-zA-Z0-9!"#$%&'()*+,-./:;<=>?@[\\\]^_`{|}~]/g, '_');
+    const max = 128;
+    const sep = 'sha1::body:';
+    const hash = (0, crypto_1.createHash)('sha1').update(dedupContent).digest('hex');
+    const truncated = dedupContent.slice(0, max - sep.length - 42);
+    const id = `sha1:{${hash}}:body:${truncated}`;
+    return id;
+}
+/**
+ * Returns the cache key given a deduplication id.
+ * @param {String} dedupId - a deduplication id returned from getDeduplicationId
+ * @param opt - Opt object from getOptionsWithDefaults()
+ * @returns the cache key
+ */
+function getCacheKey(dedupId, opt) {
+    const cacheKey = opt.cachePrefix + 'dedup:' + dedupId;
+    debug({ getCacheKey: { cacheKey } });
+    return cacheKey;
+}
+/**
+ * Modifies a message (parameters to SendMessageCommand) to add the parameters
+ * for whatever deduplication options the caller has set.
+ * @param {String} message - parameters to SendMessageCommand
+ * @param {Object} opt - Opt object from getOptionsWithDefaults()
+ * @param {Object} [messageOptions] - optional per message options. We only care about the key deduplicationId.
+ * @returns {Object} the modified parameters/message object
+ */
+function addDedupParamsToMessage(message, opt, messageOptions) {
+    // Either of these means we need to calculate an id
+    if (opt.fifo || opt.externalDedup) {
+        const uuidFunction = opt.uuidFunction || uuid_1.v1;
+        if (opt.deduplicationId)
+            message.MessageDeduplicationId = opt.deduplicationId;
+        if (opt.dedupIdPerMessage)
+            message.MessageDeduplicationId = uuidFunction();
+        if (messageOptions?.deduplicationId)
+            message.MessageDeduplicationId = messageOptions.deduplicationId;
+        // Fallback to using the message body
+        if (!message.MessageDeduplicationId) {
+            message.MessageDeduplicationId = getDeduplicationId(message.MessageBody, opt);
+        }
+        else {
+            // Transform the incoming ID so that it fits SPS spec and is suitable for Redis
+            message.MessageDeduplicationId = getDeduplicationId(message.MessageDeduplicationId, opt);
+        }
+        // Track our own dedup id so we can look it up upon ReceiveMessage
+        if (opt.externalDedup) {
+            message.MessageAttributes = {
+                QdoneDeduplicationId: {
+                    StringValue: message.MessageDeduplicationId,
+                    DataType: 'String'
+                }
+            };
+            // If we are using our own dedup, then we must disable the SQS dedup by
+            // providing a different unique ID. Otherwise SQS will interact with us.
+            if (opt.fifo)
+                message.MessageDeduplicationId = uuidFunction();
+        }
+        // Non fifo can't have this parameter
+        if (!opt.fifo)
+            delete message.MessageDeduplicationId;
+    }
+    return message;
+}
+/**
+ * Updates statistics in redis, of which there are two:
+ * 1. duplicateSet - a set who's members are cache keys and scores are the number of duplicate
+ *                   runs prevented by dedup.
+ * 2. expirationSet - a set who's members are cache keys and scores are when the cache key expires
+ * @param {String} cacheKey
+ * @param {Number} duplicates - the number of duplicates, must be at least 1 to gather stats
+ * @param {Number} expireAt - timestamp for when this key's dedupPeriod expires
+ * @param {Object} opt - Opt object from getOptionsWithDefaults()
+ * @param {Object} pipeline - (Optional) redis pipeline you will exec() yourself
+ */
+async function updateStats(cacheKey, duplicates, expireAt, opt, pipeline) {
+    if (duplicates >= 1) {
+        const duplicateSet = opt.cachePrefix + 'dedup-stats:duplicateSet';
+        const expirationSet = opt.cachePrefix + 'dedup-stats:expirationSet';
+        const hadPipeline = !!pipeline;
+        if (!hadPipeline)
+            pipeline = (0, cache_js_1.getCacheClient)(opt).multi();
+        pipeline.zadd(duplicateSet, 'GT', duplicates, cacheKey);
+        pipeline.zadd(expirationSet, 'GT', expireAt, cacheKey);
+        if (!hadPipeline)
+            await pipeline.exec();
+    }
+}
+/**
+ * Removes expired items from stats.
+ */
+async function statMaintenance(opt) {
+    const duplicateSet = opt.cachePrefix + 'dedup-stats:duplicateSet';
+    const expirationSet = opt.cachePrefix + 'dedup-stats:expirationSet';
+    const client = (0, cache_js_1.getCacheClient)(opt);
+    const now = new Date().getTime();
+    // Grab a batch of expired keys
+    debug({ statMaintenance: { aboutToGo: true, expirationSet } });
+    const expiredStats = await client.zrange(expirationSet, '-inf', now, 'BYSCORE');
+    debug({ statMaintenance: { expiredStats } });
+    // And remove them from indexes, main storage
+    if (expiredStats.length) {
+        const result = await client.multi()
+            .zrem(expirationSet, expiredStats)
+            .zrem(duplicateSet, expiredStats)
+            .exec();
+        debug({ statMaintenance: { result } });
+    }
+}
+/**
+ * Determines whether we should enqueue this message or whether it is a duplicate.
+ * Returns true if enqueuing the message would not result in a duplicate.
+ * @param {Object} message - Parameters to SendMessageCommand
+ * @param {Object} opt - Opt object from getOptionsWithDefaults()
+ * @returns {Boolean} true if the message can be enqueued without duplicate, else false
+ */
+async function dedupShouldEnqueue(message, opt) {
+    const client = (0, cache_js_1.getCacheClient)(opt);
+    const dedupId = message?.MessageAttributes?.QdoneDeduplicationId?.StringValue;
+    const cacheKey = getCacheKey(dedupId, opt);
+    const expireAt = new Date().getTime() + opt.dedupPeriod;
+    const copies = await client.incr(cacheKey);
+    debug({ action: 'shouldEnqueue', cacheKey, copies });
+    if (copies === 1) {
+        await client.expireat(cacheKey, expireAt);
+        return true;
+    }
+    if (opt.dedupStats) {
+        const duplicates = copies - 1;
+        await updateStats(cacheKey, duplicates, expireAt, opt);
+    }
+    return false;
+}
+/**
+ * Determines which messages we should enqueue, returning only those that
+ * would not be duplicates.
+ * @param {Array[Object]} messages - Entries array for the SendMessageBatchCommand
+ * @param {Object} opt - Opt object from getOptionsWithDefaults()
+ * @returns {Array[Object]} an array of messages that can be safely enqueued. Could be empty.
+ */
+async function dedupShouldEnqueueMulti(messages, opt) {
+    debug({ dedupShouldEnqueueMulti: { messages, opt } });
+    const expireAt = new Date().getTime() + opt.dedupPeriod;
+    // Increment all
+    const incrPipeline = (0, cache_js_1.getCacheClient)(opt).pipeline();
+    for (const message of messages) {
+        const dedupId = message?.MessageAttributes?.QdoneDeduplicationId?.StringValue;
+        const cacheKey = getCacheKey(dedupId, opt);
+        incrPipeline.incr(cacheKey);
+    }
+    const responses = await incrPipeline.exec();
+    debug({ dedupShouldEnqueueMulti: { messages, responses } });
+    // Interpret responses and expire keys for races we won
+    const expirePipeline = (0, cache_js_1.getCacheClient)(opt).pipeline();
+    const statsPipeline = opt.dedupStats ? (0, cache_js_1.getCacheClient)(opt).pipeline() : undefined;
+    const messagesToEnqueue = [];
+    for (let i = 0; i < messages.length; i++) {
+        const message = messages[i];
+        const [, copies] = responses[i];
+        const dedupId = message?.MessageAttributes?.QdoneDeduplicationId?.StringValue;
+        const cacheKey = getCacheKey(dedupId, opt);
+        if (copies === 1) {
+            messagesToEnqueue.push(message);
+            expirePipeline.expireat(cacheKey, expireAt);
+        }
+        else if (opt.dedupStats) {
+            const duplicates = copies - 1;
+            updateStats(cacheKey, duplicates, expireAt, opt, statsPipeline);
+        }
+    }
+    await expirePipeline.exec();
+    if (opt.dedupStats)
+        await statsPipeline.exec();
+    return messagesToEnqueue;
+}
+/**
+ * Marks a message as processed so that subsequent calls to dedupShouldEnqueue
+ * and dedupShouldEnqueueMulti will allow a message to be enqueued again
+ * without waiting for dedupPeriod to expire.
+ * @param {Object} message - Return value from RecieveMessageCommand
+ * @param {Object} opt - Opt object from getOptionsWithDefaults()
+ * @returns {Number} 1 if a cache key was deleted, otherwise 0
+ */
+async function dedupSuccessfullyProcessed(message, opt) {
+    debug({ dedupSuccessfullyProcessed: { message, opt } });
+    const client = (0, cache_js_1.getCacheClient)(opt);
+    const dedupId = message?.MessageAttributes?.QdoneDeduplicationId?.StringValue;
+    if (dedupId) {
+        const cacheKey = getCacheKey(dedupId, opt);
+        const count = await client.del(cacheKey);
+        // Probabalistic stat maintenance
+        if (opt.dedupStats) {
+            const chance = 1 / 100.0;
+            if (Math.random() < chance)
+                await statMaintenance(opt);
+        }
+        return count;
+    }
+    return 0;
+}
+/**
+ * Marks an array of messages as processed so that subsequent calls to
+ * dedupShouldEnqueue and dedupShouldEnqueueMulti will allow a message to be
+ * enqueued again without waiting for dedupPeriod to expire.
+ * @param {Array[Object]} messages - Return values from RecieveMessageCommand
+ * @param {Object} opt - Opt object from getOptionsWithDefaults()
+ * @returns {Number} number of deleted keys
+ */
+async function dedupSuccessfullyProcessedMulti(messages, opt) {
+    debug({ messages, dedupSuccessfullyProcessedMulti: { messages, opt } });
+    const cacheKeys = [];
+    for (const message of messages) {
+        const dedupId = message?.MessageAttributes?.QdoneDeduplicationId?.StringValue;
+        if (dedupId) {
+            const cacheKey = getCacheKey(dedupId, opt);
+            cacheKeys.push(cacheKey);
+        }
+    }
+    debug({ dedupSuccessfullyProcessedMulti: { cacheKeys } });
+    if (cacheKeys.length) {
+        const numDeleted = await (0, cache_js_1.getCacheClient)(opt).del(cacheKeys);
+        // const numDeleted = results.map(([, val]) => val).reduce((a, b) => a + b, 0)
+        debug({ dedupSuccessfullyProcessedMulti: { cacheKeys, numDeleted } });
+        // Probabalistic stat maintenance
+        if (opt.dedupStats) {
+            const chance = numDeleted / 100.0;
+            if (Math.random() < chance)
+                await statMaintenance(opt);
+        }
+        return numDeleted;
+    }
+    return 0;
+}