qdone 2.0.35-alpha → 2.0.37-alpha

package/commonjs/src/cache.js

@@ -16,17 +16,18 @@ let client;
  * how to connect.
  */
 function getCacheClient(opt) {
+    const RedisClass = opt.Redis || ioredis_1.default;
     if (client) {
         return client;
     }
     else if (opt.cacheUri) {
         const url = new url_1.URL(opt.cacheUri);
         if (url.protocol === 'redis:') {
-            client = new ioredis_1.default(url.toString());
+            client = new RedisClass(url.toString());
         }
         else if (url.protocol === 'redis-cluster:') {
             url.protocol = 'redis:';
-            client = new ioredis_1.default.Cluster([url.toString()], { slotsRefreshInterval: 60 * 1000 });
+            client = new RedisClass.Cluster([url.toString()], { slotsRefreshInterval: 60 * 1000 });
         }
         else {
             throw new UsageError(`Only redis:// or redis-cluster:// URLs are currently supported. Got: ${url.protocol}`);

package/commonjs/src/dedup.js

@@ -0,0 +1,261 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.dedupSuccessfullyProcessedMulti = exports.dedupSuccessfullyProcessed = exports.dedupShouldEnqueueMulti = exports.dedupShouldEnqueue = exports.statMaintenance = exports.updateStats = exports.addDedupParamsToMessage = exports.getCacheKey = exports.getDeduplicationId = void 0;
+const crypto_1 = require("crypto");
+const uuid_1 = require("uuid");
+const cache_js_1 = require("./cache.js");
+const debug_1 = __importDefault(require("debug"));
+const debug = (0, debug_1.default)('qdone:dedup');
+/**
+ * Returns a MessageDeduplicationId key appropriate for using with Amazon SQS
+ * for the given message. The passed dedupContent will be returned untouched
+ * if it meets all the requirements for SQS's MessageDeduplicationId,
+ * otherwise disallowed characters will be replaced by `_` and content longer
+ * than 128 characters will be truncated and a hash of the content appended.
+ * @param {String} dedupContent - Content used to construct the deduplication id.
+ * @param {Object} opt - Opt object from getOptionsWithDefaults()
+ * @returns {String} the cache key
+ */
+function getDeduplicationId(dedupContent, opt) {
+    debug({ getDeduplicationId: { dedupContent } });
+    // Don't transmit long keys to redis
+    dedupContent = dedupContent.trim().replace(/[^a-zA-Z0-9!"#$%&'()*+,-./:;<=>?@[\\\]^_`{|}~]/g, '_');
+    const max = 128;
+    const sep = '...sha1:';
+    if (dedupContent.length > max) {
+        dedupContent = dedupContent.slice(0, max - sep.length - 40) + '...sha1:' + (0, crypto_1.createHash)('sha1').update(dedupContent).digest('hex');
+    }
+    return dedupContent;
+}
+exports.getDeduplicationId = getDeduplicationId;
+/**
+ * Returns the cache key given a deduplication id.
+ * @param {String} dedupId - a deduplication id returned from getDeduplicationId
+ * @param opt - Opt object from getOptionsWithDefaults()
+ * @returns the cache key
+ */
+function getCacheKey(dedupId, opt) {
+    const cacheKey = opt.cachePrefix + 'dedup:' + dedupId;
+    debug({ getCacheKey: { cacheKey } });
+    return cacheKey;
+}
+exports.getCacheKey = getCacheKey;
+/**
+ * Modifies a message (parameters to SendMessageCommand) to add the parameters
+ * for whatever deduplication options the caller has set.
+ * @param {String} message - parameters to SendMessageCommand
+ * @param {Object} opt - Opt object from getOptionsWithDefaults()
+ * @param {Object} [messageOptions] - optional per message options. We only care about the key deduplicationId.
+ * @returns {Object} the modified parameters/message object
+ */
+function addDedupParamsToMessage(message, opt, messageOptions) {
+    // Either of these means we need to calculate an id
+    if (opt.fifo || opt.externalDedup) {
+        const uuidFunction = opt.uuidFunction || uuid_1.v1;
+        if (opt.deduplicationId)
+            message.MessageDeduplicationId = opt.deduplicationId;
+        if (opt.dedupIdPerMessage)
+            message.MessageDeduplicationId = uuidFunction();
+        if (messageOptions?.deduplicationId)
+            message.MessageDeduplicationId = messageOptions.deduplicationId;
+        // Fallback to using the message body
+        if (!message.MessageDeduplicationId) {
+            message.MessageDeduplicationId = getDeduplicationId(message.MessageBody, opt);
+        }
+        // Track our own dedup id so we can look it up upon ReceiveMessage
+        if (opt.externalDedup) {
+            message.MessageAttributes = {
+                QdoneDeduplicationId: {
+                    StringValue: message.MessageDeduplicationId,
+                    DataType: 'String'
+                }
+            };
+            // If we are using our own dedup, then we must disable the SQS dedup by
+            // providing a different unique ID. Otherwise SQS will interact with us.
+            if (opt.fifo)
+                message.MessageDeduplicationId = uuidFunction();
+        }
+        // Non fifo can't have this parameter
+        if (!opt.fifo)
+            delete message.MessageDeduplicationId;
+    }
+    return message;
+}
+exports.addDedupParamsToMessage = addDedupParamsToMessage;
+/**
+ * Updates statistics in redis, of which there are two:
+ * 1. duplicateSet - a set who's members are cache keys and scores are the number of duplicate
+ *                   runs prevented by dedup.
+ * 2. expirationSet - a set who's members are cache keys and scores are when the cache key expires
+ * @param {String} cacheKey
+ * @param {Number} duplicates - the number of duplicates, must be at least 1 to gather stats
+ * @param {Number} expireAt - timestamp for when this key's dedupPeriod expires
+ * @param {Object} opt - Opt object from getOptionsWithDefaults()
+ * @param {Object} pipeline - (Optional) redis pipeline you will exec() yourself
+ */
+async function updateStats(cacheKey, duplicates, expireAt, opt, pipeline) {
+    if (duplicates >= 1) {
+        const duplicateSet = opt.cachePrefix + 'dedup-stats:duplicateSet';
+        const expirationSet = opt.cachePrefix + 'dedup-stats:expirationSet';
+        const hadPipeline = !!pipeline;
+        if (!hadPipeline)
+            pipeline = (0, cache_js_1.getCacheClient)(opt).multi();
+        pipeline.zadd(duplicateSet, 'GT', duplicates, cacheKey);
+        pipeline.zadd(expirationSet, 'GT', expireAt, cacheKey);
+        if (!hadPipeline)
+            await pipeline.exec();
+    }
+}
+exports.updateStats = updateStats;
+/**
+ * Removes expired items from stats.
+ */
+async function statMaintenance(opt) {
+    const duplicateSet = opt.cachePrefix + 'dedup-stats:duplicateSet';
+    const expirationSet = opt.cachePrefix + 'dedup-stats:expirationSet';
+    const client = (0, cache_js_1.getCacheClient)(opt);
+    const now = new Date().getTime();
+    // Grab a batch of expired keys
+    debug({ statMaintenance: { aboutToGo: true, expirationSet } });
+    const expiredStats = await client.zrange(expirationSet, '-inf', now, 'BYSCORE');
+    debug({ statMaintenance: { expiredStats } });
+    // And remove them from indexes, main storage
+    if (expiredStats.length) {
+        const result = await client.multi()
+            .zrem(expirationSet, expiredStats)
+            .zrem(duplicateSet, expiredStats)
+            .exec();
+        debug({ statMaintenance: { result } });
+    }
+}
+exports.statMaintenance = statMaintenance;
+/**
+ * Determines whether we should enqueue this message or whether it is a duplicate.
+ * Returns true if enqueuing the message would not result in a duplicate.
+ * @param {Object} message - Parameters to SendMessageCommand
+ * @param {Object} opt - Opt object from getOptionsWithDefaults()
+ * @returns {Boolean} true if the message can be enqueued without duplicate, else false
+ */
+async function dedupShouldEnqueue(message, opt) {
+    const client = (0, cache_js_1.getCacheClient)(opt);
+    const dedupId = message?.MessageAttributes?.QdoneDeduplicationId?.StringValue;
+    const cacheKey = getCacheKey(dedupId, opt);
+    const expireAt = new Date().getTime() + opt.dedupPeriod;
+    const copies = await client.incr(cacheKey);
+    debug({ action: 'shouldEnqueue', cacheKey, copies });
+    if (copies === 1) {
+        await client.expireat(cacheKey, expireAt);
+        return true;
+    }
+    if (opt.dedupStats) {
+        const duplicates = copies - 1;
+        await updateStats(cacheKey, duplicates, expireAt, opt);
+    }
+    return false;
+}
+exports.dedupShouldEnqueue = dedupShouldEnqueue;
+/**
+ * Determines which messages we should enqueue, returning only those that
+ * would not be duplicates.
+ * @param {Array[Object]} messages - Entries array for the SendMessageBatchCommand
+ * @param {Object} opt - Opt object from getOptionsWithDefaults()
+ * @returns {Array[Object]} an array of messages that can be safely enqueued. Could be empty.
+ */
+async function dedupShouldEnqueueMulti(messages, opt) {
+    debug({ dedupShouldEnqueueMulti: { messages, opt } });
+    const expireAt = new Date().getTime() + opt.dedupPeriod;
+    // Increment all
+    const incrPipeline = (0, cache_js_1.getCacheClient)(opt).pipeline();
+    for (const message of messages) {
+        const dedupId = message?.MessageAttributes?.QdoneDeduplicationId?.StringValue;
+        const cacheKey = getCacheKey(dedupId, opt);
+        incrPipeline.incr(cacheKey);
+    }
+    const responses = await incrPipeline.exec();
+    debug({ dedupShouldEnqueueMulti: { messages, responses } });
+    // Interpret responses and expire keys for races we won
+    const expirePipeline = (0, cache_js_1.getCacheClient)(opt).pipeline();
+    const statsPipeline = opt.dedupStats ? (0, cache_js_1.getCacheClient)(opt).pipeline() : undefined;
+    const messagesToEnqueue = [];
+    for (let i = 0; i < messages.length; i++) {
+        const message = messages[i];
+        const [, copies] = responses[i];
+        const dedupId = message?.MessageAttributes?.QdoneDeduplicationId?.StringValue;
+        const cacheKey = getCacheKey(dedupId, opt);
+        if (copies === 1) {
+            messagesToEnqueue.push(message);
+            expirePipeline.expireat(cacheKey, expireAt);
+        }
+        else if (opt.dedupStats) {
+            const duplicates = copies - 1;
+            updateStats(cacheKey, duplicates, expireAt, opt, statsPipeline);
+        }
+    }
+    await expirePipeline.exec();
+    if (opt.dedupStats)
+        await statsPipeline.exec();
+    return messagesToEnqueue;
+}
+exports.dedupShouldEnqueueMulti = dedupShouldEnqueueMulti;
+/**
+ * Marks a message as processed so that subsequent calls to dedupShouldEnqueue
+ * and dedupShouldEnqueueMulti will allow a message to be enqueued again
+ * without waiting for dedupPeriod to expire.
+ * @param {Object} message - Return value from RecieveMessageCommand
+ * @param {Object} opt - Opt object from getOptionsWithDefaults()
+ * @returns {Number} 1 if a cache key was deleted, otherwise 0
+ */
+async function dedupSuccessfullyProcessed(message, opt) {
+    const client = (0, cache_js_1.getCacheClient)(opt);
+    const dedupId = message?.MessageAttributes?.QdoneDeduplicationId?.StringValue;
+    if (dedupId) {
+        const cacheKey = getCacheKey(dedupId, opt);
+        const count = await client.del(cacheKey);
+        // Probabalistic stat maintenance
+        if (opt.dedupStats) {
+            const chance = 1 / 100.0;
+            if (Math.random() < chance)
+                await statMaintenance(opt);
+        }
+        return count;
+    }
+    return 0;
+}
+exports.dedupSuccessfullyProcessed = dedupSuccessfullyProcessed;
+/**
+ * Marks an array of messages as processed so that subsequent calls to
+ * dedupShouldEnqueue and dedupShouldEnqueueMulti will allow a message to be
+ * enqueued again without waiting for dedupPeriod to expire.
+ * @param {Array[Object]} messages - Return values from RecieveMessageCommand
+ * @param {Object} opt - Opt object from getOptionsWithDefaults()
+ * @returns {Number} number of deleted keys
+ */
+async function dedupSuccessfullyProcessedMulti(messages, opt) {
+    debug({ messages, dedupSuccessfullyProcessedMulti: { messages, opt } });
+    const cacheKeys = [];
+    for (const message of messages) {
+        const dedupId = message?.MessageAttributes?.QdoneDeduplicationId?.StringValue;
+        if (dedupId) {
+            const cacheKey = getCacheKey(dedupId, opt);
+            cacheKeys.push(cacheKey);
+        }
+    }
+    debug({ dedupSuccessfullyProcessedMulti: { cacheKeys } });
+    if (cacheKeys.length) {
+        const numDeleted = await (0, cache_js_1.getCacheClient)(opt).del(cacheKeys);
+        // const numDeleted = results.map(([, val]) => val).reduce((a, b) => a + b, 0)
+        debug({ dedupSuccessfullyProcessedMulti: { cacheKeys, numDeleted } });
+        // Probabalistic stat maintenance
+        if (opt.dedupStats) {
+            const chance = numDeleted / 100.0;
+            if (Math.random() < chance)
+                await statMaintenance(opt);
+        }
+        return numDeleted;
+    }
+    return 0;
+}
+exports.dedupSuccessfullyProcessedMulti = dedupSuccessfullyProcessedMulti;

package/commonjs/src/defaults.js

@@ -1,6 +1,6 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.setupVerbose = exports.setupAWS = exports.getOptionsWithDefaults = exports.defaults = void 0;
+exports.setupVerbose = exports.setupAWS = exports.getOptionsWithDefaults = exports.validateMessageOptions = exports.defaults = void 0;
 /**
  * Default options for qdone. Accepts a command line options object and
  * returns nicely-named options.
@@ -21,10 +21,14 @@ exports.defaults = Object.freeze({
     disableLog: false,
     includeFailed: false,
     includeDead: false,
+    externalDedup: false,
+    dedupPeriod: 60 * 5,
+    dedupStats: false,
     // Enqueue
     groupId: (0, uuid_1.v1)(),
     groupIdPerMessage: false,
     deduplicationId: undefined,
+    dedupIdPerMessage: false,
     messageRetentionPeriod: 1209600,
     delay: 0,
     sendRetries: 6,
@@ -42,7 +46,9 @@ exports.defaults = Object.freeze({
     // Idle Queues
     idleFor: 60,
     delete: false,
-    unpair: false
+    unpair: false,
+    // Check
+    create: false
 });
 function validateInteger(opt, name) {
     const parsed = parseInt(opt[name], 10);
@@ -50,6 +56,20 @@ function validateInteger(opt, name) {
         throw new Error(`${name} needs to be an integer.`);
     return parsed;
 }
+function validateMessageOptions(messageOptions) {
+    const validKeys = ['deduplicationId', 'groupId'];
+    if (typeof messageOptions === 'object' &&
+        !Array.isArray(messageOptions) &&
+        messageOptions !== null) {
+        for (const key in messageOptions) {
+            if (!validKeys.includes(key))
+                throw new Error(`Invalid message option ${key}`);
+        }
+        return messageOptions;
+    }
+    return {};
+}
+exports.validateMessageOptions = validateMessageOptions;
 /**
  * This function should be called by each exposed API entry point on the
  * options passed in from the caller. It supports options named in camelCase
@@ -63,7 +83,7 @@ function getOptionsWithDefaults(options) {
     if (!options)
         options = {};
     // Activate DLQ if any option is set
-    const dlq = options.dlq || !!(options['dlq-suffix'] || options['dlq-after'] || options['dlq-name']);
+    const dlq = options.dlq || !!(options['dlq-suffix'] || options['dlq-after'] || options['dlq-name'] || options.dlqSuffix || options.dlqAfter || options.dlqName);
     const opt = {
         // Shared
         prefix: options.prefix === '' ? options.prefix : (options.prefix || exports.defaults.prefix),
@@ -76,6 +96,9 @@ function getOptionsWithDefaults(options) {
         disableLog: options.disableLog || options['disable-log'] || exports.defaults.disableLog,
         includeFailed: options.includeFailed || options['include-failed'] || exports.defaults.includeFailed,
         includeDead: options.includeDead || options['include-dead'] || exports.defaults.includeDead,
+        externalDedup: options.externalDedup || options['external-dedup'] || exports.defaults.externalDedup,
+        dedupPeriod: options.dedupPeriod || options['dedup-period'] || exports.defaults.dedupPeriod,
+        dedupStats: options.dedupStats || options['dedup-stats'] || exports.defaults.dedupStats,
         // Cache
         cacheUri: options.cacheUri || options['cache-uri'] || exports.defaults.cacheUri,
         cachePrefix: options.cachePrefix || options['cache-prefix'] || exports.defaults.cachePrefix,
@@ -84,6 +107,7 @@ function getOptionsWithDefaults(options) {
         groupId: options.groupId || options['group-id'] || exports.defaults.groupId,
         groupIdPerMessage: false,
         deduplicationId: options.deduplicationId || options['deduplication-id'] || exports.defaults.deduplicationId,
+        dedupIdPerMessage: options.dedupIdPerMessage || options['dedup-id-per-message'] || exports.defaults.dedupIdPerMessage,
         messageRetentionPeriod: options.messageRetentionPeriod || options['message-retention-period'] || exports.defaults.messageRetentionPeriod,
         delay: options.delay || exports.defaults.delay,
         sendRetries: options['send-retries'] || exports.defaults.sendRetries,
@@ -102,7 +126,10 @@ function getOptionsWithDefaults(options) {
         // Idle Queues
         idleFor: options.idleFor || options['idle-for'] || exports.defaults.idleFor,
         delete: options.delete || exports.defaults.delete,
-        unpair: options.delete || exports.defaults.unpair
+        unpair: options.delete || exports.defaults.unpair,
+        // Check
+        create: options.create || exports.defaults.create,
+        overwrite: options.overwrite || exports.defaults.overwrite
     };
     // Setting this env here means we don't have to in AWS SDK constructors
     process.env.AWS_REGION = opt.region;
@@ -111,6 +138,7 @@ function getOptionsWithDefaults(options) {
     opt.messageRetentionPeriod = validateInteger(opt, 'messageRetentionPeriod');
     opt.delay = validateInteger(opt, 'delay');
     opt.sendRetries = validateInteger(opt, 'sendRetries');
+    opt.dedupPeriod = validateInteger(opt, 'dedupPeriod');
     opt.failDelay = validateInteger(opt, 'failDelay');
     opt.dlqAfter = validateInteger(opt, 'dlqAfter');
     opt.waitTime = validateInteger(opt, 'waitTime');
@@ -118,6 +146,13 @@ function getOptionsWithDefaults(options) {
     opt.maxConcurrentJobs = validateInteger(opt, 'maxConcurrentJobs');
     opt.maxMemoryPercent = validateInteger(opt, 'maxMemoryPercent');
     opt.idleFor = validateInteger(opt, 'idleFor');
+    // Validate dedup args
+    if (opt.externalDedup && !opt.cacheUri)
+        throw new Error('--external-dedup requires the --cache-uri argument');
+    if (opt.externalDedup && (!opt.dedupPeriod || opt.dedupPeriod < 1))
+        throw new Error('--external-dedup of redis requires a --dedup-period > 1 second');
+    if (opt.dedupIdPerMessage && opt.deduplicationId)
+        throw new Error('Use either --deduplication-id or --dedup-id-per-message but not both');
     return opt;
 }
 exports.getOptionsWithDefaults = getOptionsWithDefaults;