qdone 2.1.0 → 2.2.0

package/commonjs/src/enqueue.js

@@ -0,0 +1,520 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.getDLQParams = getDLQParams;
+exports.getOrCreateDLQ = getOrCreateDLQ;
+exports.getFailParams = getFailParams;
+exports.getOrCreateFailQueue = getOrCreateFailQueue;
+exports.getQueueParams = getQueueParams;
+exports.getOrCreateQueue = getOrCreateQueue;
+exports.getQueueAttributes = getQueueAttributes;
+exports.formatMessage = formatMessage;
+exports.sendMessage = sendMessage;
+exports.sendMessageBatch = sendMessageBatch;
+exports.flushMessages = flushMessages;
+exports.addMessage = addMessage;
+exports.enqueue = enqueue;
+exports.enqueueBatch = enqueueBatch;
+const node_1 = require("@sentry/node");
+const uuid_1 = require("uuid");
+const chalk_1 = __importDefault(require("chalk"));
+const debug_1 = __importDefault(require("debug"));
+const client_sqs_1 = require("@aws-sdk/client-sqs");
+const qrlCache_js_1 = require("./qrlCache.js");
+const sqs_js_1 = require("./sqs.js");
+const dedup_js_1 = require("./dedup.js");
+const defaults_js_1 = require("./defaults.js");
+const exponentialBackoff_js_1 = require("./exponentialBackoff.js");
+const debug = (0, debug_1.default)('qdone:enqueue');
+function getDLQParams(queue, opt) {
+    const dqname = (0, qrlCache_js_1.normalizeDLQName)(queue, opt);
+    const params = {
+        Attributes: { MessageRetentionPeriod: opt.messageRetentionPeriod + '' },
+        QueueName: dqname
+    };
+    if (opt.tags)
+        params.tags = opt.tags;
+    if (opt.fifo)
+        params.Attributes.FifoQueue = 'true';
+    return { dqname, params };
+}
+async function getOrCreateDLQ(queue, opt) {
+    debug('getOrCreateDLQ(', queue, ')');
+    const { dqname, params } = getDLQParams(queue, opt);
+    try {
+        const dqrl = await (0, qrlCache_js_1.qrlCacheGet)(dqname);
+        return dqrl;
+    }
+    catch (err) {
+        // Anything other than queue doesn't exist gets re-thrown
+        if (!(err instanceof client_sqs_1.QueueDoesNotExist))
+            throw err;
+        // Create our DLQ
+        const client = (0, sqs_js_1.getSQSClient)();
+        const cmd = new client_sqs_1.CreateQueueCommand(params);
+        if (opt.verbose)
+            console.error(chalk_1.default.blue('Creating dead letter queue ') + dqname);
+        const data = await client.send(cmd);
+        debug('createQueue returned', data);
+        const dqrl = data.QueueUrl;
+        (0, qrlCache_js_1.qrlCacheSet)(dqname, dqrl);
+        return dqrl;
+    }
+}
+/**
+ * Returns the parameters needed for creating a failed queue. If DLQ options
+ * are set, it makes an API call to get this DLQ's ARN.
+ */
+async function getFailParams(queue, opt) {
+    const fqname = (0, qrlCache_js_1.normalizeFailQueueName)(queue, opt);
+    const params = {
+        Attributes: { MessageRetentionPeriod: opt.messageRetentionPeriod + '' },
+        QueueName: fqname
+    };
+    // If we have a dlq, we grab it and set a redrive policy
+    if (opt.dlq) {
+        const dqname = (0, qrlCache_js_1.normalizeDLQName)(queue, opt);
+        const dqrl = await (0, qrlCache_js_1.qrlCacheGet)(dqname);
+        const dqa = await getQueueAttributes(dqrl);
+        debug('dqa', dqa);
+        params.Attributes.RedrivePolicy = JSON.stringify({
+            deadLetterTargetArn: dqa.Attributes.QueueArn,
+            maxReceiveCount: opt.dlqAfter
+        });
+    }
+    if (opt.failDelay)
+        params.Attributes.DelaySeconds = opt.failDelay + '';
+    if (opt.tags)
+        params.tags = opt.tags;
+    if (opt.fifo)
+        params.Attributes.FifoQueue = 'true';
+    return params;
+}
+/**
+ * Returns the qrl for the failed queue for the given queue. Creates the queue
+ * if it does not exist.
+ */
+async function getOrCreateFailQueue(queue, opt, doesNotExist) {
+    debug('getOrCreateFailQueue(', queue, ')');
+    const fqname = (0, qrlCache_js_1.normalizeFailQueueName)(queue, opt);
+    try {
+        // Bail early if the caller knew we didn't have a queue
+        if (doesNotExist)
+            throw new client_sqs_1.QueueDoesNotExist(fqname);
+        const fqrl = await (0, qrlCache_js_1.qrlCacheGet)(fqname);
+        return fqrl;
+    }
+    catch (err) {
+        // Anything other than queue doesn't exist gets re-thrown
+        if (!(err instanceof client_sqs_1.QueueDoesNotExist))
+            throw err;
+        // Grab params, creating DLQ if needed
+        let params;
+        try {
+            params = await getFailParams(queue, opt);
+        }
+        catch (e) {
+            // If DLQ doesn't exist, create it
+            if (!(opt.dlq && e instanceof client_sqs_1.QueueDoesNotExist))
+                throw e;
+            await getOrCreateDLQ(queue, opt);
+            params = await getFailParams(queue, opt);
+        }
+        // Create our fail queue
+        const client = (0, sqs_js_1.getSQSClient)();
+        const cmd = new client_sqs_1.CreateQueueCommand(params);
+        if (opt.verbose)
+            console.error(chalk_1.default.blue('Creating fail queue ') + fqname);
+        const data = await client.send(cmd);
+        debug('createQueue returned', data);
+        const fqrl = data.QueueUrl;
+        (0, qrlCache_js_1.qrlCacheSet)(fqname, fqrl);
+        return fqrl;
+    }
+}
+/**
+ * Returns the parameters needed for creating a queue. If fail options
+ * are set, it makes an API call to get the fail queue's ARN.
+ */
+async function getQueueParams(queue, opt) {
+    const qname = (0, qrlCache_js_1.normalizeQueueName)(queue, opt);
+    const fqname = (0, qrlCache_js_1.normalizeFailQueueName)(queue, opt);
+    const fqrl = await (0, qrlCache_js_1.qrlCacheGet)(fqname, opt);
+    const fqa = await getQueueAttributes(fqrl);
+    const params = {
+        Attributes: {
+            MessageRetentionPeriod: opt.messageRetentionPeriod + '',
+            RedrivePolicy: JSON.stringify({
+                deadLetterTargetArn: fqa.Attributes.QueueArn,
+                maxReceiveCount: 1
+            })
+        },
+        QueueName: qname
+    };
+    if (opt.tags)
+        params.tags = opt.tags;
+    if (opt.fifo)
+        params.Attributes.FifoQueue = 'true';
+    return params;
+}
+/**
+ * Returns a qrl for a queue that either exists or does not
+ */
+async function getOrCreateQueue(queue, opt) {
+    debug('getOrCreateQueue(', queue, ')');
+    const qname = (0, qrlCache_js_1.normalizeQueueName)(queue, opt);
+    try {
+        const qrl = await (0, qrlCache_js_1.qrlCacheGet)(qname);
+        return qrl;
+    }
+    catch (err) {
+        // Anything other than queue doesn't exist gets re-thrown
+        if (!(err instanceof client_sqs_1.QueueDoesNotExist))
+            throw err;
+        // Grab params, creating fail queue if needed
+        let params;
+        try {
+            params = await getQueueParams(qname, opt);
+        }
+        catch (e) {
+            // If fail queue doesn't exist, create it
+            if (!(e instanceof client_sqs_1.QueueDoesNotExist))
+                throw e;
+            await getOrCreateFailQueue(qname, opt, true);
+            params = await getQueueParams(qname, opt);
+        }
+        debug({ getOrCreateQueue: { qname, params } });
+        // Create our queue
+        const client = (0, sqs_js_1.getSQSClient)();
+        const cmd = new client_sqs_1.CreateQueueCommand(params);
+        if (opt.verbose)
+            console.error(chalk_1.default.blue('Creating fail queue ') + qname);
+        const data = await client.send(cmd);
+        debug('AWS createQueue returned', data);
+        const qrl = data.QueueUrl;
+        (0, qrlCache_js_1.qrlCacheSet)(qname, qrl);
+        return qrl;
+    }
+}
+async function getQueueAttributes(qrl) {
+    debug('getQueueAttributes(', qrl, ')');
+    const client = (0, sqs_js_1.getSQSClient)();
+    const params = { AttributeNames: ['All'], QueueUrl: qrl };
+    const cmd = new client_sqs_1.GetQueueAttributesCommand(params);
+    // debug({ cmd })
+    const data = await client.send(cmd);
+    debug('GetQueueAttributes returned', data);
+    return data;
+}
+function formatMessage(body, id, opt, messageOptions) {
+    const message = { MessageBody: body };
+    if (typeof id !== 'undefined')
+        message.Id = '' + id;
+    if (opt.fifo) {
+        message.MessageGroupId = messageOptions?.groupId || opt?.groupId;
+    }
+    (0, dedup_js_1.addDedupParamsToMessage)(message, opt, messageOptions);
+    if (opt.delay)
+        message.DelaySeconds = opt.delay;
+    if (messageOptions?.delay)
+        message.DelaySeconds = messageOptions.delay;
+    return message;
+}
+// Retry happens within the context of the send functions
+const retryableExceptions = [
+    client_sqs_1.RequestThrottled,
+    client_sqs_1.KmsThrottled,
+    client_sqs_1.QueueDoesNotExist // Queue could temporarily not exist due to eventual consistency, let it retry
+];
+async function sendMessage(qrl, queue, command, opt, messageOptions) {
+    debug('sendMessage(', qrl, command, ')');
+    const uuidFunction = opt.uuidFunction || uuid_1.v1;
+    const params = {
+        QueueUrl: qrl,
+        ...formatMessage(command, null, opt, messageOptions)
+    };
+    // See if we even have to send it
+    if (opt.externalDedup) {
+        const shouldEnqueue = await (0, dedup_js_1.dedupShouldEnqueue)(params, opt);
+        if (!shouldEnqueue)
+            return { MessageId: uuidFunction() };
+    }
+    // Send it
+    const client = (0, sqs_js_1.getSQSClient)();
+    let cmd = new client_sqs_1.SendMessageCommand(params);
+    debug({ cmd });
+    const backoff = new exponentialBackoff_js_1.ExponentialBackoff(opt.sendRetries);
+    const send = async (attemptNumber) => {
+        cmd.input.attemptNumber = attemptNumber;
+        const data = await client.send(cmd);
+        debug('sendMessage returned', data);
+        return data;
+    };
+    const shouldRetry = async (result, error) => {
+        if (!error)
+            return false;
+        if (error instanceof client_sqs_1.QueueDoesNotExist) {
+            const qname = (0, qrlCache_js_1.normalizeQueueName)(queue, opt);
+            (0, qrlCache_js_1.qrlCacheInvalidate)(qname);
+            // clear cache in case cache does not reflect reality, then try recreating the queue before sending message again
+            const newQrl = await getOrCreateQueue(queue, opt);
+            params.QueueUrl = newQrl;
+            cmd = new client_sqs_1.SendMessageCommand(params);
+        }
+        for (const exceptionClass of retryableExceptions) {
+            if (error instanceof exceptionClass) {
+                debug({ sendMessageRetryingBecause: { error, result } });
+                return true;
+            }
+        }
+        // If we could not send it, we also need to remove our dedup flag
+        await (0, dedup_js_1.dedupSuccessfullyProcessed)(params, opt);
+        return false;
+    };
+    const result = await backoff.run(send, shouldRetry);
+    debug({ sendMessageResult: result });
+    return result;
+}
+async function sendMessageBatch(qrl, queue, messages, opt) {
+    debug('sendMessageBatch(', qrl, messages.map(e => Object.assign(Object.assign({}, e), { MessageBody: e.MessageBody.slice(0, 10) + '...' })), ')');
+    const params = { Entries: messages, QueueUrl: qrl };
+    if (opt.sentryDsn) {
+        (0, node_1.addBreadcrumb)({ category: 'sendMessageBatch', message: JSON.stringify({ params }), level: 'debug' });
+    }
+    debug({ params });
+    // See which messages we even have to send
+    if (opt.externalDedup) {
+        const promises = params.Entries.map(async (m) => ({ m, shouldEnqueue: await (0, dedup_js_1.dedupShouldEnqueue)(m, opt) }));
+        const results = await Promise.all(promises);
+        params.Entries = results.filter(({ shouldEnqueue }) => shouldEnqueue).map(({ m }) => m);
+        if (!params.Entries.length) {
+            const result = {
+                Failed: [],
+                Successful: results.map(({ m: { Id: id, MessageAttributes: ma } }) => ({
+                    Id: id,
+                    MessageId: 'duplicate',
+                    QdoneDeduplicationId: ma?.QdoneDeduplicationId?.StringValue
+                }))
+            };
+            return result;
+        }
+    }
+    // Send them
+    const client = (0, sqs_js_1.getSQSClient)();
+    let cmd = new client_sqs_1.SendMessageBatchCommand(params);
+    debug({ cmd });
+    const backoff = new exponentialBackoff_js_1.ExponentialBackoff(opt.sendRetries);
+    const send = async (attemptNumber) => {
+        debug({ sendMessageBatchSend: { attemptNumber, params } });
+        const data = await client.send(cmd);
+        return data;
+    };
+    const shouldRetry = async (result, error) => {
+        debug({ shouldRetry: { error, result } });
+        if (result) {
+            // Handle failed result of one or more messages in the batch
+            if (result.Failed && result.Failed.length) {
+                for (const failed of result.Failed) {
+                    // Find corresponding messages
+                    const original = params.Entries.find((e) => e.Id === failed.Id);
+                    const info = { failed, original, opt };
+                    if (opt.sentryDsn) {
+                        (0, node_1.addBreadcrumb)({ category: 'sendMessageBatch', message: 'Failed message: ' + JSON.stringify(info), level: 'error' });
+                    }
+                    else {
+                        console.error(info);
+                    }
+                }
+                throw new Error('One or more message failures: ' + JSON.stringify(result.Failed));
+            }
+        }
+        if (error) {
+            // Handle a failed result from an overall error on request
+            if (opt.sentryDsn) {
+                (0, node_1.addBreadcrumb)({ category: 'sendMessageBatch', message: JSON.stringify({ error }), level: 'error' });
+            }
+            if (error instanceof client_sqs_1.QueueDoesNotExist) {
+                const qname = (0, qrlCache_js_1.normalizeQueueName)(queue, opt);
+                (0, qrlCache_js_1.qrlCacheInvalidate)(qname);
+                // Clear stale cache entry and recreate queue before retrying
+                const newQrl = await getOrCreateQueue(queue, opt);
+                params.QueueUrl = newQrl;
+                cmd = new client_sqs_1.SendMessageBatchCommand(params);
+            }
+            for (const exceptionClass of retryableExceptions) {
+                debug({ exceptionClass, retryableExceptions });
+                if (error instanceof exceptionClass) {
+                    debug({ sendMessageRetryingBecause: { error, result } });
+                    return true;
+                }
+            }
+        }
+    };
+    return backoff.run(send, shouldRetry);
+}
+let requestCount = 0;
+//
+// Flushes the internal message buffer for qrl.
+// If the message is too large, batch is retried with half the messages.
+// Returns number of messages flushed.
+//
+async function flushMessages(qrl, queue, opt, sendBuffer) {
+    debug('flushMessages', { qrl, queue, sendBuffer });
+    // Track our outgoing messages to map with Failed / Successful returns
+    const messagesById = new Map();
+    const resultsById = new Map();
+    const results = [];
+    if (sendBuffer[qrl] && sendBuffer[qrl].length) {
+        for (const message of sendBuffer[qrl]) {
+            const Id = message.Id;
+            messagesById.set(Id, message);
+            // Pre-prepare results
+            const result = { Id };
+            resultsById.set(Id, result);
+            results.push(result);
+        }
+    }
+    // Flush until empty
+    let numFlushed = 0;
+    async function whileNotEmpty() {
+        if (!(sendBuffer[qrl] && sendBuffer[qrl].length)) {
+            return { numFlushed, results };
+        }
+        // Construct batch until full
+        const batch = [];
+        let nextSize = JSON.stringify(sendBuffer[qrl][0]).length;
+        let totalSize = 0;
+        while ((totalSize + nextSize) < 262144 && sendBuffer[qrl].length && batch.length < 10) {
+            batch.push(sendBuffer[qrl].shift());
+            totalSize += nextSize;
+            if (sendBuffer[qrl].length)
+                nextSize = JSON.stringify(sendBuffer[qrl][0]).length;
+            else
+                nextSize = 0;
+        }
+        // Send batch
+        const data = await sendMessageBatch(qrl, queue, batch, opt);
+        // Fail if there are any individual message failures
+        if (data?.Failed && data?.Failed.length) {
+            const err = new Error('One or more message failures: ' + JSON.stringify(data.Failed));
+            err.Failed = data.Failed;
+            throw err;
+        }
+        // If we actually managed to flush any of them
+        if (batch.length) {
+            requestCount += 1;
+            if (data?.Successful) {
+                for (const { Id, MessageId } of data.Successful) {
+                    const result = resultsById.get(Id);
+                    const message = messagesById.get(Id);
+                    result.MessageId = MessageId;
+                    result.Id = Id;
+                    if (message?.MessageAttributes?.QdoneDeduplicationId?.StringValue) {
+                        result.QdoneDeduplicationId = message?.MessageAttributes?.QdoneDeduplicationId?.StringValue;
+                    }
+                    if (opt.verbose)
+                        console.error(chalk_1.default.blue('Enqueued job ') + MessageId + chalk_1.default.blue(' request ' + requestCount));
+                }
+            }
+            numFlushed += batch.length;
+        }
+        return whileNotEmpty();
+    }
+    return whileNotEmpty();
+}
+//
+// Adds a message to the inernal message buffer for the given qrl.
+// Automaticaly flushes if queue has >= 10 messages.
+// Returns number of messages flushed.
+//
+const debugAddMessage = (0, debug_1.default)('qdone:enqueue:addMessage');
+async function addMessage(qrl, queue, command, messageIndex, opt, sendBuffer, messageOptions) {
+    const message = formatMessage(command, messageIndex, opt, messageOptions);
+    sendBuffer[qrl] = sendBuffer[qrl] || [];
+    sendBuffer[qrl].push(message);
+    debugAddMessage({ location: 'addMessage', messageIndex, sendBuffer });
+    if (sendBuffer[qrl].length >= 10) {
+        return flushMessages(qrl, queue, opt, sendBuffer);
+    }
+    return { numFlushed: 0, results: [] };
+}
+//
+// Enqueue a single command
+// Returns a promise for the SQS API response.
+//
+async function enqueue(queue, command, options) {
+    debug('enqueue(', { queue, command }, ')');
+    const opt = (0, defaults_js_1.getOptionsWithDefaults)(options);
+    if (opt.sentryDsn) {
+        (0, node_1.setExtra)({ qdoneOperation: 'enqueue', args: { queue, command, opt } });
+    }
+    try {
+        const qrl = await getOrCreateQueue(queue, opt);
+        return sendMessage(qrl, queue, command, opt);
+    }
+    catch (e) {
+        console.log(e);
+        throw e;
+    }
+}
+//
+// Enqueue many commands formatted as an array of {queue: ..., command: ...} pairs.
+// Returns a promise for the total number of messages enqueued.
+//
+async function enqueueBatch(pairs, options) {
+    debug('enqueueBatch(', pairs, ')');
+    const opt = (0, defaults_js_1.getOptionsWithDefaults)(options);
+    if (opt.sentryDsn) {
+        (0, node_1.setExtra)({ qdoneOperation: 'enqueueBatch', args: { pairs, opt } });
+    }
+    try {
+        const allResults = [];
+        // Find unique queues so we can pre-fetch qrls. We do this so that all
+        // queues are created prior to going through our flush logic
+        const normalizedPairs = pairs.map(({ queue, command, messageOptions }) => ({
+            qname: (0, qrlCache_js_1.normalizeQueueName)(queue, opt),
+            command,
+            messageOptions: (0, defaults_js_1.validateMessageOptions)(messageOptions)
+        }));
+        const uniqueQnames = new Set(normalizedPairs.map(p => p.qname));
+        // Prefetch qrls / create queues in parallel, building a reverse map for cache invalidation
+        const createPromises = [];
+        const qrlToQname = new Map();
+        for (const qname of uniqueQnames) {
+            createPromises.push(getOrCreateQueue(qname, opt).then(qrl => qrlToQname.set(qrl, qname)));
+        }
+        await Promise.all(createPromises);
+        // After we've prefetched, all qrls are in cache
+        // so go back through the list of pairs and fire off messages
+        requestCount = 0;
+        const sendBuffer = {};
+        let messageIndex = 0;
+        let initialFlushTotal = 0;
+        for (const { qname, command, messageOptions } of normalizedPairs) {
+            const qrl = await getOrCreateQueue(qname, opt);
+            const { numFlushed, results } = await addMessage(qrl, qname, command, messageIndex++, opt, sendBuffer, messageOptions);
+            initialFlushTotal += numFlushed;
+            allResults.push(...results);
+        }
+        // And flush any remaining messages
+        const extraFlushPromises = [];
+        for (const qrl in sendBuffer) {
+            extraFlushPromises.push(flushMessages(qrl, qrlToQname.get(qrl), opt, sendBuffer));
+        }
+        let extraFlushTotal = 0;
+        for (const { numFlushed, results } of await Promise.all(extraFlushPromises)) {
+            allResults.push(...results);
+            extraFlushTotal += numFlushed;
+        }
+        const totalFlushed = initialFlushTotal + extraFlushTotal;
+        debug({ initialFlushTotal, extraFlushTotal, totalFlushed });
+        return { numFlushed: totalFlushed, results: allResults };
+    }
+    catch (e) {
+        console.log(e);
+        throw e;
+    }
+}
+debug('loaded');

package/commonjs/src/exponentialBackoff.js

@@ -0,0 +1,101 @@
+"use strict";
+/**
+ * Exponential backoff controller.
+ * usage:
+ *   const exp = new ExponentialBackoff()
+ *   const result = await exp.run(
+ *     function action (attemptNumber) {
+ *        console.log(attemptNumber) // 1, 2, 3, ...
+ *        return axios.post(...)
+ *     },
+ *     function shouldRetry (returnValue, error) {
+ *       if (returnValue && return value.code = 500) return true
+ *       if (error && error.message === 'Internal Server Error') return true
+ *     }
+ *   )
+ */
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.ExponentialBackoff = void 0;
+class ExponentialBackoff {
+    /**
+     * Creates various behaviors for backoff.
+     * @param {number} maxRetries - Number of times to attempt the action before
+     *  throwing an error. Defaults to 3.
+     * @param {number} maxJitterPercent - Jitter as a percentage of the delay.
+     *  For example, if the exponential delay is 2 seconds, then a jitter of
+     *  0.5 could lead to a delay as low as 1 second and as high as 3 seconds,
+     *  since 0.5 * 2 = 1. Defaults to 0.5.
+     * @param {number} exponentBase - The base for the exponent. Defaults to 2,
+     *  which means the delay doubles every attempt.
+     */
+    constructor(maxRetries = 3, maxJitterPercent = 0.5, exponentBase = 2) {
+        if (maxRetries < 1)
+            throw new Error('maxRetries must be >= 1');
+        if (maxJitterPercent < 0.1 || maxJitterPercent > 1)
+            throw new Error('maxJitterPercent must be in the interval [0.1, 1]');
+        if (exponentBase < 1 || exponentBase > 10)
+            throw new Error('exponentBase must be in the range [1, 10]');
+        this.maxRetries = parseInt(maxRetries);
+        this.maxJitterPercent = parseFloat(maxJitterPercent);
+        this.exponentBase = parseFloat(exponentBase);
+        this.attemptNumber = 0;
+    }
+    /**
+     * Calculates how many ms to delay based on the current attempt number.
+     */
+    calculateDelayMs(attemptNumber) {
+        const secondsRaw = this.exponentBase ** attemptNumber; // 2, 4, 8, 16, ....
+        const jitter = this.maxJitterPercent * (Math.random() - 0.5); // [-0.5, 0.5]
+        const delayMs = Math.round(secondsRaw * (1 + jitter) * 1000);
+        // console.log({ secondsRaw, jitter, delayMs })
+        return delayMs;
+    }
+    /**
+     * Resolves after a delay set by the current attempt.
+     */
+    async delay(attemptNumber) {
+        // console.log(attemptNumber)
+        const delay = this.calculateDelayMs(attemptNumber);
+        // console.log({ function: 'delay', attemptNumber, delay })
+        return new Promise((resolve, reject) => setTimeout(resolve, delay));
+    }
+    /**
+     * Call another function repeatedly, retrying with exponential backoff and
+     * jitter if not successful.
+     * @param {ExponentialBackoff~action} action - Callback that does the action
+     *  to be attempted (web request, rpc, database call, etc). Will be called
+     *  again after the exponential dealy if shouldRetry() returns true.
+     * @param {ExponentialBackoff~shouldRetry} shouldRetry - Callback that gets
+     *  to look at the return value of action() and any potential exception. If
+     *  this returns true then the action will be retried with the appropriate
+     *  backoff delay. Defaults to a function that returns true if an exception
+     *  is thrown.
+     */
+    async run(action = async (attemptNumber) => undefined, shouldRetry = async (returnValue, error) => !!error) {
+        let attemptNumber = 0;
+        while (attemptNumber++ < this.maxRetries) {
+            try {
+                const result = await action(attemptNumber);
+                if (await shouldRetry(result, undefined)) {
+                    if (attemptNumber >= this.maxRetries)
+                        throw new Error('Maximum number of attempts reached');
+                    await this.delay(attemptNumber);
+                }
+                else {
+                    return result;
+                }
+            }
+            catch (e) {
+                if (await shouldRetry(undefined, e)) {
+                    if (attemptNumber >= this.maxRetries)
+                        throw e;
+                    await this.delay(attemptNumber);
+                }
+                else {
+                    throw e;
+                }
+            }
+        }
+    }
+}
+exports.ExponentialBackoff = ExponentialBackoff;