qdone 2.0.5-alpha → 2.0.7-alpha

package/commonjs/src/defaults.js

@@ -23,9 +23,10 @@ exports.defaults = Object.freeze({
     // Enqueue
     groupId: (0, uuid_1.v1)(),
     groupIdPerMessage: false,
-    deduplicationId: (0, uuid_1.v1)(),
+    deduplicationId: undefined,
     messageRetentionPeriod: 1209600,
     delay: 0,
+    sendRetries: 6,
     failDelay: 0,
     dlq: false,
     dlqSuffix: '_dead',
@@ -74,6 +75,7 @@ function getOptionsWithDefaults(options) {
         deduplicationId: options.deduplicationId || options['deduplication-id'] || exports.defaults.deduplicationId,
         messageRetentionPeriod: options.messageRetentionPeriod || options['message-retention-period'] || exports.defaults.messageRetentionPeriod,
         delay: options.delay || exports.defaults.delay,
+        sendRetries: options['send-retries'] || exports.defaults.sendRetries,
         failDelay: options.failDelay || options['fail-delay'] || exports.defaults.failDelay,
         dlq: dlq || exports.defaults.dlq,
         dlqSuffix: options.dlqSuffix || options['dlq-suffix'] || exports.defaults.dlqSuffix,

package/commonjs/src/enqueue.js

@@ -1,10 +1,4 @@
 "use strict";
-// const Q = require('q')
-// const debug = require('debug')('qdone:enqueue')
-// const chalk = require('chalk')
-// const uuid = require('uuid')
-// const qrlCache = require('./qrlCache')
-// const AWS = require('aws-sdk')
 var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
     function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
     return new (P || (P = Promise))(function (resolve, reject) {
@@ -46,6 +40,7 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
 };
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.enqueueBatch = exports.enqueue = exports.addMessage = exports.flushMessages = exports.sendMessageBatch = exports.sendMessage = exports.formatMessage = exports.getQueueAttributes = exports.getOrCreateQueue = exports.getOrCreateFailQueue = exports.getOrCreateDLQ = void 0;
+var node_1 = require("@sentry/node");
 var uuid_1 = require("uuid");
 var chalk_1 = __importDefault(require("chalk"));
 var debug_1 = __importDefault(require("debug"));
@@ -53,6 +48,7 @@ var client_sqs_1 = require("@aws-sdk/client-sqs");
 var qrlCache_js_1 = require("./qrlCache.js");
 var sqs_js_1 = require("./sqs.js");
 var defaults_js_1 = require("./defaults.js");
+var exponentialBackoff_js_1 = require("./exponentialBackoff.js");
 var debug = (0, debug_1.default)('qdone:enqueue');
 function getOrCreateDLQ(queue, opt) {
     return __awaiter(this, void 0, void 0, function () {
@@ -258,29 +254,63 @@ function formatMessage(command, id) {
     return message;
 }
 exports.formatMessage = formatMessage;
+// Retry happens within the context of the send functions
+var retryableExceptions = [
+    client_sqs_1.RequestThrottled,
+    client_sqs_1.KmsThrottled,
+    client_sqs_1.QueueDoesNotExist // Queue could temporarily not exist due to eventual consistency, let it retry
+];
 function sendMessage(qrl, command, opt) {
     return __awaiter(this, void 0, void 0, function () {
-        var params, client, cmd, data;
+        var params, client, cmd, backoff, send, shouldRetry, result;
+        var _this = this;
         return __generator(this, function (_a) {
             switch (_a.label) {
                 case 0:
                     debug('sendMessage(', qrl, command, ')');
                     params = Object.assign({ QueueUrl: qrl }, formatMessage(command));
-                    // Add in group id if we're using fifo
                     if (opt.fifo) {
                         params.MessageGroupId = opt.groupId;
-                        params.MessageDeduplicationId = opt.deduplicationId;
+                        params.MessageDeduplicationId = opt.deduplicationId || (0, uuid_1.v1)();
                     }
                     if (opt.delay)
                         params.DelaySeconds = opt.delay;
                     client = (0, sqs_js_1.getSQSClient)();
                     cmd = new client_sqs_1.SendMessageCommand(params);
                     debug({ cmd: cmd });
-                    return [4 /*yield*/, client.send(cmd)];
+                    backoff = new exponentialBackoff_js_1.ExponentialBackoff(opt.sendRetries);
+                    send = function (attemptNumber) { return __awaiter(_this, void 0, void 0, function () {
+                        var data;
+                        return __generator(this, function (_a) {
+                            switch (_a.label) {
+                                case 0:
+                                    cmd.input.attemptNumber = attemptNumber;
+                                    return [4 /*yield*/, client.send(cmd)];
+                                case 1:
+                                    data = _a.sent();
+                                    debug('sendMessage returned', data);
+                                    return [2 /*return*/, data];
+                            }
+                        });
+                    }); };
+                    shouldRetry = function (result, error) { return __awaiter(_this, void 0, void 0, function () {
+                        var _i, retryableExceptions_1, exceptionClass;
+                        return __generator(this, function (_a) {
+                            for (_i = 0, retryableExceptions_1 = retryableExceptions; _i < retryableExceptions_1.length; _i++) {
+                                exceptionClass = retryableExceptions_1[_i];
+                                if (error instanceof exceptionClass) {
+                                    debug({ sendMessageRetryingBecause: { error: error, result: result } });
+                                    return [2 /*return*/, true];
+                                }
+                            }
+                            return [2 /*return*/, false];
+                        });
+                    }); };
+                    return [4 /*yield*/, backoff.run(send, shouldRetry)];
                 case 1:
-                    data = _a.sent();
-                    debug('sendMessage returned', data);
-                    return [2 /*return*/, data];
+                    result = _a.sent();
+                    debug({ sendMessageResult: result });
+                    return [2 /*return*/, result];
             }
         });
     });
@@ -288,34 +318,84 @@ function sendMessage(qrl, command, opt) {
 exports.sendMessage = sendMessage;
 function sendMessageBatch(qrl, messages, opt) {
     return __awaiter(this, void 0, void 0, function () {
-        var params, uuidFunction, client, cmd, data;
+        var params, uuidFunction, client, cmd, backoff, send, shouldRetry;
+        var _this = this;
         return __generator(this, function (_a) {
-            switch (_a.label) {
-                case 0:
-                    debug('sendMessageBatch(', qrl, messages.map(function (e) { return Object.assign(Object.assign({}, e), { MessageBody: e.MessageBody.slice(0, 10) + '...' }); }), ')');
-                    params = { Entries: messages, QueueUrl: qrl };
-                    uuidFunction = opt.uuidFunction || uuid_1.v1;
-                    // Add in group id if we're using fifo
-                    if (opt.fifo) {
-                        params.Entries = params.Entries.map(function (message) { return Object.assign({
-                            MessageGroupId: opt.groupIdPerMessage ? uuidFunction() : opt.groupId,
-                            MessageDeduplicationId: uuidFunction()
-                        }, message); });
+            debug('sendMessageBatch(', qrl, messages.map(function (e) { return Object.assign(Object.assign({}, e), { MessageBody: e.MessageBody.slice(0, 10) + '...' }); }), ')');
+            params = { Entries: messages, QueueUrl: qrl };
+            uuidFunction = opt.uuidFunction || uuid_1.v1;
+            // Add in group id if we're using fifo
+            if (opt.fifo) {
+                params.Entries = params.Entries.map(function (message) { return Object.assign({
+                    MessageGroupId: opt.groupIdPerMessage ? uuidFunction() : opt.groupId,
+                    MessageDeduplicationId: uuidFunction()
+                }, message); });
+            }
+            if (opt.delay) {
+                params.Entries = params.Entries.map(function (message) {
+                    return Object.assign({ DelaySeconds: opt.delay }, message);
+                });
+            }
+            if (opt.sentryDsn) {
+                (0, node_1.addBreadcrumb)({ category: 'sendMessageBatch', message: JSON.stringify({ params: params }), level: 'debug' });
+            }
+            debug({ params: params });
+            client = (0, sqs_js_1.getSQSClient)();
+            cmd = new client_sqs_1.SendMessageBatchCommand(params);
+            debug({ cmd: cmd });
+            backoff = new exponentialBackoff_js_1.ExponentialBackoff(opt.sendRetries);
+            send = function (attemptNumber) { return __awaiter(_this, void 0, void 0, function () {
+                var data;
+                return __generator(this, function (_a) {
+                    switch (_a.label) {
+                        case 0:
+                            debug({ sendMessageBatchSend: { attemptNumber: attemptNumber, params: params } });
+                            return [4 /*yield*/, client.send(cmd)];
+                        case 1:
+                            data = _a.sent();
+                            return [2 /*return*/, data];
                     }
-                    if (opt.delay) {
-                        params.Entries = params.Entries.map(function (message) {
-                            return Object.assign({ DelaySeconds: opt.delay }, message);
-                        });
+                });
+            }); };
+            shouldRetry = function (result, error) {
+                debug({ shouldRetry: { error: error, result: result } });
+                if (result) {
+                    // Handle failed result of one or more messages in the batch
+                    if (result.Failed && result.Failed.length) {
+                        var _loop_1 = function (failed) {
+                            // Find corresponding messages
+                            var original = params.Entries.find(function (e) { return e.Id === failed.Id; });
+                            var info = { failed: failed, original: original, opt: opt };
+                            if (opt.sentryDsn) {
+                                (0, node_1.addBreadcrumb)({ category: 'sendMessageBatch', message: 'Failed message: ' + JSON.stringify(info), level: 'error' });
+                            }
+                            else {
+                                console.error(info);
+                            }
+                        };
+                        for (var _i = 0, _a = result.Failed; _i < _a.length; _i++) {
+                            var failed = _a[_i];
+                            _loop_1(failed);
+                        }
+                        throw new Error('One or more message failures: ' + JSON.stringify(result.Failed));
                     }
-                    client = (0, sqs_js_1.getSQSClient)();
-                    cmd = new client_sqs_1.SendMessageBatchCommand(params);
-                    debug({ cmd: cmd });
-                    return [4 /*yield*/, client.send(cmd)];
-                case 1:
-                    data = _a.sent();
-                    debug('sendMessageBatch returned', data);
-                    return [2 /*return*/, data];
-            }
+                }
+                if (error) {
+                    // Handle a failed result from an overall error on request
+                    if (opt.sentryDsn) {
+                        (0, node_1.addBreadcrumb)({ category: 'sendMessageBatch', message: JSON.stringify({ error: error }), level: 'error' });
+                    }
+                    for (var _b = 0, retryableExceptions_2 = retryableExceptions; _b < retryableExceptions_2.length; _b++) {
+                        var exceptionClass = retryableExceptions_2[_b];
+                        debug({ exceptionClass: exceptionClass, retryableExceptions: retryableExceptions });
+                        if (error instanceof exceptionClass) {
+                            debug({ sendMessageRetryingBecause: { error: error, result: result } });
+                            return true;
+                        }
+                    }
+                }
+            };
+            return [2 /*return*/, backoff.run(send, shouldRetry)];
         });
     });
 }
@@ -396,6 +476,7 @@ function addMessage(qrl, command, opt) {
             message = formatMessage(command, messageIndex++);
             messages[qrl] = messages[qrl] || [];
             messages[qrl].push(message);
+            debug({ location: 'addMessage', messages: messages });
             if (messages[qrl].length >= 10) {
                 return [2 /*return*/, flushMessages(qrl, opt)];
             }
@@ -431,9 +512,9 @@ exports.enqueue = enqueue;
 //
 function enqueueBatch(pairs, options) {
     return __awaiter(this, void 0, void 0, function () {
-        var opt, normalizedPairs, uniqueQnames, createPromises, _i, uniqueQnames_1, qname, addMessagePromises, _a, normalizedPairs_1, _b, qname, command, qrl, flushCounts, initialFlushTotal, extraFlushPromises, qrl, extraFlushCounts, extraFlushTotal, totalFlushed;
-        return __generator(this, function (_c) {
-            switch (_c.label) {
+        var opt, normalizedPairs, uniqueQnames, createPromises, _i, uniqueQnames_1, qname, initialFlushTotal, _a, normalizedPairs_1, _b, qname, command, qrl, _c, extraFlushPromises, qrl, extraFlushCounts, extraFlushTotal, totalFlushed;
+        return __generator(this, function (_d) {
+            switch (_d.label) {
                 case 0:
                     debug('enqueueBatch(', pairs, ')');
                     opt = (0, defaults_js_1.getOptionsWithDefaults)(options);
@@ -455,39 +536,35 @@ function enqueueBatch(pairs, options) {
                         // so go back through the list of pairs and fire off messages
                     ];
                 case 1:
-                    _c.sent();
+                    _d.sent();
                     // After we've prefetched, all qrls are in cache
                     // so go back through the list of pairs and fire off messages
                     requestCount = 0;
-                    addMessagePromises = [];
+                    initialFlushTotal = 0;
                     _a = 0, normalizedPairs_1 = normalizedPairs;
-                    _c.label = 2;
+                    _d.label = 2;
                 case 2:
-                    if (!(_a < normalizedPairs_1.length)) return [3 /*break*/, 5];
+                    if (!(_a < normalizedPairs_1.length)) return [3 /*break*/, 6];
                     _b = normalizedPairs_1[_a], qname = _b.qname, command = _b.command;
                     return [4 /*yield*/, getOrCreateQueue(qname, opt)];
                 case 3:
-                    qrl = _c.sent();
-                    addMessagePromises.push(addMessage(qrl, command, opt));
-                    _c.label = 4;
+                    qrl = _d.sent();
+                    _c = initialFlushTotal;
+                    return [4 /*yield*/, addMessage(qrl, command, opt)];
                 case 4:
+                    initialFlushTotal = _c + _d.sent();
+                    _d.label = 5;
+                case 5:
                     _a++;
                     return [3 /*break*/, 2];
-                case 5: return [4 /*yield*/, Promise.all(addMessagePromises)
-                    // Count up how many were flushed during add
-                ];
                 case 6:
-                    flushCounts = _c.sent();
-                    // Count up how many were flushed during add
-                    debug('flushCounts', flushCounts);
-                    initialFlushTotal = flushCounts.reduce(function (a, b) { return a + b; }, 0);
                     extraFlushPromises = [];
                     for (qrl in messages) {
                         extraFlushPromises.push(flushMessages(qrl, opt));
                     }
                     return [4 /*yield*/, Promise.all(extraFlushPromises)];
                 case 7:
-                    extraFlushCounts = _c.sent();
+                    extraFlushCounts = _d.sent();
                     extraFlushTotal = extraFlushCounts.reduce(function (a, b) { return a + b; }, 0);
                     totalFlushed = initialFlushTotal + extraFlushTotal;
                     debug({ initialFlushTotal: initialFlushTotal, extraFlushTotal: extraFlushTotal, totalFlushed: totalFlushed });

package/commonjs/src/exponentialBackoff.js

@@ -0,0 +1,172 @@
+"use strict";
+/**
+ * Exponential backoff controller.
+ * usage:
+ *   const exp = new ExponentialBackoff()
+ *   const result = await exp.run(
+ *     function action (attemptNumber) {
+ *        console.log(attemptNumber) // 1, 2, 3, ...
+ *        return axios.post(...)
+ *     },
+ *     function shouldRetry (returnValue, error) {
+ *       if (returnValue && return value.code = 500) return true
+ *       if (error && error.message === 'Internal Server Error') return true
+ *     }
+ *   )
+ */
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+var __generator = (this && this.__generator) || function (thisArg, body) {
+    var _ = { label: 0, sent: function() { if (t[0] & 1) throw t[1]; return t[1]; }, trys: [], ops: [] }, f, y, t, g;
+    return g = { next: verb(0), "throw": verb(1), "return": verb(2) }, typeof Symbol === "function" && (g[Symbol.iterator] = function() { return this; }), g;
+    function verb(n) { return function (v) { return step([n, v]); }; }
+    function step(op) {
+        if (f) throw new TypeError("Generator is already executing.");
+        while (g && (g = 0, op[0] && (_ = 0)), _) try {
+            if (f = 1, y && (t = op[0] & 2 ? y["return"] : op[0] ? y["throw"] || ((t = y["return"]) && t.call(y), 0) : y.next) && !(t = t.call(y, op[1])).done) return t;
+            if (y = 0, t) op = [op[0] & 2, t.value];
+            switch (op[0]) {
+                case 0: case 1: t = op; break;
+                case 4: _.label++; return { value: op[1], done: false };
+                case 5: _.label++; y = op[1]; op = [0]; continue;
+                case 7: op = _.ops.pop(); _.trys.pop(); continue;
+                default:
+                    if (!(t = _.trys, t = t.length > 0 && t[t.length - 1]) && (op[0] === 6 || op[0] === 2)) { _ = 0; continue; }
+                    if (op[0] === 3 && (!t || (op[1] > t[0] && op[1] < t[3]))) { _.label = op[1]; break; }
+                    if (op[0] === 6 && _.label < t[1]) { _.label = t[1]; t = op; break; }
+                    if (t && _.label < t[2]) { _.label = t[2]; _.ops.push(op); break; }
+                    if (t[2]) _.ops.pop();
+                    _.trys.pop(); continue;
+            }
+            op = body.call(thisArg, _);
+        } catch (e) { op = [6, e]; y = 0; } finally { f = t = 0; }
+        if (op[0] & 5) throw op[1]; return { value: op[0] ? op[1] : void 0, done: true };
+    }
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.ExponentialBackoff = void 0;
+var ExponentialBackoff = /** @class */ (function () {
+    /**
+     * Creates various behaviors for backoff.
+     * @param {number} maxRetries - Number of times to attempt the action before
+     *  throwing an error. Defaults to 3.
+     * @param {number} maxJitterPercent - Jitter as a percentage of the delay.
+     *  For example, if the exponential delay is 2 seconds, then a jitter of
+     *  0.5 could lead to a delay as low as 1 second and as high as 3 seconds,
+     *  since 0.5 * 2 = 1. Defaults to 0.5.
+     * @param {number} exponentBase - The base for the exponent. Defaults to 2,
+     *  which means the delay doubles every attempt.
+     */
+    function ExponentialBackoff(maxRetries, maxJitterPercent, exponentBase) {
+        if (maxRetries === void 0) { maxRetries = 3; }
+        if (maxJitterPercent === void 0) { maxJitterPercent = 0.5; }
+        if (exponentBase === void 0) { exponentBase = 2; }
+        if (maxRetries < 1)
+            throw new Error('maxRetries must be >= 1');
+        if (maxJitterPercent < 0.1 || maxJitterPercent > 1)
+            throw new Error('maxJitterPercent must be in the interval [0.1, 1]');
+        if (exponentBase < 1 || exponentBase > 10)
+            throw new Error('exponentBase must be in the range [1, 10]');
+        this.maxRetries = parseInt(maxRetries);
+        this.maxJitterPercent = parseFloat(maxJitterPercent);
+        this.exponentBase = parseFloat(exponentBase);
+        this.attemptNumber = 0;
+    }
+    /**
+     * Calculates how many ms to delay based on the current attempt number.
+     */
+    ExponentialBackoff.prototype.calculateDelayMs = function (attemptNumber) {
+        var secondsRaw = Math.pow(this.exponentBase, attemptNumber); // 2, 4, 8, 16, ....
+        var jitter = this.maxJitterPercent * (Math.random() - 0.5); // [-0.5, 0.5]
+        var delayMs = Math.round(secondsRaw * (1 + jitter) * 1000);
+        // console.log({ secondsRaw, jitter, delayMs })
+        return delayMs;
+    };
+    /**
+     * Resolves after a delay set by the current attempt.
+     */
+    ExponentialBackoff.prototype.delay = function (attemptNumber) {
+        return __awaiter(this, void 0, void 0, function () {
+            var delay;
+            return __generator(this, function (_a) {
+                delay = this.calculateDelayMs(attemptNumber);
+                // console.log({ function: 'delay', attemptNumber, delay })
+                return [2 /*return*/, new Promise(function (resolve, reject) { return setTimeout(resolve, delay); })];
+            });
+        });
+    };
+    /**
+     * Call another function repeatedly, retrying with exponential backoff and
+     * jitter if not successful.
+     * @param {ExponentialBackoff~action} action - Callback that does the action
+     *  to be attempted (web request, rpc, database call, etc). Will be called
+     *  again after the exponential dealy if shouldRetry() returns true.
+     * @param {ExponentialBackoff~shouldRetry} shouldRetry - Callback that gets
+     *  to look at the return value of action() and any potential exception. If
+     *  this returns true then the action will be retried with the appropriate
+     *  backoff delay. Defaults to a function that returns true if an exception
+     *  is thrown.
+     */
+    ExponentialBackoff.prototype.run = function (action, shouldRetry) {
+        var _this = this;
+        if (action === void 0) { action = function (attemptNumber) { return __awaiter(_this, void 0, void 0, function () { return __generator(this, function (_a) {
+            return [2 /*return*/, undefined];
+        }); }); }; }
+        if (shouldRetry === void 0) { shouldRetry = function (returnValue, error) { return __awaiter(_this, void 0, void 0, function () { return __generator(this, function (_a) {
+            return [2 /*return*/, !!error];
+        }); }); }; }
+        return __awaiter(this, void 0, void 0, function () {
+            var attemptNumber, result, e_1;
+            return __generator(this, function (_a) {
+                switch (_a.label) {
+                    case 0:
+                        attemptNumber = 0;
+                        _a.label = 1;
+                    case 1:
+                        if (!(attemptNumber++ < this.maxRetries)) return [3 /*break*/, 14];
+                        _a.label = 2;
+                    case 2:
+                        _a.trys.push([2, 8, , 13]);
+                        return [4 /*yield*/, action(attemptNumber)];
+                    case 3:
+                        result = _a.sent();
+                        return [4 /*yield*/, shouldRetry(result, undefined)];
+                    case 4:
+                        if (!_a.sent()) return [3 /*break*/, 6];
+                        if (attemptNumber >= this.maxRetries)
+                            throw new Error('Maximum number of attempts reached');
+                        return [4 /*yield*/, this.delay(attemptNumber)];
+                    case 5:
+                        _a.sent();
+                        return [3 /*break*/, 7];
+                    case 6: return [2 /*return*/, result];
+                    case 7: return [3 /*break*/, 13];
+                    case 8:
+                        e_1 = _a.sent();
+                        return [4 /*yield*/, shouldRetry(undefined, e_1)];
+                    case 9:
+                        if (!_a.sent()) return [3 /*break*/, 11];
+                        if (attemptNumber >= this.maxRetries)
+                            throw e_1;
+                        return [4 /*yield*/, this.delay(attemptNumber)];
+                    case 10:
+                        _a.sent();
+                        return [3 /*break*/, 12];
+                    case 11: throw e_1;
+                    case 12: return [3 /*break*/, 13];
+                    case 13: return [3 /*break*/, 1];
+                    case 14: return [2 /*return*/];
+                }
+            });
+        });
+    };
+    return ExponentialBackoff;
+}());
+exports.ExponentialBackoff = ExponentialBackoff;

package/commonjs/src/idleQueues.js

@@ -328,7 +328,7 @@ function processQueuePair(qname, qrl, opt) {
                     isFifo = qname.endsWith('.fifo');
                     normalizeOptions = Object.assign({}, opt, { fifo: isFifo });
                     fqname = (0, qrlCache_js_1.normalizeFailQueueName)(qname, normalizeOptions);
-                    fqrl = (0, qrlCache_js_1.normalizeFailQueueName)(qrl, normalizeOptions);
+                    fqrl = (0, qrlCache_js_1.normalizeFailQueueName)(fqname, normalizeOptions);
                     return [4 /*yield*/, checkIdle(qname, qrl, opt)];
                 case 1:
                     result = _b.sent();

package/npm-shrinkwrap.json

@@ -1,12 +1,12 @@
 {
   "name": "qdone",
-  "version": "2.0.5-alpha",
+  "version": "2.0.7-alpha",
   "lockfileVersion": 2,
   "requires": true,
   "packages": {
     "": {
       "name": "qdone",
-      "version": "2.0.5-alpha",
+      "version": "2.0.7-alpha",
       "license": "MIT",
       "dependencies": {
         "@aws-sdk/client-cloudwatch": "^3.465.0",

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "qdone",
-  "version": "2.0.5-alpha",
+  "version": "2.0.7-alpha",
   "description": "Language agnostic job queue for SQS",
   "type": "module",
   "main": "./index.js",

package/src/defaults.js

@@ -22,9 +22,10 @@ export const defaults = Object.freeze({
   // Enqueue
   groupId: uuidv1(),
   groupIdPerMessage: false,
-  deduplicationId: uuidv1(),
+  deduplicationId: undefined,
   messageRetentionPeriod: 1209600,
   delay: 0,
+  sendRetries: 6,
   failDelay: 0,
   dlq: false,
   dlqSuffix: '_dead',
@@ -79,6 +80,7 @@ export function getOptionsWithDefaults (options) {
     deduplicationId: options.deduplicationId || options['deduplication-id'] || defaults.deduplicationId,
     messageRetentionPeriod: options.messageRetentionPeriod || options['message-retention-period'] || defaults.messageRetentionPeriod,
     delay: options.delay || defaults.delay,
+    sendRetries: options['send-retries'] || defaults.sendRetries,
     failDelay: options.failDelay || options['fail-delay'] || defaults.failDelay,
     dlq: dlq || defaults.dlq,
     dlqSuffix: options.dlqSuffix || options['dlq-suffix'] || defaults.dlqSuffix,

package/src/enqueue.js

@@ -1,10 +1,4 @@
-// const Q = require('q')
-// const debug = require('debug')('qdone:enqueue')
-// const chalk = require('chalk')
-// const uuid = require('uuid')
-// const qrlCache = require('./qrlCache')
-// const AWS = require('aws-sdk')
-
+import { addBreadcrumb } from '@sentry/node'
 import { v1 as uuidV1 } from 'uuid'
 import chalk from 'chalk'
 import Debug from 'debug'
@@ -13,7 +7,9 @@ import {
   GetQueueAttributesCommand,
   SendMessageCommand,
   SendMessageBatchCommand,
-  QueueDoesNotExist
+  QueueDoesNotExist,
+  RequestThrottled,
+  KmsThrottled
 } from '@aws-sdk/client-sqs'
 
 import {
@@ -25,6 +21,7 @@ import {
 } from './qrlCache.js'
 import { getSQSClient } from './sqs.js'
 import { getOptionsWithDefaults } from './defaults.js'
+import { ExponentialBackoff } from './exponentialBackoff.js'
 
 const debug = Debug('qdone:enqueue')
 
@@ -162,21 +159,45 @@ export function formatMessage (command, id) {
   return message
 }
 
+// Retry happens within the context of the send functions
+const retryableExceptions = [
+  RequestThrottled,
+  KmsThrottled,
+  QueueDoesNotExist // Queue could temporarily not exist due to eventual consistency, let it retry
+]
+
 export async function sendMessage (qrl, command, opt) {
   debug('sendMessage(', qrl, command, ')')
   const params = Object.assign({ QueueUrl: qrl }, formatMessage(command))
-  // Add in group id if we're using fifo
   if (opt.fifo) {
     params.MessageGroupId = opt.groupId
-    params.MessageDeduplicationId = opt.deduplicationId
+    params.MessageDeduplicationId = opt.deduplicationId || uuidV1()
   }
   if (opt.delay) params.DelaySeconds = opt.delay
+
+  // Send it
   const client = getSQSClient()
   const cmd = new SendMessageCommand(params)
   debug({ cmd })
-  const data = await client.send(cmd)
-  debug('sendMessage returned', data)
-  return data
+  const backoff = new ExponentialBackoff(opt.sendRetries)
+  const send = async (attemptNumber) => {
+    cmd.input.attemptNumber = attemptNumber
+    const data = await client.send(cmd)
+    debug('sendMessage returned', data)
+    return data
+  }
+  const shouldRetry = async (result, error) => {
+    for (const exceptionClass of retryableExceptions) {
+      if (error instanceof exceptionClass) {
+        debug({ sendMessageRetryingBecause: { error, result } })
+        return true
+      }
+    }
+    return false
+  }
+  const result = await backoff.run(send, shouldRetry)
+  debug({ sendMessageResult: result })
+  return result
 }
 
 export async function sendMessageBatch (qrl, messages, opt) {
@@ -196,12 +217,54 @@ export async function sendMessageBatch (qrl, messages, opt) {
     params.Entries = params.Entries.map(message =>
       Object.assign({ DelaySeconds: opt.delay }, message))
   }
+  if (opt.sentryDsn) {
+    addBreadcrumb({ category: 'sendMessageBatch', message: JSON.stringify({ params }), level: 'debug' })
+  }
+  debug({ params })
+
+  // Send them
   const client = getSQSClient()
   const cmd = new SendMessageBatchCommand(params)
   debug({ cmd })
-  const data = await client.send(cmd)
-  debug('sendMessageBatch returned', data)
-  return data
+  const backoff = new ExponentialBackoff(opt.sendRetries)
+  const send = async (attemptNumber) => {
+    debug({ sendMessageBatchSend: { attemptNumber, params } })
+    const data = await client.send(cmd)
+    return data
+  }
+  const shouldRetry = (result, error) => {
+    debug({ shouldRetry: { error, result } })
+    if (result) {
+      // Handle failed result of one or more messages in the batch
+      if (result.Failed && result.Failed.length) {
+        for (const failed of result.Failed) {
+          // Find corresponding messages
+          const original = params.Entries.find((e) => e.Id === failed.Id)
+          const info = { failed, original, opt }
+          if (opt.sentryDsn) {
+            addBreadcrumb({ category: 'sendMessageBatch', message: 'Failed message: ' + JSON.stringify(info), level: 'error' })
+          } else {
+            console.error(info)
+          }
+        }
+        throw new Error('One or more message failures: ' + JSON.stringify(result.Failed))
+      }
+    }
+    if (error) {
+      // Handle a failed result from an overall error on request
+      if (opt.sentryDsn) {
+        addBreadcrumb({ category: 'sendMessageBatch', message: JSON.stringify({ error }), level: 'error' })
+      }
+      for (const exceptionClass of retryableExceptions) {
+        debug({ exceptionClass, retryableExceptions })
+        if (error instanceof exceptionClass) {
+          debug({ sendMessageRetryingBecause: { error, result } })
+          return true
+        }
+      }
+    }
+  }
+  return backoff.run(send, shouldRetry)
 }
 
 const messages = {}
@@ -263,6 +326,7 @@ export async function addMessage (qrl, command, opt) {
   const message = formatMessage(command, messageIndex++)
   messages[qrl] = messages[qrl] || []
   messages[qrl].push(message)
+  debug({ location: 'addMessage', messages })
   if (messages[qrl].length >= 10) {
     return flushMessages(qrl, opt)
   }
@@ -306,16 +370,11 @@ export async function enqueueBatch (pairs, options) {
   // After we've prefetched, all qrls are in cache
   // so go back through the list of pairs and fire off messages
   requestCount = 0
-  const addMessagePromises = []
+  let initialFlushTotal = 0
   for (const { qname, command } of normalizedPairs) {
     const qrl = await getOrCreateQueue(qname, opt)
-    addMessagePromises.push(addMessage(qrl, command, opt))
+    initialFlushTotal += await addMessage(qrl, command, opt)
   }
-  const flushCounts = await Promise.all(addMessagePromises)
-
-  // Count up how many were flushed during add
-  debug('flushCounts', flushCounts)
-  const initialFlushTotal = flushCounts.reduce((a, b) => a + b, 0)
 
   // And flush any remaining messages
   const extraFlushPromises = []

package/src/exponentialBackoff.js

@@ -0,0 +1,111 @@
+/**
+ * Exponential backoff controller.
+ * usage:
+ *   const exp = new ExponentialBackoff()
+ *   const result = await exp.run(
+ *     function action (attemptNumber) {
+ *        console.log(attemptNumber) // 1, 2, 3, ...
+ *        return axios.post(...)
+ *     },
+ *     function shouldRetry (returnValue, error) {
+ *       if (returnValue && return value.code = 500) return true
+ *       if (error && error.message === 'Internal Server Error') return true
+ *     }
+ *   )
+ */
+
+export class ExponentialBackoff {
+  /**
+   * Creates various behaviors for backoff.
+   * @param {number} maxRetries - Number of times to attempt the action before
+   *  throwing an error. Defaults to 3.
+   * @param {number} maxJitterPercent - Jitter as a percentage of the delay.
+   *  For example, if the exponential delay is 2 seconds, then a jitter of
+   *  0.5 could lead to a delay as low as 1 second and as high as 3 seconds,
+   *  since 0.5 * 2 = 1. Defaults to 0.5.
+   * @param {number} exponentBase - The base for the exponent. Defaults to 2,
+   *  which means the delay doubles every attempt.
+   */
+  constructor (maxRetries = 3, maxJitterPercent = 0.5, exponentBase = 2) {
+    if (maxRetries < 1) throw new Error('maxRetries must be >= 1')
+    if (maxJitterPercent < 0.1 || maxJitterPercent > 1) throw new Error('maxJitterPercent must be in the interval [0.1, 1]')
+    if (exponentBase < 1 || exponentBase > 10) throw new Error('exponentBase must be in the range [1, 10]')
+    this.maxRetries = parseInt(maxRetries)
+    this.maxJitterPercent = parseFloat(maxJitterPercent)
+    this.exponentBase = parseFloat(exponentBase)
+    this.attemptNumber = 0
+  }
+
+  /**
+   * Calculates how many ms to delay based on the current attempt number.
+   */
+  calculateDelayMs (attemptNumber) {
+    const secondsRaw = this.exponentBase ** attemptNumber // 2, 4, 8, 16, ....
+    const jitter = this.maxJitterPercent * (Math.random() - 0.5) // [-0.5, 0.5]
+    const delayMs = Math.round(secondsRaw * (1 + jitter) * 1000)
+    // console.log({ secondsRaw, jitter, delayMs })
+    return delayMs
+  }
+
+  /**
+   * Resolves after a delay set by the current attempt.
+   */
+  async delay (attemptNumber) {
+    // console.log(attemptNumber)
+    const delay = this.calculateDelayMs(attemptNumber)
+    // console.log({ function: 'delay', attemptNumber, delay })
+    return new Promise((resolve, reject) => setTimeout(resolve, delay))
+  }
+
+  /**
+   * Call another function repeatedly, retrying with exponential backoff and
+   * jitter if not successful.
+   * @param {ExponentialBackoff~action} action - Callback that does the action
+   *  to be attempted (web request, rpc, database call, etc). Will be called
+   *  again after the exponential dealy if shouldRetry() returns true.
+   * @param {ExponentialBackoff~shouldRetry} shouldRetry - Callback that gets
+   *  to look at the return value of action() and any potential exception. If
+   *  this returns true then the action will be retried with the appropriate
+   *  backoff delay. Defaults to a function that returns true if an exception
+   *  is thrown.
+   */
+  async run (
+    action = async (attemptNumber) => undefined,
+    shouldRetry = async (returnValue, error) => !!error
+  ) {
+    let attemptNumber = 0
+    while (attemptNumber++ < this.maxRetries) {
+      try {
+        const result = await action(attemptNumber)
+        if (await shouldRetry(result, undefined)) {
+          if (attemptNumber >= this.maxRetries) throw new Error('Maximum number of attempts reached')
+          await this.delay(attemptNumber)
+        } else {
+          return result
+        }
+      } catch (e) {
+        if (await shouldRetry(undefined, e)) {
+          if (attemptNumber >= this.maxRetries) throw e
+          await this.delay(attemptNumber)
+        } else {
+          throw e
+        }
+      }
+    }
+  }
+
+  /**
+   * Callback used by run().
+   * @callback ExponentialBackoff~action
+   * @param {number} attemptNumber - Which attempt this is, i.e. 1, 2, 3, ...
+   */
+
+  /**
+   * Callback used by run().
+   * @callback ExponentialBackoff~shouldRetry
+   * @param returnValue - The value returned by your action. If an exception
+   *  was thrown by the action then this is undefined.
+   * @param error - The exception thrown by your action. If there was no
+   *  exception, this is undefined.
+   */
+}

package/src/idleQueues.js

@@ -208,7 +208,7 @@ export async function processQueuePair (qname, qrl, opt) {
   const normalizeOptions = Object.assign({}, opt, { fifo: isFifo })
   // Generate fail queue name/url
   const fqname = normalizeFailQueueName(qname, normalizeOptions)
-  const fqrl = normalizeFailQueueName(qrl, normalizeOptions)
+  const fqrl = normalizeFailQueueName(fqname, normalizeOptions)
 
   // Idle check
   const result = await checkIdle(qname, qrl, opt)

package/src/worker.js

@@ -5,7 +5,8 @@
 import {
   ChangeMessageVisibilityCommand,
   ReceiveMessageCommand,
-  DeleteMessageCommand
+  DeleteMessageCommand,
+  QueueDoesNotExist
 } from '@aws-sdk/client-sqs'
 import { exec } from 'child_process' // node:child_process
 import treeKill from 'tree-kill'
@@ -208,11 +209,25 @@ export async function listen (queues, options) {
           chalk.blue(' (' + qrl + ')')
         )
       }
-      // Aggregate the results
-      const { noJobs, jobsSucceeded, jobsFailed } = await pollForJobs(qname, qrl, opt)
-      stats.noJobs += noJobs
-      stats.jobsFailed += jobsFailed
-      stats.jobsSucceeded += jobsSucceeded
+      try {
+        // Aggregate the results
+        const { noJobs, jobsSucceeded, jobsFailed } = await pollForJobs(qname, qrl, opt)
+        stats.noJobs += noJobs
+        stats.jobsFailed += jobsFailed
+        stats.jobsSucceeded += jobsSucceeded
+      } catch (e) {
+        if (e instanceof QueueDoesNotExist) {
+          if (opt.verbose) {
+            console.error(
+              chalk.yellow('Warning: Queue ') +
+              qname.slice(opt.prefix.length) +
+              chalk.yellow(' does not exist.')
+            )
+          }
+        } else {
+          throw e
+        }
+      }
     }
     return stats
   }