qdone 1.7.0 → 2.0.0-alpha

package/commonjs/src/enqueue.js

@@ -0,0 +1,498 @@
+"use strict";
+// const Q = require('q')
+// const debug = require('debug')('qdone:enqueue')
+// const chalk = require('chalk')
+// const uuid = require('uuid')
+// const qrlCache = require('./qrlCache')
+// const AWS = require('aws-sdk')
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+var __generator = (this && this.__generator) || function (thisArg, body) {
+    var _ = { label: 0, sent: function() { if (t[0] & 1) throw t[1]; return t[1]; }, trys: [], ops: [] }, f, y, t, g;
+    return g = { next: verb(0), "throw": verb(1), "return": verb(2) }, typeof Symbol === "function" && (g[Symbol.iterator] = function() { return this; }), g;
+    function verb(n) { return function (v) { return step([n, v]); }; }
+    function step(op) {
+        if (f) throw new TypeError("Generator is already executing.");
+        while (g && (g = 0, op[0] && (_ = 0)), _) try {
+            if (f = 1, y && (t = op[0] & 2 ? y["return"] : op[0] ? y["throw"] || ((t = y["return"]) && t.call(y), 0) : y.next) && !(t = t.call(y, op[1])).done) return t;
+            if (y = 0, t) op = [op[0] & 2, t.value];
+            switch (op[0]) {
+                case 0: case 1: t = op; break;
+                case 4: _.label++; return { value: op[1], done: false };
+                case 5: _.label++; y = op[1]; op = [0]; continue;
+                case 7: op = _.ops.pop(); _.trys.pop(); continue;
+                default:
+                    if (!(t = _.trys, t = t.length > 0 && t[t.length - 1]) && (op[0] === 6 || op[0] === 2)) { _ = 0; continue; }
+                    if (op[0] === 3 && (!t || (op[1] > t[0] && op[1] < t[3]))) { _.label = op[1]; break; }
+                    if (op[0] === 6 && _.label < t[1]) { _.label = t[1]; t = op; break; }
+                    if (t && _.label < t[2]) { _.label = t[2]; _.ops.push(op); break; }
+                    if (t[2]) _.ops.pop();
+                    _.trys.pop(); continue;
+            }
+            op = body.call(thisArg, _);
+        } catch (e) { op = [6, e]; y = 0; } finally { f = t = 0; }
+        if (op[0] & 5) throw op[1]; return { value: op[0] ? op[1] : void 0, done: true };
+    }
+};
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.enqueueBatch = exports.enqueue = exports.addMessage = exports.flushMessages = exports.sendMessageBatch = exports.sendMessage = exports.formatMessage = exports.getQueueAttributes = exports.getOrCreateQueue = exports.getOrCreateFailQueue = exports.getOrCreateDLQ = void 0;
+var uuid_1 = require("uuid");
+var chalk_1 = __importDefault(require("chalk"));
+var debug_1 = __importDefault(require("debug"));
+var client_sqs_1 = require("@aws-sdk/client-sqs");
+var qrlCache_js_1 = require("./qrlCache.js");
+var sqs_js_1 = require("./sqs.js");
+var defaults_js_1 = require("./defaults.js");
+var debug = (0, debug_1.default)('qdone:enqueue');
+function getOrCreateDLQ(queue, opt) {
+    return __awaiter(this, void 0, void 0, function () {
+        var dqname, dqrl, err_1, client, params, cmd, data, dqrl;
+        return __generator(this, function (_a) {
+            switch (_a.label) {
+                case 0:
+                    debug('getOrCreateDLQ(', queue, ')');
+                    dqname = (0, qrlCache_js_1.normalizeDLQName)(queue, opt);
+                    _a.label = 1;
+                case 1:
+                    _a.trys.push([1, 3, , 5]);
+                    return [4 /*yield*/, (0, qrlCache_js_1.qrlCacheGet)(dqname)];
+                case 2:
+                    dqrl = _a.sent();
+                    return [2 /*return*/, dqrl];
+                case 3:
+                    err_1 = _a.sent();
+                    // Anything other than queue doesn't exist gets re-thrown
+                    if (!(err_1 instanceof client_sqs_1.QueueDoesNotExist))
+                        throw err_1;
+                    client = (0, sqs_js_1.getSQSClient)();
+                    params = {
+                        Attributes: { MessageRetentionPeriod: opt.messageRetentionPeriod + '' },
+                        QueueName: dqname
+                    };
+                    if (opt.tags)
+                        params.tags = opt.tags;
+                    if (opt.fifo)
+                        params.Attributes.FifoQueue = 'true';
+                    cmd = new client_sqs_1.CreateQueueCommand(params);
+                    if (opt.verbose)
+                        console.error(chalk_1.default.blue('Creating dead letter queue ') + dqname);
+                    return [4 /*yield*/, client.send(cmd)];
+                case 4:
+                    data = _a.sent();
+                    debug('createQueue returned', data);
+                    dqrl = data.QueueUrl;
+                    (0, qrlCache_js_1.qrlCacheSet)(dqname, dqrl);
+                    return [2 /*return*/, dqrl];
+                case 5: return [2 /*return*/];
+            }
+        });
+    });
+}
+exports.getOrCreateDLQ = getOrCreateDLQ;
+function getOrCreateFailQueue(queue, opt) {
+    return __awaiter(this, void 0, void 0, function () {
+        var fqname, fqrl, err_2, client, params, dqrl, dqa, cmd, data, fqrl;
+        return __generator(this, function (_a) {
+            switch (_a.label) {
+                case 0:
+                    debug('getOrCreateFailQueue(', queue, ')');
+                    fqname = (0, qrlCache_js_1.normalizeFailQueueName)(queue, opt);
+                    _a.label = 1;
+                case 1:
+                    _a.trys.push([1, 3, , 8]);
+                    return [4 /*yield*/, (0, qrlCache_js_1.qrlCacheGet)(fqname)];
+                case 2:
+                    fqrl = _a.sent();
+                    return [2 /*return*/, fqrl];
+                case 3:
+                    err_2 = _a.sent();
+                    // Anything other than queue doesn't exist gets re-thrown
+                    if (!(err_2 instanceof client_sqs_1.QueueDoesNotExist))
+                        throw err_2;
+                    client = (0, sqs_js_1.getSQSClient)();
+                    params = {
+                        Attributes: { MessageRetentionPeriod: opt.messageRetentionPeriod + '' },
+                        QueueName: fqname
+                    };
+                    if (!opt.dlq) return [3 /*break*/, 6];
+                    return [4 /*yield*/, getOrCreateDLQ(queue, opt)];
+                case 4:
+                    dqrl = _a.sent();
+                    return [4 /*yield*/, getQueueAttributes(dqrl)];
+                case 5:
+                    dqa = _a.sent();
+                    debug('dqa', dqa);
+                    params.Attributes.RedrivePolicy = JSON.stringify({
+                        deadLetterTargetArn: dqa.Attributes.QueueArn,
+                        maxReceiveCount: opt.dlqAfter + ''
+                    });
+                    _a.label = 6;
+                case 6:
+                    if (opt.tags)
+                        params.tags = opt.tags;
+                    if (opt.fifo)
+                        params.Attributes.FifoQueue = 'true';
+                    cmd = new client_sqs_1.CreateQueueCommand(params);
+                    if (opt.verbose)
+                        console.error(chalk_1.default.blue('Creating fail queue ') + fqname);
+                    return [4 /*yield*/, client.send(cmd)];
+                case 7:
+                    data = _a.sent();
+                    debug('createQueue returned', data);
+                    fqrl = data.QueueUrl;
+                    (0, qrlCache_js_1.qrlCacheSet)(fqname, fqrl);
+                    return [2 /*return*/, fqrl];
+                case 8: return [2 /*return*/];
+            }
+        });
+    });
+}
+exports.getOrCreateFailQueue = getOrCreateFailQueue;
+/**
+ * Returns a qrl for a queue that either exists or does not
+ */
+function getOrCreateQueue(queue, opt) {
+    return __awaiter(this, void 0, void 0, function () {
+        var qname, qrl, err_3, fqrl, fqa, client, params, cmd, data, qrl;
+        return __generator(this, function (_a) {
+            switch (_a.label) {
+                case 0:
+                    debug('getOrCreateQueue(', queue, ')');
+                    qname = (0, qrlCache_js_1.normalizeQueueName)(queue, opt);
+                    _a.label = 1;
+                case 1:
+                    _a.trys.push([1, 3, , 7]);
+                    return [4 /*yield*/, (0, qrlCache_js_1.qrlCacheGet)(qname)];
+                case 2:
+                    qrl = _a.sent();
+                    return [2 /*return*/, qrl];
+                case 3:
+                    err_3 = _a.sent();
+                    // Anything other than queue doesn't exist gets re-thrown
+                    if (!(err_3 instanceof client_sqs_1.QueueDoesNotExist))
+                        throw err_3;
+                    return [4 /*yield*/, getOrCreateFailQueue(qname, opt)];
+                case 4:
+                    fqrl = _a.sent();
+                    return [4 /*yield*/, getQueueAttributes(fqrl)
+                        // Create our queue
+                    ];
+                case 5:
+                    fqa = _a.sent();
+                    client = (0, sqs_js_1.getSQSClient)();
+                    params = {
+                        Attributes: {
+                            MessageRetentionPeriod: opt.messageRetentionPeriod + '',
+                            RedrivePolicy: JSON.stringify({
+                                deadLetterTargetArn: fqa.Attributes.QueueArn,
+                                maxReceiveCount: '1'
+                            })
+                        },
+                        QueueName: qname
+                    };
+                    if (opt.tags)
+                        params.tags = opt.tags;
+                    if (opt.fifo)
+                        params.Attributes.FifoQueue = 'true';
+                    cmd = new client_sqs_1.CreateQueueCommand(params);
+                    debug({ params: params });
+                    if (opt.verbose)
+                        console.error(chalk_1.default.blue('Creating queue ') + qname);
+                    return [4 /*yield*/, client.send(cmd)];
+                case 6:
+                    data = _a.sent();
+                    debug('createQueue returned', data);
+                    qrl = data.QueueUrl;
+                    (0, qrlCache_js_1.qrlCacheSet)(qname, qrl);
+                    return [2 /*return*/, qrl];
+                case 7: return [2 /*return*/];
+            }
+        });
+    });
+}
+exports.getOrCreateQueue = getOrCreateQueue;
+function getQueueAttributes(qrl) {
+    return __awaiter(this, void 0, void 0, function () {
+        var client, params, cmd, data;
+        return __generator(this, function (_a) {
+            switch (_a.label) {
+                case 0:
+                    debug('getQueueAttributes(', qrl, ')');
+                    client = (0, sqs_js_1.getSQSClient)();
+                    params = { AttributeNames: ['All'], QueueUrl: qrl };
+                    cmd = new client_sqs_1.GetQueueAttributesCommand(params);
+                    return [4 /*yield*/, client.send(cmd)];
+                case 1:
+                    data = _a.sent();
+                    debug('GetQueueAttributes returned', data);
+                    return [2 /*return*/, data];
+            }
+        });
+    });
+}
+exports.getQueueAttributes = getQueueAttributes;
+function formatMessage(command, id) {
+    var message = {
+        /*
+        MessageAttributes: {
+          City: { DataType: 'String', StringValue: 'Any City' },
+          Population: { DataType: 'Number', StringValue: '1250800' }
+        },
+        */
+        MessageBody: command
+    };
+    if (typeof id !== 'undefined')
+        message.Id = '' + id;
+    return message;
+}
+exports.formatMessage = formatMessage;
+function sendMessage(qrl, command, opt) {
+    return __awaiter(this, void 0, void 0, function () {
+        var params, client, cmd, data;
+        return __generator(this, function (_a) {
+            switch (_a.label) {
+                case 0:
+                    debug('sendMessage(', qrl, command, ')');
+                    params = Object.assign({ QueueUrl: qrl }, formatMessage(command));
+                    // Add in group id if we're using fifo
+                    if (opt.fifo) {
+                        params.MessageGroupId = opt.groupId;
+                        params.MessageDeduplicationId = opt.deduplicationId;
+                    }
+                    if (opt.delay)
+                        params.DelaySeconds = opt.delay;
+                    client = (0, sqs_js_1.getSQSClient)();
+                    cmd = new client_sqs_1.SendMessageCommand(params);
+                    debug({ cmd: cmd });
+                    return [4 /*yield*/, client.send(cmd)];
+                case 1:
+                    data = _a.sent();
+                    debug('sendMessage returned', data);
+                    return [2 /*return*/, data];
+            }
+        });
+    });
+}
+exports.sendMessage = sendMessage;
+function sendMessageBatch(qrl, messages, opt) {
+    return __awaiter(this, void 0, void 0, function () {
+        var params, uuidFunction, client, cmd, data;
+        return __generator(this, function (_a) {
+            switch (_a.label) {
+                case 0:
+                    debug('sendMessageBatch(', qrl, messages.map(function (e) { return Object.assign(Object.assign({}, e), { MessageBody: e.MessageBody.slice(0, 10) + '...' }); }), ')');
+                    params = { Entries: messages, QueueUrl: qrl };
+                    uuidFunction = opt.uuidFunction || uuid_1.v1;
+                    // Add in group id if we're using fifo
+                    if (opt.fifo) {
+                        params.Entries = params.Entries.map(function (message) { return Object.assign({
+                            MessageGroupId: opt.groupIdPerMessage ? uuidFunction() : opt.groupId,
+                            MessageDeduplicationId: uuidFunction()
+                        }, message); });
+                    }
+                    if (opt.delay) {
+                        params.Entries = params.Entries.map(function (message) {
+                            return Object.assign({ DelaySeconds: opt.delay }, message);
+                        });
+                    }
+                    client = (0, sqs_js_1.getSQSClient)();
+                    cmd = new client_sqs_1.SendMessageBatchCommand(params);
+                    debug({ cmd: cmd });
+                    return [4 /*yield*/, client.send(cmd)];
+                case 1:
+                    data = _a.sent();
+                    debug('sendMessageBatch returned', data);
+                    return [2 /*return*/, data];
+            }
+        });
+    });
+}
+exports.sendMessageBatch = sendMessageBatch;
+var messages = {};
+var requestCount = 0;
+//
+// Flushes the internal message buffer for qrl.
+// If the message is too large, batch is retried with half the messages.
+// Returns number of messages flushed.
+//
+function flushMessages(qrl, opt) {
+    return __awaiter(this, void 0, void 0, function () {
+        function whileNotEmpty() {
+            return __awaiter(this, void 0, void 0, function () {
+                var batch, nextSize, totalSize, data, err;
+                return __generator(this, function (_a) {
+                    switch (_a.label) {
+                        case 0:
+                            if (!(messages[qrl] && messages[qrl].length))
+                                return [2 /*return*/, numFlushed
+                                    // Construct batch until full
+                                ];
+                            batch = [];
+                            nextSize = JSON.stringify(messages[qrl][0]).length;
+                            totalSize = 0;
+                            while ((totalSize + nextSize) < 262144 && messages[qrl].length && batch.length < 10) {
+                                batch.push(messages[qrl].shift());
+                                totalSize += nextSize;
+                                if (messages[qrl].length)
+                                    nextSize = JSON.stringify(messages[qrl][0]).length;
+                                else
+                                    nextSize = 0;
+                            }
+                            return [4 /*yield*/, sendMessageBatch(qrl, batch, opt)];
+                        case 1:
+                            data = _a.sent();
+                            debug({ data: data });
+                            // Fail if there are any individual message failures
+                            if (data.Failed && data.Failed.length) {
+                                err = new Error('One or more message failures: ' + JSON.stringify(data.Failed));
+                                err.Failed = data.Failed;
+                                throw err;
+                            }
+                            // If we actually managed to flush any of them
+                            if (batch.length) {
+                                requestCount += 1;
+                                data.Successful.forEach(function (message) {
+                                    if (opt.verbose)
+                                        console.error(chalk_1.default.blue('Enqueued job ') + message.MessageId + chalk_1.default.blue(' request ' + requestCount));
+                                });
+                                numFlushed += batch.length;
+                            }
+                            return [2 /*return*/, whileNotEmpty()];
+                    }
+                });
+            });
+        }
+        var numFlushed;
+        return __generator(this, function (_a) {
+            debug('flushMessages', qrl);
+            numFlushed = 0;
+            return [2 /*return*/, whileNotEmpty()];
+        });
+    });
+}
+exports.flushMessages = flushMessages;
+//
+// Adds a message to the inernal message buffer for the given qrl.
+// Automaticaly flushes if queue has >= 10 messages.
+// Returns number of messages flushed.
+//
+var messageIndex = 0;
+function addMessage(qrl, command, opt) {
+    return __awaiter(this, void 0, void 0, function () {
+        var message;
+        return __generator(this, function (_a) {
+            message = formatMessage(command, messageIndex++);
+            messages[qrl] = messages[qrl] || [];
+            messages[qrl].push(message);
+            if (messages[qrl].length >= 10) {
+                return [2 /*return*/, flushMessages(qrl, opt)];
+            }
+            return [2 /*return*/, 0];
+        });
+    });
+}
+exports.addMessage = addMessage;
+//
+// Enqueue a single command
+// Returns a promise for the SQS API response.
+//
+function enqueue(queue, command, options) {
+    return __awaiter(this, void 0, void 0, function () {
+        var opt, qrl;
+        return __generator(this, function (_a) {
+            switch (_a.label) {
+                case 0:
+                    debug('enqueue(', { queue: queue, command: command }, ')');
+                    opt = (0, defaults_js_1.getOptionsWithDefaults)(options);
+                    return [4 /*yield*/, getOrCreateQueue(queue, opt)];
+                case 1:
+                    qrl = _a.sent();
+                    return [2 /*return*/, sendMessage(qrl, command, opt)];
+            }
+        });
+    });
+}
+exports.enqueue = enqueue;
+//
+// Enqueue many commands formatted as an array of {queue: ..., command: ...} pairs.
+// Returns a promise for the total number of messages enqueued.
+//
+function enqueueBatch(pairs, options) {
+    return __awaiter(this, void 0, void 0, function () {
+        var opt, normalizedPairs, uniqueQnames, createPromises, _i, uniqueQnames_1, qname, addMessagePromises, _a, normalizedPairs_1, _b, qname, command, qrl, flushCounts, initialFlushTotal, extraFlushPromises, qrl, extraFlushCounts, extraFlushTotal, totalFlushed;
+        return __generator(this, function (_c) {
+            switch (_c.label) {
+                case 0:
+                    debug('enqueueBatch(', pairs, ')');
+                    opt = (0, defaults_js_1.getOptionsWithDefaults)(options);
+                    normalizedPairs = pairs.map(function (_a) {
+                        var queue = _a.queue, command = _a.command;
+                        return ({
+                            qname: (0, qrlCache_js_1.normalizeQueueName)(queue, opt),
+                            command: command
+                        });
+                    });
+                    uniqueQnames = new Set(normalizedPairs.map(function (p) { return p.qname; }));
+                    createPromises = [];
+                    for (_i = 0, uniqueQnames_1 = uniqueQnames; _i < uniqueQnames_1.length; _i++) {
+                        qname = uniqueQnames_1[_i];
+                        createPromises.push(getOrCreateQueue(qname, opt));
+                    }
+                    return [4 /*yield*/, Promise.all(createPromises)
+                        // After we've prefetched, all qrls are in cache
+                        // so go back through the list of pairs and fire off messages
+                    ];
+                case 1:
+                    _c.sent();
+                    // After we've prefetched, all qrls are in cache
+                    // so go back through the list of pairs and fire off messages
+                    requestCount = 0;
+                    addMessagePromises = [];
+                    _a = 0, normalizedPairs_1 = normalizedPairs;
+                    _c.label = 2;
+                case 2:
+                    if (!(_a < normalizedPairs_1.length)) return [3 /*break*/, 5];
+                    _b = normalizedPairs_1[_a], qname = _b.qname, command = _b.command;
+                    return [4 /*yield*/, getOrCreateQueue(qname, opt)];
+                case 3:
+                    qrl = _c.sent();
+                    addMessagePromises.push(addMessage(qrl, command, opt));
+                    _c.label = 4;
+                case 4:
+                    _a++;
+                    return [3 /*break*/, 2];
+                case 5: return [4 /*yield*/, Promise.all(addMessagePromises)
+                    // Count up how many were flushed during add
+                ];
+                case 6:
+                    flushCounts = _c.sent();
+                    // Count up how many were flushed during add
+                    debug('flushCounts', flushCounts);
+                    initialFlushTotal = flushCounts.reduce(function (a, b) { return a + b; }, 0);
+                    extraFlushPromises = [];
+                    for (qrl in messages) {
+                        extraFlushPromises.push(flushMessages(qrl, opt));
+                    }
+                    return [4 /*yield*/, Promise.all(extraFlushPromises)];
+                case 7:
+                    extraFlushCounts = _c.sent();
+                    extraFlushTotal = extraFlushCounts.reduce(function (a, b) { return a + b; }, 0);
+                    totalFlushed = initialFlushTotal + extraFlushTotal;
+                    debug({ initialFlushTotal: initialFlushTotal, extraFlushTotal: extraFlushTotal, totalFlushed: totalFlushed });
+                    return [2 /*return*/, totalFlushed];
+            }
+        });
+    });
+}
+exports.enqueueBatch = enqueueBatch;
+debug('loaded');