qdone 1.6.0 → 2.0.0-alpha

package/commonjs/src/qrlCache.js

@@ -0,0 +1,250 @@
+"use strict";
+/**
+ * A cache for maintaining queue urls. Avoids having to call the GetQueueUrl
+ * api too often.
+ */
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+var __generator = (this && this.__generator) || function (thisArg, body) {
+    var _ = { label: 0, sent: function() { if (t[0] & 1) throw t[1]; return t[1]; }, trys: [], ops: [] }, f, y, t, g;
+    return g = { next: verb(0), "throw": verb(1), "return": verb(2) }, typeof Symbol === "function" && (g[Symbol.iterator] = function() { return this; }), g;
+    function verb(n) { return function (v) { return step([n, v]); }; }
+    function step(op) {
+        if (f) throw new TypeError("Generator is already executing.");
+        while (g && (g = 0, op[0] && (_ = 0)), _) try {
+            if (f = 1, y && (t = op[0] & 2 ? y["return"] : op[0] ? y["throw"] || ((t = y["return"]) && t.call(y), 0) : y.next) && !(t = t.call(y, op[1])).done) return t;
+            if (y = 0, t) op = [op[0] & 2, t.value];
+            switch (op[0]) {
+                case 0: case 1: t = op; break;
+                case 4: _.label++; return { value: op[1], done: false };
+                case 5: _.label++; y = op[1]; op = [0]; continue;
+                case 7: op = _.ops.pop(); _.trys.pop(); continue;
+                default:
+                    if (!(t = _.trys, t = t.length > 0 && t[t.length - 1]) && (op[0] === 6 || op[0] === 2)) { _ = 0; continue; }
+                    if (op[0] === 3 && (!t || (op[1] > t[0] && op[1] < t[3]))) { _.label = op[1]; break; }
+                    if (op[0] === 6 && _.label < t[1]) { _.label = t[1]; t = op; break; }
+                    if (t && _.label < t[2]) { _.label = t[2]; _.ops.push(op); break; }
+                    if (t[2]) _.ops.pop();
+                    _.trys.pop(); continue;
+            }
+            op = body.call(thisArg, _);
+        } catch (e) { op = [6, e]; y = 0; } finally { f = t = 0; }
+        if (op[0] & 5) throw op[1]; return { value: op[0] ? op[1] : void 0, done: true };
+    }
+};
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.getQnameUrlPairs = exports.getMatchingQueues = exports.ingestQRLs = exports.qrlCacheInvalidate = exports.qrlCacheSet = exports.qrlCacheGet = exports.qrlCacheClear = exports.normalizeDLQName = exports.normalizeFailQueueName = exports.normalizeQueueName = exports.fifoSuffix = void 0;
+var url_1 = require("url");
+var path_1 = require("path");
+var client_sqs_1 = require("@aws-sdk/client-sqs");
+var sqs_js_1 = require("./sqs.js");
+var debug_1 = __importDefault(require("debug"));
+var debug = (0, debug_1.default)('qdone:qrlCache');
+// The actual cache is shared across callers.
+var qcache = new Map();
+//
+// Normalizes a queue name to end with .fifo if opt.fifo is set
+//
+exports.fifoSuffix = '.fifo';
+function normalizeQueueName(qname, opt) {
+    var hasFifo = qname.endsWith(exports.fifoSuffix);
+    var needsFifo = opt.fifo && qname.slice(-1) !== '*';
+    var needsPrefix = !qname.startsWith(opt.prefix);
+    var base = hasFifo ? qname.slice(0, -exports.fifoSuffix.length) : qname;
+    return (needsPrefix ? opt.prefix : '') + base + (needsFifo ? exports.fifoSuffix : '');
+}
+exports.normalizeQueueName = normalizeQueueName;
+//
+// Normalizes fail queue name, appending both --fail-suffix and .fifo depending on opt
+//
+function normalizeFailQueueName(fqname, opt) {
+    var newopt = Object.assign({}, opt, { fifo: false });
+    var base = normalizeQueueName(fqname, newopt); // strip fifo
+    debug({ fqname: fqname, base: base, opt: opt });
+    var needsFail = !base.endsWith(opt.failSuffix);
+    return base + (needsFail ? opt.failSuffix : '') + (opt.fifo ? exports.fifoSuffix : '');
+}
+exports.normalizeFailQueueName = normalizeFailQueueName;
+//
+// Normalizes dlq queue name, appending both --dlq-suffix and .fifo depending on opt
+//
+function normalizeDLQName(dqname, opt) {
+    var newopt = Object.assign({}, opt, { fifo: false });
+    var base = normalizeQueueName(dqname, newopt); // strip fifo
+    var needsDead = !base.endsWith(opt.dlqSuffix);
+    return base + (needsDead ? opt.dlqSuffix : '') + (opt.fifo ? exports.fifoSuffix : '');
+}
+exports.normalizeDLQName = normalizeDLQName;
+//
+// Clear cache
+//
+function qrlCacheClear() {
+    qcache.clear();
+}
+exports.qrlCacheClear = qrlCacheClear;
+//
+// Get QRL (Queue URL)
+// Returns a promise for the queue name
+//
+function qrlCacheGet(qname) {
+    return __awaiter(this, void 0, void 0, function () {
+        var client, input, cmd, result, qrl;
+        return __generator(this, function (_a) {
+            switch (_a.label) {
+                case 0:
+                    debug('get', qname, qcache);
+                    if (qcache.has(qname))
+                        return [2 /*return*/, qcache.get(qname)];
+                    client = (0, sqs_js_1.getSQSClient)();
+                    input = { QueueName: qname };
+                    cmd = new client_sqs_1.GetQueueUrlCommand(input);
+                    return [4 /*yield*/, client.send(cmd)
+                        // debug('result', result)
+                        // if (!result) throw new Error(`No such queue ${qname}`)
+                    ];
+                case 1:
+                    result = _a.sent();
+                    qrl = result.QueueUrl;
+                    // debug('getQueueUrl returned', data)
+                    qcache.set(qname, qrl);
+                    // debug('qcache', Object.keys(qcache), 'get', qname, ' => ', qcache[qname])
+                    return [2 /*return*/, qrl];
+            }
+        });
+    });
+}
+exports.qrlCacheGet = qrlCacheGet;
+//
+// Set QRL (Queue URL)
+// Immediately updates the cache
+//
+function qrlCacheSet(qname, qrl) {
+    qcache.set(qname, qrl);
+    // debug('qcache', Object.keys(qcache), 'set', qname, ' => ', qcache[qname])
+}
+exports.qrlCacheSet = qrlCacheSet;
+//
+// Invalidate cache for given qname
+//
+function qrlCacheInvalidate(qname) {
+    // debug('qcache', Object.keys(qcache), 'delete', qname, ' (was ', qcache[qname], ')')
+    qcache.delete(qname);
+}
+exports.qrlCacheInvalidate = qrlCacheInvalidate;
+//
+// Ingets multiple QRLs
+// Extracts queue names from an array of QRLs and immediately updates the cache.
+//
+function ingestQRLs(qrls) {
+    var pairs = [];
+    debug('ingestQRLs', qrls);
+    for (var _i = 0, qrls_1 = qrls; _i < qrls_1.length; _i++) {
+        var qrl = qrls_1[_i];
+        var qname = (0, path_1.basename)((new url_1.URL(qrl)).pathname);
+        qrlCacheSet(qname, qrl);
+        pairs.push({ qname: qname, qrl: qrl });
+        // debug('qcache', Object.keys(qcache), 'ingestQRLs', qname, ' => ', qcache[qname])
+    }
+    return pairs;
+}
+exports.ingestQRLs = ingestQRLs;
+/**
+ * Returns qrls for queues matching the given prefix and regex.
+ */
+function getMatchingQueues(prefix, nextToken) {
+    return __awaiter(this, void 0, void 0, function () {
+        var input, client, command, result, _a, qrls, keepGoing, _b, _c, _d;
+        var _e;
+        return __generator(this, function (_f) {
+            switch (_f.label) {
+                case 0:
+                    debug('getMatchingQueues', prefix, nextToken);
+                    input = { QueueNamePrefix: prefix, MaxResults: 1000 };
+                    if (nextToken)
+                        input.NextToken = nextToken;
+                    client = (0, sqs_js_1.getSQSClient)();
+                    command = new client_sqs_1.ListQueuesCommand(input);
+                    return [4 /*yield*/, client.send(command)];
+                case 1:
+                    result = _f.sent();
+                    debug({ result: result });
+                    _a = result || {}, qrls = _a.QueueUrls, keepGoing = _a.NextToken;
+                    debug({ qrls: qrls, keepGoing: keepGoing });
+                    if (!keepGoing) return [3 /*break*/, 3];
+                    _c = (_b = (_e = (qrls || [])).push).apply;
+                    _d = [_e];
+                    return [4 /*yield*/, getMatchingQueues(prefix, keepGoing)];
+                case 2:
+                    _c.apply(_b, _d.concat([_f.sent()]));
+                    _f.label = 3;
+                case 3: return [2 /*return*/, qrls || []];
+            }
+        });
+    });
+}
+exports.getMatchingQueues = getMatchingQueues;
+//
+// Resolves into a flattened aray of {qname: ..., qrl: ...} objects.
+//
+function getQnameUrlPairs(qnames, opt) {
+    return __awaiter(this, void 0, void 0, function () {
+        var promises, results;
+        return __generator(this, function (_a) {
+            switch (_a.label) {
+                case 0:
+                    promises = qnames.map(function getQueueUrlOrUrls(qname) {
+                        return __awaiter(this, void 0, void 0, function () {
+                            var prefix, queues, filteredQueues, err_1;
+                            var _a;
+                            return __generator(this, function (_b) {
+                                switch (_b.label) {
+                                    case 0:
+                                        if (!(qname.slice(-1) === '*')) return [3 /*break*/, 2];
+                                        prefix = qname.slice(0, -1);
+                                        return [4 /*yield*/, getMatchingQueues(prefix)];
+                                    case 1:
+                                        queues = _b.sent();
+                                        debug('listQueues return', queues);
+                                        if (opt.fifo) {
+                                            filteredQueues = queues.filter(function (url) { return url.slice(-exports.fifoSuffix.length) === exports.fifoSuffix; });
+                                            return [2 /*return*/, ingestQRLs(filteredQueues)];
+                                        }
+                                        else {
+                                            return [2 /*return*/, ingestQRLs(queues)];
+                                        }
+                                        return [3 /*break*/, 5];
+                                    case 2:
+                                        _b.trys.push([2, 4, , 5]);
+                                        _a = { qname: qname };
+                                        return [4 /*yield*/, qrlCacheGet(qname)];
+                                    case 3: return [2 /*return*/, (_a.qrl = _b.sent(), _a)];
+                                    case 4:
+                                        err_1 = _b.sent();
+                                        if (!(err_1 instanceof client_sqs_1.QueueDoesNotExist))
+                                            throw err_1;
+                                        return [3 /*break*/, 5];
+                                    case 5: return [2 /*return*/];
+                                }
+                            });
+                        });
+                    });
+                    return [4 /*yield*/, Promise.all(promises)];
+                case 1:
+                    results = _a.sent();
+                    return [2 /*return*/, ([].concat.apply([], results)).filter(function (r) { return r; })];
+            }
+        });
+    });
+}
+exports.getQnameUrlPairs = getQnameUrlPairs;
+debug('loaded');

package/commonjs/src/sqs.js

@@ -0,0 +1,160 @@
+"use strict";
+/**
+ * Functions that deal with SQS
+ */
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+var __generator = (this && this.__generator) || function (thisArg, body) {
+    var _ = { label: 0, sent: function() { if (t[0] & 1) throw t[1]; return t[1]; }, trys: [], ops: [] }, f, y, t, g;
+    return g = { next: verb(0), "throw": verb(1), "return": verb(2) }, typeof Symbol === "function" && (g[Symbol.iterator] = function() { return this; }), g;
+    function verb(n) { return function (v) { return step([n, v]); }; }
+    function step(op) {
+        if (f) throw new TypeError("Generator is already executing.");
+        while (g && (g = 0, op[0] && (_ = 0)), _) try {
+            if (f = 1, y && (t = op[0] & 2 ? y["return"] : op[0] ? y["throw"] || ((t = y["return"]) && t.call(y), 0) : y.next) && !(t = t.call(y, op[1])).done) return t;
+            if (y = 0, t) op = [op[0] & 2, t.value];
+            switch (op[0]) {
+                case 0: case 1: t = op; break;
+                case 4: _.label++; return { value: op[1], done: false };
+                case 5: _.label++; y = op[1]; op = [0]; continue;
+                case 7: op = _.ops.pop(); _.trys.pop(); continue;
+                default:
+                    if (!(t = _.trys, t = t.length > 0 && t[t.length - 1]) && (op[0] === 6 || op[0] === 2)) { _ = 0; continue; }
+                    if (op[0] === 3 && (!t || (op[1] > t[0] && op[1] < t[3]))) { _.label = op[1]; break; }
+                    if (op[0] === 6 && _.label < t[1]) { _.label = t[1]; t = op; break; }
+                    if (t && _.label < t[2]) { _.label = t[2]; _.ops.push(op); break; }
+                    if (t[2]) _.ops.pop();
+                    _.trys.pop(); continue;
+            }
+            op = body.call(thisArg, _);
+        } catch (e) { op = [6, e]; y = 0; } finally { f = t = 0; }
+        if (op[0] & 5) throw op[1]; return { value: op[0] ? op[1] : void 0, done: true };
+    }
+};
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.getQueueAttributes = exports.getMatchingQueues = exports.setSQSClient = exports.getSQSClient = void 0;
+var client_sqs_1 = require("@aws-sdk/client-sqs");
+var path_1 = require("path");
+var debug_1 = __importDefault(require("debug"));
+var debug = (0, debug_1.default)('qdone:sqs');
+/**
+ * Utility function to return an instantiated, shared SQSClient.
+ */
+var client;
+function getSQSClient() {
+    if (client)
+        return client;
+    client = new client_sqs_1.SQSClient();
+    return client;
+}
+exports.getSQSClient = getSQSClient;
+/**
+ * Utility function to set the client explicitly, used in testing.
+ */
+function setSQSClient(explicitClient) {
+    client = explicitClient;
+}
+exports.setSQSClient = setSQSClient;
+/**
+ * Returns qrls for queues matching the given prefix and regex.
+ */
+function getMatchingQueues(prefix, regex) {
+    return __awaiter(this, void 0, void 0, function () {
+        function processQueues(nextToken) {
+            return __awaiter(this, void 0, void 0, function () {
+                var command, result, qrls, nextToken2, _a, _b, _c;
+                return __generator(this, function (_d) {
+                    switch (_d.label) {
+                        case 0:
+                            if (nextToken)
+                                input.NextToken = nextToken;
+                            command = new client_sqs_1.ListQueuesCommand(input);
+                            return [4 /*yield*/, client.send(command)
+                                // debug({ result })
+                            ];
+                        case 1:
+                            result = _d.sent();
+                            qrls = result.QueueUrls, nextToken2 = result.NextToken;
+                            _b = (_a = (qrls || []).filter(function (q) { return regex.test(q); })).concat;
+                            if (!nextToken2) return [3 /*break*/, 3];
+                            return [4 /*yield*/, processQueues(nextToken2)];
+                        case 2:
+                            _c = _d.sent();
+                            return [3 /*break*/, 4];
+                        case 3:
+                            _c = [];
+                            _d.label = 4;
+                        case 4: 
+                        // debug({ qrls, nextToken2 })
+                        return [2 /*return*/, _b.apply(_a, [_c])];
+                    }
+                });
+            });
+        }
+        var input, client;
+        return __generator(this, function (_a) {
+            input = { QueueNamePrefix: prefix, MaxResults: 1000 };
+            client = getSQSClient();
+            return [2 /*return*/, processQueues()];
+        });
+    });
+}
+exports.getMatchingQueues = getMatchingQueues;
+/**
+ * Gets attributes on every queue in parallel.
+ */
+function getQueueAttributes(qrls) {
+    return __awaiter(this, void 0, void 0, function () {
+        var promises, _loop_1, _i, qrls_1, qrl;
+        var _this = this;
+        return __generator(this, function (_a) {
+            promises = [];
+            _loop_1 = function (qrl) {
+                var input = {
+                    QueueUrl: qrl,
+                    AttributeNames: [
+                        'ApproximateNumberOfMessages',
+                        'ApproximateNumberOfMessagesNotVisible',
+                        'ApproximateNumberOfMessagesDelayed'
+                    ]
+                };
+                var command = new client_sqs_1.GetQueueAttributesCommand(input);
+                // debug({ input, command })
+                promises.push((function () { return __awaiter(_this, void 0, void 0, function () {
+                    var queue, result;
+                    return __generator(this, function (_a) {
+                        switch (_a.label) {
+                            case 0:
+                                queue = (0, path_1.basename)(qrl);
+                                return [4 /*yield*/, client.send(command)
+                                    // debug({ queue, result })
+                                ];
+                            case 1:
+                                result = _a.sent();
+                                // debug({ queue, result })
+                                return [2 /*return*/, { queue: queue, result: result }];
+                        }
+                    });
+                }); })());
+            };
+            // debug({ qrls })
+            for (_i = 0, qrls_1 = qrls; _i < qrls_1.length; _i++) {
+                qrl = qrls_1[_i];
+                _loop_1(qrl);
+            }
+            return [2 /*return*/, Promise.all(promises)];
+        });
+    });
+}
+exports.getQueueAttributes = getQueueAttributes;
+debug('loaded');