qdone 2.0.29-alpha → 2.0.30-alpha

package/commonjs/src/monitor.js

@@ -2,100 +2,42 @@
 /**
  * Multi-queue monitoring functionaliry
  */
-var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
-    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
-    return new (P || (P = Promise))(function (resolve, reject) {
-        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
-        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
-        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
-        step((generator = generator.apply(thisArg, _arguments || [])).next());
-    });
-};
-var __generator = (this && this.__generator) || function (thisArg, body) {
-    var _ = { label: 0, sent: function() { if (t[0] & 1) throw t[1]; return t[1]; }, trys: [], ops: [] }, f, y, t, g;
-    return g = { next: verb(0), "throw": verb(1), "return": verb(2) }, typeof Symbol === "function" && (g[Symbol.iterator] = function() { return this; }), g;
-    function verb(n) { return function (v) { return step([n, v]); }; }
-    function step(op) {
-        if (f) throw new TypeError("Generator is already executing.");
-        while (g && (g = 0, op[0] && (_ = 0)), _) try {
-            if (f = 1, y && (t = op[0] & 2 ? y["return"] : op[0] ? y["throw"] || ((t = y["return"]) && t.call(y), 0) : y.next) && !(t = t.call(y, op[1])).done) return t;
-            if (y = 0, t) op = [op[0] & 2, t.value];
-            switch (op[0]) {
-                case 0: case 1: t = op; break;
-                case 4: _.label++; return { value: op[1], done: false };
-                case 5: _.label++; y = op[1]; op = [0]; continue;
-                case 7: op = _.ops.pop(); _.trys.pop(); continue;
-                default:
-                    if (!(t = _.trys, t = t.length > 0 && t[t.length - 1]) && (op[0] === 6 || op[0] === 2)) { _ = 0; continue; }
-                    if (op[0] === 3 && (!t || (op[1] > t[0] && op[1] < t[3]))) { _.label = op[1]; break; }
-                    if (op[0] === 6 && _.label < t[1]) { _.label = t[1]; t = op; break; }
-                    if (t && _.label < t[2]) { _.label = t[2]; _.ops.push(op); break; }
-                    if (t[2]) _.ops.pop();
-                    _.trys.pop(); continue;
-            }
-            op = body.call(thisArg, _);
-        } catch (e) { op = [6, e]; y = 0; } finally { f = t = 0; }
-        if (op[0] & 5) throw op[1]; return { value: op[0] ? op[1] : void 0, done: true };
-    }
-};
-var __spreadArray = (this && this.__spreadArray) || function (to, from, pack) {
-    if (pack || arguments.length === 2) for (var i = 0, l = from.length, ar; i < l; i++) {
-        if (ar || !(i in from)) {
-            if (!ar) ar = Array.prototype.slice.call(from, 0, i);
-            ar[i] = from[i];
-        }
-    }
-    return to.concat(ar || Array.prototype.slice.call(from));
-};
 var __importDefault = (this && this.__importDefault) || function (mod) {
     return (mod && mod.__esModule) ? mod : { "default": mod };
 };
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.getAggregateData = exports.interpretWildcard = exports.monitor = void 0;
-var sqs_js_1 = require("./sqs.js");
-var cloudWatch_js_1 = require("./cloudWatch.js");
-var defaults_js_1 = require("./defaults.js");
-var debug_1 = __importDefault(require("debug"));
-var debug = (0, debug_1.default)('sd:utils:qmonitor:index');
+const sqs_js_1 = require("./sqs.js");
+const cloudWatch_js_1 = require("./cloudWatch.js");
+const defaults_js_1 = require("./defaults.js");
+const debug_1 = __importDefault(require("debug"));
+const debug = (0, debug_1.default)('sd:utils:qmonitor:index');
 /**
  * Splits a queue name with a single wildcard into prefix and suffix regex.
  */
-function monitor(queue, save, options) {
-    return __awaiter(this, void 0, void 0, function () {
-        var opt, data;
-        return __generator(this, function (_a) {
-            switch (_a.label) {
-                case 0:
-                    opt = (0, defaults_js_1.getOptionsWithDefaults)(options);
-                    return [4 /*yield*/, getAggregateData(queue)];
-                case 1:
-                    data = _a.sent();
-                    console.log(data);
-                    if (!save) return [3 /*break*/, 3];
-                    if (opt.verbose)
-                        process.stderr.write('Saving to CloudWatch...');
-                    return [4 /*yield*/, (0, cloudWatch_js_1.putAggregateData)(data)];
-                case 2:
-                    _a.sent();
-                    if (opt.verbose)
-                        process.stderr.write('done\n');
-                    _a.label = 3;
-                case 3: return [2 /*return*/];
-            }
-        });
-    });
+async function monitor(queue, save, options) {
+    const opt = (0, defaults_js_1.getOptionsWithDefaults)(options);
+    const data = await getAggregateData(queue);
+    console.log(data);
+    if (save) {
+        if (opt.verbose)
+            process.stderr.write('Saving to CloudWatch...');
+        await (0, cloudWatch_js_1.putAggregateData)(data);
+        if (opt.verbose)
+            process.stderr.write('done\n');
+    }
 }
 exports.monitor = monitor;
 /**
  * Splits a queue name with a single wildcard into prefix and suffix regex.
  */
 function interpretWildcard(queueName) {
-    var _a = queueName.split('*'), prefix = _a[0], suffix = _a[1];
+    const [prefix, suffix] = queueName.split('*');
     // Strip anything that could cause backreferences
-    var safeSuffix = (suffix || '').replace(/[^a-zA-Z0-9_.]+/g, '').replace(/\./g, '\\.');
-    var suffixRegex = new RegExp("".concat(safeSuffix, "$"));
+    const safeSuffix = (suffix || '').replace(/[^a-zA-Z0-9_.]+/g, '').replace(/\./g, '\\.');
+    const suffixRegex = new RegExp(`${safeSuffix}$`);
     // debug({ prefix, suffix, safeSuffix, suffixRegex })
-    return { prefix: prefix, suffix: suffix, safeSuffix: safeSuffix, suffixRegex: suffixRegex };
+    return { prefix, suffix, safeSuffix, suffixRegex };
 }
 exports.interpretWildcard = interpretWildcard;
 /**
@@ -105,44 +47,29 @@ exports.interpretWildcard = interpretWildcard;
  *  - ApproximateNumberOfMessagesDelayed: Sum
  *  - ApproximateNumberOfMessagesNotVisible: Sum
  */
-function getAggregateData(queueName) {
-    return __awaiter(this, void 0, void 0, function () {
-        var _a, prefix, suffixRegex, qrls, data, total, _i, data_1, _b, queue, result, key, newAtrribute;
-        return __generator(this, function (_c) {
-            switch (_c.label) {
-                case 0:
-                    _a = interpretWildcard(queueName), prefix = _a.prefix, suffixRegex = _a.suffixRegex;
-                    return [4 /*yield*/, (0, sqs_js_1.getMatchingQueues)(prefix, suffixRegex)
-                        // debug({ qrls })
-                    ];
-                case 1:
-                    qrls = _c.sent();
-                    return [4 /*yield*/, (0, sqs_js_1.getQueueAttributes)(qrls)
-                        // debug({ data })
-                    ];
-                case 2:
-                    data = _c.sent();
-                    total = { totalQueues: 0, contributingQueueNames: new Set() };
-                    for (_i = 0, data_1 = data; _i < data_1.length; _i++) {
-                        _b = data_1[_i], queue = _b.queue, result = _b.result;
-                        // debug({ row })
-                        total.totalQueues++;
-                        for (key in result.Attributes) {
-                            newAtrribute = parseInt(result.Attributes[key], 10);
-                            if (newAtrribute > 0) {
-                                total.contributingQueueNames.add(queue);
-                                total[key] = (total[key] || 0) + newAtrribute;
-                            }
-                        }
-                    }
-                    // debug({ total })
-                    // convert set to array
-                    total.contributingQueueNames = __spreadArray([], total.contributingQueueNames.values(), true);
-                    total.queueName = queueName;
-                    return [2 /*return*/, total];
+async function getAggregateData(queueName) {
+    const { prefix, suffixRegex } = interpretWildcard(queueName);
+    const qrls = await (0, sqs_js_1.getMatchingQueues)(prefix, suffixRegex);
+    // debug({ qrls })
+    const data = await (0, sqs_js_1.getQueueAttributes)(qrls);
+    // debug({ data })
+    const total = { totalQueues: 0, contributingQueueNames: new Set() };
+    for (const { queue, result } of data) {
+        // debug({ row })
+        total.totalQueues++;
+        for (const key in result.Attributes) {
+            const newAtrribute = parseInt(result.Attributes[key], 10);
+            if (newAtrribute > 0) {
+                total.contributingQueueNames.add(queue);
+                total[key] = (total[key] || 0) + newAtrribute;
             }
-        });
-    });
+        }
+    }
+    // debug({ total })
+    // convert set to array
+    total.contributingQueueNames = [...total.contributingQueueNames.values()];
+    total.queueName = queueName;
+    return total;
 }
 exports.getAggregateData = getAggregateData;
 debug('loaded');

package/commonjs/src/qrlCache.js

@@ -3,64 +3,28 @@
  * A cache for maintaining queue urls. Avoids having to call the GetQueueUrl
  * api too often.
  */
-var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
-    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
-    return new (P || (P = Promise))(function (resolve, reject) {
-        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
-        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
-        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
-        step((generator = generator.apply(thisArg, _arguments || [])).next());
-    });
-};
-var __generator = (this && this.__generator) || function (thisArg, body) {
-    var _ = { label: 0, sent: function() { if (t[0] & 1) throw t[1]; return t[1]; }, trys: [], ops: [] }, f, y, t, g;
-    return g = { next: verb(0), "throw": verb(1), "return": verb(2) }, typeof Symbol === "function" && (g[Symbol.iterator] = function() { return this; }), g;
-    function verb(n) { return function (v) { return step([n, v]); }; }
-    function step(op) {
-        if (f) throw new TypeError("Generator is already executing.");
-        while (g && (g = 0, op[0] && (_ = 0)), _) try {
-            if (f = 1, y && (t = op[0] & 2 ? y["return"] : op[0] ? y["throw"] || ((t = y["return"]) && t.call(y), 0) : y.next) && !(t = t.call(y, op[1])).done) return t;
-            if (y = 0, t) op = [op[0] & 2, t.value];
-            switch (op[0]) {
-                case 0: case 1: t = op; break;
-                case 4: _.label++; return { value: op[1], done: false };
-                case 5: _.label++; y = op[1]; op = [0]; continue;
-                case 7: op = _.ops.pop(); _.trys.pop(); continue;
-                default:
-                    if (!(t = _.trys, t = t.length > 0 && t[t.length - 1]) && (op[0] === 6 || op[0] === 2)) { _ = 0; continue; }
-                    if (op[0] === 3 && (!t || (op[1] > t[0] && op[1] < t[3]))) { _.label = op[1]; break; }
-                    if (op[0] === 6 && _.label < t[1]) { _.label = t[1]; t = op; break; }
-                    if (t && _.label < t[2]) { _.label = t[2]; _.ops.push(op); break; }
-                    if (t[2]) _.ops.pop();
-                    _.trys.pop(); continue;
-            }
-            op = body.call(thisArg, _);
-        } catch (e) { op = [6, e]; y = 0; } finally { f = t = 0; }
-        if (op[0] & 5) throw op[1]; return { value: op[0] ? op[1] : void 0, done: true };
-    }
-};
 var __importDefault = (this && this.__importDefault) || function (mod) {
     return (mod && mod.__esModule) ? mod : { "default": mod };
 };
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.getQnameUrlPairs = exports.getMatchingQueues = exports.ingestQRLs = exports.qrlCacheInvalidate = exports.qrlCacheSet = exports.qrlCacheGet = exports.qrlCacheClear = exports.normalizeDLQName = exports.normalizeFailQueueName = exports.normalizeQueueName = exports.fifoSuffix = void 0;
-var url_1 = require("url");
-var path_1 = require("path");
-var client_sqs_1 = require("@aws-sdk/client-sqs");
-var sqs_js_1 = require("./sqs.js");
-var debug_1 = __importDefault(require("debug"));
-var debug = (0, debug_1.default)('qdone:qrlCache');
+const url_1 = require("url");
+const path_1 = require("path");
+const client_sqs_1 = require("@aws-sdk/client-sqs");
+const sqs_js_1 = require("./sqs.js");
+const debug_1 = __importDefault(require("debug"));
+const debug = (0, debug_1.default)('qdone:qrlCache');
 // The actual cache is shared across callers.
-var qcache = new Map();
+const qcache = new Map();
 //
 // Normalizes a queue name to end with .fifo if opt.fifo is set
 //
 exports.fifoSuffix = '.fifo';
 function normalizeQueueName(qname, opt) {
-    var hasFifo = qname.endsWith(exports.fifoSuffix);
-    var needsFifo = opt.fifo && qname.slice(-1) !== '*';
-    var needsPrefix = !qname.startsWith(opt.prefix);
-    var base = hasFifo ? qname.slice(0, -exports.fifoSuffix.length) : qname;
+    const hasFifo = qname.endsWith(exports.fifoSuffix);
+    const needsFifo = opt.fifo && qname.slice(-1) !== '*';
+    const needsPrefix = !qname.startsWith(opt.prefix);
+    const base = hasFifo ? qname.slice(0, -exports.fifoSuffix.length) : qname;
     return (needsPrefix ? opt.prefix : '') + base + (needsFifo ? exports.fifoSuffix : '');
 }
 exports.normalizeQueueName = normalizeQueueName;
@@ -68,10 +32,10 @@ exports.normalizeQueueName = normalizeQueueName;
 // Normalizes fail queue name, appending both --fail-suffix and .fifo depending on opt
 //
 function normalizeFailQueueName(fqname, opt) {
-    var newopt = Object.assign({}, opt, { fifo: false });
-    var base = normalizeQueueName(fqname, newopt); // strip fifo
-    debug({ fqname: fqname, base: base, opt: opt });
-    var needsFail = !base.endsWith(opt.failSuffix);
+    const newopt = Object.assign({}, opt, { fifo: false });
+    const base = normalizeQueueName(fqname, newopt); // strip fifo
+    debug({ fqname, base, opt });
+    const needsFail = !base.endsWith(opt.failSuffix);
     return base + (needsFail ? opt.failSuffix : '') + (opt.fifo ? exports.fifoSuffix : '');
 }
 exports.normalizeFailQueueName = normalizeFailQueueName;
@@ -79,9 +43,9 @@ exports.normalizeFailQueueName = normalizeFailQueueName;
 // Normalizes dlq queue name, appending both --dlq-suffix and .fifo depending on opt
 //
 function normalizeDLQName(dqname, opt) {
-    var newopt = Object.assign({}, opt, { fifo: false });
-    var base = normalizeQueueName(dqname, newopt); // strip fifo
-    var needsDead = !base.endsWith(opt.dlqSuffix);
+    const newopt = Object.assign({}, opt, { fifo: false });
+    const base = normalizeQueueName(dqname, newopt); // strip fifo
+    const needsDead = !base.endsWith(opt.dlqSuffix);
     return base + (needsDead ? opt.dlqSuffix : '') + (opt.fifo ? exports.fifoSuffix : '');
 }
 exports.normalizeDLQName = normalizeDLQName;
@@ -96,32 +60,22 @@ exports.qrlCacheClear = qrlCacheClear;
 // Get QRL (Queue URL)
 // Returns a promise for the queue name
 //
-function qrlCacheGet(qname) {
-    return __awaiter(this, void 0, void 0, function () {
-        var client, input, cmd, result, qrl;
-        return __generator(this, function (_a) {
-            switch (_a.label) {
-                case 0:
-                    debug('get', qname, qcache);
-                    if (qcache.has(qname))
-                        return [2 /*return*/, qcache.get(qname)];
-                    client = (0, sqs_js_1.getSQSClient)();
-                    input = { QueueName: qname };
-                    cmd = new client_sqs_1.GetQueueUrlCommand(input);
-                    return [4 /*yield*/, client.send(cmd)
-                        // debug('result', result)
-                        // if (!result) throw new Error(`No such queue ${qname}`)
-                    ];
-                case 1:
-                    result = _a.sent();
-                    qrl = result.QueueUrl;
-                    // debug('getQueueUrl returned', data)
-                    qcache.set(qname, qrl);
-                    // debug('qcache', Object.keys(qcache), 'get', qname, ' => ', qcache[qname])
-                    return [2 /*return*/, qrl];
-            }
-        });
-    });
+async function qrlCacheGet(qname) {
+    debug('get', qname, qcache);
+    if (qcache.has(qname))
+        return qcache.get(qname);
+    const client = (0, sqs_js_1.getSQSClient)();
+    const input = { QueueName: qname };
+    const cmd = new client_sqs_1.GetQueueUrlCommand(input);
+    // debug({ cmd })
+    const result = await client.send(cmd);
+    // debug('result', result)
+    // if (!result) throw new Error(`No such queue ${qname}`)
+    const { QueueUrl: qrl } = result;
+    // debug('getQueueUrl returned', data)
+    qcache.set(qname, qrl);
+    // debug('qcache', Object.keys(qcache), 'get', qname, ' => ', qcache[qname])
+    return qrl;
 }
 exports.qrlCacheGet = qrlCacheGet;
 //
@@ -146,13 +100,12 @@ exports.qrlCacheInvalidate = qrlCacheInvalidate;
 // Extracts queue names from an array of QRLs and immediately updates the cache.
 //
 function ingestQRLs(qrls) {
-    var pairs = [];
+    const pairs = [];
     debug('ingestQRLs', qrls);
-    for (var _i = 0, qrls_1 = qrls; _i < qrls_1.length; _i++) {
-        var qrl = qrls_1[_i];
-        var qname = (0, path_1.basename)((new url_1.URL(qrl)).pathname);
+    for (const qrl of qrls) {
+        const qname = (0, path_1.basename)((new url_1.URL(qrl)).pathname);
         qrlCacheSet(qname, qrl);
-        pairs.push({ qname: qname, qrl: qrl });
+        pairs.push({ qname, qrl });
         // debug('qcache', Object.keys(qcache), 'ingestQRLs', qname, ' => ', qcache[qname])
     }
     return pairs;
@@ -161,90 +114,55 @@ exports.ingestQRLs = ingestQRLs;
 /**
  * Returns qrls for queues matching the given prefix and regex.
  */
-function getMatchingQueues(prefix, nextToken) {
-    return __awaiter(this, void 0, void 0, function () {
-        var input, client, command, result, _a, qrls, keepGoing, _b, _c, _d;
-        var _e;
-        return __generator(this, function (_f) {
-            switch (_f.label) {
-                case 0:
-                    debug('getMatchingQueues', prefix, nextToken);
-                    input = { QueueNamePrefix: prefix, MaxResults: 1000 };
-                    if (nextToken)
-                        input.NextToken = nextToken;
-                    client = (0, sqs_js_1.getSQSClient)();
-                    command = new client_sqs_1.ListQueuesCommand(input);
-                    return [4 /*yield*/, client.send(command)];
-                case 1:
-                    result = _f.sent();
-                    debug({ result: result });
-                    _a = result || {}, qrls = _a.QueueUrls, keepGoing = _a.NextToken;
-                    debug({ qrls: qrls, keepGoing: keepGoing });
-                    if (!keepGoing) return [3 /*break*/, 3];
-                    _c = (_b = (_e = (qrls || [])).push).apply;
-                    _d = [_e];
-                    return [4 /*yield*/, getMatchingQueues(prefix, keepGoing)];
-                case 2:
-                    _c.apply(_b, _d.concat([_f.sent()]));
-                    _f.label = 3;
-                case 3: return [2 /*return*/, qrls || []];
-            }
-        });
-    });
+async function getMatchingQueues(prefix, nextToken) {
+    debug('getMatchingQueues', prefix, nextToken);
+    const input = { QueueNamePrefix: prefix, MaxResults: 1000 };
+    if (nextToken)
+        input.NextToken = nextToken;
+    const client = (0, sqs_js_1.getSQSClient)();
+    const command = new client_sqs_1.ListQueuesCommand(input);
+    const result = await client.send(command);
+    debug({ result });
+    const { QueueUrls: qrls, NextToken: keepGoing } = result || {};
+    debug({ qrls, keepGoing });
+    if (keepGoing)
+        (qrls || []).push(...await getMatchingQueues(prefix, keepGoing));
+    return qrls || [];
 }
 exports.getMatchingQueues = getMatchingQueues;
 //
 // Resolves into a flattened aray of {qname: ..., qrl: ...} objects.
 //
-function getQnameUrlPairs(qnames, opt) {
-    return __awaiter(this, void 0, void 0, function () {
-        var promises, results;
-        return __generator(this, function (_a) {
-            switch (_a.label) {
-                case 0:
-                    promises = qnames.map(function getQueueUrlOrUrls(qname) {
-                        return __awaiter(this, void 0, void 0, function () {
-                            var prefix, queues, filteredQueues, err_1;
-                            var _a;
-                            return __generator(this, function (_b) {
-                                switch (_b.label) {
-                                    case 0:
-                                        if (!(qname.slice(-1) === '*')) return [3 /*break*/, 2];
-                                        prefix = qname.slice(0, -1);
-                                        return [4 /*yield*/, getMatchingQueues(prefix)];
-                                    case 1:
-                                        queues = _b.sent();
-                                        debug('listQueues return', queues);
-                                        if (opt.fifo) {
-                                            filteredQueues = queues.filter(function (url) { return url.slice(-exports.fifoSuffix.length) === exports.fifoSuffix; });
-                                            return [2 /*return*/, ingestQRLs(filteredQueues)];
-                                        }
-                                        else {
-                                            return [2 /*return*/, ingestQRLs(queues)];
-                                        }
-                                        return [3 /*break*/, 5];
-                                    case 2:
-                                        _b.trys.push([2, 4, , 5]);
-                                        _a = { qname: qname };
-                                        return [4 /*yield*/, qrlCacheGet(qname)];
-                                    case 3: return [2 /*return*/, (_a.qrl = _b.sent(), _a)];
-                                    case 4:
-                                        err_1 = _b.sent();
-                                        if (!(err_1 instanceof client_sqs_1.QueueDoesNotExist))
-                                            throw err_1;
-                                        return [3 /*break*/, 5];
-                                    case 5: return [2 /*return*/];
-                                }
-                            });
-                        });
-                    });
-                    return [4 /*yield*/, Promise.all(promises)];
-                case 1:
-                    results = _a.sent();
-                    return [2 /*return*/, ([].concat.apply([], results)).filter(function (r) { return r; })];
+async function getQnameUrlPairs(qnames, opt) {
+    const promises = qnames.map(async function getQueueUrlOrUrls(qname) {
+        if (qname.slice(-1) === '*') {
+            // Wildcard queues support
+            const prefix = qname.slice(0, -1);
+            const queues = await getMatchingQueues(prefix);
+            debug('listQueues return', queues);
+            if (opt.fifo) {
+                // Remove non-fifo queues
+                const filteredQueues = queues.filter(url => url.slice(-exports.fifoSuffix.length) === exports.fifoSuffix);
+                return ingestQRLs(filteredQueues);
+            }
+            else {
+                return ingestQRLs(queues);
+            }
+        }
+        else {
+            // Normal, non wildcard queue support
+            try {
+                return { qname, qrl: await qrlCacheGet(qname) };
+            }
+            catch (err) {
+                if (!(err instanceof client_sqs_1.QueueDoesNotExist))
+                    throw err;
             }
-        });
+        }
     });
+    // Flatten nested results
+    const results = await Promise.all(promises);
+    return ([].concat.apply([], results)).filter(r => r);
 }
 exports.getQnameUrlPairs = getQnameUrlPairs;
 debug('loaded');