qdone 2.0.19-alpha → 2.0.20-alpha

package/commonjs/index.js

@@ -1,9 +1,11 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.requestShutdown = exports.processMessages = exports.enqueueBatch = exports.enqueue = void 0;
+exports.monitor = exports.requestShutdown = exports.processMessages = exports.enqueueBatch = exports.enqueue = void 0;
 var enqueue_js_1 = require("./src/enqueue.js");
 Object.defineProperty(exports, "enqueue", { enumerable: true, get: function () { return enqueue_js_1.enqueue; } });
 Object.defineProperty(exports, "enqueueBatch", { enumerable: true, get: function () { return enqueue_js_1.enqueueBatch; } });
 var consumer_js_1 = require("./src/consumer.js");
 Object.defineProperty(exports, "processMessages", { enumerable: true, get: function () { return consumer_js_1.processMessages; } });
 Object.defineProperty(exports, "requestShutdown", { enumerable: true, get: function () { return consumer_js_1.requestShutdown; } });
+var monitor_js_1 = require("./src/monitor.js");
+Object.defineProperty(exports, "monitor", { enumerable: true, get: function () { return monitor_js_1.monitor; } });

package/commonjs/src/monitor.js

@@ -0,0 +1,148 @@
+"use strict";
+/**
+ * Multi-queue monitoring functionaliry
+ */
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+var __generator = (this && this.__generator) || function (thisArg, body) {
+    var _ = { label: 0, sent: function() { if (t[0] & 1) throw t[1]; return t[1]; }, trys: [], ops: [] }, f, y, t, g;
+    return g = { next: verb(0), "throw": verb(1), "return": verb(2) }, typeof Symbol === "function" && (g[Symbol.iterator] = function() { return this; }), g;
+    function verb(n) { return function (v) { return step([n, v]); }; }
+    function step(op) {
+        if (f) throw new TypeError("Generator is already executing.");
+        while (g && (g = 0, op[0] && (_ = 0)), _) try {
+            if (f = 1, y && (t = op[0] & 2 ? y["return"] : op[0] ? y["throw"] || ((t = y["return"]) && t.call(y), 0) : y.next) && !(t = t.call(y, op[1])).done) return t;
+            if (y = 0, t) op = [op[0] & 2, t.value];
+            switch (op[0]) {
+                case 0: case 1: t = op; break;
+                case 4: _.label++; return { value: op[1], done: false };
+                case 5: _.label++; y = op[1]; op = [0]; continue;
+                case 7: op = _.ops.pop(); _.trys.pop(); continue;
+                default:
+                    if (!(t = _.trys, t = t.length > 0 && t[t.length - 1]) && (op[0] === 6 || op[0] === 2)) { _ = 0; continue; }
+                    if (op[0] === 3 && (!t || (op[1] > t[0] && op[1] < t[3]))) { _.label = op[1]; break; }
+                    if (op[0] === 6 && _.label < t[1]) { _.label = t[1]; t = op; break; }
+                    if (t && _.label < t[2]) { _.label = t[2]; _.ops.push(op); break; }
+                    if (t[2]) _.ops.pop();
+                    _.trys.pop(); continue;
+            }
+            op = body.call(thisArg, _);
+        } catch (e) { op = [6, e]; y = 0; } finally { f = t = 0; }
+        if (op[0] & 5) throw op[1]; return { value: op[0] ? op[1] : void 0, done: true };
+    }
+};
+var __spreadArray = (this && this.__spreadArray) || function (to, from, pack) {
+    if (pack || arguments.length === 2) for (var i = 0, l = from.length, ar; i < l; i++) {
+        if (ar || !(i in from)) {
+            if (!ar) ar = Array.prototype.slice.call(from, 0, i);
+            ar[i] = from[i];
+        }
+    }
+    return to.concat(ar || Array.prototype.slice.call(from));
+};
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.getAggregateData = exports.interpretWildcard = exports.monitor = void 0;
+var sqs_js_1 = require("./sqs.js");
+var cloudWatch_js_1 = require("./cloudWatch.js");
+var defaults_js_1 = require("./defaults.js");
+var debug_1 = __importDefault(require("debug"));
+var debug = (0, debug_1.default)('sd:utils:qmonitor:index');
+/**
+ * Splits a queue name with a single wildcard into prefix and suffix regex.
+ */
+function monitor(queue, save, options) {
+    return __awaiter(this, void 0, void 0, function () {
+        var opt, data;
+        return __generator(this, function (_a) {
+            switch (_a.label) {
+                case 0:
+                    opt = (0, defaults_js_1.getOptionsWithDefaults)(options);
+                    return [4 /*yield*/, getAggregateData(queue)];
+                case 1:
+                    data = _a.sent();
+                    console.log(data);
+                    if (!save) return [3 /*break*/, 3];
+                    if (opt.verbose)
+                        process.stderr.write('Saving to CloudWatch...');
+                    return [4 /*yield*/, (0, cloudWatch_js_1.putAggregateData)(data)];
+                case 2:
+                    _a.sent();
+                    if (opt.verbose)
+                        process.stderr.write('done\n');
+                    _a.label = 3;
+                case 3: return [2 /*return*/];
+            }
+        });
+    });
+}
+exports.monitor = monitor;
+/**
+ * Splits a queue name with a single wildcard into prefix and suffix regex.
+ */
+function interpretWildcard(queueName) {
+    var _a = queueName.split('*'), prefix = _a[0], suffix = _a[1];
+    // Strip anything that could cause backreferences
+    var safeSuffix = (suffix || '').replace(/[^a-zA-Z0-9_.]+/g, '').replace(/\./g, '\\.');
+    var suffixRegex = new RegExp("".concat(safeSuffix, "$"));
+    // debug({ prefix, suffix, safeSuffix, suffixRegex })
+    return { prefix: prefix, suffix: suffix, safeSuffix: safeSuffix, suffixRegex: suffixRegex };
+}
+exports.interpretWildcard = interpretWildcard;
+/**
+ * Aggregates inmportant attributes across queues and reports a summary.
+ * Attributes:
+ *  - ApproximateNumberOfMessages: Sum
+ *  - ApproximateNumberOfMessagesDelayed: Sum
+ *  - ApproximateNumberOfMessagesNotVisible: Sum
+ */
+function getAggregateData(queueName) {
+    return __awaiter(this, void 0, void 0, function () {
+        var _a, prefix, suffixRegex, qrls, data, total, _i, data_1, _b, queue, result, key, newAtrribute;
+        return __generator(this, function (_c) {
+            switch (_c.label) {
+                case 0:
+                    _a = interpretWildcard(queueName), prefix = _a.prefix, suffixRegex = _a.suffixRegex;
+                    return [4 /*yield*/, (0, sqs_js_1.getMatchingQueues)(prefix, suffixRegex)
+                        // debug({ qrls })
+                    ];
+                case 1:
+                    qrls = _c.sent();
+                    return [4 /*yield*/, (0, sqs_js_1.getQueueAttributes)(qrls)
+                        // debug({ data })
+                    ];
+                case 2:
+                    data = _c.sent();
+                    total = { totalQueues: 0, contributingQueueNames: new Set() };
+                    for (_i = 0, data_1 = data; _i < data_1.length; _i++) {
+                        _b = data_1[_i], queue = _b.queue, result = _b.result;
+                        // debug({ row })
+                        total.totalQueues++;
+                        for (key in result.Attributes) {
+                            newAtrribute = parseInt(result.Attributes[key], 10);
+                            if (newAtrribute > 0) {
+                                total.contributingQueueNames.add(queue);
+                                total[key] = (total[key] || 0) + newAtrribute;
+                            }
+                        }
+                    }
+                    // debug({ total })
+                    // convert set to array
+                    total.contributingQueueNames = __spreadArray([], total.contributingQueueNames.values(), true);
+                    total.queueName = queueName;
+                    return [2 /*return*/, total];
+            }
+        });
+    });
+}
+exports.getAggregateData = getAggregateData;
+debug('loaded');

package/commonjs/src/scheduler/jobExecutor.js

@@ -47,7 +47,6 @@ exports.JobExecutor = void 0;
 var client_sqs_1 = require("@aws-sdk/client-sqs");
 var chalk_1 = __importDefault(require("chalk"));
 var debug_1 = __importDefault(require("debug"));
-var sentry_js_1 = require("../sentry.js");
 var sqs_js_1 = require("../sqs.js");
 var debug = (0, debug_1.default)('qdone:jobExecutor');
 var maxJobSeconds = 12 * 60 * 60;

package/commonjs/src/sqs.js

@@ -131,18 +131,38 @@ function getQueueAttributes(qrls) {
                 var command = new client_sqs_1.GetQueueAttributesCommand(input);
                 // debug({ input, command })
                 promises.push((function () { return __awaiter(_this, void 0, void 0, function () {
-                    var queue, result;
+                    var queue, result, e_1;
                     return __generator(this, function (_a) {
                         switch (_a.label) {
                             case 0:
                                 queue = (0, path_1.basename)(qrl);
+                                _a.label = 1;
+                            case 1:
+                                _a.trys.push([1, 3, , 4]);
                                 return [4 /*yield*/, client.send(command)
                                     // debug({ queue, result })
                                 ];
-                            case 1:
+                            case 2:
                                 result = _a.sent();
                                 // debug({ queue, result })
                                 return [2 /*return*/, { queue: queue, result: result }];
+                            case 3:
+                                e_1 = _a.sent();
+                                if (e_1 instanceof client_sqs_1.QueueDoesNotExist) {
+                                    // For queues that have been deleted in the meantime for whatever
+                                    // reason, just show as having no messages instead of failing the
+                                    // whole batch
+                                    return [2 /*return*/, {
+                                            queue: queue,
+                                            Attributes: {
+                                                ApproximateNumberOfMessages: '0',
+                                                ApproximateNumberOfMessagesNotVisible: '0',
+                                                ApproximateNumberOfMessagesDelayed: '0'
+                                            }
+                                        }];
+                                }
+                                throw e_1;
+                            case 4: return [2 /*return*/];
                         }
                     });
                 }); })());

package/index.js

@@ -1,2 +1,3 @@
 export { enqueue, enqueueBatch } from './src/enqueue.js'
 export { processMessages, requestShutdown } from './src/consumer.js'
+export { monitor } from './src/monitor.js'

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "qdone",
-  "version": "2.0.19-alpha",
+  "version": "2.0.20-alpha",
   "description": "Language agnostic job queue for SQS",
   "type": "module",
   "main": "./index.js",

package/src/monitor.js

@@ -3,9 +3,25 @@
  */
 
 import { getMatchingQueues, getQueueAttributes } from './sqs.js'
+import { putAggregateData } from './cloudWatch.js'
+import { getOptionsWithDefaults } from './defaults.js'
 import Debug from 'debug'
 const debug = Debug('sd:utils:qmonitor:index')
 
+/**
+ * Splits a queue name with a single wildcard into prefix and suffix regex.
+ */
+export async function monitor (queue, save, options) {
+  const opt = getOptionsWithDefaults(options)
+  const data = await getAggregateData(queue)
+  console.log(data)
+  if (save) {
+    if (opt.verbose) process.stderr.write('Saving to CloudWatch...')
+    await putAggregateData(data)
+    if (opt.verbose) process.stderr.write('done\n')
+  }
+}
+
 /**
  * Splits a queue name with a single wildcard into prefix and suffix regex.
  */

package/src/scheduler/jobExecutor.js

@@ -8,7 +8,6 @@ import { ChangeMessageVisibilityBatchCommand, DeleteMessageBatchCommand } from '
 import chalk from 'chalk'
 import Debug from 'debug'
 
-import { withSentry } from '../sentry.js'
 import { getSQSClient } from '../sqs.js'
 
 const debug = Debug('qdone:jobExecutor')

package/src/sqs.js

@@ -2,7 +2,7 @@
  * Functions that deal with SQS
  */
 
-import { SQSClient, ListQueuesCommand, GetQueueAttributesCommand } from '@aws-sdk/client-sqs'
+import { SQSClient, ListQueuesCommand, GetQueueAttributesCommand, QueueDoesNotExist } from '@aws-sdk/client-sqs'
 import { basename } from 'path'
 import Debug from 'debug'
 const debug = Debug('qdone:sqs')
@@ -62,9 +62,26 @@ export async function getQueueAttributes (qrls) {
     // debug({ input, command })
     promises.push((async () => {
       const queue = basename(qrl)
-      const result = await client.send(command)
-      // debug({ queue, result })
-      return { queue, result }
+      try {
+        const result = await client.send(command)
+        // debug({ queue, result })
+        return { queue, result }
+      } catch (e) {
+        if (e instanceof QueueDoesNotExist) {
+          // For queues that have been deleted in the meantime for whatever
+          // reason, just show as having no messages instead of failing the
+          // whole batch
+          return {
+            queue,
+            Attributes: {
+              ApproximateNumberOfMessages: '0',
+              ApproximateNumberOfMessagesNotVisible: '0',
+              ApproximateNumberOfMessagesDelayed: '0'
+            }
+          }
+        }
+        throw e
+      }
     })())
   }
   return Promise.all(promises)

package/commonjs/src/sentry.js

@@ -1,111 +0,0 @@
-"use strict";
-/**
- * Routines for handling Sentry instrumentation
- */
-var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
-    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
-    return new (P || (P = Promise))(function (resolve, reject) {
-        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
-        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
-        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
-        step((generator = generator.apply(thisArg, _arguments || [])).next());
-    });
-};
-var __generator = (this && this.__generator) || function (thisArg, body) {
-    var _ = { label: 0, sent: function() { if (t[0] & 1) throw t[1]; return t[1]; }, trys: [], ops: [] }, f, y, t, g;
-    return g = { next: verb(0), "throw": verb(1), "return": verb(2) }, typeof Symbol === "function" && (g[Symbol.iterator] = function() { return this; }), g;
-    function verb(n) { return function (v) { return step([n, v]); }; }
-    function step(op) {
-        if (f) throw new TypeError("Generator is already executing.");
-        while (g && (g = 0, op[0] && (_ = 0)), _) try {
-            if (f = 1, y && (t = op[0] & 2 ? y["return"] : op[0] ? y["throw"] || ((t = y["return"]) && t.call(y), 0) : y.next) && !(t = t.call(y, op[1])).done) return t;
-            if (y = 0, t) op = [op[0] & 2, t.value];
-            switch (op[0]) {
-                case 0: case 1: t = op; break;
-                case 4: _.label++; return { value: op[1], done: false };
-                case 5: _.label++; y = op[1]; op = [0]; continue;
-                case 7: op = _.ops.pop(); _.trys.pop(); continue;
-                default:
-                    if (!(t = _.trys, t = t.length > 0 && t[t.length - 1]) && (op[0] === 6 || op[0] === 2)) { _ = 0; continue; }
-                    if (op[0] === 3 && (!t || (op[1] > t[0] && op[1] < t[3]))) { _.label = op[1]; break; }
-                    if (op[0] === 6 && _.label < t[1]) { _.label = t[1]; t = op; break; }
-                    if (t && _.label < t[2]) { _.label = t[2]; _.ops.push(op); break; }
-                    if (t[2]) _.ops.pop();
-                    _.trys.pop(); continue;
-            }
-            op = body.call(thisArg, _);
-        } catch (e) { op = [6, e]; y = 0; } finally { f = t = 0; }
-        if (op[0] & 5) throw op[1]; return { value: op[0] ? op[1] : void 0, done: true };
-    }
-};
-var __importDefault = (this && this.__importDefault) || function (mod) {
-    return (mod && mod.__esModule) ? mod : { "default": mod };
-};
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.withSentry = void 0;
-var debug_1 = __importDefault(require("debug"));
-var node_1 = require("@sentry/node");
-var debug = (0, debug_1.default)('qdone:sentry');
-var sentryWasInit = false;
-function withSentry(callback, opt, contexts) {
-    return __awaiter(this, void 0, void 0, function () {
-        var result, err_1;
-        return __generator(this, function (_a) {
-            switch (_a.label) {
-                case 0:
-                    debug({ withSentry: { callback: callback, opt: opt, contexts: contexts } });
-                    // Bail if sentry isn't enabled
-                    if (!opt.sentryDsn)
-                        return [2 /*return*/, callback()
-                            // Init sentry if it's not already
-                        ];
-                    // Init sentry if it's not already
-                    if (!sentryWasInit) {
-                        (0, node_1.init)({ dsn: opt.sentryDsn, traceSampleRate: 0 });
-                        sentryWasInit = true;
-                    }
-                    _a.label = 1;
-                case 1:
-                    _a.trys.push([1, 3, , 5]);
-                    return [4 /*yield*/, callback()];
-                case 2:
-                    result = _a.sent();
-                    debug({ result: result });
-                    return [2 /*return*/, result];
-                case 3:
-                    err_1 = _a.sent();
-                    debug({ err: err_1 });
-                    return [4 /*yield*/, (0, node_1.withScope)(function (scope) {
-                            return __awaiter(this, void 0, void 0, function () {
-                                var key, stdout, stderr, sentryResult;
-                                return __generator(this, function (_a) {
-                                    switch (_a.label) {
-                                        case 0:
-                                            scope.setContext('opt', opt);
-                                            if (contexts instanceof Object) {
-                                                for (key in contexts)
-                                                    scope.setContext(key, contexts[key]);
-                                                if (err_1.stdout && err_1.stderr) {
-                                                    stdout = err_1.stdout, stderr = err_1.stderr;
-                                                    scope.setContext('IO', { stdout: stdout, stderr: stderr });
-                                                }
-                                            }
-                                            return [4 /*yield*/, (0, node_1.captureException)(err_1)];
-                                        case 1:
-                                            sentryResult = _a.sent();
-                                            debug({ sentryResult: sentryResult });
-                                            return [2 /*return*/];
-                                    }
-                                });
-                            });
-                        })];
-                case 4:
-                    _a.sent();
-                    throw err_1;
-                case 5: return [2 /*return*/];
-            }
-        });
-    });
-}
-exports.withSentry = withSentry;
-debug('loaded');