qdone 2.0.14-alpha → 2.0.16-alpha

package/commonjs/src/consumer.js

@@ -134,7 +134,7 @@ function processMessages(queues, callback, options) {
                         lastLatency = latency;
                     });
                     jobExecutor = new jobExecutor_js_1.JobExecutor(opt);
-                    queueManager = new queueManager_js_1.QueueManager(opt, queues);
+                    queueManager = new queueManager_js_1.QueueManager(opt, queues, 60);
                     delay = function (ms) { return new Promise(function (resolve) {
                         delayTimeout = setTimeout(resolve, ms);
                     }); };
@@ -206,15 +206,17 @@ function processMessages(queues, callback, options) {
                     _c.label = 1;
                 case 1:
                     if (!!shutdownRequested) return [3 /*break*/, 3];
-                    allowedJobs = opt.maxConcurrentJobs - jobExecutor.activeJobCount() - maxReturnCount;
+                    allowedJobs = Math.max(0, opt.maxConcurrentJobs - jobExecutor.activeJobCount() - maxReturnCount);
                     maxLatency = 100;
-                    latency = systemMonitor.getLatency().setTimeout;
+                    latency = systemMonitor.getLatency() || 10;
                     latencyFactor = 1 - Math.abs(Math.min(latency / maxLatency, 1)) // 0 if latency is at max, 1 if latency 0
                     ;
                     targetJobs = Math.round(allowedJobs * latencyFactor);
                     jobsLeft = targetJobs;
+                    debug({ jobCount: jobExecutor.activeJobCount(), maxReturnCount: maxReturnCount, allowedJobs: allowedJobs, maxLatency: maxLatency, latency: latency, latencyFactor: latencyFactor, targetJobs: targetJobs, activeQrls: activeQrls });
                     for (_i = 0, _a = queueManager.getPairs(); _i < _a.length; _i++) {
                         _b = _a[_i], qname = _b.qname, qrl = _b.qrl;
+                        debug({ evaluating: { qname: qname, qrl: qrl, jobsLeft: jobsLeft, activeQrlsHasQrl: activeQrls.has(qrl) } });
                         if (jobsLeft <= 0 || activeQrls.has(qrl))
                             continue;
                         maxMessages = Math.min(10, jobsLeft);
@@ -223,7 +225,7 @@ function processMessages(queues, callback, options) {
                         if (opt.verbose) {
                             console.error(chalk_1.default.blue('Listening on: '), qname);
                         }
-                        // debug({ listenedTo: { qname, maxMessages, jobsLeft } })
+                        debug({ listenedTo: { qname: qname, maxMessages: maxMessages, jobsLeft: jobsLeft } });
                     }
                     return [4 /*yield*/, delay(1000)];
                 case 2:

package/commonjs/src/scheduler/jobExecutor.js

@@ -47,6 +47,7 @@ exports.JobExecutor = void 0;
 var client_sqs_1 = require("@aws-sdk/client-sqs");
 var chalk_1 = __importDefault(require("chalk"));
 var debug_1 = __importDefault(require("debug"));
+var sentry_js_1 = require("../sentry.js");
 var sqs_js_1 = require("../sqs.js");
 var debug = (0, debug_1.default)('qdone:jobExecutor');
 var maxJobSeconds = 12 * 60 * 60;
@@ -268,7 +269,7 @@ var JobExecutor = /** @class */ (function () {
     };
     JobExecutor.prototype.executeJob = function (message, callback, qname, qrl) {
         return __awaiter(this, void 0, void 0, function () {
-            var payload, visibilityTimeout, job, oldJob, e, queue, result, err_1;
+            var payload, visibilityTimeout, job, oldJob, e, queue_1, result, err_1;
             return __generator(this, function (_a) {
                 switch (_a.label) {
                     case 0:
@@ -315,8 +316,8 @@ var JobExecutor = /** @class */ (function () {
                         _a.label = 1;
                     case 1:
                         _a.trys.push([1, 3, , 4]);
-                        queue = qname.slice(this.opt.prefix.length);
-                        return [4 /*yield*/, callback(queue, payload)];
+                        queue_1 = qname.slice(this.opt.prefix.length);
+                        return [4 /*yield*/, (0, sentry_js_1.withSentry)(function () { return callback(queue_1, payload); }, this.opt, { job: job })];
                     case 2:
                         result = _a.sent();
                         debug('executeJob callback finished', { payload: payload, result: result });
@@ -340,7 +341,6 @@ var JobExecutor = /** @class */ (function () {
                         return [3 /*break*/, 4];
                     case 3:
                         err_1 = _a.sent();
-                        // debug('exec.catch', err)
                         // Fail path for job execution
                         if (this.opt.verbose) {
                             console.error(chalk_1.default.red('FAILED'), message.Body);

package/commonjs/src/scheduler/queueManager.js

@@ -204,7 +204,12 @@ var QueueManager = /** @class */ (function () {
         return pair;
     };
     QueueManager.prototype.getPairs = function () {
-        return this.selectedPairs;
+        var _this = this;
+        var now = new Date();
+        return this.selectedPairs.filter(function (_a) {
+            var qname = _a.qname, qrl = _a.qrl;
+            return !_this.keepInIcehouse(qrl, now);
+        });
     };
     QueueManager.prototype.shutdown = function () {
         return __awaiter(this, void 0, void 0, function () {

package/commonjs/src/sentry.js

@@ -0,0 +1,111 @@
+"use strict";
+/**
+ * Routines for handling Sentry instrumentation
+ */
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+var __generator = (this && this.__generator) || function (thisArg, body) {
+    var _ = { label: 0, sent: function() { if (t[0] & 1) throw t[1]; return t[1]; }, trys: [], ops: [] }, f, y, t, g;
+    return g = { next: verb(0), "throw": verb(1), "return": verb(2) }, typeof Symbol === "function" && (g[Symbol.iterator] = function() { return this; }), g;
+    function verb(n) { return function (v) { return step([n, v]); }; }
+    function step(op) {
+        if (f) throw new TypeError("Generator is already executing.");
+        while (g && (g = 0, op[0] && (_ = 0)), _) try {
+            if (f = 1, y && (t = op[0] & 2 ? y["return"] : op[0] ? y["throw"] || ((t = y["return"]) && t.call(y), 0) : y.next) && !(t = t.call(y, op[1])).done) return t;
+            if (y = 0, t) op = [op[0] & 2, t.value];
+            switch (op[0]) {
+                case 0: case 1: t = op; break;
+                case 4: _.label++; return { value: op[1], done: false };
+                case 5: _.label++; y = op[1]; op = [0]; continue;
+                case 7: op = _.ops.pop(); _.trys.pop(); continue;
+                default:
+                    if (!(t = _.trys, t = t.length > 0 && t[t.length - 1]) && (op[0] === 6 || op[0] === 2)) { _ = 0; continue; }
+                    if (op[0] === 3 && (!t || (op[1] > t[0] && op[1] < t[3]))) { _.label = op[1]; break; }
+                    if (op[0] === 6 && _.label < t[1]) { _.label = t[1]; t = op; break; }
+                    if (t && _.label < t[2]) { _.label = t[2]; _.ops.push(op); break; }
+                    if (t[2]) _.ops.pop();
+                    _.trys.pop(); continue;
+            }
+            op = body.call(thisArg, _);
+        } catch (e) { op = [6, e]; y = 0; } finally { f = t = 0; }
+        if (op[0] & 5) throw op[1]; return { value: op[0] ? op[1] : void 0, done: true };
+    }
+};
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.withSentry = void 0;
+var debug_1 = __importDefault(require("debug"));
+var node_1 = require("@sentry/node");
+var debug = (0, debug_1.default)('qdone:sentry');
+var sentryWasInit = false;
+function withSentry(callback, opt, contexts) {
+    return __awaiter(this, void 0, void 0, function () {
+        var result, err_1;
+        return __generator(this, function (_a) {
+            switch (_a.label) {
+                case 0:
+                    debug({ withSentry: { callback: callback, opt: opt, contexts: contexts } });
+                    // Bail if sentry isn't enabled
+                    if (!opt.sentryDsn)
+                        return [2 /*return*/, callback()
+                            // Init sentry if it's not already
+                        ];
+                    // Init sentry if it's not already
+                    if (!sentryWasInit) {
+                        (0, node_1.init)({ dsn: opt.sentryDsn, traceSampleRate: 0 });
+                        sentryWasInit = true;
+                    }
+                    _a.label = 1;
+                case 1:
+                    _a.trys.push([1, 3, , 5]);
+                    return [4 /*yield*/, callback()];
+                case 2:
+                    result = _a.sent();
+                    debug({ result: result });
+                    return [2 /*return*/, result];
+                case 3:
+                    err_1 = _a.sent();
+                    debug({ err: err_1 });
+                    return [4 /*yield*/, (0, node_1.withScope)(function (scope) {
+                            return __awaiter(this, void 0, void 0, function () {
+                                var key, stdout, stderr, sentryResult;
+                                return __generator(this, function (_a) {
+                                    switch (_a.label) {
+                                        case 0:
+                                            scope.setContext('opt', opt);
+                                            if (contexts instanceof Object) {
+                                                for (key in contexts)
+                                                    scope.setContext(key, contexts[key]);
+                                                if (err_1.stdout && err_1.stderr) {
+                                                    stdout = err_1.stdout, stderr = err_1.stderr;
+                                                    scope.setContext('IO', { stdout: stdout, stderr: stderr });
+                                                }
+                                            }
+                                            return [4 /*yield*/, (0, node_1.captureException)(err_1)];
+                                        case 1:
+                                            sentryResult = _a.sent();
+                                            debug({ sentryResult: sentryResult });
+                                            return [2 /*return*/];
+                                    }
+                                });
+                            });
+                        })];
+                case 4:
+                    _a.sent();
+                    throw err_1;
+                case 5: return [2 /*return*/];
+            }
+        });
+    });
+}
+exports.withSentry = withSentry;
+debug('loaded');

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "qdone",
-  "version": "2.0.14-alpha",
+  "version": "2.0.16-alpha",
   "description": "Language agnostic job queue for SQS",
   "type": "module",
   "main": "./index.js",

package/src/consumer.js

@@ -60,7 +60,7 @@ export async function processMessages (queues, callback, options) {
     lastLatency = latency
   })
   const jobExecutor = new JobExecutor(opt)
-  const queueManager = new QueueManager(opt, queues)
+  const queueManager = new QueueManager(opt, queues, 60)
   // debug({ systemMonitor, jobExecutor, queueManager })
 
   // This delay function keeps a timeout reference around so it can be
@@ -113,15 +113,15 @@ export async function processMessages (queues, callback, options) {
 
   while (!shutdownRequested) { // eslint-disable-line
     // Figure out how we are running
-    const allowedJobs = opt.maxConcurrentJobs - jobExecutor.activeJobCount() - maxReturnCount
+    const allowedJobs = Math.max(0, opt.maxConcurrentJobs - jobExecutor.activeJobCount() - maxReturnCount)
     const maxLatency = 100
-    const latency = systemMonitor.getLatency().setTimeout
+    const latency = systemMonitor.getLatency() || 10
     const latencyFactor = 1 - Math.abs(Math.min(latency / maxLatency, 1)) // 0 if latency is at max, 1 if latency 0
     const targetJobs = Math.round(allowedJobs * latencyFactor)
-    // debug({ allowedJobs, maxLatency, latency, latencyFactor, targetJobs, activeQrls })
-
     let jobsLeft = targetJobs
+    debug({ jobCount: jobExecutor.activeJobCount(), maxReturnCount, allowedJobs, maxLatency, latency, latencyFactor, targetJobs, activeQrls })
     for (const { qname, qrl } of queueManager.getPairs()) {
+      debug({ evaluating: { qname, qrl, jobsLeft, activeQrlsHasQrl: activeQrls.has(qrl) } })
       if (jobsLeft <= 0 || activeQrls.has(qrl)) continue
       const maxMessages = Math.min(10, jobsLeft)
       listen(qname, qrl, maxMessages)
@@ -129,7 +129,7 @@ export async function processMessages (queues, callback, options) {
       if (opt.verbose) {
         console.error(chalk.blue('Listening on: '), qname)
       }
-      // debug({ listenedTo: { qname, maxMessages, jobsLeft } })
+      debug({ listenedTo: { qname, maxMessages, jobsLeft } })
     }
     await delay(1000)
   }

package/src/scheduler/jobExecutor.js

@@ -3,13 +3,12 @@
  * their visibility timeouts and deleting them when they are successful.
  */
 
-import {
-  ChangeMessageVisibilityBatchCommand,
-  DeleteMessageBatchCommand
-} from '@aws-sdk/client-sqs'
+import { ChangeMessageVisibilityBatchCommand, DeleteMessageBatchCommand } from '@aws-sdk/client-sqs'
+
 import chalk from 'chalk'
 import Debug from 'debug'
 
+import { withSentry } from '../sentry.js'
 import { getSQSClient } from '../sqs.js'
 
 const debug = Debug('qdone:jobExecutor')
@@ -243,7 +242,7 @@ export class JobExecutor {
     // Execute job
     try {
       const queue = qname.slice(this.opt.prefix.length)
-      const result = await callback(queue, payload)
+      const result = await withSentry(() => callback(queue, payload), this.opt, { job })
       debug('executeJob callback finished', { payload, result })
       if (this.opt.verbose) {
         console.error(chalk.green('SUCCESS'), message.Body)
@@ -263,7 +262,6 @@ export class JobExecutor {
       }
       this.stats.jobsSucceeded++
     } catch (err) {
-      // debug('exec.catch', err)
       // Fail path for job execution
       if (this.opt.verbose) {
         console.error(chalk.red('FAILED'), message.Body)

package/src/scheduler/queueManager.js

@@ -140,7 +140,8 @@ export class QueueManager {
   }
 
   getPairs () {
-    return this.selectedPairs
+    const now = new Date()
+    return this.selectedPairs.filter(({ qname, qrl }) => !this.keepInIcehouse(qrl, now))
   }
 
   async shutdown () {

package/src/sentry.js

@@ -3,12 +3,13 @@
  */
 
 import Debug from 'debug'
-import { init, captureException } from '@sentry/node'
+import { init, withScope, captureException } from '@sentry/node'
 
 const debug = Debug('qdone:sentry')
 
 let sentryWasInit = false
-export async function withSentry (callback, opt) {
+export async function withSentry (callback, opt, contexts) {
+  debug({ withSentry: { callback, opt, contexts } })
   // Bail if sentry isn't enabled
   if (!opt.sentryDsn) return callback()
 
@@ -23,8 +24,20 @@ export async function withSentry (callback, opt) {
     return result
   } catch (err) {
     debug({ err })
-    const sentryResult = await captureException(err)
-    debug({ sentryResult })
+    await withScope(async function (scope) {
+      scope.setContext('opt', opt)
+      if (contexts instanceof Object) {
+        for (const key in contexts) scope.setContext(key, contexts[key])
+        if (err.stdout && err.stderr) {
+          const { stdout, stderr } = err
+          scope.setContext('IO', { stdout, stderr })
+        }
+      }
+      const sentryResult = await captureException(err)
+      debug({ sentryResult })
+    })
     throw err
   }
 }
+
+debug('loaded')