qdone 2.0.11-alpha → 2.0.13-alpha

package/commonjs/src/consumer.js

@@ -98,7 +98,7 @@ exports.getMessages = getMessages;
 //
 function processMessages(queues, callback, options) {
     return __awaiter(this, void 0, void 0, function () {
-        var opt, systemMonitor, jobExecutor, queueManager, maxActiveJobs, delayTimeout, delay, activeQrls, maxReturnCount, listen, allowedJobs, maxLatency, latency, latencyFactor, targetJobs, jobsLeft, _i, _a, _b, qname, qrl, maxMessages;
+        var opt, systemMonitor, jobExecutor, queueManager, delayTimeout, delay, activeQrls, maxReturnCount, listen, allowedJobs, maxLatency, latency, latencyFactor, targetJobs, jobsLeft, _i, _a, _b, qname, qrl, maxMessages;
         var _this = this;
         return __generator(this, function (_c) {
             switch (_c.label) {
@@ -108,7 +108,6 @@ function processMessages(queues, callback, options) {
                     systemMonitor = new systemMonitor_js_1.SystemMonitor(opt);
                     jobExecutor = new jobExecutor_js_1.JobExecutor(opt);
                     queueManager = new queueManager_js_1.QueueManager(opt, queues);
-                    maxActiveJobs = 100;
                     debug({ systemMonitor: systemMonitor, jobExecutor: jobExecutor, queueManager: queueManager });
                     shutdownCallbacks.push(function () {
                         systemMonitor.shutdown();
@@ -143,7 +142,7 @@ function processMessages(queues, callback, options) {
                     _c.label = 1;
                 case 1:
                     if (!!shutdownRequested) return [3 /*break*/, 3];
-                    allowedJobs = maxActiveJobs - jobExecutor.activeJobCount() - maxReturnCount;
+                    allowedJobs = opt.maxConcurrentJobs - jobExecutor.activeJobCount() - maxReturnCount;
                     maxLatency = 100;
                     latency = systemMonitor.getLatency().setTimeout;
                     latencyFactor = 1 - Math.abs(Math.min(latency / maxLatency, 1)) // 0 if latency is at max, 1 if latency 0

package/commonjs/src/defaults.js

@@ -36,6 +36,7 @@ exports.defaults = Object.freeze({
     killAfter: 30,
     archive: false,
     activeOnly: false,
+    maxConcurrentJobs: 100,
     // Idle Queues
     idleFor: 60,
     delete: false,
@@ -87,6 +88,7 @@ function getOptionsWithDefaults(options) {
         archive: options.archive || exports.defaults.archive,
         activeOnly: options.activeOnly || options['active-only'] || exports.defaults.activeOnly,
         includeFailed: options.includeFailed || options['include-failed'] || exports.defaults.includeFailed,
+        maxConcurrentJobs: options.maxConcurrentJobs || exports.defaults.maxConcurrentJobs,
         // Idle Queues
         idleFor: options.idleFor || options['idle-for'] || exports.defaults.idleFor,
         delete: options.delete || exports.defaults.delete,

package/commonjs/src/scheduler/jobExecutor.js

@@ -67,9 +67,9 @@ var JobExecutor = /** @class */ (function () {
     }
     JobExecutor.prototype.shutdown = function () {
         this.shutdownRequested = true;
-        if (this.stats.activeJobs === 0 && this.jobs.length === 0) {
-            clearTimeout(this.maintainVisibilityTimeout);
-        }
+        // Trigger a maintenance run right away in case it speeds us up
+        clearTimeout(this.maintainVisibilityTimeout);
+        this.maintainVisibility();
     };
     JobExecutor.prototype.activeJobCount = function () {
         return this.stats.activeJobs;
@@ -79,7 +79,7 @@ var JobExecutor = /** @class */ (function () {
      */
     JobExecutor.prototype.maintainVisibility = function () {
         return __awaiter(this, void 0, void 0, function () {
-            var now, jobsToExtendByQrl, jobsToDeleteByQrl, jobsToCleanup, _i, _a, job, jobRunTime, jobsToDelete, jobsToExtend, doubled, secondsUntilMax, secondsUntilKill, _b, _c, _d, _e, qrl, jobsToExtend, entries, messageId, job, entry, input, result, _f, _g, _h, _j, qrl, jobsToDelete, entries, messageId, job, entry, input, result;
+            var now, jobsToExtendByQrl, jobsToDeleteByQrl, jobsToCleanup, _i, _a, job, jobRunTime, jobsToDelete, jobsToExtend, doubled, secondsUntilMax, secondsUntilKill, _b, _c, _d, _e, qrl, jobsToExtend, entries, messageId, job, entry, input, result, count, _f, _g, _h, _j, qrl, jobsToDelete, entries, messageId, job, entry, input, result, count, nextCheckInMs;
             var _this = this;
             return __generator(this, function (_k) {
                 switch (_k.label) {
@@ -127,9 +127,9 @@ var JobExecutor = /** @class */ (function () {
                         _e = 0;
                         _k.label = 1;
                     case 1:
-                        if (!(_e < _c.length)) return [3 /*break*/, 6];
+                        if (!(_e < _c.length)) return [3 /*break*/, 5];
                         _d = _c[_e];
-                        if (!(_d in _b)) return [3 /*break*/, 5];
+                        if (!(_d in _b)) return [3 /*break*/, 4];
                         qrl = _d;
                         jobsToExtend = jobsToExtendByQrl[qrl];
                         debug({ qrl: qrl, jobsToExtend: jobsToExtend });
@@ -154,42 +154,36 @@ var JobExecutor = /** @class */ (function () {
                         result = _k.sent();
                         debug('ChangeMessageVisibilityBatch returned', result);
                         this.stats.sqsCalls++;
-                        this.stats.timeoutsExtended += 10;
+                        if (result.Successful) {
+                            count = result.Successful.length || 0;
+                            this.stats.timeoutsExtended += count;
+                            if (this.opt.verbose) {
+                                console.error(chalk_1.default.blue('Extended'), count, chalk_1.default.blue('jobs'));
+                            }
+                            else if (!this.opt.disableLog) {
+                                console.log(JSON.stringify({ event: 'EXTEND_VISIBILITY_TIMEOUTS', timestamp: now, count: count, qrl: qrl }));
+                            }
+                        }
                         return [3 /*break*/, 2];
                     case 4:
-                        if (this.opt.verbose) {
-                            console.error(chalk_1.default.blue('Extended these jobs: '), jobsToExtend);
-                        }
-                        else if (!this.opt.disableLog) {
-                            console.log(JSON.stringify({
-                                event: 'EXTEND_VISIBILITY_TIMEOUTS',
-                                timestamp: now,
-                                messageIds: jobsToExtend.map(function (_a) {
-                                    var message = _a.message;
-                                    return message.MessageId;
-                                })
-                            }));
-                        }
-                        _k.label = 5;
-                    case 5:
                         _e++;
                         return [3 /*break*/, 1];
-                    case 6:
+                    case 5:
                         _f = jobsToDeleteByQrl;
                         _g = [];
                         for (_h in _f)
                             _g.push(_h);
                         _j = 0;
-                        _k.label = 7;
-                    case 7:
-                        if (!(_j < _g.length)) return [3 /*break*/, 12];
+                        _k.label = 6;
+                    case 6:
+                        if (!(_j < _g.length)) return [3 /*break*/, 10];
                         _h = _g[_j];
-                        if (!(_h in _f)) return [3 /*break*/, 11];
+                        if (!(_h in _f)) return [3 /*break*/, 9];
                         qrl = _h;
                         jobsToDelete = jobsToDeleteByQrl[qrl];
-                        _k.label = 8;
-                    case 8:
-                        if (!jobsToDelete.length) return [3 /*break*/, 10];
+                        _k.label = 7;
+                    case 7:
+                        if (!jobsToDelete.length) return [3 /*break*/, 9];
                         entries = [];
                         messageId = 0;
                         while (messageId++ < 10 && jobsToDelete.length) {
@@ -204,37 +198,32 @@ var JobExecutor = /** @class */ (function () {
                         input = { QueueUrl: qrl, Entries: entries };
                         debug({ DeleteMessageBatch: input });
                         return [4 /*yield*/, (0, sqs_js_1.getSQSClient)().send(new client_sqs_1.DeleteMessageBatchCommand(input))];
-                    case 9:
+                    case 8:
                         result = _k.sent();
                         this.stats.sqsCalls++;
-                        this.stats.jobsDeleted += 10;
-                        debug('DeleteMessageBatch returned', result);
-                        return [3 /*break*/, 8];
-                    case 10:
-                        if (this.opt.verbose) {
-                            console.error(chalk_1.default.blue('Deleted these finished jobs: '), jobsToDelete);
-                        }
-                        else if (!this.opt.disableLog) {
-                            console.log(JSON.stringify({
-                                event: 'DELETE_MESSAGES',
-                                timestamp: now,
-                                messageIds: jobsToDelete.map(function (_a) {
-                                    var message = _a.message;
-                                    return message.MessageId;
-                                })
-                            }));
+                        if (result.Successful) {
+                            count = result.Successful.length || 0;
+                            this.stats.jobsDeleted += count;
+                            if (this.opt.verbose) {
+                                console.error(chalk_1.default.blue('Deleted'), count, chalk_1.default.blue('jobs'));
+                            }
+                            else if (!this.opt.disableLog) {
+                                console.log(JSON.stringify({ event: 'DELETE_MESSAGES', timestamp: now, count: count, qrl: qrl }));
+                            }
                         }
-                        _k.label = 11;
-                    case 11:
-                        _j++;
+                        debug('DeleteMessageBatch returned', result);
                         return [3 /*break*/, 7];
-                    case 12:
+                    case 9:
+                        _j++;
+                        return [3 /*break*/, 6];
+                    case 10:
                         // Get rid of deleted and failed jobs
                         this.jobs = this.jobs.filter(function (j) { return j.status === 'processing'; });
                         // Check again later, unless we are shutting down and nothing's left
                         if (this.shutdownRequested && this.stats.activeJobs === 0 && this.jobs.length === 0)
                             return [2 /*return*/];
-                        this.maintainVisibilityTimeout = setTimeout(function () { return _this.maintainVisibility(); }, 10 * 1000);
+                        nextCheckInMs = this.shutdownRequested ? 1 * 1000 : 10 * 1000;
+                        this.maintainVisibilityTimeout = setTimeout(function () { return _this.maintainVisibility(); }, nextCheckInMs);
                         return [2 /*return*/];
                 }
             });
@@ -269,7 +258,7 @@ var JobExecutor = /** @class */ (function () {
                             console.log(JSON.stringify({
                                 event: 'MESSAGE_PROCESSING_START',
                                 timestamp: new Date(),
-                                qname: qname,
+                                qrl: qrl,
                                 messageId: message.MessageId,
                                 payload: job.payload
                             }));
@@ -313,7 +302,7 @@ var JobExecutor = /** @class */ (function () {
                             console.log(JSON.stringify({
                                 event: 'MESSAGE_PROCESSING_FAILED',
                                 reason: 'exception thrown',
-                                qname: qname,
+                                qrl: qrl,
                                 timestamp: new Date(),
                                 messageId: message.MessageId,
                                 payload: payload,

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "qdone",
-  "version": "2.0.11-alpha",
+  "version": "2.0.13-alpha",
   "description": "Language agnostic job queue for SQS",
   "type": "module",
   "main": "./index.js",

package/src/consumer.js

@@ -53,7 +53,6 @@ export async function processMessages (queues, callback, options) {
   const systemMonitor = new SystemMonitor(opt)
   const jobExecutor = new JobExecutor(opt)
   const queueManager = new QueueManager(opt, queues)
-  const maxActiveJobs = 100
   debug({ systemMonitor, jobExecutor, queueManager })
 
   shutdownCallbacks.push(() => {
@@ -85,7 +84,7 @@ export async function processMessages (queues, callback, options) {
 
   while (!shutdownRequested) { // eslint-disable-line
     // Figure out how we are running
-    const allowedJobs = maxActiveJobs - jobExecutor.activeJobCount() - maxReturnCount
+    const allowedJobs = opt.maxConcurrentJobs - jobExecutor.activeJobCount() - maxReturnCount
     const maxLatency = 100
     const latency = systemMonitor.getLatency().setTimeout
     const latencyFactor = 1 - Math.abs(Math.min(latency / maxLatency, 1)) // 0 if latency is at max, 1 if latency 0

package/src/defaults.js

@@ -36,6 +36,7 @@ export const defaults = Object.freeze({
   killAfter: 30,
   archive: false,
   activeOnly: false,
+  maxConcurrentJobs: 100,
 
   // Idle Queues
   idleFor: 60,
@@ -93,6 +94,7 @@ export function getOptionsWithDefaults (options) {
     archive: options.archive || defaults.archive,
     activeOnly: options.activeOnly || options['active-only'] || defaults.activeOnly,
     includeFailed: options.includeFailed || options['include-failed'] || defaults.includeFailed,
+    maxConcurrentJobs: options.maxConcurrentJobs || defaults.maxConcurrentJobs,
 
     // Idle Queues
     idleFor: options.idleFor || options['idle-for'] || defaults.idleFor,

package/src/scheduler/jobExecutor.js

@@ -34,9 +34,9 @@ export class JobExecutor {
 
   shutdown () {
     this.shutdownRequested = true
-    if (this.stats.activeJobs === 0 && this.jobs.length === 0) {
-      clearTimeout(this.maintainVisibilityTimeout)
-    }
+    // Trigger a maintenance run right away in case it speeds us up
+    clearTimeout(this.maintainVisibilityTimeout)
+    this.maintainVisibility()
   }
 
   activeJobCount () {
@@ -107,18 +107,17 @@ export class JobExecutor {
         const result = await getSQSClient().send(new ChangeMessageVisibilityBatchCommand(input))
         debug('ChangeMessageVisibilityBatch returned', result)
         this.stats.sqsCalls++
-        this.stats.timeoutsExtended += 10
+        if (result.Successful) {
+          const count = result.Successful.length || 0
+          this.stats.timeoutsExtended += count
+          if (this.opt.verbose) {
+            console.error(chalk.blue('Extended'), count, chalk.blue('jobs'))
+          } else if (!this.opt.disableLog) {
+            console.log(JSON.stringify({ event: 'EXTEND_VISIBILITY_TIMEOUTS', timestamp: now, count, qrl }))
+          }
+        }
         // TODO Sentry
       }
-      if (this.opt.verbose) {
-        console.error(chalk.blue('Extended these jobs: '), jobsToExtend)
-      } else if (!this.opt.disableLog) {
-        console.log(JSON.stringify({
-          event: 'EXTEND_VISIBILITY_TIMEOUTS',
-          timestamp: now,
-          messageIds: jobsToExtend.map(({ message }) => message.MessageId)
-        }))
-      }
     }
 
     // Delete in batches for each queue
@@ -143,19 +142,18 @@ export class JobExecutor {
         debug({ DeleteMessageBatch: input })
         const result = await getSQSClient().send(new DeleteMessageBatchCommand(input))
         this.stats.sqsCalls++
-        this.stats.jobsDeleted += 10
+        if (result.Successful) {
+          const count = result.Successful.length || 0
+          this.stats.jobsDeleted += count
+          if (this.opt.verbose) {
+            console.error(chalk.blue('Deleted'), count, chalk.blue('jobs'))
+          } else if (!this.opt.disableLog) {
+            console.log(JSON.stringify({ event: 'DELETE_MESSAGES', timestamp: now, count, qrl }))
+          }
+        }
         debug('DeleteMessageBatch returned', result)
         // TODO Sentry
       }
-      if (this.opt.verbose) {
-        console.error(chalk.blue('Deleted these finished jobs: '), jobsToDelete)
-      } else if (!this.opt.disableLog) {
-        console.log(JSON.stringify({
-          event: 'DELETE_MESSAGES',
-          timestamp: now,
-          messageIds: jobsToDelete.map(({ message }) => message.MessageId)
-        }))
-      }
     }
 
     // Get rid of deleted and failed jobs
@@ -163,7 +161,8 @@ export class JobExecutor {
 
     // Check again later, unless we are shutting down and nothing's left
     if (this.shutdownRequested && this.stats.activeJobs === 0 && this.jobs.length === 0) return
-    this.maintainVisibilityTimeout = setTimeout(() => this.maintainVisibility(), 10 * 1000)
+    const nextCheckInMs = this.shutdownRequested ? 1 * 1000 : 10 * 1000
+    this.maintainVisibilityTimeout = setTimeout(() => this.maintainVisibility(), nextCheckInMs)
   }
 
   async executeJob (message, callback, qname, qrl) {
@@ -190,7 +189,7 @@ export class JobExecutor {
       console.log(JSON.stringify({
         event: 'MESSAGE_PROCESSING_START',
         timestamp: new Date(),
-        qname,
+        qrl,
         messageId: message.MessageId,
         payload: job.payload
       }))
@@ -229,7 +228,7 @@ export class JobExecutor {
         console.log(JSON.stringify({
           event: 'MESSAGE_PROCESSING_FAILED',
           reason: 'exception thrown',
-          qname,
+          qrl,
           timestamp: new Date(),
           messageId: message.MessageId,
           payload,