qdone 2.0.12-alpha → 2.0.14-alpha

package/commonjs/src/consumer.js

@@ -56,15 +56,34 @@ var debug = (0, debug_1.default)('qdone:consumer');
 var shutdownRequested = false;
 var shutdownCallbacks = [];
 function requestShutdown() {
-    debug('requestShutdown');
-    shutdownRequested = true;
-    for (var _i = 0, shutdownCallbacks_1 = shutdownCallbacks; _i < shutdownCallbacks_1.length; _i++) {
-        var callback = shutdownCallbacks_1[_i];
-        debug('callback', callback);
-        callback();
-        // try { callback() } catch (e) { }
-    }
-    debug('requestShutdown done');
+    return __awaiter(this, void 0, void 0, function () {
+        var _i, shutdownCallbacks_1, callback;
+        return __generator(this, function (_a) {
+            switch (_a.label) {
+                case 0:
+                    debug('requestShutdown');
+                    shutdownRequested = true;
+                    _i = 0, shutdownCallbacks_1 = shutdownCallbacks;
+                    _a.label = 1;
+                case 1:
+                    if (!(_i < shutdownCallbacks_1.length)) return [3 /*break*/, 4];
+                    callback = shutdownCallbacks_1[_i];
+                    debug('callback', callback);
+                    return [4 /*yield*/, callback()
+                        // try { callback() } catch (e) { }
+                    ];
+                case 2:
+                    _a.sent();
+                    _a.label = 3;
+                case 3:
+                    _i++;
+                    return [3 /*break*/, 1];
+                case 4:
+                    debug('requestShutdown done');
+                    return [2 /*return*/];
+            }
+        });
+    });
 }
 exports.requestShutdown = requestShutdown;
 function getMessages(qrl, opt, maxMessages) {
@@ -78,7 +97,7 @@ function getMessages(qrl, opt, maxMessages) {
                         MaxNumberOfMessages: maxMessages,
                         MessageAttributeNames: ['All'],
                         QueueUrl: qrl,
-                        VisibilityTimeout: 30,
+                        VisibilityTimeout: 60,
                         WaitTimeSeconds: opt.waitTime
                     };
                     return [4 /*yield*/, (0, sqs_js_1.getSQSClient)().send(new client_sqs_1.ReceiveMessageCommand(params))
@@ -98,58 +117,101 @@ exports.getMessages = getMessages;
 //
 function processMessages(queues, callback, options) {
     return __awaiter(this, void 0, void 0, function () {
-        var opt, systemMonitor, jobExecutor, queueManager, maxActiveJobs, delayTimeout, delay, activeQrls, maxReturnCount, listen, allowedJobs, maxLatency, latency, latencyFactor, targetJobs, jobsLeft, _i, _a, _b, qname, qrl, maxMessages;
+        var opt, lastLatency, systemMonitor, jobExecutor, queueManager, delayTimeout, delay, activeQrls, maxReturnCount, listen, allowedJobs, maxLatency, latency, latencyFactor, targetJobs, jobsLeft, _i, _a, _b, qname, qrl, maxMessages;
         var _this = this;
         return __generator(this, function (_c) {
             switch (_c.label) {
                 case 0:
+                    debug({ options: options });
                     opt = (0, defaults_js_1.getOptionsWithDefaults)(options);
-                    debug('processMessages', { queues: queues, callback: callback, options: options, opt: opt });
-                    systemMonitor = new systemMonitor_js_1.SystemMonitor(opt);
+                    debug('processMessages', { queues: queues, callback: callback, options: options, opt: opt, argv: process.argv });
+                    lastLatency = 10;
+                    systemMonitor = new systemMonitor_js_1.SystemMonitor(function (latency) {
+                        var percentDifference = 100 * Math.abs(lastLatency - latency) / lastLatency;
+                        if (percentDifference > 10 && opt.verbose) {
+                            console.error(chalk_1.default.blue('Latency:', Math.round(latency), 'ms'));
+                        }
+                        lastLatency = latency;
+                    });
                     jobExecutor = new jobExecutor_js_1.JobExecutor(opt);
                     queueManager = new queueManager_js_1.QueueManager(opt, queues);
-                    maxActiveJobs = 100;
-                    debug({ systemMonitor: systemMonitor, jobExecutor: jobExecutor, queueManager: queueManager });
-                    shutdownCallbacks.push(function () {
-                        systemMonitor.shutdown();
-                        queueManager.shutdown();
-                        jobExecutor.shutdown();
-                    });
                     delay = function (ms) { return new Promise(function (resolve) {
                         delayTimeout = setTimeout(resolve, ms);
                     }); };
+                    shutdownCallbacks.push(function () { return __awaiter(_this, void 0, void 0, function () {
+                        return __generator(this, function (_a) {
+                            switch (_a.label) {
+                                case 0:
+                                    clearTimeout(delayTimeout);
+                                    return [4 /*yield*/, queueManager.shutdown()];
+                                case 1:
+                                    _a.sent();
+                                    debug({ queueManager: 'done' });
+                                    return [4 /*yield*/, jobExecutor.shutdown()];
+                                case 2:
+                                    _a.sent();
+                                    debug({ jobExecutor: 'done' });
+                                    return [4 /*yield*/, systemMonitor.shutdown()];
+                                case 3:
+                                    _a.sent();
+                                    debug({ systemMonitor: 'done' });
+                                    return [2 /*return*/];
+                            }
+                        });
+                    }); });
                     activeQrls = new Set();
                     maxReturnCount = 0;
                     listen = function (qname, qrl, maxMessages) { return __awaiter(_this, void 0, void 0, function () {
-                        var messages, _i, messages_1, message;
+                        var messages, _i, messages_1, message, e_1;
                         return __generator(this, function (_a) {
                             switch (_a.label) {
                                 case 0:
                                     activeQrls.add(qrl);
                                     maxReturnCount += maxMessages;
-                                    return [4 /*yield*/, getMessages(qrl, opt, maxMessages)];
+                                    _a.label = 1;
                                 case 1:
+                                    _a.trys.push([1, 3, , 4]);
+                                    return [4 /*yield*/, getMessages(qrl, opt, maxMessages)];
+                                case 2:
                                     messages = _a.sent();
-                                    for (_i = 0, messages_1 = messages; _i < messages_1.length; _i++) {
-                                        message = messages_1[_i];
-                                        jobExecutor.executeJob(message, callback, qname, qrl);
+                                    if (messages.length) {
+                                        for (_i = 0, messages_1 = messages; _i < messages_1.length; _i++) {
+                                            message = messages_1[_i];
+                                            jobExecutor.executeJob(message, callback, qname, qrl);
+                                        }
+                                        queueManager.updateIcehouse(qrl, false);
+                                    }
+                                    else {
+                                        // If we didn't get any, update the icehouse so we can back off
+                                        queueManager.updateIcehouse(qrl, true);
                                     }
+                                    // Max job accounting
                                     maxReturnCount -= maxMessages;
                                     activeQrls.delete(qrl);
-                                    return [2 /*return*/];
+                                    return [3 /*break*/, 4];
+                                case 3:
+                                    e_1 = _a.sent();
+                                    // If the queue has been cleaned up, we should back off anyway
+                                    if (e_1 instanceof client_sqs_1.QueueDoesNotExist) {
+                                        queueManager.updateIcehouse(qrl, true);
+                                    }
+                                    else {
+                                        throw e_1;
+                                    }
+                                    return [3 /*break*/, 4];
+                                case 4: return [2 /*return*/];
                             }
                         });
                     }); };
                     _c.label = 1;
                 case 1:
                     if (!!shutdownRequested) return [3 /*break*/, 3];
-                    allowedJobs = maxActiveJobs - jobExecutor.activeJobCount() - maxReturnCount;
+                    allowedJobs = opt.maxConcurrentJobs - jobExecutor.activeJobCount() - maxReturnCount;
                     maxLatency = 100;
                     latency = systemMonitor.getLatency().setTimeout;
                     latencyFactor = 1 - Math.abs(Math.min(latency / maxLatency, 1)) // 0 if latency is at max, 1 if latency 0
                     ;
                     targetJobs = Math.round(allowedJobs * latencyFactor);
-                    debug({ allowedJobs: allowedJobs, maxLatency: maxLatency, latency: latency, latencyFactor: latencyFactor, targetJobs: targetJobs, activeQrls: activeQrls });
                     jobsLeft = targetJobs;
                     for (_i = 0, _a = queueManager.getPairs(); _i < _a.length; _i++) {
                         _b = _a[_i], qname = _b.qname, qrl = _b.qrl;
@@ -161,7 +223,7 @@ function processMessages(queues, callback, options) {
                         if (opt.verbose) {
                             console.error(chalk_1.default.blue('Listening on: '), qname);
                         }
-                        debug({ listenedTo: { qname: qname, maxMessages: maxMessages, jobsLeft: jobsLeft } });
+                        // debug({ listenedTo: { qname, maxMessages, jobsLeft } })
                     }
                     return [4 /*yield*/, delay(1000)];
                 case 2:

package/commonjs/src/defaults.js

@@ -36,11 +36,18 @@ exports.defaults = Object.freeze({
     killAfter: 30,
     archive: false,
     activeOnly: false,
+    maxConcurrentJobs: 100,
     // Idle Queues
     idleFor: 60,
     delete: false,
     unpair: false
 });
+function validateInteger(opt, name) {
+    var parsed = parseInt(opt[name], 10);
+    if (isNaN(parsed))
+        throw new Error("".concat(name, " needs to be an integer."));
+    return parsed;
+}
 /**
  * This function should be called by each exposed API entry point on the
  * options passed in from the caller. It supports options named in camelCase
@@ -87,13 +94,25 @@ function getOptionsWithDefaults(options) {
         archive: options.archive || exports.defaults.archive,
         activeOnly: options.activeOnly || options['active-only'] || exports.defaults.activeOnly,
         includeFailed: options.includeFailed || options['include-failed'] || exports.defaults.includeFailed,
+        maxConcurrentJobs: options.maxConcurrentJobs || exports.defaults.maxConcurrentJobs,
         // Idle Queues
         idleFor: options.idleFor || options['idle-for'] || exports.defaults.idleFor,
         delete: options.delete || exports.defaults.delete,
         unpair: options.delete || exports.defaults.unpair
     };
+    // Setting this env here means we don't have to in AWS SDK constructors
     process.env.AWS_REGION = opt.region;
-    // TODO: validate options
+    // Validation
+    opt.cacheTtlSeconds = validateInteger(opt, 'cacheTtlSeconds');
+    opt.messageRetentionPeriod = validateInteger(opt, 'messageRetentionPeriod');
+    opt.delay = validateInteger(opt, 'delay');
+    opt.sendRetries = validateInteger(opt, 'sendRetries');
+    opt.failDelay = validateInteger(opt, 'failDelay');
+    opt.dlqAfter = validateInteger(opt, 'dlqAfter');
+    opt.waitTime = validateInteger(opt, 'waitTime');
+    opt.killAfter = validateInteger(opt, 'killAfter');
+    opt.maxConcurrentJobs = validateInteger(opt, 'maxConcurrentJobs');
+    opt.idleFor = validateInteger(opt, 'idleFor');
     return opt;
 }
 exports.getOptionsWithDefaults = getOptionsWithDefaults;

package/commonjs/src/idleQueues.js

@@ -91,11 +91,10 @@ function _cheapIdleCheck(qname, qrl, opt) {
                 case 0:
                     client = (0, sqs_js_1.getSQSClient)();
                     cmd = new client_sqs_1.GetQueueAttributesCommand({ AttributeNames: exports.attributeNames, QueueUrl: qrl });
-                    return [4 /*yield*/, client.send(cmd)
-                        // debug('data', data)
-                    ];
+                    return [4 /*yield*/, client.send(cmd)];
                 case 1:
                     data = _a.sent();
+                    debug('data', data);
                     result = data.Attributes;
                     result.queue = qname.slice(opt.prefix.length);
                     // We are idle if all the messages attributes are zero

package/commonjs/src/scheduler/jobExecutor.js

@@ -54,6 +54,7 @@ var JobExecutor = /** @class */ (function () {
     function JobExecutor(opt) {
         this.opt = opt;
         this.jobs = [];
+        this.jobsByMessageId = {};
         this.stats = {
             activeJobs: 0,
             sqsCalls: 0,
@@ -62,14 +63,27 @@ var JobExecutor = /** @class */ (function () {
             jobsFailed: 0,
             jobsDeleted: 0
         };
-        this.maintainVisibility();
+        this.maintainPromise = this.maintainVisibility();
         debug({ this: this });
     }
     JobExecutor.prototype.shutdown = function () {
-        this.shutdownRequested = true;
-        if (this.stats.activeJobs === 0 && this.jobs.length === 0) {
-            clearTimeout(this.maintainVisibilityTimeout);
-        }
+        return __awaiter(this, void 0, void 0, function () {
+            return __generator(this, function (_a) {
+                switch (_a.label) {
+                    case 0:
+                        this.shutdownRequested = true;
+                        // Trigger a maintenance run right away in case it speeds us up
+                        clearTimeout(this.maintainVisibilityTimeout);
+                        return [4 /*yield*/, this.maintainPromise];
+                    case 1:
+                        _a.sent();
+                        return [4 /*yield*/, this.maintainVisibility()];
+                    case 2:
+                        _a.sent();
+                        return [2 /*return*/];
+                }
+            });
+        });
     };
     JobExecutor.prototype.activeJobCount = function () {
         return this.stats.activeJobs;
@@ -79,13 +93,13 @@ var JobExecutor = /** @class */ (function () {
      */
     JobExecutor.prototype.maintainVisibility = function () {
         return __awaiter(this, void 0, void 0, function () {
-            var now, jobsToExtendByQrl, jobsToDeleteByQrl, jobsToCleanup, _i, _a, job, jobRunTime, jobsToDelete, jobsToExtend, doubled, secondsUntilMax, secondsUntilKill, _b, _c, _d, _e, qrl, jobsToExtend, entries, messageId, job, entry, input, result, _f, _g, _h, _j, qrl, jobsToDelete, entries, messageId, job, entry, input, result;
+            var start, jobsToExtendByQrl, jobsToDeleteByQrl, jobsToCleanup, i, job, jobRunTime, jobsToDelete, jobsToExtend, doubled, secondsUntilMax, _a, _b, _c, _i, qrl, jobsToExtend, entries, messageId, job, entry, input, result, count, _d, _e, _f, _g, qrl, jobsToDelete, entries, messageId, job, entry, input, result, count, msElapsed, msPeriod, msLeft, msMin, nextCheckInMs;
             var _this = this;
-            return __generator(this, function (_k) {
-                switch (_k.label) {
+            return __generator(this, function (_h) {
+                switch (_h.label) {
                     case 0:
-                        debug('maintainVisibility', this.jobs);
-                        now = new Date();
+                        clearTimeout(this.maintainVisibilityTimeout);
+                        start = new Date();
                         jobsToExtendByQrl = {};
                         jobsToDeleteByQrl = {};
                         jobsToCleanup = new Set();
@@ -97,43 +111,48 @@ var JobExecutor = /** @class */ (function () {
                             }));
                         }
                         // Build list of jobs we need to deal with
-                        for (_i = 0, _a = this.jobs; _i < _a.length; _i++) {
-                            job = _a[_i];
-                            jobRunTime = (now - job.start) / 1000;
+                        for (i = 0; i < this.jobs.length; i++) {
+                            job = this.jobs[i];
+                            jobRunTime = Math.round((start - job.start) / 1000);
+                            // debug('considering job', job)
                             if (job.status === 'complete') {
                                 jobsToDelete = jobsToDeleteByQrl[job.qrl] || [];
+                                job.status = 'deleting';
                                 jobsToDelete.push(job);
                                 jobsToDeleteByQrl[job.qrl] = jobsToDelete;
                             }
                             else if (job.status === 'failed') {
                                 jobsToCleanup.add(job);
                             }
-                            else if (jobRunTime >= job.exendAtSecond) {
-                                jobsToExtend = jobsToExtendByQrl[job.qrl] || [];
-                                jobsToExtend.push(job);
-                                jobsToExtendByQrl[job.qrl] = jobsToExtend;
-                                doubled = job.visibilityTimeout * 2;
-                                secondsUntilMax = maxJobSeconds - jobRunTime;
-                                secondsUntilKill = this.opt.killAfter - jobRunTime;
-                                job.visibilityTimeout = Math.min(double, secondsUntilMax, secondsUntilKill);
-                                job.extendAtSecond = jobRunTime + job.visibilityTimeout; // this is what we use next time
+                            else if (job.status === 'processing') {
+                                debug('processing', { job: job, jobRunTime: jobRunTime });
+                                if (jobRunTime >= job.extendAtSecond) {
+                                    jobsToExtend = jobsToExtendByQrl[job.qrl] || [];
+                                    jobsToExtend.push(job);
+                                    jobsToExtendByQrl[job.qrl] = jobsToExtend;
+                                    doubled = job.visibilityTimeout * 2;
+                                    secondsUntilMax = Math.max(1, maxJobSeconds - jobRunTime);
+                                    // const secondsUntilKill = Math.max(1, this.opt.killAfter - jobRunTime)
+                                    job.visibilityTimeout = Math.min(doubled, secondsUntilMax); //, secondsUntilKill)
+                                    job.extendAtSecond = Math.round(jobRunTime + job.visibilityTimeout); // this is what we use next time
+                                    debug({ doubled: doubled, secondsUntilMax: secondsUntilMax, job: job });
+                                }
                             }
                         }
-                        debug('maintainVisibility', { jobsToDeleteByQrl: jobsToDeleteByQrl, jobsToExtendByQrl: jobsToExtendByQrl });
-                        _b = jobsToExtendByQrl;
-                        _c = [];
-                        for (_d in _b)
-                            _c.push(_d);
-                        _e = 0;
-                        _k.label = 1;
+                        _a = jobsToExtendByQrl;
+                        _b = [];
+                        for (_c in _a)
+                            _b.push(_c);
+                        _i = 0;
+                        _h.label = 1;
                     case 1:
-                        if (!(_e < _c.length)) return [3 /*break*/, 6];
-                        _d = _c[_e];
-                        if (!(_d in _b)) return [3 /*break*/, 5];
-                        qrl = _d;
+                        if (!(_i < _b.length)) return [3 /*break*/, 5];
+                        _c = _b[_i];
+                        if (!(_c in _a)) return [3 /*break*/, 4];
+                        qrl = _c;
                         jobsToExtend = jobsToExtendByQrl[qrl];
                         debug({ qrl: qrl, jobsToExtend: jobsToExtend });
-                        _k.label = 2;
+                        _h.label = 2;
                     case 2:
                         if (!jobsToExtend.length) return [3 /*break*/, 4];
                         entries = [];
@@ -147,50 +166,44 @@ var JobExecutor = /** @class */ (function () {
                             };
                             entries.push(entry);
                         }
+                        debug({ entries: entries });
                         input = { QueueUrl: qrl, Entries: entries };
                         debug({ ChangeMessageVisibilityBatch: input });
                         return [4 /*yield*/, (0, sqs_js_1.getSQSClient)().send(new client_sqs_1.ChangeMessageVisibilityBatchCommand(input))];
                     case 3:
-                        result = _k.sent();
+                        result = _h.sent();
                         debug('ChangeMessageVisibilityBatch returned', result);
                         this.stats.sqsCalls++;
-                        if (result.Successful)
-                            this.stats.timeoutsExtended += result.Successful.length || 0;
+                        if (result.Successful) {
+                            count = result.Successful.length || 0;
+                            this.stats.timeoutsExtended += count;
+                            if (this.opt.verbose) {
+                                console.error(chalk_1.default.blue('Extended'), count, chalk_1.default.blue('jobs'));
+                            }
+                            else if (!this.opt.disableLog) {
+                                console.log(JSON.stringify({ event: 'EXTEND_VISIBILITY_TIMEOUTS', timestamp: start, count: count, qrl: qrl }));
+                            }
+                        }
                         return [3 /*break*/, 2];
                     case 4:
-                        if (this.opt.verbose) {
-                            console.error(chalk_1.default.blue('Extended these jobs: '), jobsToExtend);
-                        }
-                        else if (!this.opt.disableLog) {
-                            console.log(JSON.stringify({
-                                event: 'EXTEND_VISIBILITY_TIMEOUTS',
-                                timestamp: now,
-                                messageIds: jobsToExtend.map(function (_a) {
-                                    var message = _a.message;
-                                    return message.MessageId;
-                                })
-                            }));
-                        }
-                        _k.label = 5;
-                    case 5:
-                        _e++;
+                        _i++;
                         return [3 /*break*/, 1];
+                    case 5:
+                        _d = jobsToDeleteByQrl;
+                        _e = [];
+                        for (_f in _d)
+                            _e.push(_f);
+                        _g = 0;
+                        _h.label = 6;
                     case 6:
-                        _f = jobsToDeleteByQrl;
-                        _g = [];
-                        for (_h in _f)
-                            _g.push(_h);
-                        _j = 0;
-                        _k.label = 7;
-                    case 7:
-                        if (!(_j < _g.length)) return [3 /*break*/, 12];
-                        _h = _g[_j];
-                        if (!(_h in _f)) return [3 /*break*/, 11];
-                        qrl = _h;
+                        if (!(_g < _e.length)) return [3 /*break*/, 10];
+                        _f = _e[_g];
+                        if (!(_f in _d)) return [3 /*break*/, 9];
+                        qrl = _f;
                         jobsToDelete = jobsToDeleteByQrl[qrl];
-                        _k.label = 8;
-                    case 8:
-                        if (!jobsToDelete.length) return [3 /*break*/, 10];
+                        _h.label = 7;
+                    case 7:
+                        if (!jobsToDelete.length) return [3 /*break*/, 9];
                         entries = [];
                         messageId = 0;
                         while (messageId++ < 10 && jobsToDelete.length) {
@@ -202,41 +215,52 @@ var JobExecutor = /** @class */ (function () {
                             };
                             entries.push(entry);
                         }
+                        debug({ entries: entries });
                         input = { QueueUrl: qrl, Entries: entries };
                         debug({ DeleteMessageBatch: input });
                         return [4 /*yield*/, (0, sqs_js_1.getSQSClient)().send(new client_sqs_1.DeleteMessageBatchCommand(input))];
-                    case 9:
-                        result = _k.sent();
+                    case 8:
+                        result = _h.sent();
                         this.stats.sqsCalls++;
-                        if (result.Successful)
-                            this.stats.jobsDeleted += result.Successful.length || 0;
-                        debug('DeleteMessageBatch returned', result);
-                        return [3 /*break*/, 8];
-                    case 10:
-                        if (this.opt.verbose) {
-                            console.error(chalk_1.default.blue('Deleted these finished jobs: '), jobsToDelete);
-                        }
-                        else if (!this.opt.disableLog) {
-                            console.log(JSON.stringify({
-                                event: 'DELETE_MESSAGES',
-                                timestamp: now,
-                                messageIds: jobsToDelete.map(function (_a) {
-                                    var message = _a.message;
-                                    return message.MessageId;
-                                })
-                            }));
+                        if (result.Successful) {
+                            count = result.Successful.length || 0;
+                            this.stats.jobsDeleted += count;
+                            if (this.opt.verbose) {
+                                console.error(chalk_1.default.blue('Deleted'), count, chalk_1.default.blue('jobs'));
+                            }
+                            else if (!this.opt.disableLog) {
+                                console.log(JSON.stringify({ event: 'DELETE_MESSAGES', timestamp: start, count: count, qrl: qrl }));
+                            }
                         }
-                        _k.label = 11;
-                    case 11:
-                        _j++;
+                        debug('DeleteMessageBatch returned', result);
                         return [3 /*break*/, 7];
-                    case 12:
+                    case 9:
+                        _g++;
+                        return [3 /*break*/, 6];
+                    case 10:
                         // Get rid of deleted and failed jobs
-                        this.jobs = this.jobs.filter(function (j) { return j.status === 'processing'; });
-                        // Check again later, unless we are shutting down and nothing's left
+                        this.jobs = this.jobs.filter(function (job) {
+                            if (job.status === 'deleting' || job.status === 'failed') {
+                                debug('removed', job.message.MessageId);
+                                delete _this.jobsByMessageId[job.message.MessageId];
+                                return false;
+                            }
+                            else {
+                                return true;
+                            }
+                        });
+                        // Bail if we are shutting down
                         if (this.shutdownRequested && this.stats.activeJobs === 0 && this.jobs.length === 0)
                             return [2 /*return*/];
-                        this.maintainVisibilityTimeout = setTimeout(function () { return _this.maintainVisibility(); }, 10 * 1000);
+                        msElapsed = new Date() - start;
+                        msPeriod = this.shutdownRequested ? 1 * 1000 : 10 * 1000;
+                        msLeft = Math.max(0, msPeriod - msElapsed);
+                        msMin = this.shutdownRequested ? 1000 : 0;
+                        nextCheckInMs = Math.max(msMin, msLeft);
+                        debug({ msElapsed: msElapsed, msPeriod: msPeriod, msLeft: msLeft, msMin: msMin, nextCheckInMs: nextCheckInMs });
+                        this.maintainVisibilityTimeout = setTimeout(function () {
+                            _this.maintainPromise = _this.maintainVisibility();
+                        }, nextCheckInMs);
                         return [2 /*return*/];
                 }
             });
@@ -244,16 +268,16 @@ var JobExecutor = /** @class */ (function () {
     };
     JobExecutor.prototype.executeJob = function (message, callback, qname, qrl) {
         return __awaiter(this, void 0, void 0, function () {
-            var payload, visibilityTimeout, job, queue, result, err_1;
+            var payload, visibilityTimeout, job, oldJob, e, queue, result, err_1;
             return __generator(this, function (_a) {
                 switch (_a.label) {
                     case 0:
                         payload = this.opt.json ? JSON.parse(message.Body) : message.Body;
-                        visibilityTimeout = 30;
+                        visibilityTimeout = 60;
                         job = {
                             status: 'processing',
                             start: new Date(),
-                            visibilityTimeout: 30,
+                            visibilityTimeout: visibilityTimeout,
                             extendAtSecond: visibilityTimeout / 2,
                             payload: this.opt.json ? JSON.parse(message.Body) : message.Body,
                             message: message,
@@ -261,8 +285,20 @@ var JobExecutor = /** @class */ (function () {
                             qname: qname,
                             qrl: qrl
                         };
-                        debug('executeJob', job);
+                        oldJob = this.jobsByMessageId[job.message.MessageId];
+                        if (oldJob) {
+                            // If we actually see the same job again, we fucked up, probably due to
+                            // the system being overloaded and us missing our extension call. So
+                            // we'll celebrate this occasion by throwing a big fat error.
+                            debug({ oldJob: oldJob });
+                            e = new Error("Saw job ".concat(oldJob.message.MessageId, " twice"));
+                            e.job = oldJob;
+                            // TODO: sentry breadcrumb
+                            throw e;
+                        }
+                        // debug('executeJob', job)
                         this.jobs.push(job);
+                        this.jobsByMessageId[job.message.MessageId] = job;
                         this.stats.activeJobs++;
                         if (this.opt.verbose) {
                             console.error(chalk_1.default.blue('Executing:'), qname, chalk_1.default.blue('-->'), job.payload);
@@ -271,7 +307,7 @@ var JobExecutor = /** @class */ (function () {
                             console.log(JSON.stringify({
                                 event: 'MESSAGE_PROCESSING_START',
                                 timestamp: new Date(),
-                                qname: qname,
+                                qrl: qrl,
                                 messageId: message.MessageId,
                                 payload: job.payload
                             }));
@@ -304,7 +340,7 @@ var JobExecutor = /** @class */ (function () {
                         return [3 /*break*/, 4];
                     case 3:
                         err_1 = _a.sent();
-                        debug('exec.catch');
+                        // debug('exec.catch', err)
                         // Fail path for job execution
                         if (this.opt.verbose) {
                             console.error(chalk_1.default.red('FAILED'), message.Body);
@@ -315,7 +351,7 @@ var JobExecutor = /** @class */ (function () {
                             console.log(JSON.stringify({
                                 event: 'MESSAGE_PROCESSING_FAILED',
                                 reason: 'exception thrown',
-                                qname: qname,
+                                qrl: qrl,
                                 timestamp: new Date(),
                                 messageId: message.MessageId,
                                 payload: payload,