qdone 2.0.13-alpha → 2.0.14-alpha

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (13) hide show
  1. package/commonjs/src/consumer.js +90 -27
  2. package/commonjs/src/defaults.js +18 -1
  3. package/commonjs/src/idleQueues.js +2 -3
  4. package/commonjs/src/scheduler/jobExecutor.js +109 -60
  5. package/commonjs/src/scheduler/queueManager.js +116 -35
  6. package/commonjs/src/scheduler/systemMonitor.js +21 -37
  7. package/package.json +1 -1
  8. package/src/consumer.js +49 -20
  9. package/src/defaults.js +20 -1
  10. package/src/idleQueues.js +1 -1
  11. package/src/scheduler/jobExecutor.js +75 -30
  12. package/src/scheduler/queueManager.js +86 -23
  13. package/src/scheduler/systemMonitor.js +18 -32
package/commonjs/src/consumer.js CHANGED
@@ -56,15 +56,34 @@ var debug = (0, debug_1.default)('qdone:consumer');
56
56
  var shutdownRequested = false;
57
57
  var shutdownCallbacks = [];
58
58
  function requestShutdown() {
59
- debug('requestShutdown');
60
- shutdownRequested = true;
61
- for (var _i = 0, shutdownCallbacks_1 = shutdownCallbacks; _i < shutdownCallbacks_1.length; _i++) {
62
- var callback = shutdownCallbacks_1[_i];
63
- debug('callback', callback);
64
- callback();
65
- // try { callback() } catch (e) { }
66
- }
67
- debug('requestShutdown done');
59
+ return __awaiter(this, void 0, void 0, function () {
60
+ var _i, shutdownCallbacks_1, callback;
61
+ return __generator(this, function (_a) {
62
+ switch (_a.label) {
63
+ case 0:
64
+ debug('requestShutdown');
65
+ shutdownRequested = true;
66
+ _i = 0, shutdownCallbacks_1 = shutdownCallbacks;
67
+ _a.label = 1;
68
+ case 1:
69
+ if (!(_i < shutdownCallbacks_1.length)) return [3 /*break*/, 4];
70
+ callback = shutdownCallbacks_1[_i];
71
+ debug('callback', callback);
72
+ return [4 /*yield*/, callback()
73
+ // try { callback() } catch (e) { }
74
+ ];
75
+ case 2:
76
+ _a.sent();
77
+ _a.label = 3;
78
+ case 3:
79
+ _i++;
80
+ return [3 /*break*/, 1];
81
+ case 4:
82
+ debug('requestShutdown done');
83
+ return [2 /*return*/];
84
+ }
85
+ });
86
+ });
68
87
  }
69
88
  exports.requestShutdown = requestShutdown;
70
89
  function getMessages(qrl, opt, maxMessages) {
@@ -78,7 +97,7 @@ function getMessages(qrl, opt, maxMessages) {
78
97
  MaxNumberOfMessages: maxMessages,
79
98
  MessageAttributeNames: ['All'],
80
99
  QueueUrl: qrl,
81
- VisibilityTimeout: 30,
100
+ VisibilityTimeout: 60,
82
101
  WaitTimeSeconds: opt.waitTime
83
102
  };
84
103
  return [4 /*yield*/, (0, sqs_js_1.getSQSClient)().send(new client_sqs_1.ReceiveMessageCommand(params))
@@ -98,44 +117,89 @@ exports.getMessages = getMessages;
98
117
  //
99
118
  function processMessages(queues, callback, options) {
100
119
  return __awaiter(this, void 0, void 0, function () {
101
- var opt, systemMonitor, jobExecutor, queueManager, delayTimeout, delay, activeQrls, maxReturnCount, listen, allowedJobs, maxLatency, latency, latencyFactor, targetJobs, jobsLeft, _i, _a, _b, qname, qrl, maxMessages;
120
+ var opt, lastLatency, systemMonitor, jobExecutor, queueManager, delayTimeout, delay, activeQrls, maxReturnCount, listen, allowedJobs, maxLatency, latency, latencyFactor, targetJobs, jobsLeft, _i, _a, _b, qname, qrl, maxMessages;
102
121
  var _this = this;
103
122
  return __generator(this, function (_c) {
104
123
  switch (_c.label) {
105
124
  case 0:
125
+ debug({ options: options });
106
126
  opt = (0, defaults_js_1.getOptionsWithDefaults)(options);
107
- debug('processMessages', { queues: queues, callback: callback, options: options, opt: opt });
108
- systemMonitor = new systemMonitor_js_1.SystemMonitor(opt);
127
+ debug('processMessages', { queues: queues, callback: callback, options: options, opt: opt, argv: process.argv });
128
+ lastLatency = 10;
129
+ systemMonitor = new systemMonitor_js_1.SystemMonitor(function (latency) {
130
+ var percentDifference = 100 * Math.abs(lastLatency - latency) / lastLatency;
131
+ if (percentDifference > 10 && opt.verbose) {
132
+ console.error(chalk_1.default.blue('Latency:', Math.round(latency), 'ms'));
133
+ }
134
+ lastLatency = latency;
135
+ });
109
136
  jobExecutor = new jobExecutor_js_1.JobExecutor(opt);
110
137
  queueManager = new queueManager_js_1.QueueManager(opt, queues);
111
- debug({ systemMonitor: systemMonitor, jobExecutor: jobExecutor, queueManager: queueManager });
112
- shutdownCallbacks.push(function () {
113
- systemMonitor.shutdown();
114
- queueManager.shutdown();
115
- jobExecutor.shutdown();
116
- });
117
138
  delay = function (ms) { return new Promise(function (resolve) {
118
139
  delayTimeout = setTimeout(resolve, ms);
119
140
  }); };
141
+ shutdownCallbacks.push(function () { return __awaiter(_this, void 0, void 0, function () {
142
+ return __generator(this, function (_a) {
143
+ switch (_a.label) {
144
+ case 0:
145
+ clearTimeout(delayTimeout);
146
+ return [4 /*yield*/, queueManager.shutdown()];
147
+ case 1:
148
+ _a.sent();
149
+ debug({ queueManager: 'done' });
150
+ return [4 /*yield*/, jobExecutor.shutdown()];
151
+ case 2:
152
+ _a.sent();
153
+ debug({ jobExecutor: 'done' });
154
+ return [4 /*yield*/, systemMonitor.shutdown()];
155
+ case 3:
156
+ _a.sent();
157
+ debug({ systemMonitor: 'done' });
158
+ return [2 /*return*/];
159
+ }
160
+ });
161
+ }); });
120
162
  activeQrls = new Set();
121
163
  maxReturnCount = 0;
122
164
  listen = function (qname, qrl, maxMessages) { return __awaiter(_this, void 0, void 0, function () {
123
- var messages, _i, messages_1, message;
165
+ var messages, _i, messages_1, message, e_1;
124
166
  return __generator(this, function (_a) {
125
167
  switch (_a.label) {
126
168
  case 0:
127
169
  activeQrls.add(qrl);
128
170
  maxReturnCount += maxMessages;
129
- return [4 /*yield*/, getMessages(qrl, opt, maxMessages)];
171
+ _a.label = 1;
130
172
  case 1:
173
+ _a.trys.push([1, 3, , 4]);
174
+ return [4 /*yield*/, getMessages(qrl, opt, maxMessages)];
175
+ case 2:
131
176
  messages = _a.sent();
132
- for (_i = 0, messages_1 = messages; _i < messages_1.length; _i++) {
133
- message = messages_1[_i];
134
- jobExecutor.executeJob(message, callback, qname, qrl);
177
+ if (messages.length) {
178
+ for (_i = 0, messages_1 = messages; _i < messages_1.length; _i++) {
179
+ message = messages_1[_i];
180
+ jobExecutor.executeJob(message, callback, qname, qrl);
181
+ }
182
+ queueManager.updateIcehouse(qrl, false);
183
+ }
184
+ else {
185
+ // If we didn't get any, update the icehouse so we can back off
186
+ queueManager.updateIcehouse(qrl, true);
135
187
  }
188
+ // Max job accounting
136
189
  maxReturnCount -= maxMessages;
137
190
  activeQrls.delete(qrl);
138
- return [2 /*return*/];
191
+ return [3 /*break*/, 4];
192
+ case 3:
193
+ e_1 = _a.sent();
194
+ // If the queue has been cleaned up, we should back off anyway
195
+ if (e_1 instanceof client_sqs_1.QueueDoesNotExist) {
196
+ queueManager.updateIcehouse(qrl, true);
197
+ }
198
+ else {
199
+ throw e_1;
200
+ }
201
+ return [3 /*break*/, 4];
202
+ case 4: return [2 /*return*/];
139
203
  }
140
204
  });
141
205
  }); };
@@ -148,7 +212,6 @@ function processMessages(queues, callback, options) {
148
212
  latencyFactor = 1 - Math.abs(Math.min(latency / maxLatency, 1)) // 0 if latency is at max, 1 if latency 0
149
213
  ;
150
214
  targetJobs = Math.round(allowedJobs * latencyFactor);
151
- debug({ allowedJobs: allowedJobs, maxLatency: maxLatency, latency: latency, latencyFactor: latencyFactor, targetJobs: targetJobs, activeQrls: activeQrls });
152
215
  jobsLeft = targetJobs;
153
216
  for (_i = 0, _a = queueManager.getPairs(); _i < _a.length; _i++) {
154
217
  _b = _a[_i], qname = _b.qname, qrl = _b.qrl;
@@ -160,7 +223,7 @@ function processMessages(queues, callback, options) {
160
223
  if (opt.verbose) {
161
224
  console.error(chalk_1.default.blue('Listening on: '), qname);
162
225
  }
163
- debug({ listenedTo: { qname: qname, maxMessages: maxMessages, jobsLeft: jobsLeft } });
226
+ // debug({ listenedTo: { qname, maxMessages, jobsLeft } })
164
227
  }
165
228
  return [4 /*yield*/, delay(1000)];
166
229
  case 2:
package/commonjs/src/defaults.js CHANGED
@@ -42,6 +42,12 @@ exports.defaults = Object.freeze({
42
42
  delete: false,
43
43
  unpair: false
44
44
  });
45
+ function validateInteger(opt, name) {
46
+ var parsed = parseInt(opt[name], 10);
47
+ if (isNaN(parsed))
48
+ throw new Error("".concat(name, " needs to be an integer."));
49
+ return parsed;
50
+ }
45
51
  /**
46
52
  * This function should be called by each exposed API entry point on the
47
53
  * options passed in from the caller. It supports options named in camelCase
@@ -94,8 +100,19 @@ function getOptionsWithDefaults(options) {
94
100
  delete: options.delete || exports.defaults.delete,
95
101
  unpair: options.delete || exports.defaults.unpair
96
102
  };
103
+ // Setting this env here means we don't have to in AWS SDK constructors
97
104
  process.env.AWS_REGION = opt.region;
98
- // TODO: validate options
105
+ // Validation
106
+ opt.cacheTtlSeconds = validateInteger(opt, 'cacheTtlSeconds');
107
+ opt.messageRetentionPeriod = validateInteger(opt, 'messageRetentionPeriod');
108
+ opt.delay = validateInteger(opt, 'delay');
109
+ opt.sendRetries = validateInteger(opt, 'sendRetries');
110
+ opt.failDelay = validateInteger(opt, 'failDelay');
111
+ opt.dlqAfter = validateInteger(opt, 'dlqAfter');
112
+ opt.waitTime = validateInteger(opt, 'waitTime');
113
+ opt.killAfter = validateInteger(opt, 'killAfter');
114
+ opt.maxConcurrentJobs = validateInteger(opt, 'maxConcurrentJobs');
115
+ opt.idleFor = validateInteger(opt, 'idleFor');
99
116
  return opt;
100
117
  }
101
118
  exports.getOptionsWithDefaults = getOptionsWithDefaults;
package/commonjs/src/idleQueues.js CHANGED
@@ -91,11 +91,10 @@ function _cheapIdleCheck(qname, qrl, opt) {
91
91
  case 0:
92
92
  client = (0, sqs_js_1.getSQSClient)();
93
93
  cmd = new client_sqs_1.GetQueueAttributesCommand({ AttributeNames: exports.attributeNames, QueueUrl: qrl });
94
- return [4 /*yield*/, client.send(cmd)
95
- // debug('data', data)
96
- ];
94
+ return [4 /*yield*/, client.send(cmd)];
97
95
  case 1:
98
96
  data = _a.sent();
97
+ debug('data', data);
99
98
  result = data.Attributes;
100
99
  result.queue = qname.slice(opt.prefix.length);
101
100
  // We are idle if all the messages attributes are zero
package/commonjs/src/scheduler/jobExecutor.js CHANGED
@@ -54,6 +54,7 @@ var JobExecutor = /** @class */ (function () {
54
54
  function JobExecutor(opt) {
55
55
  this.opt = opt;
56
56
  this.jobs = [];
57
+ this.jobsByMessageId = {};
57
58
  this.stats = {
58
59
  activeJobs: 0,
59
60
  sqsCalls: 0,
@@ -62,14 +63,27 @@ var JobExecutor = /** @class */ (function () {
62
63
  jobsFailed: 0,
63
64
  jobsDeleted: 0
64
65
  };
65
- this.maintainVisibility();
66
+ this.maintainPromise = this.maintainVisibility();
66
67
  debug({ this: this });
67
68
  }
68
69
  JobExecutor.prototype.shutdown = function () {
69
- this.shutdownRequested = true;
70
- // Trigger a maintenance run right away in case it speeds us up
71
- clearTimeout(this.maintainVisibilityTimeout);
72
- this.maintainVisibility();
70
+ return __awaiter(this, void 0, void 0, function () {
71
+ return __generator(this, function (_a) {
72
+ switch (_a.label) {
73
+ case 0:
74
+ this.shutdownRequested = true;
75
+ // Trigger a maintenance run right away in case it speeds us up
76
+ clearTimeout(this.maintainVisibilityTimeout);
77
+ return [4 /*yield*/, this.maintainPromise];
78
+ case 1:
79
+ _a.sent();
80
+ return [4 /*yield*/, this.maintainVisibility()];
81
+ case 2:
82
+ _a.sent();
83
+ return [2 /*return*/];
84
+ }
85
+ });
86
+ });
73
87
  };
74
88
  JobExecutor.prototype.activeJobCount = function () {
75
89
  return this.stats.activeJobs;
@@ -79,13 +93,13 @@ var JobExecutor = /** @class */ (function () {
79
93
  */
80
94
  JobExecutor.prototype.maintainVisibility = function () {
81
95
  return __awaiter(this, void 0, void 0, function () {
82
- var now, jobsToExtendByQrl, jobsToDeleteByQrl, jobsToCleanup, _i, _a, job, jobRunTime, jobsToDelete, jobsToExtend, doubled, secondsUntilMax, secondsUntilKill, _b, _c, _d, _e, qrl, jobsToExtend, entries, messageId, job, entry, input, result, count, _f, _g, _h, _j, qrl, jobsToDelete, entries, messageId, job, entry, input, result, count, nextCheckInMs;
96
+ var start, jobsToExtendByQrl, jobsToDeleteByQrl, jobsToCleanup, i, job, jobRunTime, jobsToDelete, jobsToExtend, doubled, secondsUntilMax, _a, _b, _c, _i, qrl, jobsToExtend, entries, messageId, job, entry, input, result, count, _d, _e, _f, _g, qrl, jobsToDelete, entries, messageId, job, entry, input, result, count, msElapsed, msPeriod, msLeft, msMin, nextCheckInMs;
83
97
  var _this = this;
84
- return __generator(this, function (_k) {
85
- switch (_k.label) {
98
+ return __generator(this, function (_h) {
99
+ switch (_h.label) {
86
100
  case 0:
87
- debug('maintainVisibility', this.jobs);
88
- now = new Date();
101
+ clearTimeout(this.maintainVisibilityTimeout);
102
+ start = new Date();
89
103
  jobsToExtendByQrl = {};
90
104
  jobsToDeleteByQrl = {};
91
105
  jobsToCleanup = new Set();
@@ -97,43 +111,48 @@ var JobExecutor = /** @class */ (function () {
97
111
  }));
98
112
  }
99
113
  // Build list of jobs we need to deal with
100
- for (_i = 0, _a = this.jobs; _i < _a.length; _i++) {
101
- job = _a[_i];
102
- jobRunTime = (now - job.start) / 1000;
114
+ for (i = 0; i < this.jobs.length; i++) {
115
+ job = this.jobs[i];
116
+ jobRunTime = Math.round((start - job.start) / 1000);
117
+ // debug('considering job', job)
103
118
  if (job.status === 'complete') {
104
119
  jobsToDelete = jobsToDeleteByQrl[job.qrl] || [];
120
+ job.status = 'deleting';
105
121
  jobsToDelete.push(job);
106
122
  jobsToDeleteByQrl[job.qrl] = jobsToDelete;
107
123
  }
108
124
  else if (job.status === 'failed') {
109
125
  jobsToCleanup.add(job);
110
126
  }
111
- else if (jobRunTime >= job.exendAtSecond) {
112
- jobsToExtend = jobsToExtendByQrl[job.qrl] || [];
113
- jobsToExtend.push(job);
114
- jobsToExtendByQrl[job.qrl] = jobsToExtend;
115
- doubled = job.visibilityTimeout * 2;
116
- secondsUntilMax = maxJobSeconds - jobRunTime;
117
- secondsUntilKill = this.opt.killAfter - jobRunTime;
118
- job.visibilityTimeout = Math.min(double, secondsUntilMax, secondsUntilKill);
119
- job.extendAtSecond = jobRunTime + job.visibilityTimeout; // this is what we use next time
127
+ else if (job.status === 'processing') {
128
+ debug('processing', { job: job, jobRunTime: jobRunTime });
129
+ if (jobRunTime >= job.extendAtSecond) {
130
+ jobsToExtend = jobsToExtendByQrl[job.qrl] || [];
131
+ jobsToExtend.push(job);
132
+ jobsToExtendByQrl[job.qrl] = jobsToExtend;
133
+ doubled = job.visibilityTimeout * 2;
134
+ secondsUntilMax = Math.max(1, maxJobSeconds - jobRunTime);
135
+ // const secondsUntilKill = Math.max(1, this.opt.killAfter - jobRunTime)
136
+ job.visibilityTimeout = Math.min(doubled, secondsUntilMax); //, secondsUntilKill)
137
+ job.extendAtSecond = Math.round(jobRunTime + job.visibilityTimeout); // this is what we use next time
138
+ debug({ doubled: doubled, secondsUntilMax: secondsUntilMax, job: job });
139
+ }
120
140
  }
121
141
  }
122
- debug('maintainVisibility', { jobsToDeleteByQrl: jobsToDeleteByQrl, jobsToExtendByQrl: jobsToExtendByQrl });
123
- _b = jobsToExtendByQrl;
124
- _c = [];
125
- for (_d in _b)
126
- _c.push(_d);
127
- _e = 0;
128
- _k.label = 1;
142
+ _a = jobsToExtendByQrl;
143
+ _b = [];
144
+ for (_c in _a)
145
+ _b.push(_c);
146
+ _i = 0;
147
+ _h.label = 1;
129
148
  case 1:
130
- if (!(_e < _c.length)) return [3 /*break*/, 5];
131
- _d = _c[_e];
132
- if (!(_d in _b)) return [3 /*break*/, 4];
133
- qrl = _d;
149
+ if (!(_i < _b.length)) return [3 /*break*/, 5];
150
+ _c = _b[_i];
151
+ if (!(_c in _a)) return [3 /*break*/, 4];
152
+ qrl = _c;
134
153
  jobsToExtend = jobsToExtendByQrl[qrl];
135
154
  debug({ qrl: qrl, jobsToExtend: jobsToExtend });
136
- _k.label = 2;
155
+ _h.label = 2;
137
156
  case 2:
138
157
  if (!jobsToExtend.length) return [3 /*break*/, 4];
139
158
  entries = [];
@@ -147,11 +166,12 @@ var JobExecutor = /** @class */ (function () {
147
166
  };
148
167
  entries.push(entry);
149
168
  }
169
+ debug({ entries: entries });
150
170
  input = { QueueUrl: qrl, Entries: entries };
151
171
  debug({ ChangeMessageVisibilityBatch: input });
152
172
  return [4 /*yield*/, (0, sqs_js_1.getSQSClient)().send(new client_sqs_1.ChangeMessageVisibilityBatchCommand(input))];
153
173
  case 3:
154
- result = _k.sent();
174
+ result = _h.sent();
155
175
  debug('ChangeMessageVisibilityBatch returned', result);
156
176
  this.stats.sqsCalls++;
157
177
  if (result.Successful) {
@@ -161,27 +181,27 @@ var JobExecutor = /** @class */ (function () {
161
181
  console.error(chalk_1.default.blue('Extended'), count, chalk_1.default.blue('jobs'));
162
182
  }
163
183
  else if (!this.opt.disableLog) {
164
- console.log(JSON.stringify({ event: 'EXTEND_VISIBILITY_TIMEOUTS', timestamp: now, count: count, qrl: qrl }));
184
+ console.log(JSON.stringify({ event: 'EXTEND_VISIBILITY_TIMEOUTS', timestamp: start, count: count, qrl: qrl }));
165
185
  }
166
186
  }
167
187
  return [3 /*break*/, 2];
168
188
  case 4:
169
- _e++;
189
+ _i++;
170
190
  return [3 /*break*/, 1];
171
191
  case 5:
172
- _f = jobsToDeleteByQrl;
173
- _g = [];
174
- for (_h in _f)
175
- _g.push(_h);
176
- _j = 0;
177
- _k.label = 6;
192
+ _d = jobsToDeleteByQrl;
193
+ _e = [];
194
+ for (_f in _d)
195
+ _e.push(_f);
196
+ _g = 0;
197
+ _h.label = 6;
178
198
  case 6:
179
- if (!(_j < _g.length)) return [3 /*break*/, 10];
180
- _h = _g[_j];
181
- if (!(_h in _f)) return [3 /*break*/, 9];
182
- qrl = _h;
199
+ if (!(_g < _e.length)) return [3 /*break*/, 10];
200
+ _f = _e[_g];
201
+ if (!(_f in _d)) return [3 /*break*/, 9];
202
+ qrl = _f;
183
203
  jobsToDelete = jobsToDeleteByQrl[qrl];
184
- _k.label = 7;
204
+ _h.label = 7;
185
205
  case 7:
186
206
  if (!jobsToDelete.length) return [3 /*break*/, 9];
187
207
  entries = [];
@@ -195,11 +215,12 @@ var JobExecutor = /** @class */ (function () {
195
215
  };
196
216
  entries.push(entry);
197
217
  }
218
+ debug({ entries: entries });
198
219
  input = { QueueUrl: qrl, Entries: entries };
199
220
  debug({ DeleteMessageBatch: input });
200
221
  return [4 /*yield*/, (0, sqs_js_1.getSQSClient)().send(new client_sqs_1.DeleteMessageBatchCommand(input))];
201
222
  case 8:
202
- result = _k.sent();
223
+ result = _h.sent();
203
224
  this.stats.sqsCalls++;
204
225
  if (result.Successful) {
205
226
  count = result.Successful.length || 0;
@@ -208,22 +229,38 @@ var JobExecutor = /** @class */ (function () {
208
229
  console.error(chalk_1.default.blue('Deleted'), count, chalk_1.default.blue('jobs'));
209
230
  }
210
231
  else if (!this.opt.disableLog) {
211
- console.log(JSON.stringify({ event: 'DELETE_MESSAGES', timestamp: now, count: count, qrl: qrl }));
232
+ console.log(JSON.stringify({ event: 'DELETE_MESSAGES', timestamp: start, count: count, qrl: qrl }));
212
233
  }
213
234
  }
214
235
  debug('DeleteMessageBatch returned', result);
215
236
  return [3 /*break*/, 7];
216
237
  case 9:
217
- _j++;
238
+ _g++;
218
239
  return [3 /*break*/, 6];
219
240
  case 10:
220
241
  // Get rid of deleted and failed jobs
221
- this.jobs = this.jobs.filter(function (j) { return j.status === 'processing'; });
222
- // Check again later, unless we are shutting down and nothing's left
242
+ this.jobs = this.jobs.filter(function (job) {
243
+ if (job.status === 'deleting' || job.status === 'failed') {
244
+ debug('removed', job.message.MessageId);
245
+ delete _this.jobsByMessageId[job.message.MessageId];
246
+ return false;
247
+ }
248
+ else {
249
+ return true;
250
+ }
251
+ });
252
+ // Bail if we are shutting down
223
253
  if (this.shutdownRequested && this.stats.activeJobs === 0 && this.jobs.length === 0)
224
254
  return [2 /*return*/];
225
- nextCheckInMs = this.shutdownRequested ? 1 * 1000 : 10 * 1000;
226
- this.maintainVisibilityTimeout = setTimeout(function () { return _this.maintainVisibility(); }, nextCheckInMs);
255
+ msElapsed = new Date() - start;
256
+ msPeriod = this.shutdownRequested ? 1 * 1000 : 10 * 1000;
257
+ msLeft = Math.max(0, msPeriod - msElapsed);
258
+ msMin = this.shutdownRequested ? 1000 : 0;
259
+ nextCheckInMs = Math.max(msMin, msLeft);
260
+ debug({ msElapsed: msElapsed, msPeriod: msPeriod, msLeft: msLeft, msMin: msMin, nextCheckInMs: nextCheckInMs });
261
+ this.maintainVisibilityTimeout = setTimeout(function () {
262
+ _this.maintainPromise = _this.maintainVisibility();
263
+ }, nextCheckInMs);
227
264
  return [2 /*return*/];
228
265
  }
229
266
  });
@@ -231,16 +268,16 @@ var JobExecutor = /** @class */ (function () {
231
268
  };
232
269
  JobExecutor.prototype.executeJob = function (message, callback, qname, qrl) {
233
270
  return __awaiter(this, void 0, void 0, function () {
234
- var payload, visibilityTimeout, job, queue, result, err_1;
271
+ var payload, visibilityTimeout, job, oldJob, e, queue, result, err_1;
235
272
  return __generator(this, function (_a) {
236
273
  switch (_a.label) {
237
274
  case 0:
238
275
  payload = this.opt.json ? JSON.parse(message.Body) : message.Body;
239
- visibilityTimeout = 30;
276
+ visibilityTimeout = 60;
240
277
  job = {
241
278
  status: 'processing',
242
279
  start: new Date(),
243
- visibilityTimeout: 30,
280
+ visibilityTimeout: visibilityTimeout,
244
281
  extendAtSecond: visibilityTimeout / 2,
245
282
  payload: this.opt.json ? JSON.parse(message.Body) : message.Body,
246
283
  message: message,
@@ -248,8 +285,20 @@ var JobExecutor = /** @class */ (function () {
248
285
  qname: qname,
249
286
  qrl: qrl
250
287
  };
251
- debug('executeJob', job);
288
+ oldJob = this.jobsByMessageId[job.message.MessageId];
289
+ if (oldJob) {
290
+ // If we actually see the same job again, we fucked up, probably due to
291
+ // the system being overloaded and us missing our extension call. So
292
+ // we'll celebrate this occasion by throwing a big fat error.
293
+ debug({ oldJob: oldJob });
294
+ e = new Error("Saw job ".concat(oldJob.message.MessageId, " twice"));
295
+ e.job = oldJob;
296
+ // TODO: sentry breadcrumb
297
+ throw e;
298
+ }
299
+ // debug('executeJob', job)
252
300
  this.jobs.push(job);
301
+ this.jobsByMessageId[job.message.MessageId] = job;
253
302
  this.stats.activeJobs++;
254
303
  if (this.opt.verbose) {
255
304
  console.error(chalk_1.default.blue('Executing:'), qname, chalk_1.default.blue('-->'), job.payload);
@@ -291,7 +340,7 @@ var JobExecutor = /** @class */ (function () {
291
340
  return [3 /*break*/, 4];
292
341
  case 3:
293
342
  err_1 = _a.sent();
294
- debug('exec.catch');
343
+ // debug('exec.catch', err)
295
344
  // Fail path for job execution
296
345
  if (this.opt.verbose) {
297
346
  console.error(chalk_1.default.red('FAILED'), message.Body);