qdone 2.0.12-alpha → 2.0.13-alpha

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (7) hide show
  1. package/commonjs/src/consumer.js +2 -3
  2. package/commonjs/src/defaults.js +2 -0
  3. package/commonjs/src/scheduler/jobExecutor.js +43 -56
  4. package/package.json +1 -1
  5. package/src/consumer.js +1 -2
  6. package/src/defaults.js +2 -0
  7. package/src/scheduler/jobExecutor.js +25 -26
package/commonjs/src/consumer.js CHANGED
@@ -98,7 +98,7 @@ exports.getMessages = getMessages;
98
98
  //
99
99
  function processMessages(queues, callback, options) {
100
100
  return __awaiter(this, void 0, void 0, function () {
101
- var opt, systemMonitor, jobExecutor, queueManager, maxActiveJobs, delayTimeout, delay, activeQrls, maxReturnCount, listen, allowedJobs, maxLatency, latency, latencyFactor, targetJobs, jobsLeft, _i, _a, _b, qname, qrl, maxMessages;
101
+ var opt, systemMonitor, jobExecutor, queueManager, delayTimeout, delay, activeQrls, maxReturnCount, listen, allowedJobs, maxLatency, latency, latencyFactor, targetJobs, jobsLeft, _i, _a, _b, qname, qrl, maxMessages;
102
102
  var _this = this;
103
103
  return __generator(this, function (_c) {
104
104
  switch (_c.label) {
@@ -108,7 +108,6 @@ function processMessages(queues, callback, options) {
108
108
  systemMonitor = new systemMonitor_js_1.SystemMonitor(opt);
109
109
  jobExecutor = new jobExecutor_js_1.JobExecutor(opt);
110
110
  queueManager = new queueManager_js_1.QueueManager(opt, queues);
111
- maxActiveJobs = 100;
112
111
  debug({ systemMonitor: systemMonitor, jobExecutor: jobExecutor, queueManager: queueManager });
113
112
  shutdownCallbacks.push(function () {
114
113
  systemMonitor.shutdown();
@@ -143,7 +142,7 @@ function processMessages(queues, callback, options) {
143
142
  _c.label = 1;
144
143
  case 1:
145
144
  if (!!shutdownRequested) return [3 /*break*/, 3];
146
- allowedJobs = maxActiveJobs - jobExecutor.activeJobCount() - maxReturnCount;
145
+ allowedJobs = opt.maxConcurrentJobs - jobExecutor.activeJobCount() - maxReturnCount;
147
146
  maxLatency = 100;
148
147
  latency = systemMonitor.getLatency().setTimeout;
149
148
  latencyFactor = 1 - Math.abs(Math.min(latency / maxLatency, 1)) // 0 if latency is at max, 1 if latency 0
package/commonjs/src/defaults.js CHANGED
@@ -36,6 +36,7 @@ exports.defaults = Object.freeze({
36
36
  killAfter: 30,
37
37
  archive: false,
38
38
  activeOnly: false,
39
+ maxConcurrentJobs: 100,
39
40
  // Idle Queues
40
41
  idleFor: 60,
41
42
  delete: false,
@@ -87,6 +88,7 @@ function getOptionsWithDefaults(options) {
87
88
  archive: options.archive || exports.defaults.archive,
88
89
  activeOnly: options.activeOnly || options['active-only'] || exports.defaults.activeOnly,
89
90
  includeFailed: options.includeFailed || options['include-failed'] || exports.defaults.includeFailed,
91
+ maxConcurrentJobs: options.maxConcurrentJobs || exports.defaults.maxConcurrentJobs,
90
92
  // Idle Queues
91
93
  idleFor: options.idleFor || options['idle-for'] || exports.defaults.idleFor,
92
94
  delete: options.delete || exports.defaults.delete,
package/commonjs/src/scheduler/jobExecutor.js CHANGED
@@ -67,9 +67,9 @@ var JobExecutor = /** @class */ (function () {
67
67
  }
68
68
  JobExecutor.prototype.shutdown = function () {
69
69
  this.shutdownRequested = true;
70
- if (this.stats.activeJobs === 0 && this.jobs.length === 0) {
71
- clearTimeout(this.maintainVisibilityTimeout);
72
- }
70
+ // Trigger a maintenance run right away in case it speeds us up
71
+ clearTimeout(this.maintainVisibilityTimeout);
72
+ this.maintainVisibility();
73
73
  };
74
74
  JobExecutor.prototype.activeJobCount = function () {
75
75
  return this.stats.activeJobs;
@@ -79,7 +79,7 @@ var JobExecutor = /** @class */ (function () {
79
79
  */
80
80
  JobExecutor.prototype.maintainVisibility = function () {
81
81
  return __awaiter(this, void 0, void 0, function () {
82
- var now, jobsToExtendByQrl, jobsToDeleteByQrl, jobsToCleanup, _i, _a, job, jobRunTime, jobsToDelete, jobsToExtend, doubled, secondsUntilMax, secondsUntilKill, _b, _c, _d, _e, qrl, jobsToExtend, entries, messageId, job, entry, input, result, _f, _g, _h, _j, qrl, jobsToDelete, entries, messageId, job, entry, input, result;
82
+ var now, jobsToExtendByQrl, jobsToDeleteByQrl, jobsToCleanup, _i, _a, job, jobRunTime, jobsToDelete, jobsToExtend, doubled, secondsUntilMax, secondsUntilKill, _b, _c, _d, _e, qrl, jobsToExtend, entries, messageId, job, entry, input, result, count, _f, _g, _h, _j, qrl, jobsToDelete, entries, messageId, job, entry, input, result, count, nextCheckInMs;
83
83
  var _this = this;
84
84
  return __generator(this, function (_k) {
85
85
  switch (_k.label) {
@@ -127,9 +127,9 @@ var JobExecutor = /** @class */ (function () {
127
127
  _e = 0;
128
128
  _k.label = 1;
129
129
  case 1:
130
- if (!(_e < _c.length)) return [3 /*break*/, 6];
130
+ if (!(_e < _c.length)) return [3 /*break*/, 5];
131
131
  _d = _c[_e];
132
- if (!(_d in _b)) return [3 /*break*/, 5];
132
+ if (!(_d in _b)) return [3 /*break*/, 4];
133
133
  qrl = _d;
134
134
  jobsToExtend = jobsToExtendByQrl[qrl];
135
135
  debug({ qrl: qrl, jobsToExtend: jobsToExtend });
@@ -154,43 +154,36 @@ var JobExecutor = /** @class */ (function () {
154
154
  result = _k.sent();
155
155
  debug('ChangeMessageVisibilityBatch returned', result);
156
156
  this.stats.sqsCalls++;
157
- if (result.Successful)
158
- this.stats.timeoutsExtended += result.Successful.length || 0;
157
+ if (result.Successful) {
158
+ count = result.Successful.length || 0;
159
+ this.stats.timeoutsExtended += count;
160
+ if (this.opt.verbose) {
161
+ console.error(chalk_1.default.blue('Extended'), count, chalk_1.default.blue('jobs'));
162
+ }
163
+ else if (!this.opt.disableLog) {
164
+ console.log(JSON.stringify({ event: 'EXTEND_VISIBILITY_TIMEOUTS', timestamp: now, count: count, qrl: qrl }));
165
+ }
166
+ }
159
167
  return [3 /*break*/, 2];
160
168
  case 4:
161
- if (this.opt.verbose) {
162
- console.error(chalk_1.default.blue('Extended these jobs: '), jobsToExtend);
163
- }
164
- else if (!this.opt.disableLog) {
165
- console.log(JSON.stringify({
166
- event: 'EXTEND_VISIBILITY_TIMEOUTS',
167
- timestamp: now,
168
- messageIds: jobsToExtend.map(function (_a) {
169
- var message = _a.message;
170
- return message.MessageId;
171
- })
172
- }));
173
- }
174
- _k.label = 5;
175
- case 5:
176
169
  _e++;
177
170
  return [3 /*break*/, 1];
178
- case 6:
171
+ case 5:
179
172
  _f = jobsToDeleteByQrl;
180
173
  _g = [];
181
174
  for (_h in _f)
182
175
  _g.push(_h);
183
176
  _j = 0;
184
- _k.label = 7;
185
- case 7:
186
- if (!(_j < _g.length)) return [3 /*break*/, 12];
177
+ _k.label = 6;
178
+ case 6:
179
+ if (!(_j < _g.length)) return [3 /*break*/, 10];
187
180
  _h = _g[_j];
188
- if (!(_h in _f)) return [3 /*break*/, 11];
181
+ if (!(_h in _f)) return [3 /*break*/, 9];
189
182
  qrl = _h;
190
183
  jobsToDelete = jobsToDeleteByQrl[qrl];
191
- _k.label = 8;
192
- case 8:
193
- if (!jobsToDelete.length) return [3 /*break*/, 10];
184
+ _k.label = 7;
185
+ case 7:
186
+ if (!jobsToDelete.length) return [3 /*break*/, 9];
194
187
  entries = [];
195
188
  messageId = 0;
196
189
  while (messageId++ < 10 && jobsToDelete.length) {
@@ -205,38 +198,32 @@ var JobExecutor = /** @class */ (function () {
205
198
  input = { QueueUrl: qrl, Entries: entries };
206
199
  debug({ DeleteMessageBatch: input });
207
200
  return [4 /*yield*/, (0, sqs_js_1.getSQSClient)().send(new client_sqs_1.DeleteMessageBatchCommand(input))];
208
- case 9:
201
+ case 8:
209
202
  result = _k.sent();
210
203
  this.stats.sqsCalls++;
211
- if (result.Successful)
212
- this.stats.jobsDeleted += result.Successful.length || 0;
213
- debug('DeleteMessageBatch returned', result);
214
- return [3 /*break*/, 8];
215
- case 10:
216
- if (this.opt.verbose) {
217
- console.error(chalk_1.default.blue('Deleted these finished jobs: '), jobsToDelete);
218
- }
219
- else if (!this.opt.disableLog) {
220
- console.log(JSON.stringify({
221
- event: 'DELETE_MESSAGES',
222
- timestamp: now,
223
- messageIds: jobsToDelete.map(function (_a) {
224
- var message = _a.message;
225
- return message.MessageId;
226
- })
227
- }));
204
+ if (result.Successful) {
205
+ count = result.Successful.length || 0;
206
+ this.stats.jobsDeleted += count;
207
+ if (this.opt.verbose) {
208
+ console.error(chalk_1.default.blue('Deleted'), count, chalk_1.default.blue('jobs'));
209
+ }
210
+ else if (!this.opt.disableLog) {
211
+ console.log(JSON.stringify({ event: 'DELETE_MESSAGES', timestamp: now, count: count, qrl: qrl }));
212
+ }
228
213
  }
229
- _k.label = 11;
230
- case 11:
231
- _j++;
214
+ debug('DeleteMessageBatch returned', result);
232
215
  return [3 /*break*/, 7];
233
- case 12:
216
+ case 9:
217
+ _j++;
218
+ return [3 /*break*/, 6];
219
+ case 10:
234
220
  // Get rid of deleted and failed jobs
235
221
  this.jobs = this.jobs.filter(function (j) { return j.status === 'processing'; });
236
222
  // Check again later, unless we are shutting down and nothing's left
237
223
  if (this.shutdownRequested && this.stats.activeJobs === 0 && this.jobs.length === 0)
238
224
  return [2 /*return*/];
239
- this.maintainVisibilityTimeout = setTimeout(function () { return _this.maintainVisibility(); }, 10 * 1000);
225
+ nextCheckInMs = this.shutdownRequested ? 1 * 1000 : 10 * 1000;
226
+ this.maintainVisibilityTimeout = setTimeout(function () { return _this.maintainVisibility(); }, nextCheckInMs);
240
227
  return [2 /*return*/];
241
228
  }
242
229
  });
@@ -271,7 +258,7 @@ var JobExecutor = /** @class */ (function () {
271
258
  console.log(JSON.stringify({
272
259
  event: 'MESSAGE_PROCESSING_START',
273
260
  timestamp: new Date(),
274
- qname: qname,
261
+ qrl: qrl,
275
262
  messageId: message.MessageId,
276
263
  payload: job.payload
277
264
  }));
@@ -315,7 +302,7 @@ var JobExecutor = /** @class */ (function () {
315
302
  console.log(JSON.stringify({
316
303
  event: 'MESSAGE_PROCESSING_FAILED',
317
304
  reason: 'exception thrown',
318
- qname: qname,
305
+ qrl: qrl,
319
306
  timestamp: new Date(),
320
307
  messageId: message.MessageId,
321
308
  payload: payload,
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "qdone",
3
- "version": "2.0.12-alpha",
3
+ "version": "2.0.13-alpha",
4
4
  "description": "Language agnostic job queue for SQS",
5
5
  "type": "module",
6
6
  "main": "./index.js",
package/src/consumer.js CHANGED
@@ -53,7 +53,6 @@ export async function processMessages (queues, callback, options) {
53
53
  const systemMonitor = new SystemMonitor(opt)
54
54
  const jobExecutor = new JobExecutor(opt)
55
55
  const queueManager = new QueueManager(opt, queues)
56
- const maxActiveJobs = 100
57
56
  debug({ systemMonitor, jobExecutor, queueManager })
58
57
 
59
58
  shutdownCallbacks.push(() => {
@@ -85,7 +84,7 @@ export async function processMessages (queues, callback, options) {
85
84
 
86
85
  while (!shutdownRequested) { // eslint-disable-line
87
86
  // Figure out how we are running
88
- const allowedJobs = maxActiveJobs - jobExecutor.activeJobCount() - maxReturnCount
87
+ const allowedJobs = opt.maxConcurrentJobs - jobExecutor.activeJobCount() - maxReturnCount
89
88
  const maxLatency = 100
90
89
  const latency = systemMonitor.getLatency().setTimeout
91
90
  const latencyFactor = 1 - Math.abs(Math.min(latency / maxLatency, 1)) // 0 if latency is at max, 1 if latency 0
package/src/defaults.js CHANGED
@@ -36,6 +36,7 @@ export const defaults = Object.freeze({
36
36
  killAfter: 30,
37
37
  archive: false,
38
38
  activeOnly: false,
39
+ maxConcurrentJobs: 100,
39
40
 
40
41
  // Idle Queues
41
42
  idleFor: 60,
@@ -93,6 +94,7 @@ export function getOptionsWithDefaults (options) {
93
94
  archive: options.archive || defaults.archive,
94
95
  activeOnly: options.activeOnly || options['active-only'] || defaults.activeOnly,
95
96
  includeFailed: options.includeFailed || options['include-failed'] || defaults.includeFailed,
97
+ maxConcurrentJobs: options.maxConcurrentJobs || defaults.maxConcurrentJobs,
96
98
 
97
99
  // Idle Queues
98
100
  idleFor: options.idleFor || options['idle-for'] || defaults.idleFor,
package/src/scheduler/jobExecutor.js CHANGED
@@ -34,9 +34,9 @@ export class JobExecutor {
34
34
 
35
35
  shutdown () {
36
36
  this.shutdownRequested = true
37
- if (this.stats.activeJobs === 0 && this.jobs.length === 0) {
38
- clearTimeout(this.maintainVisibilityTimeout)
39
- }
37
+ // Trigger a maintenance run right away in case it speeds us up
38
+ clearTimeout(this.maintainVisibilityTimeout)
39
+ this.maintainVisibility()
40
40
  }
41
41
 
42
42
  activeJobCount () {
@@ -107,18 +107,17 @@ export class JobExecutor {
107
107
  const result = await getSQSClient().send(new ChangeMessageVisibilityBatchCommand(input))
108
108
  debug('ChangeMessageVisibilityBatch returned', result)
109
109
  this.stats.sqsCalls++
110
- if (result.Successful) this.stats.timeoutsExtended += result.Successful.length || 0
110
+ if (result.Successful) {
111
+ const count = result.Successful.length || 0
112
+ this.stats.timeoutsExtended += count
113
+ if (this.opt.verbose) {
114
+ console.error(chalk.blue('Extended'), count, chalk.blue('jobs'))
115
+ } else if (!this.opt.disableLog) {
116
+ console.log(JSON.stringify({ event: 'EXTEND_VISIBILITY_TIMEOUTS', timestamp: now, count, qrl }))
117
+ }
118
+ }
111
119
  // TODO Sentry
112
120
  }
113
- if (this.opt.verbose) {
114
- console.error(chalk.blue('Extended these jobs: '), jobsToExtend)
115
- } else if (!this.opt.disableLog) {
116
- console.log(JSON.stringify({
117
- event: 'EXTEND_VISIBILITY_TIMEOUTS',
118
- timestamp: now,
119
- messageIds: jobsToExtend.map(({ message }) => message.MessageId)
120
- }))
121
- }
122
121
  }
123
122
 
124
123
  // Delete in batches for each queue
@@ -143,19 +142,18 @@ export class JobExecutor {
143
142
  debug({ DeleteMessageBatch: input })
144
143
  const result = await getSQSClient().send(new DeleteMessageBatchCommand(input))
145
144
  this.stats.sqsCalls++
146
- if (result.Successful) this.stats.jobsDeleted += result.Successful.length || 0
145
+ if (result.Successful) {
146
+ const count = result.Successful.length || 0
147
+ this.stats.jobsDeleted += count
148
+ if (this.opt.verbose) {
149
+ console.error(chalk.blue('Deleted'), count, chalk.blue('jobs'))
150
+ } else if (!this.opt.disableLog) {
151
+ console.log(JSON.stringify({ event: 'DELETE_MESSAGES', timestamp: now, count, qrl }))
152
+ }
153
+ }
147
154
  debug('DeleteMessageBatch returned', result)
148
155
  // TODO Sentry
149
156
  }
150
- if (this.opt.verbose) {
151
- console.error(chalk.blue('Deleted these finished jobs: '), jobsToDelete)
152
- } else if (!this.opt.disableLog) {
153
- console.log(JSON.stringify({
154
- event: 'DELETE_MESSAGES',
155
- timestamp: now,
156
- messageIds: jobsToDelete.map(({ message }) => message.MessageId)
157
- }))
158
- }
159
157
  }
160
158
 
161
159
  // Get rid of deleted and failed jobs
@@ -163,7 +161,8 @@ export class JobExecutor {
163
161
 
164
162
  // Check again later, unless we are shutting down and nothing's left
165
163
  if (this.shutdownRequested && this.stats.activeJobs === 0 && this.jobs.length === 0) return
166
- this.maintainVisibilityTimeout = setTimeout(() => this.maintainVisibility(), 10 * 1000)
164
+ const nextCheckInMs = this.shutdownRequested ? 1 * 1000 : 10 * 1000
165
+ this.maintainVisibilityTimeout = setTimeout(() => this.maintainVisibility(), nextCheckInMs)
167
166
  }
168
167
 
169
168
  async executeJob (message, callback, qname, qrl) {
@@ -190,7 +189,7 @@ export class JobExecutor {
190
189
  console.log(JSON.stringify({
191
190
  event: 'MESSAGE_PROCESSING_START',
192
191
  timestamp: new Date(),
193
- qname,
192
+ qrl,
194
193
  messageId: message.MessageId,
195
194
  payload: job.payload
196
195
  }))
@@ -229,7 +228,7 @@ export class JobExecutor {
229
228
  console.log(JSON.stringify({
230
229
  event: 'MESSAGE_PROCESSING_FAILED',
231
230
  reason: 'exception thrown',
232
- qname,
231
+ qrl,
233
232
  timestamp: new Date(),
234
233
  messageId: message.MessageId,
235
234
  payload,