qdone 2.0.10-alpha → 2.0.12-alpha

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (5) hide show
  1. package/commonjs/src/consumer.js +1 -1
  2. package/commonjs/src/scheduler/jobExecutor.js +45 -28
  3. package/package.json +1 -1
  4. package/src/consumer.js +1 -1
  5. package/src/scheduler/jobExecutor.js +14 -11
package/commonjs/src/consumer.js CHANGED
@@ -158,7 +158,7 @@ function processMessages(queues, callback, options) {
158
158
  maxMessages = Math.min(10, jobsLeft);
159
159
  listen(qname, qrl, maxMessages);
160
160
  jobsLeft -= maxMessages;
161
- if (this.opt.verbose) {
161
+ if (opt.verbose) {
162
162
  console.error(chalk_1.default.blue('Listening on: '), qname);
163
163
  }
164
164
  debug({ listenedTo: { qname: qname, maxMessages: maxMessages, jobsLeft: jobsLeft } });
package/commonjs/src/scheduler/jobExecutor.js CHANGED
@@ -79,19 +79,19 @@ var JobExecutor = /** @class */ (function () {
79
79
  */
80
80
  JobExecutor.prototype.maintainVisibility = function () {
81
81
  return __awaiter(this, void 0, void 0, function () {
82
- var now, jobsToExtendByQrl, jobsToDeleteByQrl, jobsToCleanup, _i, _a, job, jobRunTime, jobsToDelete, jobsToExtend, doubled, secondsUntilMax, secondsUntilKill, _b, jobsToExtendByQrl_1, _c, qrl, jobsToExtend, entries, messageId, job, entry, input, result, _d, jobsToDeleteByQrl_1, _e, qrl, jobsToDelete, entries, messageId, job, entry, input, result;
82
+ var now, jobsToExtendByQrl, jobsToDeleteByQrl, jobsToCleanup, _i, _a, job, jobRunTime, jobsToDelete, jobsToExtend, doubled, secondsUntilMax, secondsUntilKill, _b, _c, _d, _e, qrl, jobsToExtend, entries, messageId, job, entry, input, result, _f, _g, _h, _j, qrl, jobsToDelete, entries, messageId, job, entry, input, result;
83
83
  var _this = this;
84
- return __generator(this, function (_f) {
85
- switch (_f.label) {
84
+ return __generator(this, function (_k) {
85
+ switch (_k.label) {
86
86
  case 0:
87
87
  debug('maintainVisibility', this.jobs);
88
88
  now = new Date();
89
- jobsToExtendByQrl = new Map();
90
- jobsToDeleteByQrl = new Map();
89
+ jobsToExtendByQrl = {};
90
+ jobsToDeleteByQrl = {};
91
91
  jobsToCleanup = new Set();
92
92
  if (this.opt.verbose) {
93
93
  console.error(chalk_1.default.blue('Stats: '), this.stats);
94
- console.error(chalk_1.default.blue('Running: '), this.jobs.map(function (_a) {
94
+ console.error(chalk_1.default.blue('Running: '), this.jobs.filter(function (j) { return j.status === 'processing'; }).map(function (_a) {
95
95
  var qname = _a.qname, message = _a.message;
96
96
  return ({ qname: qname, payload: message.Body });
97
97
  }));
@@ -101,17 +101,17 @@ var JobExecutor = /** @class */ (function () {
101
101
  job = _a[_i];
102
102
  jobRunTime = (now - job.start) / 1000;
103
103
  if (job.status === 'complete') {
104
- jobsToDelete = jobsToDeleteByQrl.get(job.qrl) || [];
104
+ jobsToDelete = jobsToDeleteByQrl[job.qrl] || [];
105
105
  jobsToDelete.push(job);
106
- jobsToDeleteByQrl.set(job.qrl, jobsToDelete);
106
+ jobsToDeleteByQrl[job.qrl] = jobsToDelete;
107
107
  }
108
108
  else if (job.status === 'failed') {
109
109
  jobsToCleanup.add(job);
110
110
  }
111
111
  else if (jobRunTime >= job.exendAtSecond) {
112
- jobsToExtend = jobsToExtendByQrl.get(job.qrl) || [];
112
+ jobsToExtend = jobsToExtendByQrl[job.qrl] || [];
113
113
  jobsToExtend.push(job);
114
- jobsToExtendByQrl.set(job.qrl, jobsToExtend);
114
+ jobsToExtendByQrl[job.qrl] = jobsToExtend;
115
115
  doubled = job.visibilityTimeout * 2;
116
116
  secondsUntilMax = maxJobSeconds - jobRunTime;
117
117
  secondsUntilKill = this.opt.killAfter - jobRunTime;
@@ -120,12 +120,20 @@ var JobExecutor = /** @class */ (function () {
120
120
  }
121
121
  }
122
122
  debug('maintainVisibility', { jobsToDeleteByQrl: jobsToDeleteByQrl, jobsToExtendByQrl: jobsToExtendByQrl });
123
- _b = 0, jobsToExtendByQrl_1 = jobsToExtendByQrl;
124
- _f.label = 1;
123
+ _b = jobsToExtendByQrl;
124
+ _c = [];
125
+ for (_d in _b)
126
+ _c.push(_d);
127
+ _e = 0;
128
+ _k.label = 1;
125
129
  case 1:
126
- if (!(_b < jobsToExtendByQrl_1.length)) return [3 /*break*/, 6];
127
- _c = jobsToExtendByQrl_1[_b], qrl = _c[0], jobsToExtend = _c[1];
128
- _f.label = 2;
130
+ if (!(_e < _c.length)) return [3 /*break*/, 6];
131
+ _d = _c[_e];
132
+ if (!(_d in _b)) return [3 /*break*/, 5];
133
+ qrl = _d;
134
+ jobsToExtend = jobsToExtendByQrl[qrl];
135
+ debug({ qrl: qrl, jobsToExtend: jobsToExtend });
136
+ _k.label = 2;
129
137
  case 2:
130
138
  if (!jobsToExtend.length) return [3 /*break*/, 4];
131
139
  entries = [];
@@ -143,10 +151,11 @@ var JobExecutor = /** @class */ (function () {
143
151
  debug({ ChangeMessageVisibilityBatch: input });
144
152
  return [4 /*yield*/, (0, sqs_js_1.getSQSClient)().send(new client_sqs_1.ChangeMessageVisibilityBatchCommand(input))];
145
153
  case 3:
146
- result = _f.sent();
154
+ result = _k.sent();
147
155
  debug('ChangeMessageVisibilityBatch returned', result);
148
156
  this.stats.sqsCalls++;
149
- this.stats.timeoutsExtended += 10;
157
+ if (result.Successful)
158
+ this.stats.timeoutsExtended += result.Successful.length || 0;
150
159
  return [3 /*break*/, 2];
151
160
  case 4:
152
161
  if (this.opt.verbose) {
@@ -162,17 +171,24 @@ var JobExecutor = /** @class */ (function () {
162
171
  })
163
172
  }));
164
173
  }
165
- _f.label = 5;
174
+ _k.label = 5;
166
175
  case 5:
167
- _b++;
176
+ _e++;
168
177
  return [3 /*break*/, 1];
169
178
  case 6:
170
- _d = 0, jobsToDeleteByQrl_1 = jobsToDeleteByQrl;
171
- _f.label = 7;
179
+ _f = jobsToDeleteByQrl;
180
+ _g = [];
181
+ for (_h in _f)
182
+ _g.push(_h);
183
+ _j = 0;
184
+ _k.label = 7;
172
185
  case 7:
173
- if (!(_d < jobsToDeleteByQrl_1.length)) return [3 /*break*/, 12];
174
- _e = jobsToDeleteByQrl_1[_d], qrl = _e[0], jobsToDelete = _e[1];
175
- _f.label = 8;
186
+ if (!(_j < _g.length)) return [3 /*break*/, 12];
187
+ _h = _g[_j];
188
+ if (!(_h in _f)) return [3 /*break*/, 11];
189
+ qrl = _h;
190
+ jobsToDelete = jobsToDeleteByQrl[qrl];
191
+ _k.label = 8;
176
192
  case 8:
177
193
  if (!jobsToDelete.length) return [3 /*break*/, 10];
178
194
  entries = [];
@@ -190,9 +206,10 @@ var JobExecutor = /** @class */ (function () {
190
206
  debug({ DeleteMessageBatch: input });
191
207
  return [4 /*yield*/, (0, sqs_js_1.getSQSClient)().send(new client_sqs_1.DeleteMessageBatchCommand(input))];
192
208
  case 9:
193
- result = _f.sent();
209
+ result = _k.sent();
194
210
  this.stats.sqsCalls++;
195
- this.stats.jobsDeleted += 10;
211
+ if (result.Successful)
212
+ this.stats.jobsDeleted += result.Successful.length || 0;
196
213
  debug('DeleteMessageBatch returned', result);
197
214
  return [3 /*break*/, 8];
198
215
  case 10:
@@ -209,9 +226,9 @@ var JobExecutor = /** @class */ (function () {
209
226
  })
210
227
  }));
211
228
  }
212
- _f.label = 11;
229
+ _k.label = 11;
213
230
  case 11:
214
- _d++;
231
+ _j++;
215
232
  return [3 /*break*/, 7];
216
233
  case 12:
217
234
  // Get rid of deleted and failed jobs
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "qdone",
3
- "version": "2.0.10-alpha",
3
+ "version": "2.0.12-alpha",
4
4
  "description": "Language agnostic job queue for SQS",
5
5
  "type": "module",
6
6
  "main": "./index.js",
package/src/consumer.js CHANGED
@@ -98,7 +98,7 @@ export async function processMessages (queues, callback, options) {
98
98
  const maxMessages = Math.min(10, jobsLeft)
99
99
  listen(qname, qrl, maxMessages)
100
100
  jobsLeft -= maxMessages
101
- if (this.opt.verbose) {
101
+ if (opt.verbose) {
102
102
  console.error(chalk.blue('Listening on: '), qname)
103
103
  }
104
104
  debug({ listenedTo: { qname, maxMessages, jobsLeft } })
package/src/scheduler/jobExecutor.js CHANGED
@@ -49,29 +49,29 @@ export class JobExecutor {
49
49
  async maintainVisibility () {
50
50
  debug('maintainVisibility', this.jobs)
51
51
  const now = new Date()
52
- const jobsToExtendByQrl = new Map()
53
- const jobsToDeleteByQrl = new Map()
52
+ const jobsToExtendByQrl = {}
53
+ const jobsToDeleteByQrl = {}
54
54
  const jobsToCleanup = new Set()
55
55
 
56
56
  if (this.opt.verbose) {
57
57
  console.error(chalk.blue('Stats: '), this.stats)
58
- console.error(chalk.blue('Running: '), this.jobs.map(({ qname, message }) => ({ qname, payload: message.Body })))
58
+ console.error(chalk.blue('Running: '), this.jobs.filter(j => j.status === 'processing').map(({ qname, message }) => ({ qname, payload: message.Body })))
59
59
  }
60
60
 
61
61
  // Build list of jobs we need to deal with
62
62
  for (const job of this.jobs) {
63
63
  const jobRunTime = (now - job.start) / 1000
64
64
  if (job.status === 'complete') {
65
- const jobsToDelete = jobsToDeleteByQrl.get(job.qrl) || []
65
+ const jobsToDelete = jobsToDeleteByQrl[job.qrl] || []
66
66
  jobsToDelete.push(job)
67
- jobsToDeleteByQrl.set(job.qrl, jobsToDelete)
67
+ jobsToDeleteByQrl[job.qrl] = jobsToDelete
68
68
  } else if (job.status === 'failed') {
69
69
  jobsToCleanup.add(job)
70
70
  } else if (jobRunTime >= job.exendAtSecond) {
71
71
  // Add it to our organized list of jobs
72
- const jobsToExtend = jobsToExtendByQrl.get(job.qrl) || []
72
+ const jobsToExtend = jobsToExtendByQrl[job.qrl] || []
73
73
  jobsToExtend.push(job)
74
- jobsToExtendByQrl.set(job.qrl, jobsToExtend)
74
+ jobsToExtendByQrl[job.qrl] = jobsToExtend
75
75
 
76
76
  // Update the visibility timeout, double every time, up to max
77
77
  const doubled = job.visibilityTimeout * 2
@@ -84,7 +84,9 @@ export class JobExecutor {
84
84
  debug('maintainVisibility', { jobsToDeleteByQrl, jobsToExtendByQrl })
85
85
 
86
86
  // Extend in batches for each queue
87
- for (const [qrl, jobsToExtend] of jobsToExtendByQrl) {
87
+ for (const qrl in jobsToExtendByQrl) {
88
+ const jobsToExtend = jobsToExtendByQrl[qrl]
89
+ debug({ qrl, jobsToExtend })
88
90
  while (jobsToExtend.length) {
89
91
  // Build list of messages to go in this batch
90
92
  const entries = []
@@ -105,7 +107,7 @@ export class JobExecutor {
105
107
  const result = await getSQSClient().send(new ChangeMessageVisibilityBatchCommand(input))
106
108
  debug('ChangeMessageVisibilityBatch returned', result)
107
109
  this.stats.sqsCalls++
108
- this.stats.timeoutsExtended += 10
110
+ if (result.Successful) this.stats.timeoutsExtended += result.Successful.length || 0
109
111
  // TODO Sentry
110
112
  }
111
113
  if (this.opt.verbose) {
@@ -120,7 +122,8 @@ export class JobExecutor {
120
122
  }
121
123
 
122
124
  // Delete in batches for each queue
123
- for (const [qrl, jobsToDelete] of jobsToDeleteByQrl) {
125
+ for (const qrl in jobsToDeleteByQrl) {
126
+ const jobsToDelete = jobsToDeleteByQrl[qrl]
124
127
  while (jobsToDelete.length) {
125
128
  // Build list of messages to go in this batch
126
129
  const entries = []
@@ -140,7 +143,7 @@ export class JobExecutor {
140
143
  debug({ DeleteMessageBatch: input })
141
144
  const result = await getSQSClient().send(new DeleteMessageBatchCommand(input))
142
145
  this.stats.sqsCalls++
143
- this.stats.jobsDeleted += 10
146
+ if (result.Successful) this.stats.jobsDeleted += result.Successful.length || 0
144
147
  debug('DeleteMessageBatch returned', result)
145
148
  // TODO Sentry
146
149
  }