qdone 2.0.9-alpha → 2.0.10-alpha
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/commonjs/index.js +1 -2
- package/commonjs/src/consumer.js +73 -420
- package/commonjs/src/scheduler/jobExecutor.js +321 -0
- package/commonjs/src/scheduler/queueManager.js +145 -0
- package/commonjs/src/scheduler/systemMonitor.js +65 -0
- package/index.js +1 -1
- package/package.json +1 -1
- package/src/consumer.js +48 -317
- package/src/scheduler/jobExecutor.js +242 -0
- package/src/scheduler/queueManager.js +88 -0
- package/src/scheduler/systemMonitor.js +58 -0
|
@@ -0,0 +1,321 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
/**
|
|
3
|
+
* Component to manage all the currently executing jobs, including extending
|
|
4
|
+
* their visibility timeouts and deleting them when they are successful.
|
|
5
|
+
*/
|
|
6
|
+
var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
|
|
7
|
+
function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
|
|
8
|
+
return new (P || (P = Promise))(function (resolve, reject) {
|
|
9
|
+
function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
|
|
10
|
+
function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
|
|
11
|
+
function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
|
|
12
|
+
step((generator = generator.apply(thisArg, _arguments || [])).next());
|
|
13
|
+
});
|
|
14
|
+
};
|
|
15
|
+
var __generator = (this && this.__generator) || function (thisArg, body) {
|
|
16
|
+
var _ = { label: 0, sent: function() { if (t[0] & 1) throw t[1]; return t[1]; }, trys: [], ops: [] }, f, y, t, g;
|
|
17
|
+
return g = { next: verb(0), "throw": verb(1), "return": verb(2) }, typeof Symbol === "function" && (g[Symbol.iterator] = function() { return this; }), g;
|
|
18
|
+
function verb(n) { return function (v) { return step([n, v]); }; }
|
|
19
|
+
function step(op) {
|
|
20
|
+
if (f) throw new TypeError("Generator is already executing.");
|
|
21
|
+
while (g && (g = 0, op[0] && (_ = 0)), _) try {
|
|
22
|
+
if (f = 1, y && (t = op[0] & 2 ? y["return"] : op[0] ? y["throw"] || ((t = y["return"]) && t.call(y), 0) : y.next) && !(t = t.call(y, op[1])).done) return t;
|
|
23
|
+
if (y = 0, t) op = [op[0] & 2, t.value];
|
|
24
|
+
switch (op[0]) {
|
|
25
|
+
case 0: case 1: t = op; break;
|
|
26
|
+
case 4: _.label++; return { value: op[1], done: false };
|
|
27
|
+
case 5: _.label++; y = op[1]; op = [0]; continue;
|
|
28
|
+
case 7: op = _.ops.pop(); _.trys.pop(); continue;
|
|
29
|
+
default:
|
|
30
|
+
if (!(t = _.trys, t = t.length > 0 && t[t.length - 1]) && (op[0] === 6 || op[0] === 2)) { _ = 0; continue; }
|
|
31
|
+
if (op[0] === 3 && (!t || (op[1] > t[0] && op[1] < t[3]))) { _.label = op[1]; break; }
|
|
32
|
+
if (op[0] === 6 && _.label < t[1]) { _.label = t[1]; t = op; break; }
|
|
33
|
+
if (t && _.label < t[2]) { _.label = t[2]; _.ops.push(op); break; }
|
|
34
|
+
if (t[2]) _.ops.pop();
|
|
35
|
+
_.trys.pop(); continue;
|
|
36
|
+
}
|
|
37
|
+
op = body.call(thisArg, _);
|
|
38
|
+
} catch (e) { op = [6, e]; y = 0; } finally { f = t = 0; }
|
|
39
|
+
if (op[0] & 5) throw op[1]; return { value: op[0] ? op[1] : void 0, done: true };
|
|
40
|
+
}
|
|
41
|
+
};
|
|
42
|
+
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
43
|
+
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
44
|
+
};
|
|
45
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
46
|
+
exports.JobExecutor = void 0;
|
|
47
|
+
var client_sqs_1 = require("@aws-sdk/client-sqs");
|
|
48
|
+
var chalk_1 = __importDefault(require("chalk"));
|
|
49
|
+
var debug_1 = __importDefault(require("debug"));
|
|
50
|
+
var sqs_js_1 = require("../sqs.js");
|
|
51
|
+
var debug = (0, debug_1.default)('qdone:jobExecutor');
|
|
52
|
+
var maxJobSeconds = 12 * 60 * 60;
|
|
53
|
+
var JobExecutor = /** @class */ (function () {
|
|
54
|
+
function JobExecutor(opt) {
|
|
55
|
+
this.opt = opt;
|
|
56
|
+
this.jobs = [];
|
|
57
|
+
this.stats = {
|
|
58
|
+
activeJobs: 0,
|
|
59
|
+
sqsCalls: 0,
|
|
60
|
+
timeoutsExtended: 0,
|
|
61
|
+
jobsSucceeded: 0,
|
|
62
|
+
jobsFailed: 0,
|
|
63
|
+
jobsDeleted: 0
|
|
64
|
+
};
|
|
65
|
+
this.maintainVisibility();
|
|
66
|
+
debug({ this: this });
|
|
67
|
+
}
|
|
68
|
+
JobExecutor.prototype.shutdown = function () {
|
|
69
|
+
this.shutdownRequested = true;
|
|
70
|
+
if (this.stats.activeJobs === 0 && this.jobs.length === 0) {
|
|
71
|
+
clearTimeout(this.maintainVisibilityTimeout);
|
|
72
|
+
}
|
|
73
|
+
};
|
|
74
|
+
JobExecutor.prototype.activeJobCount = function () {
|
|
75
|
+
return this.stats.activeJobs;
|
|
76
|
+
};
|
|
77
|
+
/**
|
|
78
|
+
* Changes message visibility on all running jobs using as few calls as possible.
|
|
79
|
+
*/
|
|
80
|
+
JobExecutor.prototype.maintainVisibility = function () {
|
|
81
|
+
return __awaiter(this, void 0, void 0, function () {
|
|
82
|
+
var now, jobsToExtendByQrl, jobsToDeleteByQrl, jobsToCleanup, _i, _a, job, jobRunTime, jobsToDelete, jobsToExtend, doubled, secondsUntilMax, secondsUntilKill, _b, jobsToExtendByQrl_1, _c, qrl, jobsToExtend, entries, messageId, job, entry, input, result, _d, jobsToDeleteByQrl_1, _e, qrl, jobsToDelete, entries, messageId, job, entry, input, result;
|
|
83
|
+
var _this = this;
|
|
84
|
+
return __generator(this, function (_f) {
|
|
85
|
+
switch (_f.label) {
|
|
86
|
+
case 0:
|
|
87
|
+
debug('maintainVisibility', this.jobs);
|
|
88
|
+
now = new Date();
|
|
89
|
+
jobsToExtendByQrl = new Map();
|
|
90
|
+
jobsToDeleteByQrl = new Map();
|
|
91
|
+
jobsToCleanup = new Set();
|
|
92
|
+
if (this.opt.verbose) {
|
|
93
|
+
console.error(chalk_1.default.blue('Stats: '), this.stats);
|
|
94
|
+
console.error(chalk_1.default.blue('Running: '), this.jobs.map(function (_a) {
|
|
95
|
+
var qname = _a.qname, message = _a.message;
|
|
96
|
+
return ({ qname: qname, payload: message.Body });
|
|
97
|
+
}));
|
|
98
|
+
}
|
|
99
|
+
// Build list of jobs we need to deal with
|
|
100
|
+
for (_i = 0, _a = this.jobs; _i < _a.length; _i++) {
|
|
101
|
+
job = _a[_i];
|
|
102
|
+
jobRunTime = (now - job.start) / 1000;
|
|
103
|
+
if (job.status === 'complete') {
|
|
104
|
+
jobsToDelete = jobsToDeleteByQrl.get(job.qrl) || [];
|
|
105
|
+
jobsToDelete.push(job);
|
|
106
|
+
jobsToDeleteByQrl.set(job.qrl, jobsToDelete);
|
|
107
|
+
}
|
|
108
|
+
else if (job.status === 'failed') {
|
|
109
|
+
jobsToCleanup.add(job);
|
|
110
|
+
}
|
|
111
|
+
else if (jobRunTime >= job.exendAtSecond) {
|
|
112
|
+
jobsToExtend = jobsToExtendByQrl.get(job.qrl) || [];
|
|
113
|
+
jobsToExtend.push(job);
|
|
114
|
+
jobsToExtendByQrl.set(job.qrl, jobsToExtend);
|
|
115
|
+
doubled = job.visibilityTimeout * 2;
|
|
116
|
+
secondsUntilMax = maxJobSeconds - jobRunTime;
|
|
117
|
+
secondsUntilKill = this.opt.killAfter - jobRunTime;
|
|
118
|
+
job.visibilityTimeout = Math.min(double, secondsUntilMax, secondsUntilKill);
|
|
119
|
+
job.extendAtSecond = jobRunTime + job.visibilityTimeout; // this is what we use next time
|
|
120
|
+
}
|
|
121
|
+
}
|
|
122
|
+
debug('maintainVisibility', { jobsToDeleteByQrl: jobsToDeleteByQrl, jobsToExtendByQrl: jobsToExtendByQrl });
|
|
123
|
+
_b = 0, jobsToExtendByQrl_1 = jobsToExtendByQrl;
|
|
124
|
+
_f.label = 1;
|
|
125
|
+
case 1:
|
|
126
|
+
if (!(_b < jobsToExtendByQrl_1.length)) return [3 /*break*/, 6];
|
|
127
|
+
_c = jobsToExtendByQrl_1[_b], qrl = _c[0], jobsToExtend = _c[1];
|
|
128
|
+
_f.label = 2;
|
|
129
|
+
case 2:
|
|
130
|
+
if (!jobsToExtend.length) return [3 /*break*/, 4];
|
|
131
|
+
entries = [];
|
|
132
|
+
messageId = 0;
|
|
133
|
+
while (messageId++ < 10 && jobsToExtend.length) {
|
|
134
|
+
job = jobsToExtend.shift();
|
|
135
|
+
entry = {
|
|
136
|
+
Id: '' + messageId,
|
|
137
|
+
ReceiptHandle: job.message.ReceiptHandle,
|
|
138
|
+
VisibilityTimeout: job.visibilityTimeout
|
|
139
|
+
};
|
|
140
|
+
entries.push(entry);
|
|
141
|
+
}
|
|
142
|
+
input = { QueueUrl: qrl, Entries: entries };
|
|
143
|
+
debug({ ChangeMessageVisibilityBatch: input });
|
|
144
|
+
return [4 /*yield*/, (0, sqs_js_1.getSQSClient)().send(new client_sqs_1.ChangeMessageVisibilityBatchCommand(input))];
|
|
145
|
+
case 3:
|
|
146
|
+
result = _f.sent();
|
|
147
|
+
debug('ChangeMessageVisibilityBatch returned', result);
|
|
148
|
+
this.stats.sqsCalls++;
|
|
149
|
+
this.stats.timeoutsExtended += 10;
|
|
150
|
+
return [3 /*break*/, 2];
|
|
151
|
+
case 4:
|
|
152
|
+
if (this.opt.verbose) {
|
|
153
|
+
console.error(chalk_1.default.blue('Extended these jobs: '), jobsToExtend);
|
|
154
|
+
}
|
|
155
|
+
else if (!this.opt.disableLog) {
|
|
156
|
+
console.log(JSON.stringify({
|
|
157
|
+
event: 'EXTEND_VISIBILITY_TIMEOUTS',
|
|
158
|
+
timestamp: now,
|
|
159
|
+
messageIds: jobsToExtend.map(function (_a) {
|
|
160
|
+
var message = _a.message;
|
|
161
|
+
return message.MessageId;
|
|
162
|
+
})
|
|
163
|
+
}));
|
|
164
|
+
}
|
|
165
|
+
_f.label = 5;
|
|
166
|
+
case 5:
|
|
167
|
+
_b++;
|
|
168
|
+
return [3 /*break*/, 1];
|
|
169
|
+
case 6:
|
|
170
|
+
_d = 0, jobsToDeleteByQrl_1 = jobsToDeleteByQrl;
|
|
171
|
+
_f.label = 7;
|
|
172
|
+
case 7:
|
|
173
|
+
if (!(_d < jobsToDeleteByQrl_1.length)) return [3 /*break*/, 12];
|
|
174
|
+
_e = jobsToDeleteByQrl_1[_d], qrl = _e[0], jobsToDelete = _e[1];
|
|
175
|
+
_f.label = 8;
|
|
176
|
+
case 8:
|
|
177
|
+
if (!jobsToDelete.length) return [3 /*break*/, 10];
|
|
178
|
+
entries = [];
|
|
179
|
+
messageId = 0;
|
|
180
|
+
while (messageId++ < 10 && jobsToDelete.length) {
|
|
181
|
+
job = jobsToDelete.shift();
|
|
182
|
+
entry = {
|
|
183
|
+
Id: '' + messageId,
|
|
184
|
+
ReceiptHandle: job.message.ReceiptHandle,
|
|
185
|
+
VisibilityTimeout: job.visibilityTimeout
|
|
186
|
+
};
|
|
187
|
+
entries.push(entry);
|
|
188
|
+
}
|
|
189
|
+
input = { QueueUrl: qrl, Entries: entries };
|
|
190
|
+
debug({ DeleteMessageBatch: input });
|
|
191
|
+
return [4 /*yield*/, (0, sqs_js_1.getSQSClient)().send(new client_sqs_1.DeleteMessageBatchCommand(input))];
|
|
192
|
+
case 9:
|
|
193
|
+
result = _f.sent();
|
|
194
|
+
this.stats.sqsCalls++;
|
|
195
|
+
this.stats.jobsDeleted += 10;
|
|
196
|
+
debug('DeleteMessageBatch returned', result);
|
|
197
|
+
return [3 /*break*/, 8];
|
|
198
|
+
case 10:
|
|
199
|
+
if (this.opt.verbose) {
|
|
200
|
+
console.error(chalk_1.default.blue('Deleted these finished jobs: '), jobsToDelete);
|
|
201
|
+
}
|
|
202
|
+
else if (!this.opt.disableLog) {
|
|
203
|
+
console.log(JSON.stringify({
|
|
204
|
+
event: 'DELETE_MESSAGES',
|
|
205
|
+
timestamp: now,
|
|
206
|
+
messageIds: jobsToDelete.map(function (_a) {
|
|
207
|
+
var message = _a.message;
|
|
208
|
+
return message.MessageId;
|
|
209
|
+
})
|
|
210
|
+
}));
|
|
211
|
+
}
|
|
212
|
+
_f.label = 11;
|
|
213
|
+
case 11:
|
|
214
|
+
_d++;
|
|
215
|
+
return [3 /*break*/, 7];
|
|
216
|
+
case 12:
|
|
217
|
+
// Get rid of deleted and failed jobs
|
|
218
|
+
this.jobs = this.jobs.filter(function (j) { return j.status === 'processing'; });
|
|
219
|
+
// Check again later, unless we are shutting down and nothing's left
|
|
220
|
+
if (this.shutdownRequested && this.stats.activeJobs === 0 && this.jobs.length === 0)
|
|
221
|
+
return [2 /*return*/];
|
|
222
|
+
this.maintainVisibilityTimeout = setTimeout(function () { return _this.maintainVisibility(); }, 10 * 1000);
|
|
223
|
+
return [2 /*return*/];
|
|
224
|
+
}
|
|
225
|
+
});
|
|
226
|
+
});
|
|
227
|
+
};
|
|
228
|
+
JobExecutor.prototype.executeJob = function (message, callback, qname, qrl) {
|
|
229
|
+
return __awaiter(this, void 0, void 0, function () {
|
|
230
|
+
var payload, visibilityTimeout, job, queue, result, err_1;
|
|
231
|
+
return __generator(this, function (_a) {
|
|
232
|
+
switch (_a.label) {
|
|
233
|
+
case 0:
|
|
234
|
+
payload = this.opt.json ? JSON.parse(message.Body) : message.Body;
|
|
235
|
+
visibilityTimeout = 30;
|
|
236
|
+
job = {
|
|
237
|
+
status: 'processing',
|
|
238
|
+
start: new Date(),
|
|
239
|
+
visibilityTimeout: 30,
|
|
240
|
+
extendAtSecond: visibilityTimeout / 2,
|
|
241
|
+
payload: this.opt.json ? JSON.parse(message.Body) : message.Body,
|
|
242
|
+
message: message,
|
|
243
|
+
callback: callback,
|
|
244
|
+
qname: qname,
|
|
245
|
+
qrl: qrl
|
|
246
|
+
};
|
|
247
|
+
debug('executeJob', job);
|
|
248
|
+
this.jobs.push(job);
|
|
249
|
+
this.stats.activeJobs++;
|
|
250
|
+
if (this.opt.verbose) {
|
|
251
|
+
console.error(chalk_1.default.blue('Executing:'), qname, chalk_1.default.blue('-->'), job.payload);
|
|
252
|
+
}
|
|
253
|
+
else if (!this.opt.disableLog) {
|
|
254
|
+
console.log(JSON.stringify({
|
|
255
|
+
event: 'MESSAGE_PROCESSING_START',
|
|
256
|
+
timestamp: new Date(),
|
|
257
|
+
qname: qname,
|
|
258
|
+
messageId: message.MessageId,
|
|
259
|
+
payload: job.payload
|
|
260
|
+
}));
|
|
261
|
+
}
|
|
262
|
+
_a.label = 1;
|
|
263
|
+
case 1:
|
|
264
|
+
_a.trys.push([1, 3, , 4]);
|
|
265
|
+
queue = qname.slice(this.opt.prefix.length);
|
|
266
|
+
return [4 /*yield*/, callback(queue, payload)];
|
|
267
|
+
case 2:
|
|
268
|
+
result = _a.sent();
|
|
269
|
+
debug('executeJob callback finished', { payload: payload, result: result });
|
|
270
|
+
if (this.opt.verbose) {
|
|
271
|
+
console.error(chalk_1.default.green('SUCCESS'), message.Body);
|
|
272
|
+
}
|
|
273
|
+
job.status = 'complete';
|
|
274
|
+
if (this.opt.verbose) {
|
|
275
|
+
console.error(chalk_1.default.blue(' done'));
|
|
276
|
+
console.error();
|
|
277
|
+
}
|
|
278
|
+
else if (!this.opt.disableLog) {
|
|
279
|
+
console.log(JSON.stringify({
|
|
280
|
+
event: 'MESSAGE_PROCESSING_COMPLETE',
|
|
281
|
+
timestamp: new Date(),
|
|
282
|
+
messageId: message.MessageId,
|
|
283
|
+
payload: payload
|
|
284
|
+
}));
|
|
285
|
+
}
|
|
286
|
+
this.stats.jobsSucceeded++;
|
|
287
|
+
return [3 /*break*/, 4];
|
|
288
|
+
case 3:
|
|
289
|
+
err_1 = _a.sent();
|
|
290
|
+
debug('exec.catch');
|
|
291
|
+
// Fail path for job execution
|
|
292
|
+
if (this.opt.verbose) {
|
|
293
|
+
console.error(chalk_1.default.red('FAILED'), message.Body);
|
|
294
|
+
console.error(chalk_1.default.blue(' error : ') + err_1);
|
|
295
|
+
}
|
|
296
|
+
else if (!this.opt.disableLog) {
|
|
297
|
+
// Production error logging
|
|
298
|
+
console.log(JSON.stringify({
|
|
299
|
+
event: 'MESSAGE_PROCESSING_FAILED',
|
|
300
|
+
reason: 'exception thrown',
|
|
301
|
+
qname: qname,
|
|
302
|
+
timestamp: new Date(),
|
|
303
|
+
messageId: message.MessageId,
|
|
304
|
+
payload: payload,
|
|
305
|
+
errorMessage: err_1.toString().split('\n').slice(1).join('\n').trim() || undefined,
|
|
306
|
+
err: err_1
|
|
307
|
+
}));
|
|
308
|
+
}
|
|
309
|
+
job.status = 'failed';
|
|
310
|
+
this.stats.jobsFailed++;
|
|
311
|
+
return [3 /*break*/, 4];
|
|
312
|
+
case 4:
|
|
313
|
+
this.stats.activeJobs--;
|
|
314
|
+
return [2 /*return*/];
|
|
315
|
+
}
|
|
316
|
+
});
|
|
317
|
+
});
|
|
318
|
+
};
|
|
319
|
+
return JobExecutor;
|
|
320
|
+
}());
|
|
321
|
+
exports.JobExecutor = JobExecutor;
|
|
@@ -0,0 +1,145 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
/**
|
|
3
|
+
* Component to manange what queues are being listened to and in what order.
|
|
4
|
+
*/
|
|
5
|
+
var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
|
|
6
|
+
function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
|
|
7
|
+
return new (P || (P = Promise))(function (resolve, reject) {
|
|
8
|
+
function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
|
|
9
|
+
function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
|
|
10
|
+
function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
|
|
11
|
+
step((generator = generator.apply(thisArg, _arguments || [])).next());
|
|
12
|
+
});
|
|
13
|
+
};
|
|
14
|
+
var __generator = (this && this.__generator) || function (thisArg, body) {
|
|
15
|
+
var _ = { label: 0, sent: function() { if (t[0] & 1) throw t[1]; return t[1]; }, trys: [], ops: [] }, f, y, t, g;
|
|
16
|
+
return g = { next: verb(0), "throw": verb(1), "return": verb(2) }, typeof Symbol === "function" && (g[Symbol.iterator] = function() { return this; }), g;
|
|
17
|
+
function verb(n) { return function (v) { return step([n, v]); }; }
|
|
18
|
+
function step(op) {
|
|
19
|
+
if (f) throw new TypeError("Generator is already executing.");
|
|
20
|
+
while (g && (g = 0, op[0] && (_ = 0)), _) try {
|
|
21
|
+
if (f = 1, y && (t = op[0] & 2 ? y["return"] : op[0] ? y["throw"] || ((t = y["return"]) && t.call(y), 0) : y.next) && !(t = t.call(y, op[1])).done) return t;
|
|
22
|
+
if (y = 0, t) op = [op[0] & 2, t.value];
|
|
23
|
+
switch (op[0]) {
|
|
24
|
+
case 0: case 1: t = op; break;
|
|
25
|
+
case 4: _.label++; return { value: op[1], done: false };
|
|
26
|
+
case 5: _.label++; y = op[1]; op = [0]; continue;
|
|
27
|
+
case 7: op = _.ops.pop(); _.trys.pop(); continue;
|
|
28
|
+
default:
|
|
29
|
+
if (!(t = _.trys, t = t.length > 0 && t[t.length - 1]) && (op[0] === 6 || op[0] === 2)) { _ = 0; continue; }
|
|
30
|
+
if (op[0] === 3 && (!t || (op[1] > t[0] && op[1] < t[3]))) { _.label = op[1]; break; }
|
|
31
|
+
if (op[0] === 6 && _.label < t[1]) { _.label = t[1]; t = op; break; }
|
|
32
|
+
if (t && _.label < t[2]) { _.label = t[2]; _.ops.push(op); break; }
|
|
33
|
+
if (t[2]) _.ops.pop();
|
|
34
|
+
_.trys.pop(); continue;
|
|
35
|
+
}
|
|
36
|
+
op = body.call(thisArg, _);
|
|
37
|
+
} catch (e) { op = [6, e]; y = 0; } finally { f = t = 0; }
|
|
38
|
+
if (op[0] & 5) throw op[1]; return { value: op[0] ? op[1] : void 0, done: true };
|
|
39
|
+
}
|
|
40
|
+
};
|
|
41
|
+
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
42
|
+
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
43
|
+
};
|
|
44
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
45
|
+
exports.QueueManager = void 0;
|
|
46
|
+
var chalk_1 = __importDefault(require("chalk"));
|
|
47
|
+
var debug_1 = __importDefault(require("debug"));
|
|
48
|
+
var qrlCache_js_1 = require("../qrlCache.js");
|
|
49
|
+
var idleQueues_js_1 = require("../idleQueues.js");
|
|
50
|
+
var sqs_js_1 = require("../sqs.js");
|
|
51
|
+
var debug = (0, debug_1.default)('qdone:queueManager');
|
|
52
|
+
var QueueManager = /** @class */ (function () {
|
|
53
|
+
function QueueManager(opt, queues, resolveSeconds) {
|
|
54
|
+
if (resolveSeconds === void 0) { resolveSeconds = 10; }
|
|
55
|
+
this.opt = opt;
|
|
56
|
+
this.queues = queues;
|
|
57
|
+
this.resolveSeconds = resolveSeconds;
|
|
58
|
+
this.selectedPairs = [];
|
|
59
|
+
this.resolveTimeout = undefined;
|
|
60
|
+
this.shutdownRequested = false;
|
|
61
|
+
this.resolveQueues();
|
|
62
|
+
}
|
|
63
|
+
QueueManager.prototype.resolveQueues = function () {
|
|
64
|
+
return __awaiter(this, void 0, void 0, function () {
|
|
65
|
+
var qnames, pairs, activePairs;
|
|
66
|
+
var _this = this;
|
|
67
|
+
return __generator(this, function (_a) {
|
|
68
|
+
switch (_a.label) {
|
|
69
|
+
case 0:
|
|
70
|
+
if (this.shutdownRequested)
|
|
71
|
+
return [2 /*return*/];
|
|
72
|
+
// Start processing
|
|
73
|
+
if (this.opt.verbose)
|
|
74
|
+
console.error(chalk_1.default.blue('Resolving queues: ') + this.queues.join(' '));
|
|
75
|
+
qnames = this.queues.map(function (queue) { return (0, qrlCache_js_1.normalizeQueueName)(queue, _this.opt); });
|
|
76
|
+
return [4 /*yield*/, (0, qrlCache_js_1.getQnameUrlPairs)(qnames, this.opt)];
|
|
77
|
+
case 1:
|
|
78
|
+
pairs = _a.sent();
|
|
79
|
+
if (this.shutdownRequested)
|
|
80
|
+
return [2 /*return*/];
|
|
81
|
+
activePairs = [];
|
|
82
|
+
if (!this.opt.activeOnly) return [3 /*break*/, 3];
|
|
83
|
+
debug({ pairsBeforeCheck: pairs });
|
|
84
|
+
return [4 /*yield*/, Promise.all(pairs.map(function (pair) { return __awaiter(_this, void 0, void 0, function () {
|
|
85
|
+
var idle;
|
|
86
|
+
return __generator(this, function (_a) {
|
|
87
|
+
switch (_a.label) {
|
|
88
|
+
case 0: return [4 /*yield*/, (0, idleQueues_js_1.cheapIdleCheck)(pair.qname, pair.qrl, this.opt)];
|
|
89
|
+
case 1:
|
|
90
|
+
idle = (_a.sent()).idle;
|
|
91
|
+
if (!idle)
|
|
92
|
+
activePairs.push(pair);
|
|
93
|
+
return [2 /*return*/];
|
|
94
|
+
}
|
|
95
|
+
});
|
|
96
|
+
}); }))];
|
|
97
|
+
case 2:
|
|
98
|
+
_a.sent();
|
|
99
|
+
_a.label = 3;
|
|
100
|
+
case 3:
|
|
101
|
+
if (this.shutdownRequested)
|
|
102
|
+
return [2 /*return*/];
|
|
103
|
+
// Finished resolving
|
|
104
|
+
if (this.opt.verbose) {
|
|
105
|
+
console.error(chalk_1.default.blue(' done'));
|
|
106
|
+
console.error();
|
|
107
|
+
}
|
|
108
|
+
// Figure out which queues we want to listen on, choosing between active and
|
|
109
|
+
// all, filtering out failed queues if the user wants that
|
|
110
|
+
this.selectedPairs = (this.opt.activeOnly ? activePairs : pairs)
|
|
111
|
+
.filter(function (_a) {
|
|
112
|
+
var qname = _a.qname;
|
|
113
|
+
var suf = _this.opt.failSuffix + (_this.opt.fifo ? '.fifo' : '');
|
|
114
|
+
var isFailQueue = qname.slice(-suf.length) === suf;
|
|
115
|
+
var shouldInclude = _this.opt.includeFailed ? true : !isFailQueue;
|
|
116
|
+
return shouldInclude;
|
|
117
|
+
});
|
|
118
|
+
// Randomize order
|
|
119
|
+
this.selectedPairs.sort(function () { return 0.5 - Math.random(); });
|
|
120
|
+
debug('selectedPairs', this.selectedPairs);
|
|
121
|
+
if (this.opt.verbose) {
|
|
122
|
+
console.error(chalk_1.default.blue('Will resolve queues again in ' + this.resolveSeconds + ' seconds'));
|
|
123
|
+
}
|
|
124
|
+
this.resolveTimeout = setTimeout(function () { return _this.resolveQueues(); }, this.resolveSeconds * 1000);
|
|
125
|
+
return [2 /*return*/];
|
|
126
|
+
}
|
|
127
|
+
});
|
|
128
|
+
});
|
|
129
|
+
};
|
|
130
|
+
// Return the next queue in the lineup
|
|
131
|
+
QueueManager.prototype.nextPair = function () {
|
|
132
|
+
var pair = this.selectedPairs.shift();
|
|
133
|
+
this.selectedPairs.push(pair);
|
|
134
|
+
return pair;
|
|
135
|
+
};
|
|
136
|
+
QueueManager.prototype.getPairs = function () {
|
|
137
|
+
return this.selectedPairs;
|
|
138
|
+
};
|
|
139
|
+
QueueManager.prototype.shutdown = function () {
|
|
140
|
+
clearTimeout(this.resolveTimeout);
|
|
141
|
+
this.shutdownRequested = true;
|
|
142
|
+
};
|
|
143
|
+
return QueueManager;
|
|
144
|
+
}());
|
|
145
|
+
exports.QueueManager = QueueManager;
|
|
@@ -0,0 +1,65 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
/**
|
|
3
|
+
* Component to track event loop latency, which can be used as a metric for
|
|
4
|
+
* backpressure.
|
|
5
|
+
*/
|
|
6
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
7
|
+
exports.SystemMonitor = void 0;
|
|
8
|
+
var SystemMonitor = /** @class */ (function () {
|
|
9
|
+
function SystemMonitor(opt, smoothingFactor, reportSeconds) {
|
|
10
|
+
if (smoothingFactor === void 0) { smoothingFactor = 0.5; }
|
|
11
|
+
if (reportSeconds === void 0) { reportSeconds = 5; }
|
|
12
|
+
this.opt = opt;
|
|
13
|
+
this.smoothingFactor = smoothingFactor;
|
|
14
|
+
this.reportSeconds = reportSeconds;
|
|
15
|
+
this.measurements = {
|
|
16
|
+
setTimeout: [],
|
|
17
|
+
setImmediate: []
|
|
18
|
+
};
|
|
19
|
+
this.timeouts = {
|
|
20
|
+
setTimeout: undefined,
|
|
21
|
+
setImmediate: undefined,
|
|
22
|
+
reportLatency: undefined
|
|
23
|
+
};
|
|
24
|
+
this.measureLatencySetTimeout();
|
|
25
|
+
this.reportLatency();
|
|
26
|
+
}
|
|
27
|
+
SystemMonitor.prototype.measureLatencySetTimeout = function () {
|
|
28
|
+
var _this = this;
|
|
29
|
+
var start = new Date();
|
|
30
|
+
this.timeouts.setTimeout = setTimeout(function () {
|
|
31
|
+
var latency = new Date() - start;
|
|
32
|
+
_this.measurements.setTimeout.push(latency);
|
|
33
|
+
if (_this.measurements.setTimeout.length > 1000)
|
|
34
|
+
_this.measurements.setTimeout.shift();
|
|
35
|
+
_this.measureLatencySetTimeout();
|
|
36
|
+
});
|
|
37
|
+
};
|
|
38
|
+
SystemMonitor.prototype.getLatency = function () {
|
|
39
|
+
var results = {};
|
|
40
|
+
for (var k in this.measurements) {
|
|
41
|
+
var values = this.measurements[k];
|
|
42
|
+
results[k] = values.length ? values.reduce(function (a, b) { return a + b; }, 0) / values.length : 0;
|
|
43
|
+
}
|
|
44
|
+
return results;
|
|
45
|
+
};
|
|
46
|
+
SystemMonitor.prototype.reportLatency = function () {
|
|
47
|
+
var _this = this;
|
|
48
|
+
this.timeouts.reportLatency = setTimeout(function () {
|
|
49
|
+
var _a;
|
|
50
|
+
for (var k in _this.measurements) {
|
|
51
|
+
var values = _this.measurements[k];
|
|
52
|
+
var mean = values.length ? values.reduce(function (a, b) { return a + b; }, 0) / values.length : 0;
|
|
53
|
+
console.log((_a = {}, _a[k] = mean, _a));
|
|
54
|
+
}
|
|
55
|
+
_this.reportLatency();
|
|
56
|
+
}, this.reportSeconds * 1000);
|
|
57
|
+
};
|
|
58
|
+
SystemMonitor.prototype.shutdown = function () {
|
|
59
|
+
console.log(this.measurements);
|
|
60
|
+
for (var k in this.timeouts)
|
|
61
|
+
clearTimeout(this.timeouts[k]);
|
|
62
|
+
};
|
|
63
|
+
return SystemMonitor;
|
|
64
|
+
}());
|
|
65
|
+
exports.SystemMonitor = SystemMonitor;
|
package/index.js
CHANGED
|
@@ -1,2 +1,2 @@
|
|
|
1
1
|
export { enqueue, enqueueBatch } from './src/enqueue.js'
|
|
2
|
-
export { processMessages, requestShutdown
|
|
2
|
+
export { processMessages, requestShutdown } from './src/consumer.js'
|