qdone 2.0.8-alpha → 2.0.10-alpha
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/commonjs/index.js +1 -2
- package/commonjs/src/consumer.js +80 -386
- package/commonjs/src/scheduler/jobExecutor.js +321 -0
- package/commonjs/src/scheduler/queueManager.js +145 -0
- package/commonjs/src/scheduler/systemMonitor.js +65 -0
- package/index.js +1 -1
- package/package.json +1 -1
- package/src/consumer.js +55 -292
- package/src/scheduler/jobExecutor.js +242 -0
- package/src/scheduler/queueManager.js +88 -0
- package/src/scheduler/systemMonitor.js +58 -0
package/commonjs/index.js
CHANGED
|
@@ -1,10 +1,9 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
-
exports.
|
|
3
|
+
exports.requestShutdown = exports.processMessages = exports.enqueueBatch = exports.enqueue = void 0;
|
|
4
4
|
var enqueue_js_1 = require("./src/enqueue.js");
|
|
5
5
|
Object.defineProperty(exports, "enqueue", { enumerable: true, get: function () { return enqueue_js_1.enqueue; } });
|
|
6
6
|
Object.defineProperty(exports, "enqueueBatch", { enumerable: true, get: function () { return enqueue_js_1.enqueueBatch; } });
|
|
7
7
|
var consumer_js_1 = require("./src/consumer.js");
|
|
8
8
|
Object.defineProperty(exports, "processMessages", { enumerable: true, get: function () { return consumer_js_1.processMessages; } });
|
|
9
9
|
Object.defineProperty(exports, "requestShutdown", { enumerable: true, get: function () { return consumer_js_1.requestShutdown; } });
|
|
10
|
-
Object.defineProperty(exports, "DoNotProcess", { enumerable: true, get: function () { return consumer_js_1.DoNotProcess; } });
|
package/commonjs/src/consumer.js
CHANGED
|
@@ -2,21 +2,6 @@
|
|
|
2
2
|
/**
|
|
3
3
|
* Consumer implementation.
|
|
4
4
|
*/
|
|
5
|
-
var __extends = (this && this.__extends) || (function () {
|
|
6
|
-
var extendStatics = function (d, b) {
|
|
7
|
-
extendStatics = Object.setPrototypeOf ||
|
|
8
|
-
({ __proto__: [] } instanceof Array && function (d, b) { d.__proto__ = b; }) ||
|
|
9
|
-
function (d, b) { for (var p in b) if (Object.prototype.hasOwnProperty.call(b, p)) d[p] = b[p]; };
|
|
10
|
-
return extendStatics(d, b);
|
|
11
|
-
};
|
|
12
|
-
return function (d, b) {
|
|
13
|
-
if (typeof b !== "function" && b !== null)
|
|
14
|
-
throw new TypeError("Class extends value " + String(b) + " is not a constructor or null");
|
|
15
|
-
extendStatics(d, b);
|
|
16
|
-
function __() { this.constructor = d; }
|
|
17
|
-
d.prototype = b === null ? Object.create(b) : (__.prototype = b.prototype, new __());
|
|
18
|
-
};
|
|
19
|
-
})();
|
|
20
5
|
var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
|
|
21
6
|
function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
|
|
22
7
|
return new (P || (P = Promise))(function (resolve, reject) {
|
|
@@ -57,427 +42,136 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
|
57
42
|
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
58
43
|
};
|
|
59
44
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
60
|
-
exports.processMessages = exports.
|
|
45
|
+
exports.processMessages = exports.getMessages = exports.requestShutdown = void 0;
|
|
61
46
|
var client_sqs_1 = require("@aws-sdk/client-sqs");
|
|
62
47
|
var chalk_1 = __importDefault(require("chalk"));
|
|
63
48
|
var debug_1 = __importDefault(require("debug"));
|
|
64
|
-
var
|
|
65
|
-
var
|
|
49
|
+
var systemMonitor_js_1 = require("./scheduler/systemMonitor.js");
|
|
50
|
+
var queueManager_js_1 = require("./scheduler/queueManager.js");
|
|
51
|
+
var jobExecutor_js_1 = require("./scheduler/jobExecutor.js");
|
|
66
52
|
var defaults_js_1 = require("./defaults.js");
|
|
67
53
|
var sqs_js_1 = require("./sqs.js");
|
|
68
|
-
|
|
69
|
-
// Throwing an instance of this Error allows the processMessages callback to
|
|
70
|
-
// refuse a message which then gets immediately returned to the queue.
|
|
71
|
-
//
|
|
72
|
-
// This has the side effect of throtting the queue since it stops polling on
|
|
73
|
-
// the queue until the next queue resolution in processMessages.
|
|
74
|
-
//
|
|
75
|
-
// This is useful for implementing schedulers on top of qdone, for example, to
|
|
76
|
-
// look at the queue name and decide whether to take on a new message.
|
|
77
|
-
//
|
|
78
|
-
var DoNotProcess = /** @class */ (function (_super) {
|
|
79
|
-
__extends(DoNotProcess, _super);
|
|
80
|
-
function DoNotProcess() {
|
|
81
|
-
return _super !== null && _super.apply(this, arguments) || this;
|
|
82
|
-
}
|
|
83
|
-
return DoNotProcess;
|
|
84
|
-
}(Error));
|
|
85
|
-
exports.DoNotProcess = DoNotProcess;
|
|
86
|
-
var debug = (0, debug_1.default)('qdone:worker');
|
|
54
|
+
var debug = (0, debug_1.default)('qdone:consumer');
|
|
87
55
|
// Global flag for shutdown request
|
|
88
56
|
var shutdownRequested = false;
|
|
89
57
|
var shutdownCallbacks = [];
|
|
90
58
|
function requestShutdown() {
|
|
59
|
+
debug('requestShutdown');
|
|
91
60
|
shutdownRequested = true;
|
|
92
61
|
for (var _i = 0, shutdownCallbacks_1 = shutdownCallbacks; _i < shutdownCallbacks_1.length; _i++) {
|
|
93
62
|
var callback = shutdownCallbacks_1[_i];
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
}
|
|
97
|
-
catch (e) { }
|
|
63
|
+
debug('callback', callback);
|
|
64
|
+
callback();
|
|
65
|
+
// try { callback() } catch (e) { }
|
|
98
66
|
}
|
|
67
|
+
debug('requestShutdown done');
|
|
99
68
|
}
|
|
100
69
|
exports.requestShutdown = requestShutdown;
|
|
101
|
-
function
|
|
102
|
-
return __awaiter(this, void 0, void 0, function () {
|
|
103
|
-
function extendTimeout() {
|
|
104
|
-
return __awaiter(this, void 0, void 0, function () {
|
|
105
|
-
var maxJobRun, jobRunTime, result, err_2;
|
|
106
|
-
return __generator(this, function (_a) {
|
|
107
|
-
switch (_a.label) {
|
|
108
|
-
case 0:
|
|
109
|
-
debug('extendTimeout');
|
|
110
|
-
maxJobRun = 12 * 60 * 60;
|
|
111
|
-
jobRunTime = ((new Date()) - jobStart) / 1000;
|
|
112
|
-
// Double every time, up to max
|
|
113
|
-
visibilityTimeout = Math.min(visibilityTimeout * 2, maxJobRun - jobRunTime, opt.killAfter - jobRunTime);
|
|
114
|
-
if (opt.verbose) {
|
|
115
|
-
console.error(chalk_1.default.blue(' Ran for ') + jobRunTime +
|
|
116
|
-
chalk_1.default.blue(' seconds, requesting another ') + visibilityTimeout +
|
|
117
|
-
chalk_1.default.blue(' seconds'));
|
|
118
|
-
}
|
|
119
|
-
_a.label = 1;
|
|
120
|
-
case 1:
|
|
121
|
-
_a.trys.push([1, 3, , 4]);
|
|
122
|
-
return [4 /*yield*/, (0, sqs_js_1.getSQSClient)().send(new client_sqs_1.ChangeMessageVisibilityCommand({
|
|
123
|
-
QueueUrl: qrl,
|
|
124
|
-
ReceiptHandle: message.ReceiptHandle,
|
|
125
|
-
VisibilityTimeout: visibilityTimeout
|
|
126
|
-
}))];
|
|
127
|
-
case 2:
|
|
128
|
-
result = _a.sent();
|
|
129
|
-
debug('ChangeMessageVisibility returned', result);
|
|
130
|
-
if (jobRunTime + visibilityTimeout >= maxJobRun ||
|
|
131
|
-
jobRunTime + visibilityTimeout >= opt.killAfter) {
|
|
132
|
-
if (opt.verbose)
|
|
133
|
-
console.error(chalk_1.default.yellow(' warning: this is our last time extension'));
|
|
134
|
-
}
|
|
135
|
-
else {
|
|
136
|
-
// Extend when we get 50% of the way to timeout
|
|
137
|
-
timeoutExtender = setTimeout(extendTimeout, visibilityTimeout * 1000 * 0.5);
|
|
138
|
-
}
|
|
139
|
-
return [3 /*break*/, 4];
|
|
140
|
-
case 3:
|
|
141
|
-
err_2 = _a.sent();
|
|
142
|
-
debug('ChangeMessageVisibility threw', err_2);
|
|
143
|
-
// Rejection means we're ouuta time, whatever, let the job die
|
|
144
|
-
if (opt.verbose) {
|
|
145
|
-
console.error(chalk_1.default.red(' failed to extend job: ') + err_2);
|
|
146
|
-
}
|
|
147
|
-
else if (!opt.disableLog) {
|
|
148
|
-
// Production error logging
|
|
149
|
-
console.log(JSON.stringify({
|
|
150
|
-
event: 'MESSAGE_PROCESSING_FAILED',
|
|
151
|
-
reason: 'ran longer than --kill-after',
|
|
152
|
-
timestamp: new Date(),
|
|
153
|
-
messageId: message.MessageId,
|
|
154
|
-
payload: payload,
|
|
155
|
-
errorMessage: err_2.toString().split('\n').slice(1).join('\n').trim() || undefined,
|
|
156
|
-
err: err_2
|
|
157
|
-
}));
|
|
158
|
-
}
|
|
159
|
-
return [3 /*break*/, 4];
|
|
160
|
-
case 4: return [2 /*return*/];
|
|
161
|
-
}
|
|
162
|
-
});
|
|
163
|
-
});
|
|
164
|
-
}
|
|
165
|
-
var payload, jobStart, visibilityTimeout, timeoutExtender, queue, result, err_1, result;
|
|
166
|
-
return __generator(this, function (_a) {
|
|
167
|
-
switch (_a.label) {
|
|
168
|
-
case 0:
|
|
169
|
-
debug('processMessage', message, qname, qrl);
|
|
170
|
-
payload = JSON.parse(message.Body);
|
|
171
|
-
if (opt.verbose) {
|
|
172
|
-
console.error(chalk_1.default.blue(' Processing payload:'), payload);
|
|
173
|
-
}
|
|
174
|
-
else if (!opt.disableLog) {
|
|
175
|
-
console.log(JSON.stringify({
|
|
176
|
-
event: 'MESSAGE_PROCESSING_START',
|
|
177
|
-
timestamp: new Date(),
|
|
178
|
-
messageId: message.MessageId,
|
|
179
|
-
payload: payload
|
|
180
|
-
}));
|
|
181
|
-
}
|
|
182
|
-
jobStart = new Date();
|
|
183
|
-
visibilityTimeout = 30 // this should be the queue timeout
|
|
184
|
-
;
|
|
185
|
-
// Extend when we get 50% of the way to timeout
|
|
186
|
-
timeoutExtender = setTimeout(extendTimeout, visibilityTimeout * 1000 * 0.5);
|
|
187
|
-
debug('timeout', visibilityTimeout * 1000 * 0.5);
|
|
188
|
-
_a.label = 1;
|
|
189
|
-
case 1:
|
|
190
|
-
_a.trys.push([1, 4, , 7]);
|
|
191
|
-
queue = qname.slice(opt.prefix.length);
|
|
192
|
-
return [4 /*yield*/, callback(queue, payload)];
|
|
193
|
-
case 2:
|
|
194
|
-
result = _a.sent();
|
|
195
|
-
debug('processMessage callback finished', { payload: payload, result: result });
|
|
196
|
-
clearTimeout(timeoutExtender);
|
|
197
|
-
if (opt.verbose) {
|
|
198
|
-
console.error(chalk_1.default.green(' SUCCESS'));
|
|
199
|
-
console.error(chalk_1.default.blue(' cleaning up (removing message) ...'));
|
|
200
|
-
}
|
|
201
|
-
return [4 /*yield*/, (0, sqs_js_1.getSQSClient)().send(new client_sqs_1.DeleteMessageCommand({
|
|
202
|
-
QueueUrl: qrl,
|
|
203
|
-
ReceiptHandle: message.ReceiptHandle
|
|
204
|
-
}))];
|
|
205
|
-
case 3:
|
|
206
|
-
_a.sent();
|
|
207
|
-
if (opt.verbose) {
|
|
208
|
-
console.error(chalk_1.default.blue(' done'));
|
|
209
|
-
console.error();
|
|
210
|
-
}
|
|
211
|
-
else if (!opt.disableLog) {
|
|
212
|
-
console.log(JSON.stringify({
|
|
213
|
-
event: 'MESSAGE_PROCESSING_COMPLETE',
|
|
214
|
-
timestamp: new Date(),
|
|
215
|
-
messageId: message.MessageId,
|
|
216
|
-
payload: payload
|
|
217
|
-
}));
|
|
218
|
-
}
|
|
219
|
-
return [2 /*return*/, { noJobs: 0, jobsSucceeded: 1, jobsFailed: 0 }];
|
|
220
|
-
case 4:
|
|
221
|
-
err_1 = _a.sent();
|
|
222
|
-
debug('exec.catch');
|
|
223
|
-
clearTimeout(timeoutExtender);
|
|
224
|
-
if (!(err_1 instanceof DoNotProcess)) return [3 /*break*/, 6];
|
|
225
|
-
if (opt.verbose) {
|
|
226
|
-
console.error(chalk_1.default.blue(' callback ') + chalk_1.default.yellow('REFUSED'));
|
|
227
|
-
console.error(chalk_1.default.blue(' cleaning up (removing message) ...'));
|
|
228
|
-
}
|
|
229
|
-
return [4 /*yield*/, (0, sqs_js_1.getSQSClient)().send(new client_sqs_1.ChangeMessageVisibilityCommand({
|
|
230
|
-
QueueUrl: qrl,
|
|
231
|
-
ReceiptHandle: message.ReceiptHandle,
|
|
232
|
-
VisibilityTimeout: 0
|
|
233
|
-
}))];
|
|
234
|
-
case 5:
|
|
235
|
-
result = _a.sent();
|
|
236
|
-
debug('ChangeMessageVisibility returned', result);
|
|
237
|
-
return [2 /*return*/, { noJobs: 1, jobsSucceeded: 0, jobsFailed: 0 }];
|
|
238
|
-
case 6:
|
|
239
|
-
// Fail path for job execution
|
|
240
|
-
if (opt.verbose) {
|
|
241
|
-
console.error(chalk_1.default.red(' FAILED'));
|
|
242
|
-
console.error(chalk_1.default.blue(' error : ') + err_1);
|
|
243
|
-
}
|
|
244
|
-
else if (!opt.disableLog) {
|
|
245
|
-
// Production error logging
|
|
246
|
-
console.log(JSON.stringify({
|
|
247
|
-
event: 'MESSAGE_PROCESSING_FAILED',
|
|
248
|
-
reason: 'exception thrown',
|
|
249
|
-
timestamp: new Date(),
|
|
250
|
-
messageId: message.MessageId,
|
|
251
|
-
payload: payload,
|
|
252
|
-
errorMessage: err_1.toString().split('\n').slice(1).join('\n').trim() || undefined,
|
|
253
|
-
err: err_1
|
|
254
|
-
}));
|
|
255
|
-
}
|
|
256
|
-
return [2 /*return*/, { noJobs: 0, jobsSucceeded: 0, jobsFailed: 1 }];
|
|
257
|
-
case 7: return [2 /*return*/];
|
|
258
|
-
}
|
|
259
|
-
});
|
|
260
|
-
});
|
|
261
|
-
}
|
|
262
|
-
exports.processMessage = processMessage;
|
|
263
|
-
//
|
|
264
|
-
// Pull work off of a single queue
|
|
265
|
-
//
|
|
266
|
-
function pollSingleQueue(qname, qrl, callback, opt) {
|
|
70
|
+
function getMessages(qrl, opt, maxMessages) {
|
|
267
71
|
return __awaiter(this, void 0, void 0, function () {
|
|
268
|
-
var params, response
|
|
72
|
+
var params, response;
|
|
269
73
|
return __generator(this, function (_a) {
|
|
270
74
|
switch (_a.label) {
|
|
271
75
|
case 0:
|
|
272
|
-
debug('pollSingleQueue', { qname: qname, qrl: qrl, callback: callback, opt: opt });
|
|
273
76
|
params = {
|
|
274
77
|
AttributeNames: ['All'],
|
|
275
|
-
MaxNumberOfMessages:
|
|
78
|
+
MaxNumberOfMessages: maxMessages,
|
|
276
79
|
MessageAttributeNames: ['All'],
|
|
277
80
|
QueueUrl: qrl,
|
|
278
81
|
VisibilityTimeout: 30,
|
|
279
82
|
WaitTimeSeconds: opt.waitTime
|
|
280
83
|
};
|
|
281
|
-
return [4 /*yield*/, (0, sqs_js_1.getSQSClient)().send(new client_sqs_1.ReceiveMessageCommand(params))
|
|
282
|
-
|
|
283
|
-
response = _a.sent();
|
|
284
|
-
debug('ReceiveMessage response', response);
|
|
285
|
-
if (shutdownRequested)
|
|
286
|
-
return [2 /*return*/, { noJobs: 0, jobsSucceeded: 0, jobsFailed: 0 }];
|
|
287
|
-
if (response.Messages) {
|
|
288
|
-
message = response.Messages[0];
|
|
289
|
-
if (opt.verbose)
|
|
290
|
-
console.error(chalk_1.default.blue(' Found message ' + message.MessageId));
|
|
291
|
-
return [2 /*return*/, processMessage(message, callback, qname, qrl, opt)];
|
|
292
|
-
}
|
|
293
|
-
else {
|
|
294
|
-
return [2 /*return*/, { noJobs: 1, jobsSucceeded: 0, jobsFailed: 0 }];
|
|
295
|
-
}
|
|
296
|
-
return [2 /*return*/];
|
|
297
|
-
}
|
|
298
|
-
});
|
|
299
|
-
});
|
|
300
|
-
}
|
|
301
|
-
exports.pollSingleQueue = pollSingleQueue;
|
|
302
|
-
//
|
|
303
|
-
// Resolve a set of queues
|
|
304
|
-
//
|
|
305
|
-
function resolveQueues(queues, opt) {
|
|
306
|
-
return __awaiter(this, void 0, void 0, function () {
|
|
307
|
-
var qnames, pairs, activePairs, selectedPairs;
|
|
308
|
-
var _this = this;
|
|
309
|
-
return __generator(this, function (_a) {
|
|
310
|
-
switch (_a.label) {
|
|
311
|
-
case 0:
|
|
312
|
-
// Start processing
|
|
313
|
-
if (opt.verbose)
|
|
314
|
-
console.error(chalk_1.default.blue('Resolving queues: ') + queues.join(' '));
|
|
315
|
-
qnames = queues.map(function (queue) { return (0, qrlCache_js_1.normalizeQueueName)(queue, opt); });
|
|
316
|
-
return [4 /*yield*/, (0, qrlCache_js_1.getQnameUrlPairs)(qnames, opt)
|
|
317
|
-
// Figure out which pairs are active
|
|
84
|
+
return [4 /*yield*/, (0, sqs_js_1.getSQSClient)().send(new client_sqs_1.ReceiveMessageCommand(params))
|
|
85
|
+
// debug('ReceiveMessage response', response)
|
|
318
86
|
];
|
|
319
87
|
case 1:
|
|
320
|
-
|
|
321
|
-
|
|
322
|
-
|
|
323
|
-
debug({ pairsBeforeCheck: pairs });
|
|
324
|
-
return [4 /*yield*/, Promise.all(pairs.map(function (pair) { return __awaiter(_this, void 0, void 0, function () {
|
|
325
|
-
var idle;
|
|
326
|
-
return __generator(this, function (_a) {
|
|
327
|
-
switch (_a.label) {
|
|
328
|
-
case 0: return [4 /*yield*/, (0, idleQueues_js_1.cheapIdleCheck)(pair.qname, pair.qrl, opt)];
|
|
329
|
-
case 1:
|
|
330
|
-
idle = (_a.sent()).idle;
|
|
331
|
-
if (!idle)
|
|
332
|
-
activePairs.push(pair);
|
|
333
|
-
return [2 /*return*/];
|
|
334
|
-
}
|
|
335
|
-
});
|
|
336
|
-
}); }))];
|
|
337
|
-
case 2:
|
|
338
|
-
_a.sent();
|
|
339
|
-
_a.label = 3;
|
|
340
|
-
case 3:
|
|
341
|
-
// Finished resolving
|
|
342
|
-
debug('getQnameUrlPairs.then');
|
|
343
|
-
if (opt.verbose) {
|
|
344
|
-
console.error(chalk_1.default.blue(' done'));
|
|
345
|
-
console.error();
|
|
346
|
-
}
|
|
347
|
-
selectedPairs = (opt.activeOnly ? activePairs : pairs)
|
|
348
|
-
.filter(function (_a) {
|
|
349
|
-
var qname = _a.qname;
|
|
350
|
-
var suf = opt.failSuffix + (opt.fifo ? '.fifo' : '');
|
|
351
|
-
var isFailQueue = qname.slice(-suf.length) === suf;
|
|
352
|
-
var shouldInclude = opt.includeFailed ? true : !isFailQueue;
|
|
353
|
-
return shouldInclude;
|
|
354
|
-
});
|
|
355
|
-
return [2 /*return*/, selectedPairs];
|
|
88
|
+
response = _a.sent();
|
|
89
|
+
// debug('ReceiveMessage response', response)
|
|
90
|
+
return [2 /*return*/, response.Messages || []];
|
|
356
91
|
}
|
|
357
92
|
});
|
|
358
93
|
});
|
|
359
94
|
}
|
|
360
|
-
exports.
|
|
95
|
+
exports.getMessages = getMessages;
|
|
361
96
|
//
|
|
362
97
|
// Consumer
|
|
363
98
|
//
|
|
364
99
|
function processMessages(queues, callback, options) {
|
|
365
100
|
return __awaiter(this, void 0, void 0, function () {
|
|
366
|
-
|
|
367
|
-
|
|
368
|
-
|
|
369
|
-
|
|
370
|
-
var activeQueues = Object.keys(activeLoops).filter(function (q) { return activeLoops[q]; }).map(function (q) { return q.slice(opt.prefix.length); });
|
|
371
|
-
if (activeQueues.length) {
|
|
372
|
-
console.error(chalk_1.default.blue('Waiting for work to finish on the following queues: ') + activeQueues.join(chalk_1.default.blue(', ')));
|
|
373
|
-
}
|
|
374
|
-
clearTimeout(delayTimeout);
|
|
375
|
-
}
|
|
376
|
-
}
|
|
377
|
-
// Listen to a queue until it is out of messages
|
|
378
|
-
function listenLoop(qname, qrl) {
|
|
379
|
-
return __awaiter(this, void 0, void 0, function () {
|
|
380
|
-
var _a, noJobs, jobsSucceeded, jobsFailed, err_3;
|
|
381
|
-
return __generator(this, function (_b) {
|
|
382
|
-
switch (_b.label) {
|
|
383
|
-
case 0:
|
|
384
|
-
_b.trys.push([0, 2, 3, 4]);
|
|
385
|
-
if (shutdownRequested)
|
|
386
|
-
return [2 /*return*/];
|
|
387
|
-
if (opt.verbose) {
|
|
388
|
-
console.error(chalk_1.default.blue('Looking for work on ') +
|
|
389
|
-
qname.slice(opt.prefix.length) +
|
|
390
|
-
chalk_1.default.blue(' (' + qrl + ')'));
|
|
391
|
-
}
|
|
392
|
-
return [4 /*yield*/, pollSingleQueue(qname, qrl, callback, opt)];
|
|
393
|
-
case 1:
|
|
394
|
-
_a = _b.sent(), noJobs = _a.noJobs, jobsSucceeded = _a.jobsSucceeded, jobsFailed = _a.jobsFailed;
|
|
395
|
-
stats.noJobs += noJobs;
|
|
396
|
-
stats.jobsFailed += jobsFailed;
|
|
397
|
-
stats.jobsSucceeded += jobsSucceeded;
|
|
398
|
-
// No work? return to outer loop
|
|
399
|
-
if (noJobs)
|
|
400
|
-
return [2 /*return*/];
|
|
401
|
-
// Otherwise keep going
|
|
402
|
-
return [2 /*return*/, listenLoop(qname, qrl)];
|
|
403
|
-
case 2:
|
|
404
|
-
err_3 = _b.sent();
|
|
405
|
-
// TODO: Sentry
|
|
406
|
-
console.error(chalk_1.default.red(' ERROR in listenLoop'));
|
|
407
|
-
console.error(chalk_1.default.blue(' error : ') + err_3);
|
|
408
|
-
return [3 /*break*/, 4];
|
|
409
|
-
case 3:
|
|
410
|
-
delete activeLoops[qname];
|
|
411
|
-
return [7 /*endfinally*/];
|
|
412
|
-
case 4: return [2 /*return*/];
|
|
413
|
-
}
|
|
414
|
-
});
|
|
415
|
-
});
|
|
416
|
-
}
|
|
417
|
-
var opt, stats, activeLoops, delayTimeout, delay, start, selectedPairs, _i, selectedPairs_1, _a, qname, qrl, msSoFar, msUntilNextResolve;
|
|
418
|
-
return __generator(this, function (_b) {
|
|
419
|
-
switch (_b.label) {
|
|
101
|
+
var opt, systemMonitor, jobExecutor, queueManager, maxActiveJobs, delayTimeout, delay, activeQrls, maxReturnCount, listen, allowedJobs, maxLatency, latency, latencyFactor, targetJobs, jobsLeft, _i, _a, _b, qname, qrl, maxMessages;
|
|
102
|
+
var _this = this;
|
|
103
|
+
return __generator(this, function (_c) {
|
|
104
|
+
switch (_c.label) {
|
|
420
105
|
case 0:
|
|
421
106
|
opt = (0, defaults_js_1.getOptionsWithDefaults)(options);
|
|
422
107
|
debug('processMessages', { queues: queues, callback: callback, options: options, opt: opt });
|
|
423
|
-
|
|
424
|
-
|
|
108
|
+
systemMonitor = new systemMonitor_js_1.SystemMonitor(opt);
|
|
109
|
+
jobExecutor = new jobExecutor_js_1.JobExecutor(opt);
|
|
110
|
+
queueManager = new queueManager_js_1.QueueManager(opt, queues);
|
|
111
|
+
maxActiveJobs = 100;
|
|
112
|
+
debug({ systemMonitor: systemMonitor, jobExecutor: jobExecutor, queueManager: queueManager });
|
|
113
|
+
shutdownCallbacks.push(function () {
|
|
114
|
+
systemMonitor.shutdown();
|
|
115
|
+
queueManager.shutdown();
|
|
116
|
+
jobExecutor.shutdown();
|
|
117
|
+
});
|
|
425
118
|
delay = function (ms) { return new Promise(function (resolve) {
|
|
426
119
|
delayTimeout = setTimeout(resolve, ms);
|
|
427
120
|
}); };
|
|
428
|
-
|
|
429
|
-
|
|
121
|
+
activeQrls = new Set();
|
|
122
|
+
maxReturnCount = 0;
|
|
123
|
+
listen = function (qname, qrl, maxMessages) { return __awaiter(_this, void 0, void 0, function () {
|
|
124
|
+
var messages, _i, messages_1, message;
|
|
125
|
+
return __generator(this, function (_a) {
|
|
126
|
+
switch (_a.label) {
|
|
127
|
+
case 0:
|
|
128
|
+
activeQrls.add(qrl);
|
|
129
|
+
maxReturnCount += maxMessages;
|
|
130
|
+
return [4 /*yield*/, getMessages(qrl, opt, maxMessages)];
|
|
131
|
+
case 1:
|
|
132
|
+
messages = _a.sent();
|
|
133
|
+
for (_i = 0, messages_1 = messages; _i < messages_1.length; _i++) {
|
|
134
|
+
message = messages_1[_i];
|
|
135
|
+
jobExecutor.executeJob(message, callback, qname, qrl);
|
|
136
|
+
}
|
|
137
|
+
maxReturnCount -= maxMessages;
|
|
138
|
+
activeQrls.delete(qrl);
|
|
139
|
+
return [2 /*return*/];
|
|
140
|
+
}
|
|
141
|
+
});
|
|
142
|
+
}); };
|
|
143
|
+
_c.label = 1;
|
|
430
144
|
case 1:
|
|
431
|
-
if (!!shutdownRequested) return [3 /*break*/,
|
|
432
|
-
|
|
433
|
-
|
|
434
|
-
|
|
435
|
-
|
|
436
|
-
|
|
437
|
-
|
|
438
|
-
|
|
439
|
-
|
|
440
|
-
|
|
441
|
-
|
|
442
|
-
if (
|
|
443
|
-
|
|
444
|
-
|
|
445
|
-
|
|
446
|
-
|
|
447
|
-
|
|
448
|
-
console.error();
|
|
449
|
-
}
|
|
450
|
-
// Launch listen loop for each queue
|
|
451
|
-
for (_i = 0, selectedPairs_1 = selectedPairs; _i < selectedPairs_1.length; _i++) {
|
|
452
|
-
_a = selectedPairs_1[_i], qname = _a.qname, qrl = _a.qrl;
|
|
453
|
-
if (!activeLoops[qname])
|
|
454
|
-
activeLoops[qname] = listenLoop(qname, qrl);
|
|
145
|
+
if (!!shutdownRequested) return [3 /*break*/, 3];
|
|
146
|
+
allowedJobs = maxActiveJobs - jobExecutor.activeJobCount() - maxReturnCount;
|
|
147
|
+
maxLatency = 100;
|
|
148
|
+
latency = systemMonitor.getLatency().setTimeout;
|
|
149
|
+
latencyFactor = 1 - Math.abs(Math.min(latency / maxLatency, 1)) // 0 if latency is at max, 1 if latency 0
|
|
150
|
+
;
|
|
151
|
+
targetJobs = Math.round(allowedJobs * latencyFactor);
|
|
152
|
+
debug({ allowedJobs: allowedJobs, maxLatency: maxLatency, latency: latency, latencyFactor: latencyFactor, targetJobs: targetJobs, activeQrls: activeQrls });
|
|
153
|
+
jobsLeft = targetJobs;
|
|
154
|
+
for (_i = 0, _a = queueManager.getPairs(); _i < _a.length; _i++) {
|
|
155
|
+
_b = _a[_i], qname = _b.qname, qrl = _b.qrl;
|
|
156
|
+
if (jobsLeft <= 0 || activeQrls.has(qrl))
|
|
157
|
+
continue;
|
|
158
|
+
maxMessages = Math.min(10, jobsLeft);
|
|
159
|
+
listen(qname, qrl, maxMessages);
|
|
160
|
+
jobsLeft -= maxMessages;
|
|
161
|
+
if (this.opt.verbose) {
|
|
162
|
+
console.error(chalk_1.default.blue('Listening on: '), qname);
|
|
455
163
|
}
|
|
164
|
+
debug({ listenedTo: { qname: qname, maxMessages: maxMessages, jobsLeft: jobsLeft } });
|
|
456
165
|
}
|
|
457
|
-
|
|
458
|
-
|
|
459
|
-
|
|
460
|
-
|
|
461
|
-
if (!msUntilNextResolve) return [3 /*break*/, 4];
|
|
462
|
-
if (opt.verbose)
|
|
463
|
-
console.error(chalk_1.default.blue('Will resolve queues again in ' + Math.round(msUntilNextResolve / 1000) + ' seconds'));
|
|
464
|
-
return [4 /*yield*/, delay(msUntilNextResolve)];
|
|
166
|
+
return [4 /*yield*/, delay(1000)];
|
|
167
|
+
case 2:
|
|
168
|
+
_c.sent();
|
|
169
|
+
return [3 /*break*/, 1];
|
|
465
170
|
case 3:
|
|
466
|
-
|
|
467
|
-
_b.label = 4;
|
|
468
|
-
case 4: return [3 /*break*/, 1];
|
|
469
|
-
case 5:
|
|
470
|
-
// Wait on all work to finish
|
|
471
|
-
// shutdownCallback()
|
|
472
|
-
return [4 /*yield*/, Promise.all(Object.values(activeLoops))];
|
|
473
|
-
case 6:
|
|
474
|
-
// Wait on all work to finish
|
|
475
|
-
// shutdownCallback()
|
|
476
|
-
_b.sent();
|
|
171
|
+
debug('after all');
|
|
477
172
|
return [2 /*return*/];
|
|
478
173
|
}
|
|
479
174
|
});
|
|
480
175
|
});
|
|
481
176
|
}
|
|
482
177
|
exports.processMessages = processMessages;
|
|
483
|
-
debug('loaded');
|