qdone 2.0.28-alpha → 2.0.30-alpha
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/commonjs/src/cache.js +22 -93
- package/commonjs/src/cloudWatch.js +65 -111
- package/commonjs/src/consumer.js +135 -237
- package/commonjs/src/defaults.js +11 -11
- package/commonjs/src/enqueue.js +311 -503
- package/commonjs/src/exponentialBackoff.js +39 -110
- package/commonjs/src/idleQueues.js +255 -396
- package/commonjs/src/monitor.js +42 -115
- package/commonjs/src/qrlCache.js +80 -162
- package/commonjs/src/scheduler/jobExecutor.js +305 -363
- package/commonjs/src/scheduler/queueManager.js +116 -190
- package/commonjs/src/scheduler/systemMonitor.js +24 -28
- package/commonjs/src/sqs.js +58 -141
- package/package.json +2 -2
- package/src/consumer.js +3 -5
- package/src/defaults.js +3 -3
- package/src/enqueue.js +1 -1
- package/src/scheduler/jobExecutor.js +77 -33
- package/src/scheduler/queueManager.js +13 -4
- package/npm-shrinkwrap.json +0 -15999
|
@@ -1,49 +1,4 @@
|
|
|
1
1
|
"use strict";
|
|
2
|
-
var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
|
|
3
|
-
function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
|
|
4
|
-
return new (P || (P = Promise))(function (resolve, reject) {
|
|
5
|
-
function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
|
|
6
|
-
function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
|
|
7
|
-
function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
|
|
8
|
-
step((generator = generator.apply(thisArg, _arguments || [])).next());
|
|
9
|
-
});
|
|
10
|
-
};
|
|
11
|
-
var __generator = (this && this.__generator) || function (thisArg, body) {
|
|
12
|
-
var _ = { label: 0, sent: function() { if (t[0] & 1) throw t[1]; return t[1]; }, trys: [], ops: [] }, f, y, t, g;
|
|
13
|
-
return g = { next: verb(0), "throw": verb(1), "return": verb(2) }, typeof Symbol === "function" && (g[Symbol.iterator] = function() { return this; }), g;
|
|
14
|
-
function verb(n) { return function (v) { return step([n, v]); }; }
|
|
15
|
-
function step(op) {
|
|
16
|
-
if (f) throw new TypeError("Generator is already executing.");
|
|
17
|
-
while (g && (g = 0, op[0] && (_ = 0)), _) try {
|
|
18
|
-
if (f = 1, y && (t = op[0] & 2 ? y["return"] : op[0] ? y["throw"] || ((t = y["return"]) && t.call(y), 0) : y.next) && !(t = t.call(y, op[1])).done) return t;
|
|
19
|
-
if (y = 0, t) op = [op[0] & 2, t.value];
|
|
20
|
-
switch (op[0]) {
|
|
21
|
-
case 0: case 1: t = op; break;
|
|
22
|
-
case 4: _.label++; return { value: op[1], done: false };
|
|
23
|
-
case 5: _.label++; y = op[1]; op = [0]; continue;
|
|
24
|
-
case 7: op = _.ops.pop(); _.trys.pop(); continue;
|
|
25
|
-
default:
|
|
26
|
-
if (!(t = _.trys, t = t.length > 0 && t[t.length - 1]) && (op[0] === 6 || op[0] === 2)) { _ = 0; continue; }
|
|
27
|
-
if (op[0] === 3 && (!t || (op[1] > t[0] && op[1] < t[3]))) { _.label = op[1]; break; }
|
|
28
|
-
if (op[0] === 6 && _.label < t[1]) { _.label = t[1]; t = op; break; }
|
|
29
|
-
if (t && _.label < t[2]) { _.label = t[2]; _.ops.push(op); break; }
|
|
30
|
-
if (t[2]) _.ops.pop();
|
|
31
|
-
_.trys.pop(); continue;
|
|
32
|
-
}
|
|
33
|
-
op = body.call(thisArg, _);
|
|
34
|
-
} catch (e) { op = [6, e]; y = 0; } finally { f = t = 0; }
|
|
35
|
-
if (op[0] & 5) throw op[1]; return { value: op[0] ? op[1] : void 0, done: true };
|
|
36
|
-
}
|
|
37
|
-
};
|
|
38
|
-
var __spreadArray = (this && this.__spreadArray) || function (to, from, pack) {
|
|
39
|
-
if (pack || arguments.length === 2) for (var i = 0, l = from.length, ar; i < l; i++) {
|
|
40
|
-
if (ar || !(i in from)) {
|
|
41
|
-
if (!ar) ar = Array.prototype.slice.call(from, 0, i);
|
|
42
|
-
ar[i] = from[i];
|
|
43
|
-
}
|
|
44
|
-
}
|
|
45
|
-
return to.concat(ar || Array.prototype.slice.call(from));
|
|
46
|
-
};
|
|
47
2
|
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
48
3
|
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
49
4
|
};
|
|
@@ -52,17 +7,17 @@ exports.idleQueues = exports.processQueuePair = exports.processQueue = exports.d
|
|
|
52
7
|
/**
|
|
53
8
|
* Implementation of checks and caching of checks to determine if queues are idle.
|
|
54
9
|
*/
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
10
|
+
const chalk_1 = __importDefault(require("chalk"));
|
|
11
|
+
const sqs_js_1 = require("./sqs.js");
|
|
12
|
+
const cloudWatch_js_1 = require("./cloudWatch.js");
|
|
13
|
+
const defaults_js_1 = require("./defaults.js");
|
|
14
|
+
const client_sqs_1 = require("@aws-sdk/client-sqs");
|
|
15
|
+
const client_cloudwatch_1 = require("@aws-sdk/client-cloudwatch");
|
|
16
|
+
const qrlCache_js_1 = require("./qrlCache.js");
|
|
17
|
+
const cache_js_1 = require("./cache.js");
|
|
63
18
|
// const AWS = require('aws-sdk')
|
|
64
|
-
|
|
65
|
-
|
|
19
|
+
const debug_1 = __importDefault(require("debug"));
|
|
20
|
+
const debug = (0, debug_1.default)('qdone:idleQueues');
|
|
66
21
|
// Queue attributes we check to determine idle
|
|
67
22
|
exports.attributeNames = [
|
|
68
23
|
'ApproximateNumberOfMessages',
|
|
@@ -70,7 +25,7 @@ exports.attributeNames = [
|
|
|
70
25
|
'ApproximateNumberOfMessagesDelayed'
|
|
71
26
|
];
|
|
72
27
|
// CloudWatch metrics we check to determine idle
|
|
73
|
-
|
|
28
|
+
const metricNames = [
|
|
74
29
|
'NumberOfMessagesSent',
|
|
75
30
|
'NumberOfMessagesReceived',
|
|
76
31
|
'NumberOfMessagesDeleted',
|
|
@@ -83,114 +38,79 @@ var metricNames = [
|
|
|
83
38
|
/**
|
|
84
39
|
* Actual SQS call, used in conjunction with cache.
|
|
85
40
|
*/
|
|
86
|
-
function _cheapIdleCheck(qname, qrl, opt) {
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
// Count deleted queues as idle
|
|
108
|
-
return [2 /*return*/, { idle: true, SQS: 1 }];
|
|
109
|
-
}
|
|
110
|
-
else {
|
|
111
|
-
throw e_1;
|
|
112
|
-
}
|
|
113
|
-
return [3 /*break*/, 3];
|
|
114
|
-
case 3: return [2 /*return*/];
|
|
115
|
-
}
|
|
116
|
-
});
|
|
117
|
-
});
|
|
41
|
+
async function _cheapIdleCheck(qname, qrl, opt) {
|
|
42
|
+
try {
|
|
43
|
+
const client = (0, sqs_js_1.getSQSClient)();
|
|
44
|
+
const cmd = new client_sqs_1.GetQueueAttributesCommand({ AttributeNames: exports.attributeNames, QueueUrl: qrl });
|
|
45
|
+
const data = await client.send(cmd);
|
|
46
|
+
debug('data', data);
|
|
47
|
+
const result = data.Attributes;
|
|
48
|
+
result.queue = qname.slice(opt.prefix.length);
|
|
49
|
+
// We are idle if all the messages attributes are zero
|
|
50
|
+
result.idle = exports.attributeNames.filter(k => result[k] === '0').length === exports.attributeNames.length;
|
|
51
|
+
return { result, SQS: 1 };
|
|
52
|
+
}
|
|
53
|
+
catch (e) {
|
|
54
|
+
if (e instanceof client_sqs_1.QueueDoesNotExist) {
|
|
55
|
+
// Count deleted queues as idle
|
|
56
|
+
return { idle: true, SQS: 1 };
|
|
57
|
+
}
|
|
58
|
+
else {
|
|
59
|
+
throw e;
|
|
60
|
+
}
|
|
61
|
+
}
|
|
118
62
|
}
|
|
119
63
|
exports._cheapIdleCheck = _cheapIdleCheck;
|
|
120
64
|
/**
|
|
121
65
|
* Gets queue attributes from the SQS api and assesses whether queue is idle
|
|
122
66
|
* at this immediate moment.
|
|
123
67
|
*/
|
|
124
|
-
function cheapIdleCheck(qname, qrl, opt) {
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
return
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
debug({ action: 'do real check' });
|
|
146
|
-
return [4 /*yield*/, _cheapIdleCheck(qname, qrl, opt)];
|
|
147
|
-
case 3:
|
|
148
|
-
_a = _b.sent(), result = _a.result, SQS = _a.SQS;
|
|
149
|
-
debug({ action: 'setCache', key: key, result: result });
|
|
150
|
-
return [4 /*yield*/, (0, cache_js_1.setCache)(key, result, opt)];
|
|
151
|
-
case 4:
|
|
152
|
-
ok = _b.sent();
|
|
153
|
-
debug({ action: 'return result of set cache', ok: ok });
|
|
154
|
-
return [2 /*return*/, { result: result, SQS: SQS }];
|
|
155
|
-
}
|
|
156
|
-
});
|
|
157
|
-
});
|
|
68
|
+
async function cheapIdleCheck(qname, qrl, opt) {
|
|
69
|
+
// Just call the API if we don't have a cache
|
|
70
|
+
if (!opt.cacheUri)
|
|
71
|
+
return _cheapIdleCheck(qname, qrl, opt);
|
|
72
|
+
// Otherwise check cache
|
|
73
|
+
const key = 'cheap-idle-check:' + qrl;
|
|
74
|
+
const cacheResult = await (0, cache_js_1.getCache)(key, opt);
|
|
75
|
+
debug({ cacheResult });
|
|
76
|
+
if (cacheResult) {
|
|
77
|
+
debug({ action: 'return resolved' });
|
|
78
|
+
return { result: cacheResult, SQS: 0 };
|
|
79
|
+
}
|
|
80
|
+
else {
|
|
81
|
+
// Cache miss, make call
|
|
82
|
+
debug({ action: 'do real check' });
|
|
83
|
+
const { result, SQS } = await _cheapIdleCheck(qname, qrl, opt);
|
|
84
|
+
debug({ action: 'setCache', key, result });
|
|
85
|
+
const ok = await (0, cache_js_1.setCache)(key, result, opt);
|
|
86
|
+
debug({ action: 'return result of set cache', ok });
|
|
87
|
+
return { result, SQS };
|
|
88
|
+
}
|
|
158
89
|
}
|
|
159
90
|
exports.cheapIdleCheck = cheapIdleCheck;
|
|
160
91
|
/**
|
|
161
92
|
* Gets a single metric from the CloudWatch api.
|
|
162
93
|
*/
|
|
163
|
-
function getMetric(qname, qrl, metricName, opt) {
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
cmd = new client_cloudwatch_1.GetMetricStatisticsCommand(params);
|
|
184
|
-
return [4 /*yield*/, client.send(cmd)];
|
|
185
|
-
case 1:
|
|
186
|
-
data = _b.sent();
|
|
187
|
-
debug('getMetric data', data);
|
|
188
|
-
return [2 /*return*/, (_a = {},
|
|
189
|
-
_a[metricName] = data.Datapoints.map(function (d) { return d.Sum; }).reduce(function (a, b) { return a + b; }, 0),
|
|
190
|
-
_a)];
|
|
191
|
-
}
|
|
192
|
-
});
|
|
193
|
-
});
|
|
94
|
+
async function getMetric(qname, qrl, metricName, opt) {
|
|
95
|
+
debug('getMetric', qname, qrl, metricName);
|
|
96
|
+
const now = new Date();
|
|
97
|
+
const params = {
|
|
98
|
+
StartTime: new Date(now.getTime() - 1000 * 60 * opt.idleFor),
|
|
99
|
+
EndTime: now,
|
|
100
|
+
MetricName: metricName,
|
|
101
|
+
Namespace: 'AWS/SQS',
|
|
102
|
+
Period: 3600,
|
|
103
|
+
Dimensions: [{ Name: 'QueueName', Value: qname }],
|
|
104
|
+
Statistics: ['Sum']
|
|
105
|
+
// Unit: ['']
|
|
106
|
+
};
|
|
107
|
+
const client = (0, cloudWatch_js_1.getCloudWatchClient)();
|
|
108
|
+
const cmd = new client_cloudwatch_1.GetMetricStatisticsCommand(params);
|
|
109
|
+
const data = await client.send(cmd);
|
|
110
|
+
debug('getMetric data', data);
|
|
111
|
+
return {
|
|
112
|
+
[metricName]: data.Datapoints.map(d => d.Sum).reduce((a, b) => a + b, 0)
|
|
113
|
+
};
|
|
194
114
|
}
|
|
195
115
|
exports.getMetric = getMetric;
|
|
196
116
|
/**
|
|
@@ -204,274 +124,213 @@ exports.getMetric = getMetric;
|
|
|
204
124
|
* We could randomize the order, but for my test use case, it's always cheaper
|
|
205
125
|
* to check NumberOfMessagesSent first, and is the primary indicator of use.
|
|
206
126
|
*/
|
|
207
|
-
function checkIdle(qname, qrl, opt) {
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
|
|
239
|
-
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
case 4:
|
|
248
|
-
_i++;
|
|
249
|
-
return [3 /*break*/, 2];
|
|
250
|
-
case 5:
|
|
251
|
-
stats = Object.assign.apply(Object, __spreadArray([{
|
|
252
|
-
queue: qname.slice(opt.prefix.length),
|
|
253
|
-
cheap: cheapResult,
|
|
254
|
-
apiCalls: apiCalls,
|
|
255
|
-
idle: idle
|
|
256
|
-
}], results // merge in results from CloudWatch
|
|
257
|
-
, false));
|
|
258
|
-
debug('checkIdle stats', stats);
|
|
259
|
-
return [2 /*return*/, stats];
|
|
260
|
-
}
|
|
261
|
-
});
|
|
262
|
-
});
|
|
127
|
+
async function checkIdle(qname, qrl, opt) {
|
|
128
|
+
// Do the cheap check first to make sure there is no data in flight at the moment
|
|
129
|
+
debug('checkIdle', qname, qrl);
|
|
130
|
+
const { result: cheapResult, SQS } = await cheapIdleCheck(qname, qrl, opt);
|
|
131
|
+
debug('cheapResult', cheapResult);
|
|
132
|
+
// Short circuit further calls if cheap result shows data
|
|
133
|
+
if (cheapResult.idle === false) {
|
|
134
|
+
return {
|
|
135
|
+
queue: qname.slice(opt.prefix.length),
|
|
136
|
+
cheap: cheapResult,
|
|
137
|
+
idle: false,
|
|
138
|
+
apiCalls: { SQS, CloudWatch: 0 }
|
|
139
|
+
};
|
|
140
|
+
}
|
|
141
|
+
// If we get here, there's nothing in the queue at the moment,
|
|
142
|
+
// so we have to check metrics one at a time
|
|
143
|
+
const apiCalls = { SQS: 1, CloudWatch: 0 };
|
|
144
|
+
const results = [];
|
|
145
|
+
let idle = true;
|
|
146
|
+
for (const metricName of metricNames) {
|
|
147
|
+
// Check metrics in order
|
|
148
|
+
const result = await getMetric(qname, qrl, metricName, opt);
|
|
149
|
+
results.push(result);
|
|
150
|
+
debug('getMetric result', result);
|
|
151
|
+
apiCalls.CloudWatch++;
|
|
152
|
+
// Recalculate idle
|
|
153
|
+
idle = result[metricName] === 0;
|
|
154
|
+
if (!idle)
|
|
155
|
+
break; // and stop checking metrics if we find evidence of activity
|
|
156
|
+
}
|
|
157
|
+
// Calculate stats
|
|
158
|
+
const stats = Object.assign({
|
|
159
|
+
queue: qname.slice(opt.prefix.length),
|
|
160
|
+
cheap: cheapResult,
|
|
161
|
+
apiCalls,
|
|
162
|
+
idle
|
|
163
|
+
}, ...results // merge in results from CloudWatch
|
|
164
|
+
);
|
|
165
|
+
debug('checkIdle stats', stats);
|
|
166
|
+
return stats;
|
|
263
167
|
}
|
|
264
168
|
exports.checkIdle = checkIdle;
|
|
265
169
|
/**
|
|
266
170
|
* Just deletes a queue.
|
|
267
171
|
*/
|
|
268
|
-
function deleteQueue(qname, qrl, opt) {
|
|
269
|
-
|
|
270
|
-
|
|
271
|
-
|
|
272
|
-
|
|
273
|
-
|
|
274
|
-
|
|
275
|
-
|
|
276
|
-
|
|
277
|
-
|
|
278
|
-
debug(result);
|
|
279
|
-
if (opt.verbose)
|
|
280
|
-
console.error(chalk_1.default.blue('Deleted ') + qname.slice(opt.prefix.length));
|
|
281
|
-
return [2 /*return*/, {
|
|
282
|
-
deleted: true,
|
|
283
|
-
apiCalls: { SQS: 1, CloudWatch: 0 }
|
|
284
|
-
}];
|
|
285
|
-
}
|
|
286
|
-
});
|
|
287
|
-
});
|
|
172
|
+
async function deleteQueue(qname, qrl, opt) {
|
|
173
|
+
const cmd = new client_sqs_1.DeleteQueueCommand({ QueueUrl: qrl });
|
|
174
|
+
const result = await (0, sqs_js_1.getSQSClient)().send(cmd);
|
|
175
|
+
debug(result);
|
|
176
|
+
if (opt.verbose)
|
|
177
|
+
console.error(chalk_1.default.blue('Deleted ') + qname.slice(opt.prefix.length));
|
|
178
|
+
return {
|
|
179
|
+
deleted: true,
|
|
180
|
+
apiCalls: { SQS: 1, CloudWatch: 0 }
|
|
181
|
+
};
|
|
288
182
|
}
|
|
289
183
|
exports.deleteQueue = deleteQueue;
|
|
290
184
|
/**
|
|
291
185
|
* Processes a single queue, checking for idle, deleting if applicable.
|
|
292
186
|
*/
|
|
293
|
-
function processQueue(qname, qrl, opt) {
|
|
294
|
-
|
|
295
|
-
|
|
296
|
-
|
|
297
|
-
|
|
298
|
-
|
|
299
|
-
|
|
300
|
-
|
|
301
|
-
|
|
302
|
-
|
|
303
|
-
|
|
304
|
-
|
|
305
|
-
|
|
306
|
-
|
|
307
|
-
|
|
308
|
-
|
|
309
|
-
|
|
310
|
-
|
|
311
|
-
|
|
312
|
-
|
|
313
|
-
return [4 /*yield*/, deleteQueue(qname, qrl, opt)];
|
|
314
|
-
case 2:
|
|
315
|
-
deleteResult = _a.sent();
|
|
316
|
-
resultIncludingDelete = Object.assign(result, {
|
|
317
|
-
deleted: deleteResult.deleted,
|
|
318
|
-
apiCalls: {
|
|
319
|
-
SQS: result.apiCalls.SQS + deleteResult.apiCalls.SQS,
|
|
320
|
-
CloudWatch: result.apiCalls.CloudWatch + deleteResult.apiCalls.CloudWatch
|
|
321
|
-
}
|
|
322
|
-
});
|
|
323
|
-
return [2 /*return*/, resultIncludingDelete];
|
|
324
|
-
case 3: return [2 /*return*/];
|
|
187
|
+
async function processQueue(qname, qrl, opt) {
|
|
188
|
+
const result = await checkIdle(qname, qrl, opt);
|
|
189
|
+
debug(qname, result);
|
|
190
|
+
// Queue is active
|
|
191
|
+
if (!result.idle) {
|
|
192
|
+
// Notify and return
|
|
193
|
+
if (opt.verbose)
|
|
194
|
+
console.error(chalk_1.default.blue('Queue ') + qname.slice(opt.prefix.length) + chalk_1.default.blue(' has been ') + 'active' + chalk_1.default.blue(' in the last ') + opt.idleFor + chalk_1.default.blue(' minutes.'));
|
|
195
|
+
return result;
|
|
196
|
+
}
|
|
197
|
+
// Queue is idle
|
|
198
|
+
if (opt.verbose)
|
|
199
|
+
console.error(chalk_1.default.blue('Queue ') + qname.slice(opt.prefix.length) + chalk_1.default.blue(' has been ') + 'idle' + chalk_1.default.blue(' for the last ') + opt.idleFor + chalk_1.default.blue(' minutes.'));
|
|
200
|
+
if (opt.delete) {
|
|
201
|
+
const deleteResult = await deleteQueue(qname, qrl, opt);
|
|
202
|
+
const resultIncludingDelete = Object.assign(result, {
|
|
203
|
+
deleted: deleteResult.deleted,
|
|
204
|
+
apiCalls: {
|
|
205
|
+
SQS: result.apiCalls.SQS + deleteResult.apiCalls.SQS,
|
|
206
|
+
CloudWatch: result.apiCalls.CloudWatch + deleteResult.apiCalls.CloudWatch
|
|
325
207
|
}
|
|
326
208
|
});
|
|
327
|
-
|
|
209
|
+
return resultIncludingDelete;
|
|
210
|
+
}
|
|
328
211
|
}
|
|
329
212
|
exports.processQueue = processQueue;
|
|
330
213
|
/**
|
|
331
214
|
* Processes a queue and its fail queue, treating them as a unit.
|
|
332
215
|
*/
|
|
333
|
-
function processQueuePair(qname, qrl, opt) {
|
|
334
|
-
|
|
335
|
-
|
|
336
|
-
|
|
337
|
-
|
|
338
|
-
|
|
339
|
-
|
|
340
|
-
|
|
341
|
-
|
|
342
|
-
|
|
343
|
-
|
|
344
|
-
|
|
345
|
-
|
|
346
|
-
|
|
347
|
-
|
|
348
|
-
|
|
349
|
-
|
|
350
|
-
|
|
351
|
-
|
|
352
|
-
|
|
353
|
-
|
|
354
|
-
|
|
355
|
-
|
|
356
|
-
|
|
357
|
-
|
|
358
|
-
|
|
359
|
-
|
|
360
|
-
case 3:
|
|
361
|
-
fresult = _b.sent();
|
|
362
|
-
debug('fresult', fresult);
|
|
363
|
-
idleCheckResult = Object.assign(result, { idle: result.idle && fresult.idle, failq: fresult }, {
|
|
364
|
-
apiCalls: {
|
|
365
|
-
SQS: result.apiCalls.SQS + fresult.apiCalls.SQS,
|
|
366
|
-
CloudWatch: result.apiCalls.CloudWatch + fresult.apiCalls.CloudWatch
|
|
367
|
-
}
|
|
368
|
-
});
|
|
369
|
-
factive = !fresult.idle;
|
|
370
|
-
if (factive) {
|
|
371
|
-
if (opt.verbose)
|
|
372
|
-
console.error(chalk_1.default.blue('Queue ') + fqname.slice(opt.prefix.length) + chalk_1.default.blue(' has been ') + 'active' + chalk_1.default.blue(' in the last ') + opt.idleFor + chalk_1.default.blue(' minutes.'));
|
|
373
|
-
return [2 /*return*/, idleCheckResult];
|
|
374
|
-
}
|
|
375
|
-
// Queue is idle
|
|
376
|
-
if (opt.verbose)
|
|
377
|
-
console.error(chalk_1.default.blue('Queue ') + fqname.slice(opt.prefix.length) + chalk_1.default.blue(' has been ') + 'idle' + chalk_1.default.blue(' for the last ') + opt.idleFor + chalk_1.default.blue(' minutes.'));
|
|
378
|
-
// Trigger a delete if the user wants it
|
|
379
|
-
if (!opt.delete)
|
|
380
|
-
return [2 /*return*/, idleCheckResult];
|
|
381
|
-
return [4 /*yield*/, Promise.all([
|
|
382
|
-
deleteQueue(qname, qrl, opt),
|
|
383
|
-
deleteQueue(fqname, fqrl, opt)
|
|
384
|
-
])];
|
|
385
|
-
case 4:
|
|
386
|
-
_a = _b.sent(), dresult = _a[0], dfresult = _a[1];
|
|
387
|
-
return [2 /*return*/, Object.assign(idleCheckResult, {
|
|
388
|
-
apiCalls: {
|
|
389
|
-
// Sum the SQS calls across all four
|
|
390
|
-
SQS: [result, fresult, dresult, dfresult]
|
|
391
|
-
.map(function (r) { return r.apiCalls.SQS; })
|
|
392
|
-
.reduce(function (a, b) { return a + b; }, 0),
|
|
393
|
-
// Sum the CloudWatch calls across all four
|
|
394
|
-
CloudWatch: [result, fresult, dresult, dfresult]
|
|
395
|
-
.map(function (r) { return r.apiCalls.CloudWatch; })
|
|
396
|
-
.reduce(function (a, b) { return a + b; }, 0)
|
|
397
|
-
}
|
|
398
|
-
})];
|
|
399
|
-
case 5:
|
|
400
|
-
e_2 = _b.sent();
|
|
401
|
-
// Handle the case where the fail queue has been deleted or was never
|
|
402
|
-
// created for some reason
|
|
403
|
-
if (!(e_2 instanceof client_sqs_1.QueueDoesNotExist))
|
|
404
|
-
throw e_2;
|
|
405
|
-
// Fail queue doesn't exist if we get here
|
|
406
|
-
if (opt.verbose)
|
|
407
|
-
console.error(chalk_1.default.blue('Queue ') + fqname.slice(opt.prefix.length) + chalk_1.default.blue(' does not exist.'));
|
|
408
|
-
// Handle delete
|
|
409
|
-
if (!opt.delete)
|
|
410
|
-
return [2 /*return*/, result];
|
|
411
|
-
return [4 /*yield*/, deleteQueue(qname, qrl, opt)];
|
|
412
|
-
case 6:
|
|
413
|
-
deleteResult = _b.sent();
|
|
414
|
-
resultIncludingDelete = Object.assign(result, {
|
|
415
|
-
deleted: deleteResult.deleted,
|
|
416
|
-
apiCalls: {
|
|
417
|
-
SQS: result.apiCalls.SQS + deleteResult.apiCalls.SQS,
|
|
418
|
-
CloudWatch: result.apiCalls.CloudWatch + deleteResult.apiCalls.CloudWatch
|
|
419
|
-
}
|
|
420
|
-
});
|
|
421
|
-
return [2 /*return*/, resultIncludingDelete];
|
|
422
|
-
case 7: return [2 /*return*/];
|
|
216
|
+
async function processQueuePair(qname, qrl, opt) {
|
|
217
|
+
const isFifo = qname.endsWith('.fifo');
|
|
218
|
+
const normalizeOptions = Object.assign({}, opt, { fifo: isFifo });
|
|
219
|
+
// Generate fail queue name/url
|
|
220
|
+
const fqname = (0, qrlCache_js_1.normalizeFailQueueName)(qname, normalizeOptions);
|
|
221
|
+
const fqrl = (0, qrlCache_js_1.normalizeFailQueueName)(fqname, normalizeOptions);
|
|
222
|
+
// Idle check
|
|
223
|
+
const result = await checkIdle(qname, qrl, opt);
|
|
224
|
+
debug('result', result);
|
|
225
|
+
// Queue is active
|
|
226
|
+
const active = !result.idle;
|
|
227
|
+
if (active) {
|
|
228
|
+
if (opt.verbose)
|
|
229
|
+
console.error(chalk_1.default.blue('Queue ') + qname.slice(opt.prefix.length) + chalk_1.default.blue(' has been ') + 'active' + chalk_1.default.blue(' in the last ') + opt.idleFor + chalk_1.default.blue(' minutes.'));
|
|
230
|
+
return result;
|
|
231
|
+
}
|
|
232
|
+
// Queue is idle
|
|
233
|
+
if (opt.verbose)
|
|
234
|
+
console.error(chalk_1.default.blue('Queue ') + qname.slice(opt.prefix.length) + chalk_1.default.blue(' has been ') + 'idle' + chalk_1.default.blue(' for the last ') + opt.idleFor + chalk_1.default.blue(' minutes.'));
|
|
235
|
+
// Check fail queue
|
|
236
|
+
try {
|
|
237
|
+
const fresult = await checkIdle(fqname, fqrl, opt);
|
|
238
|
+
debug('fresult', fresult);
|
|
239
|
+
const idleCheckResult = Object.assign(result, { idle: result.idle && fresult.idle, failq: fresult }, {
|
|
240
|
+
apiCalls: {
|
|
241
|
+
SQS: result.apiCalls.SQS + fresult.apiCalls.SQS,
|
|
242
|
+
CloudWatch: result.apiCalls.CloudWatch + fresult.apiCalls.CloudWatch
|
|
423
243
|
}
|
|
424
244
|
});
|
|
425
|
-
|
|
245
|
+
// Queue is active
|
|
246
|
+
const factive = !fresult.idle;
|
|
247
|
+
if (factive) {
|
|
248
|
+
if (opt.verbose)
|
|
249
|
+
console.error(chalk_1.default.blue('Queue ') + fqname.slice(opt.prefix.length) + chalk_1.default.blue(' has been ') + 'active' + chalk_1.default.blue(' in the last ') + opt.idleFor + chalk_1.default.blue(' minutes.'));
|
|
250
|
+
return idleCheckResult;
|
|
251
|
+
}
|
|
252
|
+
// Queue is idle
|
|
253
|
+
if (opt.verbose)
|
|
254
|
+
console.error(chalk_1.default.blue('Queue ') + fqname.slice(opt.prefix.length) + chalk_1.default.blue(' has been ') + 'idle' + chalk_1.default.blue(' for the last ') + opt.idleFor + chalk_1.default.blue(' minutes.'));
|
|
255
|
+
// Trigger a delete if the user wants it
|
|
256
|
+
if (!opt.delete)
|
|
257
|
+
return idleCheckResult;
|
|
258
|
+
const [dresult, dfresult] = await Promise.all([
|
|
259
|
+
deleteQueue(qname, qrl, opt),
|
|
260
|
+
deleteQueue(fqname, fqrl, opt)
|
|
261
|
+
]);
|
|
262
|
+
return Object.assign(idleCheckResult, {
|
|
263
|
+
apiCalls: {
|
|
264
|
+
// Sum the SQS calls across all four
|
|
265
|
+
SQS: [result, fresult, dresult, dfresult]
|
|
266
|
+
.map(r => r.apiCalls.SQS)
|
|
267
|
+
.reduce((a, b) => a + b, 0),
|
|
268
|
+
// Sum the CloudWatch calls across all four
|
|
269
|
+
CloudWatch: [result, fresult, dresult, dfresult]
|
|
270
|
+
.map(r => r.apiCalls.CloudWatch)
|
|
271
|
+
.reduce((a, b) => a + b, 0)
|
|
272
|
+
}
|
|
273
|
+
});
|
|
274
|
+
}
|
|
275
|
+
catch (e) {
|
|
276
|
+
// Handle the case where the fail queue has been deleted or was never
|
|
277
|
+
// created for some reason
|
|
278
|
+
if (!(e instanceof client_sqs_1.QueueDoesNotExist))
|
|
279
|
+
throw e;
|
|
280
|
+
// Fail queue doesn't exist if we get here
|
|
281
|
+
if (opt.verbose)
|
|
282
|
+
console.error(chalk_1.default.blue('Queue ') + fqname.slice(opt.prefix.length) + chalk_1.default.blue(' does not exist.'));
|
|
283
|
+
// Handle delete
|
|
284
|
+
if (!opt.delete)
|
|
285
|
+
return result;
|
|
286
|
+
const deleteResult = await deleteQueue(qname, qrl, opt);
|
|
287
|
+
const resultIncludingDelete = Object.assign(result, {
|
|
288
|
+
deleted: deleteResult.deleted,
|
|
289
|
+
apiCalls: {
|
|
290
|
+
SQS: result.apiCalls.SQS + deleteResult.apiCalls.SQS,
|
|
291
|
+
CloudWatch: result.apiCalls.CloudWatch + deleteResult.apiCalls.CloudWatch
|
|
292
|
+
}
|
|
293
|
+
});
|
|
294
|
+
return resultIncludingDelete;
|
|
295
|
+
}
|
|
426
296
|
}
|
|
427
297
|
exports.processQueuePair = processQueuePair;
|
|
428
298
|
//
|
|
429
299
|
// Resolve queues for listening loop listen
|
|
430
300
|
//
|
|
431
|
-
function idleQueues(queues, options) {
|
|
432
|
-
|
|
433
|
-
|
|
434
|
-
|
|
435
|
-
|
|
436
|
-
|
|
437
|
-
|
|
438
|
-
|
|
439
|
-
|
|
440
|
-
|
|
441
|
-
|
|
442
|
-
|
|
443
|
-
|
|
444
|
-
|
|
445
|
-
|
|
446
|
-
|
|
447
|
-
|
|
448
|
-
|
|
449
|
-
filteredEntries = entries.filter(function (entry) {
|
|
450
|
-
var suf = opt.failSuffix;
|
|
451
|
-
var sufFifo = opt.failSuffix + qrlCache_js_1.fifoSuffix;
|
|
452
|
-
var isFail = entry.qname.endsWith(suf);
|
|
453
|
-
var isFifoFail = entry.qname.endsWith(sufFifo);
|
|
454
|
-
return opt.includeFailed ? true : (!isFail && !isFifoFail);
|
|
455
|
-
});
|
|
456
|
-
// But only if we have queues to remove
|
|
457
|
-
if (filteredEntries.length) {
|
|
458
|
-
if (opt.verbose) {
|
|
459
|
-
console.error(chalk_1.default.blue('Checking queues (in this order):'));
|
|
460
|
-
console.error(filteredEntries.map(function (e) {
|
|
461
|
-
return ' ' + e.qname.slice(opt.prefix.length) + chalk_1.default.blue(' - ' + e.qrl);
|
|
462
|
-
}).join('\n'));
|
|
463
|
-
console.error();
|
|
464
|
-
}
|
|
465
|
-
// Check each queue in parallel
|
|
466
|
-
if (opt.unpair)
|
|
467
|
-
return [2 /*return*/, Promise.all(filteredEntries.map(function (e) { return processQueue(e.qname, e.qrl, opt); }))];
|
|
468
|
-
return [2 /*return*/, Promise.all(filteredEntries.map(function (e) { return processQueuePair(e.qname, e.qrl, opt); }))];
|
|
469
|
-
}
|
|
470
|
-
// Otherwise, let caller know
|
|
471
|
-
return [2 /*return*/, 'noQueues'];
|
|
472
|
-
}
|
|
473
|
-
});
|
|
301
|
+
async function idleQueues(queues, options) {
|
|
302
|
+
const opt = (0, defaults_js_1.getOptionsWithDefaults)(options);
|
|
303
|
+
if (opt.verbose)
|
|
304
|
+
console.error(chalk_1.default.blue('Resolving queues: ') + queues.join(' '));
|
|
305
|
+
const qnames = queues.map(queue => opt.prefix + queue);
|
|
306
|
+
const entries = await (0, qrlCache_js_1.getQnameUrlPairs)(qnames, opt);
|
|
307
|
+
debug('getQnameUrlPairs.then');
|
|
308
|
+
if (opt.verbose) {
|
|
309
|
+
console.error(chalk_1.default.blue(' done'));
|
|
310
|
+
console.error();
|
|
311
|
+
}
|
|
312
|
+
// Filter out any queue ending in suffix unless --include-failed is set
|
|
313
|
+
const filteredEntries = entries.filter(entry => {
|
|
314
|
+
const suf = opt.failSuffix;
|
|
315
|
+
const sufFifo = opt.failSuffix + qrlCache_js_1.fifoSuffix;
|
|
316
|
+
const isFail = entry.qname.endsWith(suf);
|
|
317
|
+
const isFifoFail = entry.qname.endsWith(sufFifo);
|
|
318
|
+
return opt.includeFailed ? true : (!isFail && !isFifoFail);
|
|
474
319
|
});
|
|
320
|
+
// But only if we have queues to remove
|
|
321
|
+
if (filteredEntries.length) {
|
|
322
|
+
if (opt.verbose) {
|
|
323
|
+
console.error(chalk_1.default.blue('Checking queues (in this order):'));
|
|
324
|
+
console.error(filteredEntries.map(e => ' ' + e.qname.slice(opt.prefix.length) + chalk_1.default.blue(' - ' + e.qrl)).join('\n'));
|
|
325
|
+
console.error();
|
|
326
|
+
}
|
|
327
|
+
// Check each queue in parallel
|
|
328
|
+
if (opt.unpair)
|
|
329
|
+
return Promise.all(filteredEntries.map(e => processQueue(e.qname, e.qrl, opt)));
|
|
330
|
+
return Promise.all(filteredEntries.map(e => processQueuePair(e.qname, e.qrl, opt)));
|
|
331
|
+
}
|
|
332
|
+
// Otherwise, let caller know
|
|
333
|
+
return 'noQueues';
|
|
475
334
|
}
|
|
476
335
|
exports.idleQueues = idleQueues;
|
|
477
336
|
debug('loaded');
|