qdone 2.0.29-alpha → 2.0.31-alpha
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/commonjs/src/cache.js +22 -93
- package/commonjs/src/cloudWatch.js +65 -111
- package/commonjs/src/consumer.js +136 -237
- package/commonjs/src/defaults.js +11 -11
- package/commonjs/src/enqueue.js +311 -503
- package/commonjs/src/exponentialBackoff.js +39 -110
- package/commonjs/src/idleQueues.js +255 -396
- package/commonjs/src/monitor.js +42 -115
- package/commonjs/src/qrlCache.js +80 -162
- package/commonjs/src/scheduler/jobExecutor.js +324 -363
- package/commonjs/src/scheduler/queueManager.js +111 -198
- package/commonjs/src/scheduler/systemMonitor.js +24 -28
- package/commonjs/src/sqs.js +58 -141
- package/npm-shrinkwrap.json +15999 -0
- package/package.json +2 -2
- package/src/consumer.js +6 -7
- package/src/defaults.js +3 -3
- package/src/enqueue.js +1 -1
- package/src/scheduler/jobExecutor.js +97 -33
- package/src/scheduler/queueManager.js +1 -1
package/commonjs/src/sqs.js
CHANGED
|
@@ -2,55 +2,19 @@
|
|
|
2
2
|
/**
|
|
3
3
|
* Functions that deal with SQS
|
|
4
4
|
*/
|
|
5
|
-
var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
|
|
6
|
-
function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
|
|
7
|
-
return new (P || (P = Promise))(function (resolve, reject) {
|
|
8
|
-
function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
|
|
9
|
-
function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
|
|
10
|
-
function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
|
|
11
|
-
step((generator = generator.apply(thisArg, _arguments || [])).next());
|
|
12
|
-
});
|
|
13
|
-
};
|
|
14
|
-
var __generator = (this && this.__generator) || function (thisArg, body) {
|
|
15
|
-
var _ = { label: 0, sent: function() { if (t[0] & 1) throw t[1]; return t[1]; }, trys: [], ops: [] }, f, y, t, g;
|
|
16
|
-
return g = { next: verb(0), "throw": verb(1), "return": verb(2) }, typeof Symbol === "function" && (g[Symbol.iterator] = function() { return this; }), g;
|
|
17
|
-
function verb(n) { return function (v) { return step([n, v]); }; }
|
|
18
|
-
function step(op) {
|
|
19
|
-
if (f) throw new TypeError("Generator is already executing.");
|
|
20
|
-
while (g && (g = 0, op[0] && (_ = 0)), _) try {
|
|
21
|
-
if (f = 1, y && (t = op[0] & 2 ? y["return"] : op[0] ? y["throw"] || ((t = y["return"]) && t.call(y), 0) : y.next) && !(t = t.call(y, op[1])).done) return t;
|
|
22
|
-
if (y = 0, t) op = [op[0] & 2, t.value];
|
|
23
|
-
switch (op[0]) {
|
|
24
|
-
case 0: case 1: t = op; break;
|
|
25
|
-
case 4: _.label++; return { value: op[1], done: false };
|
|
26
|
-
case 5: _.label++; y = op[1]; op = [0]; continue;
|
|
27
|
-
case 7: op = _.ops.pop(); _.trys.pop(); continue;
|
|
28
|
-
default:
|
|
29
|
-
if (!(t = _.trys, t = t.length > 0 && t[t.length - 1]) && (op[0] === 6 || op[0] === 2)) { _ = 0; continue; }
|
|
30
|
-
if (op[0] === 3 && (!t || (op[1] > t[0] && op[1] < t[3]))) { _.label = op[1]; break; }
|
|
31
|
-
if (op[0] === 6 && _.label < t[1]) { _.label = t[1]; t = op; break; }
|
|
32
|
-
if (t && _.label < t[2]) { _.label = t[2]; _.ops.push(op); break; }
|
|
33
|
-
if (t[2]) _.ops.pop();
|
|
34
|
-
_.trys.pop(); continue;
|
|
35
|
-
}
|
|
36
|
-
op = body.call(thisArg, _);
|
|
37
|
-
} catch (e) { op = [6, e]; y = 0; } finally { f = t = 0; }
|
|
38
|
-
if (op[0] & 5) throw op[1]; return { value: op[0] ? op[1] : void 0, done: true };
|
|
39
|
-
}
|
|
40
|
-
};
|
|
41
5
|
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
42
6
|
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
43
7
|
};
|
|
44
8
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
45
9
|
exports.getQueueAttributes = exports.getMatchingQueues = exports.setSQSClient = exports.getSQSClient = void 0;
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
10
|
+
const client_sqs_1 = require("@aws-sdk/client-sqs");
|
|
11
|
+
const path_1 = require("path");
|
|
12
|
+
const debug_1 = __importDefault(require("debug"));
|
|
13
|
+
const debug = (0, debug_1.default)('qdone:sqs');
|
|
50
14
|
/**
|
|
51
15
|
* Utility function to return an instantiated, shared SQSClient.
|
|
52
16
|
*/
|
|
53
|
-
|
|
17
|
+
let client;
|
|
54
18
|
function getSQSClient() {
|
|
55
19
|
if (client)
|
|
56
20
|
return client;
|
|
@@ -68,113 +32,66 @@ exports.setSQSClient = setSQSClient;
|
|
|
68
32
|
/**
|
|
69
33
|
* Returns qrls for queues matching the given prefix and regex.
|
|
70
34
|
*/
|
|
71
|
-
function getMatchingQueues(prefix, regex) {
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
result = _d.sent();
|
|
87
|
-
qrls = result.QueueUrls, nextToken2 = result.NextToken;
|
|
88
|
-
_b = (_a = (qrls || []).filter(function (q) { return regex.test(q); })).concat;
|
|
89
|
-
if (!nextToken2) return [3 /*break*/, 3];
|
|
90
|
-
return [4 /*yield*/, processQueues(nextToken2)];
|
|
91
|
-
case 2:
|
|
92
|
-
_c = _d.sent();
|
|
93
|
-
return [3 /*break*/, 4];
|
|
94
|
-
case 3:
|
|
95
|
-
_c = [];
|
|
96
|
-
_d.label = 4;
|
|
97
|
-
case 4:
|
|
98
|
-
// debug({ qrls, nextToken2 })
|
|
99
|
-
return [2 /*return*/, _b.apply(_a, [_c])];
|
|
100
|
-
}
|
|
101
|
-
});
|
|
102
|
-
});
|
|
103
|
-
}
|
|
104
|
-
var input, client;
|
|
105
|
-
return __generator(this, function (_a) {
|
|
106
|
-
input = { QueueNamePrefix: prefix, MaxResults: 1000 };
|
|
107
|
-
client = getSQSClient();
|
|
108
|
-
return [2 /*return*/, processQueues()];
|
|
109
|
-
});
|
|
110
|
-
});
|
|
35
|
+
async function getMatchingQueues(prefix, regex) {
|
|
36
|
+
const input = { QueueNamePrefix: prefix, MaxResults: 1000 };
|
|
37
|
+
const client = getSQSClient();
|
|
38
|
+
async function processQueues(nextToken) {
|
|
39
|
+
if (nextToken)
|
|
40
|
+
input.NextToken = nextToken;
|
|
41
|
+
const command = new client_sqs_1.ListQueuesCommand(input);
|
|
42
|
+
// debug({ nextToken, input, command })
|
|
43
|
+
const result = await client.send(command);
|
|
44
|
+
// debug({ result })
|
|
45
|
+
const { QueueUrls: qrls, NextToken: nextToken2 } = result;
|
|
46
|
+
// debug({ qrls, nextToken2 })
|
|
47
|
+
return (qrls || []).filter(q => regex.test(q)).concat(nextToken2 ? await processQueues(nextToken2) : []);
|
|
48
|
+
}
|
|
49
|
+
return processQueues();
|
|
111
50
|
}
|
|
112
51
|
exports.getMatchingQueues = getMatchingQueues;
|
|
113
52
|
/**
|
|
114
53
|
* Gets attributes on every queue in parallel.
|
|
115
54
|
*/
|
|
116
|
-
function getQueueAttributes(qrls) {
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
return [2 /*return*/, { queue: queue, result: result }];
|
|
149
|
-
case 3:
|
|
150
|
-
e_1 = _a.sent();
|
|
151
|
-
if (e_1 instanceof client_sqs_1.QueueDoesNotExist) {
|
|
152
|
-
// For queues that have been deleted in the meantime for whatever
|
|
153
|
-
// reason, just show as having no messages instead of failing the
|
|
154
|
-
// whole batch
|
|
155
|
-
return [2 /*return*/, {
|
|
156
|
-
queue: queue,
|
|
157
|
-
Attributes: {
|
|
158
|
-
ApproximateNumberOfMessages: '0',
|
|
159
|
-
ApproximateNumberOfMessagesNotVisible: '0',
|
|
160
|
-
ApproximateNumberOfMessagesDelayed: '0'
|
|
161
|
-
}
|
|
162
|
-
}];
|
|
163
|
-
}
|
|
164
|
-
throw e_1;
|
|
165
|
-
case 4: return [2 /*return*/];
|
|
55
|
+
async function getQueueAttributes(qrls) {
|
|
56
|
+
const promises = [];
|
|
57
|
+
// debug({ qrls })
|
|
58
|
+
for (const qrl of qrls) {
|
|
59
|
+
const input = {
|
|
60
|
+
QueueUrl: qrl,
|
|
61
|
+
AttributeNames: [
|
|
62
|
+
'ApproximateNumberOfMessages',
|
|
63
|
+
'ApproximateNumberOfMessagesNotVisible',
|
|
64
|
+
'ApproximateNumberOfMessagesDelayed'
|
|
65
|
+
]
|
|
66
|
+
};
|
|
67
|
+
const command = new client_sqs_1.GetQueueAttributesCommand(input);
|
|
68
|
+
// debug({ input, command })
|
|
69
|
+
promises.push((async () => {
|
|
70
|
+
const queue = (0, path_1.basename)(qrl);
|
|
71
|
+
try {
|
|
72
|
+
const result = await client.send(command);
|
|
73
|
+
// debug({ queue, result })
|
|
74
|
+
return { queue, result };
|
|
75
|
+
}
|
|
76
|
+
catch (e) {
|
|
77
|
+
if (e instanceof client_sqs_1.QueueDoesNotExist) {
|
|
78
|
+
// For queues that have been deleted in the meantime for whatever
|
|
79
|
+
// reason, just show as having no messages instead of failing the
|
|
80
|
+
// whole batch
|
|
81
|
+
return {
|
|
82
|
+
queue,
|
|
83
|
+
Attributes: {
|
|
84
|
+
ApproximateNumberOfMessages: '0',
|
|
85
|
+
ApproximateNumberOfMessagesNotVisible: '0',
|
|
86
|
+
ApproximateNumberOfMessagesDelayed: '0'
|
|
166
87
|
}
|
|
167
|
-
}
|
|
168
|
-
}
|
|
169
|
-
|
|
170
|
-
// debug({ qrls })
|
|
171
|
-
for (_i = 0, qrls_1 = qrls; _i < qrls_1.length; _i++) {
|
|
172
|
-
qrl = qrls_1[_i];
|
|
173
|
-
_loop_1(qrl);
|
|
88
|
+
};
|
|
89
|
+
}
|
|
90
|
+
throw e;
|
|
174
91
|
}
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
92
|
+
})());
|
|
93
|
+
}
|
|
94
|
+
return Promise.all(promises);
|
|
178
95
|
}
|
|
179
96
|
exports.getQueueAttributes = getQueueAttributes;
|
|
180
97
|
debug('loaded');
|