qdone 2.1.0 → 2.1.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +2 -2
- package/README.md +1 -4
- package/commonjs/index.js +11 -0
- package/commonjs/src/cache.js +72 -0
- package/commonjs/src/cloudWatch.js +102 -0
- package/commonjs/src/consumer.js +173 -0
- package/commonjs/src/dedup.js +266 -0
- package/commonjs/src/defaults.js +182 -0
- package/commonjs/src/enqueue.js +521 -0
- package/commonjs/src/exponentialBackoff.js +101 -0
- package/commonjs/src/idleQueues.js +333 -0
- package/commonjs/src/monitor.js +80 -0
- package/commonjs/src/qrlCache.js +172 -0
- package/commonjs/src/scheduler/jobExecutor.js +383 -0
- package/commonjs/src/scheduler/queueManager.js +161 -0
- package/commonjs/src/scheduler/systemMonitor.js +94 -0
- package/commonjs/src/sqs.js +97 -0
- package/package.json +7 -3
|
@@ -0,0 +1,97 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
/**
|
|
3
|
+
* Functions that deal with SQS
|
|
4
|
+
*/
|
|
5
|
+
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
6
|
+
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
7
|
+
};
|
|
8
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
9
|
+
exports.getQueueAttributes = exports.getMatchingQueues = exports.setSQSClient = exports.getSQSClient = void 0;
|
|
10
|
+
const client_sqs_1 = require("@aws-sdk/client-sqs");
|
|
11
|
+
const path_1 = require("path");
|
|
12
|
+
const debug_1 = __importDefault(require("debug"));
|
|
13
|
+
const debug = (0, debug_1.default)('qdone:sqs');
|
|
14
|
+
/**
|
|
15
|
+
* Utility function to return an instantiated, shared SQSClient.
|
|
16
|
+
*/
|
|
17
|
+
let client;
|
|
18
|
+
function getSQSClient() {
|
|
19
|
+
if (client)
|
|
20
|
+
return client;
|
|
21
|
+
client = new client_sqs_1.SQSClient();
|
|
22
|
+
return client;
|
|
23
|
+
}
|
|
24
|
+
exports.getSQSClient = getSQSClient;
|
|
25
|
+
/**
|
|
26
|
+
* Utility function to set the client explicitly, used in testing.
|
|
27
|
+
*/
|
|
28
|
+
function setSQSClient(explicitClient) {
|
|
29
|
+
client = explicitClient;
|
|
30
|
+
}
|
|
31
|
+
exports.setSQSClient = setSQSClient;
|
|
32
|
+
/**
|
|
33
|
+
* Returns qrls for queues matching the given prefix and regex.
|
|
34
|
+
*/
|
|
35
|
+
async function getMatchingQueues(prefix, regex) {
|
|
36
|
+
const input = { QueueNamePrefix: prefix, MaxResults: 1000 };
|
|
37
|
+
const client = getSQSClient();
|
|
38
|
+
async function processQueues(nextToken) {
|
|
39
|
+
if (nextToken)
|
|
40
|
+
input.NextToken = nextToken;
|
|
41
|
+
const command = new client_sqs_1.ListQueuesCommand(input);
|
|
42
|
+
// debug({ nextToken, input, command })
|
|
43
|
+
const result = await client.send(command);
|
|
44
|
+
// debug({ result })
|
|
45
|
+
const { QueueUrls: qrls, NextToken: nextToken2 } = result;
|
|
46
|
+
// debug({ qrls, nextToken2 })
|
|
47
|
+
return (qrls || []).filter(q => regex.test(q)).concat(nextToken2 ? await processQueues(nextToken2) : []);
|
|
48
|
+
}
|
|
49
|
+
return processQueues();
|
|
50
|
+
}
|
|
51
|
+
exports.getMatchingQueues = getMatchingQueues;
|
|
52
|
+
/**
|
|
53
|
+
* Gets attributes on every queue in parallel.
|
|
54
|
+
*/
|
|
55
|
+
async function getQueueAttributes(qrls) {
|
|
56
|
+
const promises = [];
|
|
57
|
+
// debug({ qrls })
|
|
58
|
+
for (const qrl of qrls) {
|
|
59
|
+
const input = {
|
|
60
|
+
QueueUrl: qrl,
|
|
61
|
+
AttributeNames: [
|
|
62
|
+
'ApproximateNumberOfMessages',
|
|
63
|
+
'ApproximateNumberOfMessagesNotVisible',
|
|
64
|
+
'ApproximateNumberOfMessagesDelayed'
|
|
65
|
+
]
|
|
66
|
+
};
|
|
67
|
+
const command = new client_sqs_1.GetQueueAttributesCommand(input);
|
|
68
|
+
// debug({ input, command })
|
|
69
|
+
promises.push((async () => {
|
|
70
|
+
const queue = (0, path_1.basename)(qrl);
|
|
71
|
+
try {
|
|
72
|
+
const result = await client.send(command);
|
|
73
|
+
// debug({ queue, result })
|
|
74
|
+
return { queue, result };
|
|
75
|
+
}
|
|
76
|
+
catch (e) {
|
|
77
|
+
if (e instanceof client_sqs_1.QueueDoesNotExist) {
|
|
78
|
+
// For queues that have been deleted in the meantime for whatever
|
|
79
|
+
// reason, just show as having no messages instead of failing the
|
|
80
|
+
// whole batch
|
|
81
|
+
return {
|
|
82
|
+
queue,
|
|
83
|
+
Attributes: {
|
|
84
|
+
ApproximateNumberOfMessages: '0',
|
|
85
|
+
ApproximateNumberOfMessagesNotVisible: '0',
|
|
86
|
+
ApproximateNumberOfMessagesDelayed: '0'
|
|
87
|
+
}
|
|
88
|
+
};
|
|
89
|
+
}
|
|
90
|
+
throw e;
|
|
91
|
+
}
|
|
92
|
+
})());
|
|
93
|
+
}
|
|
94
|
+
return Promise.all(promises);
|
|
95
|
+
}
|
|
96
|
+
exports.getQueueAttributes = getQueueAttributes;
|
|
97
|
+
debug('loaded');
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "qdone",
|
|
3
|
-
"version": "2.1.
|
|
3
|
+
"version": "2.1.1",
|
|
4
4
|
"description": "A distributed scheduler for SQS",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"main": "./index.js",
|
|
@@ -57,8 +57,12 @@
|
|
|
57
57
|
"publish-latest": "npm publish --tag latest",
|
|
58
58
|
"publish-next": "npm publish --tag next"
|
|
59
59
|
},
|
|
60
|
-
"author": "
|
|
61
|
-
"
|
|
60
|
+
"author": "SureDone, Inc.",
|
|
61
|
+
"contributors": [
|
|
62
|
+
"Ryan Witt <onecreativenerd@gmail.com>",
|
|
63
|
+
"Shane O'Hanlon <shane.c.ohanlon@gmail.com>"
|
|
64
|
+
],
|
|
65
|
+
"license": "ISC",
|
|
62
66
|
"repository": {
|
|
63
67
|
"type": "git",
|
|
64
68
|
"url": "git+https://github.com/suredone/qdone.git"
|