qdone 2.0.28-alpha → 2.0.30-alpha
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/commonjs/src/cache.js +22 -93
- package/commonjs/src/cloudWatch.js +65 -111
- package/commonjs/src/consumer.js +135 -237
- package/commonjs/src/defaults.js +11 -11
- package/commonjs/src/enqueue.js +311 -503
- package/commonjs/src/exponentialBackoff.js +39 -110
- package/commonjs/src/idleQueues.js +255 -396
- package/commonjs/src/monitor.js +42 -115
- package/commonjs/src/qrlCache.js +80 -162
- package/commonjs/src/scheduler/jobExecutor.js +305 -363
- package/commonjs/src/scheduler/queueManager.js +116 -190
- package/commonjs/src/scheduler/systemMonitor.js +24 -28
- package/commonjs/src/sqs.js +58 -141
- package/package.json +2 -2
- package/src/consumer.js +3 -5
- package/src/defaults.js +3 -3
- package/src/enqueue.js +1 -1
- package/src/scheduler/jobExecutor.js +77 -33
- package/src/scheduler/queueManager.js +13 -4
- package/npm-shrinkwrap.json +0 -15999
package/commonjs/src/cache.js
CHANGED
|
@@ -1,72 +1,16 @@
|
|
|
1
1
|
"use strict";
|
|
2
|
-
var __extends = (this && this.__extends) || (function () {
|
|
3
|
-
var extendStatics = function (d, b) {
|
|
4
|
-
extendStatics = Object.setPrototypeOf ||
|
|
5
|
-
({ __proto__: [] } instanceof Array && function (d, b) { d.__proto__ = b; }) ||
|
|
6
|
-
function (d, b) { for (var p in b) if (Object.prototype.hasOwnProperty.call(b, p)) d[p] = b[p]; };
|
|
7
|
-
return extendStatics(d, b);
|
|
8
|
-
};
|
|
9
|
-
return function (d, b) {
|
|
10
|
-
if (typeof b !== "function" && b !== null)
|
|
11
|
-
throw new TypeError("Class extends value " + String(b) + " is not a constructor or null");
|
|
12
|
-
extendStatics(d, b);
|
|
13
|
-
function __() { this.constructor = d; }
|
|
14
|
-
d.prototype = b === null ? Object.create(b) : (__.prototype = b.prototype, new __());
|
|
15
|
-
};
|
|
16
|
-
})();
|
|
17
|
-
var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
|
|
18
|
-
function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
|
|
19
|
-
return new (P || (P = Promise))(function (resolve, reject) {
|
|
20
|
-
function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
|
|
21
|
-
function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
|
|
22
|
-
function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
|
|
23
|
-
step((generator = generator.apply(thisArg, _arguments || [])).next());
|
|
24
|
-
});
|
|
25
|
-
};
|
|
26
|
-
var __generator = (this && this.__generator) || function (thisArg, body) {
|
|
27
|
-
var _ = { label: 0, sent: function() { if (t[0] & 1) throw t[1]; return t[1]; }, trys: [], ops: [] }, f, y, t, g;
|
|
28
|
-
return g = { next: verb(0), "throw": verb(1), "return": verb(2) }, typeof Symbol === "function" && (g[Symbol.iterator] = function() { return this; }), g;
|
|
29
|
-
function verb(n) { return function (v) { return step([n, v]); }; }
|
|
30
|
-
function step(op) {
|
|
31
|
-
if (f) throw new TypeError("Generator is already executing.");
|
|
32
|
-
while (g && (g = 0, op[0] && (_ = 0)), _) try {
|
|
33
|
-
if (f = 1, y && (t = op[0] & 2 ? y["return"] : op[0] ? y["throw"] || ((t = y["return"]) && t.call(y), 0) : y.next) && !(t = t.call(y, op[1])).done) return t;
|
|
34
|
-
if (y = 0, t) op = [op[0] & 2, t.value];
|
|
35
|
-
switch (op[0]) {
|
|
36
|
-
case 0: case 1: t = op; break;
|
|
37
|
-
case 4: _.label++; return { value: op[1], done: false };
|
|
38
|
-
case 5: _.label++; y = op[1]; op = [0]; continue;
|
|
39
|
-
case 7: op = _.ops.pop(); _.trys.pop(); continue;
|
|
40
|
-
default:
|
|
41
|
-
if (!(t = _.trys, t = t.length > 0 && t[t.length - 1]) && (op[0] === 6 || op[0] === 2)) { _ = 0; continue; }
|
|
42
|
-
if (op[0] === 3 && (!t || (op[1] > t[0] && op[1] < t[3]))) { _.label = op[1]; break; }
|
|
43
|
-
if (op[0] === 6 && _.label < t[1]) { _.label = t[1]; t = op; break; }
|
|
44
|
-
if (t && _.label < t[2]) { _.label = t[2]; _.ops.push(op); break; }
|
|
45
|
-
if (t[2]) _.ops.pop();
|
|
46
|
-
_.trys.pop(); continue;
|
|
47
|
-
}
|
|
48
|
-
op = body.call(thisArg, _);
|
|
49
|
-
} catch (e) { op = [6, e]; y = 0; } finally { f = t = 0; }
|
|
50
|
-
if (op[0] & 5) throw op[1]; return { value: op[0] ? op[1] : void 0, done: true };
|
|
51
|
-
}
|
|
52
|
-
};
|
|
53
2
|
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
54
3
|
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
55
4
|
};
|
|
56
5
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
57
6
|
exports.setCache = exports.getCache = exports.shutdownCache = exports.getCacheClient = void 0;
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
return _super !== null && _super.apply(this, arguments) || this;
|
|
66
|
-
}
|
|
67
|
-
return UsageError;
|
|
68
|
-
}(Error));
|
|
69
|
-
var client;
|
|
7
|
+
const ioredis_1 = __importDefault(require("ioredis"));
|
|
8
|
+
const url_1 = require("url");
|
|
9
|
+
const debug_1 = __importDefault(require("debug"));
|
|
10
|
+
const debug = (0, debug_1.default)('qdone:cache');
|
|
11
|
+
class UsageError extends Error {
|
|
12
|
+
}
|
|
13
|
+
let client;
|
|
70
14
|
/**
|
|
71
15
|
* Internal function to setup redis client. Parses URI to figure out
|
|
72
16
|
* how to connect.
|
|
@@ -76,7 +20,7 @@ function getCacheClient(opt) {
|
|
|
76
20
|
return client;
|
|
77
21
|
}
|
|
78
22
|
else if (opt.cacheUri) {
|
|
79
|
-
|
|
23
|
+
const url = new url_1.URL(opt.cacheUri);
|
|
80
24
|
if (url.protocol === 'redis:') {
|
|
81
25
|
client = new ioredis_1.default(url.toString());
|
|
82
26
|
}
|
|
@@ -85,7 +29,7 @@ function getCacheClient(opt) {
|
|
|
85
29
|
client = new ioredis_1.default.Cluster([url.toString()], { slotsRefreshInterval: 60 * 1000 });
|
|
86
30
|
}
|
|
87
31
|
else {
|
|
88
|
-
throw new UsageError(
|
|
32
|
+
throw new UsageError(`Only redis:// or redis-cluster:// URLs are currently supported. Got: ${url.protocol}`);
|
|
89
33
|
}
|
|
90
34
|
return client;
|
|
91
35
|
}
|
|
@@ -104,39 +48,24 @@ exports.shutdownCache = shutdownCache;
|
|
|
104
48
|
* Returns a promise for the item. Resolves to false if cache is empty, object
|
|
105
49
|
* if it is found.
|
|
106
50
|
*/
|
|
107
|
-
function getCache(key, opt) {
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
cacheKey = opt.cachePrefix + key;
|
|
115
|
-
debug({ action: 'getCache', cacheKey: cacheKey });
|
|
116
|
-
return [4 /*yield*/, client.get(cacheKey)];
|
|
117
|
-
case 1:
|
|
118
|
-
result = _a.sent();
|
|
119
|
-
debug({ action: 'getCache got', cacheKey: cacheKey, result: result });
|
|
120
|
-
return [2 /*return*/, result ? JSON.parse(result) : undefined];
|
|
121
|
-
}
|
|
122
|
-
});
|
|
123
|
-
});
|
|
51
|
+
async function getCache(key, opt) {
|
|
52
|
+
const client = getCacheClient(opt);
|
|
53
|
+
const cacheKey = opt.cachePrefix + key;
|
|
54
|
+
debug({ action: 'getCache', cacheKey });
|
|
55
|
+
const result = await client.get(cacheKey);
|
|
56
|
+
debug({ action: 'getCache got', cacheKey, result });
|
|
57
|
+
return result ? JSON.parse(result) : undefined;
|
|
124
58
|
}
|
|
125
59
|
exports.getCache = getCache;
|
|
126
60
|
/**
|
|
127
61
|
* Returns a promise for the status. Encodes object as JSON
|
|
128
62
|
*/
|
|
129
|
-
function setCache(key, value, opt) {
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
cacheKey = opt.cachePrefix + key;
|
|
136
|
-
debug({ action: 'setCache', cacheKey: cacheKey, value: value });
|
|
137
|
-
return [2 /*return*/, client.setex(cacheKey, opt.cacheTtlSeconds, encoded)];
|
|
138
|
-
});
|
|
139
|
-
});
|
|
63
|
+
async function setCache(key, value, opt) {
|
|
64
|
+
const client = getCacheClient(opt);
|
|
65
|
+
const encoded = JSON.stringify(value);
|
|
66
|
+
const cacheKey = opt.cachePrefix + key;
|
|
67
|
+
debug({ action: 'setCache', cacheKey, value });
|
|
68
|
+
return client.setex(cacheKey, opt.cacheTtlSeconds, encoded);
|
|
140
69
|
}
|
|
141
70
|
exports.setCache = setCache;
|
|
142
71
|
debug('loaded');
|
|
@@ -2,54 +2,18 @@
|
|
|
2
2
|
/**
|
|
3
3
|
* Functions that deal with CloudWatch
|
|
4
4
|
*/
|
|
5
|
-
var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
|
|
6
|
-
function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
|
|
7
|
-
return new (P || (P = Promise))(function (resolve, reject) {
|
|
8
|
-
function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
|
|
9
|
-
function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
|
|
10
|
-
function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
|
|
11
|
-
step((generator = generator.apply(thisArg, _arguments || [])).next());
|
|
12
|
-
});
|
|
13
|
-
};
|
|
14
|
-
var __generator = (this && this.__generator) || function (thisArg, body) {
|
|
15
|
-
var _ = { label: 0, sent: function() { if (t[0] & 1) throw t[1]; return t[1]; }, trys: [], ops: [] }, f, y, t, g;
|
|
16
|
-
return g = { next: verb(0), "throw": verb(1), "return": verb(2) }, typeof Symbol === "function" && (g[Symbol.iterator] = function() { return this; }), g;
|
|
17
|
-
function verb(n) { return function (v) { return step([n, v]); }; }
|
|
18
|
-
function step(op) {
|
|
19
|
-
if (f) throw new TypeError("Generator is already executing.");
|
|
20
|
-
while (g && (g = 0, op[0] && (_ = 0)), _) try {
|
|
21
|
-
if (f = 1, y && (t = op[0] & 2 ? y["return"] : op[0] ? y["throw"] || ((t = y["return"]) && t.call(y), 0) : y.next) && !(t = t.call(y, op[1])).done) return t;
|
|
22
|
-
if (y = 0, t) op = [op[0] & 2, t.value];
|
|
23
|
-
switch (op[0]) {
|
|
24
|
-
case 0: case 1: t = op; break;
|
|
25
|
-
case 4: _.label++; return { value: op[1], done: false };
|
|
26
|
-
case 5: _.label++; y = op[1]; op = [0]; continue;
|
|
27
|
-
case 7: op = _.ops.pop(); _.trys.pop(); continue;
|
|
28
|
-
default:
|
|
29
|
-
if (!(t = _.trys, t = t.length > 0 && t[t.length - 1]) && (op[0] === 6 || op[0] === 2)) { _ = 0; continue; }
|
|
30
|
-
if (op[0] === 3 && (!t || (op[1] > t[0] && op[1] < t[3]))) { _.label = op[1]; break; }
|
|
31
|
-
if (op[0] === 6 && _.label < t[1]) { _.label = t[1]; t = op; break; }
|
|
32
|
-
if (t && _.label < t[2]) { _.label = t[2]; _.ops.push(op); break; }
|
|
33
|
-
if (t[2]) _.ops.pop();
|
|
34
|
-
_.trys.pop(); continue;
|
|
35
|
-
}
|
|
36
|
-
op = body.call(thisArg, _);
|
|
37
|
-
} catch (e) { op = [6, e]; y = 0; } finally { f = t = 0; }
|
|
38
|
-
if (op[0] & 5) throw op[1]; return { value: op[0] ? op[1] : void 0, done: true };
|
|
39
|
-
}
|
|
40
|
-
};
|
|
41
5
|
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
42
6
|
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
43
7
|
};
|
|
44
8
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
45
9
|
exports.putAggregateData = exports.setCloudWatchClient = exports.getCloudWatchClient = void 0;
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
10
|
+
const client_cloudwatch_1 = require("@aws-sdk/client-cloudwatch");
|
|
11
|
+
const debug_1 = __importDefault(require("debug"));
|
|
12
|
+
const debug = (0, debug_1.default)('qdone:cloudWatch');
|
|
49
13
|
/**
|
|
50
14
|
* Utility function to return an instantiated, shared CloudWatchClient.
|
|
51
15
|
*/
|
|
52
|
-
|
|
16
|
+
let client;
|
|
53
17
|
function getCloudWatchClient() {
|
|
54
18
|
if (client)
|
|
55
19
|
return client;
|
|
@@ -71,78 +35,68 @@ exports.setCloudWatchClient = setCloudWatchClient;
|
|
|
71
35
|
* @param queueName {String} - The name of the wildcard queue these metrics are for.
|
|
72
36
|
* @param total {Object} - returned object from getAggregateData()
|
|
73
37
|
*/
|
|
74
|
-
function putAggregateData(total, timestamp) {
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
Value: total.queueName
|
|
130
|
-
}],
|
|
131
|
-
Timestamp: now,
|
|
132
|
-
Value: total.ApproximateNumberOfMessagesNotVisible || 0,
|
|
133
|
-
Unit: 'Count'
|
|
134
|
-
}
|
|
135
|
-
]
|
|
136
|
-
};
|
|
137
|
-
command = new client_cloudwatch_1.PutMetricDataCommand(input);
|
|
138
|
-
return [4 /*yield*/, client.send(command)];
|
|
139
|
-
case 1:
|
|
140
|
-
response = _a.sent();
|
|
141
|
-
debug({ response: response });
|
|
142
|
-
return [2 /*return*/];
|
|
38
|
+
async function putAggregateData(total, timestamp) {
|
|
39
|
+
const client = getCloudWatchClient();
|
|
40
|
+
const now = timestamp || new Date();
|
|
41
|
+
const input = {
|
|
42
|
+
Namespace: 'qmonitor',
|
|
43
|
+
MetricData: [
|
|
44
|
+
{
|
|
45
|
+
MetricName: 'totalQueues',
|
|
46
|
+
Dimensions: [{
|
|
47
|
+
Name: 'queueName',
|
|
48
|
+
Value: total.queueName
|
|
49
|
+
}],
|
|
50
|
+
Timestamp: now,
|
|
51
|
+
Value: total.totalQueues,
|
|
52
|
+
Unit: 'Count'
|
|
53
|
+
},
|
|
54
|
+
{
|
|
55
|
+
MetricName: 'contributingQueueCount',
|
|
56
|
+
Dimensions: [{
|
|
57
|
+
Name: 'queueName',
|
|
58
|
+
Value: total.queueName
|
|
59
|
+
}],
|
|
60
|
+
Timestamp: now,
|
|
61
|
+
Value: total.contributingQueueNames.length,
|
|
62
|
+
Unit: 'Count'
|
|
63
|
+
},
|
|
64
|
+
{
|
|
65
|
+
MetricName: 'ApproximateNumberOfMessages',
|
|
66
|
+
Dimensions: [{
|
|
67
|
+
Name: 'queueName',
|
|
68
|
+
Value: total.queueName
|
|
69
|
+
}],
|
|
70
|
+
Timestamp: now,
|
|
71
|
+
Value: total.ApproximateNumberOfMessages || 0,
|
|
72
|
+
Unit: 'Count'
|
|
73
|
+
},
|
|
74
|
+
{
|
|
75
|
+
MetricName: 'ApproximateNumberOfMessagesDelayed',
|
|
76
|
+
Dimensions: [{
|
|
77
|
+
Name: 'queueName',
|
|
78
|
+
Value: total.queueName
|
|
79
|
+
}],
|
|
80
|
+
Timestamp: now,
|
|
81
|
+
Value: total.ApproximateNumberOfMessagesDelayed || 0,
|
|
82
|
+
Unit: 'Count'
|
|
83
|
+
},
|
|
84
|
+
{
|
|
85
|
+
MetricName: 'ApproximateNumberOfMessagesNotVisible',
|
|
86
|
+
Dimensions: [{
|
|
87
|
+
Name: 'queueName',
|
|
88
|
+
Value: total.queueName
|
|
89
|
+
}],
|
|
90
|
+
Timestamp: now,
|
|
91
|
+
Value: total.ApproximateNumberOfMessagesNotVisible || 0,
|
|
92
|
+
Unit: 'Count'
|
|
143
93
|
}
|
|
144
|
-
|
|
145
|
-
}
|
|
94
|
+
]
|
|
95
|
+
};
|
|
96
|
+
const command = new client_cloudwatch_1.PutMetricDataCommand(input);
|
|
97
|
+
// debug({ input, command })
|
|
98
|
+
const response = await client.send(command);
|
|
99
|
+
debug({ response });
|
|
146
100
|
}
|
|
147
101
|
exports.putAggregateData = putAggregateData;
|
|
148
102
|
debug('loaded');
|