qdone 1.7.0 → 2.0.0-alpha

package/README.md

@@ -425,6 +425,14 @@ Options
     -f, --fifo                Create new queues as FIFOs
     -g, --group-id string     FIFO Group ID to use for all messages enqueued in current command. Defaults to an string unique to this invocation.
     --group-id-per-message    Use a unique Group ID for every message, even messages in the same batch.
+    --deduplication-id string A Message Deduplication ID to give SQS when sending a message. Use this
+                              option if you are managing retries outside of qdone, and make sure the ID is
+                              the same for each retry in the deduplication window. Defaults to a string
+                              unique to this invocation.
+    -d, --delay number        Delays delivery of each message by the given number of seconds (up to 900 seconds,
+                              or 15 minutes). Defaults to immediate delivery (no delay).
+    --dlq-after number        Sends messages from the failed queue to a dead letter queue after this many failures. [default: 3]
+    --dlq-suffix string       Suffix to append to each queue to generate DLQ name [default: _dead]
     --prefix string           Prefix to place at the front of each SQS queue name [default: qdone_]
     --fail-suffix string      Suffix to append to each queue to generate fail queue name [default: _failed]
     --region string           AWS region for Queues [default: us-east-1]
@@ -478,4 +486,4 @@ If a queue name ends with the * (wildcard) character, worker will listen on all
     -q, --quiet             Turn on production logging. Automatically set if stderr is not a tty.
     -v, --verbose           Turn on verbose output. Automatically set if stderr is a tty.
     -V, --version           Show version number
-    --help                  Print full help message.
+    --help                  Print full help message.

package/commonjs/index.js

@@ -0,0 +1,10 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.DoNotProcess = exports.requestShutdown = exports.processMessages = exports.enqueueBatch = exports.enqueue = void 0;
+var enqueue_js_1 = require("./src/enqueue.js");
+Object.defineProperty(exports, "enqueue", { enumerable: true, get: function () { return enqueue_js_1.enqueue; } });
+Object.defineProperty(exports, "enqueueBatch", { enumerable: true, get: function () { return enqueue_js_1.enqueueBatch; } });
+var consumer_js_1 = require("./src/consumer.js");
+Object.defineProperty(exports, "processMessages", { enumerable: true, get: function () { return consumer_js_1.processMessages; } });
+Object.defineProperty(exports, "requestShutdown", { enumerable: true, get: function () { return consumer_js_1.requestShutdown; } });
+Object.defineProperty(exports, "DoNotProcess", { enumerable: true, get: function () { return consumer_js_1.DoNotProcess; } });

package/commonjs/package.json

@@ -0,0 +1,3 @@
+{
+  "type": "commonjs"
+}

package/commonjs/src/cache.js

@@ -0,0 +1,142 @@
+"use strict";
+var __extends = (this && this.__extends) || (function () {
+    var extendStatics = function (d, b) {
+        extendStatics = Object.setPrototypeOf ||
+            ({ __proto__: [] } instanceof Array && function (d, b) { d.__proto__ = b; }) ||
+            function (d, b) { for (var p in b) if (Object.prototype.hasOwnProperty.call(b, p)) d[p] = b[p]; };
+        return extendStatics(d, b);
+    };
+    return function (d, b) {
+        if (typeof b !== "function" && b !== null)
+            throw new TypeError("Class extends value " + String(b) + " is not a constructor or null");
+        extendStatics(d, b);
+        function __() { this.constructor = d; }
+        d.prototype = b === null ? Object.create(b) : (__.prototype = b.prototype, new __());
+    };
+})();
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+var __generator = (this && this.__generator) || function (thisArg, body) {
+    var _ = { label: 0, sent: function() { if (t[0] & 1) throw t[1]; return t[1]; }, trys: [], ops: [] }, f, y, t, g;
+    return g = { next: verb(0), "throw": verb(1), "return": verb(2) }, typeof Symbol === "function" && (g[Symbol.iterator] = function() { return this; }), g;
+    function verb(n) { return function (v) { return step([n, v]); }; }
+    function step(op) {
+        if (f) throw new TypeError("Generator is already executing.");
+        while (g && (g = 0, op[0] && (_ = 0)), _) try {
+            if (f = 1, y && (t = op[0] & 2 ? y["return"] : op[0] ? y["throw"] || ((t = y["return"]) && t.call(y), 0) : y.next) && !(t = t.call(y, op[1])).done) return t;
+            if (y = 0, t) op = [op[0] & 2, t.value];
+            switch (op[0]) {
+                case 0: case 1: t = op; break;
+                case 4: _.label++; return { value: op[1], done: false };
+                case 5: _.label++; y = op[1]; op = [0]; continue;
+                case 7: op = _.ops.pop(); _.trys.pop(); continue;
+                default:
+                    if (!(t = _.trys, t = t.length > 0 && t[t.length - 1]) && (op[0] === 6 || op[0] === 2)) { _ = 0; continue; }
+                    if (op[0] === 3 && (!t || (op[1] > t[0] && op[1] < t[3]))) { _.label = op[1]; break; }
+                    if (op[0] === 6 && _.label < t[1]) { _.label = t[1]; t = op; break; }
+                    if (t && _.label < t[2]) { _.label = t[2]; _.ops.push(op); break; }
+                    if (t[2]) _.ops.pop();
+                    _.trys.pop(); continue;
+            }
+            op = body.call(thisArg, _);
+        } catch (e) { op = [6, e]; y = 0; } finally { f = t = 0; }
+        if (op[0] & 5) throw op[1]; return { value: op[0] ? op[1] : void 0, done: true };
+    }
+};
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.setCache = exports.getCache = exports.shutdownCache = exports.getCacheClient = void 0;
+var ioredis_1 = __importDefault(require("ioredis"));
+var url_1 = require("url");
+var debug_1 = __importDefault(require("debug"));
+var debug = (0, debug_1.default)('qdone:cache');
+var UsageError = /** @class */ (function (_super) {
+    __extends(UsageError, _super);
+    function UsageError() {
+        return _super !== null && _super.apply(this, arguments) || this;
+    }
+    return UsageError;
+}(Error));
+var client;
+/**
+ * Internal function to setup redis client. Parses URI to figure out
+ * how to connect.
+ */
+function getCacheClient(opt) {
+    if (client) {
+        return client;
+    }
+    else if (opt.cacheUri) {
+        var url = new url_1.URL(opt.cacheUri);
+        if (url.protocol === 'redis:') {
+            client = new ioredis_1.default(url.toString());
+        }
+        else if (url.protocol === 'redis-cluster:') {
+            url.protocol = 'redis:';
+            client = new ioredis_1.default.Cluster([url.toString()], { slotsRefreshInterval: 60 * 1000 });
+        }
+        else {
+            throw new UsageError("Only redis:// or redis-cluster:// URLs are currently supported. Got: ".concat(url.protocol));
+        }
+        return client;
+    }
+    else {
+        throw new UsageError('Caching requires the --cache-uri option');
+    }
+}
+exports.getCacheClient = getCacheClient;
+function shutdownCache() {
+    if (client)
+        client.quit();
+    client = undefined;
+}
+exports.shutdownCache = shutdownCache;
+/**
+ * Returns a promise for the item. Resolves to false if cache is empty, object
+ * if it is found.
+ */
+function getCache(key, opt) {
+    return __awaiter(this, void 0, void 0, function () {
+        var client, cacheKey, result;
+        return __generator(this, function (_a) {
+            switch (_a.label) {
+                case 0:
+                    client = getCacheClient(opt);
+                    cacheKey = opt.cachePrefix + key;
+                    debug({ action: 'getCache', cacheKey: cacheKey });
+                    return [4 /*yield*/, client.get(cacheKey)];
+                case 1:
+                    result = _a.sent();
+                    debug({ action: 'getCache got', cacheKey: cacheKey, result: result });
+                    return [2 /*return*/, result ? JSON.parse(result) : undefined];
+            }
+        });
+    });
+}
+exports.getCache = getCache;
+/**
+ * Returns a promise for the status. Encodes object as JSON
+ */
+function setCache(key, value, opt) {
+    return __awaiter(this, void 0, void 0, function () {
+        var client, encoded, cacheKey;
+        return __generator(this, function (_a) {
+            client = getCacheClient(opt);
+            encoded = JSON.stringify(value);
+            cacheKey = opt.cachePrefix + key;
+            debug({ action: 'setCache', cacheKey: cacheKey, value: value });
+            return [2 /*return*/, client.setex(cacheKey, opt.cacheTtlSeconds, encoded)];
+        });
+    });
+}
+exports.setCache = setCache;
+debug('loaded');

package/commonjs/src/cloudWatch.js

@@ -0,0 +1,148 @@
+"use strict";
+/**
+ * Functions that deal with CloudWatch
+ */
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+var __generator = (this && this.__generator) || function (thisArg, body) {
+    var _ = { label: 0, sent: function() { if (t[0] & 1) throw t[1]; return t[1]; }, trys: [], ops: [] }, f, y, t, g;
+    return g = { next: verb(0), "throw": verb(1), "return": verb(2) }, typeof Symbol === "function" && (g[Symbol.iterator] = function() { return this; }), g;
+    function verb(n) { return function (v) { return step([n, v]); }; }
+    function step(op) {
+        if (f) throw new TypeError("Generator is already executing.");
+        while (g && (g = 0, op[0] && (_ = 0)), _) try {
+            if (f = 1, y && (t = op[0] & 2 ? y["return"] : op[0] ? y["throw"] || ((t = y["return"]) && t.call(y), 0) : y.next) && !(t = t.call(y, op[1])).done) return t;
+            if (y = 0, t) op = [op[0] & 2, t.value];
+            switch (op[0]) {
+                case 0: case 1: t = op; break;
+                case 4: _.label++; return { value: op[1], done: false };
+                case 5: _.label++; y = op[1]; op = [0]; continue;
+                case 7: op = _.ops.pop(); _.trys.pop(); continue;
+                default:
+                    if (!(t = _.trys, t = t.length > 0 && t[t.length - 1]) && (op[0] === 6 || op[0] === 2)) { _ = 0; continue; }
+                    if (op[0] === 3 && (!t || (op[1] > t[0] && op[1] < t[3]))) { _.label = op[1]; break; }
+                    if (op[0] === 6 && _.label < t[1]) { _.label = t[1]; t = op; break; }
+                    if (t && _.label < t[2]) { _.label = t[2]; _.ops.push(op); break; }
+                    if (t[2]) _.ops.pop();
+                    _.trys.pop(); continue;
+            }
+            op = body.call(thisArg, _);
+        } catch (e) { op = [6, e]; y = 0; } finally { f = t = 0; }
+        if (op[0] & 5) throw op[1]; return { value: op[0] ? op[1] : void 0, done: true };
+    }
+};
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.putAggregateData = exports.setCloudWatchClient = exports.getCloudWatchClient = void 0;
+var client_cloudwatch_1 = require("@aws-sdk/client-cloudwatch");
+var debug_1 = __importDefault(require("debug"));
+var debug = (0, debug_1.default)('qdone:cloudWatch');
+/**
+ * Utility function to return an instantiated, shared CloudWatchClient.
+ */
+var client;
+function getCloudWatchClient() {
+    if (client)
+        return client;
+    client = new client_cloudwatch_1.CloudWatchClient();
+    return client;
+}
+exports.getCloudWatchClient = getCloudWatchClient;
+/**
+ * Utility function to set the client explicitly, used in testing.
+ */
+function setCloudWatchClient(explicitClient) {
+    client = explicitClient;
+}
+exports.setCloudWatchClient = setCloudWatchClient;
+/**
+ * Takes data in the form returned by getAggregageData() and pushes it to
+ * CloudWatch metrics under the given queueName.
+ *
+ * @param queueName {String} - The name of the wildcard queue these metrics are for.
+ * @param total {Object} - returned object from getAggregateData()
+ */
+function putAggregateData(total, timestamp) {
+    return __awaiter(this, void 0, void 0, function () {
+        var client, now, input, command, response;
+        return __generator(this, function (_a) {
+            switch (_a.label) {
+                case 0:
+                    client = getCloudWatchClient();
+                    now = timestamp || new Date();
+                    input = {
+                        Namespace: 'qmonitor',
+                        MetricData: [
+                            {
+                                MetricName: 'totalQueues',
+                                Dimensions: [{
+                                        Name: 'queueName',
+                                        Value: total.queueName
+                                    }],
+                                Timestamp: now,
+                                Value: total.totalQueues,
+                                Unit: 'Count'
+                            },
+                            {
+                                MetricName: 'contributingQueueCount',
+                                Dimensions: [{
+                                        Name: 'queueName',
+                                        Value: total.queueName
+                                    }],
+                                Timestamp: now,
+                                Value: total.contributingQueueNames.length,
+                                Unit: 'Count'
+                            },
+                            {
+                                MetricName: 'ApproximateNumberOfMessages',
+                                Dimensions: [{
+                                        Name: 'queueName',
+                                        Value: total.queueName
+                                    }],
+                                Timestamp: now,
+                                Value: total.ApproximateNumberOfMessages || 0,
+                                Unit: 'Count'
+                            },
+                            {
+                                MetricName: 'ApproximateNumberOfMessagesDelayed',
+                                Dimensions: [{
+                                        Name: 'queueName',
+                                        Value: total.queueName
+                                    }],
+                                Timestamp: now,
+                                Value: total.ApproximateNumberOfMessagesDelayed || 0,
+                                Unit: 'Count'
+                            },
+                            {
+                                MetricName: 'ApproximateNumberOfMessagesNotVisible',
+                                Dimensions: [{
+                                        Name: 'queueName',
+                                        Value: total.queueName
+                                    }],
+                                Timestamp: now,
+                                Value: total.ApproximateNumberOfMessagesNotVisible || 0,
+                                Unit: 'Count'
+                            }
+                        ]
+                    };
+                    command = new client_cloudwatch_1.PutMetricDataCommand(input);
+                    return [4 /*yield*/, client.send(command)];
+                case 1:
+                    response = _a.sent();
+                    debug({ response: response });
+                    return [2 /*return*/];
+            }
+        });
+    });
+}
+exports.putAggregateData = putAggregateData;
+debug('loaded');