qdone 2.0.29-alpha → 2.0.31-alpha

package/commonjs/src/consumer.js

@@ -2,256 +2,155 @@
 /**
  * Consumer implementation.
  */
-var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
-    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
-    return new (P || (P = Promise))(function (resolve, reject) {
-        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
-        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
-        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
-        step((generator = generator.apply(thisArg, _arguments || [])).next());
-    });
-};
-var __generator = (this && this.__generator) || function (thisArg, body) {
-    var _ = { label: 0, sent: function() { if (t[0] & 1) throw t[1]; return t[1]; }, trys: [], ops: [] }, f, y, t, g;
-    return g = { next: verb(0), "throw": verb(1), "return": verb(2) }, typeof Symbol === "function" && (g[Symbol.iterator] = function() { return this; }), g;
-    function verb(n) { return function (v) { return step([n, v]); }; }
-    function step(op) {
-        if (f) throw new TypeError("Generator is already executing.");
-        while (g && (g = 0, op[0] && (_ = 0)), _) try {
-            if (f = 1, y && (t = op[0] & 2 ? y["return"] : op[0] ? y["throw"] || ((t = y["return"]) && t.call(y), 0) : y.next) && !(t = t.call(y, op[1])).done) return t;
-            if (y = 0, t) op = [op[0] & 2, t.value];
-            switch (op[0]) {
-                case 0: case 1: t = op; break;
-                case 4: _.label++; return { value: op[1], done: false };
-                case 5: _.label++; y = op[1]; op = [0]; continue;
-                case 7: op = _.ops.pop(); _.trys.pop(); continue;
-                default:
-                    if (!(t = _.trys, t = t.length > 0 && t[t.length - 1]) && (op[0] === 6 || op[0] === 2)) { _ = 0; continue; }
-                    if (op[0] === 3 && (!t || (op[1] > t[0] && op[1] < t[3]))) { _.label = op[1]; break; }
-                    if (op[0] === 6 && _.label < t[1]) { _.label = t[1]; t = op; break; }
-                    if (t && _.label < t[2]) { _.label = t[2]; _.ops.push(op); break; }
-                    if (t[2]) _.ops.pop();
-                    _.trys.pop(); continue;
-            }
-            op = body.call(thisArg, _);
-        } catch (e) { op = [6, e]; y = 0; } finally { f = t = 0; }
-        if (op[0] & 5) throw op[1]; return { value: op[0] ? op[1] : void 0, done: true };
-    }
-};
 var __importDefault = (this && this.__importDefault) || function (mod) {
     return (mod && mod.__esModule) ? mod : { "default": mod };
 };
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.processMessages = exports.getMessages = exports.requestShutdown = void 0;
-var os_1 = require("os");
-var client_sqs_1 = require("@aws-sdk/client-sqs");
-var chalk_1 = __importDefault(require("chalk"));
-var debug_1 = __importDefault(require("debug"));
-var systemMonitor_js_1 = require("./scheduler/systemMonitor.js");
-var queueManager_js_1 = require("./scheduler/queueManager.js");
-var jobExecutor_js_1 = require("./scheduler/jobExecutor.js");
-var defaults_js_1 = require("./defaults.js");
-var sqs_js_1 = require("./sqs.js");
-var debug = (0, debug_1.default)('qdone:consumer');
+const os_1 = require("os");
+const client_sqs_1 = require("@aws-sdk/client-sqs");
+const chalk_1 = __importDefault(require("chalk"));
+const debug_1 = __importDefault(require("debug"));
+const systemMonitor_js_1 = require("./scheduler/systemMonitor.js");
+const queueManager_js_1 = require("./scheduler/queueManager.js");
+const jobExecutor_js_1 = require("./scheduler/jobExecutor.js");
+const defaults_js_1 = require("./defaults.js");
+const sqs_js_1 = require("./sqs.js");
+const debug = (0, debug_1.default)('qdone:consumer');
 // Global flag for shutdown request
-var shutdownRequested = false;
-var shutdownCallbacks = [];
-function requestShutdown() {
-    return __awaiter(this, void 0, void 0, function () {
-        var _i, shutdownCallbacks_1, callback;
-        return __generator(this, function (_a) {
-            switch (_a.label) {
-                case 0:
-                    debug('requestShutdown');
-                    shutdownRequested = true;
-                    _i = 0, shutdownCallbacks_1 = shutdownCallbacks;
-                    _a.label = 1;
-                case 1:
-                    if (!(_i < shutdownCallbacks_1.length)) return [3 /*break*/, 4];
-                    callback = shutdownCallbacks_1[_i];
-                    debug('callback', callback);
-                    return [4 /*yield*/, callback()
-                        // try { callback() } catch (e) { }
-                    ];
-                case 2:
-                    _a.sent();
-                    _a.label = 3;
-                case 3:
-                    _i++;
-                    return [3 /*break*/, 1];
-                case 4:
-                    debug('requestShutdown done');
-                    return [2 /*return*/];
-            }
-        });
-    });
+let shutdownRequested = false;
+const shutdownCallbacks = [];
+async function requestShutdown() {
+    debug('requestShutdown');
+    shutdownRequested = true;
+    for (const callback of shutdownCallbacks) {
+        debug('callback', callback);
+        await callback();
+        // try { callback() } catch (e) { }
+    }
+    debug('requestShutdown done');
 }
 exports.requestShutdown = requestShutdown;
-function getMessages(qrl, opt, maxMessages) {
-    return __awaiter(this, void 0, void 0, function () {
-        var params, response;
-        return __generator(this, function (_a) {
-            switch (_a.label) {
-                case 0:
-                    params = {
-                        AttributeNames: ['All'],
-                        MaxNumberOfMessages: maxMessages,
-                        MessageAttributeNames: ['All'],
-                        QueueUrl: qrl,
-                        VisibilityTimeout: 60,
-                        WaitTimeSeconds: opt.waitTime
-                    };
-                    return [4 /*yield*/, (0, sqs_js_1.getSQSClient)().send(new client_sqs_1.ReceiveMessageCommand(params))
-                        // debug('ReceiveMessage response', response)
-                    ];
-                case 1:
-                    response = _a.sent();
-                    // debug('ReceiveMessage response', response)
-                    return [2 /*return*/, response.Messages || []];
-            }
-        });
-    });
+async function getMessages(qrl, opt, maxMessages) {
+    const params = {
+        AttributeNames: ['All'],
+        MaxNumberOfMessages: maxMessages,
+        MessageAttributeNames: ['All'],
+        QueueUrl: qrl,
+        VisibilityTimeout: 60,
+        WaitTimeSeconds: opt.waitTime
+    };
+    const response = await (0, sqs_js_1.getSQSClient)().send(new client_sqs_1.ReceiveMessageCommand(params));
+    // debug('ReceiveMessage response', response)
+    return response.Messages || [];
 }
 exports.getMessages = getMessages;
 //
 // Consumer
 //
-function processMessages(queues, callback, options) {
-    return __awaiter(this, void 0, void 0, function () {
-        var opt, lastLatency, systemMonitor, jobExecutor, queueManager, cores, delayTimeout, delay, activeQrls, maxReturnCount, listen, allowedJobs, maxLatency, latency, latencyFactor, freeMemory, totalMemory, memoryThreshold, freememThreshold, remainingMemory, freememFactor, oneMinuteLoad, loadPerCore, loadFactor, overallFactor, targetJobs, jobsLeft, _i, _a, _b, qname, qrl, maxMessages;
-        var _this = this;
-        return __generator(this, function (_c) {
-            switch (_c.label) {
-                case 0:
-                    debug({ options: options });
-                    opt = (0, defaults_js_1.getOptionsWithDefaults)(options);
-                    debug('processMessages', { queues: queues, callback: callback, options: options, opt: opt, argv: process.argv });
-                    lastLatency = 10;
-                    systemMonitor = new systemMonitor_js_1.SystemMonitor(function (latency) {
-                        var percentDifference = 100 * Math.abs(lastLatency - latency) / lastLatency;
-                        if (percentDifference > 10 && opt.verbose) {
-                            console.error(chalk_1.default.blue('Latency:', Math.round(latency), 'ms'));
-                        }
-                        lastLatency = latency;
-                    });
-                    jobExecutor = new jobExecutor_js_1.JobExecutor(opt);
-                    queueManager = new queueManager_js_1.QueueManager(opt, queues, 60);
-                    cores = (0, os_1.cpus)().length;
-                    delay = function (ms) { return new Promise(function (resolve) {
-                        delayTimeout = setTimeout(resolve, ms);
-                    }); };
-                    shutdownCallbacks.push(function () { return __awaiter(_this, void 0, void 0, function () {
-                        return __generator(this, function (_a) {
-                            switch (_a.label) {
-                                case 0:
-                                    clearTimeout(delayTimeout);
-                                    return [4 /*yield*/, queueManager.shutdown()];
-                                case 1:
-                                    _a.sent();
-                                    debug({ queueManager: 'done' });
-                                    return [4 /*yield*/, jobExecutor.shutdown()];
-                                case 2:
-                                    _a.sent();
-                                    debug({ jobExecutor: 'done' });
-                                    return [4 /*yield*/, systemMonitor.shutdown()];
-                                case 3:
-                                    _a.sent();
-                                    debug({ systemMonitor: 'done' });
-                                    return [2 /*return*/];
-                            }
-                        });
-                    }); });
-                    activeQrls = new Set();
-                    maxReturnCount = 0;
-                    listen = function (qname, qrl, maxMessages) { return __awaiter(_this, void 0, void 0, function () {
-                        var messages, _i, messages_1, message, e_1;
-                        return __generator(this, function (_a) {
-                            switch (_a.label) {
-                                case 0:
-                                    if (opt.verbose) {
-                                        console.error(chalk_1.default.blue('Listening on: '), qname);
-                                    }
-                                    activeQrls.add(qrl);
-                                    maxReturnCount += maxMessages;
-                                    _a.label = 1;
-                                case 1:
-                                    _a.trys.push([1, 3, , 4]);
-                                    return [4 /*yield*/, getMessages(qrl, opt, maxMessages)];
-                                case 2:
-                                    messages = _a.sent();
-                                    if (!shutdownRequested) {
-                                        if (messages.length) {
-                                            for (_i = 0, messages_1 = messages; _i < messages_1.length; _i++) {
-                                                message = messages_1[_i];
-                                                jobExecutor.executeJob(message, callback, qname, qrl);
-                                            }
-                                            queueManager.updateIcehouse(qrl, false);
-                                        }
-                                        else {
-                                            // If we didn't get any, update the icehouse so we can back off
-                                            queueManager.updateIcehouse(qrl, true);
-                                        }
-                                    }
-                                    // Max job accounting
-                                    maxReturnCount -= maxMessages;
-                                    activeQrls.delete(qrl);
-                                    return [3 /*break*/, 4];
-                                case 3:
-                                    e_1 = _a.sent();
-                                    // If the queue has been cleaned up, we should back off anyway
-                                    if (e_1 instanceof client_sqs_1.QueueDoesNotExist) {
-                                        queueManager.updateIcehouse(qrl, true);
-                                    }
-                                    else {
-                                        throw e_1;
-                                    }
-                                    return [3 /*break*/, 4];
-                                case 4: return [2 /*return*/];
-                            }
-                        });
-                    }); };
-                    _c.label = 1;
-                case 1:
-                    if (!!shutdownRequested) return [3 /*break*/, 3];
-                    allowedJobs = Math.max(0, opt.maxConcurrentJobs - jobExecutor.activeJobCount() - maxReturnCount);
-                    maxLatency = 100;
-                    latency = systemMonitor.getLatency() || 10;
-                    latencyFactor = 1 - Math.abs(Math.min(latency / maxLatency, 1)) // 0 if latency is at max, 1 if latency 0
-                    ;
-                    freeMemory = (0, os_1.freemem)();
-                    totalMemory = (0, os_1.totalmem)();
-                    memoryThreshold = totalMemory * opt.maxMemoryPercent / 100;
-                    freememThreshold = totalMemory - memoryThreshold;
-                    remainingMemory = Math.max(0, freeMemory - freememThreshold);
-                    freememFactor = Math.min(1, Math.max(0, remainingMemory / memoryThreshold));
-                    oneMinuteLoad = (0, os_1.loadavg)()[0];
-                    loadPerCore = oneMinuteLoad / cores;
-                    loadFactor = 1 - Math.min(1, Math.max(0, loadPerCore / 3));
-                    overallFactor = Math.min(latencyFactor, freememFactor, loadFactor);
-                    targetJobs = Math.round(allowedJobs * overallFactor);
-                    jobsLeft = targetJobs;
-                    if (opt.verbose) {
-                        console.error({ maxConcurrentJobs: opt.maxConcurrentJobs, jobCount: jobExecutor.activeJobCount(), allowedJobs: allowedJobs, maxLatency: maxLatency, latencyFactor: latencyFactor, freememFactor: freememFactor, loadFactor: loadFactor, overallFactor: overallFactor, targetJobs: targetJobs, activeQrls: activeQrls });
-                    }
-                    for (_i = 0, _a = queueManager.getPairs(); _i < _a.length; _i++) {
-                        _b = _a[_i], qname = _b.qname, qrl = _b.qrl;
-                        // debug({ evaluating: { qname, qrl, jobsLeft, activeQrlsHasQrl: activeQrls.has(qrl) } })
-                        if (jobsLeft <= 0 || activeQrls.has(qrl))
-                            continue;
-                        maxMessages = Math.min(10, jobsLeft);
-                        listen(qname, qrl, maxMessages);
-                        jobsLeft -= maxMessages;
-                        // debug({ listenedTo: { qname, maxMessages, jobsLeft } })
-                    }
-                    return [4 /*yield*/, delay(1000)];
-                case 2:
-                    _c.sent();
-                    return [3 /*break*/, 1];
-                case 3:
-                    debug('after all');
-                    return [2 /*return*/];
-            }
-        });
+async function processMessages(queues, callback, options) {
+    debug({ options });
+    const opt = (0, defaults_js_1.getOptionsWithDefaults)(options);
+    debug('processMessages', { queues, callback, options, opt, argv: process.argv });
+    let lastLatency = 10;
+    const systemMonitor = new systemMonitor_js_1.SystemMonitor(latency => {
+        const percentDifference = 100 * Math.abs(lastLatency - latency) / lastLatency;
+        if (percentDifference > 10 && opt.verbose) {
+            console.error(chalk_1.default.blue('Latency:', Math.round(latency), 'ms'));
+        }
+        lastLatency = latency;
+    });
+    const jobExecutor = new jobExecutor_js_1.JobExecutor(opt);
+    const queueManager = new queueManager_js_1.QueueManager(opt, queues, 60);
+    const cores = (0, os_1.cpus)().length;
+    // debug({ systemMonitor, jobExecutor, queueManager })
+    // This delay function keeps a timeout reference around so it can be
+    // cancelled at shutdown
+    let delayTimeout;
+    const delay = (ms) => new Promise(resolve => {
+        delayTimeout = setTimeout(resolve, ms);
+    });
+    shutdownCallbacks.push(async () => {
+        clearTimeout(delayTimeout);
+        await queueManager.shutdown();
+        debug({ queueManager: 'done' });
+        await jobExecutor.shutdown();
+        debug({ jobExecutor: 'done' });
+        await systemMonitor.shutdown();
+        debug({ systemMonitor: 'done' });
     });
+    // Keep track of how many messages could be returned from each queue
+    const activeQrls = new Set();
+    let maxReturnCount = 0;
+    const listen = async (qname, qrl, maxMessages) => {
+        if (opt.verbose) {
+            console.error(chalk_1.default.blue('Listening on: '), qname);
+        }
+        activeQrls.add(qrl);
+        maxReturnCount += maxMessages;
+        try {
+            const messages = await getMessages(qrl, opt, maxMessages);
+            if (!shutdownRequested) {
+                if (messages.length) {
+                    jobExecutor.executeJobs(messages, callback, qname, qrl);
+                    queueManager.updateIcehouse(qrl, false);
+                }
+                else {
+                    // If we didn't get any, update the icehouse so we can back off
+                    queueManager.updateIcehouse(qrl, true);
+                }
+            }
+            // Max job accounting
+            if (messages.length)
+                maxReturnCount -= messages.length;
+            activeQrls.delete(qrl);
+        }
+        catch (e) {
+            // If the queue has been cleaned up, we should back off anyway
+            if (e instanceof client_sqs_1.QueueDoesNotExist) {
+                queueManager.updateIcehouse(qrl, true);
+            }
+            else {
+                throw e;
+            }
+        }
+    };
+    while (!shutdownRequested) { // eslint-disable-line
+        // Figure out how we are running
+        const allowedJobs = Math.max(0, opt.maxConcurrentJobs - jobExecutor.runningJobCount() - maxReturnCount);
+        // Latency
+        const maxLatency = 100;
+        const latency = systemMonitor.getLatency() || 10;
+        const latencyFactor = 1 - Math.abs(Math.min(latency / maxLatency, 1)); // 0 if latency is at max, 1 if latency 0
+        // Memory
+        const freeMemory = (0, os_1.freemem)();
+        const totalMemory = (0, os_1.totalmem)();
+        const memoryThreshold = totalMemory * opt.maxMemoryPercent / 100;
+        const freememThreshold = totalMemory - memoryThreshold;
+        const remainingMemory = Math.max(0, freeMemory - freememThreshold);
+        const freememFactor = Math.min(1, Math.max(0, remainingMemory / memoryThreshold));
+        // Load
+        const oneMinuteLoad = (0, os_1.loadavg)()[0];
+        const loadPerCore = oneMinuteLoad / cores;
+        const loadFactor = 1 - Math.min(1, Math.max(0, loadPerCore / 3));
+        const overallFactor = Math.min(latencyFactor, freememFactor, loadFactor);
+        const targetJobs = Math.round(allowedJobs * overallFactor);
+        let jobsLeft = targetJobs;
+        if (opt.verbose) {
+            // console.error({ maxConcurrentJobs: opt.maxConcurrentJobs, jobCount: jobExecutor.activeJobCount(), allowedJobs, maxLatency, latencyFactor, freememFactor, loadFactor, overallFactor, targetJobs, activeQrls })
+        }
+        for (const { qname, qrl } of queueManager.getPairs()) {
+            // const qcount = jobExecutor.runningJobCountForQueue(qname)
+            // console.log({ evaluating: { qname, qrl, qcount, jobsLeft, activeQrlsHasQrl: activeQrls.has(qrl) } })
+            if (jobsLeft <= 0 || activeQrls.has(qrl))
+                continue;
+            const maxMessages = Math.min(10, jobsLeft);
+            listen(qname, qrl, maxMessages);
+            jobsLeft -= maxMessages;
+            // debug({ listenedTo: { qname, maxMessages, jobsLeft } })
+        }
+        await delay(1000);
+    }
+    debug('after all');
 }
 exports.processMessages = processMessages;

package/commonjs/src/defaults.js

@@ -5,7 +5,7 @@ exports.setupVerbose = exports.setupAWS = exports.getOptionsWithDefaults = expor
  * Default options for qdone. Accepts a command line options object and
  * returns nicely-named options.
  */
-var uuid_1 = require("uuid");
+const uuid_1 = require("uuid");
 exports.defaults = Object.freeze({
     // Shared
     prefix: 'qdone_',
@@ -27,8 +27,8 @@ exports.defaults = Object.freeze({
     messageRetentionPeriod: 1209600,
     delay: 0,
     sendRetries: 6,
-    failDelay: 0,
-    dlq: false,
+    failDelay: 120,
+    dlq: true,
     dlqSuffix: '_dead',
     dlqAfter: 3,
     // Worker
@@ -44,9 +44,9 @@ exports.defaults = Object.freeze({
     unpair: false
 });
 function validateInteger(opt, name) {
-    var parsed = parseInt(opt[name], 10);
+    const parsed = parseInt(opt[name], 10);
     if (isNaN(parsed))
-        throw new Error("".concat(name, " needs to be an integer."));
+        throw new Error(`${name} needs to be an integer.`);
     return parsed;
 }
 /**
@@ -62,8 +62,8 @@ function getOptionsWithDefaults(options) {
     if (!options)
         options = {};
     // Activate DLQ if any option is set
-    var dlq = options.dlq || !!(options['dlq-suffix'] || options['dlq-after'] || options['dlq-name']);
-    var opt = {
+    const dlq = options.dlq || !!(options['dlq-suffix'] || options['dlq-after'] || options['dlq-name']);
+    const opt = {
         // Shared
         prefix: options.prefix === '' ? options.prefix : (options.prefix || exports.defaults.prefix),
         failSuffix: options.failSuffix || options['fail-suffix'] || exports.defaults.failSuffix,
@@ -85,7 +85,7 @@ function getOptionsWithDefaults(options) {
         delay: options.delay || exports.defaults.delay,
         sendRetries: options['send-retries'] || exports.defaults.sendRetries,
         failDelay: options.failDelay || options['fail-delay'] || exports.defaults.failDelay,
-        dlq: dlq || exports.defaults.dlq,
+        dlq: dlq === false ? false : (dlq || exports.defaults.dlq),
         dlqSuffix: options.dlqSuffix || options['dlq-suffix'] || exports.defaults.dlqSuffix,
         dlqAfter: options.dlqAfter || options['dlq-after'] || exports.defaults.dlqAfter,
         tags: options.tags || undefined,
@@ -120,13 +120,13 @@ function getOptionsWithDefaults(options) {
 }
 exports.getOptionsWithDefaults = getOptionsWithDefaults;
 function setupAWS(options) {
-    var opt = getOptionsWithDefaults(options);
+    const opt = getOptionsWithDefaults(options);
     process.env.AWS_REGION = opt.region;
 }
 exports.setupAWS = setupAWS;
 function setupVerbose(options) {
-    var verbose = options.verbose || (process.stderr.isTTY && !options.quiet);
-    var quiet = options.quiet || (!process.stderr.isTTY && !options.verbose);
+    const verbose = options.verbose || (process.stderr.isTTY && !options.quiet);
+    const quiet = options.quiet || (!process.stderr.isTTY && !options.verbose);
     options.verbose = verbose;
     options.quiet = quiet;
 }