qdone 2.0.17-alpha → 2.0.19-alpha

package/commonjs/src/consumer.js

@@ -118,7 +118,7 @@ exports.getMessages = getMessages;
 //
 function processMessages(queues, callback, options) {
     return __awaiter(this, void 0, void 0, function () {
-        var opt, lastLatency, systemMonitor, jobExecutor, queueManager, delayTimeout, delay, activeQrls, maxReturnCount, listen, allowedJobs, maxLatency, latency, latencyFactor, freememFactor, targetJobs, jobsLeft, _i, _a, _b, qname, qrl, maxMessages;
+        var opt, lastLatency, systemMonitor, jobExecutor, queueManager, delayTimeout, delay, activeQrls, maxReturnCount, listen, allowedJobs, maxLatency, freeMemory, totalMemory, memoryThreshold, latency, latencyFactor, freememFactor, targetJobs, jobsLeft, _i, _a, _b, qname, qrl, maxMessages;
         var _this = this;
         return __generator(this, function (_c) {
             switch (_c.label) {
@@ -209,16 +209,19 @@ function processMessages(queues, callback, options) {
                     if (!!shutdownRequested) return [3 /*break*/, 3];
                     allowedJobs = Math.max(0, opt.maxConcurrentJobs - jobExecutor.activeJobCount() - maxReturnCount);
                     maxLatency = 100;
+                    freeMemory = (0, os_1.freemem)();
+                    totalMemory = (0, os_1.totalmem)();
+                    memoryThreshold = totalMemory * opt.maxMemoryPercent / 100;
                     latency = systemMonitor.getLatency() || 10;
                     latencyFactor = 1 - Math.abs(Math.min(latency / maxLatency, 1)) // 0 if latency is at max, 1 if latency 0
                     ;
-                    freememFactor = Math.min(1, Math.max(0, (0, os_1.freemem)() - (0, os_1.totalmem)() / 2) / (0, os_1.totalmem)());
+                    freememFactor = Math.min(1, Math.max(0, freeMemory / memoryThreshold));
                     targetJobs = Math.round(allowedJobs * latencyFactor * freememFactor);
                     jobsLeft = targetJobs;
-                    debug({ jobCount: jobExecutor.activeJobCount(), maxReturnCount: maxReturnCount, allowedJobs: allowedJobs, maxLatency: maxLatency, latency: latency, latencyFactor: latencyFactor, freememFactor: freememFactor, targetJobs: targetJobs, activeQrls: activeQrls });
+                    debug({ jobCount: jobExecutor.activeJobCount(), freeMemory: freeMemory, totalMemory: totalMemory, memoryThreshold: memoryThreshold, maxReturnCount: maxReturnCount, allowedJobs: allowedJobs, maxLatency: maxLatency, latency: latency, latencyFactor: latencyFactor, freememFactor: freememFactor, targetJobs: targetJobs, activeQrls: activeQrls });
                     for (_i = 0, _a = queueManager.getPairs(); _i < _a.length; _i++) {
                         _b = _a[_i], qname = _b.qname, qrl = _b.qrl;
-                        debug({ evaluating: { qname: qname, qrl: qrl, jobsLeft: jobsLeft, activeQrlsHasQrl: activeQrls.has(qrl) } });
+                        // debug({ evaluating: { qname, qrl, jobsLeft, activeQrlsHasQrl: activeQrls.has(qrl) } })
                         if (jobsLeft <= 0 || activeQrls.has(qrl))
                             continue;
                         maxMessages = Math.min(10, jobsLeft);
@@ -227,7 +230,7 @@ function processMessages(queues, callback, options) {
                         if (opt.verbose) {
                             console.error(chalk_1.default.blue('Listening on: '), qname);
                         }
-                        debug({ listenedTo: { qname: qname, maxMessages: maxMessages, jobsLeft: jobsLeft } });
+                        // debug({ listenedTo: { qname, maxMessages, jobsLeft } })
                     }
                     return [4 /*yield*/, delay(1000)];
                 case 2:

package/commonjs/src/defaults.js

@@ -37,6 +37,7 @@ exports.defaults = Object.freeze({
     archive: false,
     activeOnly: false,
     maxConcurrentJobs: 100,
+    maxMemoryPercent: 70,
     // Idle Queues
     idleFor: 60,
     delete: false,
@@ -95,6 +96,7 @@ function getOptionsWithDefaults(options) {
         activeOnly: options.activeOnly || options['active-only'] || exports.defaults.activeOnly,
         includeFailed: options.includeFailed || options['include-failed'] || exports.defaults.includeFailed,
         maxConcurrentJobs: options.maxConcurrentJobs || exports.defaults.maxConcurrentJobs,
+        maxMemoryPercent: options.maxMemoryPercent || exports.defaults.maxMemoryPercent,
         // Idle Queues
         idleFor: options.idleFor || options['idle-for'] || exports.defaults.idleFor,
         delete: options.delete || exports.defaults.delete,
@@ -112,6 +114,7 @@ function getOptionsWithDefaults(options) {
     opt.waitTime = validateInteger(opt, 'waitTime');
     opt.killAfter = validateInteger(opt, 'killAfter');
     opt.maxConcurrentJobs = validateInteger(opt, 'maxConcurrentJobs');
+    opt.maxMemoryPercent = validateInteger(opt, 'maxMemoryPercent');
     opt.idleFor = validateInteger(opt, 'idleFor');
     return opt;
 }

package/commonjs/src/idleQueues.js

@@ -85,21 +85,33 @@ var metricNames = [
  */
 function _cheapIdleCheck(qname, qrl, opt) {
     return __awaiter(this, void 0, void 0, function () {
-        var client, cmd, data, result;
+        var client, cmd, data, result_1, e_1;
         return __generator(this, function (_a) {
             switch (_a.label) {
                 case 0:
+                    _a.trys.push([0, 2, , 3]);
                     client = (0, sqs_js_1.getSQSClient)();
                     cmd = new client_sqs_1.GetQueueAttributesCommand({ AttributeNames: exports.attributeNames, QueueUrl: qrl });
                     return [4 /*yield*/, client.send(cmd)];
                 case 1:
                     data = _a.sent();
                     debug('data', data);
-                    result = data.Attributes;
-                    result.queue = qname.slice(opt.prefix.length);
+                    result_1 = data.Attributes;
+                    result_1.queue = qname.slice(opt.prefix.length);
                     // We are idle if all the messages attributes are zero
-                    result.idle = exports.attributeNames.filter(function (k) { return result[k] === '0'; }).length === exports.attributeNames.length;
-                    return [2 /*return*/, { result: result, SQS: 1 }];
+                    result_1.idle = exports.attributeNames.filter(function (k) { return result_1[k] === '0'; }).length === exports.attributeNames.length;
+                    return [2 /*return*/, { result: result_1, SQS: 1 }];
+                case 2:
+                    e_1 = _a.sent();
+                    if (e_1 instanceof client_sqs_1.QueueDoesNotExist) {
+                        // Count deleted queues as idle
+                        return [2 /*return*/, { idle: true, SQS: 1 }];
+                    }
+                    else {
+                        throw e_1;
+                    }
+                    return [3 /*break*/, 3];
+                case 3: return [2 /*return*/];
             }
         });
     });
@@ -320,7 +332,7 @@ exports.processQueue = processQueue;
  */
 function processQueuePair(qname, qrl, opt) {
     return __awaiter(this, void 0, void 0, function () {
-        var isFifo, normalizeOptions, fqname, fqrl, result, active, fresult, idleCheckResult, factive, _a, dresult, dfresult, e_1, deleteResult, resultIncludingDelete;
+        var isFifo, normalizeOptions, fqname, fqrl, result, active, fresult, idleCheckResult, factive, _a, dresult, dfresult, e_2, deleteResult, resultIncludingDelete;
         return __generator(this, function (_b) {
             switch (_b.label) {
                 case 0:
@@ -385,11 +397,11 @@ function processQueuePair(qname, qrl, opt) {
                             }
                         })];
                 case 5:
-                    e_1 = _b.sent();
+                    e_2 = _b.sent();
                     // Handle the case where the fail queue has been deleted or was never
                     // created for some reason
-                    if (!(e_1 instanceof client_sqs_1.QueueDoesNotExist))
-                        throw e_1;
+                    if (!(e_2 instanceof client_sqs_1.QueueDoesNotExist))
+                        throw e_2;
                     // Fail queue doesn't exist if we get here
                     if (opt.verbose)
                         console.error(chalk_1.default.blue('Queue ') + fqname.slice(opt.prefix.length) + chalk_1.default.blue(' does not exist.'));

package/commonjs/src/scheduler/jobExecutor.js

@@ -269,7 +269,7 @@ var JobExecutor = /** @class */ (function () {
     };
     JobExecutor.prototype.executeJob = function (message, callback, qname, qrl) {
         return __awaiter(this, void 0, void 0, function () {
-            var payload, visibilityTimeout, job, oldJob, e, queue_1, result, err_1;
+            var payload, visibilityTimeout, job, oldJob, e, queue, result, err_1;
             return __generator(this, function (_a) {
                 switch (_a.label) {
                     case 0:
@@ -316,8 +316,8 @@ var JobExecutor = /** @class */ (function () {
                         _a.label = 1;
                     case 1:
                         _a.trys.push([1, 3, , 4]);
-                        queue_1 = qname.slice(this.opt.prefix.length);
-                        return [4 /*yield*/, (0, sentry_js_1.withSentry)(function () { return callback(queue_1, payload); }, this.opt, { job: job })];
+                        queue = qname.slice(this.opt.prefix.length);
+                        return [4 /*yield*/, callback(queue, payload)];
                     case 2:
                         result = _a.sent();
                         debug('executeJob callback finished', { payload: payload, result: result });

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "qdone",
-  "version": "2.0.17-alpha",
+  "version": "2.0.19-alpha",
   "description": "Language agnostic job queue for SQS",
   "type": "module",
   "main": "./index.js",

package/src/consumer.js

@@ -116,14 +116,17 @@ export async function processMessages (queues, callback, options) {
     // Figure out how we are running
     const allowedJobs = Math.max(0, opt.maxConcurrentJobs - jobExecutor.activeJobCount() - maxReturnCount)
     const maxLatency = 100
+    const freeMemory = freemem()
+    const totalMemory = totalmem()
+    const memoryThreshold = totalMemory * opt.maxMemoryPercent / 100
     const latency = systemMonitor.getLatency() || 10
     const latencyFactor = 1 - Math.abs(Math.min(latency / maxLatency, 1)) // 0 if latency is at max, 1 if latency 0
-    const freememFactor = Math.min(1, Math.max(0, freemem() - totalmem() / 2) / totalmem())
+    const freememFactor = Math.min(1, Math.max(0, freeMemory / memoryThreshold))
     const targetJobs = Math.round(allowedJobs * latencyFactor * freememFactor)
     let jobsLeft = targetJobs
-    debug({ jobCount: jobExecutor.activeJobCount(), maxReturnCount, allowedJobs, maxLatency, latency, latencyFactor, freememFactor, targetJobs, activeQrls })
+    debug({ jobCount: jobExecutor.activeJobCount(), freeMemory, totalMemory, memoryThreshold, maxReturnCount, allowedJobs, maxLatency, latency, latencyFactor, freememFactor, targetJobs, activeQrls })
     for (const { qname, qrl } of queueManager.getPairs()) {
-      debug({ evaluating: { qname, qrl, jobsLeft, activeQrlsHasQrl: activeQrls.has(qrl) } })
+      // debug({ evaluating: { qname, qrl, jobsLeft, activeQrlsHasQrl: activeQrls.has(qrl) } })
       if (jobsLeft <= 0 || activeQrls.has(qrl)) continue
       const maxMessages = Math.min(10, jobsLeft)
       listen(qname, qrl, maxMessages)
@@ -131,7 +134,7 @@ export async function processMessages (queues, callback, options) {
       if (opt.verbose) {
         console.error(chalk.blue('Listening on: '), qname)
       }
-      debug({ listenedTo: { qname, maxMessages, jobsLeft } })
+      // debug({ listenedTo: { qname, maxMessages, jobsLeft } })
     }
     await delay(1000)
   }

package/src/defaults.js

@@ -37,6 +37,7 @@ export const defaults = Object.freeze({
   archive: false,
   activeOnly: false,
   maxConcurrentJobs: 100,
+  maxMemoryPercent: 70,
 
   // Idle Queues
   idleFor: 60,
@@ -101,6 +102,7 @@ export function getOptionsWithDefaults (options) {
     activeOnly: options.activeOnly || options['active-only'] || defaults.activeOnly,
     includeFailed: options.includeFailed || options['include-failed'] || defaults.includeFailed,
     maxConcurrentJobs: options.maxConcurrentJobs || defaults.maxConcurrentJobs,
+    maxMemoryPercent: options.maxMemoryPercent || defaults.maxMemoryPercent,
 
     // Idle Queues
     idleFor: options.idleFor || options['idle-for'] || defaults.idleFor,
@@ -121,6 +123,7 @@ export function getOptionsWithDefaults (options) {
   opt.waitTime = validateInteger(opt, 'waitTime')
   opt.killAfter = validateInteger(opt, 'killAfter')
   opt.maxConcurrentJobs = validateInteger(opt, 'maxConcurrentJobs')
+  opt.maxMemoryPercent = validateInteger(opt, 'maxMemoryPercent')
   opt.idleFor = validateInteger(opt, 'idleFor')
 
   return opt

package/src/idleQueues.js

@@ -37,15 +37,24 @@ const metricNames = [
  * Actual SQS call, used in conjunction with cache.
  */
 export async function _cheapIdleCheck (qname, qrl, opt) {
-  const client = getSQSClient()
-  const cmd = new GetQueueAttributesCommand({ AttributeNames: attributeNames, QueueUrl: qrl })
-  const data = await client.send(cmd)
-  debug('data', data)
-  const result = data.Attributes
-  result.queue = qname.slice(opt.prefix.length)
-  // We are idle if all the messages attributes are zero
-  result.idle = attributeNames.filter(k => result[k] === '0').length === attributeNames.length
-  return { result, SQS: 1 }
+  try {
+    const client = getSQSClient()
+    const cmd = new GetQueueAttributesCommand({ AttributeNames: attributeNames, QueueUrl: qrl })
+    const data = await client.send(cmd)
+    debug('data', data)
+    const result = data.Attributes
+    result.queue = qname.slice(opt.prefix.length)
+    // We are idle if all the messages attributes are zero
+    result.idle = attributeNames.filter(k => result[k] === '0').length === attributeNames.length
+    return { result, SQS: 1 }
+  } catch (e) {
+    if (e instanceof QueueDoesNotExist) {
+      // Count deleted queues as idle
+      return { idle: true, SQS: 1 }
+    } else {
+      throw e
+    }
+  }
 }
 
 /**

package/src/scheduler/jobExecutor.js

@@ -242,7 +242,7 @@ export class JobExecutor {
     // Execute job
     try {
       const queue = qname.slice(this.opt.prefix.length)
-      const result = await withSentry(() => callback(queue, payload), this.opt, { job })
+      const result = await callback(queue, payload)
       debug('executeJob callback finished', { payload, result })
       if (this.opt.verbose) {
         console.error(chalk.green('SUCCESS'), message.Body)