qdone 2.0.22-alpha → 2.0.24-alpha

package/commonjs/src/consumer.js

@@ -118,7 +118,7 @@ exports.getMessages = getMessages;
 //
 function processMessages(queues, callback, options) {
     return __awaiter(this, void 0, void 0, function () {
-        var opt, lastLatency, systemMonitor, jobExecutor, queueManager, delayTimeout, delay, activeQrls, maxReturnCount, listen, allowedJobs, maxLatency, freeMemory, totalMemory, memoryThreshold, latency, latencyFactor, freememFactor, targetJobs, jobsLeft, _i, _a, _b, qname, qrl, maxMessages;
+        var opt, lastLatency, systemMonitor, jobExecutor, queueManager, delayTimeout, delay, activeQrls, maxReturnCount, listen, allowedJobs, maxLatency, freeMemory, totalMemory, remainingMemory, memoryThreshold, latency, latencyFactor, freememFactor, targetJobs, jobsLeft, _i, _a, _b, qname, qrl, maxMessages;
         var _this = this;
         return __generator(this, function (_c) {
             switch (_c.label) {
@@ -178,7 +178,7 @@ function processMessages(queues, callback, options) {
                                     if (messages.length) {
                                         for (_i = 0, messages_1 = messages; _i < messages_1.length; _i++) {
                                             message = messages_1[_i];
-                                            jobExecutor.executeJob(message, callback, qname, qrl);
+                                            jobExecutor.executeJob(message, callback, qname, qrl, function () { return queueManager.updateIcehouse(qrl, true); });
                                         }
                                         queueManager.updateIcehouse(qrl, false);
                                     }
@@ -211,14 +211,15 @@ function processMessages(queues, callback, options) {
                     maxLatency = 100;
                     freeMemory = (0, os_1.freemem)();
                     totalMemory = (0, os_1.totalmem)();
+                    remainingMemory = totalMemory - freeMemory;
                     memoryThreshold = totalMemory * opt.maxMemoryPercent / 100;
                     latency = systemMonitor.getLatency() || 10;
                     latencyFactor = 1 - Math.abs(Math.min(latency / maxLatency, 1)) // 0 if latency is at max, 1 if latency 0
                     ;
-                    freememFactor = Math.min(1, Math.max(0, freeMemory / memoryThreshold));
+                    freememFactor = Math.min(1, Math.max(0, remainingMemory / memoryThreshold));
                     targetJobs = Math.round(allowedJobs * latencyFactor * freememFactor);
                     jobsLeft = targetJobs;
-                    debug({ jobCount: jobExecutor.activeJobCount(), freeMemory: freeMemory, totalMemory: totalMemory, memoryThreshold: memoryThreshold, maxReturnCount: maxReturnCount, allowedJobs: allowedJobs, maxLatency: maxLatency, latency: latency, latencyFactor: latencyFactor, freememFactor: freememFactor, targetJobs: targetJobs, activeQrls: activeQrls });
+                    debug({ jobCount: jobExecutor.activeJobCount(), freeMemory: freeMemory, totalMemory: totalMemory, remainingMemory: remainingMemory, memoryThreshold: memoryThreshold, maxReturnCount: maxReturnCount, allowedJobs: allowedJobs, maxLatency: maxLatency, latency: latency, latencyFactor: latencyFactor, freememFactor: freememFactor, targetJobs: targetJobs, activeQrls: activeQrls });
                     for (_i = 0, _a = queueManager.getPairs(); _i < _a.length; _i++) {
                         _b = _a[_i], qname = _b.qname, qrl = _b.qrl;
                         // debug({ evaluating: { qname, qrl, jobsLeft, activeQrlsHasQrl: activeQrls.has(qrl) } })

package/commonjs/src/scheduler/jobExecutor.js

@@ -266,7 +266,7 @@ var JobExecutor = /** @class */ (function () {
             });
         });
     };
-    JobExecutor.prototype.executeJob = function (message, callback, qname, qrl) {
+    JobExecutor.prototype.executeJob = function (message, callback, qname, qrl, failedCallback) {
         return __awaiter(this, void 0, void 0, function () {
             var payload, visibilityTimeout, job, oldJob, e, queue, result, err_1;
             return __generator(this, function (_a) {
@@ -340,6 +340,9 @@ var JobExecutor = /** @class */ (function () {
                         return [3 /*break*/, 4];
                     case 3:
                         err_1 = _a.sent();
+                        // Notify caller that we failed
+                        if (failedCallback)
+                            failedCallback(message, qname, qrl);
                         // Fail path for job execution
                         if (this.opt.verbose) {
                             console.error(chalk_1.default.red('FAILED'), message.Body);

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "qdone",
-  "version": "2.0.22-alpha",
+  "version": "2.0.24-alpha",
   "description": "Language agnostic job queue for SQS",
   "type": "module",
   "main": "./index.js",

package/src/consumer.js

@@ -91,7 +91,7 @@ export async function processMessages (queues, callback, options) {
       const messages = await getMessages(qrl, opt, maxMessages)
       if (messages.length) {
         for (const message of messages) {
-          jobExecutor.executeJob(message, callback, qname, qrl)
+          jobExecutor.executeJob(message, callback, qname, qrl, () => queueManager.updateIcehouse(qrl, true))
         }
         queueManager.updateIcehouse(qrl, false)
       } else {
@@ -118,13 +118,14 @@ export async function processMessages (queues, callback, options) {
     const maxLatency = 100
     const freeMemory = freemem()
     const totalMemory = totalmem()
+    const remainingMemory = totalMemory - freeMemory
     const memoryThreshold = totalMemory * opt.maxMemoryPercent / 100
     const latency = systemMonitor.getLatency() || 10
     const latencyFactor = 1 - Math.abs(Math.min(latency / maxLatency, 1)) // 0 if latency is at max, 1 if latency 0
-    const freememFactor = Math.min(1, Math.max(0, freeMemory / memoryThreshold))
+    const freememFactor = Math.min(1, Math.max(0, remainingMemory / memoryThreshold))
     const targetJobs = Math.round(allowedJobs * latencyFactor * freememFactor)
     let jobsLeft = targetJobs
-    debug({ jobCount: jobExecutor.activeJobCount(), freeMemory, totalMemory, memoryThreshold, maxReturnCount, allowedJobs, maxLatency, latency, latencyFactor, freememFactor, targetJobs, activeQrls })
+    debug({ jobCount: jobExecutor.activeJobCount(), freeMemory, totalMemory, remainingMemory, memoryThreshold, maxReturnCount, allowedJobs, maxLatency, latency, latencyFactor, freememFactor, targetJobs, activeQrls })
     for (const { qname, qrl } of queueManager.getPairs()) {
       // debug({ evaluating: { qname, qrl, jobsLeft, activeQrlsHasQrl: activeQrls.has(qrl) } })
       if (jobsLeft <= 0 || activeQrls.has(qrl)) continue

package/src/scheduler/jobExecutor.js

@@ -193,7 +193,7 @@ export class JobExecutor {
     }, nextCheckInMs)
   }
 
-  async executeJob (message, callback, qname, qrl) {
+  async executeJob (message, callback, qname, qrl, failedCallback) {
     // Create job entry and track it
     const payload = this.opt.json ? JSON.parse(message.Body) : message.Body
     const visibilityTimeout = 60
@@ -261,6 +261,8 @@ export class JobExecutor {
       }
       this.stats.jobsSucceeded++
     } catch (err) {
+      // Notify caller that we failed
+      if (failedCallback) failedCallback(message, qname, qrl)
       // Fail path for job execution
       if (this.opt.verbose) {
         console.error(chalk.red('FAILED'), message.Body)