qdone 2.0.14-alpha → 2.0.15-alpha

package/commonjs/src/consumer.js

@@ -134,7 +134,7 @@ function processMessages(queues, callback, options) {
                         lastLatency = latency;
                     });
                     jobExecutor = new jobExecutor_js_1.JobExecutor(opt);
-                    queueManager = new queueManager_js_1.QueueManager(opt, queues);
+                    queueManager = new queueManager_js_1.QueueManager(opt, queues, 60);
                     delay = function (ms) { return new Promise(function (resolve) {
                         delayTimeout = setTimeout(resolve, ms);
                     }); };
@@ -206,15 +206,17 @@ function processMessages(queues, callback, options) {
                     _c.label = 1;
                 case 1:
                     if (!!shutdownRequested) return [3 /*break*/, 3];
-                    allowedJobs = opt.maxConcurrentJobs - jobExecutor.activeJobCount() - maxReturnCount;
+                    allowedJobs = Math.max(0, opt.maxConcurrentJobs - jobExecutor.activeJobCount() - maxReturnCount);
                     maxLatency = 100;
-                    latency = systemMonitor.getLatency().setTimeout;
+                    latency = systemMonitor.getLatency() || 10;
                     latencyFactor = 1 - Math.abs(Math.min(latency / maxLatency, 1)) // 0 if latency is at max, 1 if latency 0
                     ;
                     targetJobs = Math.round(allowedJobs * latencyFactor);
                     jobsLeft = targetJobs;
+                    debug({ jobCount: jobExecutor.activeJobCount(), maxReturnCount: maxReturnCount, allowedJobs: allowedJobs, maxLatency: maxLatency, latency: latency, latencyFactor: latencyFactor, targetJobs: targetJobs, activeQrls: activeQrls });
                     for (_i = 0, _a = queueManager.getPairs(); _i < _a.length; _i++) {
                         _b = _a[_i], qname = _b.qname, qrl = _b.qrl;
+                        debug({ evaluating: { qname: qname, qrl: qrl, jobsLeft: jobsLeft, activeQrlsHasQrl: activeQrls.has(qrl) } });
                         if (jobsLeft <= 0 || activeQrls.has(qrl))
                             continue;
                         maxMessages = Math.min(10, jobsLeft);
@@ -223,7 +225,7 @@ function processMessages(queues, callback, options) {
                         if (opt.verbose) {
                             console.error(chalk_1.default.blue('Listening on: '), qname);
                         }
-                        // debug({ listenedTo: { qname, maxMessages, jobsLeft } })
+                        debug({ listenedTo: { qname: qname, maxMessages: maxMessages, jobsLeft: jobsLeft } });
                     }
                     return [4 /*yield*/, delay(1000)];
                 case 2:

package/commonjs/src/scheduler/queueManager.js

@@ -204,7 +204,12 @@ var QueueManager = /** @class */ (function () {
         return pair;
     };
     QueueManager.prototype.getPairs = function () {
-        return this.selectedPairs;
+        var _this = this;
+        var now = new Date();
+        return this.selectedPairs.filter(function (_a) {
+            var qname = _a.qname, qrl = _a.qrl;
+            return !_this.keepInIcehouse(qrl, now);
+        });
     };
     QueueManager.prototype.shutdown = function () {
         return __awaiter(this, void 0, void 0, function () {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "qdone",
-  "version": "2.0.14-alpha",
+  "version": "2.0.15-alpha",
   "description": "Language agnostic job queue for SQS",
   "type": "module",
   "main": "./index.js",

package/src/consumer.js

@@ -60,7 +60,7 @@ export async function processMessages (queues, callback, options) {
     lastLatency = latency
   })
   const jobExecutor = new JobExecutor(opt)
-  const queueManager = new QueueManager(opt, queues)
+  const queueManager = new QueueManager(opt, queues, 60)
   // debug({ systemMonitor, jobExecutor, queueManager })
 
   // This delay function keeps a timeout reference around so it can be
@@ -113,15 +113,15 @@ export async function processMessages (queues, callback, options) {
 
   while (!shutdownRequested) { // eslint-disable-line
     // Figure out how we are running
-    const allowedJobs = opt.maxConcurrentJobs - jobExecutor.activeJobCount() - maxReturnCount
+    const allowedJobs = Math.max(0, opt.maxConcurrentJobs - jobExecutor.activeJobCount() - maxReturnCount)
     const maxLatency = 100
-    const latency = systemMonitor.getLatency().setTimeout
+    const latency = systemMonitor.getLatency() || 10
     const latencyFactor = 1 - Math.abs(Math.min(latency / maxLatency, 1)) // 0 if latency is at max, 1 if latency 0
     const targetJobs = Math.round(allowedJobs * latencyFactor)
-    // debug({ allowedJobs, maxLatency, latency, latencyFactor, targetJobs, activeQrls })
-
     let jobsLeft = targetJobs
+    debug({ jobCount: jobExecutor.activeJobCount(), maxReturnCount, allowedJobs, maxLatency, latency, latencyFactor, targetJobs, activeQrls })
     for (const { qname, qrl } of queueManager.getPairs()) {
+      debug({ evaluating: { qname, qrl, jobsLeft, activeQrlsHasQrl: activeQrls.has(qrl) } })
       if (jobsLeft <= 0 || activeQrls.has(qrl)) continue
       const maxMessages = Math.min(10, jobsLeft)
       listen(qname, qrl, maxMessages)
@@ -129,7 +129,7 @@ export async function processMessages (queues, callback, options) {
       if (opt.verbose) {
         console.error(chalk.blue('Listening on: '), qname)
       }
-      // debug({ listenedTo: { qname, maxMessages, jobsLeft } })
+      debug({ listenedTo: { qname, maxMessages, jobsLeft } })
     }
     await delay(1000)
   }

package/src/scheduler/queueManager.js

@@ -140,7 +140,8 @@ export class QueueManager {
   }
 
   getPairs () {
-    return this.selectedPairs
+    const now = new Date()
+    return this.selectedPairs.filter(({ qname, qrl }) => !this.keepInIcehouse(qrl, now))
   }
 
   async shutdown () {