qdone 2.0.27-alpha → 2.0.29-alpha

package/commonjs/src/consumer.js

@@ -183,7 +183,7 @@ function processMessages(queues, callback, options) {
                                         if (messages.length) {
                                             for (_i = 0, messages_1 = messages; _i < messages_1.length; _i++) {
                                                 message = messages_1[_i];
-                                                jobExecutor.executeJob(message, callback, qname, qrl, function () { return queueManager.updateIcehouse(qrl, true); });
+                                                jobExecutor.executeJob(message, callback, qname, qrl);
                                             }
                                             queueManager.updateIcehouse(qrl, false);
                                         }
@@ -231,7 +231,7 @@ function processMessages(queues, callback, options) {
                     targetJobs = Math.round(allowedJobs * overallFactor);
                     jobsLeft = targetJobs;
                     if (opt.verbose) {
-                        console.error({ jobCount: jobExecutor.activeJobCount(), freeMemory: freeMemory, totalMemory: totalMemory, freememThreshold: freememThreshold, remainingMemory: remainingMemory, memoryThreshold: memoryThreshold, maxReturnCount: maxReturnCount, allowedJobs: allowedJobs, maxLatency: maxLatency, latency: latency, latencyFactor: latencyFactor, freememFactor: freememFactor, oneMinuteLoad: oneMinuteLoad, loadPerCore: loadPerCore, loadFactor: loadFactor, overallFactor: overallFactor, targetJobs: targetJobs, activeQrls: activeQrls });
+                        console.error({ maxConcurrentJobs: opt.maxConcurrentJobs, jobCount: jobExecutor.activeJobCount(), allowedJobs: allowedJobs, maxLatency: maxLatency, latencyFactor: latencyFactor, freememFactor: freememFactor, loadFactor: loadFactor, overallFactor: overallFactor, targetJobs: targetJobs, activeQrls: activeQrls });
                     }
                     for (_i = 0, _a = queueManager.getPairs(); _i < _a.length; _i++) {
                         _b = _a[_i], qname = _b.qname, qrl = _b.qrl;

package/commonjs/src/scheduler/jobExecutor.js

@@ -96,10 +96,10 @@ var JobExecutor = /** @class */ (function () {
      */
     JobExecutor.prototype.maintainVisibility = function () {
         return __awaiter(this, void 0, void 0, function () {
-            var nextCheckInMs, start, jobsToExtendByQrl, jobsToDeleteByQrl, jobsToCleanup, i, job, jobRunTime, jobsToDelete, jobsToExtend, doubled, secondsUntilMax, _a, _b, _c, _i, qrl, jobsToExtend, entries, messageId, job, entry, input, result, count, _d, _e, _f, _g, qrl, jobsToDelete, entries, messageId, job, entry, input, result, count;
+            var nextCheckInMs, start, jobsToExtendByQrl, jobsToDeleteByQrl, jobsToCleanup, i, job, jobRunTime, jobsToDelete, jobsToExtend, doubled, secondsUntilMax, _a, _b, _c, _i, qrl, jobsToExtend, entries, messageId, job, entry, input, result, _d, _e, failed, count, _f, _g, _h, _j, qrl, jobsToDelete, entries, messageId, job, entry, input, result, _k, _l, failed, count;
             var _this = this;
-            return __generator(this, function (_h) {
-                switch (_h.label) {
+            return __generator(this, function (_m) {
+                switch (_m.label) {
                     case 0:
                         // Bail if we are shutting down
                         if (this.shutdownRequested && this.stats.activeJobs === 0 && this.jobs.length === 0) {
@@ -149,7 +149,7 @@ var JobExecutor = /** @class */ (function () {
                                     secondsUntilMax = Math.max(1, maxJobSeconds - jobRunTime);
                                     // const secondsUntilKill = Math.max(1, this.opt.killAfter - jobRunTime)
                                     job.visibilityTimeout = Math.min(doubled, secondsUntilMax); //, secondsUntilKill)
-                                    job.extendAtSecond = Math.round(jobRunTime + job.visibilityTimeout); // this is what we use next time
+                                    job.extendAtSecond = Math.round(jobRunTime + job.visibilityTimeout / 2); // this is what we use next time
                                     debug({ doubled: doubled, secondsUntilMax: secondsUntilMax, job: job });
                                 }
                             }
@@ -159,7 +159,7 @@ var JobExecutor = /** @class */ (function () {
                         for (_c in _a)
                             _b.push(_c);
                         _i = 0;
-                        _h.label = 1;
+                        _m.label = 1;
                     case 1:
                         if (!(_i < _b.length)) return [3 /*break*/, 5];
                         _c = _b[_i];
@@ -167,7 +167,7 @@ var JobExecutor = /** @class */ (function () {
                         qrl = _c;
                         jobsToExtend = jobsToExtendByQrl[qrl];
                         debug({ qrl: qrl, jobsToExtend: jobsToExtend });
-                        _h.label = 2;
+                        _m.label = 2;
                     case 2:
                         if (!jobsToExtend.length) return [3 /*break*/, 4];
                         entries = [];
@@ -175,7 +175,7 @@ var JobExecutor = /** @class */ (function () {
                         while (messageId++ < 10 && jobsToExtend.length) {
                             job = jobsToExtend.shift();
                             entry = {
-                                Id: '' + messageId,
+                                Id: job.message.MessageId,
                                 ReceiptHandle: job.message.ReceiptHandle,
                                 VisibilityTimeout: job.visibilityTimeout
                             };
@@ -186,9 +186,16 @@ var JobExecutor = /** @class */ (function () {
                         debug({ ChangeMessageVisibilityBatch: input });
                         return [4 /*yield*/, (0, sqs_js_1.getSQSClient)().send(new client_sqs_1.ChangeMessageVisibilityBatchCommand(input))];
                     case 3:
-                        result = _h.sent();
+                        result = _m.sent();
                         debug('ChangeMessageVisibilityBatch returned', result);
                         this.stats.sqsCalls++;
+                        if (result.Failed) {
+                            console.error('FAILED_MESSAGES', result.Failed);
+                            for (_d = 0, _e = result.Failed; _d < _e.length; _d++) {
+                                failed = _e[_d];
+                                console.error('FAILED_TO_EXTEND_JOB', this.jobsByMessageId[failed.Id]);
+                            }
+                        }
                         if (result.Successful) {
                             count = result.Successful.length || 0;
                             this.stats.timeoutsExtended += count;
@@ -204,19 +211,19 @@ var JobExecutor = /** @class */ (function () {
                         _i++;
                         return [3 /*break*/, 1];
                     case 5:
-                        _d = jobsToDeleteByQrl;
-                        _e = [];
-                        for (_f in _d)
-                            _e.push(_f);
-                        _g = 0;
-                        _h.label = 6;
+                        _f = jobsToDeleteByQrl;
+                        _g = [];
+                        for (_h in _f)
+                            _g.push(_h);
+                        _j = 0;
+                        _m.label = 6;
                     case 6:
-                        if (!(_g < _e.length)) return [3 /*break*/, 10];
-                        _f = _e[_g];
-                        if (!(_f in _d)) return [3 /*break*/, 9];
-                        qrl = _f;
+                        if (!(_j < _g.length)) return [3 /*break*/, 10];
+                        _h = _g[_j];
+                        if (!(_h in _f)) return [3 /*break*/, 9];
+                        qrl = _h;
                         jobsToDelete = jobsToDeleteByQrl[qrl];
-                        _h.label = 7;
+                        _m.label = 7;
                     case 7:
                         if (!jobsToDelete.length) return [3 /*break*/, 9];
                         entries = [];
@@ -224,7 +231,7 @@ var JobExecutor = /** @class */ (function () {
                         while (messageId++ < 10 && jobsToDelete.length) {
                             job = jobsToDelete.shift();
                             entry = {
-                                Id: '' + messageId,
+                                Id: job.message.MessageId,
                                 ReceiptHandle: job.message.ReceiptHandle,
                                 VisibilityTimeout: job.visibilityTimeout
                             };
@@ -235,8 +242,15 @@ var JobExecutor = /** @class */ (function () {
                         debug({ DeleteMessageBatch: input });
                         return [4 /*yield*/, (0, sqs_js_1.getSQSClient)().send(new client_sqs_1.DeleteMessageBatchCommand(input))];
                     case 8:
-                        result = _h.sent();
+                        result = _m.sent();
                         this.stats.sqsCalls++;
+                        if (result.Failed) {
+                            console.error('FAILED_MESSAGES', result.Failed);
+                            for (_k = 0, _l = result.Failed; _k < _l.length; _k++) {
+                                failed = _l[_k];
+                                console.error('FAILED_TO_DELETE_JOB', this.jobsByMessageId[failed.Id]);
+                            }
+                        }
                         if (result.Successful) {
                             count = result.Successful.length || 0;
                             this.stats.jobsDeleted += count;
@@ -250,7 +264,7 @@ var JobExecutor = /** @class */ (function () {
                         debug('DeleteMessageBatch returned', result);
                         return [3 /*break*/, 7];
                     case 9:
-                        _g++;
+                        _j++;
                         return [3 /*break*/, 6];
                     case 10:
                         // Get rid of deleted and failed jobs

package/commonjs/src/scheduler/queueManager.js

@@ -95,8 +95,11 @@ var QueueManager = /** @class */ (function () {
             var secondsElapsed = Math.round((now - lastCheck) / 1000);
             debug({ icehouseCheck: { qrl: qrl, lastCheck: lastCheck, secondsToWait: secondsToWait, secondsElapsed: secondsElapsed } });
             var letOut = secondsElapsed > secondsToWait;
-            if (letOut && this.opt.verbose) {
-                console.error(chalk_1.default.blue('Coming out of icehouse:'), qrl);
+            if (letOut) {
+                delete this.icehouse[qrl];
+                if (this.opt.verbose) {
+                    console.error(chalk_1.default.blue('Coming out of icehouse:'), qrl);
+                }
             }
             return !letOut;
         }
@@ -125,7 +128,10 @@ var QueueManager = /** @class */ (function () {
                     case 1:
                         pairs = _a.sent();
                         if (this.opt.verbose)
-                            console.error(chalk_1.default.blue('Resolving queues:'));
+                            console.error(chalk_1.default.blue('Resolving queues:'), pairs.map(function (_a) {
+                                var qname = _a.qname;
+                                return qname;
+                            }));
                         if (this.shutdownRequested)
                             return [2 /*return*/];
                         now = new Date();
@@ -142,6 +148,13 @@ var QueueManager = /** @class */ (function () {
                             var qname = _a.qname, qrl = _a.qrl;
                             var isFifo = qname.endsWith('.fifo');
                             return _this.opt.fifo ? isFifo : true;
+                        })
+                            // next dead
+                            .filter(function (_a) {
+                            var qname = _a.qname, qrl = _a.qrl;
+                            var isFifo = qname.endsWith('.fifo');
+                            var isDead = isFifo ? qname.endsWith('_dead.fifo') : qname.endsWith('_dead');
+                            return !isDead;
                         })
                             // then icehouse
                             .filter(function (_a) {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "qdone",
-  "version": "2.0.27-alpha",
+  "version": "2.0.29-alpha",
   "description": "Language agnostic job queue for SQS",
   "type": "module",
   "main": "./index.js",

package/src/consumer.js

@@ -97,7 +97,7 @@ export async function processMessages (queues, callback, options) {
       if (!shutdownRequested) {
         if (messages.length) {
           for (const message of messages) {
-            jobExecutor.executeJob(message, callback, qname, qrl, () => queueManager.updateIcehouse(qrl, true))
+            jobExecutor.executeJob(message, callback, qname, qrl)
           }
           queueManager.updateIcehouse(qrl, false)
         } else {
@@ -146,7 +146,7 @@ export async function processMessages (queues, callback, options) {
     let jobsLeft = targetJobs
 
     if (opt.verbose) {
-      console.error({ jobCount: jobExecutor.activeJobCount(), freeMemory, totalMemory, freememThreshold, remainingMemory, memoryThreshold, maxReturnCount, allowedJobs, maxLatency, latency, latencyFactor, freememFactor, oneMinuteLoad, loadPerCore, loadFactor, overallFactor, targetJobs, activeQrls })
+      console.error({ maxConcurrentJobs: opt.maxConcurrentJobs, jobCount: jobExecutor.activeJobCount(), allowedJobs, maxLatency, latencyFactor, freememFactor, loadFactor, overallFactor, targetJobs, activeQrls })
     }
     for (const { qname, qrl } of queueManager.getPairs()) {
       // debug({ evaluating: { qname, qrl, jobsLeft, activeQrlsHasQrl: activeQrls.has(qrl) } })

package/src/scheduler/jobExecutor.js

@@ -101,7 +101,7 @@ export class JobExecutor {
           const secondsUntilMax = Math.max(1, maxJobSeconds - jobRunTime)
           // const secondsUntilKill = Math.max(1, this.opt.killAfter - jobRunTime)
           job.visibilityTimeout = Math.min(doubled, secondsUntilMax) //, secondsUntilKill)
-          job.extendAtSecond = Math.round(jobRunTime + job.visibilityTimeout) // this is what we use next time
+          job.extendAtSecond = Math.round(jobRunTime + job.visibilityTimeout / 2) // this is what we use next time
           debug({ doubled, secondsUntilMax, job })
         }
       }
@@ -119,7 +119,7 @@ export class JobExecutor {
         while (messageId++ < 10 && jobsToExtend.length) {
           const job = jobsToExtend.shift()
           const entry = {
-            Id: '' + messageId,
+            Id: job.message.MessageId,
             ReceiptHandle: job.message.ReceiptHandle,
             VisibilityTimeout: job.visibilityTimeout
           }
@@ -133,6 +133,12 @@ export class JobExecutor {
         const result = await getSQSClient().send(new ChangeMessageVisibilityBatchCommand(input))
         debug('ChangeMessageVisibilityBatch returned', result)
         this.stats.sqsCalls++
+        if (result.Failed) {
+          console.error('FAILED_MESSAGES', result.Failed)
+          for (const failed of result.Failed) {
+            console.error('FAILED_TO_EXTEND_JOB', this.jobsByMessageId[failed.Id])
+          }
+        }
         if (result.Successful) {
           const count = result.Successful.length || 0
           this.stats.timeoutsExtended += count
@@ -156,7 +162,7 @@ export class JobExecutor {
         while (messageId++ < 10 && jobsToDelete.length) {
           const job = jobsToDelete.shift()
           const entry = {
-            Id: '' + messageId,
+            Id: job.message.MessageId,
             ReceiptHandle: job.message.ReceiptHandle,
             VisibilityTimeout: job.visibilityTimeout
           }
@@ -169,6 +175,12 @@ export class JobExecutor {
         debug({ DeleteMessageBatch: input })
         const result = await getSQSClient().send(new DeleteMessageBatchCommand(input))
         this.stats.sqsCalls++
+        if (result.Failed) {
+          console.error('FAILED_MESSAGES', result.Failed)
+          for (const failed of result.Failed) {
+            console.error('FAILED_TO_DELETE_JOB', this.jobsByMessageId[failed.Id])
+          }
+        }
         if (result.Successful) {
           const count = result.Successful.length || 0
           this.stats.jobsDeleted += count

package/src/scheduler/queueManager.js

@@ -57,8 +57,11 @@ export class QueueManager {
       const secondsElapsed = Math.round((now - lastCheck) / 1000)
       debug({ icehouseCheck: { qrl, lastCheck, secondsToWait, secondsElapsed } })
       const letOut = secondsElapsed > secondsToWait
-      if (letOut && this.opt.verbose) {
-        console.error(chalk.blue('Coming out of icehouse:'), qrl)
+      if (letOut) {
+        delete this.icehouse[qrl]
+        if (this.opt.verbose) {
+          console.error(chalk.blue('Coming out of icehouse:'), qrl)
+        }
       }
       return !letOut
     } else {
@@ -79,7 +82,7 @@ export class QueueManager {
     // Start processing
     const qnames = this.queues.map(queue => normalizeQueueName(queue, this.opt))
     const pairs = await getQnameUrlPairs(qnames, this.opt)
-    if (this.opt.verbose) console.error(chalk.blue('Resolving queues:'))
+    if (this.opt.verbose) console.error(chalk.blue('Resolving queues:'), pairs.map(({ qname }) => qname))
 
     if (this.shutdownRequested) return
 
@@ -97,6 +100,12 @@ export class QueueManager {
         const isFifo = qname.endsWith('.fifo')
         return this.opt.fifo ? isFifo : true
       })
+      // next dead
+      .filter(({ qname, qrl }) => {
+        const isFifo = qname.endsWith('.fifo')
+        const isDead = isFifo ? qname.endsWith('_dead.fifo') : qname.endsWith('_dead')
+        return !isDead
+      })
       // then icehouse
       .filter(({ qname, qrl }) => !this.keepInIcehouse(qrl, now))