qdone 2.0.28-alpha → 2.0.30-alpha

package/commonjs/src/sqs.js

@@ -2,55 +2,19 @@
 /**
  * Functions that deal with SQS
  */
-var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
-    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
-    return new (P || (P = Promise))(function (resolve, reject) {
-        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
-        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
-        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
-        step((generator = generator.apply(thisArg, _arguments || [])).next());
-    });
-};
-var __generator = (this && this.__generator) || function (thisArg, body) {
-    var _ = { label: 0, sent: function() { if (t[0] & 1) throw t[1]; return t[1]; }, trys: [], ops: [] }, f, y, t, g;
-    return g = { next: verb(0), "throw": verb(1), "return": verb(2) }, typeof Symbol === "function" && (g[Symbol.iterator] = function() { return this; }), g;
-    function verb(n) { return function (v) { return step([n, v]); }; }
-    function step(op) {
-        if (f) throw new TypeError("Generator is already executing.");
-        while (g && (g = 0, op[0] && (_ = 0)), _) try {
-            if (f = 1, y && (t = op[0] & 2 ? y["return"] : op[0] ? y["throw"] || ((t = y["return"]) && t.call(y), 0) : y.next) && !(t = t.call(y, op[1])).done) return t;
-            if (y = 0, t) op = [op[0] & 2, t.value];
-            switch (op[0]) {
-                case 0: case 1: t = op; break;
-                case 4: _.label++; return { value: op[1], done: false };
-                case 5: _.label++; y = op[1]; op = [0]; continue;
-                case 7: op = _.ops.pop(); _.trys.pop(); continue;
-                default:
-                    if (!(t = _.trys, t = t.length > 0 && t[t.length - 1]) && (op[0] === 6 || op[0] === 2)) { _ = 0; continue; }
-                    if (op[0] === 3 && (!t || (op[1] > t[0] && op[1] < t[3]))) { _.label = op[1]; break; }
-                    if (op[0] === 6 && _.label < t[1]) { _.label = t[1]; t = op; break; }
-                    if (t && _.label < t[2]) { _.label = t[2]; _.ops.push(op); break; }
-                    if (t[2]) _.ops.pop();
-                    _.trys.pop(); continue;
-            }
-            op = body.call(thisArg, _);
-        } catch (e) { op = [6, e]; y = 0; } finally { f = t = 0; }
-        if (op[0] & 5) throw op[1]; return { value: op[0] ? op[1] : void 0, done: true };
-    }
-};
 var __importDefault = (this && this.__importDefault) || function (mod) {
     return (mod && mod.__esModule) ? mod : { "default": mod };
 };
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.getQueueAttributes = exports.getMatchingQueues = exports.setSQSClient = exports.getSQSClient = void 0;
-var client_sqs_1 = require("@aws-sdk/client-sqs");
-var path_1 = require("path");
-var debug_1 = __importDefault(require("debug"));
-var debug = (0, debug_1.default)('qdone:sqs');
+const client_sqs_1 = require("@aws-sdk/client-sqs");
+const path_1 = require("path");
+const debug_1 = __importDefault(require("debug"));
+const debug = (0, debug_1.default)('qdone:sqs');
 /**
  * Utility function to return an instantiated, shared SQSClient.
  */
-var client;
+let client;
 function getSQSClient() {
     if (client)
         return client;
@@ -68,113 +32,66 @@ exports.setSQSClient = setSQSClient;
 /**
  * Returns qrls for queues matching the given prefix and regex.
  */
-function getMatchingQueues(prefix, regex) {
-    return __awaiter(this, void 0, void 0, function () {
-        function processQueues(nextToken) {
-            return __awaiter(this, void 0, void 0, function () {
-                var command, result, qrls, nextToken2, _a, _b, _c;
-                return __generator(this, function (_d) {
-                    switch (_d.label) {
-                        case 0:
-                            if (nextToken)
-                                input.NextToken = nextToken;
-                            command = new client_sqs_1.ListQueuesCommand(input);
-                            return [4 /*yield*/, client.send(command)
-                                // debug({ result })
-                            ];
-                        case 1:
-                            result = _d.sent();
-                            qrls = result.QueueUrls, nextToken2 = result.NextToken;
-                            _b = (_a = (qrls || []).filter(function (q) { return regex.test(q); })).concat;
-                            if (!nextToken2) return [3 /*break*/, 3];
-                            return [4 /*yield*/, processQueues(nextToken2)];
-                        case 2:
-                            _c = _d.sent();
-                            return [3 /*break*/, 4];
-                        case 3:
-                            _c = [];
-                            _d.label = 4;
-                        case 4: 
-                        // debug({ qrls, nextToken2 })
-                        return [2 /*return*/, _b.apply(_a, [_c])];
-                    }
-                });
-            });
-        }
-        var input, client;
-        return __generator(this, function (_a) {
-            input = { QueueNamePrefix: prefix, MaxResults: 1000 };
-            client = getSQSClient();
-            return [2 /*return*/, processQueues()];
-        });
-    });
+async function getMatchingQueues(prefix, regex) {
+    const input = { QueueNamePrefix: prefix, MaxResults: 1000 };
+    const client = getSQSClient();
+    async function processQueues(nextToken) {
+        if (nextToken)
+            input.NextToken = nextToken;
+        const command = new client_sqs_1.ListQueuesCommand(input);
+        // debug({ nextToken, input, command })
+        const result = await client.send(command);
+        // debug({ result })
+        const { QueueUrls: qrls, NextToken: nextToken2 } = result;
+        // debug({ qrls, nextToken2 })
+        return (qrls || []).filter(q => regex.test(q)).concat(nextToken2 ? await processQueues(nextToken2) : []);
+    }
+    return processQueues();
 }
 exports.getMatchingQueues = getMatchingQueues;
 /**
  * Gets attributes on every queue in parallel.
  */
-function getQueueAttributes(qrls) {
-    return __awaiter(this, void 0, void 0, function () {
-        var promises, _loop_1, _i, qrls_1, qrl;
-        var _this = this;
-        return __generator(this, function (_a) {
-            promises = [];
-            _loop_1 = function (qrl) {
-                var input = {
-                    QueueUrl: qrl,
-                    AttributeNames: [
-                        'ApproximateNumberOfMessages',
-                        'ApproximateNumberOfMessagesNotVisible',
-                        'ApproximateNumberOfMessagesDelayed'
-                    ]
-                };
-                var command = new client_sqs_1.GetQueueAttributesCommand(input);
-                // debug({ input, command })
-                promises.push((function () { return __awaiter(_this, void 0, void 0, function () {
-                    var queue, result, e_1;
-                    return __generator(this, function (_a) {
-                        switch (_a.label) {
-                            case 0:
-                                queue = (0, path_1.basename)(qrl);
-                                _a.label = 1;
-                            case 1:
-                                _a.trys.push([1, 3, , 4]);
-                                return [4 /*yield*/, client.send(command)
-                                    // debug({ queue, result })
-                                ];
-                            case 2:
-                                result = _a.sent();
-                                // debug({ queue, result })
-                                return [2 /*return*/, { queue: queue, result: result }];
-                            case 3:
-                                e_1 = _a.sent();
-                                if (e_1 instanceof client_sqs_1.QueueDoesNotExist) {
-                                    // For queues that have been deleted in the meantime for whatever
-                                    // reason, just show as having no messages instead of failing the
-                                    // whole batch
-                                    return [2 /*return*/, {
-                                            queue: queue,
-                                            Attributes: {
-                                                ApproximateNumberOfMessages: '0',
-                                                ApproximateNumberOfMessagesNotVisible: '0',
-                                                ApproximateNumberOfMessagesDelayed: '0'
-                                            }
-                                        }];
-                                }
-                                throw e_1;
-                            case 4: return [2 /*return*/];
+async function getQueueAttributes(qrls) {
+    const promises = [];
+    // debug({ qrls })
+    for (const qrl of qrls) {
+        const input = {
+            QueueUrl: qrl,
+            AttributeNames: [
+                'ApproximateNumberOfMessages',
+                'ApproximateNumberOfMessagesNotVisible',
+                'ApproximateNumberOfMessagesDelayed'
+            ]
+        };
+        const command = new client_sqs_1.GetQueueAttributesCommand(input);
+        // debug({ input, command })
+        promises.push((async () => {
+            const queue = (0, path_1.basename)(qrl);
+            try {
+                const result = await client.send(command);
+                // debug({ queue, result })
+                return { queue, result };
+            }
+            catch (e) {
+                if (e instanceof client_sqs_1.QueueDoesNotExist) {
+                    // For queues that have been deleted in the meantime for whatever
+                    // reason, just show as having no messages instead of failing the
+                    // whole batch
+                    return {
+                        queue,
+                        Attributes: {
+                            ApproximateNumberOfMessages: '0',
+                            ApproximateNumberOfMessagesNotVisible: '0',
+                            ApproximateNumberOfMessagesDelayed: '0'
                         }
-                    });
-                }); })());
-            };
-            // debug({ qrls })
-            for (_i = 0, qrls_1 = qrls; _i < qrls_1.length; _i++) {
-                qrl = qrls_1[_i];
-                _loop_1(qrl);
+                    };
+                }
+                throw e;
             }
-            return [2 /*return*/, Promise.all(promises)];
-        });
-    });
+        })());
+    }
+    return Promise.all(promises);
 }
 exports.getQueueAttributes = getQueueAttributes;
 debug('loaded');

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "qdone",
-  "version": "2.0.28-alpha",
+  "version": "2.0.30-alpha",
   "description": "Language agnostic job queue for SQS",
   "type": "module",
   "main": "./index.js",
@@ -47,7 +47,7 @@
   "scripts": {
     "start": "./src/bin.js",
     "test": "NODE_OPTIONS='--experimental-json-modules --experimental-vm-modules --no-warnings' jest",
-    "build": "tsc --allowJs index.js --outdir commonjs --esModuleInterop --module commonjs",
+    "build": "tsc --allowJs index.js --outdir commonjs --esModuleInterop --module commonjs  --target es2021 --strict --skipLibCheck --forceConsistentCasingInFileNames",
     "clean": "rm -rf commonjs/src commonjs/*.js coverage",
     "lint": "standard",
     "coverage": "nyc report --reporter=text-lcov | coveralls",

package/src/consumer.js

@@ -96,9 +96,7 @@ export async function processMessages (queues, callback, options) {
 
       if (!shutdownRequested) {
         if (messages.length) {
-          for (const message of messages) {
-            jobExecutor.executeJob(message, callback, qname, qrl, () => queueManager.updateIcehouse(qrl, true))
-          }
+          jobExecutor.executeJobs(messages, callback, qname, qrl)
           queueManager.updateIcehouse(qrl, false)
         } else {
           // If we didn't get any, update the icehouse so we can back off
@@ -107,7 +105,7 @@ export async function processMessages (queues, callback, options) {
       }
 
       // Max job accounting
-      maxReturnCount -= maxMessages
+      if (messages.length) maxReturnCount -= messages.length
       activeQrls.delete(qrl)
     } catch (e) {
       // If the queue has been cleaned up, we should back off anyway
@@ -146,7 +144,7 @@ export async function processMessages (queues, callback, options) {
     let jobsLeft = targetJobs
 
     if (opt.verbose) {
-      console.error({ jobCount: jobExecutor.activeJobCount(), freeMemory, totalMemory, freememThreshold, remainingMemory, memoryThreshold, maxReturnCount, allowedJobs, maxLatency, latency, latencyFactor, freememFactor, oneMinuteLoad, loadPerCore, loadFactor, overallFactor, targetJobs, activeQrls })
+      // console.error({ maxConcurrentJobs: opt.maxConcurrentJobs, jobCount: jobExecutor.activeJobCount(), allowedJobs, maxLatency, latencyFactor, freememFactor, loadFactor, overallFactor, targetJobs, activeQrls })
     }
     for (const { qname, qrl } of queueManager.getPairs()) {
       // debug({ evaluating: { qname, qrl, jobsLeft, activeQrlsHasQrl: activeQrls.has(qrl) } })

package/src/defaults.js

@@ -26,8 +26,8 @@ export const defaults = Object.freeze({
   messageRetentionPeriod: 1209600,
   delay: 0,
   sendRetries: 6,
-  failDelay: 0,
-  dlq: false,
+  failDelay: 120,
+  dlq: true,
   dlqSuffix: '_dead',
   dlqAfter: 3,
 
@@ -90,7 +90,7 @@ export function getOptionsWithDefaults (options) {
     delay: options.delay || defaults.delay,
     sendRetries: options['send-retries'] || defaults.sendRetries,
     failDelay: options.failDelay || options['fail-delay'] || defaults.failDelay,
-    dlq: dlq || defaults.dlq,
+    dlq: dlq === false ? false : (dlq || defaults.dlq),
     dlqSuffix: options.dlqSuffix || options['dlq-suffix'] || defaults.dlqSuffix,
     dlqAfter: options.dlqAfter || options['dlq-after'] || defaults.dlqAfter,
     tags: options.tags || undefined,

package/src/enqueue.js

@@ -79,7 +79,7 @@ export async function getOrCreateFailQueue (queue, opt) {
         maxReceiveCount: opt.dlqAfter + ''
       })
     }
-    if (opt.failDelay) params.Attributes.DelaySeconds = opt.failDelay
+    if (opt.failDelay) params.Attributes.DelaySeconds = opt.failDelay + ''
     if (opt.tags) params.tags = opt.tags
     if (opt.fifo) params.Attributes.FifoQueue = 'true'
     const cmd = new CreateQueueCommand(params)

package/src/scheduler/jobExecutor.js

@@ -21,6 +21,8 @@ export class JobExecutor {
     this.jobsByMessageId = {}
     this.stats = {
       activeJobs: 0,
+      waitingJobs: 0,
+      runningJobs: 0,
       sqsCalls: 0,
       timeoutsExtended: 0,
       jobsSucceeded: 0,
@@ -71,15 +73,12 @@ export class JobExecutor {
     const jobsToDeleteByQrl = {}
     const jobsToCleanup = new Set()
 
-    if (this.opt.verbose) {
-      console.error(chalk.blue('Stats: '), this.stats)
-      console.error(chalk.blue('Running: '), this.jobs.filter(j => j.status === 'processing').map(({ qname, message }) => ({ qname, payload: message.Body })))
-    }
-
     // Build list of jobs we need to deal with
+    const jobStatuses = {}
     for (let i = 0; i < this.jobs.length; i++) {
       const job = this.jobs[i]
       const jobRunTime = Math.round((start - job.start) / 1000)
+      jobStatuses[job.status] = (jobStatuses[job.status] || 0) + 1
       // debug('considering job', job)
       if (job.status === 'complete') {
         const jobsToDelete = jobsToDeleteByQrl[job.qrl] || []
@@ -88,7 +87,8 @@ export class JobExecutor {
         jobsToDeleteByQrl[job.qrl] = jobsToDelete
       } else if (job.status === 'failed') {
         jobsToCleanup.add(job)
-      } else if (job.status === 'processing') {
+      } else if (job.status !== 'deleting') {
+        // Any other job state gets visibility accounting
         debug('processing', { job, jobRunTime })
         if (jobRunTime >= job.extendAtSecond) {
           // Add it to our organized list of jobs
@@ -106,7 +106,11 @@ export class JobExecutor {
         }
       }
     }
-    // debug('maintainVisibility', { jobsToDeleteByQrl, jobsToExtendByQrl })
+
+    if (this.opt.verbose) {
+      console.error(chalk.blue('Stats: '), { stats: this.stats, jobStatuses })
+      console.error(chalk.blue('Running: '), this.jobs.filter(j => j.status === 'processing').map(({ qname, message }) => ({ qname, payload: message.Body })))
+    }
 
     // Extend in batches for each queue
     for (const qrl in jobsToExtendByQrl) {
@@ -207,20 +211,19 @@ export class JobExecutor {
     })
   }
 
-  async executeJob (message, callback, qname, qrl, failedCallback) {
-    if (this.shutdownRequested) throw new Error('jobExecutor is shutting down so cannot execute new job')
+  addJob (message, callback, qname, qrl) {
     // Create job entry and track it
-    const payload = this.opt.json ? JSON.parse(message.Body) : message.Body
-    const visibilityTimeout = 60
+    const defaultVisibilityTimeout = 60
     const job = {
-      status: 'processing',
+      status: 'waiting',
       start: new Date(),
-      visibilityTimeout,
-      extendAtSecond: visibilityTimeout / 2,
+      visibilityTimeout: defaultVisibilityTimeout,
+      extendAtSecond: defaultVisibilityTimeout / 2,
       payload: this.opt.json ? JSON.parse(message.Body) : message.Body,
       message,
       callback,
       qname,
+      prettyQname: qname.slice(this.opt.prefix.length),
       qrl
     }
 
@@ -237,29 +240,45 @@ export class JobExecutor {
       throw e
     }
 
-    // debug('executeJob', job)
     this.jobs.push(job)
     this.jobsByMessageId[job.message.MessageId] = job
     this.stats.activeJobs++
+    this.stats.waitingJobs++
     if (this.opt.verbose) {
-      console.error(chalk.blue('Executing:'), qname, chalk.blue('-->'), job.payload)
+      console.error(chalk.blue('Got message:'), job.prettyQname, chalk.blue('-->'), job.payload, job.message.MessageId)
     } else if (!this.opt.disableLog) {
       console.log(JSON.stringify({
-        event: 'MESSAGE_PROCESSING_START',
+        event: 'MESSAGE_RECEIVED',
         timestamp: new Date(),
-        qrl,
+        queue: job.qname,
         messageId: message.MessageId,
         payload: job.payload
       }))
     }
+    return job
+  }
 
-    // Execute job
+  async runJob (job) {
     try {
-      const queue = qname.slice(this.opt.prefix.length)
-      const result = await callback(queue, payload)
-      debug('executeJob callback finished', { payload, result })
       if (this.opt.verbose) {
-        console.error(chalk.green('SUCCESS'), message.Body)
+        console.error(chalk.blue('Running:'), job.prettyQname, chalk.blue('-->'), job.payload, job.message.MessageId)
+      } else if (!this.opt.disableLog) {
+        console.log(JSON.stringify({
+          event: 'MESSAGE_PROCESSING_START',
+          timestamp: new Date(),
+          queue: job.qname,
+          messageId: job.message.MessageId,
+          payload: job.payload
+        }))
+      }
+      job.status = 'running'
+      this.stats.runningJobs++
+      this.stats.waitingJobs--
+      const queue = job.qname.slice(this.opt.prefix.length)
+      const result = await job.callback(queue, job.payload)
+      debug('executeJob callback finished', { payload: job.payload, result })
+      if (this.opt.verbose) {
+        console.error(chalk.green('SUCCESS'), job.payload)
       }
       job.status = 'complete'
 
@@ -269,35 +288,60 @@ export class JobExecutor {
       } else if (!this.opt.disableLog) {
         console.log(JSON.stringify({
           event: 'MESSAGE_PROCESSING_COMPLETE',
+          queue: job.qname,
           timestamp: new Date(),
-          messageId: message.MessageId,
-          payload
+          messageId: job.message.MessageId,
+          payload: job.payload
         }))
       }
       this.stats.jobsSucceeded++
     } catch (err) {
-      // Notify caller that we failed
-      if (failedCallback) failedCallback(message, qname, qrl)
+      job.status = 'failed'
+      this.stats.jobsFailed++
       // Fail path for job execution
       if (this.opt.verbose) {
-        console.error(chalk.red('FAILED'), message.Body)
+        console.error(chalk.red('FAILED'), job.payload)
         console.error(chalk.blue('  error : ') + err)
       } else if (!this.opt.disableLog) {
         // Production error logging
         console.log(JSON.stringify({
           event: 'MESSAGE_PROCESSING_FAILED',
           reason: 'exception thrown',
-          qrl,
+          queue: job.qname,
           timestamp: new Date(),
-          messageId: message.MessageId,
-          payload,
+          messageId: job.message.MessageId,
+          payload: job.payload,
           errorMessage: err.toString().split('\n').slice(1).join('\n').trim() || undefined,
           err
         }))
       }
-      job.status = 'failed'
-      this.stats.jobsFailed++
     }
     this.stats.activeJobs--
+    this.stats.runningJobs--
+  }
+
+  async executeJobs (messages, callback, qname, qrl) {
+    if (this.shutdownRequested) throw new Error('jobExecutor is shutting down so cannot execute new jobs')
+
+    // Begin tracking jobs
+    const jobs = messages.map(message => this.addJob(message, callback, qname, qrl))
+    const isFifo = qrl.endsWith('.fifo')
+
+    // console.log(jobs)
+
+    // Begin executing
+    for (const [job, i] of jobs.map((job, i) => [job, i])) {
+      // Figure out if the next job needs to happen in serial, otherwise we can parallel execute
+      // const job = jobs[i]
+      const nextJob = jobs[i + 1]
+      const nextJobIsSerial =
+         isFifo && nextJob &&
+         job.message?.Attributes?.GroupId === nextJob.message?.Attributes?.GroupId
+
+      console.log({ i, nextJobAtt: nextJob.message.Attributes, nextJobIsSerial })
+      // Execute serial or parallel
+      if (nextJobIsSerial) await this.runJob(job)
+      else this.runJob(job)
+    }
   }
 }

package/src/scheduler/queueManager.js

@@ -35,7 +35,7 @@ export class QueueManager {
       const now = new Date()
       const secondsElapsed = lastCheck - now
       const minWait = 10
-      const maxWait = 600
+      const maxWait = 120
       const baseSeconds = numEmptyReceives ** 2 * 20
       const jitterSeconds = Math.round((Math.random() - 0.5) * baseSeconds)
       const newSecondsToWait = Math.max(minWait, Math.min(maxWait, baseSeconds + jitterSeconds))
@@ -57,8 +57,11 @@ export class QueueManager {
       const secondsElapsed = Math.round((now - lastCheck) / 1000)
       debug({ icehouseCheck: { qrl, lastCheck, secondsToWait, secondsElapsed } })
       const letOut = secondsElapsed > secondsToWait
-      if (letOut && this.opt.verbose) {
-        console.error(chalk.blue('Coming out of icehouse:'), qrl)
+      if (letOut) {
+        delete this.icehouse[qrl]
+        if (this.opt.verbose) {
+          console.error(chalk.blue('Coming out of icehouse:'), qrl)
+        }
       }
       return !letOut
     } else {
@@ -79,7 +82,7 @@ export class QueueManager {
     // Start processing
     const qnames = this.queues.map(queue => normalizeQueueName(queue, this.opt))
     const pairs = await getQnameUrlPairs(qnames, this.opt)
-    if (this.opt.verbose) console.error(chalk.blue('Resolving queues:'))
+    if (this.opt.verbose) console.error(chalk.blue('Resolving queues:'), pairs.map(({ qname }) => qname))
 
     if (this.shutdownRequested) return
 
@@ -97,6 +100,12 @@ export class QueueManager {
         const isFifo = qname.endsWith('.fifo')
         return this.opt.fifo ? isFifo : true
       })
+      // next dead
+      .filter(({ qname, qrl }) => {
+        const isFifo = qname.endsWith('.fifo')
+        const isDead = isFifo ? qname.endsWith('_dead.fifo') : qname.endsWith('_dead')
+        return !isDead
+      })
       // then icehouse
       .filter(({ qname, qrl }) => !this.keepInIcehouse(qrl, now))