qdone 2.0.29-alpha → 2.0.31-alpha

package/commonjs/src/scheduler/jobExecutor.js

@@ -3,60 +3,27 @@
  * Component to manage all the currently executing jobs, including extending
  * their visibility timeouts and deleting them when they are successful.
  */
-var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
-    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
-    return new (P || (P = Promise))(function (resolve, reject) {
-        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
-        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
-        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
-        step((generator = generator.apply(thisArg, _arguments || [])).next());
-    });
-};
-var __generator = (this && this.__generator) || function (thisArg, body) {
-    var _ = { label: 0, sent: function() { if (t[0] & 1) throw t[1]; return t[1]; }, trys: [], ops: [] }, f, y, t, g;
-    return g = { next: verb(0), "throw": verb(1), "return": verb(2) }, typeof Symbol === "function" && (g[Symbol.iterator] = function() { return this; }), g;
-    function verb(n) { return function (v) { return step([n, v]); }; }
-    function step(op) {
-        if (f) throw new TypeError("Generator is already executing.");
-        while (g && (g = 0, op[0] && (_ = 0)), _) try {
-            if (f = 1, y && (t = op[0] & 2 ? y["return"] : op[0] ? y["throw"] || ((t = y["return"]) && t.call(y), 0) : y.next) && !(t = t.call(y, op[1])).done) return t;
-            if (y = 0, t) op = [op[0] & 2, t.value];
-            switch (op[0]) {
-                case 0: case 1: t = op; break;
-                case 4: _.label++; return { value: op[1], done: false };
-                case 5: _.label++; y = op[1]; op = [0]; continue;
-                case 7: op = _.ops.pop(); _.trys.pop(); continue;
-                default:
-                    if (!(t = _.trys, t = t.length > 0 && t[t.length - 1]) && (op[0] === 6 || op[0] === 2)) { _ = 0; continue; }
-                    if (op[0] === 3 && (!t || (op[1] > t[0] && op[1] < t[3]))) { _.label = op[1]; break; }
-                    if (op[0] === 6 && _.label < t[1]) { _.label = t[1]; t = op; break; }
-                    if (t && _.label < t[2]) { _.label = t[2]; _.ops.push(op); break; }
-                    if (t[2]) _.ops.pop();
-                    _.trys.pop(); continue;
-            }
-            op = body.call(thisArg, _);
-        } catch (e) { op = [6, e]; y = 0; } finally { f = t = 0; }
-        if (op[0] & 5) throw op[1]; return { value: op[0] ? op[1] : void 0, done: true };
-    }
-};
 var __importDefault = (this && this.__importDefault) || function (mod) {
     return (mod && mod.__esModule) ? mod : { "default": mod };
 };
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.JobExecutor = void 0;
-var client_sqs_1 = require("@aws-sdk/client-sqs");
-var chalk_1 = __importDefault(require("chalk"));
-var debug_1 = __importDefault(require("debug"));
-var sqs_js_1 = require("../sqs.js");
-var debug = (0, debug_1.default)('qdone:jobExecutor');
-var maxJobSeconds = 12 * 60 * 60;
-var JobExecutor = /** @class */ (function () {
-    function JobExecutor(opt) {
+const client_sqs_1 = require("@aws-sdk/client-sqs");
+const chalk_1 = __importDefault(require("chalk"));
+const debug_1 = __importDefault(require("debug"));
+const sqs_js_1 = require("../sqs.js");
+const debug = (0, debug_1.default)('qdone:jobExecutor');
+const maxJobSeconds = 12 * 60 * 60;
+class JobExecutor {
+    constructor(opt) {
         this.opt = opt;
         this.jobs = [];
         this.jobsByMessageId = {};
+        this.jobsByQueue = new Map();
         this.stats = {
             activeJobs: 0,
+            waitingJobs: 0,
+            runningJobs: 0,
             sqsCalls: 0,
             timeoutsExtended: 0,
             jobsSucceeded: 0,
@@ -66,330 +33,324 @@ var JobExecutor = /** @class */ (function () {
         this.maintainPromise = this.maintainVisibility();
         debug({ this: this });
     }
-    JobExecutor.prototype.shutdown = function () {
-        return __awaiter(this, void 0, void 0, function () {
-            return __generator(this, function (_a) {
-                switch (_a.label) {
-                    case 0:
-                        this.shutdownRequested = true;
-                        // Trigger a maintenance run right away in case it speeds us up
-                        clearTimeout(this.maintainVisibilityTimeout);
-                        if (this.opt.verbose) {
-                            console.error(chalk_1.default.blue('Shutting down jobExecutor'));
-                        }
-                        return [4 /*yield*/, this.maintainPromise];
-                    case 1:
-                        _a.sent();
-                        return [4 /*yield*/, this.maintainVisibility()];
-                    case 2:
-                        _a.sent();
-                        return [2 /*return*/];
-                }
-            });
-        });
-    };
-    JobExecutor.prototype.activeJobCount = function () {
+    async shutdown() {
+        this.shutdownRequested = true;
+        // Trigger a maintenance run right away in case it speeds us up
+        clearTimeout(this.maintainVisibilityTimeout);
+        if (this.opt.verbose) {
+            console.error(chalk_1.default.blue('Shutting down jobExecutor'));
+        }
+        await this.maintainPromise;
+        await this.maintainVisibility();
+    }
+    activeJobCount() {
         return this.stats.activeJobs;
-    };
+    }
+    runningJobCount() {
+        return this.stats.runningJobs;
+    }
+    /**
+     * Returns the number of jobs running in a queue.
+     */
+    runningJobCountForQueue(qname) {
+        const jobs = this.jobsByQueue.get(qname) || new Set();
+        let runningCount = 0;
+        for (const job of jobs.values())
+            runningCount += job.status === 'running';
+        return runningCount;
+    }
     /**
      * Changes message visibility on all running jobs using as few calls as possible.
      */
-    JobExecutor.prototype.maintainVisibility = function () {
-        return __awaiter(this, void 0, void 0, function () {
-            var nextCheckInMs, start, jobsToExtendByQrl, jobsToDeleteByQrl, jobsToCleanup, i, job, jobRunTime, jobsToDelete, jobsToExtend, doubled, secondsUntilMax, _a, _b, _c, _i, qrl, jobsToExtend, entries, messageId, job, entry, input, result, _d, _e, failed, count, _f, _g, _h, _j, qrl, jobsToDelete, entries, messageId, job, entry, input, result, _k, _l, failed, count;
-            var _this = this;
-            return __generator(this, function (_m) {
-                switch (_m.label) {
-                    case 0:
-                        // Bail if we are shutting down
-                        if (this.shutdownRequested && this.stats.activeJobs === 0 && this.jobs.length === 0) {
-                            if (this.opt.verbose) {
-                                console.error(chalk_1.default.blue('All workers done, finishing shutdown of jobExecutor'));
-                            }
-                            return [2 /*return*/];
-                        }
-                        // Reset our timeout
-                        clearTimeout(this.maintainVisibilityTimeout);
-                        nextCheckInMs = this.shutdownRequested ? 1000 : 10 * 1000;
-                        this.maintainVisibilityTimeout = setTimeout(function () {
-                            _this.maintainPromise = _this.maintainVisibility();
-                        }, nextCheckInMs);
-                        start = new Date();
-                        jobsToExtendByQrl = {};
-                        jobsToDeleteByQrl = {};
-                        jobsToCleanup = new Set();
-                        if (this.opt.verbose) {
-                            console.error(chalk_1.default.blue('Stats: '), this.stats);
-                            console.error(chalk_1.default.blue('Running: '), this.jobs.filter(function (j) { return j.status === 'processing'; }).map(function (_a) {
-                                var qname = _a.qname, message = _a.message;
-                                return ({ qname: qname, payload: message.Body });
-                            }));
-                        }
-                        // Build list of jobs we need to deal with
-                        for (i = 0; i < this.jobs.length; i++) {
-                            job = this.jobs[i];
-                            jobRunTime = Math.round((start - job.start) / 1000);
-                            // debug('considering job', job)
-                            if (job.status === 'complete') {
-                                jobsToDelete = jobsToDeleteByQrl[job.qrl] || [];
-                                job.status = 'deleting';
-                                jobsToDelete.push(job);
-                                jobsToDeleteByQrl[job.qrl] = jobsToDelete;
-                            }
-                            else if (job.status === 'failed') {
-                                jobsToCleanup.add(job);
-                            }
-                            else if (job.status === 'processing') {
-                                debug('processing', { job: job, jobRunTime: jobRunTime });
-                                if (jobRunTime >= job.extendAtSecond) {
-                                    jobsToExtend = jobsToExtendByQrl[job.qrl] || [];
-                                    jobsToExtend.push(job);
-                                    jobsToExtendByQrl[job.qrl] = jobsToExtend;
-                                    doubled = job.visibilityTimeout * 2;
-                                    secondsUntilMax = Math.max(1, maxJobSeconds - jobRunTime);
-                                    // const secondsUntilKill = Math.max(1, this.opt.killAfter - jobRunTime)
-                                    job.visibilityTimeout = Math.min(doubled, secondsUntilMax); //, secondsUntilKill)
-                                    job.extendAtSecond = Math.round(jobRunTime + job.visibilityTimeout / 2); // this is what we use next time
-                                    debug({ doubled: doubled, secondsUntilMax: secondsUntilMax, job: job });
-                                }
-                            }
-                        }
-                        _a = jobsToExtendByQrl;
-                        _b = [];
-                        for (_c in _a)
-                            _b.push(_c);
-                        _i = 0;
-                        _m.label = 1;
-                    case 1:
-                        if (!(_i < _b.length)) return [3 /*break*/, 5];
-                        _c = _b[_i];
-                        if (!(_c in _a)) return [3 /*break*/, 4];
-                        qrl = _c;
-                        jobsToExtend = jobsToExtendByQrl[qrl];
-                        debug({ qrl: qrl, jobsToExtend: jobsToExtend });
-                        _m.label = 2;
-                    case 2:
-                        if (!jobsToExtend.length) return [3 /*break*/, 4];
-                        entries = [];
-                        messageId = 0;
-                        while (messageId++ < 10 && jobsToExtend.length) {
-                            job = jobsToExtend.shift();
-                            entry = {
-                                Id: job.message.MessageId,
-                                ReceiptHandle: job.message.ReceiptHandle,
-                                VisibilityTimeout: job.visibilityTimeout
-                            };
-                            entries.push(entry);
-                        }
-                        debug({ entries: entries });
-                        input = { QueueUrl: qrl, Entries: entries };
-                        debug({ ChangeMessageVisibilityBatch: input });
-                        return [4 /*yield*/, (0, sqs_js_1.getSQSClient)().send(new client_sqs_1.ChangeMessageVisibilityBatchCommand(input))];
-                    case 3:
-                        result = _m.sent();
-                        debug('ChangeMessageVisibilityBatch returned', result);
-                        this.stats.sqsCalls++;
-                        if (result.Failed) {
-                            console.error('FAILED_MESSAGES', result.Failed);
-                            for (_d = 0, _e = result.Failed; _d < _e.length; _d++) {
-                                failed = _e[_d];
-                                console.error('FAILED_TO_EXTEND_JOB', this.jobsByMessageId[failed.Id]);
-                            }
-                        }
-                        if (result.Successful) {
-                            count = result.Successful.length || 0;
-                            this.stats.timeoutsExtended += count;
-                            if (this.opt.verbose) {
-                                console.error(chalk_1.default.blue('Extended'), count, chalk_1.default.blue('jobs'));
-                            }
-                            else if (!this.opt.disableLog) {
-                                console.log(JSON.stringify({ event: 'EXTEND_VISIBILITY_TIMEOUTS', timestamp: start, count: count, qrl: qrl }));
-                            }
-                        }
-                        return [3 /*break*/, 2];
-                    case 4:
-                        _i++;
-                        return [3 /*break*/, 1];
-                    case 5:
-                        _f = jobsToDeleteByQrl;
-                        _g = [];
-                        for (_h in _f)
-                            _g.push(_h);
-                        _j = 0;
-                        _m.label = 6;
-                    case 6:
-                        if (!(_j < _g.length)) return [3 /*break*/, 10];
-                        _h = _g[_j];
-                        if (!(_h in _f)) return [3 /*break*/, 9];
-                        qrl = _h;
-                        jobsToDelete = jobsToDeleteByQrl[qrl];
-                        _m.label = 7;
-                    case 7:
-                        if (!jobsToDelete.length) return [3 /*break*/, 9];
-                        entries = [];
-                        messageId = 0;
-                        while (messageId++ < 10 && jobsToDelete.length) {
-                            job = jobsToDelete.shift();
-                            entry = {
-                                Id: job.message.MessageId,
-                                ReceiptHandle: job.message.ReceiptHandle,
-                                VisibilityTimeout: job.visibilityTimeout
-                            };
-                            entries.push(entry);
-                        }
-                        debug({ entries: entries });
-                        input = { QueueUrl: qrl, Entries: entries };
-                        debug({ DeleteMessageBatch: input });
-                        return [4 /*yield*/, (0, sqs_js_1.getSQSClient)().send(new client_sqs_1.DeleteMessageBatchCommand(input))];
-                    case 8:
-                        result = _m.sent();
-                        this.stats.sqsCalls++;
-                        if (result.Failed) {
-                            console.error('FAILED_MESSAGES', result.Failed);
-                            for (_k = 0, _l = result.Failed; _k < _l.length; _k++) {
-                                failed = _l[_k];
-                                console.error('FAILED_TO_DELETE_JOB', this.jobsByMessageId[failed.Id]);
-                            }
-                        }
-                        if (result.Successful) {
-                            count = result.Successful.length || 0;
-                            this.stats.jobsDeleted += count;
-                            if (this.opt.verbose) {
-                                console.error(chalk_1.default.blue('Deleted'), count, chalk_1.default.blue('jobs'));
-                            }
-                            else if (!this.opt.disableLog) {
-                                console.log(JSON.stringify({ event: 'DELETE_MESSAGES', timestamp: start, count: count, qrl: qrl }));
-                            }
-                        }
-                        debug('DeleteMessageBatch returned', result);
-                        return [3 /*break*/, 7];
-                    case 9:
-                        _j++;
-                        return [3 /*break*/, 6];
-                    case 10:
-                        // Get rid of deleted and failed jobs
-                        this.jobs = this.jobs.filter(function (job) {
-                            if (job.status === 'deleting' || job.status === 'failed') {
-                                debug('removed', job.message.MessageId);
-                                delete _this.jobsByMessageId[job.message.MessageId];
-                                return false;
-                            }
-                            else {
-                                return true;
-                            }
-                        });
-                        return [2 /*return*/];
+    async maintainVisibility() {
+        // Bail if we are shutting down
+        if (this.shutdownRequested && this.stats.activeJobs === 0 && this.jobs.length === 0) {
+            if (this.opt.verbose) {
+                console.error(chalk_1.default.blue('All workers done, finishing shutdown of jobExecutor'));
+            }
+            return;
+        }
+        // Reset our timeout
+        clearTimeout(this.maintainVisibilityTimeout);
+        const nextCheckInMs = this.shutdownRequested ? 1000 : 10 * 1000;
+        this.maintainVisibilityTimeout = setTimeout(() => {
+            this.maintainPromise = this.maintainVisibility();
+        }, nextCheckInMs);
+        // debug('maintainVisibility', this.jobs)
+        const start = new Date();
+        const jobsToExtendByQrl = {};
+        const jobsToDeleteByQrl = {};
+        const jobsToCleanup = new Set();
+        // Build list of jobs we need to deal with
+        const jobStatuses = {};
+        for (let i = 0; i < this.jobs.length; i++) {
+            const job = this.jobs[i];
+            const jobRunTime = Math.round((start - job.start) / 1000);
+            jobStatuses[job.status] = (jobStatuses[job.status] || 0) + 1;
+            // debug('considering job', job)
+            if (job.status === 'complete') {
+                const jobsToDelete = jobsToDeleteByQrl[job.qrl] || [];
+                job.status = 'deleting';
+                jobsToDelete.push(job);
+                jobsToDeleteByQrl[job.qrl] = jobsToDelete;
+            }
+            else if (job.status === 'failed') {
+                jobsToCleanup.add(job);
+            }
+            else if (job.status !== 'deleting') {
+                // Any other job state gets visibility accounting
+                debug('processing', { job, jobRunTime });
+                if (jobRunTime >= job.extendAtSecond) {
+                    // Add it to our organized list of jobs
+                    const jobsToExtend = jobsToExtendByQrl[job.qrl] || [];
+                    jobsToExtend.push(job);
+                    jobsToExtendByQrl[job.qrl] = jobsToExtend;
+                    // Update the visibility timeout, double every time, up to max
+                    const doubled = job.visibilityTimeout * 2;
+                    const secondsUntilMax = Math.max(1, maxJobSeconds - jobRunTime);
+                    // const secondsUntilKill = Math.max(1, this.opt.killAfter - jobRunTime)
+                    job.visibilityTimeout = Math.min(doubled, secondsUntilMax); //, secondsUntilKill)
+                    job.extendAtSecond = Math.round(jobRunTime + job.visibilityTimeout / 2); // this is what we use next time
+                    debug({ doubled, secondsUntilMax, job });
                 }
-            });
-        });
-    };
-    JobExecutor.prototype.executeJob = function (message, callback, qname, qrl, failedCallback) {
-        return __awaiter(this, void 0, void 0, function () {
-            var payload, visibilityTimeout, job, oldJob, e, queue, result, err_1;
-            return __generator(this, function (_a) {
-                switch (_a.label) {
-                    case 0:
-                        if (this.shutdownRequested)
-                            throw new Error('jobExecutor is shutting down so cannot execute new job');
-                        payload = this.opt.json ? JSON.parse(message.Body) : message.Body;
-                        visibilityTimeout = 60;
-                        job = {
-                            status: 'processing',
-                            start: new Date(),
-                            visibilityTimeout: visibilityTimeout,
-                            extendAtSecond: visibilityTimeout / 2,
-                            payload: this.opt.json ? JSON.parse(message.Body) : message.Body,
-                            message: message,
-                            callback: callback,
-                            qname: qname,
-                            qrl: qrl
-                        };
-                        oldJob = this.jobsByMessageId[job.message.MessageId];
-                        if (oldJob) {
-                            // If we actually see the same job again, we fucked up, probably due to
-                            // the system being overloaded and us missing our extension call. So
-                            // we'll celebrate this occasion by throwing a big fat error.
-                            debug({ oldJob: oldJob });
-                            e = new Error("Saw job ".concat(oldJob.message.MessageId, " twice"));
-                            e.job = oldJob;
-                            // TODO: sentry breadcrumb
-                            throw e;
-                        }
-                        // debug('executeJob', job)
-                        this.jobs.push(job);
-                        this.jobsByMessageId[job.message.MessageId] = job;
-                        this.stats.activeJobs++;
-                        if (this.opt.verbose) {
-                            console.error(chalk_1.default.blue('Executing:'), qname, chalk_1.default.blue('-->'), job.payload);
-                        }
-                        else if (!this.opt.disableLog) {
-                            console.log(JSON.stringify({
-                                event: 'MESSAGE_PROCESSING_START',
-                                timestamp: new Date(),
-                                qrl: qrl,
-                                messageId: message.MessageId,
-                                payload: job.payload
-                            }));
-                        }
-                        _a.label = 1;
-                    case 1:
-                        _a.trys.push([1, 3, , 4]);
-                        queue = qname.slice(this.opt.prefix.length);
-                        return [4 /*yield*/, callback(queue, payload)];
-                    case 2:
-                        result = _a.sent();
-                        debug('executeJob callback finished', { payload: payload, result: result });
-                        if (this.opt.verbose) {
-                            console.error(chalk_1.default.green('SUCCESS'), message.Body);
-                        }
-                        job.status = 'complete';
-                        if (this.opt.verbose) {
-                            console.error(chalk_1.default.blue('  done'));
-                            console.error();
-                        }
-                        else if (!this.opt.disableLog) {
-                            console.log(JSON.stringify({
-                                event: 'MESSAGE_PROCESSING_COMPLETE',
-                                timestamp: new Date(),
-                                messageId: message.MessageId,
-                                payload: payload
-                            }));
-                        }
-                        this.stats.jobsSucceeded++;
-                        return [3 /*break*/, 4];
-                    case 3:
-                        err_1 = _a.sent();
-                        // Notify caller that we failed
-                        if (failedCallback)
-                            failedCallback(message, qname, qrl);
-                        // Fail path for job execution
-                        if (this.opt.verbose) {
-                            console.error(chalk_1.default.red('FAILED'), message.Body);
-                            console.error(chalk_1.default.blue('  error : ') + err_1);
-                        }
-                        else if (!this.opt.disableLog) {
-                            // Production error logging
-                            console.log(JSON.stringify({
-                                event: 'MESSAGE_PROCESSING_FAILED',
-                                reason: 'exception thrown',
-                                qrl: qrl,
-                                timestamp: new Date(),
-                                messageId: message.MessageId,
-                                payload: payload,
-                                errorMessage: err_1.toString().split('\n').slice(1).join('\n').trim() || undefined,
-                                err: err_1
-                            }));
-                        }
-                        job.status = 'failed';
-                        this.stats.jobsFailed++;
-                        return [3 /*break*/, 4];
-                    case 4:
-                        this.stats.activeJobs--;
-                        return [2 /*return*/];
+            }
+        }
+        if (this.opt.verbose) {
+            console.error(chalk_1.default.blue('Stats: '), { stats: this.stats, jobStatuses });
+            console.error(chalk_1.default.blue('Running: '), this.jobs.filter(j => j.status === 'processing').map(({ qname, message }) => ({ qname, payload: message.Body })));
+        }
+        // Extend in batches for each queue
+        for (const qrl in jobsToExtendByQrl) {
+            const jobsToExtend = jobsToExtendByQrl[qrl];
+            debug({ qrl, jobsToExtend });
+            while (jobsToExtend.length) {
+                // Build list of messages to go in this batch
+                const entries = [];
+                let messageId = 0;
+                while (messageId++ < 10 && jobsToExtend.length) {
+                    const job = jobsToExtend.shift();
+                    const entry = {
+                        Id: job.message.MessageId,
+                        ReceiptHandle: job.message.ReceiptHandle,
+                        VisibilityTimeout: job.visibilityTimeout
+                    };
+                    entries.push(entry);
+                }
+                debug({ entries });
+                // Change batch
+                const input = { QueueUrl: qrl, Entries: entries };
+                debug({ ChangeMessageVisibilityBatch: input });
+                const result = await (0, sqs_js_1.getSQSClient)().send(new client_sqs_1.ChangeMessageVisibilityBatchCommand(input));
+                debug('ChangeMessageVisibilityBatch returned', result);
+                this.stats.sqsCalls++;
+                if (result.Failed) {
+                    console.error('FAILED_MESSAGES', result.Failed);
+                    for (const failed of result.Failed) {
+                        console.error('FAILED_TO_EXTEND_JOB', this.jobsByMessageId[failed.Id]);
+                    }
+                }
+                if (result.Successful) {
+                    const count = result.Successful.length || 0;
+                    this.stats.timeoutsExtended += count;
+                    if (this.opt.verbose) {
+                        console.error(chalk_1.default.blue('Extended'), count, chalk_1.default.blue('jobs'));
+                    }
+                    else if (!this.opt.disableLog) {
+                        console.log(JSON.stringify({ event: 'EXTEND_VISIBILITY_TIMEOUTS', timestamp: start, count, qrl }));
+                    }
                 }
-            });
+                // TODO Sentry
+            }
+        }
+        // Delete in batches for each queue
+        for (const qrl in jobsToDeleteByQrl) {
+            const jobsToDelete = jobsToDeleteByQrl[qrl];
+            while (jobsToDelete.length) {
+                // Build list of messages to go in this batch
+                const entries = [];
+                let messageId = 0;
+                while (messageId++ < 10 && jobsToDelete.length) {
+                    const job = jobsToDelete.shift();
+                    const entry = {
+                        Id: job.message.MessageId,
+                        ReceiptHandle: job.message.ReceiptHandle,
+                        VisibilityTimeout: job.visibilityTimeout
+                    };
+                    entries.push(entry);
+                }
+                debug({ entries });
+                // Delete batch
+                const input = { QueueUrl: qrl, Entries: entries };
+                debug({ DeleteMessageBatch: input });
+                const result = await (0, sqs_js_1.getSQSClient)().send(new client_sqs_1.DeleteMessageBatchCommand(input));
+                this.stats.sqsCalls++;
+                if (result.Failed) {
+                    console.error('FAILED_MESSAGES', result.Failed);
+                    for (const failed of result.Failed) {
+                        console.error('FAILED_TO_DELETE_JOB', this.jobsByMessageId[failed.Id]);
+                    }
+                }
+                if (result.Successful) {
+                    const count = result.Successful.length || 0;
+                    this.stats.jobsDeleted += count;
+                    if (this.opt.verbose) {
+                        console.error(chalk_1.default.blue('Deleted'), count, chalk_1.default.blue('jobs'));
+                    }
+                    else if (!this.opt.disableLog) {
+                        console.log(JSON.stringify({ event: 'DELETE_MESSAGES', timestamp: start, count, qrl }));
+                    }
+                }
+                debug('DeleteMessageBatch returned', result);
+                // TODO Sentry
+            }
+        }
+        // Get rid of deleted and failed jobs
+        this.jobs = this.jobs.filter(job => {
+            if (job.status === 'deleting' || job.status === 'failed') {
+                debug('removed', job.message.MessageId);
+                // Accounting
+                delete this.jobsByMessageId[job.message.MessageId];
+                this.jobsByQueue.get(job.qname).delete(job);
+                return false;
+            }
+            else {
+                return true;
+            }
         });
-    };
-    return JobExecutor;
-}());
+    }
+    addJob(message, callback, qname, qrl) {
+        // Create job entry and track it
+        const defaultVisibilityTimeout = 60;
+        const job = {
+            status: 'waiting',
+            start: new Date(),
+            visibilityTimeout: defaultVisibilityTimeout,
+            extendAtSecond: defaultVisibilityTimeout / 2,
+            payload: this.opt.json ? JSON.parse(message.Body) : message.Body,
+            message,
+            callback,
+            qname,
+            prettyQname: qname.slice(this.opt.prefix.length),
+            qrl
+        };
+        // See if we are already executing this job
+        const oldJob = this.jobsByMessageId[job.message.MessageId];
+        if (oldJob) {
+            // If we actually see the same job again, we fucked up, probably due to
+            // the system being overloaded and us missing our extension call. So
+            // we'll celebrate this occasion by throwing a big fat error.
+            debug({ oldJob });
+            const e = new Error(`Saw job ${oldJob.message.MessageId} twice`);
+            e.job = oldJob;
+            // TODO: sentry breadcrumb
+            throw e;
+        }
+        // Accounting
+        this.jobs.push(job);
+        this.jobsByMessageId[job.message.MessageId] = job;
+        // Track all jobs for each queue
+        if (!this.jobsByQueue.has(job.qname))
+            this.jobsByQueue.set(job.qname, new Set());
+        this.jobsByQueue.get(job.qname).add(job);
+        this.stats.activeJobs++;
+        this.stats.waitingJobs++;
+        if (this.opt.verbose) {
+            console.error(chalk_1.default.blue('Got message:'), job.prettyQname, chalk_1.default.blue('-->'), job.payload, job.message.MessageId);
+        }
+        else if (!this.opt.disableLog) {
+            console.log(JSON.stringify({
+                event: 'MESSAGE_RECEIVED',
+                timestamp: new Date(),
+                queue: job.qname,
+                messageId: message.MessageId,
+                payload: job.payload
+            }));
+        }
+        return job;
+    }
+    async runJob(job) {
+        try {
+            if (this.opt.verbose) {
+                console.error(chalk_1.default.blue('Running:'), job.prettyQname, chalk_1.default.blue('-->'), job.payload, job.message.MessageId);
+            }
+            else if (!this.opt.disableLog) {
+                console.log(JSON.stringify({
+                    event: 'MESSAGE_PROCESSING_START',
+                    timestamp: new Date(),
+                    queue: job.qname,
+                    messageId: job.message.MessageId,
+                    payload: job.payload
+                }));
+            }
+            job.status = 'running';
+            this.stats.runningJobs++;
+            this.stats.waitingJobs--;
+            const queue = job.qname.slice(this.opt.prefix.length);
+            const result = await job.callback(queue, job.payload);
+            debug('executeJob callback finished', { payload: job.payload, result });
+            if (this.opt.verbose) {
+                console.error(chalk_1.default.green('SUCCESS'), job.payload);
+            }
+            job.status = 'complete';
+            if (this.opt.verbose) {
+                console.error(chalk_1.default.blue('  done'));
+                console.error();
+            }
+            else if (!this.opt.disableLog) {
+                console.log(JSON.stringify({
+                    event: 'MESSAGE_PROCESSING_COMPLETE',
+                    queue: job.qname,
+                    timestamp: new Date(),
+                    messageId: job.message.MessageId,
+                    payload: job.payload
+                }));
+            }
+            this.stats.jobsSucceeded++;
+        }
+        catch (err) {
+            job.status = 'failed';
+            this.stats.jobsFailed++;
+            // Fail path for job execution
+            if (this.opt.verbose) {
+                console.error(chalk_1.default.red('FAILED'), job.payload);
+                console.error(chalk_1.default.blue('  error : ') + err);
+            }
+            else if (!this.opt.disableLog) {
+                // Production error logging
+                console.log(JSON.stringify({
+                    event: 'MESSAGE_PROCESSING_FAILED',
+                    reason: 'exception thrown',
+                    queue: job.qname,
+                    timestamp: new Date(),
+                    messageId: job.message.MessageId,
+                    payload: job.payload,
+                    errorMessage: err.toString().split('\n').slice(1).join('\n').trim() || undefined,
+                    err
+                }));
+            }
+        }
+        this.stats.activeJobs--;
+        this.stats.runningJobs--;
+    }
+    async executeJobs(messages, callback, qname, qrl) {
+        if (this.shutdownRequested)
+            throw new Error('jobExecutor is shutting down so cannot execute new jobs');
+        // Begin tracking jobs
+        const jobs = messages.map(message => this.addJob(message, callback, qname, qrl));
+        const isFifo = qrl.endsWith('.fifo');
+        // console.log(jobs)
+        // Begin executing
+        for (const [job, i] of jobs.map((job, i) => [job, i])) {
+            // Figure out if the next job needs to happen in serial, otherwise we can parallel execute
+            const nextJob = jobs[i + 1];
+            const nextJobIsSerial = isFifo && nextJob && job.message?.Attributes?.GroupId === nextJob.message?.Attributes?.GroupId;
+            // console.log({ i, nextJobAtt: nextJob?.message?.Attributes, nextJobIsSerial })
+            // Execute serial or parallel
+            if (nextJobIsSerial)
+                await this.runJob(job);
+            else
+                this.runJob(job);
+        }
+    }
+}
 exports.JobExecutor = JobExecutor;