qdone 2.1.0 → 2.1.1

package/commonjs/src/scheduler/jobExecutor.js

@@ -0,0 +1,383 @@
+"use strict";
+/**
+ * Component to manage all the currently executing jobs, including extending
+ * their visibility timeouts and deleting them when they are successful.
+ */
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.JobExecutor = void 0;
+const client_sqs_1 = require("@aws-sdk/client-sqs");
+const chalk_1 = __importDefault(require("chalk"));
+const debug_1 = __importDefault(require("debug"));
+const dedup_js_1 = require("../dedup.js");
+const sqs_js_1 = require("../sqs.js");
+const debug = (0, debug_1.default)('qdone:jobExecutor');
+const maxJobSeconds = 12 * 60 * 60;
+class JobExecutor {
+    constructor(opt) {
+        this.opt = opt;
+        this.jobs = []; // for full traversals
+        this.jobsByMessageId = {}; // for looking up via message id
+        this.jobsByQueue = new Map(); // for looking up via queue name
+        this.stats = {
+            activeJobs: 0,
+            waitingJobs: 0,
+            runningJobs: 0,
+            sqsCalls: 0,
+            timeoutsExtended: 0,
+            jobsSucceeded: 0,
+            jobsFailed: 0,
+            jobsDeleted: 0
+        };
+        this.maintainPromise = this.maintainVisibility();
+        debug({ this: this });
+    }
+    async shutdown() {
+        this.shutdownRequested = true;
+        // Trigger a maintenance run right away in case it speeds us up
+        clearTimeout(this.maintainVisibilityTimeout);
+        if (this.opt.verbose) {
+            console.error(chalk_1.default.blue('Shutting down jobExecutor'));
+        }
+        await this.maintainPromise;
+        await this.maintainVisibility();
+    }
+    activeJobCount() {
+        return this.stats.activeJobs;
+    }
+    runningJobCount() {
+        return this.stats.runningJobs;
+    }
+    /**
+     * Returns the number of jobs running in a queue.
+     */
+    runningJobCountForQueue(qname) {
+        const jobs = this.jobsByQueue.get(qname) || new Set();
+        let runningCount = 0;
+        for (const job of jobs.values())
+            runningCount += job.status === 'running';
+        return runningCount;
+    }
+    /**
+     * Changes message visibility on all running jobs using as few calls as possible.
+     */
+    async maintainVisibility() {
+        // Bail if we are shutting down
+        if (this.shutdownRequested && this.stats.activeJobs === 0 && this.jobs.length === 0) {
+            if (this.opt.verbose) {
+                console.error(chalk_1.default.blue('All workers done, finishing shutdown of jobExecutor'));
+            }
+            return;
+        }
+        // Reset our timeout
+        clearTimeout(this.maintainVisibilityTimeout);
+        const nextCheckInMs = this.shutdownRequested ? 1000 : 10 * 1000;
+        this.maintainVisibilityTimeout = setTimeout(() => {
+            this.maintainPromise = this.maintainVisibility();
+        }, nextCheckInMs);
+        // debug('maintainVisibility', this.jobs)
+        const start = new Date();
+        const jobsToExtendByQrl = {};
+        const jobsToDeleteByQrl = {};
+        const jobsToCleanup = new Set();
+        // Build list of jobs we need to deal with
+        const jobStatuses = {};
+        for (let i = 0; i < this.jobs.length; i++) {
+            const job = this.jobs[i];
+            const jobRunTime = Math.round((start - job.start) / 1000);
+            jobStatuses[job.status] = (jobStatuses[job.status] || 0) + 1;
+            // debug('considering job', job)
+            if (job.status === 'complete') {
+                const jobsToDelete = jobsToDeleteByQrl[job.qrl] || [];
+                job.status = 'deleting';
+                jobsToDelete.push(job);
+                jobsToDeleteByQrl[job.qrl] = jobsToDelete;
+            }
+            else if (job.status === 'failed') {
+                jobsToCleanup.add(job);
+            }
+            else if (job.status !== 'deleting') {
+                // Any other job state gets visibility accounting
+                debug('processing', { job, jobRunTime });
+                if (jobRunTime >= job.extendAtSecond) {
+                    // Add it to our organized list of jobs
+                    const jobsToExtend = jobsToExtendByQrl[job.qrl] || [];
+                    jobsToExtend.push(job);
+                    jobsToExtendByQrl[job.qrl] = jobsToExtend;
+                    // Update the visibility timeout, double every time, up to max
+                    const doubled = job.visibilityTimeout * 2;
+                    const secondsUntilMax = Math.max(1, maxJobSeconds - jobRunTime);
+                    // const secondsUntilKill = Math.max(1, this.opt.killAfter - jobRunTime)
+                    job.visibilityTimeout = Math.min(doubled, secondsUntilMax); //, secondsUntilKill)
+                    job.extendAtSecond = Math.round(jobRunTime + job.visibilityTimeout / 2); // this is what we use next time
+                    debug({ doubled, secondsUntilMax, job });
+                }
+            }
+        }
+        if (this.opt.verbose) {
+            console.error(chalk_1.default.blue('Stats: '), { stats: this.stats, jobStatuses });
+            console.error(chalk_1.default.blue('Jobs: '));
+            for (const [qname, jobs] of this.jobsByQueue.entries()) {
+                if (jobs.size) {
+                    const stats = {};
+                    for (const job of jobs)
+                        stats[job.status] = (stats[job.status] || 0) + 1;
+                    console.error(chalk_1.default.blue('  queue:'), qname);
+                    if (stats.running)
+                        console.error(chalk_1.default.green('    running: '), stats.running);
+                    if (stats.waiting)
+                        console.error(chalk_1.default.yellow('    waiting: '), stats.waiting);
+                    if (stats.deleting)
+                        console.error(chalk_1.default.blue('    complete: '), stats.deleting);
+                    if (stats.failed)
+                        console.error(chalk_1.default.red('    failed: '), stats.failed);
+                }
+            }
+        }
+        // Extend in batches for each queue
+        for (const qrl in jobsToExtendByQrl) {
+            const jobsToExtend = jobsToExtendByQrl[qrl];
+            debug({ qrl, jobsToExtend });
+            while (jobsToExtend.length) {
+                // Build list of messages to go in this batch
+                const entries = [];
+                let messageId = 0;
+                while (messageId++ < 10 && jobsToExtend.length) {
+                    const job = jobsToExtend.shift();
+                    const entry = {
+                        Id: job.message.MessageId,
+                        ReceiptHandle: job.message.ReceiptHandle,
+                        VisibilityTimeout: job.visibilityTimeout
+                    };
+                    entries.push(entry);
+                }
+                debug({ entries });
+                // Change batch
+                const input = { QueueUrl: qrl, Entries: entries };
+                debug({ ChangeMessageVisibilityBatch: input });
+                const result = await (0, sqs_js_1.getSQSClient)().send(new client_sqs_1.ChangeMessageVisibilityBatchCommand(input));
+                debug('ChangeMessageVisibilityBatch returned', result);
+                this.stats.sqsCalls++;
+                if (result.Failed) {
+                    console.error('FAILED_MESSAGES', result.Failed);
+                    for (const failed of result.Failed) {
+                        console.error('FAILED_TO_EXTEND_JOB', { failedEntry: failed, job: this.jobsByMessageId[failed.Id] });
+                        // ensure that we clean this one up so it doesn't generate api calls
+                        if (this.jobsByMessageId[failed.Id])
+                            this.jobsByMessageId[failed.Id].status = 'failed';
+                    }
+                }
+                if (result.Successful) {
+                    const count = result.Successful.length || 0;
+                    this.stats.timeoutsExtended += count;
+                    if (this.opt.verbose) {
+                        console.error(chalk_1.default.blue('Extended'), count, chalk_1.default.blue('jobs'));
+                    }
+                    else if (!this.opt.disableLog) {
+                        console.log(JSON.stringify({ event: 'EXTEND_VISIBILITY_TIMEOUTS', timestamp: start, count, qrl }));
+                    }
+                }
+                // TODO Sentry
+            }
+        }
+        // Delete in batches for each queue
+        for (const qrl in jobsToDeleteByQrl) {
+            const jobsToDelete = jobsToDeleteByQrl[qrl];
+            while (jobsToDelete.length) {
+                // Build list of messages to go in this batch
+                const entries = [];
+                let messageId = 0;
+                while (messageId++ < 10 && jobsToDelete.length) {
+                    const job = jobsToDelete.shift();
+                    const entry = {
+                        Id: job.message.MessageId,
+                        ReceiptHandle: job.message.ReceiptHandle,
+                        VisibilityTimeout: job.visibilityTimeout
+                    };
+                    entries.push(entry);
+                }
+                debug({ entries });
+                // Delete batch
+                const input = { QueueUrl: qrl, Entries: entries };
+                debug({ DeleteMessageBatch: input });
+                const result = await (0, sqs_js_1.getSQSClient)().send(new client_sqs_1.DeleteMessageBatchCommand(input));
+                this.stats.sqsCalls++;
+                if (result.Failed) {
+                    console.error('FAILED_MESSAGES', result.Failed);
+                    for (const failed of result.Failed) {
+                        console.error('FAILED_TO_DELETE_JOB', { failedEntry: failed, job: this.jobsByMessageId[failed.Id] });
+                        // ensure that we clean this one up so it doesn't generate api calls
+                        if (this.jobsByMessageId[failed.Id])
+                            this.jobsByMessageId[failed.Id].status = 'failed';
+                    }
+                }
+                if (result.Successful) {
+                    const count = result.Successful.length || 0;
+                    this.stats.jobsDeleted += count;
+                    if (this.opt.verbose) {
+                        console.error(chalk_1.default.blue('Deleted'), count, chalk_1.default.blue('jobs'));
+                    }
+                    else if (!this.opt.disableLog) {
+                        console.log(JSON.stringify({ event: 'DELETE_MESSAGES', timestamp: start, count, qrl }));
+                    }
+                    // Mark batch as processed for dedup
+                    await Promise.all(result.Successful.filter(e => this.jobsByMessageId[e.Id]).map(e => (0, dedup_js_1.dedupSuccessfullyProcessed)(this.jobsByMessageId[e.Id].message, this.opt)));
+                }
+                debug('DeleteMessageBatch returned', result);
+                // TODO Sentry
+            }
+        }
+        // Get rid of deleted and failed jobs
+        this.jobs = this.jobs.filter(job => {
+            if (job.status === 'deleting' || job.status === 'failed') {
+                debug('removed', job.message.MessageId);
+                // Accounting
+                delete this.jobsByMessageId[job.message.MessageId];
+                this.jobsByQueue.get(job.qname).delete(job);
+                return false;
+            }
+            else {
+                return true;
+            }
+        });
+    }
+    addJob(message, callback, qname, qrl) {
+        // Create job entry and track it
+        const defaultVisibilityTimeout = 120;
+        const job = {
+            status: 'waiting',
+            start: new Date(),
+            visibilityTimeout: defaultVisibilityTimeout,
+            extendAtSecond: defaultVisibilityTimeout / 2,
+            payload: this.opt.json ? JSON.parse(message.Body) : message.Body,
+            message,
+            callback,
+            qname,
+            prettyQname: qname.slice(this.opt.prefix.length),
+            qrl
+        };
+        // See if we are already executing this job
+        const oldJob = this.jobsByMessageId[job.message.MessageId];
+        if (oldJob) {
+            // If we actually see the same job again, we fucked up, probably due to
+            // the system being overloaded and us missing our extension call. So
+            // we'll celebrate this occasion by throwing a big fat error.
+            debug({ oldJob });
+            const e = new Error(`Saw job ${oldJob.message.MessageId} twice`);
+            e.job = oldJob;
+            // TODO: sentry breadcrumb
+            throw e;
+        }
+        // Accounting
+        this.jobs.push(job);
+        this.jobsByMessageId[job.message.MessageId] = job;
+        // Track all jobs for each queue
+        if (!this.jobsByQueue.has(job.qname))
+            this.jobsByQueue.set(job.qname, new Set());
+        this.jobsByQueue.get(job.qname).add(job);
+        this.stats.activeJobs++;
+        this.stats.waitingJobs++;
+        if (this.opt.verbose) {
+            console.error(chalk_1.default.blue('Got message:'), job.prettyQname, chalk_1.default.blue('-->'), job.payload, job.message.MessageId);
+        }
+        else if (!this.opt.disableLog) {
+            console.log(JSON.stringify({
+                event: 'MESSAGE_RECEIVED',
+                timestamp: new Date(),
+                queue: job.qname,
+                messageId: message.MessageId,
+                payload: job.payload
+            }));
+        }
+        return job;
+    }
+    async runJob(job) {
+        try {
+            if (this.opt.verbose) {
+                console.error(chalk_1.default.blue('Running:'), job.prettyQname, chalk_1.default.blue('-->'), job.payload, job.message.MessageId);
+            }
+            else if (!this.opt.disableLog) {
+                console.log(JSON.stringify({
+                    event: 'MESSAGE_PROCESSING_START',
+                    timestamp: new Date(),
+                    queue: job.qname,
+                    messageId: job.message.MessageId,
+                    payload: job.payload
+                }));
+            }
+            job.status = 'running';
+            this.stats.runningJobs++;
+            this.stats.waitingJobs--;
+            const queue = job.qname.slice(this.opt.prefix.length);
+            const result = await job.callback(queue, job.payload);
+            debug('executeJob callback finished', { payload: job.payload, result });
+            if (this.opt.verbose) {
+                console.error(chalk_1.default.green('SUCCESS'), job.payload);
+            }
+            job.status = 'complete';
+            if (this.opt.verbose) {
+                console.error(chalk_1.default.blue('  done'));
+                console.error();
+            }
+            else if (!this.opt.disableLog) {
+                console.log(JSON.stringify({
+                    event: 'MESSAGE_PROCESSING_COMPLETE',
+                    queue: job.qname,
+                    timestamp: new Date(),
+                    messageId: job.message.MessageId,
+                    payload: job.payload
+                }));
+            }
+            this.stats.jobsSucceeded++;
+        }
+        catch (err) {
+            job.status = 'failed';
+            this.stats.jobsFailed++;
+            // Fail path for job execution
+            if (this.opt.verbose) {
+                console.error(chalk_1.default.red('FAILED'), job.payload);
+                console.error(chalk_1.default.blue('  error : ') + err);
+            }
+            else if (!this.opt.disableLog) {
+                // Production error logging
+                console.log(JSON.stringify({
+                    event: 'MESSAGE_PROCESSING_FAILED',
+                    reason: 'exception thrown',
+                    queue: job.qname,
+                    timestamp: new Date(),
+                    messageId: job.message.MessageId,
+                    payload: job.payload,
+                    errorMessage: err.toString().split('\n').slice(1).join('\n').trim() || undefined,
+                    err
+                }));
+            }
+        }
+        this.stats.activeJobs--;
+        this.stats.runningJobs--;
+    }
+    async executeJobs(messages, callback, qname, qrl) {
+        if (this.shutdownRequested)
+            throw new Error('jobExecutor is shutting down so cannot execute new jobs');
+        // Begin tracking jobs
+        const jobs = messages.map(message => this.addJob(message, callback, qname, qrl));
+        const isFifo = qrl.endsWith('.fifo');
+        const runningJobs = [];
+        // console.log(jobs)
+        // Begin executing
+        for (const [job, i] of jobs.map((job, i) => [job, i])) {
+            // Figure out if the next job needs to happen in serial, otherwise we can parallel execute
+            const nextJob = jobs[i + 1];
+            const nextJobIsSerial = isFifo && nextJob && job.message?.Attributes?.GroupId === nextJob.message?.Attributes?.GroupId;
+            // console.log({ i, nextJobAtt: nextJob?.message?.Attributes, nextJobIsSerial })
+            // Execute serial or parallel
+            if (nextJobIsSerial)
+                await this.runJob(job);
+            else
+                runningJobs.push(this.runJob(job));
+        }
+        await Promise.all(runningJobs);
+    }
+}
+exports.JobExecutor = JobExecutor;

package/commonjs/src/scheduler/queueManager.js

@@ -0,0 +1,161 @@
+"use strict";
+/**
+ * Component to manange what queues are being listened to and in what order.
+ */
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.QueueManager = void 0;
+const chalk_1 = __importDefault(require("chalk"));
+const debug_1 = __importDefault(require("debug"));
+const qrlCache_js_1 = require("../qrlCache.js");
+const idleQueues_js_1 = require("../idleQueues.js");
+const debug = (0, debug_1.default)('qdone:queueManager');
+class QueueManager {
+    constructor(opt, queues, resolveSeconds = 10) {
+        this.opt = opt;
+        this.queues = queues;
+        this.resolveSeconds = resolveSeconds;
+        this.selectedPairs = [];
+        this.icehouse = {};
+        this.resolveTimeout = undefined;
+        this.shutdownRequested = false;
+        this.resolvePromise = this.resolveQueues();
+    }
+    // Sends a queue to the icehouse, where it waits for a while before being
+    // checked again
+    updateIcehouse(qrl, emptyReceive) {
+        const foundEntry = !!this.icehouse[qrl];
+        const { lastCheck, secondsToWait, numEmptyReceives } = this.icehouse[qrl] || {
+            lastCheck: new Date(),
+            secondsToWait: Math.round(20 + 10 * Math.random()),
+            numEmptyReceives: 0 + emptyReceive
+        };
+        if (emptyReceive) {
+            const now = new Date();
+            const secondsElapsed = lastCheck - now;
+            const minWait = 10;
+            const maxWait = 120;
+            const baseSeconds = numEmptyReceives ** 2 * 20;
+            const jitterSeconds = Math.round((Math.random() - 0.5) * baseSeconds);
+            const newSecondsToWait = Math.max(minWait, Math.min(maxWait, baseSeconds + jitterSeconds));
+            const newEntry = { lastCheck: now, secondsToWait: newSecondsToWait, numEmptyReceives: numEmptyReceives + 1 };
+            this.icehouse[qrl] = newEntry;
+            if (this.opt.verbose) {
+                console.error(chalk_1.default.blue('Sending queue to icehouse'), qrl, chalk_1.default.blue('for'), newSecondsToWait, chalk_1.default.blue('seconds'));
+            }
+            debug({ foundEntry, newEntry, lastCheck, secondsToWait, now, secondsElapsed, maxWait, minWait, baseSeconds, jitterSeconds });
+        }
+        else {
+            delete this.icehouse[qrl];
+        }
+    }
+    // Returns true if the queue should be kept in the icehouse
+    keepInIcehouse(qrl, now) {
+        if (this.icehouse[qrl]) {
+            const { lastCheck, secondsToWait } = this.icehouse[qrl];
+            const secondsElapsed = Math.round((now - lastCheck) / 1000);
+            debug({ icehouseCheck: { qrl, lastCheck, secondsToWait, secondsElapsed } });
+            const letOut = secondsElapsed > secondsToWait;
+            if (letOut) {
+                delete this.icehouse[qrl];
+                if (this.opt.verbose) {
+                    console.error(chalk_1.default.blue('Coming out of icehouse:'), qrl);
+                }
+            }
+            return !letOut;
+        }
+        else {
+            return false;
+        }
+    }
+    async resolveQueues() {
+        clearTimeout(this.resolveTimeout);
+        if (this.shutdownRequested)
+            return;
+        this.resolveTimeout = setTimeout(() => {
+            this.resolvePromise = this.resolveQueues();
+        }, this.resolveSeconds * 1000);
+        if (this.opt.verbose) {
+            console.error(chalk_1.default.blue('Will resolve queues again in ' + this.resolveSeconds + ' seconds'));
+        }
+        // Start processing
+        const qnames = this.queues.map(queue => (0, qrlCache_js_1.normalizeQueueName)(queue, this.opt));
+        const pairs = await (0, qrlCache_js_1.getQnameUrlPairs)(qnames, this.opt);
+        if (this.opt.verbose)
+            console.error(chalk_1.default.blue('Resolving queues:'), pairs.map(({ qname }) => qname));
+        if (this.shutdownRequested)
+            return;
+        // Filter out queues
+        const now = new Date();
+        const filteredPairs = pairs
+            // first failed
+            .filter(({ qname, qrl }) => {
+            const suf = this.opt.failSuffix + (this.opt.fifo ? '.fifo' : '');
+            const isFailQueue = qname.slice(-suf.length) === suf;
+            return this.opt.includeFailed ? true : !isFailQueue;
+        })
+            // next fifo
+            .filter(({ qname, qrl }) => {
+            const isFifo = qname.endsWith('.fifo');
+            return this.opt.fifo ? isFifo : true;
+        })
+            // next dead
+            .filter(({ qname, qrl }) => {
+            const isFifo = qname.endsWith('.fifo');
+            const isDead = isFifo ? qname.endsWith('_dead.fifo') : qname.endsWith('_dead');
+            return this.opt.includeDead ? true : !isDead;
+        })
+            // then icehouse
+            .filter(({ qname, qrl }) => !this.keepInIcehouse(qrl, now));
+        // Figure out which pairs are active
+        const activePairs = [];
+        if (this.opt.activeOnly) {
+            if (this.opt.verbose) {
+                console.error(chalk_1.default.blue('  checking active only'));
+            }
+            await Promise.all(filteredPairs.map(async ({ qname, qrl }) => {
+                const { result: { idle } } = await (0, idleQueues_js_1.cheapIdleCheck)(qname, qrl, this.opt);
+                debug({ idle, qname });
+                if (!idle)
+                    activePairs.push({ qname, qrl });
+            }));
+        }
+        if (this.shutdownRequested)
+            return;
+        // Figure out which queues we want to listen on, choosing between active and
+        // all, filtering out failed queues if the user wants that
+        this.selectedPairs = (this.opt.activeOnly ? activePairs : filteredPairs);
+        // Randomize order
+        this.selectedPairs.sort(() => 0.5 - Math.random());
+        if (this.opt.verbose)
+            console.error(chalk_1.default.blue('  selected:\n  ') + this.selectedPairs.map(({ qname }) => qname).join('\n  '));
+        debug('selectedPairs', this.selectedPairs);
+        // Finished resolving
+        if (this.opt.verbose) {
+            console.error(chalk_1.default.blue('Done resolving'));
+            console.error();
+        }
+    }
+    // Return the next queue in the lineup
+    nextPair() {
+        const pair = this.selectedPairs.shift();
+        this.selectedPairs.push(pair);
+        return pair;
+    }
+    getPairs() {
+        const now = new Date();
+        this.selectedPairs.sort(() => 0.5 - Math.random());
+        return this.selectedPairs.filter(({ qname, qrl }) => !this.keepInIcehouse(qrl, now));
+    }
+    async shutdown() {
+        this.shutdownRequested = true;
+        clearTimeout(this.resolveTimeout);
+        if (this.opt.verbose) {
+            console.error(chalk_1.default.blue('Waiting for queues to resolve'));
+        }
+        await this.resolvePromise;
+    }
+}
+exports.QueueManager = QueueManager;

package/commonjs/src/scheduler/systemMonitor.js

@@ -0,0 +1,94 @@
+"use strict";
+/**
+ * Component to track event loop latency, which can be used as a metric for
+ * backpressure.
+ */
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.SystemMonitor = void 0;
+const os_1 = __importDefault(require("os"));
+class SystemMonitor {
+    constructor(reportCallback, reportSeconds = 1) {
+        this.reportCallback = reportCallback || console.log;
+        this.reportSeconds = reportSeconds;
+        this.latencies = [];
+        this.oneMinuteLoad = os_1.default.loadavg()[0];
+        this.instantaneousLoad = this.oneMinuteLoad;
+        this.measure();
+        this.reportLatency();
+    }
+    measure() {
+        clearTimeout(this.measureTimeout);
+        const start = new Date();
+        this.measureTimeout = setTimeout(() => {
+            this.measureLatency(start);
+            this.measureLoad();
+            this.measure();
+        });
+    }
+    measureLatency(start) {
+        const latency = new Date() - start;
+        this.latencies.push(latency);
+        if (this.latencies.length > 1000)
+            this.latencies.shift();
+    }
+    getLatency() {
+        return this.latencies.length ? this.latencies.reduce((a, b) => a + b, 0) / this.latencies.length : 0;
+    }
+    reportLatency() {
+        clearTimeout(this.reportTimeout);
+        this.reportTimeout = setTimeout(() => {
+            const latency = this.getLatency();
+            // console.log({ latency })
+            if (this.reportCallback)
+                this.reportCallback(latency);
+            this.reportLatency();
+        }, this.reportSeconds * 1000);
+    }
+    /**
+     * Measures load over the last five seconds instead of being averaged over one
+     * minute. This lets the scheduler respond much faster to dips in load.
+     *
+     * Theory:
+     *
+     *  The Linux kernel calculates the moving average something like:
+     *    A_1 = A_0 * e + A_now (1 - e)
+     *  Where:
+     *   - A_now is the number of processes active/waiting
+     *   - A_1 is the new one-minute load average after the measurement of A_now
+     *   - A_0 is the previous one-minute average
+     *   - e is 1884/2048.
+     *
+     *  Solving this for A_now, which we want to access, we get:
+     *   A_now = (A_1 - A_0 * e) / (1 - e)
+     *
+     *  We use this formula below to extract A_now when we detect a change in A_1.
+     *
+     * Note: this code assums that we are observing the average often enough to
+     * detect each change. So you have to call it at least every 5 seconds. 1
+     * second is better to reduce latency of detecting the change.
+     */
+    measureLoad() {
+        const [newLoad] = os_1.default.loadavg();
+        const previousLoad = this.oneMinuteLoad;
+        if (previousLoad !== newLoad) {
+            const e = 1884 / 2048; // see include/linux/sched/loadavg.h
+            const active = (newLoad - previousLoad * e) / (1 - e);
+            // We take the min here so that spikes up in load are averaged out. We
+            // care about detecting spikes downward so we can allow more jobs to run.
+            this.instantaneousLoad = Math.min(active, newLoad);
+            this.oneMinuteLoad = newLoad;
+            console.log({ newLoad, previousLoad, active, instantaneousLoad: this.instantaneousLoad, oneMinuteLoad: this.oneMinuteLoad });
+        }
+    }
+    getLoad() {
+        return this.instantaneousLoad;
+    }
+    shutdown() {
+        clearTimeout(this.measureTimeout);
+        clearTimeout(this.reportTimeout);
+    }
+}
+exports.SystemMonitor = SystemMonitor;