qdone 2.0.51-alpha → 2.0.53-alpha

package/README.md

@@ -561,13 +561,8 @@ If a queue name ends with the * (wildcard) character, worker will listen on all
 
     -o, --idle-for number   Minutes of inactivity after which a queue is considered
                             idle. [default: 60]
-    --delete                Delete the queue if it is idle. The fail queue also must be
-                            idle unless you use --unpair.
-    --unpair                Treat queues and their fail queues as independent. By default
-                            they are treated as a unit.
-    --include-failed        When using '*' do not ignore fail queues. This option only
-                            applies if you use --unpair. Otherwise, queues and fail queues
-                            are treated as a unit.
+    --delete                Delete the queue if it is idle along with its corresponding
+                            fail queue and dlq.
     --prefix string         Prefix to place at the front of each SQS queue name [default: qdone_]
     --fail-suffix string    Suffix to append to each queue to generate fail queue name [default: _failed]
     --region string         AWS region for Queues [default: us-east-1]

package/commonjs/src/consumer.js

@@ -130,7 +130,7 @@ async function processMessages(queues, callback, options) {
         const remainingMemory = Math.max(0, freeMemory - freememThreshold);
         const freememFactor = Math.min(1, Math.max(0, remainingMemory / memoryThreshold));
         // Load
-        const oneMinuteLoad = (0, os_1.loadavg)()[0];
+        const oneMinuteLoad = systemMonitor.getLoad();
         const loadPerCore = oneMinuteLoad / cores;
         const loadFactor = 1 - Math.min(1, Math.max(0, loadPerCore / 3));
         const overallFactor = Math.min(latencyFactor, freememFactor, loadFactor);

package/commonjs/src/defaults.js

@@ -1,6 +1,6 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.setupVerbose = exports.setupAWS = exports.getOptionsWithDefaults = exports.validateMessageOptions = exports.validateInteger = exports.defaults = void 0;
+exports.setupVerbose = exports.setupAWS = exports.getOptionsWithDefaults = exports.validateMessageOptions = exports.validateQueueName = exports.validateInteger = exports.defaults = void 0;
 /**
  * Default options for qdone. Accepts a command line options object and
  * returns nicely-named options.
@@ -46,7 +46,6 @@ exports.defaults = Object.freeze({
     // Idle Queues
     idleFor: 60,
     delete: false,
-    unpair: false,
     // Check
     create: false,
     overwrite: false
@@ -58,6 +57,14 @@ function validateInteger(opt, name) {
     return parsed;
 }
 exports.validateInteger = validateInteger;
+function validateQueueName(opt, name) {
+    if (typeof name !== 'string')
+        throw new Error(`${name} must be a string.`);
+    if (!name.match(/^[a-z0-9-_]+$/i))
+        throw new Error(`${name} can contain only numbers, letters, hypens and underscores.`);
+    return name;
+}
+exports.validateQueueName = validateQueueName;
 function validateMessageOptions(messageOptions) {
     const validKeys = ['deduplicationId', 'groupId'];
     if (typeof messageOptions === 'object' &&
@@ -128,7 +135,6 @@ function getOptionsWithDefaults(options) {
         // Idle Queues
         idleFor: options.idleFor || options['idle-for'] || process.env.QDONE_IDLE_FOR || exports.defaults.idleFor,
         delete: options.delete || process.env.QDONE_DELETE === 'true' || exports.defaults.delete,
-        unpair: options.unpair || process.env.QDONE_UNPAIR === 'true' || exports.defaults.unpair,
         // Check
         create: options.create || process.env.QDONE_CREATE === 'true' || exports.defaults.create,
         overwrite: options.overwrite || process.env.QDONE_OVERWRITE === 'true' || exports.defaults.overwrite
@@ -148,6 +154,10 @@ function getOptionsWithDefaults(options) {
     opt.maxConcurrentJobs = validateInteger(opt, 'maxConcurrentJobs');
     opt.maxMemoryPercent = validateInteger(opt, 'maxMemoryPercent');
     opt.idleFor = validateInteger(opt, 'idleFor');
+    validateQueueName(opt, 'region');
+    validateQueueName(opt, 'prefix');
+    validateQueueName(opt, 'failSuffix');
+    validateQueueName(opt, 'dlqSuffix');
     // Validate dedup args
     if (opt.externalDedup && !opt.cacheUri)
         throw new Error('--external-dedup requires the --cache-uri argument');

package/commonjs/src/idleQueues.js

@@ -3,7 +3,7 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
     return (mod && mod.__esModule) ? mod : { "default": mod };
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.idleQueues = exports.processQueueSet = exports.processQueue = exports.deleteQueue = exports.checkIdle = exports.getMetric = exports.cheapIdleCheck = exports._cheapIdleCheck = exports.attributeNames = void 0;
+exports.idleQueues = exports.stripSuffixes = exports.processQueueSet = exports.deleteQueue = exports.checkIdle = exports.getMetric = exports.cheapIdleCheck = exports._cheapIdleCheck = exports.attributeNames = void 0;
 /**
  * Implementation of checks and caching of checks to determine if queues are idle.
  */
@@ -187,35 +187,6 @@ async function deleteQueue(qname, qrl, opt) {
     };
 }
 exports.deleteQueue = deleteQueue;
-/**
- * Processes a single queue, checking for idle, deleting if applicable.
- */
-async function processQueue(qname, qrl, opt) {
-    const result = await checkIdle(qname, qrl, opt);
-    debug(qname, result);
-    // Queue is active
-    if (!result.idle) {
-        // Notify and return
-        if (opt.verbose)
-            console.error(chalk_1.default.blue('Queue ') + qname.slice(opt.prefix.length) + chalk_1.default.blue(' has been ') + 'active' + chalk_1.default.blue(' in the last ') + opt.idleFor + chalk_1.default.blue(' minutes.'));
-        return result;
-    }
-    // Queue is idle
-    if (opt.verbose)
-        console.error(chalk_1.default.blue('Queue ') + qname.slice(opt.prefix.length) + chalk_1.default.blue(' has been ') + 'idle' + chalk_1.default.blue(' for the last ') + opt.idleFor + chalk_1.default.blue(' minutes.'));
-    if (opt.delete) {
-        const deleteResult = await deleteQueue(qname, qrl, opt);
-        const resultIncludingDelete = Object.assign(result, {
-            deleted: deleteResult.deleted,
-            apiCalls: {
-                SQS: result.apiCalls.SQS + deleteResult.apiCalls.SQS,
-                CloudWatch: result.apiCalls.CloudWatch + deleteResult.apiCalls.CloudWatch
-            }
-        });
-        return resultIncludingDelete;
-    }
-}
-exports.processQueue = processQueue;
 /**
  * Processes a queue and its fail and delete queue, treating them as a unit.
  */
@@ -311,6 +282,14 @@ async function processQueueSet(qname, qrl, opt) {
 }
 exports.processQueueSet = processQueueSet;
 //
+// Strips failed and dlq suffix from a queue name or URL
+//
+function stripSuffixes(queueName, opt) {
+    const suffixFinder = new RegExp(`(${opt.dlqSuffix}|${opt.failSuffix}){1}(|${qrlCache_js_1.fifoSuffix})$`);
+    return queueName.replace(suffixFinder, '$2');
+}
+exports.stripSuffixes = stripSuffixes;
+//
 // Resolve queues for listening loop listen
 //
 async function idleQueues(queues, options) {
@@ -324,17 +303,18 @@ async function idleQueues(queues, options) {
         console.error(chalk_1.default.blue('  done'));
         console.error();
     }
-    // Filter out any queue ending in suffix unless --include-failed is set
+    // Filter out failed and dead queues, but if we have an orphaned fail or
+    // dead queue, keep the original parent queue name so that orphans can be
+    // deleted.
+    const queueNames = new Set();
     const filteredEntries = entries.filter(entry => {
-        const suf = opt.failSuffix;
-        const sufFifo = opt.failSuffix + qrlCache_js_1.fifoSuffix;
-        const isFail = entry.qname.endsWith(suf);
-        const isFifoFail = entry.qname.endsWith(sufFifo);
-        const sufDead = opt.dlqSuffix;
-        const sufFifoDead = opt.dlqSuffix + qrlCache_js_1.fifoSuffix;
-        const isDead = entry.qname.endsWith(sufDead);
-        const isFifoDead = entry.qname.endsWith(sufFifoDead);
-        return opt.includeFailed ? true : (!isFail && !isFifoFail && !isDead && !isFifoDead);
+        const stripped = stripSuffixes(entry.qname, opt);
+        if (queueNames.has(stripped))
+            return false;
+        queueNames.add(stripped);
+        entry.qname = stripped;
+        entry.qrl = stripSuffixes(entry.qrl, opt);
+        return true;
     });
     // But only if we have queues to remove
     if (filteredEntries.length) {
@@ -344,8 +324,6 @@ async function idleQueues(queues, options) {
             console.error();
         }
         // Check each queue in parallel
-        if (opt.unpair)
-            return Promise.all(filteredEntries.map(e => processQueue(e.qname, e.qrl, opt)));
         return Promise.all(filteredEntries.map(e => processQueueSet(e.qname, e.qrl, opt)));
     }
     // Otherwise, let caller know

package/commonjs/src/monitor.js

@@ -17,9 +17,11 @@ const debug = (0, debug_1.default)('qdone:monitor');
  * Splits a queue name with a single wildcard into prefix and suffix regex.
  */
 async function monitor(queue, save, options) {
+    if (queue.endsWith('.fifo'))
+        options.fifo = true;
     const opt = (0, defaults_js_1.getOptionsWithDefaults)(options);
     const queueName = (0, qrlCache_js_1.normalizeQueueName)(queue, opt);
-    debug({ opt, queueName });
+    debug({ options, opt, queue, queueName });
     const data = await getAggregateData(queueName);
     console.log(data);
     if (save) {

package/commonjs/src/qrlCache.js

@@ -70,8 +70,10 @@ async function qrlCacheGet(qname) {
     // debug({ cmd })
     const result = await client.send(cmd);
     // debug('result', result)
-    if (!result)
+    if (!result) {
+        qrlCacheInvalidate(qname);
         throw new client_sqs_1.QueueDoesNotExist(qname);
+    }
     const { QueueUrl: qrl } = result;
     // debug('getQueueUrl returned', data)
     qcache.set(qname, qrl);
@@ -84,7 +86,8 @@ exports.qrlCacheGet = qrlCacheGet;
 // Immediately updates the cache
 //
 function qrlCacheSet(qname, qrl) {
-    qcache.set(qname, qrl);
+    if (qrl)
+        qcache.set(qname, qrl);
     // debug('qcache', Object.keys(qcache), 'set', qname, ' => ', qcache[qname])
 }
 exports.qrlCacheSet = qrlCacheSet;

package/commonjs/src/scheduler/systemMonitor.js

@@ -3,13 +3,19 @@
  * Component to track event loop latency, which can be used as a metric for
  * backpressure.
  */
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.SystemMonitor = void 0;
+const os_1 = __importDefault(require("os"));
 class SystemMonitor {
     constructor(reportCallback, reportSeconds = 1) {
         this.reportCallback = reportCallback || console.log;
         this.reportSeconds = reportSeconds;
-        this.measurements = [];
+        this.latencies = [];
+        this.oneMinuteLoad = os_1.default.loadavg()[0];
+        this.instantaneousLoad = this.oneMinuteLoad;
         this.measure();
         this.reportLatency();
     }
@@ -17,15 +23,19 @@ class SystemMonitor {
         clearTimeout(this.measureTimeout);
         const start = new Date();
         this.measureTimeout = setTimeout(() => {
-            const latency = new Date() - start;
-            this.measurements.push(latency);
-            if (this.measurements.length > 1000)
-                this.measurements.shift();
+            this.measureLatency(start);
+            this.measureLoad();
             this.measure();
         });
     }
+    measureLatency(start) {
+        const latency = new Date() - start;
+        this.latencies.push(latency);
+        if (this.latencies.length > 1000)
+            this.latencies.shift();
+    }
     getLatency() {
-        return this.measurements.length ? this.measurements.reduce((a, b) => a + b, 0) / this.measurements.length : 0;
+        return this.latencies.length ? this.latencies.reduce((a, b) => a + b, 0) / this.latencies.length : 0;
     }
     reportLatency() {
         clearTimeout(this.reportTimeout);
@@ -37,6 +47,45 @@ class SystemMonitor {
             this.reportLatency();
         }, this.reportSeconds * 1000);
     }
+    /**
+     * Measures load over the last five seconds instead of being averaged over one
+     * minute. This lets the scheduler respond much faster to dips in load.
+     *
+     * Theory:
+     *
+     *  The Linux kernel calculates the moving average something like:
+     *    A_1 = A_0 * e + A_now (1 - e)
+     *  Where:
+     *   - A_now is the number of processes active/waiting
+     *   - A_1 is the new one-minute load average after the measurement of A_now
+     *   - A_0 is the previous one-minute average
+     *   - e is 1884/2048.
+     *
+     *  Solving this for A_now, which we want to access, we get:
+     *   A_now = (A_1 - A_0 * e) / (1 - e)
+     *
+     *  We use this formula below to extract A_now when we detect a change in A_1.
+     *
+     * Note: this code assums that we are observing the average often enough to
+     * detect each change. So you have to call it at least every 5 seconds. 1
+     * second is better to reduce latency of detecting the change.
+     */
+    measureLoad() {
+        const [newLoad,] = os_1.default.loadavg();
+        const previousLoad = this.oneMinuteLoad;
+        if (previousLoad !== newLoad) {
+            const e = 1884 / 2048; // see include/linux/sched/loadavg.h
+            const active = (newLoad - previousLoad * e) / (1 - e);
+            // We take the min here so that spikes up in load are averaged out. We
+            // care about detecting spikes downward so we can allow more jobs to run.
+            this.instantaneousLoad = Math.min(active, newLoad);
+            this.oneMinuteLoad = newLoad;
+            console.log({ newLoad, previousLoad, active, instantaneousLoad: this.instantaneousLoad, oneMinuteLoad: this.oneMinuteLoad });
+        }
+    }
+    getLoad() {
+        return this.instantaneousLoad;
+    }
     shutdown() {
         clearTimeout(this.measureTimeout);
         clearTimeout(this.reportTimeout);

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "qdone",
-  "version": "2.0.51-alpha",
+  "version": "2.0.53-alpha",
   "description": "A distributed scheduler for SQS",
   "type": "module",
   "main": "./index.js",

package/src/cli.js

@@ -440,9 +440,7 @@ export async function worker (argv, testHook) {
 export async function idleQueues (argv, testHook) {
   const optionDefinitions = [
     { name: 'idle-for', alias: 'o', type: Number, defaultValue: defaults.idleFor, description: `Minutes of inactivity after which a queue is considered idle. [default: ${defaults.idleFor}]` },
-    { name: 'delete', type: Boolean, description: 'Delete the queue if it is idle. The fail queue also must be idle unless you use --unpair.' },
-    { name: 'unpair', type: Boolean, description: 'Treat queues and their fail queues as independent. By default they are treated as a unit.' },
-    { name: 'include-failed', type: Boolean, description: 'When using \'*\' do not ignore fail queues. This option only applies if you use --unpair. Otherwise, queues and fail queues are treated as a unit.' }
+    { name: 'delete', type: Boolean, description: 'Delete the queue if it is idle along with its corresponding fail queue and dlq.' }
   ].concat(globalOptionDefinitions)
 
   const usageSections = [

package/src/consumer.js

@@ -136,7 +136,7 @@ export async function processMessages (queues, callback, options) {
     const freememFactor = Math.min(1, Math.max(0, remainingMemory / memoryThreshold))
 
     // Load
-    const oneMinuteLoad = loadavg()[0]
+    const oneMinuteLoad = systemMonitor.getLoad()
     const loadPerCore = oneMinuteLoad / cores
     const loadFactor = 1 - Math.min(1, Math.max(0, loadPerCore / 3))
 

package/src/defaults.js

@@ -47,7 +47,6 @@ export const defaults = Object.freeze({
   // Idle Queues
   idleFor: 60,
   delete: false,
-  unpair: false,
 
   // Check
   create: false,
@@ -60,6 +59,12 @@ export function validateInteger (opt, name) {
   return parsed
 }
 
+export function validateQueueName (opt, name) {
+  if (typeof name !== 'string') throw new Error(`${name} must be a string.`)
+  if (!name.match(/^[a-z0-9-_]+$/i)) throw new Error(`${name} can contain only numbers, letters, hypens and underscores.`)
+  return name
+}
+
 export function validateMessageOptions (messageOptions) {
   const validKeys = ['deduplicationId', 'groupId']
   if (typeof messageOptions === 'object' &&
@@ -134,7 +139,6 @@ export function getOptionsWithDefaults (options) {
     // Idle Queues
     idleFor: options.idleFor || options['idle-for'] || process.env.QDONE_IDLE_FOR || defaults.idleFor,
     delete: options.delete || process.env.QDONE_DELETE === 'true' || defaults.delete,
-    unpair: options.unpair || process.env.QDONE_UNPAIR === 'true' || defaults.unpair,
 
     // Check
     create: options.create || process.env.QDONE_CREATE === 'true' || defaults.create,
@@ -158,6 +162,11 @@ export function getOptionsWithDefaults (options) {
   opt.maxMemoryPercent = validateInteger(opt, 'maxMemoryPercent')
   opt.idleFor = validateInteger(opt, 'idleFor')
 
+  validateQueueName(opt, 'region')
+  validateQueueName(opt, 'prefix')
+  validateQueueName(opt, 'failSuffix')
+  validateQueueName(opt, 'dlqSuffix')
+
   // Validate dedup args
   if (opt.externalDedup && !opt.cacheUri) throw new Error('--external-dedup requires the --cache-uri argument')
   if (opt.externalDedup && (opt.dedupPeriod < 1)) throw new Error('--external-dedup of redis requires a --dedup-period > 1 second')

package/src/idleQueues.js

@@ -186,35 +186,6 @@ export async function deleteQueue (qname, qrl, opt) {
   }
 }
 
-/**
- * Processes a single queue, checking for idle, deleting if applicable.
- */
-export async function processQueue (qname, qrl, opt) {
-  const result = await checkIdle(qname, qrl, opt)
-  debug(qname, result)
-
-  // Queue is active
-  if (!result.idle) {
-    // Notify and return
-    if (opt.verbose) console.error(chalk.blue('Queue ') + qname.slice(opt.prefix.length) + chalk.blue(' has been ') + 'active' + chalk.blue(' in the last ') + opt.idleFor + chalk.blue(' minutes.'))
-    return result
-  }
-
-  // Queue is idle
-  if (opt.verbose) console.error(chalk.blue('Queue ') + qname.slice(opt.prefix.length) + chalk.blue(' has been ') + 'idle' + chalk.blue(' for the last ') + opt.idleFor + chalk.blue(' minutes.'))
-  if (opt.delete) {
-    const deleteResult = await deleteQueue(qname, qrl, opt)
-    const resultIncludingDelete = Object.assign(result, {
-      deleted: deleteResult.deleted,
-      apiCalls: {
-        SQS: result.apiCalls.SQS + deleteResult.apiCalls.SQS,
-        CloudWatch: result.apiCalls.CloudWatch + deleteResult.apiCalls.CloudWatch
-      }
-    })
-    return resultIncludingDelete
-  }
-}
-
 /**
  * Processes a queue and its fail and delete queue, treating them as a unit.
  */
@@ -308,6 +279,14 @@ export async function processQueueSet (qname, qrl, opt) {
   return result
 }
 
+//
+// Strips failed and dlq suffix from a queue name or URL
+//
+export function stripSuffixes (queueName, opt) {
+  const suffixFinder = new RegExp(`(${opt.dlqSuffix}|${opt.failSuffix}){1}(|${fifoSuffix})$`)
+  return queueName.replace(suffixFinder, '$2')
+}
+
 //
 // Resolve queues for listening loop listen
 //
@@ -322,17 +301,17 @@ export async function idleQueues (queues, options) {
     console.error()
   }
 
-  // Filter out any queue ending in suffix unless --include-failed is set
+  // Filter out failed and dead queues, but if we have an orphaned fail or
+  // dead queue, keep the original parent queue name so that orphans can be
+  // deleted.
+  const queueNames = new Set()
   const filteredEntries = entries.filter(entry => {
-    const suf = opt.failSuffix
-    const sufFifo = opt.failSuffix + fifoSuffix
-    const isFail = entry.qname.endsWith(suf)
-    const isFifoFail = entry.qname.endsWith(sufFifo)
-    const sufDead = opt.dlqSuffix
-    const sufFifoDead = opt.dlqSuffix + fifoSuffix
-    const isDead = entry.qname.endsWith(sufDead)
-    const isFifoDead = entry.qname.endsWith(sufFifoDead)
-    return opt.includeFailed ? true : (!isFail && !isFifoFail && !isDead && !isFifoDead)
+    const stripped = stripSuffixes(entry.qname, opt)
+    if (queueNames.has(stripped)) return false
+    queueNames.add(stripped)
+    entry.qname = stripped
+    entry.qrl = stripSuffixes(entry.qrl, opt)
+    return true
   })
 
   // But only if we have queues to remove
@@ -345,7 +324,6 @@ export async function idleQueues (queues, options) {
       console.error()
     }
     // Check each queue in parallel
-    if (opt.unpair) return Promise.all(filteredEntries.map(e => processQueue(e.qname, e.qrl, opt)))
     return Promise.all(filteredEntries.map(e => processQueueSet(e.qname, e.qrl, opt)))
   }
 

package/src/monitor.js

@@ -13,9 +13,10 @@ const debug = Debug('qdone:monitor')
  * Splits a queue name with a single wildcard into prefix and suffix regex.
  */
 export async function monitor (queue, save, options) {
+  if (queue.endsWith('.fifo')) options.fifo = true
   const opt = getOptionsWithDefaults(options)
   const queueName = normalizeQueueName(queue, opt)
-  debug({ opt, queueName })
+  debug({ options, opt, queue, queueName })
   const data = await getAggregateData(queueName)
   console.log(data)
   if (save) {

package/src/qrlCache.js

@@ -66,7 +66,10 @@ export async function qrlCacheGet (qname) {
   // debug({ cmd })
   const result = await client.send(cmd)
   // debug('result', result)
-  if (!result) throw new QueueDoesNotExist(qname)
+  if (!result) {
+    qrlCacheInvalidate(qname)
+    throw new QueueDoesNotExist(qname)
+  }
   const { QueueUrl: qrl } = result
   // debug('getQueueUrl returned', data)
   qcache.set(qname, qrl)
@@ -79,7 +82,7 @@ export async function qrlCacheGet (qname) {
 // Immediately updates the cache
 //
 export function qrlCacheSet (qname, qrl) {
-  qcache.set(qname, qrl)
+  if (qrl) qcache.set(qname, qrl)
   // debug('qcache', Object.keys(qcache), 'set', qname, ' => ', qcache[qname])
 }
 

package/src/scheduler/systemMonitor.js

@@ -3,11 +3,15 @@
  * backpressure.
  */
 
+import os from 'os'
+
 export class SystemMonitor {
   constructor (reportCallback, reportSeconds = 1) {
     this.reportCallback = reportCallback || console.log
     this.reportSeconds = reportSeconds
-    this.measurements = []
+    this.latencies = []
+    this.oneMinuteLoad = os.loadavg()[0]
+    this.instantaneousLoad = this.oneMinuteLoad
     this.measure()
     this.reportLatency()
   }
@@ -16,15 +20,20 @@ export class SystemMonitor {
     clearTimeout(this.measureTimeout)
     const start = new Date()
     this.measureTimeout = setTimeout(() => {
-      const latency = new Date() - start
-      this.measurements.push(latency)
-      if (this.measurements.length > 1000) this.measurements.shift()
+      this.measureLatency(start)
+      this.measureLoad()
       this.measure()
     })
   }
 
+  measureLatency (start) {
+    const latency = new Date() - start
+    this.latencies.push(latency)
+    if (this.latencies.length > 1000) this.latencies.shift()
+  }
+
   getLatency () {
-    return this.measurements.length ? this.measurements.reduce((a, b) => a + b, 0) / this.measurements.length : 0
+    return this.latencies.length ? this.latencies.reduce((a, b) => a + b, 0) / this.latencies.length : 0
   }
 
   reportLatency () {
@@ -37,6 +46,48 @@ export class SystemMonitor {
     }, this.reportSeconds * 1000)
   }
 
+  /**
+   * Measures load over the last five seconds instead of being averaged over one
+   * minute. This lets the scheduler respond much faster to dips in load.
+   *
+   * Theory:
+   *
+   *  The Linux kernel calculates the moving average something like:
+   *    A_1 = A_0 * e + A_now (1 - e)
+   *  Where:
+   *   - A_now is the number of processes active/waiting
+   *   - A_1 is the new one-minute load average after the measurement of A_now
+   *   - A_0 is the previous one-minute average
+   *   - e is 1884/2048.
+   *
+   *  Solving this for A_now, which we want to access, we get:
+   *   A_now = (A_1 - A_0 * e) / (1 - e)
+   *
+   *  We use this formula below to extract A_now when we detect a change in A_1.
+   *
+   * Note: this code assums that we are observing the average often enough to
+   * detect each change. So you have to call it at least every 5 seconds. 1
+   * second is better to reduce latency of detecting the change.
+   */
+
+  measureLoad () {
+    const [newLoad, ] = os.loadavg()
+    const previousLoad = this.oneMinuteLoad
+    if (previousLoad !== newLoad) {
+      const e = 1884 / 2048 // see include/linux/sched/loadavg.h
+      const active = (newLoad - previousLoad * e) / (1 - e)
+      // We take the min here so that spikes up in load are averaged out. We
+      // care about detecting spikes downward so we can allow more jobs to run.
+      this.instantaneousLoad = Math.min(active, newLoad)
+      this.oneMinuteLoad = newLoad
+      console.log({ newLoad, previousLoad, active, instantaneousLoad: this.instantaneousLoad, oneMinuteLoad: this.oneMinuteLoad })
+    }
+  }
+
+  getLoad() {
+    return this.instantaneousLoad
+  }
+
   shutdown () {
     clearTimeout(this.measureTimeout)
     clearTimeout(this.reportTimeout)