qdone 2.0.52-alpha → 2.0.54-alpha

package/commonjs/src/consumer.js

@@ -92,7 +92,7 @@ async function processMessages(queues, callback, options) {
             const messages = await getMessages(qrl, opt, maxMessages);
             if (!shutdownRequested) {
                 if (messages.length) {
-                    jobExecutor.executeJobs(messages, callback, qname, qrl);
+                    await jobExecutor.executeJobs(messages, callback, qname, qrl);
                     queueManager.updateIcehouse(qrl, false);
                 }
                 else {
@@ -130,7 +130,7 @@ async function processMessages(queues, callback, options) {
         const remainingMemory = Math.max(0, freeMemory - freememThreshold);
         const freememFactor = Math.min(1, Math.max(0, remainingMemory / memoryThreshold));
         // Load
-        const oneMinuteLoad = (0, os_1.loadavg)()[0];
+        const oneMinuteLoad = systemMonitor.getLoad();
         const loadPerCore = oneMinuteLoad / cores;
         const loadFactor = 1 - Math.min(1, Math.max(0, loadPerCore / 3));
         const overallFactor = Math.min(latencyFactor, freememFactor, loadFactor);

package/commonjs/src/idleQueues.js

@@ -137,7 +137,7 @@ async function checkIdle(qname, qrl, opt) {
     if (cheapResult.idle === false || cheapResult.exists === false) {
         return {
             queue: qname.slice(opt.prefix.length),
-            cheap: cheapResult,
+            cheap: { SQS, result: cheapResult },
             idle: cheapResult.idle,
             exists: cheapResult.exists,
             apiCalls: { SQS, CloudWatch: 0 }

package/commonjs/src/qrlCache.js

@@ -70,8 +70,10 @@ async function qrlCacheGet(qname) {
     // debug({ cmd })
     const result = await client.send(cmd);
     // debug('result', result)
-    if (!result)
+    if (!result) {
+        qrlCacheInvalidate(qname);
         throw new client_sqs_1.QueueDoesNotExist(qname);
+    }
     const { QueueUrl: qrl } = result;
     // debug('getQueueUrl returned', data)
     qcache.set(qname, qrl);
@@ -84,7 +86,8 @@ exports.qrlCacheGet = qrlCacheGet;
 // Immediately updates the cache
 //
 function qrlCacheSet(qname, qrl) {
-    qcache.set(qname, qrl);
+    if (qrl)
+        qcache.set(qname, qrl);
     // debug('qcache', Object.keys(qcache), 'set', qname, ' => ', qcache[qname])
 }
 exports.qrlCacheSet = qrlCacheSet;

package/commonjs/src/scheduler/jobExecutor.js

@@ -363,6 +363,7 @@ class JobExecutor {
         // Begin tracking jobs
         const jobs = messages.map(message => this.addJob(message, callback, qname, qrl));
         const isFifo = qrl.endsWith('.fifo');
+        const runningJobs = [];
         // console.log(jobs)
         // Begin executing
         for (const [job, i] of jobs.map((job, i) => [job, i])) {
@@ -374,8 +375,9 @@ class JobExecutor {
             if (nextJobIsSerial)
                 await this.runJob(job);
             else
-                this.runJob(job);
+                runningJobs.push(this.runJob(job));
         }
+        await Promise.all(runningJobs);
     }
 }
 exports.JobExecutor = JobExecutor;

package/commonjs/src/scheduler/systemMonitor.js

@@ -3,13 +3,19 @@
  * Component to track event loop latency, which can be used as a metric for
  * backpressure.
  */
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.SystemMonitor = void 0;
+const os_1 = __importDefault(require("os"));
 class SystemMonitor {
     constructor(reportCallback, reportSeconds = 1) {
         this.reportCallback = reportCallback || console.log;
         this.reportSeconds = reportSeconds;
-        this.measurements = [];
+        this.latencies = [];
+        this.oneMinuteLoad = os_1.default.loadavg()[0];
+        this.instantaneousLoad = this.oneMinuteLoad;
         this.measure();
         this.reportLatency();
     }
@@ -17,15 +23,19 @@ class SystemMonitor {
         clearTimeout(this.measureTimeout);
         const start = new Date();
         this.measureTimeout = setTimeout(() => {
-            const latency = new Date() - start;
-            this.measurements.push(latency);
-            if (this.measurements.length > 1000)
-                this.measurements.shift();
+            this.measureLatency(start);
+            this.measureLoad();
             this.measure();
         });
     }
+    measureLatency(start) {
+        const latency = new Date() - start;
+        this.latencies.push(latency);
+        if (this.latencies.length > 1000)
+            this.latencies.shift();
+    }
     getLatency() {
-        return this.measurements.length ? this.measurements.reduce((a, b) => a + b, 0) / this.measurements.length : 0;
+        return this.latencies.length ? this.latencies.reduce((a, b) => a + b, 0) / this.latencies.length : 0;
     }
     reportLatency() {
         clearTimeout(this.reportTimeout);
@@ -37,6 +47,45 @@ class SystemMonitor {
             this.reportLatency();
         }, this.reportSeconds * 1000);
     }
+    /**
+     * Measures load over the last five seconds instead of being averaged over one
+     * minute. This lets the scheduler respond much faster to dips in load.
+     *
+     * Theory:
+     *
+     *  The Linux kernel calculates the moving average something like:
+     *    A_1 = A_0 * e + A_now (1 - e)
+     *  Where:
+     *   - A_now is the number of processes active/waiting
+     *   - A_1 is the new one-minute load average after the measurement of A_now
+     *   - A_0 is the previous one-minute average
+     *   - e is 1884/2048.
+     *
+     *  Solving this for A_now, which we want to access, we get:
+     *   A_now = (A_1 - A_0 * e) / (1 - e)
+     *
+     *  We use this formula below to extract A_now when we detect a change in A_1.
+     *
+     * Note: this code assums that we are observing the average often enough to
+     * detect each change. So you have to call it at least every 5 seconds. 1
+     * second is better to reduce latency of detecting the change.
+     */
+    measureLoad() {
+        const [newLoad,] = os_1.default.loadavg();
+        const previousLoad = this.oneMinuteLoad;
+        if (previousLoad !== newLoad) {
+            const e = 1884 / 2048; // see include/linux/sched/loadavg.h
+            const active = (newLoad - previousLoad * e) / (1 - e);
+            // We take the min here so that spikes up in load are averaged out. We
+            // care about detecting spikes downward so we can allow more jobs to run.
+            this.instantaneousLoad = Math.min(active, newLoad);
+            this.oneMinuteLoad = newLoad;
+            console.log({ newLoad, previousLoad, active, instantaneousLoad: this.instantaneousLoad, oneMinuteLoad: this.oneMinuteLoad });
+        }
+    }
+    getLoad() {
+        return this.instantaneousLoad;
+    }
     shutdown() {
         clearTimeout(this.measureTimeout);
         clearTimeout(this.reportTimeout);

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "qdone",
-  "version": "2.0.52-alpha",
+  "version": "2.0.54-alpha",
   "description": "A distributed scheduler for SQS",
   "type": "module",
   "main": "./index.js",

package/src/cli.js

@@ -451,19 +451,14 @@ export async function idleQueues (argv, testHook) {
     {
       content: [
         { count: '1 + q + i', desc: 'q: number of queues in pattern\ni: number of idle queues' },
-        { context: 'with --delete options', count: '1 + q + 3i', desc: 'q: number of queues in pattern\ni: number of idle queues' },
-        { context: 'with --unpair option', count: '1 + q', desc: 'q: number of queues in pattern' },
-        { context: 'with --unpair and --delete options', count: '1 + q + i', desc: 'q: number of queues in pattern\ni: number of idle queues' },
-        { desc: 'NOTE: the --unpair option not cheaper if you include fail queues, because it doubles q.' }
+        { context: 'with --delete options', count: '1 + q + 3i', desc: 'q: number of queues in pattern\ni: number of idle queues' }
       ],
       long: true
     },
     { content: 'CloudWatch API Call Complexity', raw: true, long: true },
     {
       content: [
-        { count: 'min: 0 (if queue and fail queue have waiting messages)\nmax: 12q\nexpected (approximate observed): 0.5q + 12i', desc: 'q: number of queues in pattern\ni: number of idle queues' },
-        { context: 'with --unpair option', count: 'min: 0 (if queue has waiting messages)\nmax: 6q\nexpected (approximate observed): q + 6i', desc: 'q: number of queues in pattern\ni: number of idle queues' },
-        { desc: 'NOTE: the --unpair option not cheaper if you include fail queues, because it doubles q.' }
+        { count: 'min: 0 (if queue and fail queue have waiting messages)\nmax: 12q\nexpected (approximate observed): 0.5q + 12i', desc: 'q: number of queues in pattern\ni: number of idle queues' }
       ],
       long: true
     },
@@ -479,7 +474,6 @@ export async function idleQueues (argv, testHook) {
     debug('idleQueues options', options)
     if (options.help) return Promise.resolve(console.log(getUsage(usageSections)))
     if (!options._unknown || options._unknown.length === 0) throw new UsageError('idle-queues requres one or more <queue> arguments')
-    if (options['include-failed'] && !options.unpair) throw new UsageError('--include-failed should be used with --unpair')
     if (options['idle-for'] < 5) throw new UsageError('--idle-for must be at least 5 minutes (CloudWatch limitation)')
     queues = options._unknown
     debug('queues', queues)

package/src/consumer.js

@@ -96,7 +96,7 @@ export async function processMessages (queues, callback, options) {
 
       if (!shutdownRequested) {
         if (messages.length) {
-          jobExecutor.executeJobs(messages, callback, qname, qrl)
+          await jobExecutor.executeJobs(messages, callback, qname, qrl)
           queueManager.updateIcehouse(qrl, false)
         } else {
           // If we didn't get any, update the icehouse so we can back off
@@ -136,7 +136,7 @@ export async function processMessages (queues, callback, options) {
     const freememFactor = Math.min(1, Math.max(0, remainingMemory / memoryThreshold))
 
     // Load
-    const oneMinuteLoad = loadavg()[0]
+    const oneMinuteLoad = systemMonitor.getLoad()
     const loadPerCore = oneMinuteLoad / cores
     const loadFactor = 1 - Math.min(1, Math.max(0, loadPerCore / 3))
 

package/src/idleQueues.js

@@ -7,7 +7,7 @@ import { getCloudWatchClient } from './cloudWatch.js'
 import { getOptionsWithDefaults } from './defaults.js'
 import { GetQueueAttributesCommand, DeleteQueueCommand, QueueDoesNotExist } from '@aws-sdk/client-sqs'
 import { GetMetricStatisticsCommand } from '@aws-sdk/client-cloudwatch'
-import { normalizeFailQueueName, normalizeDLQName, getQnameUrlPairs, fifoSuffix } from './qrlCache.js'
+import { normalizeFailQueueName, normalizeDLQName, getQnameUrlPairs, fifoSuffix, qrlCacheSet } from './qrlCache.js'
 import { getCache, setCache } from './cache.js'
 // const AWS = require('aws-sdk')
 
@@ -133,7 +133,7 @@ export async function checkIdle (qname, qrl, opt) {
   if (cheapResult.idle === false || cheapResult.exists === false) {
     return {
       queue: qname.slice(opt.prefix.length),
-      cheap: cheapResult,
+      cheap: { SQS, result: cheapResult },
       idle: cheapResult.idle,
       exists: cheapResult.exists,
       apiCalls: { SQS, CloudWatch: 0 }

package/src/qrlCache.js

@@ -66,7 +66,10 @@ export async function qrlCacheGet (qname) {
   // debug({ cmd })
   const result = await client.send(cmd)
   // debug('result', result)
-  if (!result) throw new QueueDoesNotExist(qname)
+  if (!result) {
+    qrlCacheInvalidate(qname)
+    throw new QueueDoesNotExist(qname)
+  }
   const { QueueUrl: qrl } = result
   // debug('getQueueUrl returned', data)
   qcache.set(qname, qrl)
@@ -79,7 +82,7 @@ export async function qrlCacheGet (qname) {
 // Immediately updates the cache
 //
 export function qrlCacheSet (qname, qrl) {
-  qcache.set(qname, qrl)
+  if (qrl) qcache.set(qname, qrl)
   // debug('qcache', Object.keys(qcache), 'set', qname, ' => ', qcache[qname])
 }
 

package/src/scheduler/jobExecutor.js

@@ -373,6 +373,7 @@ export class JobExecutor {
     // Begin tracking jobs
     const jobs = messages.map(message => this.addJob(message, callback, qname, qrl))
     const isFifo = qrl.endsWith('.fifo')
+    const runningJobs = []
 
     // console.log(jobs)
 
@@ -385,7 +386,8 @@ export class JobExecutor {
       // console.log({ i, nextJobAtt: nextJob?.message?.Attributes, nextJobIsSerial })
       // Execute serial or parallel
       if (nextJobIsSerial) await this.runJob(job)
-      else this.runJob(job)
+      else runningJobs.push(this.runJob(job))
     }
+    await Promise.all(runningJobs)
   }
 }

package/src/scheduler/systemMonitor.js

@@ -3,11 +3,15 @@
  * backpressure.
  */
 
+import os from 'os'
+
 export class SystemMonitor {
   constructor (reportCallback, reportSeconds = 1) {
     this.reportCallback = reportCallback || console.log
     this.reportSeconds = reportSeconds
-    this.measurements = []
+    this.latencies = []
+    this.oneMinuteLoad = os.loadavg()[0]
+    this.instantaneousLoad = this.oneMinuteLoad
     this.measure()
     this.reportLatency()
   }
@@ -16,15 +20,20 @@ export class SystemMonitor {
     clearTimeout(this.measureTimeout)
     const start = new Date()
     this.measureTimeout = setTimeout(() => {
-      const latency = new Date() - start
-      this.measurements.push(latency)
-      if (this.measurements.length > 1000) this.measurements.shift()
+      this.measureLatency(start)
+      this.measureLoad()
       this.measure()
     })
   }
 
+  measureLatency (start) {
+    const latency = new Date() - start
+    this.latencies.push(latency)
+    if (this.latencies.length > 1000) this.latencies.shift()
+  }
+
   getLatency () {
-    return this.measurements.length ? this.measurements.reduce((a, b) => a + b, 0) / this.measurements.length : 0
+    return this.latencies.length ? this.latencies.reduce((a, b) => a + b, 0) / this.latencies.length : 0
   }
 
   reportLatency () {
@@ -37,6 +46,48 @@ export class SystemMonitor {
     }, this.reportSeconds * 1000)
   }
 
+  /**
+   * Measures load over the last five seconds instead of being averaged over one
+   * minute. This lets the scheduler respond much faster to dips in load.
+   *
+   * Theory:
+   *
+   *  The Linux kernel calculates the moving average something like:
+   *    A_1 = A_0 * e + A_now (1 - e)
+   *  Where:
+   *   - A_now is the number of processes active/waiting
+   *   - A_1 is the new one-minute load average after the measurement of A_now
+   *   - A_0 is the previous one-minute average
+   *   - e is 1884/2048.
+   *
+   *  Solving this for A_now, which we want to access, we get:
+   *   A_now = (A_1 - A_0 * e) / (1 - e)
+   *
+   *  We use this formula below to extract A_now when we detect a change in A_1.
+   *
+   * Note: this code assums that we are observing the average often enough to
+   * detect each change. So you have to call it at least every 5 seconds. 1
+   * second is better to reduce latency of detecting the change.
+   */
+
+  measureLoad () {
+    const [newLoad, ] = os.loadavg()
+    const previousLoad = this.oneMinuteLoad
+    if (previousLoad !== newLoad) {
+      const e = 1884 / 2048 // see include/linux/sched/loadavg.h
+      const active = (newLoad - previousLoad * e) / (1 - e)
+      // We take the min here so that spikes up in load are averaged out. We
+      // care about detecting spikes downward so we can allow more jobs to run.
+      this.instantaneousLoad = Math.min(active, newLoad)
+      this.oneMinuteLoad = newLoad
+      console.log({ newLoad, previousLoad, active, instantaneousLoad: this.instantaneousLoad, oneMinuteLoad: this.oneMinuteLoad })
+    }
+  }
+
+  getLoad() {
+    return this.instantaneousLoad
+  }
+
   shutdown () {
     clearTimeout(this.measureTimeout)
     clearTimeout(this.reportTimeout)