qdone 2.0.33-alpha → 2.0.35-alpha

package/commonjs/src/consumer.js

@@ -61,7 +61,7 @@ async function processMessages(queues, callback, options) {
         lastLatency = latency;
     });
     const jobExecutor = new jobExecutor_js_1.JobExecutor(opt);
-    const queueManager = new queueManager_js_1.QueueManager(opt, queues, 60);
+    const queueManager = new queueManager_js_1.QueueManager(opt, queues, 10);
     const cores = (0, os_1.cpus)().length;
     // debug({ systemMonitor, jobExecutor, queueManager })
     // This delay function keeps a timeout reference around so it can be
@@ -101,8 +101,7 @@ async function processMessages(queues, callback, options) {
                 }
             }
             // Max job accounting
-            if (messages.length)
-                maxReturnCount -= messages.length;
+            maxReturnCount -= maxMessages;
             activeQrls.delete(qrl);
         }
         catch (e) {
@@ -117,7 +116,8 @@ async function processMessages(queues, callback, options) {
     };
     while (!shutdownRequested) { // eslint-disable-line
         // Figure out how we are running
-        const allowedJobs = Math.max(0, opt.maxConcurrentJobs - jobExecutor.runningJobCount() - maxReturnCount);
+        const runningJobs = jobExecutor.runningJobCount();
+        const allowedJobs = Math.max(0, opt.maxConcurrentJobs - maxReturnCount - runningJobs);
         // Latency
         const maxLatency = 100;
         const latency = systemMonitor.getLatency() || 10;
@@ -137,12 +137,14 @@ async function processMessages(queues, callback, options) {
         const targetJobs = Math.round(allowedJobs * overallFactor);
         let jobsLeft = targetJobs;
         if (opt.verbose) {
-            // console.error({ maxConcurrentJobs: opt.maxConcurrentJobs, jobCount: jobExecutor.activeJobCount(), allowedJobs, maxLatency, latencyFactor, freememFactor, loadFactor, overallFactor, targetJobs, activeQrls })
+            console.error({ maxConcurrentJobs: opt.maxConcurrentJobs, maxReturnCount, runningJobs, allowedJobs, maxLatency, latencyFactor, freememFactor, loadFactor, overallFactor, targetJobs, activeQrls });
         }
         for (const { qname, qrl } of queueManager.getPairs()) {
             // const qcount = jobExecutor.runningJobCountForQueue(qname)
             // console.log({ evaluating: { qname, qrl, qcount, jobsLeft, activeQrlsHasQrl: activeQrls.has(qrl) } })
-            if (jobsLeft <= 0 || activeQrls.has(qrl))
+            if (jobsLeft <= 0)
+                break;
+            if (activeQrls.has(qrl))
                 continue;
             const maxMessages = Math.min(10, jobsLeft);
             listen(qname, qrl, maxMessages);

package/commonjs/src/defaults.js

@@ -20,6 +20,7 @@ exports.defaults = Object.freeze({
     fifo: false,
     disableLog: false,
     includeFailed: false,
+    includeDead: false,
     // Enqueue
     groupId: (0, uuid_1.v1)(),
     groupIdPerMessage: false,
@@ -73,6 +74,8 @@ function getOptionsWithDefaults(options) {
         fifo: options.fifo || exports.defaults.fifo,
         sentryDsn: options.sentryDsn || options['sentry-dsn'],
         disableLog: options.disableLog || options['disable-log'] || exports.defaults.disableLog,
+        includeFailed: options.includeFailed || options['include-failed'] || exports.defaults.includeFailed,
+        includeDead: options.includeDead || options['include-dead'] || exports.defaults.includeDead,
         // Cache
         cacheUri: options.cacheUri || options['cache-uri'] || exports.defaults.cacheUri,
         cachePrefix: options.cachePrefix || options['cache-prefix'] || exports.defaults.cachePrefix,
@@ -94,7 +97,6 @@ function getOptionsWithDefaults(options) {
         killAfter: options.killAfter || options['kill-after'] || exports.defaults.killAfter,
         archive: options.archive || exports.defaults.archive,
         activeOnly: options.activeOnly || options['active-only'] || exports.defaults.activeOnly,
-        includeFailed: options.includeFailed || options['include-failed'] || exports.defaults.includeFailed,
         maxConcurrentJobs: options.maxConcurrentJobs || exports.defaults.maxConcurrentJobs,
         maxMemoryPercent: options.maxMemoryPercent || exports.defaults.maxMemoryPercent,
         // Idle Queues

package/commonjs/src/scheduler/queueManager.js

@@ -105,7 +105,7 @@ class QueueManager {
             .filter(({ qname, qrl }) => {
             const isFifo = qname.endsWith('.fifo');
             const isDead = isFifo ? qname.endsWith('_dead.fifo') : qname.endsWith('_dead');
-            return !isDead;
+            return this.opt.includeDead ? true : !isDead;
         })
             // then icehouse
             .filter(({ qname, qrl }) => !this.keepInIcehouse(qrl, now));
@@ -146,6 +146,7 @@ class QueueManager {
     }
     getPairs() {
         const now = new Date();
+        this.selectedPairs.sort(() => 0.5 - Math.random());
         return this.selectedPairs.filter(({ qname, qrl }) => !this.keepInIcehouse(qrl, now));
     }
     async shutdown() {

package/npm-shrinkwrap.json

@@ -1,12 +1,12 @@
 {
   "name": "qdone",
-  "version": "2.0.33-alpha",
+  "version": "2.0.35-alpha",
   "lockfileVersion": 2,
   "requires": true,
   "packages": {
     "": {
       "name": "qdone",
-      "version": "2.0.33-alpha",
+      "version": "2.0.35-alpha",
       "license": "MIT",
       "dependencies": {
         "@aws-sdk/client-cloudwatch": "3.465.0",

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "qdone",
-  "version": "2.0.33-alpha",
-  "description": "Language agnostic job queue for SQS",
+  "version": "2.0.35-alpha",
+  "description": "A distributed scheduler for SQS",
   "type": "module",
   "main": "./index.js",
   "module": "./index.js",

package/src/consumer.js

@@ -61,7 +61,7 @@ export async function processMessages (queues, callback, options) {
     lastLatency = latency
   })
   const jobExecutor = new JobExecutor(opt)
-  const queueManager = new QueueManager(opt, queues, 60)
+  const queueManager = new QueueManager(opt, queues, 10)
   const cores = cpus().length
   // debug({ systemMonitor, jobExecutor, queueManager })
 
@@ -105,7 +105,7 @@ export async function processMessages (queues, callback, options) {
       }
 
       // Max job accounting
-      if (messages.length) maxReturnCount -= messages.length
+      maxReturnCount -= maxMessages
       activeQrls.delete(qrl)
     } catch (e) {
       // If the queue has been cleaned up, we should back off anyway
@@ -119,7 +119,8 @@ export async function processMessages (queues, callback, options) {
 
   while (!shutdownRequested) { // eslint-disable-line
     // Figure out how we are running
-    const allowedJobs = Math.max(0, opt.maxConcurrentJobs - jobExecutor.runningJobCount() - maxReturnCount)
+    const runningJobs = jobExecutor.runningJobCount()
+    const allowedJobs = Math.max(0, opt.maxConcurrentJobs - maxReturnCount - runningJobs)
 
     // Latency
     const maxLatency = 100
@@ -144,12 +145,13 @@ export async function processMessages (queues, callback, options) {
     let jobsLeft = targetJobs
 
     if (opt.verbose) {
-      // console.error({ maxConcurrentJobs: opt.maxConcurrentJobs, jobCount: jobExecutor.activeJobCount(), allowedJobs, maxLatency, latencyFactor, freememFactor, loadFactor, overallFactor, targetJobs, activeQrls })
+      console.error({ maxConcurrentJobs: opt.maxConcurrentJobs, maxReturnCount, runningJobs, allowedJobs, maxLatency, latencyFactor, freememFactor, loadFactor, overallFactor, targetJobs, activeQrls })
     }
     for (const { qname, qrl } of queueManager.getPairs()) {
       // const qcount = jobExecutor.runningJobCountForQueue(qname)
       // console.log({ evaluating: { qname, qrl, qcount, jobsLeft, activeQrlsHasQrl: activeQrls.has(qrl) } })
-      if (jobsLeft <= 0 || activeQrls.has(qrl)) continue
+      if (jobsLeft <= 0) break
+      if (activeQrls.has(qrl)) continue
       const maxMessages = Math.min(10, jobsLeft)
       listen(qname, qrl, maxMessages)
       jobsLeft -= maxMessages

package/src/defaults.js

@@ -18,6 +18,7 @@ export const defaults = Object.freeze({
   fifo: false,
   disableLog: false,
   includeFailed: false,
+  includeDead: false,
 
   // Enqueue
   groupId: uuidv1(),
@@ -76,6 +77,8 @@ export function getOptionsWithDefaults (options) {
     fifo: options.fifo || defaults.fifo,
     sentryDsn: options.sentryDsn || options['sentry-dsn'],
     disableLog: options.disableLog || options['disable-log'] || defaults.disableLog,
+    includeFailed: options.includeFailed || options['include-failed'] || defaults.includeFailed,
+    includeDead: options.includeDead || options['include-dead'] || defaults.includeDead,
 
     // Cache
     cacheUri: options.cacheUri || options['cache-uri'] || defaults.cacheUri,
@@ -100,7 +103,6 @@ export function getOptionsWithDefaults (options) {
     killAfter: options.killAfter || options['kill-after'] || defaults.killAfter,
     archive: options.archive || defaults.archive,
     activeOnly: options.activeOnly || options['active-only'] || defaults.activeOnly,
-    includeFailed: options.includeFailed || options['include-failed'] || defaults.includeFailed,
     maxConcurrentJobs: options.maxConcurrentJobs || defaults.maxConcurrentJobs,
     maxMemoryPercent: options.maxMemoryPercent || defaults.maxMemoryPercent,
 

package/src/scheduler/queueManager.js

@@ -104,7 +104,7 @@ export class QueueManager {
       .filter(({ qname, qrl }) => {
         const isFifo = qname.endsWith('.fifo')
         const isDead = isFifo ? qname.endsWith('_dead.fifo') : qname.endsWith('_dead')
-        return !isDead
+        return this.opt.includeDead ? true : !isDead
       })
       // then icehouse
       .filter(({ qname, qrl }) => !this.keepInIcehouse(qrl, now))
@@ -149,6 +149,7 @@ export class QueueManager {
 
   getPairs () {
     const now = new Date()
+    this.selectedPairs.sort(() => 0.5 - Math.random())
     return this.selectedPairs.filter(({ qname, qrl }) => !this.keepInIcehouse(qrl, now))
   }