qdone 2.0.16-alpha → 2.0.18-alpha

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "qdone",
-  "version": "2.0.16-alpha",
+  "version": "2.0.18-alpha",
   "description": "Language agnostic job queue for SQS",
   "type": "module",
   "main": "./index.js",

package/src/consumer.js

@@ -2,6 +2,7 @@
  * Consumer implementation.
  */
 
+import { freemem, totalmem } from 'os'
 import { ReceiveMessageCommand, QueueDoesNotExist } from '@aws-sdk/client-sqs'
 import chalk from 'chalk'
 import Debug from 'debug'
@@ -117,9 +118,10 @@ export async function processMessages (queues, callback, options) {
     const maxLatency = 100
     const latency = systemMonitor.getLatency() || 10
     const latencyFactor = 1 - Math.abs(Math.min(latency / maxLatency, 1)) // 0 if latency is at max, 1 if latency 0
-    const targetJobs = Math.round(allowedJobs * latencyFactor)
+    const freememFactor = Math.min(1, Math.max(0, freemem() - totalmem() / 2) / totalmem())
+    const targetJobs = Math.round(allowedJobs * latencyFactor * freememFactor)
     let jobsLeft = targetJobs
-    debug({ jobCount: jobExecutor.activeJobCount(), maxReturnCount, allowedJobs, maxLatency, latency, latencyFactor, targetJobs, activeQrls })
+    debug({ jobCount: jobExecutor.activeJobCount(), maxReturnCount, allowedJobs, maxLatency, latency, latencyFactor, freememFactor, targetJobs, activeQrls })
     for (const { qname, qrl } of queueManager.getPairs()) {
       debug({ evaluating: { qname, qrl, jobsLeft, activeQrlsHasQrl: activeQrls.has(qrl) } })
       if (jobsLeft <= 0 || activeQrls.has(qrl)) continue

package/src/idleQueues.js

@@ -37,15 +37,24 @@ const metricNames = [
  * Actual SQS call, used in conjunction with cache.
  */
 export async function _cheapIdleCheck (qname, qrl, opt) {
-  const client = getSQSClient()
-  const cmd = new GetQueueAttributesCommand({ AttributeNames: attributeNames, QueueUrl: qrl })
-  const data = await client.send(cmd)
-  debug('data', data)
-  const result = data.Attributes
-  result.queue = qname.slice(opt.prefix.length)
-  // We are idle if all the messages attributes are zero
-  result.idle = attributeNames.filter(k => result[k] === '0').length === attributeNames.length
-  return { result, SQS: 1 }
+  try {
+    const client = getSQSClient()
+    const cmd = new GetQueueAttributesCommand({ AttributeNames: attributeNames, QueueUrl: qrl })
+    const data = await client.send(cmd)
+    debug('data', data)
+    const result = data.Attributes
+    result.queue = qname.slice(opt.prefix.length)
+    // We are idle if all the messages attributes are zero
+    result.idle = attributeNames.filter(k => result[k] === '0').length === attributeNames.length
+    return { result, SQS: 1 }
+  } catch (e) {
+    if (e instanceof QueueDoesNotExist) {
+      // Count deleted queues as idle
+      return { idle: true, SQS: 1 }
+    } else {
+      throw e
+    }
+  }
 }
 
 /**

package/src/scheduler/jobExecutor.js

@@ -242,7 +242,7 @@ export class JobExecutor {
     // Execute job
     try {
       const queue = qname.slice(this.opt.prefix.length)
-      const result = await withSentry(() => callback(queue, payload), this.opt, { job })
+      const result = await callback(queue, payload)
       debug('executeJob callback finished', { payload, result })
       if (this.opt.verbose) {
         console.error(chalk.green('SUCCESS'), message.Body)