qdone 2.0.15-alpha → 2.0.17-alpha

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "qdone",
-  "version": "2.0.15-alpha",
+  "version": "2.0.17-alpha",
   "description": "Language agnostic job queue for SQS",
   "type": "module",
   "main": "./index.js",

package/src/consumer.js

@@ -2,6 +2,7 @@
  * Consumer implementation.
  */
 
+import { freemem, totalmem } from 'os'
 import { ReceiveMessageCommand, QueueDoesNotExist } from '@aws-sdk/client-sqs'
 import chalk from 'chalk'
 import Debug from 'debug'
@@ -117,9 +118,10 @@ export async function processMessages (queues, callback, options) {
     const maxLatency = 100
     const latency = systemMonitor.getLatency() || 10
     const latencyFactor = 1 - Math.abs(Math.min(latency / maxLatency, 1)) // 0 if latency is at max, 1 if latency 0
-    const targetJobs = Math.round(allowedJobs * latencyFactor)
+    const freememFactor = Math.min(1, Math.max(0, freemem() - totalmem() / 2) / totalmem())
+    const targetJobs = Math.round(allowedJobs * latencyFactor * freememFactor)
     let jobsLeft = targetJobs
-    debug({ jobCount: jobExecutor.activeJobCount(), maxReturnCount, allowedJobs, maxLatency, latency, latencyFactor, targetJobs, activeQrls })
+    debug({ jobCount: jobExecutor.activeJobCount(), maxReturnCount, allowedJobs, maxLatency, latency, latencyFactor, freememFactor, targetJobs, activeQrls })
     for (const { qname, qrl } of queueManager.getPairs()) {
       debug({ evaluating: { qname, qrl, jobsLeft, activeQrlsHasQrl: activeQrls.has(qrl) } })
       if (jobsLeft <= 0 || activeQrls.has(qrl)) continue

package/src/scheduler/jobExecutor.js

@@ -3,13 +3,12 @@
  * their visibility timeouts and deleting them when they are successful.
  */
 
-import {
-  ChangeMessageVisibilityBatchCommand,
-  DeleteMessageBatchCommand
-} from '@aws-sdk/client-sqs'
+import { ChangeMessageVisibilityBatchCommand, DeleteMessageBatchCommand } from '@aws-sdk/client-sqs'
+
 import chalk from 'chalk'
 import Debug from 'debug'
 
+import { withSentry } from '../sentry.js'
 import { getSQSClient } from '../sqs.js'
 
 const debug = Debug('qdone:jobExecutor')
@@ -243,7 +242,7 @@ export class JobExecutor {
     // Execute job
     try {
       const queue = qname.slice(this.opt.prefix.length)
-      const result = await callback(queue, payload)
+      const result = await withSentry(() => callback(queue, payload), this.opt, { job })
       debug('executeJob callback finished', { payload, result })
       if (this.opt.verbose) {
         console.error(chalk.green('SUCCESS'), message.Body)
@@ -263,7 +262,6 @@ export class JobExecutor {
       }
       this.stats.jobsSucceeded++
     } catch (err) {
-      // debug('exec.catch', err)
       // Fail path for job execution
       if (this.opt.verbose) {
         console.error(chalk.red('FAILED'), message.Body)

package/src/sentry.js

@@ -3,12 +3,13 @@
  */
 
 import Debug from 'debug'
-import { init, captureException } from '@sentry/node'
+import { init, withScope, captureException } from '@sentry/node'
 
 const debug = Debug('qdone:sentry')
 
 let sentryWasInit = false
-export async function withSentry (callback, opt) {
+export async function withSentry (callback, opt, contexts) {
+  debug({ withSentry: { callback, opt, contexts } })
   // Bail if sentry isn't enabled
   if (!opt.sentryDsn) return callback()
 
@@ -23,8 +24,20 @@ export async function withSentry (callback, opt) {
     return result
   } catch (err) {
     debug({ err })
-    const sentryResult = await captureException(err)
-    debug({ sentryResult })
+    await withScope(async function (scope) {
+      scope.setContext('opt', opt)
+      if (contexts instanceof Object) {
+        for (const key in contexts) scope.setContext(key, contexts[key])
+        if (err.stdout && err.stderr) {
+          const { stdout, stderr } = err
+          scope.setContext('IO', { stdout, stderr })
+        }
+      }
+      const sentryResult = await captureException(err)
+      debug({ sentryResult })
+    })
     throw err
   }
 }
+
+debug('loaded')