qdone 2.0.30-alpha → 2.0.32-alpha

package/commonjs/src/consumer.js

@@ -117,7 +117,7 @@ async function processMessages(queues, callback, options) {
     };
     while (!shutdownRequested) { // eslint-disable-line
         // Figure out how we are running
-        const allowedJobs = Math.max(0, opt.maxConcurrentJobs - jobExecutor.activeJobCount() - maxReturnCount);
+        const allowedJobs = Math.max(0, opt.maxConcurrentJobs - jobExecutor.runningJobCount() - maxReturnCount);
         // Latency
         const maxLatency = 100;
         const latency = systemMonitor.getLatency() || 10;
@@ -140,7 +140,8 @@ async function processMessages(queues, callback, options) {
             // console.error({ maxConcurrentJobs: opt.maxConcurrentJobs, jobCount: jobExecutor.activeJobCount(), allowedJobs, maxLatency, latencyFactor, freememFactor, loadFactor, overallFactor, targetJobs, activeQrls })
         }
         for (const { qname, qrl } of queueManager.getPairs()) {
-            // debug({ evaluating: { qname, qrl, jobsLeft, activeQrlsHasQrl: activeQrls.has(qrl) } })
+            // const qcount = jobExecutor.runningJobCountForQueue(qname)
+            // console.log({ evaluating: { qname, qrl, qcount, jobsLeft, activeQrlsHasQrl: activeQrls.has(qrl) } })
             if (jobsLeft <= 0 || activeQrls.has(qrl))
                 continue;
             const maxMessages = Math.min(10, jobsLeft);

package/commonjs/src/scheduler/jobExecutor.js

@@ -19,6 +19,7 @@ class JobExecutor {
         this.opt = opt;
         this.jobs = [];
         this.jobsByMessageId = {};
+        this.jobsByQueue = new Map();
         this.stats = {
             activeJobs: 0,
             waitingJobs: 0,
@@ -39,12 +40,27 @@ class JobExecutor {
         if (this.opt.verbose) {
             console.error(chalk_1.default.blue('Shutting down jobExecutor'));
         }
-        await this.maintainPromise;
-        await this.maintainVisibility();
+        while (this.stats.activeJobs > 0) {
+            await this.maintainPromise;
+            await this.maintainVisibility();
+        }
     }
     activeJobCount() {
         return this.stats.activeJobs;
     }
+    runningJobCount() {
+        return this.stats.runningJobs;
+    }
+    /**
+     * Returns the number of jobs running in a queue.
+     */
+    runningJobCountForQueue(qname) {
+        const jobs = this.jobsByQueue.get(qname) || new Set();
+        let runningCount = 0;
+        for (const job of jobs.values())
+            runningCount += job.status === 'running';
+        return runningCount;
+    }
     /**
      * Changes message visibility on all running jobs using as few calls as possible.
      */
@@ -103,7 +119,24 @@ class JobExecutor {
         }
         if (this.opt.verbose) {
             console.error(chalk_1.default.blue('Stats: '), { stats: this.stats, jobStatuses });
-            console.error(chalk_1.default.blue('Running: '), this.jobs.filter(j => j.status === 'processing').map(({ qname, message }) => ({ qname, payload: message.Body })));
+            console.error(chalk_1.default.blue('Jobs: '));
+            for (const [qname, jobs] of this.jobsByQueue.entries()) {
+                if (jobs.size) {
+                    console.error(chalk_1.default.blue('  queue:'), qname);
+                    for (const job of jobs)
+                        if (job.status === 'running')
+                            console.error(chalk_1.default.green('    running: '), job.payload);
+                    for (const job of jobs)
+                        if (job.status === 'waiting')
+                            console.error(chalk_1.default.gold('    waiting:'), job.payload);
+                    for (const job of jobs)
+                        if (job.status === 'complete' || job.status === 'deleting')
+                            console.error(chalk_1.default.blue('    complete:'), job.payload);
+                    for (const job of jobs)
+                        if (job.status === 'failed')
+                            console.error(chalk_1.default.red('    failed:'), job.payload);
+                }
+            }
         }
         // Extend in batches for each queue
         for (const qrl in jobsToExtendByQrl) {
@@ -194,7 +227,9 @@ class JobExecutor {
         this.jobs = this.jobs.filter(job => {
             if (job.status === 'deleting' || job.status === 'failed') {
                 debug('removed', job.message.MessageId);
+                // Accounting
                 delete this.jobsByMessageId[job.message.MessageId];
+                this.jobsByQueue.get(job.qname).delete(job);
                 return false;
             }
             else {
@@ -229,8 +264,13 @@ class JobExecutor {
             // TODO: sentry breadcrumb
             throw e;
         }
+        // Accounting
         this.jobs.push(job);
         this.jobsByMessageId[job.message.MessageId] = job;
+        // Track all jobs for each queue
+        if (!this.jobsByQueue.has(job.qname))
+            this.jobsByQueue.set(job.qname, new Set());
+        this.jobsByQueue.get(job.qname).add(job);
         this.stats.activeJobs++;
         this.stats.waitingJobs++;
         if (this.opt.verbose) {
@@ -321,11 +361,9 @@ class JobExecutor {
         // Begin executing
         for (const [job, i] of jobs.map((job, i) => [job, i])) {
             // Figure out if the next job needs to happen in serial, otherwise we can parallel execute
-            // const job = jobs[i]
             const nextJob = jobs[i + 1];
-            const nextJobIsSerial = isFifo && nextJob &&
-                job.message?.Attributes?.GroupId === nextJob.message?.Attributes?.GroupId;
-            console.log({ i, nextJobAtt: nextJob.message.Attributes, nextJobIsSerial });
+            const nextJobIsSerial = isFifo && nextJob && job.message?.Attributes?.GroupId === nextJob.message?.Attributes?.GroupId;
+            // console.log({ i, nextJobAtt: nextJob?.message?.Attributes, nextJobIsSerial })
             // Execute serial or parallel
             if (nextJobIsSerial)
                 await this.runJob(job);