qdone 2.2.4 → 2.2.5

package/commonjs/src/scheduler/jobExecutor.js

@@ -11,10 +11,13 @@ exports.JobExecutor = void 0;
 const client_sqs_1 = require("@aws-sdk/client-sqs");
 const chalk_1 = __importDefault(require("chalk"));
 const debug_1 = __importDefault(require("debug"));
+const tree_kill_1 = __importDefault(require("tree-kill"));
 const dedup_js_1 = require("../dedup.js");
 const sqs_js_1 = require("../sqs.js");
 const debug = (0, debug_1.default)('qdone:jobExecutor');
 const maxJobSeconds = 12 * 60 * 60;
+const defaultVisibilityTimeout = 120;
+const SIGKILL_DELAY_MS = 5000;
 class JobExecutor {
     constructor(opt) {
         this.opt = opt;
@@ -29,7 +32,8 @@ class JobExecutor {
             timeoutsExtended: 0,
             jobsSucceeded: 0,
             jobsFailed: 0,
-            jobsDeleted: 0
+            jobsDeleted: 0,
+            jobsKilled: 0
         };
         this.maintainPromise = this.maintainVisibility();
         debug({ this: this });
@@ -60,6 +64,103 @@ class JobExecutor {
             runningCount += job.status === 'running';
         return runningCount;
     }
+    clearJobTimers(job) {
+        clearTimeout(job.killTimer);
+        clearTimeout(job.killSignalTimer);
+    }
+    getExecutionTimeMs(job, start = new Date()) {
+        return start - job.executionStart;
+    }
+    scheduleKillAfter(job) {
+        if (!this.opt.killAfter)
+            return;
+        clearTimeout(job.killTimer);
+        job.killTimer = setTimeout(() => {
+            job.killDue = true;
+            this.killJob(job, new Date());
+        }, this.opt.killAfter * 1000);
+        job.killTimer.unref?.();
+    }
+    killJob(job, start = new Date()) {
+        if (!job.executionStart || job.status !== 'running')
+            return;
+        if (job.killed)
+            return;
+        const executionTimeMs = this.getExecutionTimeMs(job, start);
+        if (executionTimeMs < this.opt.killAfter * 1000)
+            return;
+        const executionTime = Math.floor(executionTimeMs / 1000);
+        job.killDue = true;
+        if (!job.pid) {
+            debug('killAfter reached before PID registration', { messageId: job.message?.MessageId, executionTime });
+            return;
+        }
+        job.killed = true;
+        this.stats.jobsKilled++;
+        const pid = job.pid;
+        const killTree = this.opt.killTree || tree_kill_1.default;
+        if (this.opt.verbose) {
+            console.error(chalk_1.default.red('KILLING'), job.prettyQname, chalk_1.default.red('pid'), pid, chalk_1.default.red('after'), executionTime, chalk_1.default.red('seconds (limit:'), this.opt.killAfter + ')');
+        }
+        else if (!this.opt.disableLog) {
+            console.log(JSON.stringify({
+                event: 'JOB_KILL_AFTER',
+                timestamp: start,
+                queue: job.qname,
+                messageId: job.message.MessageId,
+                pid,
+                executionTime,
+                killAfter: this.opt.killAfter,
+                payload: job.payload
+            }));
+        }
+        killTree(pid, 'SIGTERM', (err) => {
+            if (err)
+                debug('treeKill SIGTERM error', err.message);
+        });
+        clearTimeout(job.killSignalTimer);
+        job.killSignalTimer = setTimeout(() => {
+            try {
+                process.kill(pid, 0);
+            }
+            catch (e) {
+                if (e.code === 'ESRCH')
+                    return;
+            }
+            killTree(pid, 'SIGKILL', (err) => {
+                if (err)
+                    debug('treeKill SIGKILL error', err.message);
+            });
+        }, SIGKILL_DELAY_MS);
+        job.killSignalTimer.unref?.();
+    }
+    async setRunningVisibilityTimeout(job) {
+        if (!this.opt.killAfter)
+            return;
+        const visibilityTimeout = Math.max(1, Math.min(job.visibilityTimeout, this.opt.killAfter));
+        if (visibilityTimeout >= job.visibilityTimeout)
+            return;
+        job.visibilityTimeout = visibilityTimeout;
+        job.extendAtSecond = Math.round(job.visibilityTimeout / 2);
+        const input = {
+            QueueUrl: job.qrl,
+            ReceiptHandle: job.message.ReceiptHandle,
+            VisibilityTimeout: job.visibilityTimeout
+        };
+        debug({ ChangeMessageVisibility: input });
+        try {
+            const result = await (0, sqs_js_1.getSQSClient)().send(new client_sqs_1.ChangeMessageVisibilityCommand(input));
+            debug('ChangeMessageVisibility returned', result);
+            this.stats.sqsCalls++;
+            this.stats.timeoutsExtended++;
+        }
+        catch (err) {
+            debug('ChangeMessageVisibility error', err);
+            if (this.opt.verbose) {
+                console.error(chalk_1.default.red('FAILED_TO_SET_VISIBILITY_TIMEOUT'), { err, input });
+            }
+        }
+    }
     /**
      * Changes message visibility on all running jobs using as few calls as possible.
      */
@@ -101,18 +202,33 @@ class JobExecutor {
             else if (job.status !== 'deleting') {
                 // Any other job state gets visibility accounting
                 debug('processing', { job, jobRunTime });
+                // Kill-after enforcement: terminate child process if it exceeds the deadline.
+                // Uses executionStart (when runJob began) so FIFO serial jobs aren't
+                // penalized for queue wait time.
+                if (this.opt.killAfter && job.executionStart && !job.killed) {
+                    const executionTimeMs = this.getExecutionTimeMs(job, start);
+                    if (executionTimeMs >= this.opt.killAfter * 1000) {
+                        job.killDue = true;
+                        this.killJob(job, start);
+                    }
+                }
                 if (jobRunTime >= job.extendAtSecond) {
                     // Add it to our organized list of jobs
                     const jobsToExtend = jobsToExtendByQrl[job.qrl] || [];
                     jobsToExtend.push(job);
                     jobsToExtendByQrl[job.qrl] = jobsToExtend;
-                    // Update the visibility timeout, double every time, up to max
+                    // Update the visibility timeout, double every time, up to max.
+                    // Only cap at killAfter once execution has started — waiting FIFO
+                    // jobs should not have their visibility reduced prematurely.
                     const doubled = job.visibilityTimeout * 2;
                     const secondsUntilMax = Math.max(1, maxJobSeconds - jobRunTime);
-                    // const secondsUntilKill = Math.max(1, this.opt.killAfter - jobRunTime)
-                    job.visibilityTimeout = Math.min(doubled, secondsUntilMax); //, secondsUntilKill)
+                    const executionTimeMs = job.executionStart ? this.getExecutionTimeMs(job, start) : 0;
+                    const secondsUntilKill = (this.opt.killAfter && job.executionStart)
+                        ? Math.max(1, Math.ceil((this.opt.killAfter * 1000 - executionTimeMs) / 1000))
+                        : Infinity;
+                    job.visibilityTimeout = Math.min(doubled, secondsUntilMax, secondsUntilKill);
                     job.extendAtSecond = Math.round(jobRunTime + job.visibilityTimeout / 2); // this is what we use next time
-                    debug({ doubled, secondsUntilMax, job });
+                    debug({ doubled, secondsUntilMax, secondsUntilKill, job });
                 }
             }
         }
@@ -160,7 +276,7 @@ class JobExecutor {
                 const result = await (0, sqs_js_1.getSQSClient)().send(new client_sqs_1.ChangeMessageVisibilityBatchCommand(input));
                 debug('ChangeMessageVisibilityBatch returned', result);
                 this.stats.sqsCalls++;
-                if (result.Failed) {
+                if (result.Failed?.length) {
                     console.error('FAILED_MESSAGES', result.Failed);
                     for (const failed of result.Failed) {
                         console.error('FAILED_TO_EXTEND_JOB', { failedEntry: failed, job: this.jobsByMessageId[failed.Id] });
@@ -169,7 +285,7 @@ class JobExecutor {
                             this.jobsByMessageId[failed.Id].status = 'failed';
                     }
                 }
-                if (result.Successful) {
+                if (result.Successful?.length) {
                     const count = result.Successful.length || 0;
                     this.stats.timeoutsExtended += count;
                     if (this.opt.verbose) {
@@ -204,7 +320,7 @@ class JobExecutor {
                 debug({ DeleteMessageBatch: input });
                 const result = await (0, sqs_js_1.getSQSClient)().send(new client_sqs_1.DeleteMessageBatchCommand(input));
                 this.stats.sqsCalls++;
-                if (result.Failed) {
+                if (result.Failed?.length) {
                     console.error('FAILED_MESSAGES', result.Failed);
                     for (const failed of result.Failed) {
                         console.error('FAILED_TO_DELETE_JOB', { failedEntry: failed, job: this.jobsByMessageId[failed.Id] });
@@ -213,7 +329,7 @@ class JobExecutor {
                             this.jobsByMessageId[failed.Id].status = 'failed';
                     }
                 }
-                if (result.Successful) {
+                if (result.Successful?.length) {
                     const count = result.Successful.length || 0;
                     this.stats.jobsDeleted += count;
                     if (this.opt.verbose) {
@@ -245,7 +361,6 @@ class JobExecutor {
     }
     addJob(message, callback, qname, qrl) {
         // Create job entry and track it
-        const defaultVisibilityTimeout = 120;
         const job = {
             status: 'waiting',
             start: new Date(),
@@ -308,8 +423,11 @@ class JobExecutor {
                 }));
             }
             job.status = 'running';
+            job.executionStart = new Date();
             this.stats.runningJobs++;
             this.stats.waitingJobs--;
+            this.scheduleKillAfter(job);
+            await this.setRunningVisibilityTimeout(job);
             const queue = job.qname.slice(this.opt.prefix.length);
             const attributes = {
                 queueName: job.qname,
@@ -317,7 +435,17 @@ class JobExecutor {
                 receiveCount: job.message.Attributes?.ApproximateReceiveCount || '1',
                 sentTimestamp: job.message.Attributes?.SentTimestamp || '',
                 firstReceiveTimestamp: job.message.Attributes?.ApproximateFirstReceiveTimestamp || '',
-                messageGroupId: job.message.Attributes?.MessageGroupId || ''
+                messageGroupId: job.message.Attributes?.MessageGroupId || '',
+                /** Call with a child process PID to enable kill-after process termination. */
+                registerPid: (pid) => {
+                    if (typeof pid !== 'number' || !Number.isInteger(pid) || pid <= 1 || pid === process.pid) {
+                        debug('registerPid: rejected invalid PID', pid);
+                        return;
+                    }
+                    job.pid = pid;
+                    if (job.killDue && !job.killed)
+                        this.killJob(job, new Date());
+                }
             };
             const result = await job.callback(queue, job.payload, attributes);
             debug('executeJob callback finished', { payload: job.payload, result });
@@ -362,8 +490,11 @@ class JobExecutor {
                 }));
             }
         }
-        this.stats.activeJobs--;
-        this.stats.runningJobs--;
+        finally {
+            this.clearJobTimers(job);
+            this.stats.activeJobs--;
+            this.stats.runningJobs--;
+        }
     }
     async executeJobs(messages, callback, qname, qrl) {
         if (this.shutdownRequested)

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "qdone",
-  "version": "2.2.4",
+  "version": "2.2.5",
   "description": "A distributed scheduler for SQS",
   "type": "module",
   "main": "./index.js",

package/src/scheduler/jobExecutor.js

@@ -3,10 +3,15 @@
  * their visibility timeouts and deleting them when they are successful.
  */
 
-import { ChangeMessageVisibilityBatchCommand, DeleteMessageBatchCommand } from '@aws-sdk/client-sqs'
+import {
+  ChangeMessageVisibilityBatchCommand,
+  ChangeMessageVisibilityCommand,
+  DeleteMessageBatchCommand
+} from '@aws-sdk/client-sqs'
 
 import chalk from 'chalk'
 import Debug from 'debug'
+import treeKill from 'tree-kill'
 
 import { dedupSuccessfullyProcessed } from '../dedup.js'
 import { getSQSClient } from '../sqs.js'
@@ -14,6 +19,8 @@ import { getSQSClient } from '../sqs.js'
 const debug = Debug('qdone:jobExecutor')
 
 const maxJobSeconds = 12 * 60 * 60
+const defaultVisibilityTimeout = 120
+const SIGKILL_DELAY_MS = 5000
 
 export class JobExecutor {
   constructor (opt) {
@@ -29,7 +36,8 @@ export class JobExecutor {
       timeoutsExtended: 0,
       jobsSucceeded: 0,
       jobsFailed: 0,
-      jobsDeleted: 0
+      jobsDeleted: 0,
+      jobsKilled: 0
     }
     this.maintainPromise = this.maintainVisibility()
     debug({ this: this })
@@ -64,6 +72,103 @@ export class JobExecutor {
     return runningCount
   }
 
+  clearJobTimers (job) {
+    clearTimeout(job.killTimer)
+    clearTimeout(job.killSignalTimer)
+  }
+
+  getExecutionTimeMs (job, start = new Date()) {
+    return start - job.executionStart
+  }
+
+  scheduleKillAfter (job) {
+    if (!this.opt.killAfter) return
+    clearTimeout(job.killTimer)
+    job.killTimer = setTimeout(() => {
+      job.killDue = true
+      this.killJob(job, new Date())
+    }, this.opt.killAfter * 1000)
+    job.killTimer.unref?.()
+  }
+
+  killJob (job, start = new Date()) {
+    if (!job.executionStart || job.status !== 'running') return
+    if (job.killed) return
+
+    const executionTimeMs = this.getExecutionTimeMs(job, start)
+    if (executionTimeMs < this.opt.killAfter * 1000) return
+    const executionTime = Math.floor(executionTimeMs / 1000)
+
+    job.killDue = true
+    if (!job.pid) {
+      debug('killAfter reached before PID registration', { messageId: job.message?.MessageId, executionTime })
+      return
+    }
+
+    job.killed = true
+    this.stats.jobsKilled++
+    const pid = job.pid
+    const killTree = this.opt.killTree || treeKill
+
+    if (this.opt.verbose) {
+      console.error(chalk.red('KILLING'), job.prettyQname, chalk.red('pid'), pid,
+        chalk.red('after'), executionTime, chalk.red('seconds (limit:'), this.opt.killAfter + ')')
+    } else if (!this.opt.disableLog) {
+      console.log(JSON.stringify({
+        event: 'JOB_KILL_AFTER',
+        timestamp: start,
+        queue: job.qname,
+        messageId: job.message.MessageId,
+        pid,
+        executionTime,
+        killAfter: this.opt.killAfter,
+        payload: job.payload
+      }))
+    }
+
+    killTree(pid, 'SIGTERM', (err) => {
+      if (err) debug('treeKill SIGTERM error', err.message)
+    })
+
+    clearTimeout(job.killSignalTimer)
+    job.killSignalTimer = setTimeout(() => {
+      try { process.kill(pid, 0) } catch (e) { if (e.code === 'ESRCH') return }
+      killTree(pid, 'SIGKILL', (err) => {
+        if (err) debug('treeKill SIGKILL error', err.message)
+      })
+    }, SIGKILL_DELAY_MS)
+    job.killSignalTimer.unref?.()
+  }
+
+  async setRunningVisibilityTimeout (job) {
+    if (!this.opt.killAfter) return
+
+    const visibilityTimeout = Math.max(1, Math.min(job.visibilityTimeout, this.opt.killAfter))
+    if (visibilityTimeout >= job.visibilityTimeout) return
+
+    job.visibilityTimeout = visibilityTimeout
+    job.extendAtSecond = Math.round(job.visibilityTimeout / 2)
+
+    const input = {
+      QueueUrl: job.qrl,
+      ReceiptHandle: job.message.ReceiptHandle,
+      VisibilityTimeout: job.visibilityTimeout
+    }
+    debug({ ChangeMessageVisibility: input })
+
+    try {
+      const result = await getSQSClient().send(new ChangeMessageVisibilityCommand(input))
+      debug('ChangeMessageVisibility returned', result)
+      this.stats.sqsCalls++
+      this.stats.timeoutsExtended++
+    } catch (err) {
+      debug('ChangeMessageVisibility error', err)
+      if (this.opt.verbose) {
+        console.error(chalk.red('FAILED_TO_SET_VISIBILITY_TIMEOUT'), { err, input })
+      }
+    }
+  }
+
   /**
    * Changes message visibility on all running jobs using as few calls as possible.
    */
@@ -106,19 +211,36 @@ export class JobExecutor {
       } else if (job.status !== 'deleting') {
         // Any other job state gets visibility accounting
         debug('processing', { job, jobRunTime })
+
+        // Kill-after enforcement: terminate child process if it exceeds the deadline.
+        // Uses executionStart (when runJob began) so FIFO serial jobs aren't
+        // penalized for queue wait time.
+        if (this.opt.killAfter && job.executionStart && !job.killed) {
+          const executionTimeMs = this.getExecutionTimeMs(job, start)
+          if (executionTimeMs >= this.opt.killAfter * 1000) {
+            job.killDue = true
+            this.killJob(job, start)
+          }
+        }
+
         if (jobRunTime >= job.extendAtSecond) {
           // Add it to our organized list of jobs
           const jobsToExtend = jobsToExtendByQrl[job.qrl] || []
           jobsToExtend.push(job)
           jobsToExtendByQrl[job.qrl] = jobsToExtend
 
-          // Update the visibility timeout, double every time, up to max
+          // Update the visibility timeout, double every time, up to max.
+          // Only cap at killAfter once execution has started — waiting FIFO
+          // jobs should not have their visibility reduced prematurely.
           const doubled = job.visibilityTimeout * 2
           const secondsUntilMax = Math.max(1, maxJobSeconds - jobRunTime)
-          // const secondsUntilKill = Math.max(1, this.opt.killAfter - jobRunTime)
-          job.visibilityTimeout = Math.min(doubled, secondsUntilMax) //, secondsUntilKill)
+          const executionTimeMs = job.executionStart ? this.getExecutionTimeMs(job, start) : 0
+          const secondsUntilKill = (this.opt.killAfter && job.executionStart)
+            ? Math.max(1, Math.ceil((this.opt.killAfter * 1000 - executionTimeMs) / 1000))
+            : Infinity
+          job.visibilityTimeout = Math.min(doubled, secondsUntilMax, secondsUntilKill)
           job.extendAtSecond = Math.round(jobRunTime + job.visibilityTimeout / 2) // this is what we use next time
-          debug({ doubled, secondsUntilMax, job })
+          debug({ doubled, secondsUntilMax, secondsUntilKill, job })
         }
       }
     }
@@ -164,7 +286,7 @@ export class JobExecutor {
         const result = await getSQSClient().send(new ChangeMessageVisibilityBatchCommand(input))
         debug('ChangeMessageVisibilityBatch returned', result)
         this.stats.sqsCalls++
-        if (result.Failed) {
+        if (result.Failed?.length) {
           console.error('FAILED_MESSAGES', result.Failed)
           for (const failed of result.Failed) {
             console.error('FAILED_TO_EXTEND_JOB', { failedEntry: failed, job: this.jobsByMessageId[failed.Id] })
@@ -172,7 +294,7 @@ export class JobExecutor {
             if (this.jobsByMessageId[failed.Id]) this.jobsByMessageId[failed.Id].status = 'failed'
           }
         }
-        if (result.Successful) {
+        if (result.Successful?.length) {
           const count = result.Successful.length || 0
           this.stats.timeoutsExtended += count
           if (this.opt.verbose) {
@@ -208,7 +330,7 @@ export class JobExecutor {
         debug({ DeleteMessageBatch: input })
         const result = await getSQSClient().send(new DeleteMessageBatchCommand(input))
         this.stats.sqsCalls++
-        if (result.Failed) {
+        if (result.Failed?.length) {
           console.error('FAILED_MESSAGES', result.Failed)
           for (const failed of result.Failed) {
             console.error('FAILED_TO_DELETE_JOB', { failedEntry: failed, job: this.jobsByMessageId[failed.Id] })
@@ -216,7 +338,7 @@ export class JobExecutor {
             if (this.jobsByMessageId[failed.Id]) this.jobsByMessageId[failed.Id].status = 'failed'
           }
         }
-        if (result.Successful) {
+        if (result.Successful?.length) {
           const count = result.Successful.length || 0
           this.stats.jobsDeleted += count
           if (this.opt.verbose) {
@@ -254,7 +376,6 @@ export class JobExecutor {
 
   addJob (message, callback, qname, qrl) {
     // Create job entry and track it
-    const defaultVisibilityTimeout = 120
     const job = {
       status: 'waiting',
       start: new Date(),
@@ -319,8 +440,11 @@ export class JobExecutor {
         }))
       }
       job.status = 'running'
+      job.executionStart = new Date()
       this.stats.runningJobs++
       this.stats.waitingJobs--
+      this.scheduleKillAfter(job)
+      await this.setRunningVisibilityTimeout(job)
       const queue = job.qname.slice(this.opt.prefix.length)
       const attributes = {
         queueName: job.qname,
@@ -328,7 +452,16 @@ export class JobExecutor {
         receiveCount: job.message.Attributes?.ApproximateReceiveCount || '1',
         sentTimestamp: job.message.Attributes?.SentTimestamp || '',
         firstReceiveTimestamp: job.message.Attributes?.ApproximateFirstReceiveTimestamp || '',
-        messageGroupId: job.message.Attributes?.MessageGroupId || ''
+        messageGroupId: job.message.Attributes?.MessageGroupId || '',
+        /** Call with a child process PID to enable kill-after process termination. */
+        registerPid: (pid) => {
+          if (typeof pid !== 'number' || !Number.isInteger(pid) || pid <= 1 || pid === process.pid) {
+            debug('registerPid: rejected invalid PID', pid)
+            return
+          }
+          job.pid = pid
+          if (job.killDue && !job.killed) this.killJob(job, new Date())
+        }
       }
       const result = await job.callback(queue, job.payload, attributes)
       debug('executeJob callback finished', { payload: job.payload, result })
@@ -370,9 +503,11 @@ export class JobExecutor {
           err
         }))
       }
+    } finally {
+      this.clearJobTimers(job)
+      this.stats.activeJobs--
+      this.stats.runningJobs--
     }
-    this.stats.activeJobs--
-    this.stats.runningJobs--
   }
 
   async executeJobs (messages, callback, qname, qrl) {