queue-jobs-worker 1.0.0 → 1.0.2

package/dist/index.js

@@ -1,4 +1,5 @@
 import { randomUUID } from 'crypto';
+import { Cron } from 'croner';
 import { EventEmitter } from 'events';
 
 var __defProp = Object.defineProperty;
@@ -74,7 +75,7 @@ function hashToJob(h) {
   if (h["cron"]) job.cron = h["cron"];
   return job;
 }
-var PREFIX, k, CLAIM_LUA, RedisStorageAdapter;
+var PREFIX, k, CLAIM_LUA, RECOVER_STALLED_LUA, RedisStorageAdapter;
 var init_redis_adapter = __esm({
   "src/storage/redis.adapter.ts"() {
     PREFIX = "qjw:";
@@ -116,6 +117,48 @@ redis.call('HSET', prefix .. 'job:' .. job_id,
   'updatedAt',     ARGV[2]
 )
 return job_id
+`;
+    RECOVER_STALLED_LUA = `
+local job_key        = KEYS[1]
+local active_set     = KEYS[2]
+local waiting_zset   = KEYS[3]
+
+local job_id         = ARGV[1]
+local expected_exp   = ARGV[2]
+local expected_lock  = ARGV[3]
+local recovery_ts    = ARGV[4]
+local priority_score = tonumber(ARGV[5])
+
+-- Re-read the three guard fields in one atomic HMGET.
+local fields = redis.call('HMGET', job_key, 'lockExpiresAt', 'lockId', 'status')
+local current_exp    = fields[1] or ''
+local current_lock   = fields[2] or ''
+local current_status = fields[3] or ''
+
+-- Guard 1: job must still be active.
+-- If the worker already completed, failed, or moved to DLQ, skip.
+if current_status ~= 'active' then return 0 end
+
+-- Guard 2: lockId must not have changed.
+-- A different worker may have claimed the job after the original lock expired
+-- and before this script runs.
+if current_lock ~= expected_lock then return 0 end
+
+-- Guard 3: lockExpiresAt must be exactly what the caller observed.
+-- If the worker renewed its lock the timestamp will be later than observed;
+-- the CAS mismatch catches that without needing ISO\u2192epoch conversion in Lua.
+if current_exp ~= expected_exp then return 0 end
+
+-- All guards passed \u2014 recover atomically.
+redis.call('HSET', job_key,
+  'status',        'waiting',
+  'lockId',        '',
+  'lockExpiresAt', '',
+  'updatedAt',     recovery_ts
+)
+redis.call('SREM', active_set,   job_id)
+redis.call('ZADD', waiting_zset, priority_score, job_id)
+return 1
 `;
     RedisStorageAdapter = class {
       client;
@@ -292,14 +335,26 @@ return job_id
       // Release lock
       // -------------------------------------------------------------------------
       async releaseLock(jobId) {
+        const now = (/* @__PURE__ */ new Date()).toISOString();
         await this.client.hSet(k.job(jobId), {
           lockId: "",
-          lockExpiresAt: "",
-          updatedAt: (/* @__PURE__ */ new Date()).toISOString()
+          lockExpiresAt: now,
+          updatedAt: now
         });
       }
       // -------------------------------------------------------------------------
-      // Recover stalled jobs  (batched — one pipeline per stalled job)
+      // Recover stalled jobs  (atomic compare-and-swap per job via Lua)
+      //
+      // Previous implementation: two-phase read pipeline → write pipeline.
+      // Race condition: a worker could complete or renew its lock between the two
+      // phases, causing recovery to overwrite a legitimately-active job.
+      //
+      // Fix (issue #6): for each candidate job the RECOVER_STALLED_LUA script
+      // re-reads lockExpiresAt, lockId, and status atomically and only applies
+      // the recovery if all three still match what was observed in the read phase
+      // (compare-and-swap).  If the worker renewed or completed the job in the
+      // window between the read and the Lua call, the CAS mismatch causes the
+      // script to return 0 and the job is left untouched.
       // -------------------------------------------------------------------------
       async recoverStalledJobs(queue, now) {
         const nowMs = new Date(now).getTime();
@@ -307,32 +362,29 @@ return job_id
         if (activeIds.length === 0) return [];
         const fetchPipeline = this.client.multi();
         for (const jobId of activeIds) {
-          fetchPipeline.hmGet(k.job(jobId), ["lockExpiresAt", "priority"]);
+          fetchPipeline.hmGet(k.job(jobId), ["lockExpiresAt", "lockId", "priority"]);
         }
         const fetchResults = await fetchPipeline.exec();
         const recovered = [];
-        const recoverPipeline = this.client.multi();
+        const evalPromises = [];
         for (let i = 0; i < activeIds.length; i++) {
           const jobId = activeIds[i];
           const fields = fetchResults[i];
           if (!fields) continue;
-          const [lockExpiresAt, priorityStr] = fields;
+          const [lockExpiresAt, lockId, priorityStr] = fields;
           if (!lockExpiresAt) continue;
           if (new Date(lockExpiresAt).getTime() > nowMs) continue;
           const priority = Number(priorityStr ?? "0");
-          recoverPipeline.hSet(k.job(jobId), {
-            status: "waiting",
-            lockId: "",
-            lockExpiresAt: "",
-            updatedAt: now
+          const priorityScore = String(-priority);
+          const p = this.client.eval(RECOVER_STALLED_LUA, {
+            keys: [k.job(jobId), k.active(queue), k.waiting(queue)],
+            arguments: [jobId, lockExpiresAt, lockId ?? "", now, priorityScore]
+          }).then((result) => {
+            if (result === 1) recovered.push(jobId);
           });
-          recoverPipeline.sRem(k.active(queue), jobId);
-          recoverPipeline.zAdd(k.waiting(queue), { score: -priority, value: jobId });
-          recovered.push(jobId);
-        }
-        if (recovered.length > 0) {
-          await recoverPipeline.exec();
+          evalPromises.push(p);
         }
+        await Promise.all(evalPromises);
         return recovered;
       }
       // -------------------------------------------------------------------------
@@ -676,7 +728,7 @@ CREATE TABLE IF NOT EXISTS qjw_rate_limits (
       async releaseLock(jobId) {
         await this.pool.query(
           `UPDATE qjw_jobs
-       SET lock_id = NULL, lock_expires_at = NULL, updated_at = NOW()
+       SET lock_id = NULL, lock_expires_at = NOW(), updated_at = NOW()
        WHERE id = $1`,
           [jobId]
         );
@@ -1027,7 +1079,7 @@ CREATE TABLE IF NOT EXISTS qjw_rate_limits (
       async releaseLock(jobId) {
         await this.pool.query(
           `UPDATE qjw_jobs
-       SET lock_id = NULL, lock_expires_at = NULL, updated_at = NOW(3)
+       SET lock_id = NULL, lock_expires_at = NOW(3), updated_at = NOW(3)
        WHERE id = ?`,
           [jobId]
         );
@@ -1458,6 +1510,13 @@ var Worker = class {
   // -------------------------------------------------------------------------
   async claimNext() {
     const now = (/* @__PURE__ */ new Date()).toISOString();
+    const raw = await this.storage.claim({
+      queue: this.queueName,
+      lockId: this.id,
+      lockDuration: this.config.lockDuration,
+      now
+    });
+    if (!raw) return false;
     if (this.config.rateLimit) {
       const allowed = await this.storage.checkAndIncrementRateLimit(
         this.queueName,
@@ -1465,15 +1524,11 @@ var Worker = class {
         this.config.rateLimit.duration,
         now
       );
-      if (!allowed) return false;
+      if (!allowed) {
+        await this.storage.releaseLock(raw.id);
+        return false;
+      }
     }
-    const raw = await this.storage.claim({
-      queue: this.queueName,
-      lockId: this.id,
-      lockDuration: this.config.lockDuration,
-      now
-    });
-    if (!raw) return false;
     const job = new Job(raw);
     this.activeCount += 1;
     this.activeJobIds.add(job.id);
@@ -1523,25 +1578,25 @@ var Worker = class {
   // -------------------------------------------------------------------------
   async enqueueCronNext(job) {
     try {
-      let nextMs;
+      let nextDate;
       try {
-        const specifier = "croner";
-        const croner = await import(
-          /* @vite-ignore */
-          specifier
+        const cronInstance = new Cron(job.cron);
+        nextDate = cronInstance.nextRun();
+      } catch (cronErr) {
+        const error = cronErr instanceof Error ? cronErr : new Error(
+          `croner failed to initialize for expression "${job.cron}": ${String(cronErr)}`
         );
-        const CronClass = croner.Cron ?? croner.default?.Cron ?? croner.default;
-        if (typeof CronClass === "function") {
-          const cronInstance = new CronClass(job.cron);
-          const nextDate = cronInstance.nextRun();
-          nextMs = nextDate ? nextDate.getTime() : Date.now() + 6e4;
-        } else {
-          nextMs = Date.now() + 6e4;
-        }
-      } catch {
-        nextMs = Date.now() + 6e4;
+        this.emitter.emit("worker:error", this.id, error);
+        return;
       }
-      const runAt = new Date(nextMs).toISOString();
+      if (nextDate === null) {
+        const error = new Error(
+          `Cron expression "${job.cron}" has no future occurrences \u2014 job "${job.id}" will not be re-enqueued`
+        );
+        this.emitter.emit("worker:error", this.id, error);
+        return;
+      }
+      const runAt = nextDate.toISOString();
       await this.storage.enqueue({
         id: generateJobId(),
         queue: this.queueName,
@@ -1902,12 +1957,19 @@ var InMemoryStorageAdapter = class {
   async releaseLock(jobId) {
     const job = this.jobs.get(jobId);
     if (!job) return;
+    const now = (/* @__PURE__ */ new Date()).toISOString();
     job.lockId = null;
-    job.lockExpiresAt = null;
-    job.updatedAt = (/* @__PURE__ */ new Date()).toISOString();
+    job.lockExpiresAt = now;
+    job.updatedAt = now;
   }
   // -------------------------------------------------------------------------
   // Recover stalled jobs
+  //
+  // Fix (issue #6): snapshot lockId and lockExpiresAt before the eligibility
+  // check, then re-validate both values at write time.  In a single-process
+  // scenario all operations are synchronous within one event-loop tick, so
+  // the race is theoretical — but the guard makes the adapter consistent with
+  // the Redis CAS semantics and protects against any future async paths.
   // -------------------------------------------------------------------------
   async recoverStalledJobs(queue, now) {
     const nowMs = new Date(now).getTime();
@@ -1917,13 +1979,17 @@ var InMemoryStorageAdapter = class {
       if (job.status !== "active") continue;
       if (job.lockExpiresAt === null) continue;
       const lockExpiry = new Date(job.lockExpiresAt).getTime();
-      if (lockExpiry <= nowMs) {
-        job.status = "waiting";
-        job.lockId = null;
-        job.lockExpiresAt = null;
-        job.updatedAt = now;
-        recovered.push(job.id);
-      }
+      if (lockExpiry > nowMs) continue;
+      const snapshotLockId = job.lockId;
+      const snapshotLockExpiresAt = job.lockExpiresAt;
+      if (job.status !== "active") continue;
+      if (job.lockId !== snapshotLockId) continue;
+      if (job.lockExpiresAt !== snapshotLockExpiresAt) continue;
+      job.status = "waiting";
+      job.lockId = null;
+      job.lockExpiresAt = null;
+      job.updatedAt = now;
+      recovered.push(job.id);
     }
     return recovered;
   }