queue-jobs-worker 1.0.4 → 1.0.5

package/dist/index.js

@@ -12,6 +12,89 @@ var __export = (target, all) => {
     __defProp(target, name, { get: all[name], enumerable: true });
 };
 
+// src/lib/scripts/claim.lua
+var claim_default;
+var init_claim = __esm({
+  "src/lib/scripts/claim.lua"() {
+    claim_default = "local prefix    = ARGV[4]\nlocal now_ms    = tonumber(ARGV[3])\n\n-- Promote due delayed jobs into the waiting set.\nlocal due = redis.call('ZRANGEBYSCORE', KEYS[2], '-inf', now_ms)\nfor _, jid in ipairs(due) do\n  local pri_raw = redis.call('HGET', prefix .. 'job:' .. jid, 'priority')\n  local pri = tonumber(pri_raw) or 0\n  redis.call('ZADD',  KEYS[1], -pri, jid)\n  redis.call('ZREM',  KEYS[2], jid)\n  redis.call('HSET',  prefix .. 'job:' .. jid, 'status', 'waiting')\nend\n\n-- Pop the top-priority job.\nlocal items = redis.call('ZPOPMIN', KEYS[1], 1)\nif #items == 0 then return '' end\nlocal job_id = items[1]\n\n-- Lock it.\nredis.call('SADD', KEYS[3], job_id)\nredis.call('HSET', prefix .. 'job:' .. job_id,\n  'status',        'active',\n  'lockId',        ARGV[1],\n  'lockExpiresAt', ARGV[2],\n  'updatedAt',     ARGV[2]\n)\nreturn job_id\n";
+  }
+});
+
+// src/lib/scripts/recover-stalled.lua
+var recover_stalled_default;
+var init_recover_stalled = __esm({
+  "src/lib/scripts/recover-stalled.lua"() {
+    recover_stalled_default = "local job_key        = KEYS[1]\nlocal active_set     = KEYS[2]\nlocal waiting_zset   = KEYS[3]\n\nlocal job_id         = ARGV[1]\nlocal expected_exp   = ARGV[2]\nlocal expected_lock  = ARGV[3]\nlocal recovery_ts    = ARGV[4]\nlocal priority_score = tonumber(ARGV[5])\n\n-- Re-read the three guard fields in one atomic HMGET.\nlocal fields = redis.call('HMGET', job_key, 'lockExpiresAt', 'lockId', 'status')\nlocal current_exp    = fields[1] or ''\nlocal current_lock   = fields[2] or ''\nlocal current_status = fields[3] or ''\n\n-- Guard 1: job must still be active.\n-- If the worker already completed, failed, or moved to DLQ, skip.\nif current_status ~= 'active' then return 0 end\n\n-- Guard 2: lockId must not have changed.\n-- A different worker may have claimed the job after the original lock expired\n-- and before this script runs.\nif current_lock ~= expected_lock then return 0 end\n\n-- Guard 3: lockExpiresAt must be exactly what the caller observed.\n-- If the worker renewed its lock the timestamp will be later than observed;\n-- the CAS mismatch catches that without needing ISO\u2192epoch conversion in Lua.\nif current_exp ~= expected_exp then return 0 end\n\n-- All guards passed \u2014 recover atomically.\nredis.call('HSET', job_key,\n  'status',        'waiting',\n  'lockId',        '',\n  'lockExpiresAt', '',\n  'updatedAt',     recovery_ts\n)\nredis.call('SREM', active_set,   job_id)\nredis.call('ZADD', waiting_zset, priority_score, job_id)\nreturn 1\n";
+  }
+});
+
+// src/lib/scripts/renew-lock.lua
+var renew_lock_default;
+var init_renew_lock = __esm({
+  "src/lib/scripts/renew-lock.lua"() {
+    renew_lock_default = "local job_key = KEYS[1]\nlocal lock_id = ARGV[1]\nlocal new_exp = ARGV[2]\nlocal now_iso = ARGV[3]\n\nlocal fields = redis.call('HMGET', job_key, 'status', 'lockId', 'lockExpiresAt')\nlocal status = fields[1] or ''\nlocal current_lock = fields[2] or ''\nlocal current_exp = fields[3] or ''\n\nif status ~= 'active' then return 0 end\nif current_lock ~= lock_id then return 0 end\nif current_exp == '' or current_exp <= now_iso then return 0 end\n\nredis.call('HSET', job_key, 'lockExpiresAt', new_exp, 'updatedAt', now_iso)\nreturn 1\n";
+  }
+});
+
+// src/lib/scripts/rate-limit.lua
+var rate_limit_default;
+var init_rate_limit = __esm({
+  "src/lib/scripts/rate-limit.lua"() {
+    rate_limit_default = `-- RATE_LIMIT_LUA
+-- KEYS[1]: rate_count_key ("qjw:rate:{queue}")
+-- KEYS[2]: rate_ts_key    ("qjw:rate:{queue}:ts")
+-- ARGV[1]: max (number)
+-- ARGV[2]: windowMs (number)
+-- ARGV[3]: nowMs (number)
+-- ARGV[4]: ttlSec (number)
+--
+-- Returns:
+--   1: Allowed
+--   0: Rejected
+
+local ck = KEYS[1]
+local tk = KEYS[2]
+local max = tonumber(ARGV[1])
+local windowMs = tonumber(ARGV[2])
+local nowMs = tonumber(ARGV[3])
+local ttlSec = tonumber(ARGV[4])
+
+local windowStart = redis.call('GET', tk)
+
+if not windowStart or (nowMs - tonumber(windowStart)) >= windowMs then
+    redis.call('SET', ck, '1', 'EX', ttlSec)
+    redis.call('SET', tk, tostring(nowMs), 'EX', ttlSec)
+    return 1
+end
+
+local current = tonumber(redis.call('GET', ck) or '0')
+if current >= max then
+    return 0
+end
+
+redis.call('INCR', ck)
+redis.call('EXPIRE', ck, ttlSec)
+redis.call('EXPIRE', tk, ttlSec)
+return 1
+`;
+  }
+});
+
+// src/lib/scripts/index.ts
+var CLAIM_LUA, RECOVER_STALLED_LUA, RENEW_LOCK_LUA, RATE_LIMIT_LUA;
+var init_scripts = __esm({
+  "src/lib/scripts/index.ts"() {
+    init_claim();
+    init_recover_stalled();
+    init_renew_lock();
+    init_rate_limit();
+    CLAIM_LUA = claim_default;
+    RECOVER_STALLED_LUA = recover_stalled_default;
+    RENEW_LOCK_LUA = renew_lock_default;
+    RATE_LIMIT_LUA = rate_limit_default;
+  }
+});
+
 // src/storage/redis.adapter.ts
 var redis_adapter_exports = {};
 __export(redis_adapter_exports, {
@@ -75,9 +158,10 @@ function hashToJob(h) {
   if (h["cron"]) job.cron = h["cron"];
   return job;
 }
-var PREFIX, k, CLAIM_LUA, RECOVER_STALLED_LUA, RENEW_LOCK_LUA, RedisStorageAdapter;
+var PREFIX, k, RedisStorageAdapter;
 var init_redis_adapter = __esm({
   "src/storage/redis.adapter.ts"() {
+    init_scripts();
     PREFIX = "qjw:";
     k = {
       job: (id) => `${PREFIX}job:${id}`,
@@ -89,95 +173,6 @@ var init_redis_adapter = __esm({
       rateCount: (q) => `${PREFIX}rate:${q}`,
       rateTs: (q) => `${PREFIX}rate:${q}:ts`
     };
-    CLAIM_LUA = `
-local prefix    = ARGV[4]
-local now_ms    = tonumber(ARGV[3])
-
--- Promote due delayed jobs into the waiting set.
-local due = redis.call('ZRANGEBYSCORE', KEYS[2], '-inf', now_ms)
-for _, jid in ipairs(due) do
-  local pri_raw = redis.call('HGET', prefix .. 'job:' .. jid, 'priority')
-  local pri = tonumber(pri_raw) or 0
-  redis.call('ZADD',  KEYS[1], -pri, jid)
-  redis.call('ZREM',  KEYS[2], jid)
-  redis.call('HSET',  prefix .. 'job:' .. jid, 'status', 'waiting')
-end
-
--- Pop the top-priority job.
-local items = redis.call('ZPOPMIN', KEYS[1], 1)
-if #items == 0 then return '' end
-local job_id = items[1]
-
--- Lock it.
-redis.call('SADD', KEYS[3], job_id)
-redis.call('HSET', prefix .. 'job:' .. job_id,
-  'status',        'active',
-  'lockId',        ARGV[1],
-  'lockExpiresAt', ARGV[2],
-  'updatedAt',     ARGV[2]
-)
-return job_id
-`;
-    RECOVER_STALLED_LUA = `
-local job_key        = KEYS[1]
-local active_set     = KEYS[2]
-local waiting_zset   = KEYS[3]
-
-local job_id         = ARGV[1]
-local expected_exp   = ARGV[2]
-local expected_lock  = ARGV[3]
-local recovery_ts    = ARGV[4]
-local priority_score = tonumber(ARGV[5])
-
--- Re-read the three guard fields in one atomic HMGET.
-local fields = redis.call('HMGET', job_key, 'lockExpiresAt', 'lockId', 'status')
-local current_exp    = fields[1] or ''
-local current_lock   = fields[2] or ''
-local current_status = fields[3] or ''
-
--- Guard 1: job must still be active.
--- If the worker already completed, failed, or moved to DLQ, skip.
-if current_status ~= 'active' then return 0 end
-
--- Guard 2: lockId must not have changed.
--- A different worker may have claimed the job after the original lock expired
--- and before this script runs.
-if current_lock ~= expected_lock then return 0 end
-
--- Guard 3: lockExpiresAt must be exactly what the caller observed.
--- If the worker renewed its lock the timestamp will be later than observed;
--- the CAS mismatch catches that without needing ISO\u2192epoch conversion in Lua.
-if current_exp ~= expected_exp then return 0 end
-
--- All guards passed \u2014 recover atomically.
-redis.call('HSET', job_key,
-  'status',        'waiting',
-  'lockId',        '',
-  'lockExpiresAt', '',
-  'updatedAt',     recovery_ts
-)
-redis.call('SREM', active_set,   job_id)
-redis.call('ZADD', waiting_zset, priority_score, job_id)
-return 1
-`;
-    RENEW_LOCK_LUA = `
-local job_key = KEYS[1]
-local lock_id = ARGV[1]
-local new_exp = ARGV[2]
-local now_iso = ARGV[3]
-
-local fields = redis.call('HMGET', job_key, 'status', 'lockId', 'lockExpiresAt')
-local status = fields[1] or ''
-local current_lock = fields[2] or ''
-local current_exp = fields[3] or ''
-
-if status ~= 'active' then return 0 end
-if current_lock ~= lock_id then return 0 end
-if current_exp == '' or current_exp <= now_iso then return 0 end
-
-redis.call('HSET', job_key, 'lockExpiresAt', new_exp, 'updatedAt', now_iso)
-return 1
-`;
     RedisStorageAdapter = class {
       client;
       url;
@@ -500,23 +495,12 @@ return 1
         const nowMs = new Date(now).getTime();
         const ck = k.rateCount(queue);
         const tk = k.rateTs(queue);
-        const windowStart = await this.client.get(tk);
-        const ttlSec = Math.ceil(windowMs / 1e3);
-        if (!windowStart || nowMs - Number(windowStart) >= windowMs) {
-          const multi = this.client.multi();
-          multi.set(ck, "1");
-          multi.expire(ck, ttlSec);
-          multi.set(tk, String(nowMs));
-          multi.expire(tk, ttlSec);
-          await multi.exec();
-          return true;
-        }
-        const count = await this.client.incr(ck);
-        if (count > max) {
-          await this.client.decr(ck);
-          return false;
-        }
-        return true;
+        const ttlSec = Math.max(1, Math.ceil(windowMs / 1e3));
+        const res = await this.client.eval(RATE_LIMIT_LUA, {
+          keys: [ck, tk],
+          arguments: [String(max), String(windowMs), String(nowMs), String(ttlSec)]
+        });
+        return Number(res) === 1;
       }
     };
   }
@@ -911,24 +895,42 @@ CREATE TABLE IF NOT EXISTS qjw_rate_limits (
       // -------------------------------------------------------------------------
       async checkAndIncrementRateLimit(queue, max, windowMs, now) {
         const nowMs = new Date(now).getTime();
-        const res = await this.pool.query(
-          "SELECT count, window_start FROM qjw_rate_limits WHERE queue = $1",
-          [queue]
-        );
-        if (res.rows.length === 0 || nowMs - Number(res.rows[0]?.window_start ?? 0) >= windowMs) {
-          await this.pool.query(
-            `INSERT INTO qjw_rate_limits (queue, count, window_start)
-         VALUES ($1, 1, $2)
-         ON CONFLICT (queue) DO UPDATE
-           SET count = 1, window_start = EXCLUDED.window_start`,
-            [queue, nowMs]
+        const client = await this.pool.connect();
+        try {
+          await client.query("BEGIN");
+          await client.query(
+            `INSERT INTO qjw_rate_limits (queue, count, window_start) VALUES ($1, 0, 0)
+         ON CONFLICT (queue) DO NOTHING`,
+            [queue]
+          );
+          const res = await client.query(
+            "SELECT count, window_start FROM qjw_rate_limits WHERE queue = $1 FOR UPDATE",
+            [queue]
           );
+          const row = res.rows[0];
+          const windowStart = Number(row?.window_start ?? 0);
+          const count = Number(row?.count ?? 0);
+          if (!row || nowMs - windowStart >= windowMs) {
+            await client.query(
+              "UPDATE qjw_rate_limits SET count = 1, window_start = $2 WHERE queue = $1",
+              [queue, nowMs]
+            );
+            await client.query("COMMIT");
+            return true;
+          }
+          if (count >= max) {
+            await client.query("COMMIT");
+            return false;
+          }
+          await client.query("UPDATE qjw_rate_limits SET count = count + 1 WHERE queue = $1", [queue]);
+          await client.query("COMMIT");
           return true;
+        } catch (err) {
+          await client.query("ROLLBACK");
+          throw err;
+        } finally {
+          client.release();
         }
-        const current = res.rows[0].count;
-        if (current >= max) return false;
-        await this.pool.query("UPDATE qjw_rate_limits SET count = count + 1 WHERE queue = $1", [queue]);
-        return true;
       }
     };
   }
@@ -1342,22 +1344,42 @@ CREATE TABLE IF NOT EXISTS qjw_rate_limits (
       // -------------------------------------------------------------------------
       async checkAndIncrementRateLimit(queue, max, windowMs, now) {
         const nowMs = new Date(now).getTime();
-        const [rows] = await this.pool.query(
-          "SELECT count, window_start FROM qjw_rate_limits WHERE queue = ?",
-          [queue]
-        );
-        const existing = rows[0];
-        if (!existing || nowMs - Number(existing["window_start"]) >= windowMs) {
-          await this.pool.query(
-            `INSERT INTO qjw_rate_limits (queue, count, window_start) VALUES (?,1,?)
-         ON DUPLICATE KEY UPDATE count = 1, window_start = VALUES(window_start)`,
-            [queue, nowMs]
+        const conn = await this.pool.getConnection();
+        try {
+          await conn.beginTransaction();
+          await conn.query(
+            `INSERT INTO qjw_rate_limits (queue, count, window_start) VALUES (?, 0, 0)
+         ON DUPLICATE KEY UPDATE queue = queue`,
+            [queue]
+          );
+          const [rows] = await conn.query(
+            "SELECT count, window_start FROM qjw_rate_limits WHERE queue = ? FOR UPDATE",
+            [queue]
           );
+          const existing = rows[0];
+          const windowStart = Number(existing?.["window_start"] ?? 0);
+          const count = Number(existing?.["count"] ?? 0);
+          if (!existing || nowMs - windowStart >= windowMs) {
+            await conn.query("UPDATE qjw_rate_limits SET count = 1, window_start = ? WHERE queue = ?", [
+              nowMs,
+              queue
+            ]);
+            await conn.commit();
+            return true;
+          }
+          if (count >= max) {
+            await conn.commit();
+            return false;
+          }
+          await conn.query("UPDATE qjw_rate_limits SET count = count + 1 WHERE queue = ?", [queue]);
+          await conn.commit();
           return true;
+        } catch (err) {
+          await conn.rollback();
+          throw err;
+        } finally {
+          conn.release();
         }
-        if (Number(existing["count"]) >= max) return false;
-        await this.pool.query("UPDATE qjw_rate_limits SET count = count + 1 WHERE queue = ?", [queue]);
-        return true;
       }
     };
   }
@@ -1895,16 +1917,25 @@ var Queue = class {
   emitter;
   resolvedConfig;
   clientDefaults;
+  isInitialisedCheck;
   /** Registered processors keyed by job type. */
   processors = /* @__PURE__ */ new Map();
   /** Active worker instances created by this queue. */
   workers = [];
-  constructor(name, storage, emitter, options, defaults) {
+  constructor(name, storage, emitter, options, defaults, isInitialisedCheck) {
     this.name = name;
     this.storage = storage;
     this.emitter = emitter;
     this.clientDefaults = defaults;
     this.resolvedConfig = mergeQueueConfig(options, defaults);
+    this.isInitialisedCheck = isInitialisedCheck;
+  }
+  assertInitialised() {
+    if (this.isInitialisedCheck && !this.isInitialisedCheck()) {
+      throw new Error(
+        `Cannot execute queue operations on queue "${this.name}" because the QueueClient is not initialised. Call "await client.init()" first.`
+      );
+    }
   }
   // -------------------------------------------------------------------------
   // Enqueue
@@ -1917,6 +1948,7 @@ var Queue = class {
    * @param options - Per-job overrides (attempts, delay, priority, etc.).
    */
   async enqueue(type, payload, options) {
+    this.assertInitialised();
     const cfg = this.resolvedConfig;
     const now = /* @__PURE__ */ new Date();
     let runAt;
@@ -1965,6 +1997,7 @@ var Queue = class {
    * Worker-level `concurrency` overrides queue-level concurrency.
    */
   createWorker(options) {
+    this.assertInitialised();
     const worker = new Worker(
       this.name,
       this.storage,
@@ -1983,12 +2016,14 @@ var Queue = class {
   // -------------------------------------------------------------------------
   /** Fetch a single job by its ID. Returns null if not found or wrong queue. */
   async getJob(jobId) {
+    this.assertInitialised();
     const raw = await this.storage.getJob(jobId);
     if (!raw || raw.queue !== this.name) return null;
     return new Job(raw);
   }
   /** Fetch jobs from this queue, optionally filtered by status. */
   async getJobs(status, limit = 100, offset = 0) {
+    this.assertInitialised();
     const raws = await this.storage.getJobs({
       queue: this.name,
       ...status !== void 0 && { status },
@@ -1999,6 +2034,7 @@ var Queue = class {
   }
   /** Get job counts by status for this queue. */
   async getJobCounts() {
+    this.assertInitialised();
     return this.storage.getJobCounts(this.name);
   }
   // -------------------------------------------------------------------------
@@ -2355,6 +2391,9 @@ var QueueClient = class _QueueClient {
     this.dialect = options.dialect ?? "memory";
     this.connectionString = options.connectionString;
     this._storage = this.buildMemoryOrEagerAdapter();
+    if (this.dialect === "memory") {
+      this.initialised = true;
+    }
     this.emitter = new QueueEventEmitter();
   }
   // -------------------------------------------------------------------------
@@ -2433,10 +2472,19 @@ var QueueClient = class _QueueClient {
     if (this.queues.has(name)) {
       throw new Error(`A queue named "${name}" already exists on this client.`);
     }
-    if (!this.initialised && this.dialect === "memory") {
-      void this.init();
+    if (!this.initialised) {
+      throw new Error(
+        `QueueClient is not initialised. Call "await client.init()" before creating queues for dialect "${this.dialect}".`
+      );
     }
-    const queue = new Queue(name, this._storage, this.emitter, options, this.defaults);
+    const queue = new Queue(
+      name,
+      this._storage,
+      this.emitter,
+      options,
+      this.defaults,
+      () => this.initialised
+    );
     this.queues.set(name, queue);
     return queue;
   }
@@ -2505,6 +2553,7 @@ var QueueClient = class _QueueClient {
     const client = new _QueueClient(options);
     client._storage = adapter;
     client._customAdapter = true;
+    client.initialised = false;
     return client;
   }
 };