queue-jobs-worker 1.0.5 → 2.0.0

package/dist/scripts/claim-job.lua

@@ -0,0 +1,117 @@
+--[[
+  claim-job.lua
+  Atomically claim the next eligible job from a queue.
+
+  This script runs server-side in a single round-trip so no two workers
+  can ever pick up the same job, even under high concurrency.
+
+  KEYS
+    [1]  index key   — sorted set  "qjw:{queue}:index"   (score = priority)
+    [2]  waiting key — sorted set  "qjw:{queue}:waiting" (score = runAt ms)
+
+  ARGV
+    [1]  nowMs — current Unix timestamp in milliseconds (string)
+
+  Algorithm
+    1. Scan the waiting set for members whose score (runAt) <= nowMs.
+       ZRANGEBYSCORE returns by runAt ASC — but two jobs with identical runAt
+       values would be served in arbitrary order, violating priority ordering.
+    2. BUG FIX: re-sort candidates by their priority score from the index set
+       (lower score = higher priority) so high-priority jobs are always claimed
+       first even when multiple jobs become eligible at the same runAt.
+    3. Iterate the priority-sorted list.  For each candidate, atomically ZREM
+       it from the waiting set — only one worker can succeed (ZREM returns 1).
+    4. Fetch the job JSON; skip if missing (external deletion race).
+    5. Patch status → "active", bump updatedAt and processedAt in-place.
+    6. Persist and return the updated JSON string.
+
+  Returns
+    Bulk string — updated job JSON, or nil if nothing is eligible.
+--]]
+
+local indexKey   = KEYS[1]   -- "qjw:{queue}:index"
+local waitingKey = KEYS[2]   -- "qjw:{queue}:waiting"
+local nowMs      = tonumber(ARGV[1])
+
+-- ── 1. Fetch all eligible job IDs from the waiting set ──────────────────────
+-- ZRANGEBYSCORE returns members ordered by score (runAt) ASC.
+local candidates = redis.call("ZRANGEBYSCORE", waitingKey, "-inf", nowMs)
+
+if #candidates == 0 then
+  return nil
+end
+
+-- ── 2. Re-sort by priority using scores from the index set ──────────────────
+-- BUG FIX: the waiting set is ordered by runAt, not priority.  When multiple
+-- jobs are eligible at the same time (or have identical runAt values) they
+-- should be dispatched in priority ASC order (lower score = higher priority).
+-- We fetch each candidate's priority score and sort locally inside the script.
+--
+-- ZSCORE returns nil for members not in the index (should never happen in a
+-- consistent store, but we default to 0 so such jobs sort at the front rather
+-- than causing a Lua error).
+local scored = {}
+for _, jobId in ipairs(candidates) do
+  local score = redis.call("ZSCORE", indexKey, jobId)
+  scored[#scored + 1] = { id = jobId, pri = tonumber(score) or 0 }
+end
+
+-- Stable sort by priority ASC.  Lua's table.sort is not guaranteed stable,
+-- but for our purposes (priority is typically a small integer) this is fine —
+-- ties within the same priority are already broken by runAt ASC order from
+-- ZRANGEBYSCORE, and table.sort preserves relative input order for equal keys
+-- in most Lua runtimes (including Redis's embedded LuaJIT).
+table.sort(scored, function(a, b) return a.pri < b.pri end)
+
+-- ── 3. Claim the first job we can atomically remove from the waiting set ─────
+for _, entry in ipairs(scored) do
+  local jobId = entry.id
+
+  -- ZREM is atomic: returns 1 if we removed it, 0 if another worker beat us.
+  local removed = redis.call("ZREM", waitingKey, jobId)
+  if removed == 1 then
+    local hashKey = "qjw:job:" .. jobId
+    local raw     = redis.call("GET", hashKey)
+
+    if raw then
+      -- ── 4. Patch status fields in-place ─────────────────────────────────
+      -- We use targeted string substitution rather than a full JSON parse
+      -- (Redis has no built-in JSON module by default).  The job JSON is a
+      -- flat object — no nested status/updatedAt/processedAt fields exist
+      -- inside sub-objects, so string replacement is safe and unambiguous.
+      local nowStr = tostring(nowMs)
+
+      -- "status":"<anything>"  →  "status":"active"
+      raw = raw:gsub('"status":"[^"]*"', '"status":"active"', 1)
+
+      -- "updatedAt":<number>  →  current timestamp
+      raw = raw:gsub('"updatedAt":[%d%.%-]+', '"updatedAt":' .. nowStr, 1)
+
+      -- "processedAt": <string|number|null|absent>  →  current timestamp
+      -- Two-branch pattern handles both JSON string values and scalar values.
+      if raw:find('"processedAt":', 1, true) then
+        raw = raw:gsub(
+          '"processedAt":("([^"\\]|\\.)*"|[^,}]+)',
+          '"processedAt":' .. nowStr,
+          1
+        )
+      else
+        -- Field absent — append before the closing brace.
+        raw = raw:gsub('}$', ',"processedAt":' .. nowStr .. '}')
+      end
+
+      -- ── 5. Persist and return ────────────────────────────────────────────
+      redis.call("SET", hashKey, raw)
+
+      -- Only remove from the *waiting* set, NOT from the index.
+      -- The index tracks ALL jobs regardless of status so listJobs("active")
+      -- continues to find this job.  Index entries are removed only when the
+      -- job reaches a terminal state via update-job.lua or remove-job.lua.
+
+      return raw
+    end
+    -- JSON missing — job was deleted externally. Continue to next candidate.
+  end
+end
+
+return nil

package/dist/scripts/clear-queue.lua

@@ -0,0 +1,42 @@
+--[[
+  clear-queue.lua
+  Atomically delete every job in a queue and clean up all sorted sets.
+
+  Without this script clearQueue requires:
+    1 ZRANGE  — fetch all IDs
+    N DEL     — delete each job hash
+    1 DEL     — delete index
+    1 DEL     — delete waiting set
+  = N + 3 round-trips.
+
+  With this script it is a single round-trip regardless of queue size.
+
+  KEYS
+    [1]  indexKey   — "qjw:{queue}:index"
+    [2]  waitingKey — "qjw:{queue}:waiting"
+
+  ARGV
+    (none)
+
+  Returns
+    Number of job hashes that were deleted.
+--]]
+
+local indexKey   = KEYS[1]
+local waitingKey = KEYS[2]
+
+-- Collect every job ID tracked by the priority index.
+-- ZRANGE 0 -1 returns all members in score order (ASC).
+local ids     = redis.call("ZRANGE", indexKey, 0, -1)
+local deleted = 0
+
+for _, jobId in ipairs(ids) do
+  local n = redis.call("DEL", "qjw:job:" .. jobId)
+  deleted = deleted + n
+end
+
+-- Drop both sorted sets.
+redis.call("DEL", indexKey)
+redis.call("DEL", waitingKey)
+
+return deleted

package/dist/scripts/list-jobs.lua

@@ -0,0 +1,50 @@
+--[[
+  list-jobs.lua
+  Fetch all jobs for a queue in a single server-side round-trip.
+
+  Without this script listJobs requires:
+    1 ZRANGE  — fetch all IDs
+    N GET     — fetch each job hash individually  ← N+1 problem
+  = N + 1 round-trips.
+
+  With this script it is always exactly 1 round-trip.
+
+  KEYS
+    [1]  indexKey — "qjw:{queue}:index"  (sorted by priority ASC)
+
+  ARGV
+    [1]  statusFilter — status string to filter by, or "" for no filter
+
+  Algorithm
+    1. ZRANGE the index to get all job IDs in priority order.
+    2. GET each job hash in a loop (all inside the same script execution,
+       so it is pipelined on the Redis side with zero network overhead).
+    3. Optionally filter by status using a simple string search.
+    4. Return the matching JSON strings as a Redis array (multi-bulk reply).
+
+  Returns
+    Array of job JSON strings, sorted by priority ASC (then insertion order
+    within the same priority level, which matches createdAt ASC because jobs
+    are added with ZADD NX and the index preserves insertion order for ties).
+--]]
+
+local indexKey    = KEYS[1]
+local statusFilter = ARGV[1]   -- e.g. "waiting", "failed", or "" for all
+
+local ids    = redis.call("ZRANGE", indexKey, 0, -1)
+local result = {}
+
+for _, jobId in ipairs(ids) do
+  local raw = redis.call("GET", "qjw:job:" .. jobId)
+  if raw then
+    -- Apply status filter if requested.
+    -- We do a simple substring search: '"status":"<filter>"'
+    -- This is safe because status values are simple ASCII words with no
+    -- special characters that would need escaping.
+    if statusFilter == "" or raw:find('"status":"' .. statusFilter .. '"', 1, true) then
+      result[#result + 1] = raw
+    end
+  end
+end
+
+return result

package/dist/scripts/remove-job.lua

@@ -0,0 +1,41 @@
+--[[
+  remove-job.lua
+  Atomically delete a job and remove it from every sorted set.
+
+  Replaces three separate round-trips (DEL + ZREM + ZREM) with a single
+  server-side script execution.
+
+  KEYS
+    [1]  hashKey    — "qjw:job:{jobId}"
+    [2]  indexKey   — "qjw:{queue}:index"
+    [3]  waitingKey — "qjw:{queue}:waiting"
+
+  ARGV
+    [1]  jobId — the job's id
+
+  Returns
+    1 if the job existed and was deleted, 0 if it was not found.
+--]]
+
+local hashKey    = KEYS[1]
+local indexKey   = KEYS[2]
+local waitingKey = KEYS[3]
+
+local jobId      = ARGV[1]
+
+-- Check existence first so we can return a meaningful result.
+local existed = redis.call("EXISTS", hashKey)
+
+if existed == 1 then
+  redis.call("DEL",  hashKey)
+  redis.call("ZREM", indexKey,   jobId)
+  redis.call("ZREM", waitingKey, jobId)
+  return 1
+end
+
+-- Still attempt the ZREM calls in case the hash was deleted externally
+-- but the sorted-set entries were left behind (defensive cleanup).
+redis.call("ZREM", indexKey,   jobId)
+redis.call("ZREM", waitingKey, jobId)
+
+return 0

package/dist/scripts/save-job.lua

@@ -0,0 +1,52 @@
+--[[
+  save-job.lua
+  Atomically persist a new job to Redis.
+
+  Replaces three separate round-trips (SET + ZADD + ZADD) with a single
+  server-side script execution.
+
+  KEYS
+    [1]  hashKey    — "qjw:job:{jobId}"
+    [2]  indexKey   — "qjw:{queue}:index"   sorted set, score = priority
+    [3]  waitingKey — "qjw:{queue}:waiting" sorted set, score = runAt ms
+
+  ARGV
+    [1]  jobJson    — full JSON string of the Job record
+    [2]  jobId      — the job's id field
+    [3]  priority   — numeric priority (string)
+    [4]  runAt      — runAt timestamp ms (string)
+    [5]  eligible   — "1" if this job should enter the waiting set now
+                      (status is "waiting" or "retrying"), "0" otherwise
+
+  Returns
+    1 always.
+--]]
+
+local hashKey    = KEYS[1]
+local indexKey   = KEYS[2]
+local waitingKey = KEYS[3]
+
+local jobJson    = ARGV[1]
+local jobId      = ARGV[2]
+local priority   = ARGV[3]
+local runAt      = ARGV[4]
+local eligible   = ARGV[5]
+
+-- Store the full job JSON.
+redis.call("SET", hashKey, jobJson)
+
+-- Always add to the priority index so listJobs / countJobs work.
+-- BUG FIX: use ZADD NX so re-saving an existing job (e.g. after a crash
+-- recovery re-enqueue) does not silently overwrite the priority score that
+-- update-job.lua may have set.  saveJob is a new-job operation; priority
+-- updates go through updateJob / update-job.lua exclusively.
+redis.call("ZADD", indexKey, "NX", priority, jobId)
+
+-- Only add to the waiting set if the job is immediately pick-up-eligible.
+-- Use plain ZADD (not NX) so a re-save after a crash can correct the runAt
+-- score; the waiting set score must always reflect the true scheduled time.
+if eligible == "1" then
+  redis.call("ZADD", waitingKey, runAt, jobId)
+end
+
+return 1

package/dist/scripts/update-job.lua

@@ -0,0 +1,183 @@
+--[[
+  update-job.lua  v6
+  Atomically read, patch, and re-persist a job — entirely server-side.
+
+  One Redis round-trip: GET + targeted string replacement + SET.
+
+  ── KEYS ──────────────────────────────────────────────────────────────────────
+    [1]  hashKey    — "qjw:job:{jobId}"
+    [2]  indexKey   — "qjw:{queue}:index"    sorted set, score = priority
+    [3]  waitingKey — "qjw:{queue}:waiting"  sorted set, score = runAt ms
+
+  ── ARGV ──────────────────────────────────────────────────────────────────────
+    [1]  newStatus   — target status string (always patched)
+    [2]  nowMs       — current Unix timestamp ms (always written as updatedAt)
+    [3]  runAt       — runAt ms for "retrying" re-queue, or "-1" as sentinel
+                       meaning "do not touch runAt / skip ZADD waitingKey".
+    [4]  priority    — job priority integer, or "-1" sentinel ("keep existing").
+    [5]  strFields   — RS-delimited "key\x1evalue" pairs for string fields.
+    [6]  numFields   — RS-delimited "key\x1evalue" pairs for numeric fields.
+    [7]  nullFields  — RS-delimited field names to set to JSON null.
+    [8]  boolFields  — RS-delimited "key\x1evalue" pairs for boolean fields
+                       ("true" / "false" JSON literals).
+
+  ── Returns ───────────────────────────────────────────────────────────────────
+    1 on success, 0 if the job does not exist.
+--]]
+
+local hashKey    = KEYS[1]
+local indexKey   = KEYS[2]
+local waitingKey = KEYS[3]
+
+local newStatus  = ARGV[1]
+local nowMs      = ARGV[2]
+local runAt      = ARGV[3]
+local priority   = ARGV[4]
+local strFields  = ARGV[5]
+local numFields  = ARGV[6]
+local nullFields = ARGV[7]
+local boolFields = ARGV[8] or ""
+
+local RS = "\x1e"
+
+-- ── 1. Read current job JSON ──────────────────────────────────────────────────
+local raw = redis.call("GET", hashKey)
+if not raw then return 0 end
+
+-- ── helpers ───────────────────────────────────────────────────────────────────
+
+-- Escape for Lua gsub() PATTERN (not replacement).
+local function escPat(s)
+  return s:gsub("([%(%)%.%%%+%-%*%?%[%]%^%$])", "%%%1")
+end
+
+-- Escape for Lua gsub() REPLACEMENT string.
+local function escRepl(s)
+  return s:gsub("%%", "%%%%")
+end
+
+-- Split RS-delimited string into array.
+local function splitRS(s)
+  local parts = {}
+  for part in (s .. RS):gmatch("([^" .. RS .. "]*)" .. RS) do
+    parts[#parts + 1] = part
+  end
+  return parts
+end
+
+-- BUG FIX (v6): existence check uses plain literal search (4th arg = true).
+-- Previously `find('"' .. escPat(key) .. '":', 1, true)` was used, but the
+-- escPat() output contains % escapes that are meaningless in plain-text mode
+-- and could corrupt the search string if the key contained literal %.
+-- Using the raw key with plain=true is both correct and simpler.
+local function hasField(json, key)
+  return json:find('"' .. key .. '":', 1, true) ~= nil
+end
+
+-- ── 2. Patch: status (always) ─────────────────────────────────────────────────
+raw = raw:gsub('"status":"[^"]*"', '"status":"' .. newStatus .. '"', 1)
+
+-- ── 3. Patch: updatedAt (always, numeric) ────────────────────────────────────
+raw = raw:gsub('"updatedAt":[%d%.%-]+', '"updatedAt":' .. nowMs, 1)
+
+-- ── 4. Patch: string fields ───────────────────────────────────────────────────
+if strFields ~= "" then
+  local parts = splitRS(strFields)
+  local i = 1
+  while i < #parts do
+    local key = parts[i]
+    local val = parts[i + 1]
+    i = i + 2
+
+    if key ~= "" then
+      local pat  = '"' .. escPat(key) .. '":"([^"\\\\]|\\\\.)*"'
+      local repl = '"' .. key .. '":"' .. escRepl(val) .. '"'
+
+      if hasField(raw, key) then
+        raw = raw:gsub(pat, repl, 1)
+      else
+        raw = raw:gsub('}$', ',"' .. key .. '":"' .. escRepl(val) .. '"}')
+      end
+    end
+  end
+end
+
+-- ── 5. Patch: numeric fields ──────────────────────────────────────────────────
+if numFields ~= "" then
+  local parts = splitRS(numFields)
+  local i = 1
+  while i < #parts do
+    local key = parts[i]
+    local val = parts[i + 1]
+    i = i + 2
+
+    if key ~= "" then
+      local pat  = '"' .. escPat(key) .. '":[%d%.%-]+'
+      local repl = '"' .. key .. '":' .. val
+
+      if hasField(raw, key) then
+        raw = raw:gsub(pat, repl, 1)
+      else
+        raw = raw:gsub('}$', ',"' .. key .. '":' .. val .. '}')
+      end
+    end
+  end
+end
+
+-- ── 6. Patch: boolean fields ──────────────────────────────────────────────────
+if boolFields ~= "" then
+  local parts = splitRS(boolFields)
+  local i = 1
+  while i < #parts do
+    local key = parts[i]
+    local val = parts[i + 1]   -- "true" or "false"
+    i = i + 2
+
+    if key ~= "" then
+      local pat  = '"' .. escPat(key) .. '":(true|false)'
+      local repl = '"' .. key .. '":' .. val
+
+      if hasField(raw, key) then
+        raw = raw:gsub(pat, repl, 1)
+      else
+        raw = raw:gsub('}$', ',"' .. key .. '":' .. val .. '}')
+      end
+    end
+  end
+end
+
+-- ── 7. Patch: null fields ─────────────────────────────────────────────────────
+if nullFields ~= "" then
+  local parts = splitRS(nullFields)
+  for _, key in ipairs(parts) do
+    if key ~= "" and hasField(raw, key) then
+      local pat = '"' .. escPat(key) .. '":("([^"\\\\]|\\\\.)*"|[^,}]+)'
+      raw = raw:gsub(pat, '"' .. key .. '":null', 1)
+    end
+  end
+end
+
+-- ── 8. Persist updated JSON ───────────────────────────────────────────────────
+redis.call("SET", hashKey, raw)
+
+-- ── 9. Manage sorted-set membership ──────────────────────────────────────────
+local jobId = hashKey:match("qjw:job:(.+)")
+
+if newStatus == "retrying" then
+  if runAt ~= "-1" then
+    redis.call("ZADD", waitingKey, runAt, jobId)
+  end
+  if priority ~= "-1" then
+    redis.call("ZADD", indexKey, priority, jobId)
+  end
+
+elseif newStatus == "completed" or newStatus == "failed" then
+  redis.call("ZREM", waitingKey, jobId)
+  redis.call("ZREM", indexKey,   jobId)
+
+elseif newStatus == "active" then
+  redis.call("ZREM", waitingKey, jobId)
+
+end
+
+return 1