queue-jobs-worker 1.0.1 → 1.0.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +155 -122
- package/README.md +142 -386
- package/assets/queue-jobs-worker-github.png +0 -0
- package/dist/core/worker.d.ts.map +1 -1
- package/dist/index.cjs +430 -137
- package/dist/index.cjs.map +1 -1
- package/dist/index.js +430 -137
- package/dist/index.js.map +1 -1
- package/dist/storage/in-memory.adapter.d.ts +3 -2
- package/dist/storage/in-memory.adapter.d.ts.map +1 -1
- package/dist/storage/mysql.adapter.d.ts +3 -2
- package/dist/storage/mysql.adapter.d.ts.map +1 -1
- package/dist/storage/postgres.adapter.d.ts +3 -2
- package/dist/storage/postgres.adapter.d.ts.map +1 -1
- package/dist/storage/redis.adapter.d.ts +3 -2
- package/dist/storage/redis.adapter.d.ts.map +1 -1
- package/dist/types/storage.types.d.ts +20 -2
- package/dist/types/storage.types.d.ts.map +1 -1
- package/dist/types/worker.types.d.ts +1 -1
- package/dist/types/worker.types.d.ts.map +1 -1
- package/package.json +110 -97
package/dist/index.js
CHANGED
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import { randomUUID } from 'crypto';
|
|
2
|
+
import { Cron } from 'croner';
|
|
2
3
|
import { EventEmitter } from 'events';
|
|
3
4
|
|
|
4
5
|
var __defProp = Object.defineProperty;
|
|
@@ -74,7 +75,7 @@ function hashToJob(h) {
|
|
|
74
75
|
if (h["cron"]) job.cron = h["cron"];
|
|
75
76
|
return job;
|
|
76
77
|
}
|
|
77
|
-
var PREFIX, k, CLAIM_LUA, RedisStorageAdapter;
|
|
78
|
+
var PREFIX, k, CLAIM_LUA, RECOVER_STALLED_LUA, RENEW_LOCK_LUA, RedisStorageAdapter;
|
|
78
79
|
var init_redis_adapter = __esm({
|
|
79
80
|
"src/storage/redis.adapter.ts"() {
|
|
80
81
|
PREFIX = "qjw:";
|
|
@@ -116,6 +117,66 @@ redis.call('HSET', prefix .. 'job:' .. job_id,
|
|
|
116
117
|
'updatedAt', ARGV[2]
|
|
117
118
|
)
|
|
118
119
|
return job_id
|
|
120
|
+
`;
|
|
121
|
+
RECOVER_STALLED_LUA = `
|
|
122
|
+
local job_key = KEYS[1]
|
|
123
|
+
local active_set = KEYS[2]
|
|
124
|
+
local waiting_zset = KEYS[3]
|
|
125
|
+
|
|
126
|
+
local job_id = ARGV[1]
|
|
127
|
+
local expected_exp = ARGV[2]
|
|
128
|
+
local expected_lock = ARGV[3]
|
|
129
|
+
local recovery_ts = ARGV[4]
|
|
130
|
+
local priority_score = tonumber(ARGV[5])
|
|
131
|
+
|
|
132
|
+
-- Re-read the three guard fields in one atomic HMGET.
|
|
133
|
+
local fields = redis.call('HMGET', job_key, 'lockExpiresAt', 'lockId', 'status')
|
|
134
|
+
local current_exp = fields[1] or ''
|
|
135
|
+
local current_lock = fields[2] or ''
|
|
136
|
+
local current_status = fields[3] or ''
|
|
137
|
+
|
|
138
|
+
-- Guard 1: job must still be active.
|
|
139
|
+
-- If the worker already completed, failed, or moved to DLQ, skip.
|
|
140
|
+
if current_status ~= 'active' then return 0 end
|
|
141
|
+
|
|
142
|
+
-- Guard 2: lockId must not have changed.
|
|
143
|
+
-- A different worker may have claimed the job after the original lock expired
|
|
144
|
+
-- and before this script runs.
|
|
145
|
+
if current_lock ~= expected_lock then return 0 end
|
|
146
|
+
|
|
147
|
+
-- Guard 3: lockExpiresAt must be exactly what the caller observed.
|
|
148
|
+
-- If the worker renewed its lock the timestamp will be later than observed;
|
|
149
|
+
-- the CAS mismatch catches that without needing ISO\u2192epoch conversion in Lua.
|
|
150
|
+
if current_exp ~= expected_exp then return 0 end
|
|
151
|
+
|
|
152
|
+
-- All guards passed \u2014 recover atomically.
|
|
153
|
+
redis.call('HSET', job_key,
|
|
154
|
+
'status', 'waiting',
|
|
155
|
+
'lockId', '',
|
|
156
|
+
'lockExpiresAt', '',
|
|
157
|
+
'updatedAt', recovery_ts
|
|
158
|
+
)
|
|
159
|
+
redis.call('SREM', active_set, job_id)
|
|
160
|
+
redis.call('ZADD', waiting_zset, priority_score, job_id)
|
|
161
|
+
return 1
|
|
162
|
+
`;
|
|
163
|
+
RENEW_LOCK_LUA = `
|
|
164
|
+
local job_key = KEYS[1]
|
|
165
|
+
local lock_id = ARGV[1]
|
|
166
|
+
local new_exp = ARGV[2]
|
|
167
|
+
local now_iso = ARGV[3]
|
|
168
|
+
|
|
169
|
+
local fields = redis.call('HMGET', job_key, 'status', 'lockId', 'lockExpiresAt')
|
|
170
|
+
local status = fields[1] or ''
|
|
171
|
+
local current_lock = fields[2] or ''
|
|
172
|
+
local current_exp = fields[3] or ''
|
|
173
|
+
|
|
174
|
+
if status ~= 'active' then return 0 end
|
|
175
|
+
if current_lock ~= lock_id then return 0 end
|
|
176
|
+
if current_exp == '' or current_exp <= now_iso then return 0 end
|
|
177
|
+
|
|
178
|
+
redis.call('HSET', job_key, 'lockExpiresAt', new_exp, 'updatedAt', now_iso)
|
|
179
|
+
return 1
|
|
119
180
|
`;
|
|
120
181
|
RedisStorageAdapter = class {
|
|
121
182
|
client;
|
|
@@ -206,12 +267,28 @@ return job_id
|
|
|
206
267
|
return hashToJob(hash);
|
|
207
268
|
}
|
|
208
269
|
// -------------------------------------------------------------------------
|
|
270
|
+
// Renew lock
|
|
271
|
+
// -------------------------------------------------------------------------
|
|
272
|
+
async renewLock(jobId, lockId, lockDuration) {
|
|
273
|
+
const nowMs = Date.now();
|
|
274
|
+
const nowIso = new Date(nowMs).toISOString();
|
|
275
|
+
const newExpiresAt = new Date(nowMs + lockDuration).toISOString();
|
|
276
|
+
const res = await this.client.eval(RENEW_LOCK_LUA, {
|
|
277
|
+
keys: [k.job(jobId)],
|
|
278
|
+
arguments: [lockId, newExpiresAt, nowIso]
|
|
279
|
+
});
|
|
280
|
+
return Number(res) === 1;
|
|
281
|
+
}
|
|
282
|
+
// -------------------------------------------------------------------------
|
|
209
283
|
// Complete
|
|
210
284
|
// -------------------------------------------------------------------------
|
|
211
|
-
async complete(jobId) {
|
|
285
|
+
async complete(jobId, lockId) {
|
|
212
286
|
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
213
287
|
const hash = await this.client.hGetAll(k.job(jobId));
|
|
214
|
-
if (!hash) return;
|
|
288
|
+
if (!hash || Object.keys(hash).length === 0) return;
|
|
289
|
+
if (lockId !== void 0) {
|
|
290
|
+
if (hash["status"] !== "active" || hash["lockId"] !== lockId) return;
|
|
291
|
+
}
|
|
215
292
|
const queue = hash["queue"] ?? "";
|
|
216
293
|
const multi = this.client.multi();
|
|
217
294
|
multi.hSet(k.job(jobId), {
|
|
@@ -231,7 +308,10 @@ return job_id
|
|
|
231
308
|
async requeue(input) {
|
|
232
309
|
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
233
310
|
const hash = await this.client.hGetAll(k.job(input.jobId));
|
|
234
|
-
if (!hash) return;
|
|
311
|
+
if (!hash || Object.keys(hash).length === 0) return;
|
|
312
|
+
if (input.lockId !== void 0) {
|
|
313
|
+
if (hash["status"] !== "active" || hash["lockId"] !== input.lockId) return;
|
|
314
|
+
}
|
|
235
315
|
const queue = hash["queue"] ?? "";
|
|
236
316
|
const attempts = JSON.parse(hash["attempts"] ?? "[]");
|
|
237
317
|
attempts.push({
|
|
@@ -264,7 +344,10 @@ return job_id
|
|
|
264
344
|
async moveToDlq(input) {
|
|
265
345
|
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
266
346
|
const hash = await this.client.hGetAll(k.job(input.jobId));
|
|
267
|
-
if (!hash) return;
|
|
347
|
+
if (!hash || Object.keys(hash).length === 0) return;
|
|
348
|
+
if (input.lockId !== void 0) {
|
|
349
|
+
if (hash["status"] !== "active" || hash["lockId"] !== input.lockId) return;
|
|
350
|
+
}
|
|
268
351
|
const queue = hash["queue"] ?? "";
|
|
269
352
|
const attempts = JSON.parse(hash["attempts"] ?? "[]");
|
|
270
353
|
attempts.push({
|
|
@@ -291,7 +374,12 @@ return job_id
|
|
|
291
374
|
// -------------------------------------------------------------------------
|
|
292
375
|
// Release lock
|
|
293
376
|
// -------------------------------------------------------------------------
|
|
294
|
-
async releaseLock(jobId) {
|
|
377
|
+
async releaseLock(jobId, lockId) {
|
|
378
|
+
const hash = await this.client.hGetAll(k.job(jobId));
|
|
379
|
+
if (!hash || Object.keys(hash).length === 0) return;
|
|
380
|
+
if (lockId !== void 0) {
|
|
381
|
+
if (hash["status"] !== "active" || hash["lockId"] !== lockId) return;
|
|
382
|
+
}
|
|
295
383
|
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
296
384
|
await this.client.hSet(k.job(jobId), {
|
|
297
385
|
lockId: "",
|
|
@@ -300,7 +388,18 @@ return job_id
|
|
|
300
388
|
});
|
|
301
389
|
}
|
|
302
390
|
// -------------------------------------------------------------------------
|
|
303
|
-
// Recover stalled jobs (
|
|
391
|
+
// Recover stalled jobs (atomic compare-and-swap per job via Lua)
|
|
392
|
+
//
|
|
393
|
+
// Previous implementation: two-phase read pipeline → write pipeline.
|
|
394
|
+
// Race condition: a worker could complete or renew its lock between the two
|
|
395
|
+
// phases, causing recovery to overwrite a legitimately-active job.
|
|
396
|
+
//
|
|
397
|
+
// Fix (issue #6): for each candidate job the RECOVER_STALLED_LUA script
|
|
398
|
+
// re-reads lockExpiresAt, lockId, and status atomically and only applies
|
|
399
|
+
// the recovery if all three still match what was observed in the read phase
|
|
400
|
+
// (compare-and-swap). If the worker renewed or completed the job in the
|
|
401
|
+
// window between the read and the Lua call, the CAS mismatch causes the
|
|
402
|
+
// script to return 0 and the job is left untouched.
|
|
304
403
|
// -------------------------------------------------------------------------
|
|
305
404
|
async recoverStalledJobs(queue, now) {
|
|
306
405
|
const nowMs = new Date(now).getTime();
|
|
@@ -308,32 +407,29 @@ return job_id
|
|
|
308
407
|
if (activeIds.length === 0) return [];
|
|
309
408
|
const fetchPipeline = this.client.multi();
|
|
310
409
|
for (const jobId of activeIds) {
|
|
311
|
-
fetchPipeline.hmGet(k.job(jobId), ["lockExpiresAt", "priority"]);
|
|
410
|
+
fetchPipeline.hmGet(k.job(jobId), ["lockExpiresAt", "lockId", "priority"]);
|
|
312
411
|
}
|
|
313
412
|
const fetchResults = await fetchPipeline.exec();
|
|
314
413
|
const recovered = [];
|
|
315
|
-
const
|
|
414
|
+
const evalPromises = [];
|
|
316
415
|
for (let i = 0; i < activeIds.length; i++) {
|
|
317
416
|
const jobId = activeIds[i];
|
|
318
417
|
const fields = fetchResults[i];
|
|
319
418
|
if (!fields) continue;
|
|
320
|
-
const [lockExpiresAt, priorityStr] = fields;
|
|
419
|
+
const [lockExpiresAt, lockId, priorityStr] = fields;
|
|
321
420
|
if (!lockExpiresAt) continue;
|
|
322
421
|
if (new Date(lockExpiresAt).getTime() > nowMs) continue;
|
|
323
422
|
const priority = Number(priorityStr ?? "0");
|
|
324
|
-
|
|
325
|
-
|
|
326
|
-
|
|
327
|
-
|
|
328
|
-
|
|
423
|
+
const priorityScore = String(-priority);
|
|
424
|
+
const p = this.client.eval(RECOVER_STALLED_LUA, {
|
|
425
|
+
keys: [k.job(jobId), k.active(queue), k.waiting(queue)],
|
|
426
|
+
arguments: [jobId, lockExpiresAt, lockId ?? "", now, priorityScore]
|
|
427
|
+
}).then((result) => {
|
|
428
|
+
if (result === 1) recovered.push(jobId);
|
|
329
429
|
});
|
|
330
|
-
|
|
331
|
-
recoverPipeline.zAdd(k.waiting(queue), { score: -priority, value: jobId });
|
|
332
|
-
recovered.push(jobId);
|
|
333
|
-
}
|
|
334
|
-
if (recovered.length > 0) {
|
|
335
|
-
await recoverPipeline.exec();
|
|
430
|
+
evalPromises.push(p);
|
|
336
431
|
}
|
|
432
|
+
await Promise.all(evalPromises);
|
|
337
433
|
return recovered;
|
|
338
434
|
}
|
|
339
435
|
// -------------------------------------------------------------------------
|
|
@@ -610,16 +706,40 @@ CREATE TABLE IF NOT EXISTS qjw_rate_limits (
|
|
|
610
706
|
}
|
|
611
707
|
}
|
|
612
708
|
// -------------------------------------------------------------------------
|
|
613
|
-
//
|
|
709
|
+
// Renew lock
|
|
614
710
|
// -------------------------------------------------------------------------
|
|
615
|
-
async
|
|
616
|
-
|
|
711
|
+
async renewLock(jobId, lockId, lockDuration) {
|
|
712
|
+
const newLockExpiresAt = new Date(Date.now() + lockDuration).toISOString();
|
|
713
|
+
const res = await this.pool.query(
|
|
617
714
|
`UPDATE qjw_jobs
|
|
618
|
-
SET
|
|
619
|
-
|
|
620
|
-
WHERE id = $
|
|
621
|
-
[jobId]
|
|
715
|
+
SET lock_expires_at = $1::timestamptz,
|
|
716
|
+
updated_at = NOW()
|
|
717
|
+
WHERE id = $2 AND status = 'active' AND lock_id = $3 AND lock_expires_at > NOW()`,
|
|
718
|
+
[newLockExpiresAt, jobId, lockId]
|
|
622
719
|
);
|
|
720
|
+
return (res.rowCount ?? 0) > 0;
|
|
721
|
+
}
|
|
722
|
+
// -------------------------------------------------------------------------
|
|
723
|
+
// Complete
|
|
724
|
+
// -------------------------------------------------------------------------
|
|
725
|
+
async complete(jobId, lockId) {
|
|
726
|
+
if (lockId !== void 0) {
|
|
727
|
+
await this.pool.query(
|
|
728
|
+
`UPDATE qjw_jobs
|
|
729
|
+
SET status = 'completed', lock_id = NULL, lock_expires_at = NULL,
|
|
730
|
+
completed_at = NOW(), updated_at = NOW()
|
|
731
|
+
WHERE id = $1 AND status = 'active' AND lock_id = $2`,
|
|
732
|
+
[jobId, lockId]
|
|
733
|
+
);
|
|
734
|
+
} else {
|
|
735
|
+
await this.pool.query(
|
|
736
|
+
`UPDATE qjw_jobs
|
|
737
|
+
SET status = 'completed', lock_id = NULL, lock_expires_at = NULL,
|
|
738
|
+
completed_at = NOW(), updated_at = NOW()
|
|
739
|
+
WHERE id = $1`,
|
|
740
|
+
[jobId]
|
|
741
|
+
);
|
|
742
|
+
}
|
|
623
743
|
}
|
|
624
744
|
// -------------------------------------------------------------------------
|
|
625
745
|
// Requeue
|
|
@@ -633,18 +753,33 @@ CREATE TABLE IF NOT EXISTS qjw_rate_limits (
|
|
|
633
753
|
error: input.error,
|
|
634
754
|
...input.stack !== void 0 ? { stack: input.stack } : {}
|
|
635
755
|
};
|
|
636
|
-
|
|
637
|
-
|
|
638
|
-
|
|
639
|
-
|
|
640
|
-
|
|
641
|
-
|
|
642
|
-
|
|
643
|
-
|
|
644
|
-
|
|
645
|
-
|
|
646
|
-
|
|
647
|
-
|
|
756
|
+
if (input.lockId !== void 0) {
|
|
757
|
+
await this.pool.query(
|
|
758
|
+
`UPDATE qjw_jobs
|
|
759
|
+
SET status = 'waiting',
|
|
760
|
+
attempts_made = $1,
|
|
761
|
+
attempts = attempts || $2::jsonb,
|
|
762
|
+
run_at = $3::timestamptz,
|
|
763
|
+
lock_id = NULL,
|
|
764
|
+
lock_expires_at = NULL,
|
|
765
|
+
updated_at = NOW()
|
|
766
|
+
WHERE id = $4 AND status = 'active' AND lock_id = $5`,
|
|
767
|
+
[input.attemptNumber, JSON.stringify([attempt]), input.runAt, input.jobId, input.lockId]
|
|
768
|
+
);
|
|
769
|
+
} else {
|
|
770
|
+
await this.pool.query(
|
|
771
|
+
`UPDATE qjw_jobs
|
|
772
|
+
SET status = 'waiting',
|
|
773
|
+
attempts_made = $1,
|
|
774
|
+
attempts = attempts || $2::jsonb,
|
|
775
|
+
run_at = $3::timestamptz,
|
|
776
|
+
lock_id = NULL,
|
|
777
|
+
lock_expires_at = NULL,
|
|
778
|
+
updated_at = NOW()
|
|
779
|
+
WHERE id = $4`,
|
|
780
|
+
[input.attemptNumber, JSON.stringify([attempt]), input.runAt, input.jobId]
|
|
781
|
+
);
|
|
782
|
+
}
|
|
648
783
|
}
|
|
649
784
|
// -------------------------------------------------------------------------
|
|
650
785
|
// Move to DLQ
|
|
@@ -658,29 +793,53 @@ CREATE TABLE IF NOT EXISTS qjw_rate_limits (
|
|
|
658
793
|
error: input.error,
|
|
659
794
|
...input.stack !== void 0 ? { stack: input.stack } : {}
|
|
660
795
|
};
|
|
661
|
-
|
|
662
|
-
|
|
663
|
-
|
|
664
|
-
|
|
665
|
-
|
|
666
|
-
|
|
667
|
-
|
|
668
|
-
|
|
669
|
-
|
|
670
|
-
|
|
671
|
-
|
|
672
|
-
|
|
796
|
+
if (input.lockId !== void 0) {
|
|
797
|
+
await this.pool.query(
|
|
798
|
+
`UPDATE qjw_jobs
|
|
799
|
+
SET status = 'dead',
|
|
800
|
+
attempts_made = $1,
|
|
801
|
+
attempts = attempts || $2::jsonb,
|
|
802
|
+
lock_id = NULL,
|
|
803
|
+
lock_expires_at = NULL,
|
|
804
|
+
failed_at = NOW(),
|
|
805
|
+
updated_at = NOW()
|
|
806
|
+
WHERE id = $3 AND status = 'active' AND lock_id = $4`,
|
|
807
|
+
[input.attemptNumber, JSON.stringify([attempt]), input.jobId, input.lockId]
|
|
808
|
+
);
|
|
809
|
+
} else {
|
|
810
|
+
await this.pool.query(
|
|
811
|
+
`UPDATE qjw_jobs
|
|
812
|
+
SET status = 'dead',
|
|
813
|
+
attempts_made = $1,
|
|
814
|
+
attempts = attempts || $2::jsonb,
|
|
815
|
+
lock_id = NULL,
|
|
816
|
+
lock_expires_at = NULL,
|
|
817
|
+
failed_at = NOW(),
|
|
818
|
+
updated_at = NOW()
|
|
819
|
+
WHERE id = $3`,
|
|
820
|
+
[input.attemptNumber, JSON.stringify([attempt]), input.jobId]
|
|
821
|
+
);
|
|
822
|
+
}
|
|
673
823
|
}
|
|
674
824
|
// -------------------------------------------------------------------------
|
|
675
825
|
// Release lock
|
|
676
826
|
// -------------------------------------------------------------------------
|
|
677
|
-
async releaseLock(jobId) {
|
|
678
|
-
|
|
679
|
-
|
|
680
|
-
|
|
681
|
-
|
|
682
|
-
|
|
683
|
-
|
|
827
|
+
async releaseLock(jobId, lockId) {
|
|
828
|
+
if (lockId !== void 0) {
|
|
829
|
+
await this.pool.query(
|
|
830
|
+
`UPDATE qjw_jobs
|
|
831
|
+
SET lock_id = NULL, lock_expires_at = NOW(), updated_at = NOW()
|
|
832
|
+
WHERE id = $1 AND status = 'active' AND lock_id = $2`,
|
|
833
|
+
[jobId, lockId]
|
|
834
|
+
);
|
|
835
|
+
} else {
|
|
836
|
+
await this.pool.query(
|
|
837
|
+
`UPDATE qjw_jobs
|
|
838
|
+
SET lock_id = NULL, lock_expires_at = NOW(), updated_at = NOW()
|
|
839
|
+
WHERE id = $1`,
|
|
840
|
+
[jobId]
|
|
841
|
+
);
|
|
842
|
+
}
|
|
684
843
|
}
|
|
685
844
|
// -------------------------------------------------------------------------
|
|
686
845
|
// Recover stalled jobs
|
|
@@ -961,16 +1120,39 @@ CREATE TABLE IF NOT EXISTS qjw_rate_limits (
|
|
|
961
1120
|
}
|
|
962
1121
|
}
|
|
963
1122
|
// -------------------------------------------------------------------------
|
|
964
|
-
//
|
|
1123
|
+
// Renew lock
|
|
965
1124
|
// -------------------------------------------------------------------------
|
|
966
|
-
async
|
|
967
|
-
|
|
1125
|
+
async renewLock(jobId, lockId, lockDuration) {
|
|
1126
|
+
const newLockExpiresAt = new Date(Date.now() + lockDuration).toISOString().slice(0, 23).replace("T", " ");
|
|
1127
|
+
const [result] = await this.pool.query(
|
|
968
1128
|
`UPDATE qjw_jobs
|
|
969
|
-
SET
|
|
970
|
-
|
|
971
|
-
|
|
972
|
-
[jobId]
|
|
1129
|
+
SET lock_expires_at = ?, updated_at = NOW(3)
|
|
1130
|
+
WHERE id = ? AND status = 'active' AND lock_id = ? AND lock_expires_at > NOW(3)`,
|
|
1131
|
+
[newLockExpiresAt, jobId, lockId]
|
|
973
1132
|
);
|
|
1133
|
+
return result.affectedRows > 0;
|
|
1134
|
+
}
|
|
1135
|
+
// -------------------------------------------------------------------------
|
|
1136
|
+
// Complete
|
|
1137
|
+
// -------------------------------------------------------------------------
|
|
1138
|
+
async complete(jobId, lockId) {
|
|
1139
|
+
if (lockId !== void 0) {
|
|
1140
|
+
await this.pool.query(
|
|
1141
|
+
`UPDATE qjw_jobs
|
|
1142
|
+
SET status = 'completed', lock_id = NULL, lock_expires_at = NULL,
|
|
1143
|
+
completed_at = NOW(3), updated_at = NOW(3)
|
|
1144
|
+
WHERE id = ? AND status = 'active' AND lock_id = ?`,
|
|
1145
|
+
[jobId, lockId]
|
|
1146
|
+
);
|
|
1147
|
+
} else {
|
|
1148
|
+
await this.pool.query(
|
|
1149
|
+
`UPDATE qjw_jobs
|
|
1150
|
+
SET status = 'completed', lock_id = NULL, lock_expires_at = NULL,
|
|
1151
|
+
completed_at = NOW(3), updated_at = NOW(3)
|
|
1152
|
+
WHERE id = ?`,
|
|
1153
|
+
[jobId]
|
|
1154
|
+
);
|
|
1155
|
+
}
|
|
974
1156
|
}
|
|
975
1157
|
// -------------------------------------------------------------------------
|
|
976
1158
|
// Requeue
|
|
@@ -984,18 +1166,33 @@ CREATE TABLE IF NOT EXISTS qjw_rate_limits (
|
|
|
984
1166
|
error: input.error,
|
|
985
1167
|
...input.stack !== void 0 ? { stack: input.stack } : {}
|
|
986
1168
|
});
|
|
987
|
-
|
|
988
|
-
|
|
989
|
-
|
|
990
|
-
|
|
991
|
-
|
|
992
|
-
|
|
993
|
-
|
|
994
|
-
|
|
995
|
-
|
|
996
|
-
|
|
997
|
-
|
|
998
|
-
|
|
1169
|
+
if (input.lockId !== void 0) {
|
|
1170
|
+
await this.pool.query(
|
|
1171
|
+
`UPDATE qjw_jobs
|
|
1172
|
+
SET status = 'waiting',
|
|
1173
|
+
attempts_made = attempts_made + 1,
|
|
1174
|
+
attempts = JSON_ARRAY_APPEND(attempts, '$', CAST(? AS JSON)),
|
|
1175
|
+
run_at = ?,
|
|
1176
|
+
lock_id = NULL,
|
|
1177
|
+
lock_expires_at = NULL,
|
|
1178
|
+
updated_at = NOW(3)
|
|
1179
|
+
WHERE id = ? AND status = 'active' AND lock_id = ?`,
|
|
1180
|
+
[attempt, input.runAt, input.jobId, input.lockId]
|
|
1181
|
+
);
|
|
1182
|
+
} else {
|
|
1183
|
+
await this.pool.query(
|
|
1184
|
+
`UPDATE qjw_jobs
|
|
1185
|
+
SET status = 'waiting',
|
|
1186
|
+
attempts_made = attempts_made + 1,
|
|
1187
|
+
attempts = JSON_ARRAY_APPEND(attempts, '$', CAST(? AS JSON)),
|
|
1188
|
+
run_at = ?,
|
|
1189
|
+
lock_id = NULL,
|
|
1190
|
+
lock_expires_at = NULL,
|
|
1191
|
+
updated_at = NOW(3)
|
|
1192
|
+
WHERE id = ?`,
|
|
1193
|
+
[attempt, input.runAt, input.jobId]
|
|
1194
|
+
);
|
|
1195
|
+
}
|
|
999
1196
|
}
|
|
1000
1197
|
// -------------------------------------------------------------------------
|
|
1001
1198
|
// Move to DLQ
|
|
@@ -1009,29 +1206,53 @@ CREATE TABLE IF NOT EXISTS qjw_rate_limits (
|
|
|
1009
1206
|
error: input.error,
|
|
1010
1207
|
...input.stack !== void 0 ? { stack: input.stack } : {}
|
|
1011
1208
|
});
|
|
1012
|
-
|
|
1013
|
-
|
|
1014
|
-
|
|
1015
|
-
|
|
1016
|
-
|
|
1017
|
-
|
|
1018
|
-
|
|
1019
|
-
|
|
1020
|
-
|
|
1021
|
-
|
|
1022
|
-
|
|
1023
|
-
|
|
1209
|
+
if (input.lockId !== void 0) {
|
|
1210
|
+
await this.pool.query(
|
|
1211
|
+
`UPDATE qjw_jobs
|
|
1212
|
+
SET status = 'dead',
|
|
1213
|
+
attempts_made = attempts_made + 1,
|
|
1214
|
+
attempts = JSON_ARRAY_APPEND(attempts, '$', CAST(? AS JSON)),
|
|
1215
|
+
lock_id = NULL,
|
|
1216
|
+
lock_expires_at = NULL,
|
|
1217
|
+
failed_at = NOW(3),
|
|
1218
|
+
updated_at = NOW(3)
|
|
1219
|
+
WHERE id = ? AND status = 'active' AND lock_id = ?`,
|
|
1220
|
+
[attempt, input.jobId, input.lockId]
|
|
1221
|
+
);
|
|
1222
|
+
} else {
|
|
1223
|
+
await this.pool.query(
|
|
1224
|
+
`UPDATE qjw_jobs
|
|
1225
|
+
SET status = 'dead',
|
|
1226
|
+
attempts_made = attempts_made + 1,
|
|
1227
|
+
attempts = JSON_ARRAY_APPEND(attempts, '$', CAST(? AS JSON)),
|
|
1228
|
+
lock_id = NULL,
|
|
1229
|
+
lock_expires_at = NULL,
|
|
1230
|
+
failed_at = NOW(3),
|
|
1231
|
+
updated_at = NOW(3)
|
|
1232
|
+
WHERE id = ?`,
|
|
1233
|
+
[attempt, input.jobId]
|
|
1234
|
+
);
|
|
1235
|
+
}
|
|
1024
1236
|
}
|
|
1025
1237
|
// -------------------------------------------------------------------------
|
|
1026
1238
|
// Release lock
|
|
1027
1239
|
// -------------------------------------------------------------------------
|
|
1028
|
-
async releaseLock(jobId) {
|
|
1029
|
-
|
|
1030
|
-
|
|
1031
|
-
|
|
1032
|
-
|
|
1033
|
-
|
|
1034
|
-
|
|
1240
|
+
async releaseLock(jobId, lockId) {
|
|
1241
|
+
if (lockId !== void 0) {
|
|
1242
|
+
await this.pool.query(
|
|
1243
|
+
`UPDATE qjw_jobs
|
|
1244
|
+
SET lock_id = NULL, lock_expires_at = NOW(3), updated_at = NOW(3)
|
|
1245
|
+
WHERE id = ? AND status = 'active' AND lock_id = ?`,
|
|
1246
|
+
[jobId, lockId]
|
|
1247
|
+
);
|
|
1248
|
+
} else {
|
|
1249
|
+
await this.pool.query(
|
|
1250
|
+
`UPDATE qjw_jobs
|
|
1251
|
+
SET lock_id = NULL, lock_expires_at = NOW(3), updated_at = NOW(3)
|
|
1252
|
+
WHERE id = ?`,
|
|
1253
|
+
[jobId]
|
|
1254
|
+
);
|
|
1255
|
+
}
|
|
1035
1256
|
}
|
|
1036
1257
|
// -------------------------------------------------------------------------
|
|
1037
1258
|
// Recover stalled jobs (atomic — SELECT … FOR UPDATE inside transaction)
|
|
@@ -1422,7 +1643,7 @@ var Worker = class {
|
|
|
1422
1643
|
if (this.activeJobIds.size > 0) {
|
|
1423
1644
|
await Promise.all(
|
|
1424
1645
|
Array.from(this.activeJobIds).map(
|
|
1425
|
-
(jobId) => this.storage.releaseLock(jobId).catch(() => {
|
|
1646
|
+
(jobId) => this.storage.releaseLock(jobId, this.id).catch(() => {
|
|
1426
1647
|
})
|
|
1427
1648
|
)
|
|
1428
1649
|
);
|
|
@@ -1459,6 +1680,13 @@ var Worker = class {
|
|
|
1459
1680
|
// -------------------------------------------------------------------------
|
|
1460
1681
|
async claimNext() {
|
|
1461
1682
|
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
1683
|
+
const raw = await this.storage.claim({
|
|
1684
|
+
queue: this.queueName,
|
|
1685
|
+
lockId: this.id,
|
|
1686
|
+
lockDuration: this.config.lockDuration,
|
|
1687
|
+
now
|
|
1688
|
+
});
|
|
1689
|
+
if (!raw) return false;
|
|
1462
1690
|
if (this.config.rateLimit) {
|
|
1463
1691
|
const allowed = await this.storage.checkAndIncrementRateLimit(
|
|
1464
1692
|
this.queueName,
|
|
@@ -1466,15 +1694,11 @@ var Worker = class {
|
|
|
1466
1694
|
this.config.rateLimit.duration,
|
|
1467
1695
|
now
|
|
1468
1696
|
);
|
|
1469
|
-
if (!allowed)
|
|
1697
|
+
if (!allowed) {
|
|
1698
|
+
await this.storage.releaseLock(raw.id, this.id);
|
|
1699
|
+
return false;
|
|
1700
|
+
}
|
|
1470
1701
|
}
|
|
1471
|
-
const raw = await this.storage.claim({
|
|
1472
|
-
queue: this.queueName,
|
|
1473
|
-
lockId: this.id,
|
|
1474
|
-
lockDuration: this.config.lockDuration,
|
|
1475
|
-
now
|
|
1476
|
-
});
|
|
1477
|
-
if (!raw) return false;
|
|
1478
1702
|
const job = new Job(raw);
|
|
1479
1703
|
this.activeCount += 1;
|
|
1480
1704
|
this.activeJobIds.add(job.id);
|
|
@@ -1491,16 +1715,35 @@ var Worker = class {
|
|
|
1491
1715
|
await this.handleFailure(job, error);
|
|
1492
1716
|
return;
|
|
1493
1717
|
}
|
|
1718
|
+
const abortController = new AbortController();
|
|
1494
1719
|
let timeoutHandle = null;
|
|
1720
|
+
let lockRenewTimer = null;
|
|
1721
|
+
const lockRenewInterval = Math.max(100, Math.floor(this.config.lockDuration / 2));
|
|
1722
|
+
lockRenewTimer = setInterval(async () => {
|
|
1723
|
+
try {
|
|
1724
|
+
const renewed = await this.storage.renewLock(job.id, this.id, this.config.lockDuration);
|
|
1725
|
+
if (!renewed && lockRenewTimer) {
|
|
1726
|
+
clearInterval(lockRenewTimer);
|
|
1727
|
+
lockRenewTimer = null;
|
|
1728
|
+
}
|
|
1729
|
+
} catch {
|
|
1730
|
+
}
|
|
1731
|
+
}, lockRenewInterval);
|
|
1495
1732
|
try {
|
|
1496
1733
|
await new Promise((resolve, reject) => {
|
|
1497
1734
|
timeoutHandle = setTimeout(() => {
|
|
1498
|
-
|
|
1735
|
+
const timeoutError = new Error(`Job timed out after ${job.timeout}ms`);
|
|
1736
|
+
abortController.abort(timeoutError);
|
|
1737
|
+
reject(timeoutError);
|
|
1499
1738
|
}, job.timeout);
|
|
1500
|
-
Promise.resolve(processor(job)).then(resolve, reject);
|
|
1739
|
+
Promise.resolve(processor(job, abortController.signal)).then(resolve, reject);
|
|
1501
1740
|
});
|
|
1502
1741
|
if (timeoutHandle) clearTimeout(timeoutHandle);
|
|
1503
|
-
|
|
1742
|
+
if (lockRenewTimer) {
|
|
1743
|
+
clearInterval(lockRenewTimer);
|
|
1744
|
+
lockRenewTimer = null;
|
|
1745
|
+
}
|
|
1746
|
+
await this.storage.complete(job.id, this.id);
|
|
1504
1747
|
const completedData = await this.storage.getJob(job.id) ?? job._data;
|
|
1505
1748
|
this.emitter.emit("job:completed", completedData);
|
|
1506
1749
|
if (job.cron) {
|
|
@@ -1508,9 +1751,17 @@ var Worker = class {
|
|
|
1508
1751
|
}
|
|
1509
1752
|
} catch (err) {
|
|
1510
1753
|
if (timeoutHandle) clearTimeout(timeoutHandle);
|
|
1754
|
+
if (lockRenewTimer) {
|
|
1755
|
+
clearInterval(lockRenewTimer);
|
|
1756
|
+
lockRenewTimer = null;
|
|
1757
|
+
}
|
|
1511
1758
|
const error = err instanceof Error ? err : new Error(String(err));
|
|
1512
1759
|
await this.handleFailure(job, error);
|
|
1513
1760
|
} finally {
|
|
1761
|
+
if (lockRenewTimer) {
|
|
1762
|
+
clearInterval(lockRenewTimer);
|
|
1763
|
+
lockRenewTimer = null;
|
|
1764
|
+
}
|
|
1514
1765
|
this.activeJobIds.delete(job.id);
|
|
1515
1766
|
this.activeCount -= 1;
|
|
1516
1767
|
if (this.activeCount === 0 && this.drainResolve) {
|
|
@@ -1524,25 +1775,25 @@ var Worker = class {
|
|
|
1524
1775
|
// -------------------------------------------------------------------------
|
|
1525
1776
|
async enqueueCronNext(job) {
|
|
1526
1777
|
try {
|
|
1527
|
-
let
|
|
1778
|
+
let nextDate;
|
|
1528
1779
|
try {
|
|
1529
|
-
const
|
|
1530
|
-
|
|
1531
|
-
|
|
1532
|
-
|
|
1780
|
+
const cronInstance = new Cron(job.cron);
|
|
1781
|
+
nextDate = cronInstance.nextRun();
|
|
1782
|
+
} catch (cronErr) {
|
|
1783
|
+
const error = cronErr instanceof Error ? cronErr : new Error(
|
|
1784
|
+
`croner failed to initialize for expression "${job.cron}": ${String(cronErr)}`
|
|
1533
1785
|
);
|
|
1534
|
-
|
|
1535
|
-
|
|
1536
|
-
|
|
1537
|
-
|
|
1538
|
-
|
|
1539
|
-
|
|
1540
|
-
|
|
1541
|
-
|
|
1542
|
-
|
|
1543
|
-
nextMs = Date.now() + 6e4;
|
|
1786
|
+
this.emitter.emit("worker:error", this.id, error);
|
|
1787
|
+
return;
|
|
1788
|
+
}
|
|
1789
|
+
if (nextDate === null) {
|
|
1790
|
+
const error = new Error(
|
|
1791
|
+
`Cron expression "${job.cron}" has no future occurrences \u2014 job "${job.id}" will not be re-enqueued`
|
|
1792
|
+
);
|
|
1793
|
+
this.emitter.emit("worker:error", this.id, error);
|
|
1794
|
+
return;
|
|
1544
1795
|
}
|
|
1545
|
-
const runAt =
|
|
1796
|
+
const runAt = nextDate.toISOString();
|
|
1546
1797
|
await this.storage.enqueue({
|
|
1547
1798
|
id: generateJobId(),
|
|
1548
1799
|
queue: this.queueName,
|
|
@@ -1576,6 +1827,7 @@ var Worker = class {
|
|
|
1576
1827
|
runAt,
|
|
1577
1828
|
error: error.message,
|
|
1578
1829
|
attemptNumber,
|
|
1830
|
+
lockId: this.id,
|
|
1579
1831
|
...error.stack !== void 0 && { stack: error.stack }
|
|
1580
1832
|
});
|
|
1581
1833
|
const updated = await this.storage.getJob(job.id);
|
|
@@ -1587,6 +1839,7 @@ var Worker = class {
|
|
|
1587
1839
|
jobId: job.id,
|
|
1588
1840
|
error: error.message,
|
|
1589
1841
|
attemptNumber,
|
|
1842
|
+
lockId: this.id,
|
|
1590
1843
|
...error.stack !== void 0 && { stack: error.stack }
|
|
1591
1844
|
});
|
|
1592
1845
|
const dead = await this.storage.getJob(job.id);
|
|
@@ -1846,10 +2099,29 @@ var InMemoryStorageAdapter = class {
|
|
|
1846
2099
|
return job;
|
|
1847
2100
|
}
|
|
1848
2101
|
// -------------------------------------------------------------------------
|
|
2102
|
+
// Renew lock
|
|
2103
|
+
// -------------------------------------------------------------------------
|
|
2104
|
+
async renewLock(jobId, lockId, lockDuration) {
|
|
2105
|
+
const job = this.jobs.get(jobId);
|
|
2106
|
+
if (!job) return false;
|
|
2107
|
+
if (job.status !== "active" || job.lockId !== lockId) return false;
|
|
2108
|
+
const nowMs = Date.now();
|
|
2109
|
+
if (job.lockExpiresAt !== null && new Date(job.lockExpiresAt).getTime() <= nowMs) {
|
|
2110
|
+
return false;
|
|
2111
|
+
}
|
|
2112
|
+
job.lockExpiresAt = new Date(nowMs + lockDuration).toISOString();
|
|
2113
|
+
job.updatedAt = new Date(nowMs).toISOString();
|
|
2114
|
+
return true;
|
|
2115
|
+
}
|
|
2116
|
+
// -------------------------------------------------------------------------
|
|
1849
2117
|
// Complete
|
|
1850
2118
|
// -------------------------------------------------------------------------
|
|
1851
|
-
async complete(jobId) {
|
|
1852
|
-
const job = this.
|
|
2119
|
+
async complete(jobId, lockId) {
|
|
2120
|
+
const job = this.jobs.get(jobId);
|
|
2121
|
+
if (!job) return;
|
|
2122
|
+
if (lockId !== void 0 && (job.status !== "active" || job.lockId !== lockId)) {
|
|
2123
|
+
return;
|
|
2124
|
+
}
|
|
1853
2125
|
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
1854
2126
|
job.status = "completed";
|
|
1855
2127
|
job.lockId = null;
|
|
@@ -1861,7 +2133,11 @@ var InMemoryStorageAdapter = class {
|
|
|
1861
2133
|
// Requeue (retry)
|
|
1862
2134
|
// -------------------------------------------------------------------------
|
|
1863
2135
|
async requeue(input) {
|
|
1864
|
-
const job = this.
|
|
2136
|
+
const job = this.jobs.get(input.jobId);
|
|
2137
|
+
if (!job) return;
|
|
2138
|
+
if (input.lockId !== void 0 && (job.status !== "active" || job.lockId !== input.lockId)) {
|
|
2139
|
+
return;
|
|
2140
|
+
}
|
|
1865
2141
|
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
1866
2142
|
job.attempts.push({
|
|
1867
2143
|
attempt: input.attemptNumber,
|
|
@@ -1881,7 +2157,11 @@ var InMemoryStorageAdapter = class {
|
|
|
1881
2157
|
// Move to DLQ
|
|
1882
2158
|
// -------------------------------------------------------------------------
|
|
1883
2159
|
async moveToDlq(input) {
|
|
1884
|
-
const job = this.
|
|
2160
|
+
const job = this.jobs.get(input.jobId);
|
|
2161
|
+
if (!job) return;
|
|
2162
|
+
if (input.lockId !== void 0 && (job.status !== "active" || job.lockId !== input.lockId)) {
|
|
2163
|
+
return;
|
|
2164
|
+
}
|
|
1885
2165
|
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
1886
2166
|
job.attempts.push({
|
|
1887
2167
|
attempt: input.attemptNumber,
|
|
@@ -1900,9 +2180,12 @@ var InMemoryStorageAdapter = class {
|
|
|
1900
2180
|
// -------------------------------------------------------------------------
|
|
1901
2181
|
// Release lock
|
|
1902
2182
|
// -------------------------------------------------------------------------
|
|
1903
|
-
async releaseLock(jobId) {
|
|
2183
|
+
async releaseLock(jobId, lockId) {
|
|
1904
2184
|
const job = this.jobs.get(jobId);
|
|
1905
2185
|
if (!job) return;
|
|
2186
|
+
if (lockId !== void 0 && (job.status !== "active" || job.lockId !== lockId)) {
|
|
2187
|
+
return;
|
|
2188
|
+
}
|
|
1906
2189
|
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
1907
2190
|
job.lockId = null;
|
|
1908
2191
|
job.lockExpiresAt = now;
|
|
@@ -1910,6 +2193,12 @@ var InMemoryStorageAdapter = class {
|
|
|
1910
2193
|
}
|
|
1911
2194
|
// -------------------------------------------------------------------------
|
|
1912
2195
|
// Recover stalled jobs
|
|
2196
|
+
//
|
|
2197
|
+
// Fix (issue #6): snapshot lockId and lockExpiresAt before the eligibility
|
|
2198
|
+
// check, then re-validate both values at write time. In a single-process
|
|
2199
|
+
// scenario all operations are synchronous within one event-loop tick, so
|
|
2200
|
+
// the race is theoretical — but the guard makes the adapter consistent with
|
|
2201
|
+
// the Redis CAS semantics and protects against any future async paths.
|
|
1913
2202
|
// -------------------------------------------------------------------------
|
|
1914
2203
|
async recoverStalledJobs(queue, now) {
|
|
1915
2204
|
const nowMs = new Date(now).getTime();
|
|
@@ -1919,13 +2208,17 @@ var InMemoryStorageAdapter = class {
|
|
|
1919
2208
|
if (job.status !== "active") continue;
|
|
1920
2209
|
if (job.lockExpiresAt === null) continue;
|
|
1921
2210
|
const lockExpiry = new Date(job.lockExpiresAt).getTime();
|
|
1922
|
-
if (lockExpiry
|
|
1923
|
-
|
|
1924
|
-
|
|
1925
|
-
|
|
1926
|
-
|
|
1927
|
-
|
|
1928
|
-
|
|
2211
|
+
if (lockExpiry > nowMs) continue;
|
|
2212
|
+
const snapshotLockId = job.lockId;
|
|
2213
|
+
const snapshotLockExpiresAt = job.lockExpiresAt;
|
|
2214
|
+
if (job.status !== "active") continue;
|
|
2215
|
+
if (job.lockId !== snapshotLockId) continue;
|
|
2216
|
+
if (job.lockExpiresAt !== snapshotLockExpiresAt) continue;
|
|
2217
|
+
job.status = "waiting";
|
|
2218
|
+
job.lockId = null;
|
|
2219
|
+
job.lockExpiresAt = null;
|
|
2220
|
+
job.updatedAt = now;
|
|
2221
|
+
recovered.push(job.id);
|
|
1929
2222
|
}
|
|
1930
2223
|
return recovered;
|
|
1931
2224
|
}
|