queue-jobs-worker 1.0.1 → 1.0.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +155 -122
- package/README.md +142 -386
- package/assets/queue-jobs-worker-github.png +0 -0
- package/dist/core/worker.d.ts.map +1 -1
- package/dist/index.cjs +430 -137
- package/dist/index.cjs.map +1 -1
- package/dist/index.js +430 -137
- package/dist/index.js.map +1 -1
- package/dist/storage/in-memory.adapter.d.ts +3 -2
- package/dist/storage/in-memory.adapter.d.ts.map +1 -1
- package/dist/storage/mysql.adapter.d.ts +3 -2
- package/dist/storage/mysql.adapter.d.ts.map +1 -1
- package/dist/storage/postgres.adapter.d.ts +3 -2
- package/dist/storage/postgres.adapter.d.ts.map +1 -1
- package/dist/storage/redis.adapter.d.ts +3 -2
- package/dist/storage/redis.adapter.d.ts.map +1 -1
- package/dist/types/storage.types.d.ts +20 -2
- package/dist/types/storage.types.d.ts.map +1 -1
- package/dist/types/worker.types.d.ts +1 -1
- package/dist/types/worker.types.d.ts.map +1 -1
- package/package.json +110 -97
package/dist/index.cjs
CHANGED
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
'use strict';
|
|
2
2
|
|
|
3
3
|
var crypto = require('crypto');
|
|
4
|
+
var croner = require('croner');
|
|
4
5
|
var events = require('events');
|
|
5
6
|
|
|
6
7
|
var __defProp = Object.defineProperty;
|
|
@@ -76,7 +77,7 @@ function hashToJob(h) {
|
|
|
76
77
|
if (h["cron"]) job.cron = h["cron"];
|
|
77
78
|
return job;
|
|
78
79
|
}
|
|
79
|
-
var PREFIX, k, CLAIM_LUA; exports.RedisStorageAdapter = void 0;
|
|
80
|
+
var PREFIX, k, CLAIM_LUA, RECOVER_STALLED_LUA, RENEW_LOCK_LUA; exports.RedisStorageAdapter = void 0;
|
|
80
81
|
var init_redis_adapter = __esm({
|
|
81
82
|
"src/storage/redis.adapter.ts"() {
|
|
82
83
|
PREFIX = "qjw:";
|
|
@@ -118,6 +119,66 @@ redis.call('HSET', prefix .. 'job:' .. job_id,
|
|
|
118
119
|
'updatedAt', ARGV[2]
|
|
119
120
|
)
|
|
120
121
|
return job_id
|
|
122
|
+
`;
|
|
123
|
+
RECOVER_STALLED_LUA = `
|
|
124
|
+
local job_key = KEYS[1]
|
|
125
|
+
local active_set = KEYS[2]
|
|
126
|
+
local waiting_zset = KEYS[3]
|
|
127
|
+
|
|
128
|
+
local job_id = ARGV[1]
|
|
129
|
+
local expected_exp = ARGV[2]
|
|
130
|
+
local expected_lock = ARGV[3]
|
|
131
|
+
local recovery_ts = ARGV[4]
|
|
132
|
+
local priority_score = tonumber(ARGV[5])
|
|
133
|
+
|
|
134
|
+
-- Re-read the three guard fields in one atomic HMGET.
|
|
135
|
+
local fields = redis.call('HMGET', job_key, 'lockExpiresAt', 'lockId', 'status')
|
|
136
|
+
local current_exp = fields[1] or ''
|
|
137
|
+
local current_lock = fields[2] or ''
|
|
138
|
+
local current_status = fields[3] or ''
|
|
139
|
+
|
|
140
|
+
-- Guard 1: job must still be active.
|
|
141
|
+
-- If the worker already completed, failed, or moved to DLQ, skip.
|
|
142
|
+
if current_status ~= 'active' then return 0 end
|
|
143
|
+
|
|
144
|
+
-- Guard 2: lockId must not have changed.
|
|
145
|
+
-- A different worker may have claimed the job after the original lock expired
|
|
146
|
+
-- and before this script runs.
|
|
147
|
+
if current_lock ~= expected_lock then return 0 end
|
|
148
|
+
|
|
149
|
+
-- Guard 3: lockExpiresAt must be exactly what the caller observed.
|
|
150
|
+
-- If the worker renewed its lock the timestamp will be later than observed;
|
|
151
|
+
-- the CAS mismatch catches that without needing ISO\u2192epoch conversion in Lua.
|
|
152
|
+
if current_exp ~= expected_exp then return 0 end
|
|
153
|
+
|
|
154
|
+
-- All guards passed \u2014 recover atomically.
|
|
155
|
+
redis.call('HSET', job_key,
|
|
156
|
+
'status', 'waiting',
|
|
157
|
+
'lockId', '',
|
|
158
|
+
'lockExpiresAt', '',
|
|
159
|
+
'updatedAt', recovery_ts
|
|
160
|
+
)
|
|
161
|
+
redis.call('SREM', active_set, job_id)
|
|
162
|
+
redis.call('ZADD', waiting_zset, priority_score, job_id)
|
|
163
|
+
return 1
|
|
164
|
+
`;
|
|
165
|
+
RENEW_LOCK_LUA = `
|
|
166
|
+
local job_key = KEYS[1]
|
|
167
|
+
local lock_id = ARGV[1]
|
|
168
|
+
local new_exp = ARGV[2]
|
|
169
|
+
local now_iso = ARGV[3]
|
|
170
|
+
|
|
171
|
+
local fields = redis.call('HMGET', job_key, 'status', 'lockId', 'lockExpiresAt')
|
|
172
|
+
local status = fields[1] or ''
|
|
173
|
+
local current_lock = fields[2] or ''
|
|
174
|
+
local current_exp = fields[3] or ''
|
|
175
|
+
|
|
176
|
+
if status ~= 'active' then return 0 end
|
|
177
|
+
if current_lock ~= lock_id then return 0 end
|
|
178
|
+
if current_exp == '' or current_exp <= now_iso then return 0 end
|
|
179
|
+
|
|
180
|
+
redis.call('HSET', job_key, 'lockExpiresAt', new_exp, 'updatedAt', now_iso)
|
|
181
|
+
return 1
|
|
121
182
|
`;
|
|
122
183
|
exports.RedisStorageAdapter = class {
|
|
123
184
|
client;
|
|
@@ -208,12 +269,28 @@ return job_id
|
|
|
208
269
|
return hashToJob(hash);
|
|
209
270
|
}
|
|
210
271
|
// -------------------------------------------------------------------------
|
|
272
|
+
// Renew lock
|
|
273
|
+
// -------------------------------------------------------------------------
|
|
274
|
+
async renewLock(jobId, lockId, lockDuration) {
|
|
275
|
+
const nowMs = Date.now();
|
|
276
|
+
const nowIso = new Date(nowMs).toISOString();
|
|
277
|
+
const newExpiresAt = new Date(nowMs + lockDuration).toISOString();
|
|
278
|
+
const res = await this.client.eval(RENEW_LOCK_LUA, {
|
|
279
|
+
keys: [k.job(jobId)],
|
|
280
|
+
arguments: [lockId, newExpiresAt, nowIso]
|
|
281
|
+
});
|
|
282
|
+
return Number(res) === 1;
|
|
283
|
+
}
|
|
284
|
+
// -------------------------------------------------------------------------
|
|
211
285
|
// Complete
|
|
212
286
|
// -------------------------------------------------------------------------
|
|
213
|
-
async complete(jobId) {
|
|
287
|
+
async complete(jobId, lockId) {
|
|
214
288
|
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
215
289
|
const hash = await this.client.hGetAll(k.job(jobId));
|
|
216
|
-
if (!hash) return;
|
|
290
|
+
if (!hash || Object.keys(hash).length === 0) return;
|
|
291
|
+
if (lockId !== void 0) {
|
|
292
|
+
if (hash["status"] !== "active" || hash["lockId"] !== lockId) return;
|
|
293
|
+
}
|
|
217
294
|
const queue = hash["queue"] ?? "";
|
|
218
295
|
const multi = this.client.multi();
|
|
219
296
|
multi.hSet(k.job(jobId), {
|
|
@@ -233,7 +310,10 @@ return job_id
|
|
|
233
310
|
async requeue(input) {
|
|
234
311
|
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
235
312
|
const hash = await this.client.hGetAll(k.job(input.jobId));
|
|
236
|
-
if (!hash) return;
|
|
313
|
+
if (!hash || Object.keys(hash).length === 0) return;
|
|
314
|
+
if (input.lockId !== void 0) {
|
|
315
|
+
if (hash["status"] !== "active" || hash["lockId"] !== input.lockId) return;
|
|
316
|
+
}
|
|
237
317
|
const queue = hash["queue"] ?? "";
|
|
238
318
|
const attempts = JSON.parse(hash["attempts"] ?? "[]");
|
|
239
319
|
attempts.push({
|
|
@@ -266,7 +346,10 @@ return job_id
|
|
|
266
346
|
async moveToDlq(input) {
|
|
267
347
|
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
268
348
|
const hash = await this.client.hGetAll(k.job(input.jobId));
|
|
269
|
-
if (!hash) return;
|
|
349
|
+
if (!hash || Object.keys(hash).length === 0) return;
|
|
350
|
+
if (input.lockId !== void 0) {
|
|
351
|
+
if (hash["status"] !== "active" || hash["lockId"] !== input.lockId) return;
|
|
352
|
+
}
|
|
270
353
|
const queue = hash["queue"] ?? "";
|
|
271
354
|
const attempts = JSON.parse(hash["attempts"] ?? "[]");
|
|
272
355
|
attempts.push({
|
|
@@ -293,7 +376,12 @@ return job_id
|
|
|
293
376
|
// -------------------------------------------------------------------------
|
|
294
377
|
// Release lock
|
|
295
378
|
// -------------------------------------------------------------------------
|
|
296
|
-
async releaseLock(jobId) {
|
|
379
|
+
async releaseLock(jobId, lockId) {
|
|
380
|
+
const hash = await this.client.hGetAll(k.job(jobId));
|
|
381
|
+
if (!hash || Object.keys(hash).length === 0) return;
|
|
382
|
+
if (lockId !== void 0) {
|
|
383
|
+
if (hash["status"] !== "active" || hash["lockId"] !== lockId) return;
|
|
384
|
+
}
|
|
297
385
|
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
298
386
|
await this.client.hSet(k.job(jobId), {
|
|
299
387
|
lockId: "",
|
|
@@ -302,7 +390,18 @@ return job_id
|
|
|
302
390
|
});
|
|
303
391
|
}
|
|
304
392
|
// -------------------------------------------------------------------------
|
|
305
|
-
// Recover stalled jobs (
|
|
393
|
+
// Recover stalled jobs (atomic compare-and-swap per job via Lua)
|
|
394
|
+
//
|
|
395
|
+
// Previous implementation: two-phase read pipeline → write pipeline.
|
|
396
|
+
// Race condition: a worker could complete or renew its lock between the two
|
|
397
|
+
// phases, causing recovery to overwrite a legitimately-active job.
|
|
398
|
+
//
|
|
399
|
+
// Fix (issue #6): for each candidate job the RECOVER_STALLED_LUA script
|
|
400
|
+
// re-reads lockExpiresAt, lockId, and status atomically and only applies
|
|
401
|
+
// the recovery if all three still match what was observed in the read phase
|
|
402
|
+
// (compare-and-swap). If the worker renewed or completed the job in the
|
|
403
|
+
// window between the read and the Lua call, the CAS mismatch causes the
|
|
404
|
+
// script to return 0 and the job is left untouched.
|
|
306
405
|
// -------------------------------------------------------------------------
|
|
307
406
|
async recoverStalledJobs(queue, now) {
|
|
308
407
|
const nowMs = new Date(now).getTime();
|
|
@@ -310,32 +409,29 @@ return job_id
|
|
|
310
409
|
if (activeIds.length === 0) return [];
|
|
311
410
|
const fetchPipeline = this.client.multi();
|
|
312
411
|
for (const jobId of activeIds) {
|
|
313
|
-
fetchPipeline.hmGet(k.job(jobId), ["lockExpiresAt", "priority"]);
|
|
412
|
+
fetchPipeline.hmGet(k.job(jobId), ["lockExpiresAt", "lockId", "priority"]);
|
|
314
413
|
}
|
|
315
414
|
const fetchResults = await fetchPipeline.exec();
|
|
316
415
|
const recovered = [];
|
|
317
|
-
const
|
|
416
|
+
const evalPromises = [];
|
|
318
417
|
for (let i = 0; i < activeIds.length; i++) {
|
|
319
418
|
const jobId = activeIds[i];
|
|
320
419
|
const fields = fetchResults[i];
|
|
321
420
|
if (!fields) continue;
|
|
322
|
-
const [lockExpiresAt, priorityStr] = fields;
|
|
421
|
+
const [lockExpiresAt, lockId, priorityStr] = fields;
|
|
323
422
|
if (!lockExpiresAt) continue;
|
|
324
423
|
if (new Date(lockExpiresAt).getTime() > nowMs) continue;
|
|
325
424
|
const priority = Number(priorityStr ?? "0");
|
|
326
|
-
|
|
327
|
-
|
|
328
|
-
|
|
329
|
-
|
|
330
|
-
|
|
425
|
+
const priorityScore = String(-priority);
|
|
426
|
+
const p = this.client.eval(RECOVER_STALLED_LUA, {
|
|
427
|
+
keys: [k.job(jobId), k.active(queue), k.waiting(queue)],
|
|
428
|
+
arguments: [jobId, lockExpiresAt, lockId ?? "", now, priorityScore]
|
|
429
|
+
}).then((result) => {
|
|
430
|
+
if (result === 1) recovered.push(jobId);
|
|
331
431
|
});
|
|
332
|
-
|
|
333
|
-
recoverPipeline.zAdd(k.waiting(queue), { score: -priority, value: jobId });
|
|
334
|
-
recovered.push(jobId);
|
|
335
|
-
}
|
|
336
|
-
if (recovered.length > 0) {
|
|
337
|
-
await recoverPipeline.exec();
|
|
432
|
+
evalPromises.push(p);
|
|
338
433
|
}
|
|
434
|
+
await Promise.all(evalPromises);
|
|
339
435
|
return recovered;
|
|
340
436
|
}
|
|
341
437
|
// -------------------------------------------------------------------------
|
|
@@ -612,16 +708,40 @@ CREATE TABLE IF NOT EXISTS qjw_rate_limits (
|
|
|
612
708
|
}
|
|
613
709
|
}
|
|
614
710
|
// -------------------------------------------------------------------------
|
|
615
|
-
//
|
|
711
|
+
// Renew lock
|
|
616
712
|
// -------------------------------------------------------------------------
|
|
617
|
-
async
|
|
618
|
-
|
|
713
|
+
async renewLock(jobId, lockId, lockDuration) {
|
|
714
|
+
const newLockExpiresAt = new Date(Date.now() + lockDuration).toISOString();
|
|
715
|
+
const res = await this.pool.query(
|
|
619
716
|
`UPDATE qjw_jobs
|
|
620
|
-
SET
|
|
621
|
-
|
|
622
|
-
WHERE id = $
|
|
623
|
-
[jobId]
|
|
717
|
+
SET lock_expires_at = $1::timestamptz,
|
|
718
|
+
updated_at = NOW()
|
|
719
|
+
WHERE id = $2 AND status = 'active' AND lock_id = $3 AND lock_expires_at > NOW()`,
|
|
720
|
+
[newLockExpiresAt, jobId, lockId]
|
|
624
721
|
);
|
|
722
|
+
return (res.rowCount ?? 0) > 0;
|
|
723
|
+
}
|
|
724
|
+
// -------------------------------------------------------------------------
|
|
725
|
+
// Complete
|
|
726
|
+
// -------------------------------------------------------------------------
|
|
727
|
+
async complete(jobId, lockId) {
|
|
728
|
+
if (lockId !== void 0) {
|
|
729
|
+
await this.pool.query(
|
|
730
|
+
`UPDATE qjw_jobs
|
|
731
|
+
SET status = 'completed', lock_id = NULL, lock_expires_at = NULL,
|
|
732
|
+
completed_at = NOW(), updated_at = NOW()
|
|
733
|
+
WHERE id = $1 AND status = 'active' AND lock_id = $2`,
|
|
734
|
+
[jobId, lockId]
|
|
735
|
+
);
|
|
736
|
+
} else {
|
|
737
|
+
await this.pool.query(
|
|
738
|
+
`UPDATE qjw_jobs
|
|
739
|
+
SET status = 'completed', lock_id = NULL, lock_expires_at = NULL,
|
|
740
|
+
completed_at = NOW(), updated_at = NOW()
|
|
741
|
+
WHERE id = $1`,
|
|
742
|
+
[jobId]
|
|
743
|
+
);
|
|
744
|
+
}
|
|
625
745
|
}
|
|
626
746
|
// -------------------------------------------------------------------------
|
|
627
747
|
// Requeue
|
|
@@ -635,18 +755,33 @@ CREATE TABLE IF NOT EXISTS qjw_rate_limits (
|
|
|
635
755
|
error: input.error,
|
|
636
756
|
...input.stack !== void 0 ? { stack: input.stack } : {}
|
|
637
757
|
};
|
|
638
|
-
|
|
639
|
-
|
|
640
|
-
|
|
641
|
-
|
|
642
|
-
|
|
643
|
-
|
|
644
|
-
|
|
645
|
-
|
|
646
|
-
|
|
647
|
-
|
|
648
|
-
|
|
649
|
-
|
|
758
|
+
if (input.lockId !== void 0) {
|
|
759
|
+
await this.pool.query(
|
|
760
|
+
`UPDATE qjw_jobs
|
|
761
|
+
SET status = 'waiting',
|
|
762
|
+
attempts_made = $1,
|
|
763
|
+
attempts = attempts || $2::jsonb,
|
|
764
|
+
run_at = $3::timestamptz,
|
|
765
|
+
lock_id = NULL,
|
|
766
|
+
lock_expires_at = NULL,
|
|
767
|
+
updated_at = NOW()
|
|
768
|
+
WHERE id = $4 AND status = 'active' AND lock_id = $5`,
|
|
769
|
+
[input.attemptNumber, JSON.stringify([attempt]), input.runAt, input.jobId, input.lockId]
|
|
770
|
+
);
|
|
771
|
+
} else {
|
|
772
|
+
await this.pool.query(
|
|
773
|
+
`UPDATE qjw_jobs
|
|
774
|
+
SET status = 'waiting',
|
|
775
|
+
attempts_made = $1,
|
|
776
|
+
attempts = attempts || $2::jsonb,
|
|
777
|
+
run_at = $3::timestamptz,
|
|
778
|
+
lock_id = NULL,
|
|
779
|
+
lock_expires_at = NULL,
|
|
780
|
+
updated_at = NOW()
|
|
781
|
+
WHERE id = $4`,
|
|
782
|
+
[input.attemptNumber, JSON.stringify([attempt]), input.runAt, input.jobId]
|
|
783
|
+
);
|
|
784
|
+
}
|
|
650
785
|
}
|
|
651
786
|
// -------------------------------------------------------------------------
|
|
652
787
|
// Move to DLQ
|
|
@@ -660,29 +795,53 @@ CREATE TABLE IF NOT EXISTS qjw_rate_limits (
|
|
|
660
795
|
error: input.error,
|
|
661
796
|
...input.stack !== void 0 ? { stack: input.stack } : {}
|
|
662
797
|
};
|
|
663
|
-
|
|
664
|
-
|
|
665
|
-
|
|
666
|
-
|
|
667
|
-
|
|
668
|
-
|
|
669
|
-
|
|
670
|
-
|
|
671
|
-
|
|
672
|
-
|
|
673
|
-
|
|
674
|
-
|
|
798
|
+
if (input.lockId !== void 0) {
|
|
799
|
+
await this.pool.query(
|
|
800
|
+
`UPDATE qjw_jobs
|
|
801
|
+
SET status = 'dead',
|
|
802
|
+
attempts_made = $1,
|
|
803
|
+
attempts = attempts || $2::jsonb,
|
|
804
|
+
lock_id = NULL,
|
|
805
|
+
lock_expires_at = NULL,
|
|
806
|
+
failed_at = NOW(),
|
|
807
|
+
updated_at = NOW()
|
|
808
|
+
WHERE id = $3 AND status = 'active' AND lock_id = $4`,
|
|
809
|
+
[input.attemptNumber, JSON.stringify([attempt]), input.jobId, input.lockId]
|
|
810
|
+
);
|
|
811
|
+
} else {
|
|
812
|
+
await this.pool.query(
|
|
813
|
+
`UPDATE qjw_jobs
|
|
814
|
+
SET status = 'dead',
|
|
815
|
+
attempts_made = $1,
|
|
816
|
+
attempts = attempts || $2::jsonb,
|
|
817
|
+
lock_id = NULL,
|
|
818
|
+
lock_expires_at = NULL,
|
|
819
|
+
failed_at = NOW(),
|
|
820
|
+
updated_at = NOW()
|
|
821
|
+
WHERE id = $3`,
|
|
822
|
+
[input.attemptNumber, JSON.stringify([attempt]), input.jobId]
|
|
823
|
+
);
|
|
824
|
+
}
|
|
675
825
|
}
|
|
676
826
|
// -------------------------------------------------------------------------
|
|
677
827
|
// Release lock
|
|
678
828
|
// -------------------------------------------------------------------------
|
|
679
|
-
async releaseLock(jobId) {
|
|
680
|
-
|
|
681
|
-
|
|
682
|
-
|
|
683
|
-
|
|
684
|
-
|
|
685
|
-
|
|
829
|
+
async releaseLock(jobId, lockId) {
|
|
830
|
+
if (lockId !== void 0) {
|
|
831
|
+
await this.pool.query(
|
|
832
|
+
`UPDATE qjw_jobs
|
|
833
|
+
SET lock_id = NULL, lock_expires_at = NOW(), updated_at = NOW()
|
|
834
|
+
WHERE id = $1 AND status = 'active' AND lock_id = $2`,
|
|
835
|
+
[jobId, lockId]
|
|
836
|
+
);
|
|
837
|
+
} else {
|
|
838
|
+
await this.pool.query(
|
|
839
|
+
`UPDATE qjw_jobs
|
|
840
|
+
SET lock_id = NULL, lock_expires_at = NOW(), updated_at = NOW()
|
|
841
|
+
WHERE id = $1`,
|
|
842
|
+
[jobId]
|
|
843
|
+
);
|
|
844
|
+
}
|
|
686
845
|
}
|
|
687
846
|
// -------------------------------------------------------------------------
|
|
688
847
|
// Recover stalled jobs
|
|
@@ -963,16 +1122,39 @@ CREATE TABLE IF NOT EXISTS qjw_rate_limits (
|
|
|
963
1122
|
}
|
|
964
1123
|
}
|
|
965
1124
|
// -------------------------------------------------------------------------
|
|
966
|
-
//
|
|
1125
|
+
// Renew lock
|
|
967
1126
|
// -------------------------------------------------------------------------
|
|
968
|
-
async
|
|
969
|
-
|
|
1127
|
+
async renewLock(jobId, lockId, lockDuration) {
|
|
1128
|
+
const newLockExpiresAt = new Date(Date.now() + lockDuration).toISOString().slice(0, 23).replace("T", " ");
|
|
1129
|
+
const [result] = await this.pool.query(
|
|
970
1130
|
`UPDATE qjw_jobs
|
|
971
|
-
SET
|
|
972
|
-
|
|
973
|
-
|
|
974
|
-
[jobId]
|
|
1131
|
+
SET lock_expires_at = ?, updated_at = NOW(3)
|
|
1132
|
+
WHERE id = ? AND status = 'active' AND lock_id = ? AND lock_expires_at > NOW(3)`,
|
|
1133
|
+
[newLockExpiresAt, jobId, lockId]
|
|
975
1134
|
);
|
|
1135
|
+
return result.affectedRows > 0;
|
|
1136
|
+
}
|
|
1137
|
+
// -------------------------------------------------------------------------
|
|
1138
|
+
// Complete
|
|
1139
|
+
// -------------------------------------------------------------------------
|
|
1140
|
+
async complete(jobId, lockId) {
|
|
1141
|
+
if (lockId !== void 0) {
|
|
1142
|
+
await this.pool.query(
|
|
1143
|
+
`UPDATE qjw_jobs
|
|
1144
|
+
SET status = 'completed', lock_id = NULL, lock_expires_at = NULL,
|
|
1145
|
+
completed_at = NOW(3), updated_at = NOW(3)
|
|
1146
|
+
WHERE id = ? AND status = 'active' AND lock_id = ?`,
|
|
1147
|
+
[jobId, lockId]
|
|
1148
|
+
);
|
|
1149
|
+
} else {
|
|
1150
|
+
await this.pool.query(
|
|
1151
|
+
`UPDATE qjw_jobs
|
|
1152
|
+
SET status = 'completed', lock_id = NULL, lock_expires_at = NULL,
|
|
1153
|
+
completed_at = NOW(3), updated_at = NOW(3)
|
|
1154
|
+
WHERE id = ?`,
|
|
1155
|
+
[jobId]
|
|
1156
|
+
);
|
|
1157
|
+
}
|
|
976
1158
|
}
|
|
977
1159
|
// -------------------------------------------------------------------------
|
|
978
1160
|
// Requeue
|
|
@@ -986,18 +1168,33 @@ CREATE TABLE IF NOT EXISTS qjw_rate_limits (
|
|
|
986
1168
|
error: input.error,
|
|
987
1169
|
...input.stack !== void 0 ? { stack: input.stack } : {}
|
|
988
1170
|
});
|
|
989
|
-
|
|
990
|
-
|
|
991
|
-
|
|
992
|
-
|
|
993
|
-
|
|
994
|
-
|
|
995
|
-
|
|
996
|
-
|
|
997
|
-
|
|
998
|
-
|
|
999
|
-
|
|
1000
|
-
|
|
1171
|
+
if (input.lockId !== void 0) {
|
|
1172
|
+
await this.pool.query(
|
|
1173
|
+
`UPDATE qjw_jobs
|
|
1174
|
+
SET status = 'waiting',
|
|
1175
|
+
attempts_made = attempts_made + 1,
|
|
1176
|
+
attempts = JSON_ARRAY_APPEND(attempts, '$', CAST(? AS JSON)),
|
|
1177
|
+
run_at = ?,
|
|
1178
|
+
lock_id = NULL,
|
|
1179
|
+
lock_expires_at = NULL,
|
|
1180
|
+
updated_at = NOW(3)
|
|
1181
|
+
WHERE id = ? AND status = 'active' AND lock_id = ?`,
|
|
1182
|
+
[attempt, input.runAt, input.jobId, input.lockId]
|
|
1183
|
+
);
|
|
1184
|
+
} else {
|
|
1185
|
+
await this.pool.query(
|
|
1186
|
+
`UPDATE qjw_jobs
|
|
1187
|
+
SET status = 'waiting',
|
|
1188
|
+
attempts_made = attempts_made + 1,
|
|
1189
|
+
attempts = JSON_ARRAY_APPEND(attempts, '$', CAST(? AS JSON)),
|
|
1190
|
+
run_at = ?,
|
|
1191
|
+
lock_id = NULL,
|
|
1192
|
+
lock_expires_at = NULL,
|
|
1193
|
+
updated_at = NOW(3)
|
|
1194
|
+
WHERE id = ?`,
|
|
1195
|
+
[attempt, input.runAt, input.jobId]
|
|
1196
|
+
);
|
|
1197
|
+
}
|
|
1001
1198
|
}
|
|
1002
1199
|
// -------------------------------------------------------------------------
|
|
1003
1200
|
// Move to DLQ
|
|
@@ -1011,29 +1208,53 @@ CREATE TABLE IF NOT EXISTS qjw_rate_limits (
|
|
|
1011
1208
|
error: input.error,
|
|
1012
1209
|
...input.stack !== void 0 ? { stack: input.stack } : {}
|
|
1013
1210
|
});
|
|
1014
|
-
|
|
1015
|
-
|
|
1016
|
-
|
|
1017
|
-
|
|
1018
|
-
|
|
1019
|
-
|
|
1020
|
-
|
|
1021
|
-
|
|
1022
|
-
|
|
1023
|
-
|
|
1024
|
-
|
|
1025
|
-
|
|
1211
|
+
if (input.lockId !== void 0) {
|
|
1212
|
+
await this.pool.query(
|
|
1213
|
+
`UPDATE qjw_jobs
|
|
1214
|
+
SET status = 'dead',
|
|
1215
|
+
attempts_made = attempts_made + 1,
|
|
1216
|
+
attempts = JSON_ARRAY_APPEND(attempts, '$', CAST(? AS JSON)),
|
|
1217
|
+
lock_id = NULL,
|
|
1218
|
+
lock_expires_at = NULL,
|
|
1219
|
+
failed_at = NOW(3),
|
|
1220
|
+
updated_at = NOW(3)
|
|
1221
|
+
WHERE id = ? AND status = 'active' AND lock_id = ?`,
|
|
1222
|
+
[attempt, input.jobId, input.lockId]
|
|
1223
|
+
);
|
|
1224
|
+
} else {
|
|
1225
|
+
await this.pool.query(
|
|
1226
|
+
`UPDATE qjw_jobs
|
|
1227
|
+
SET status = 'dead',
|
|
1228
|
+
attempts_made = attempts_made + 1,
|
|
1229
|
+
attempts = JSON_ARRAY_APPEND(attempts, '$', CAST(? AS JSON)),
|
|
1230
|
+
lock_id = NULL,
|
|
1231
|
+
lock_expires_at = NULL,
|
|
1232
|
+
failed_at = NOW(3),
|
|
1233
|
+
updated_at = NOW(3)
|
|
1234
|
+
WHERE id = ?`,
|
|
1235
|
+
[attempt, input.jobId]
|
|
1236
|
+
);
|
|
1237
|
+
}
|
|
1026
1238
|
}
|
|
1027
1239
|
// -------------------------------------------------------------------------
|
|
1028
1240
|
// Release lock
|
|
1029
1241
|
// -------------------------------------------------------------------------
|
|
1030
|
-
async releaseLock(jobId) {
|
|
1031
|
-
|
|
1032
|
-
|
|
1033
|
-
|
|
1034
|
-
|
|
1035
|
-
|
|
1036
|
-
|
|
1242
|
+
async releaseLock(jobId, lockId) {
|
|
1243
|
+
if (lockId !== void 0) {
|
|
1244
|
+
await this.pool.query(
|
|
1245
|
+
`UPDATE qjw_jobs
|
|
1246
|
+
SET lock_id = NULL, lock_expires_at = NOW(3), updated_at = NOW(3)
|
|
1247
|
+
WHERE id = ? AND status = 'active' AND lock_id = ?`,
|
|
1248
|
+
[jobId, lockId]
|
|
1249
|
+
);
|
|
1250
|
+
} else {
|
|
1251
|
+
await this.pool.query(
|
|
1252
|
+
`UPDATE qjw_jobs
|
|
1253
|
+
SET lock_id = NULL, lock_expires_at = NOW(3), updated_at = NOW(3)
|
|
1254
|
+
WHERE id = ?`,
|
|
1255
|
+
[jobId]
|
|
1256
|
+
);
|
|
1257
|
+
}
|
|
1037
1258
|
}
|
|
1038
1259
|
// -------------------------------------------------------------------------
|
|
1039
1260
|
// Recover stalled jobs (atomic — SELECT … FOR UPDATE inside transaction)
|
|
@@ -1424,7 +1645,7 @@ var Worker = class {
|
|
|
1424
1645
|
if (this.activeJobIds.size > 0) {
|
|
1425
1646
|
await Promise.all(
|
|
1426
1647
|
Array.from(this.activeJobIds).map(
|
|
1427
|
-
(jobId) => this.storage.releaseLock(jobId).catch(() => {
|
|
1648
|
+
(jobId) => this.storage.releaseLock(jobId, this.id).catch(() => {
|
|
1428
1649
|
})
|
|
1429
1650
|
)
|
|
1430
1651
|
);
|
|
@@ -1461,6 +1682,13 @@ var Worker = class {
|
|
|
1461
1682
|
// -------------------------------------------------------------------------
|
|
1462
1683
|
async claimNext() {
|
|
1463
1684
|
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
1685
|
+
const raw = await this.storage.claim({
|
|
1686
|
+
queue: this.queueName,
|
|
1687
|
+
lockId: this.id,
|
|
1688
|
+
lockDuration: this.config.lockDuration,
|
|
1689
|
+
now
|
|
1690
|
+
});
|
|
1691
|
+
if (!raw) return false;
|
|
1464
1692
|
if (this.config.rateLimit) {
|
|
1465
1693
|
const allowed = await this.storage.checkAndIncrementRateLimit(
|
|
1466
1694
|
this.queueName,
|
|
@@ -1468,15 +1696,11 @@ var Worker = class {
|
|
|
1468
1696
|
this.config.rateLimit.duration,
|
|
1469
1697
|
now
|
|
1470
1698
|
);
|
|
1471
|
-
if (!allowed)
|
|
1699
|
+
if (!allowed) {
|
|
1700
|
+
await this.storage.releaseLock(raw.id, this.id);
|
|
1701
|
+
return false;
|
|
1702
|
+
}
|
|
1472
1703
|
}
|
|
1473
|
-
const raw = await this.storage.claim({
|
|
1474
|
-
queue: this.queueName,
|
|
1475
|
-
lockId: this.id,
|
|
1476
|
-
lockDuration: this.config.lockDuration,
|
|
1477
|
-
now
|
|
1478
|
-
});
|
|
1479
|
-
if (!raw) return false;
|
|
1480
1704
|
const job = new Job(raw);
|
|
1481
1705
|
this.activeCount += 1;
|
|
1482
1706
|
this.activeJobIds.add(job.id);
|
|
@@ -1493,16 +1717,35 @@ var Worker = class {
|
|
|
1493
1717
|
await this.handleFailure(job, error);
|
|
1494
1718
|
return;
|
|
1495
1719
|
}
|
|
1720
|
+
const abortController = new AbortController();
|
|
1496
1721
|
let timeoutHandle = null;
|
|
1722
|
+
let lockRenewTimer = null;
|
|
1723
|
+
const lockRenewInterval = Math.max(100, Math.floor(this.config.lockDuration / 2));
|
|
1724
|
+
lockRenewTimer = setInterval(async () => {
|
|
1725
|
+
try {
|
|
1726
|
+
const renewed = await this.storage.renewLock(job.id, this.id, this.config.lockDuration);
|
|
1727
|
+
if (!renewed && lockRenewTimer) {
|
|
1728
|
+
clearInterval(lockRenewTimer);
|
|
1729
|
+
lockRenewTimer = null;
|
|
1730
|
+
}
|
|
1731
|
+
} catch {
|
|
1732
|
+
}
|
|
1733
|
+
}, lockRenewInterval);
|
|
1497
1734
|
try {
|
|
1498
1735
|
await new Promise((resolve, reject) => {
|
|
1499
1736
|
timeoutHandle = setTimeout(() => {
|
|
1500
|
-
|
|
1737
|
+
const timeoutError = new Error(`Job timed out after ${job.timeout}ms`);
|
|
1738
|
+
abortController.abort(timeoutError);
|
|
1739
|
+
reject(timeoutError);
|
|
1501
1740
|
}, job.timeout);
|
|
1502
|
-
Promise.resolve(processor(job)).then(resolve, reject);
|
|
1741
|
+
Promise.resolve(processor(job, abortController.signal)).then(resolve, reject);
|
|
1503
1742
|
});
|
|
1504
1743
|
if (timeoutHandle) clearTimeout(timeoutHandle);
|
|
1505
|
-
|
|
1744
|
+
if (lockRenewTimer) {
|
|
1745
|
+
clearInterval(lockRenewTimer);
|
|
1746
|
+
lockRenewTimer = null;
|
|
1747
|
+
}
|
|
1748
|
+
await this.storage.complete(job.id, this.id);
|
|
1506
1749
|
const completedData = await this.storage.getJob(job.id) ?? job._data;
|
|
1507
1750
|
this.emitter.emit("job:completed", completedData);
|
|
1508
1751
|
if (job.cron) {
|
|
@@ -1510,9 +1753,17 @@ var Worker = class {
|
|
|
1510
1753
|
}
|
|
1511
1754
|
} catch (err) {
|
|
1512
1755
|
if (timeoutHandle) clearTimeout(timeoutHandle);
|
|
1756
|
+
if (lockRenewTimer) {
|
|
1757
|
+
clearInterval(lockRenewTimer);
|
|
1758
|
+
lockRenewTimer = null;
|
|
1759
|
+
}
|
|
1513
1760
|
const error = err instanceof Error ? err : new Error(String(err));
|
|
1514
1761
|
await this.handleFailure(job, error);
|
|
1515
1762
|
} finally {
|
|
1763
|
+
if (lockRenewTimer) {
|
|
1764
|
+
clearInterval(lockRenewTimer);
|
|
1765
|
+
lockRenewTimer = null;
|
|
1766
|
+
}
|
|
1516
1767
|
this.activeJobIds.delete(job.id);
|
|
1517
1768
|
this.activeCount -= 1;
|
|
1518
1769
|
if (this.activeCount === 0 && this.drainResolve) {
|
|
@@ -1526,25 +1777,25 @@ var Worker = class {
|
|
|
1526
1777
|
// -------------------------------------------------------------------------
|
|
1527
1778
|
async enqueueCronNext(job) {
|
|
1528
1779
|
try {
|
|
1529
|
-
let
|
|
1780
|
+
let nextDate;
|
|
1530
1781
|
try {
|
|
1531
|
-
const
|
|
1532
|
-
|
|
1533
|
-
|
|
1534
|
-
|
|
1782
|
+
const cronInstance = new croner.Cron(job.cron);
|
|
1783
|
+
nextDate = cronInstance.nextRun();
|
|
1784
|
+
} catch (cronErr) {
|
|
1785
|
+
const error = cronErr instanceof Error ? cronErr : new Error(
|
|
1786
|
+
`croner failed to initialize for expression "${job.cron}": ${String(cronErr)}`
|
|
1535
1787
|
);
|
|
1536
|
-
|
|
1537
|
-
|
|
1538
|
-
|
|
1539
|
-
|
|
1540
|
-
|
|
1541
|
-
|
|
1542
|
-
|
|
1543
|
-
|
|
1544
|
-
|
|
1545
|
-
nextMs = Date.now() + 6e4;
|
|
1788
|
+
this.emitter.emit("worker:error", this.id, error);
|
|
1789
|
+
return;
|
|
1790
|
+
}
|
|
1791
|
+
if (nextDate === null) {
|
|
1792
|
+
const error = new Error(
|
|
1793
|
+
`Cron expression "${job.cron}" has no future occurrences \u2014 job "${job.id}" will not be re-enqueued`
|
|
1794
|
+
);
|
|
1795
|
+
this.emitter.emit("worker:error", this.id, error);
|
|
1796
|
+
return;
|
|
1546
1797
|
}
|
|
1547
|
-
const runAt =
|
|
1798
|
+
const runAt = nextDate.toISOString();
|
|
1548
1799
|
await this.storage.enqueue({
|
|
1549
1800
|
id: generateJobId(),
|
|
1550
1801
|
queue: this.queueName,
|
|
@@ -1578,6 +1829,7 @@ var Worker = class {
|
|
|
1578
1829
|
runAt,
|
|
1579
1830
|
error: error.message,
|
|
1580
1831
|
attemptNumber,
|
|
1832
|
+
lockId: this.id,
|
|
1581
1833
|
...error.stack !== void 0 && { stack: error.stack }
|
|
1582
1834
|
});
|
|
1583
1835
|
const updated = await this.storage.getJob(job.id);
|
|
@@ -1589,6 +1841,7 @@ var Worker = class {
|
|
|
1589
1841
|
jobId: job.id,
|
|
1590
1842
|
error: error.message,
|
|
1591
1843
|
attemptNumber,
|
|
1844
|
+
lockId: this.id,
|
|
1592
1845
|
...error.stack !== void 0 && { stack: error.stack }
|
|
1593
1846
|
});
|
|
1594
1847
|
const dead = await this.storage.getJob(job.id);
|
|
@@ -1848,10 +2101,29 @@ var InMemoryStorageAdapter = class {
|
|
|
1848
2101
|
return job;
|
|
1849
2102
|
}
|
|
1850
2103
|
// -------------------------------------------------------------------------
|
|
2104
|
+
// Renew lock
|
|
2105
|
+
// -------------------------------------------------------------------------
|
|
2106
|
+
async renewLock(jobId, lockId, lockDuration) {
|
|
2107
|
+
const job = this.jobs.get(jobId);
|
|
2108
|
+
if (!job) return false;
|
|
2109
|
+
if (job.status !== "active" || job.lockId !== lockId) return false;
|
|
2110
|
+
const nowMs = Date.now();
|
|
2111
|
+
if (job.lockExpiresAt !== null && new Date(job.lockExpiresAt).getTime() <= nowMs) {
|
|
2112
|
+
return false;
|
|
2113
|
+
}
|
|
2114
|
+
job.lockExpiresAt = new Date(nowMs + lockDuration).toISOString();
|
|
2115
|
+
job.updatedAt = new Date(nowMs).toISOString();
|
|
2116
|
+
return true;
|
|
2117
|
+
}
|
|
2118
|
+
// -------------------------------------------------------------------------
|
|
1851
2119
|
// Complete
|
|
1852
2120
|
// -------------------------------------------------------------------------
|
|
1853
|
-
async complete(jobId) {
|
|
1854
|
-
const job = this.
|
|
2121
|
+
async complete(jobId, lockId) {
|
|
2122
|
+
const job = this.jobs.get(jobId);
|
|
2123
|
+
if (!job) return;
|
|
2124
|
+
if (lockId !== void 0 && (job.status !== "active" || job.lockId !== lockId)) {
|
|
2125
|
+
return;
|
|
2126
|
+
}
|
|
1855
2127
|
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
1856
2128
|
job.status = "completed";
|
|
1857
2129
|
job.lockId = null;
|
|
@@ -1863,7 +2135,11 @@ var InMemoryStorageAdapter = class {
|
|
|
1863
2135
|
// Requeue (retry)
|
|
1864
2136
|
// -------------------------------------------------------------------------
|
|
1865
2137
|
async requeue(input) {
|
|
1866
|
-
const job = this.
|
|
2138
|
+
const job = this.jobs.get(input.jobId);
|
|
2139
|
+
if (!job) return;
|
|
2140
|
+
if (input.lockId !== void 0 && (job.status !== "active" || job.lockId !== input.lockId)) {
|
|
2141
|
+
return;
|
|
2142
|
+
}
|
|
1867
2143
|
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
1868
2144
|
job.attempts.push({
|
|
1869
2145
|
attempt: input.attemptNumber,
|
|
@@ -1883,7 +2159,11 @@ var InMemoryStorageAdapter = class {
|
|
|
1883
2159
|
// Move to DLQ
|
|
1884
2160
|
// -------------------------------------------------------------------------
|
|
1885
2161
|
async moveToDlq(input) {
|
|
1886
|
-
const job = this.
|
|
2162
|
+
const job = this.jobs.get(input.jobId);
|
|
2163
|
+
if (!job) return;
|
|
2164
|
+
if (input.lockId !== void 0 && (job.status !== "active" || job.lockId !== input.lockId)) {
|
|
2165
|
+
return;
|
|
2166
|
+
}
|
|
1887
2167
|
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
1888
2168
|
job.attempts.push({
|
|
1889
2169
|
attempt: input.attemptNumber,
|
|
@@ -1902,9 +2182,12 @@ var InMemoryStorageAdapter = class {
|
|
|
1902
2182
|
// -------------------------------------------------------------------------
|
|
1903
2183
|
// Release lock
|
|
1904
2184
|
// -------------------------------------------------------------------------
|
|
1905
|
-
async releaseLock(jobId) {
|
|
2185
|
+
async releaseLock(jobId, lockId) {
|
|
1906
2186
|
const job = this.jobs.get(jobId);
|
|
1907
2187
|
if (!job) return;
|
|
2188
|
+
if (lockId !== void 0 && (job.status !== "active" || job.lockId !== lockId)) {
|
|
2189
|
+
return;
|
|
2190
|
+
}
|
|
1908
2191
|
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
1909
2192
|
job.lockId = null;
|
|
1910
2193
|
job.lockExpiresAt = now;
|
|
@@ -1912,6 +2195,12 @@ var InMemoryStorageAdapter = class {
|
|
|
1912
2195
|
}
|
|
1913
2196
|
// -------------------------------------------------------------------------
|
|
1914
2197
|
// Recover stalled jobs
|
|
2198
|
+
//
|
|
2199
|
+
// Fix (issue #6): snapshot lockId and lockExpiresAt before the eligibility
|
|
2200
|
+
// check, then re-validate both values at write time. In a single-process
|
|
2201
|
+
// scenario all operations are synchronous within one event-loop tick, so
|
|
2202
|
+
// the race is theoretical — but the guard makes the adapter consistent with
|
|
2203
|
+
// the Redis CAS semantics and protects against any future async paths.
|
|
1915
2204
|
// -------------------------------------------------------------------------
|
|
1916
2205
|
async recoverStalledJobs(queue, now) {
|
|
1917
2206
|
const nowMs = new Date(now).getTime();
|
|
@@ -1921,13 +2210,17 @@ var InMemoryStorageAdapter = class {
|
|
|
1921
2210
|
if (job.status !== "active") continue;
|
|
1922
2211
|
if (job.lockExpiresAt === null) continue;
|
|
1923
2212
|
const lockExpiry = new Date(job.lockExpiresAt).getTime();
|
|
1924
|
-
if (lockExpiry
|
|
1925
|
-
|
|
1926
|
-
|
|
1927
|
-
|
|
1928
|
-
|
|
1929
|
-
|
|
1930
|
-
|
|
2213
|
+
if (lockExpiry > nowMs) continue;
|
|
2214
|
+
const snapshotLockId = job.lockId;
|
|
2215
|
+
const snapshotLockExpiresAt = job.lockExpiresAt;
|
|
2216
|
+
if (job.status !== "active") continue;
|
|
2217
|
+
if (job.lockId !== snapshotLockId) continue;
|
|
2218
|
+
if (job.lockExpiresAt !== snapshotLockExpiresAt) continue;
|
|
2219
|
+
job.status = "waiting";
|
|
2220
|
+
job.lockId = null;
|
|
2221
|
+
job.lockExpiresAt = null;
|
|
2222
|
+
job.updatedAt = now;
|
|
2223
|
+
recovered.push(job.id);
|
|
1931
2224
|
}
|
|
1932
2225
|
return recovered;
|
|
1933
2226
|
}
|