power-queues 2.0.2 → 2.0.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (5) hide show
  1. package/dist/index.cjs +753 -684
  2. package/dist/index.d.cts +34 -280
  3. package/dist/index.d.ts +34 -280
  4. package/dist/index.js +752 -693
  5. package/package.json +6 -3
package/dist/index.cjs CHANGED
@@ -20,763 +20,832 @@ var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: tru
20
20
  // src/index.ts
21
21
  var index_exports = {};
22
22
  __export(index_exports, {
23
- PowerQueue: () => PowerQueue
23
+ PowerQueues: () => PowerQueues
24
24
  });
25
25
  module.exports = __toCommonJS(index_exports);
26
26
 
27
- // src/PowerQueue.ts
28
- var import_uuid = require("uuid");
27
+ // src/PowerQueues.ts
29
28
  var import_full_utils = require("full-utils");
30
- var import_power_redis = require("power-redis");
31
- var PowerQueue = class extends import_power_redis.PowerRedis {
32
- constructor() {
33
- super(...arguments);
34
- this.iterationTimeout = 1e3;
35
- this.portionLength = 1e3;
36
- this.expireStatusSec = 300;
37
- this.maxAttempts = 1;
38
- this.concurrency = 32;
39
- this.visibilityTimeoutSec = 60;
40
- this.retryBaseSec = 1;
41
- this.retryMaxSec = 3600;
42
- this.runners = /* @__PURE__ */ new Map();
43
- this.processingRaw = /* @__PURE__ */ new Map();
44
- this.heartbeatTimers = /* @__PURE__ */ new Map();
45
- }
46
- nowSec() {
47
- return Math.floor(Date.now() / 1e3);
48
- }
49
- readyKey(queueName) {
50
- return this.toKeyString(queueName);
51
- }
52
- processingKey(queueName) {
53
- return this.toKeyString(queueName, "processing");
54
- }
55
- processingVtKey(queueName) {
56
- return this.toKeyString(queueName, "processing", "vt");
57
- }
58
- delayedKey(queueName) {
59
- return this.toKeyString(queueName, "delayed");
60
- }
61
- toKeyString(...parts) {
62
- return super.toKeyString("queue", ...parts);
63
- }
64
- getReserveScriptLMOVE() {
65
- return `
66
- local source = KEYS[1]
67
- local processing = KEYS[2]
68
- local vtkey = KEYS[3]
69
- local limit = tonumber(ARGV[1])
70
- local deadline = tonumber(ARGV[2])
71
- local moved = {}
72
- for i = 1, limit do
73
- local v = redis.call('LMOVE', source, processing, 'LEFT', 'RIGHT')
74
- if not v then break end
75
- table.insert(moved, v)
76
- end
77
- if #moved > 0 then
78
- for i = 1, #moved do
79
- redis.call('ZADD', vtkey, deadline, moved[i])
29
+
30
+ // src/AddTasks.ts
31
+ var import_uuid = require("uuid");
32
+
33
+ // src/scripts.ts
34
+ var XAddBulk = `
35
+ local UNPACK = table and table.unpack or unpack
36
+
37
+ local stream = KEYS[1]
38
+ local maxlen = tonumber(ARGV[1])
39
+ local approxFlag = tonumber(ARGV[2]) == 1
40
+ local n = tonumber(ARGV[3])
41
+ local exactFlag = tonumber(ARGV[4]) == 1
42
+ local nomkstream = tonumber(ARGV[5]) == 1
43
+ local trimLimit = tonumber(ARGV[6])
44
+ local minidWindowMs = tonumber(ARGV[7]) or 0
45
+ local minidExact = tonumber(ARGV[8]) == 1
46
+ local idx = 9
47
+ local out = {}
48
+
49
+ local common_opts = {}
50
+ local co_len = 0
51
+
52
+ if nomkstream then
53
+ co_len = co_len + 1; common_opts[co_len] = 'NOMKSTREAM'
54
+ end
55
+
56
+ if minidWindowMs > 0 then
57
+ local tm = redis.call('TIME')
58
+ local now_ms = (tonumber(tm[1]) * 1000) + math.floor(tonumber(tm[2]) / 1000)
59
+ local cutoff_ms = now_ms - minidWindowMs
60
+ if cutoff_ms < 0 then cutoff_ms = 0 end
61
+ local cutoff_id = tostring(cutoff_ms) .. '-0'
62
+
63
+ co_len = co_len + 1; common_opts[co_len] = 'MINID'
64
+ co_len = co_len + 1; common_opts[co_len] = (minidExact and '=' or '~')
65
+ co_len = co_len + 1; common_opts[co_len] = cutoff_id
66
+ if trimLimit and trimLimit > 0 then
67
+ co_len = co_len + 1; common_opts[co_len] = 'LIMIT'
68
+ co_len = co_len + 1; common_opts[co_len] = trimLimit
69
+ end
70
+ elseif maxlen and maxlen > 0 then
71
+ co_len = co_len + 1; common_opts[co_len] = 'MAXLEN'
72
+ if exactFlag then
73
+ co_len = co_len + 1; common_opts[co_len] = '='
74
+ elseif approxFlag then
75
+ co_len = co_len + 1; common_opts[co_len] = '~'
76
+ end
77
+ co_len = co_len + 1; common_opts[co_len] = maxlen
78
+ if trimLimit and trimLimit > 0 then
79
+ co_len = co_len + 1; common_opts[co_len] = 'LIMIT'
80
+ co_len = co_len + 1; common_opts[co_len] = trimLimit
81
+ end
82
+ end
83
+
84
+ for e = 1, n do
85
+ local id = ARGV[idx]; idx = idx + 1
86
+ local num_pairs = tonumber(ARGV[idx]); idx = idx + 1
87
+
88
+ local a = {}
89
+ local a_len = 0
90
+
91
+ for i = 1, co_len do a_len = a_len + 1; a[a_len] = common_opts[i] end
92
+
93
+ a_len = a_len + 1; a[a_len] = id
94
+
95
+ for j = 1, (num_pairs * 2) do
96
+ a_len = a_len + 1; a[a_len] = ARGV[idx]; idx = idx + 1
97
+ end
98
+
99
+ local addedId = redis.call('XADD', stream, UNPACK(a))
100
+ out[#out+1] = addedId or ''
101
+ end
102
+
103
+ return out
104
+ `;
105
+ var Approve = `
106
+ local stream = KEYS[1]
107
+ local group = ARGV[1]
108
+ local delFlag = tonumber(ARGV[2]) == 1
109
+
110
+ local acked = 0
111
+ local nids = #ARGV - 2
112
+ if nids > 0 then
113
+ acked = tonumber(redis.call('XACK', stream, group, unpack(ARGV, 3))) or 0
114
+ if delFlag and nids > 0 then
115
+ local ok, deln = pcall(redis.call, 'XDEL', stream, unpack(ARGV, 3))
116
+ if not ok then
117
+ deln = 0
118
+ for i = 3, #ARGV do
119
+ deln = deln + (tonumber(redis.call('XDEL', stream, ARGV[i])) or 0)
80
120
  end
81
121
  end
82
- return moved
83
- `;
84
- }
85
- getReserveScriptRPOPLPUSH() {
86
- return `
87
- local source = KEYS[1]
88
- local processing = KEYS[2]
89
- local vtkey = KEYS[3]
90
- local limit = tonumber(ARGV[1])
91
- local deadline = tonumber(ARGV[2])
92
- local moved = {}
93
- for i = 1, limit do
94
- local v = redis.call('RPOPLPUSH', source, processing)
95
- if not v then break end
96
- table.insert(moved, v)
97
- end
98
- if #moved > 0 then
99
- for i = 1, #moved do
100
- redis.call('ZADD', vtkey, deadline, moved[i])
101
- end
122
+ end
123
+ end
124
+ return acked
125
+ `;
126
+ var IdempotencyAllow = `
127
+ local doneKey = KEYS[1]
128
+ local lockKey = KEYS[2]
129
+ local startKey = KEYS[3]
130
+
131
+ if redis.call('EXISTS', doneKey) == 1 then
132
+ return 1
133
+ end
134
+
135
+ local ttl = tonumber(ARGV[1]) or 0
136
+ if ttl <= 0 then return 0 end
137
+
138
+ local ok = redis.call('SET', lockKey, ARGV[2], 'NX', 'PX', ttl)
139
+ if ok then
140
+ if startKey and startKey ~= '' then
141
+ redis.call('SET', startKey, 1, 'PX', ttl)
142
+ end
143
+ return 2
144
+ else
145
+ return 0
146
+ end
147
+ `;
148
+ var IdempotencyStart = `
149
+ local lockKey = KEYS[1]
150
+ local startKey = KEYS[2]
151
+ if redis.call('GET', lockKey) == ARGV[1] then
152
+ local ttl = tonumber(ARGV[2]) or 0
153
+ if ttl > 0 then
154
+ redis.call('SET', startKey, 1, 'PX', ttl)
155
+ redis.call('PEXPIRE', lockKey, ttl)
156
+ else
157
+ redis.call('SET', startKey, 1)
158
+ end
159
+ return 1
160
+ end
161
+ return 0
162
+ `;
163
+ var IdempotencyDone = `
164
+ local doneKey = KEYS[1]
165
+ local lockKey = KEYS[2]
166
+ local startKey = KEYS[3]
167
+ redis.call('SET', doneKey, 1)
168
+ local ttlSec = tonumber(ARGV[1]) or 0
169
+ if ttlSec > 0 then redis.call('EXPIRE', doneKey, ttlSec) end
170
+ if redis.call('GET', lockKey) == ARGV[2] then
171
+ redis.call('DEL', lockKey)
172
+ if startKey then redis.call('DEL', startKey) end
173
+ end
174
+ return 1
175
+ `;
176
+ var IdempotencyFree = `
177
+ local lockKey = KEYS[1]
178
+ local startKey = KEYS[2]
179
+ if redis.call('GET', lockKey) == ARGV[1] then
180
+ redis.call('DEL', lockKey)
181
+ if startKey then redis.call('DEL', startKey) end
182
+ return 1
183
+ end
184
+ return 0
185
+ `;
186
+ var SelectStuck = `
187
+ local stream = KEYS[1]
188
+ local group = ARGV[1]
189
+ local consumer = ARGV[2]
190
+ local pendingIdleMs = tonumber(ARGV[3])
191
+ local count = tonumber(ARGV[4]) or 0
192
+ if count < 1 then count = 1 end
193
+
194
+ local timeBudgetMs = tonumber(ARGV[5]) or 15
195
+ local t0 = redis.call('TIME')
196
+ local start_ms = (tonumber(t0[1]) * 1000) + math.floor(tonumber(t0[2]) / 1000)
197
+
198
+ local results = {}
199
+ local collected = 0
200
+ local start_id = '0-0'
201
+ local iters = 0
202
+ local max_iters = math.max(16, math.ceil(count / 100))
203
+
204
+ local function time_exceeded()
205
+ local t1 = redis.call('TIME')
206
+ local now_ms = (tonumber(t1[1]) * 1000) + math.floor(tonumber(t1[2]) / 1000)
207
+ return (now_ms - start_ms) >= timeBudgetMs
208
+ end
209
+
210
+ while (collected < count) and (iters < max_iters) do
211
+ local to_claim = count - collected
212
+ if to_claim < 1 then break end
213
+
214
+ local claim = redis.call('XAUTOCLAIM', stream, group, consumer, pendingIdleMs, start_id, 'COUNT', to_claim)
215
+ iters = iters + 1
216
+
217
+ local bucket = nil
218
+ if claim then
219
+ bucket = claim[2]
220
+ end
221
+ if bucket and #bucket > 0 then
222
+ for i = 1, #bucket do
223
+ results[#results+1] = bucket[i]
102
224
  end
103
- return moved
104
- `;
105
- }
106
- getRequeueScript() {
107
- return `
108
- -- KEYS: 1=processing, 2=processingVt, 3=ready
109
- -- ARGV: 1=now, 2=limit
110
- local processing = KEYS[1]
111
- local vt = KEYS[2]
112
- local ready = KEYS[3]
113
- local now = tonumber(ARGV[1])
114
- local limit = tonumber(ARGV[2])
115
-
116
- local members = redis.call('ZRANGEBYSCORE', vt, 0, now, 'LIMIT', 0, limit)
117
- for i=1,#members do
118
- local m = members[i]
119
- redis.call('ZREM', vt, m)
120
- redis.call('LREM', processing, 1, m)
121
- redis.call('RPUSH', ready, m)
225
+ collected = #results
226
+ end
227
+
228
+ local next_id = claim and claim[1] or start_id
229
+ if next_id == start_id then
230
+ local s, seq = string.match(start_id, '^(%d+)%-(%d+)$')
231
+ if s and seq then
232
+ start_id = s .. '-' .. tostring(tonumber(seq) + 1)
233
+ else
234
+ start_id = '0-1'
122
235
  end
123
- return #members
124
- `;
125
- }
126
- getPromoteScript() {
127
- return `
128
- -- KEYS: 1=delayed, 2=ready
129
- -- ARGV: 1=now, 2=limit
130
- local delayed = KEYS[1]
131
- local ready = KEYS[2]
132
- local now = tonumber(ARGV[1])
133
- local limit = tonumber(ARGV[2])
134
-
135
- local due = redis.call('ZRANGEBYSCORE', delayed, 0, now, 'LIMIT', 0, limit)
136
- for i=1,#due do
137
- local m = due[i]
138
- redis.call('ZREM', delayed, m)
139
- redis.call('RPUSH', ready, m)
236
+ else
237
+ start_id = next_id
238
+ end
239
+
240
+ if time_exceeded() then
241
+ break
242
+ end
243
+ end
244
+
245
+ local left = count - collected
246
+ if left > 0 then
247
+ local xr = redis.call('XREADGROUP', 'GROUP', group, consumer, 'COUNT', left, 'STREAMS', stream, '>')
248
+ if xr and xr[1] and xr[1][2] then
249
+ local entries = xr[1][2]
250
+ for i = 1, #entries do
251
+ results[#results+1] = entries[i]
140
252
  end
141
- return #due
142
- `;
143
- }
144
- async ensureReserveScript(force = false) {
145
- if (!force && (this.reserveSha || this.reserveShaRpoplpush || !(0, import_full_utils.isFunc)(this.redis?.script))) {
146
- return;
147
- }
148
- this.reserveSha = void 0;
149
- this.reserveShaRpoplpush = void 0;
150
- try {
151
- this.reserveSha = await this.redis?.script("LOAD", this.getReserveScriptLMOVE());
152
- } catch {
153
- this.reserveShaRpoplpush = await this.redis?.script("LOAD", this.getReserveScriptRPOPLPUSH());
154
- }
155
- }
156
- async ensureRequeueScript(force = false) {
157
- if (!force && this.requeueSha) {
158
- return;
159
- }
160
- const scriptFn = this.redis?.script;
161
- if (!scriptFn) {
162
- return;
163
- }
164
- this.requeueSha = await scriptFn("LOAD", this.getRequeueScript());
165
- }
166
- async ensurePromoteScript(force = false) {
167
- if (!force && this.promoteSha) {
168
- return;
169
- }
170
- const scriptFn = this.redis?.script;
171
- if (!scriptFn) {
172
- return;
173
- }
174
- this.promoteSha = await scriptFn("LOAD", this.getPromoteScript());
175
- }
176
- async moveOneToProcessing(source, processing) {
177
- const cli = this.redis;
178
- try {
179
- if ((0, import_full_utils.isFunc)(cli.lmove)) {
180
- const v = await cli.lmove(source, processing, "LEFT", "RIGHT");
181
- return (0, import_full_utils.isStr)(v) ? v : null;
253
+ end
254
+ end
255
+
256
+ return results
257
+ `;
258
+
259
+ // src/mix.ts
260
+ function mix(Base2, ...mixins) {
261
+ return mixins.reduce((acc, mixin) => mixin(acc), Base2);
262
+ }
263
+
264
+ // src/Script.ts
265
+ function Script(Base2) {
266
+ return class extends Base2 {
267
+ constructor() {
268
+ super(...arguments);
269
+ this.strictCheckingConnection = ["true", "on", "yes", "y", "1"].includes(String(process.env.REDIS_STRICT_CHECK_CONNECTION ?? "").trim().toLowerCase());
270
+ this.scripts = {};
271
+ }
272
+ async runScript(name, keys, args, defaultCode) {
273
+ if (!this.scripts[name]) {
274
+ if (typeof defaultCode !== "string" || !(defaultCode.length > 0)) {
275
+ throw new Error(`Undefined script "${name}". Save it before executing.`);
276
+ }
277
+ this.saveScript(name, defaultCode);
182
278
  }
183
- } catch {
184
- }
185
- try {
186
- if ((0, import_full_utils.isFunc)(cli.rpoplpush)) {
187
- const v = await cli.rpoplpush(source, processing);
188
- return (0, import_full_utils.isStr)(v) ? v : null;
279
+ if (!this.scripts[name].codeReady) {
280
+ this.scripts[name].codeReady = await this.loadScript(this.scripts[name].codeBody);
189
281
  }
190
- } catch {
191
- }
192
- return null;
193
- }
194
- async evalshaWithReload(shaGetter, ensure, numKeys, keysAndArgs) {
195
- await ensure(false);
196
- const sha = shaGetter();
197
- const evalshaFn = this.redis?.evalsha;
198
- if (!sha || !evalshaFn) {
199
- throw new Error("EVALSHA not available or SHA missing");
200
- }
201
- try {
202
- return await evalshaFn(sha, numKeys, ...keysAndArgs.map(String));
203
- } catch (err) {
204
- const msg = err?.message;
205
- if (typeof msg === "string" && msg.includes("NOSCRIPT")) {
206
- await ensure(true);
207
- const sha2 = shaGetter();
208
- if (!sha2) {
209
- throw new Error("EVALSHA NOSCRIPT and reload failed (no SHA)");
282
+ try {
283
+ return await this.redis.evalsha(this.scripts[name].codeReady, keys.length, ...keys, ...args);
284
+ } catch (err) {
285
+ if (String(err?.message || "").includes("NOSCRIPT")) {
286
+ this.scripts[name].codeReady = await this.loadScript(this.scripts[name].codeBody);
287
+ return await this.redis.evalsha(this.scripts[name].codeReady, keys.length, ...keys, ...args);
210
288
  }
211
- return await evalshaFn(sha2, numKeys, ...keysAndArgs.map(String));
289
+ throw err;
212
290
  }
213
- throw err;
214
291
  }
215
- }
216
- async zaddCompatXXCH(key, score, member) {
217
- const zadd = this.redis?.zadd;
218
- try {
219
- if (zadd) {
220
- await zadd.call(this.redis, key, "XX", "CH", score, member);
221
- return;
292
+ async loadScripts(full = false) {
293
+ const scripts = full ? [
294
+ ["XAddBulk", XAddBulk],
295
+ ["Approve", Approve],
296
+ ["IdempotencyAllow", IdempotencyAllow],
297
+ ["IdempotencyStart", IdempotencyStart],
298
+ ["IdempotencyDone", IdempotencyDone],
299
+ ["IdempotencyFree", IdempotencyFree],
300
+ ["SelectStuck", SelectStuck]
301
+ ] : [
302
+ ["XAddBulk", XAddBulk]
303
+ ];
304
+ for (const [name, code] of scripts) {
305
+ await this.loadScript(this.saveScript(name, code));
222
306
  }
223
- } catch {
224
307
  }
225
- try {
226
- if (zadd) {
227
- await zadd.call(this.redis, key, "CH", "XX", score, member);
228
- return;
308
+ async loadScript(code) {
309
+ for (let i = 0; i < 3; i++) {
310
+ try {
311
+ return await this.redis.script("LOAD", code);
312
+ } catch (e) {
313
+ if (i === 2) {
314
+ throw e;
315
+ }
316
+ await new Promise((r) => setTimeout(r, 10 + Math.floor(Math.random() * 40)));
317
+ }
229
318
  }
230
- } catch {
319
+ throw new Error("Load lua script failed.");
231
320
  }
232
- try {
233
- await this.redis.zadd(key, score, member);
234
- } catch {
321
+ saveScript(name, codeBody) {
322
+ if (typeof codeBody !== "string" || !(codeBody.length > 0)) {
323
+ throw new Error("Script body is empty.");
324
+ }
325
+ this.scripts[name] = { codeBody };
326
+ return codeBody;
235
327
  }
236
- }
237
- startHeartbeat(task) {
238
- const raw = this.processingRaw.get(task.id);
239
- if (!raw) {
240
- return;
241
- }
242
- const vtKey = this.processingVtKey(task.queueName);
243
- const periodMs = Math.max(1e3, Math.floor(this.visibilityTimeoutSec * 1e3 * 0.4));
244
- const t = setInterval(() => {
245
- this.extendVisibility(vtKey, raw, this.visibilityTimeoutSec).catch(() => {
328
+ };
329
+ }
330
+
331
+ // src/AddTasks.ts
332
+ function AddTasks(Base2) {
333
+ return class extends mix(Base2, Script) {
334
+ constructor() {
335
+ super(...arguments);
336
+ this.addingBatchTasksCount = 800;
337
+ this.addingBatchKeysLimit = 1e4;
338
+ this.idemOn = true;
339
+ this.idemKey = "";
340
+ }
341
+ async addTasks(queueName, data, opts) {
342
+ if (!Array.isArray(data) || !(data.length > 0)) {
343
+ throw new Error("Tasks is not filled.");
344
+ }
345
+ if (typeof queueName !== "string" || !(queueName.length > 0)) {
346
+ throw new Error("Queue name is required.");
347
+ }
348
+ const batches = this.buildBatches(data);
349
+ const result = new Array(data.length);
350
+ const promises = [];
351
+ let cursor = 0;
352
+ for (const batch of batches) {
353
+ const start = cursor;
354
+ const end = start + batch.length;
355
+ cursor = end;
356
+ promises.push(async () => {
357
+ const partIds = await this.xaddBatch(queueName, ...this.payloadBatch(batch, opts));
358
+ for (let k = 0; k < partIds.length; k++) {
359
+ result[start + k] = partIds[k];
360
+ }
361
+ });
362
+ }
363
+ const runners = Array.from({ length: promises.length }, async () => {
364
+ while (promises.length) {
365
+ const promise = promises.shift();
366
+ if (promise) {
367
+ await promise();
368
+ }
369
+ }
246
370
  });
247
- }, periodMs);
248
- t.unref?.();
249
- this.heartbeatTimers.set(task.id, t);
250
- }
251
- stopHeartbeat(task) {
252
- const t = this.heartbeatTimers.get(task.id);
253
- if (t) {
254
- clearInterval(t);
255
- }
256
- this.heartbeatTimers.delete(task.id);
257
- }
258
- async reserveMany(source, processing, processingVt, limit = 100, visibilitySec = 60) {
259
- if (!this.checkConnection()) {
260
- throw new Error("Redis connection error.");
261
- }
262
- if (!(0, import_full_utils.isStrFilled)(source) || !(0, import_full_utils.isStrFilled)(processing) || !(0, import_full_utils.isStrFilled)(processingVt)) {
263
- throw new Error("Key format error.");
264
- }
265
- if (!(0, import_full_utils.isNumP)(limit) || !(0, import_full_utils.isNumP)(visibilitySec)) {
266
- throw new Error("Limit/visibility format error.");
267
- }
268
- await this.ensureReserveScript();
269
- const deadline = this.nowSec() + visibilitySec;
270
- const tryEval = async () => {
271
- if ((0, import_full_utils.isFunc)(this.redis?.evalsha)) {
272
- if (this.reserveSha) {
273
- return await this.redis?.evalsha(this.reserveSha, 3, source, processing, processingVt, String(limit), String(deadline));
371
+ await Promise.all(runners);
372
+ return result;
373
+ }
374
+ async xaddBatch(queueName, ...batches) {
375
+ return await this.runScript("XAddBulk", [queueName], batches, XAddBulk);
376
+ }
377
+ payloadBatch(data, opts) {
378
+ const maxlen = Math.max(0, Math.floor(opts?.maxlen ?? 0));
379
+ const approx = opts?.exact ? 0 : opts?.approx !== false ? 1 : 0;
380
+ const exact = opts?.exact ? 1 : 0;
381
+ const nomkstream = opts?.nomkstream ? 1 : 0;
382
+ const trimLimit = Math.max(0, Math.floor(opts?.trimLimit ?? 0));
383
+ const minidWindowMs = Math.max(0, Math.floor(opts?.minidWindowMs ?? 0));
384
+ const minidExact = opts?.minidExact ? 1 : 0;
385
+ const argv = [
386
+ String(maxlen),
387
+ String(approx),
388
+ String(data.length),
389
+ String(exact),
390
+ String(nomkstream),
391
+ String(trimLimit),
392
+ String(minidWindowMs),
393
+ String(minidExact)
394
+ ];
395
+ for (const item of data) {
396
+ const entry = item;
397
+ const id = entry.id ?? "*";
398
+ let flat;
399
+ if ("flat" in entry && Array.isArray(entry.flat) && entry.flat.length > 0) {
400
+ flat = entry.flat;
401
+ if (flat.length % 2 !== 0) {
402
+ throw new Error('Property "flat" must contain an even number of realKeysLength (field/value pairs).');
403
+ }
404
+ } else if ("payload" in entry && typeof entry.payload === "object" && Object.keys(entry.payload || {}).length > 0) {
405
+ flat = [];
406
+ for (const [k, v] of Object.entries(entry.payload)) {
407
+ flat.push(k, v);
408
+ }
409
+ } else {
410
+ throw new Error('Task must have "payload" or "flat".');
274
411
  }
275
- if (this.reserveShaRpoplpush) {
276
- return await this.redis?.evalsha(this.reserveShaRpoplpush, 3, source, processing, processingVt, String(limit), String(deadline));
412
+ const pairs = flat.length / 2;
413
+ if (pairs <= 0) {
414
+ throw new Error('Task must have "payload" or "flat".');
415
+ }
416
+ argv.push(String(id));
417
+ argv.push(String(pairs));
418
+ for (const token of flat) {
419
+ argv.push(!token ? "" : typeof token === "string" && token.length > 0 ? token : String(token));
277
420
  }
278
421
  }
279
- return null;
280
- };
281
- try {
282
- const res = await tryEval();
283
- if ((0, import_full_utils.isArr)(res)) {
284
- return Array.from(res).map(String);
285
- }
286
- } catch (err) {
287
- if ((0, import_full_utils.isStr)(err?.message) && String(err?.message ?? "").includes("NOSCRIPT")) {
288
- await this.ensureReserveScript(true);
289
- try {
290
- const res2 = await tryEval();
291
- if ((0, import_full_utils.isArr)(res2)) {
292
- return Array.from(res2).map(String);
422
+ return argv;
423
+ }
424
+ buildBatches(tasks) {
425
+ const job = (0, import_uuid.v4)();
426
+ const batches = [];
427
+ let batch = [], realKeysLength = 0;
428
+ for (let task of tasks) {
429
+ let entry = task;
430
+ if (this.idemOn) {
431
+ const createdAt = entry?.createdAt || Date.now();
432
+ let idemKey = entry?.idemKey || (0, import_uuid.v4)();
433
+ if (typeof entry.payload === "object") {
434
+ if (this.idemKey && typeof entry.payload[this.idemKey] === "string" && entry.payload[this.idemKey].length > 0) {
435
+ idemKey = entry.payload[this.idemKey];
436
+ }
437
+ entry = {
438
+ ...entry,
439
+ payload: {
440
+ payload: JSON.stringify(entry.payload),
441
+ createdAt,
442
+ job,
443
+ idemKey
444
+ }
445
+ };
446
+ } else if (Array.isArray(entry.flat)) {
447
+ entry.flat.push("createdAt");
448
+ entry.flat.push(String(createdAt));
449
+ entry.flat.push("job");
450
+ entry.flat.push(job);
451
+ entry.flat.push("idemKey");
452
+ entry.flat.push(idemKey);
293
453
  }
294
- } catch {
295
454
  }
455
+ const reqKeysLength = this.keysLength(entry);
456
+ if (batch.length && (batch.length >= this.addingBatchTasksCount || realKeysLength + reqKeysLength > this.addingBatchKeysLimit)) {
457
+ batches.push(batch);
458
+ batch = [];
459
+ realKeysLength = 0;
460
+ }
461
+ batch.push(entry);
462
+ realKeysLength += reqKeysLength;
296
463
  }
297
- }
298
- const moved = [];
299
- for (let i = 0; i < limit; i++) {
300
- const v = await this.moveOneToProcessing(source, processing);
301
- if (!v) {
302
- break;
464
+ if (batch.length) {
465
+ batches.push(batch);
303
466
  }
304
- moved.push(v);
467
+ return batches;
305
468
  }
306
- if (moved.length) {
307
- const tx = this.redis?.multi();
308
- for (const v of moved) {
309
- tx.zadd(processingVt, deadline, v);
310
- }
311
- await tx.exec();
469
+ keysLength(task) {
470
+ return 2 + ("flat" in task && Array.isArray(task.flat) && task.flat.length ? task.flat.length : Object.keys(task).length * 2);
312
471
  }
313
- return moved;
314
- }
315
- async ackProcessing(processing, processingVt, raw) {
316
- if (!this.checkConnection()) {
317
- throw new Error("Redis connection error.");
318
- }
319
- const tx = this.redis?.multi();
320
- tx.lrem(processing, 1, raw);
321
- tx.zrem(processingVt, raw);
322
- await tx.exec();
323
- return;
324
- }
325
- async requeueExpired(processing, processingVt, ready, nowTs, chunk = 1e3) {
326
- if (!this.checkConnection()) {
327
- throw new Error("Redis connection error.");
328
- }
329
- const now = (0, import_full_utils.isNumP)(nowTs) ? nowTs : this.nowSec();
330
- try {
331
- const moved = await this.evalshaWithReload(
332
- () => this.requeueSha,
333
- (force) => this.ensureRequeueScript(!!force),
334
- 3,
335
- [processing, processingVt, ready, String(now), String(chunk)]
336
- );
337
- return (0, import_full_utils.isNumP)(moved) ? moved : 0;
338
- } catch {
339
- const expired = await this.redis?.zrangebyscore(processingVt, 0, now, "LIMIT", 0, chunk);
340
- if (!(0, import_full_utils.isArrFilled)(expired)) {
341
- return 0;
342
- }
343
- const tx = this.redis?.multi();
344
- for (const raw of expired) {
345
- tx.lrem(processing, 1, raw);
346
- tx.zrem(processingVt, raw);
347
- tx.rpush(ready, raw);
472
+ };
473
+ }
474
+
475
+ // src/SelectTasks.ts
476
+ function SelectTasks(Base2) {
477
+ return class extends mix(Base2, Script) {
478
+ constructor() {
479
+ super(...arguments);
480
+ this.consumerHost = "host";
481
+ this.stream = "stream";
482
+ this.group = "group";
483
+ this.workerBatchTasksCount = 200;
484
+ this.recoveryStuckTasksTimeoutMs = 6e4;
485
+ this.workerLoopIntervalMs = 5e3;
486
+ this.workerSelectionTimeoutMs = 80;
487
+ }
488
+ async createGroup(from = "$") {
489
+ try {
490
+ await this.redis.xgroup("CREATE", this.stream, this.group, from, "MKSTREAM");
491
+ } catch (err) {
492
+ const msg = String(err?.message || "");
493
+ if (!msg.includes("BUSYGROUP")) {
494
+ throw err;
495
+ }
348
496
  }
349
- await tx.exec();
350
- return expired.length;
351
497
  }
352
- }
353
- async promoteDelayed(delayed, ready, nowTs, chunk = 1e3) {
354
- if (!this.checkConnection()) {
355
- throw new Error("Redis connection error.");
356
- }
357
- const now = (0, import_full_utils.isNumP)(nowTs) ? nowTs : this.nowSec();
358
- try {
359
- const promoted = await this.evalshaWithReload(
360
- () => this.promoteSha,
361
- (force) => this.ensurePromoteScript(!!force),
362
- 2,
363
- [delayed, ready, String(now), String(chunk)]
364
- );
365
- return (0, import_full_utils.isNumP)(promoted) ? promoted : 0;
366
- } catch {
367
- const due = await this.redis?.zrangebyscore(delayed, 0, now, "LIMIT", 0, chunk);
368
- if (!(0, import_full_utils.isArrFilled)(due)) {
369
- return 0;
498
+ async select() {
499
+ let entries = await this.selectStuck();
500
+ if (!entries?.length) {
501
+ entries = await this.selectFresh();
370
502
  }
371
- const tx = this.redis?.multi();
372
- for (const raw of due) {
373
- tx.zrem(delayed, raw);
374
- tx.rpush(ready, raw);
503
+ return this.normalizeEntries(entries);
504
+ }
505
+ async selectStuck() {
506
+ try {
507
+ const res = await this.runScript("SelectStuck", [this.stream], [this.group, this.consumer(), String(this.recoveryStuckTasksTimeoutMs), String(this.workerBatchTasksCount), String(this.workerSelectionTimeoutMs)], SelectStuck);
508
+ return Array.isArray(res) ? res : [];
509
+ } catch (err) {
510
+ if (String(err?.message || "").includes("NOGROUP")) {
511
+ await this.createGroup();
512
+ }
375
513
  }
376
- await tx.exec();
377
- return due.length;
514
+ return [];
378
515
  }
379
- }
380
- async enqueue(ready, delayed, payload, delaySec) {
381
- if (!this.checkConnection()) {
382
- throw new Error("Redis connection error.");
516
+ async selectFresh() {
517
+ let entries = [];
518
+ try {
519
+ const res = await this.redis.xreadgroup(
520
+ "GROUP",
521
+ this.group,
522
+ this.consumer(),
523
+ "BLOCK",
524
+ Math.max(2, this.workerLoopIntervalMs | 0),
525
+ "COUNT",
526
+ this.workerBatchTasksCount,
527
+ "STREAMS",
528
+ this.stream,
529
+ ">"
530
+ );
531
+ if (!res?.[0]?.[1]?.length) {
532
+ return [];
533
+ }
534
+ entries = res?.[0]?.[1] ?? [];
535
+ if (!entries?.length) {
536
+ return [];
537
+ }
538
+ } catch (err) {
539
+ if (String(err?.message || "").includes("NOGROUP")) {
540
+ await this.createGroup();
541
+ }
542
+ }
543
+ return entries;
383
544
  }
384
- const raw = this.toPayload(payload);
385
- if ((0, import_full_utils.isNumP)(delaySec) && delaySec > 0) {
386
- const score = this.nowSec() + delaySec;
387
- return await this.redis?.zadd(delayed, score, raw);
545
+ normalizeEntries(raw) {
546
+ if (!Array.isArray(raw)) {
547
+ return [];
548
+ }
549
+ return Array.from(raw || []).map((e) => {
550
+ const id = Buffer.isBuffer(e?.[0]) ? e[0].toString() : e?.[0];
551
+ const kvRaw = e?.[1] ?? [];
552
+ const kv = Array.isArray(kvRaw) ? kvRaw.map((x) => Buffer.isBuffer(x) ? x.toString() : x) : [];
553
+ return [id, kv, 0, "", ""];
554
+ }).filter(([id, kv]) => typeof id === "string" && id.length > 0 && Array.isArray(kv) && (kv.length & 1) === 0).map(([id, kv]) => {
555
+ const values = this.values(kv);
556
+ const { idemKey = "", createdAt, job, ...data } = this.payload(values);
557
+ return [id, data, createdAt, job, idemKey];
558
+ });
388
559
  }
389
- return await this.redis?.rpush(ready, raw);
390
- }
391
- async extendVisibility(processingVt, raw, visibilitySec) {
392
- const deadline = this.nowSec() + Math.max(1, visibilitySec);
393
- await this.zaddCompatXXCH(processingVt, deadline, raw);
394
- }
395
- run(queueName) {
396
- if (!(0, import_full_utils.isStrFilled)(queueName)) {
397
- throw new Error("Queue name is not valid.");
398
- }
399
- const r = this.runners.get(queueName) ?? { running: false };
400
- if (r.running) {
401
- throw new Error(`Queue "${queueName}" already started.`);
402
- }
403
- r.running = true;
404
- this.runners.set(queueName, r);
405
- this.loop(queueName, r).catch(() => {
406
- r.running = false;
407
- });
408
- }
409
- stop(queueName) {
410
- const r = this.runners.get(queueName);
411
- if (r) {
412
- r.running = false;
413
- this.runners.delete(queueName);
560
+ consumer() {
561
+ return `${String(this.consumerHost || "host")}:${process.pid}`;
414
562
  }
415
- }
416
- buildTask(data) {
417
- if (!(0, import_full_utils.isObjFilled)(data)) {
418
- throw new Error("Data property is not valid.");
419
- }
420
- if (!(0, import_full_utils.isStrFilled)(data.queueName)) {
421
- throw new Error("Queue name is not valid.");
422
- }
423
- return {
424
- queueName: data.queueName,
425
- iterationId: (0, import_full_utils.isStrFilled)(data.iterationId) ? data.iterationId : (0, import_uuid.v4)(),
426
- iterationLength: Number(data.iterationLength || 0),
427
- id: (0, import_full_utils.isStrFilled)(data.id) ? data.id : (0, import_uuid.v4)(),
428
- maxAttempts: (0, import_full_utils.isNumPZ)(data.maxAttempts) ? data.maxAttempts : this.maxAttempts,
429
- currentAttempt: (0, import_full_utils.isNumPZ)(data.currentAttempt) ? data.currentAttempt : 0,
430
- chain: (0, import_full_utils.isObjFilled)(data.chain) && (0, import_full_utils.isArrFilled)(data.chain.queues) && (0, import_full_utils.isNumPZ)(data.chain.index) ? data.chain : {
431
- queues: [],
432
- index: 0
433
- },
434
- payload: (0, import_full_utils.isObjFilled)(data.payload) ? data.payload : {},
435
- progress: {
436
- createdAt: Date.now(),
437
- successAt: 0,
438
- errorAt: 0,
439
- failAt: 0,
440
- fatalAt: 0,
441
- retries: [],
442
- chain: [],
443
- ...(0, import_full_utils.isObjFilled)(data.progress) ? data.progress : {}
444
- },
445
- result: (0, import_full_utils.isObjFilled)(data.result) ? data.result : {}
446
- };
447
- }
448
- async addTask(data, delaySec) {
449
- const ready = this.readyKey(String(data.queueName));
450
- const delayed = this.delayedKey(String(data.queueName));
451
- return await this.enqueue(ready, delayed, this.buildTask(data), (0, import_full_utils.isNumP)(delaySec) ? delaySec : 0);
452
- }
453
- async addTasks(data) {
454
- if (!this.checkConnection()) {
455
- throw new Error("Redis connection error.");
456
- }
457
- if (!(0, import_full_utils.isObjFilled)(data) || !(0, import_full_utils.isStrFilled)(data.queueName)) {
458
- throw new Error("Queue name is not valid.");
459
- }
460
- if (!(0, import_full_utils.isArrFilled)(data.payloads)) {
461
- return 0;
462
- }
463
- const queueName = String(data.queueName);
464
- const ready = this.readyKey(queueName);
465
- const delayed = this.delayedKey(queueName);
466
- const now = this.nowSec();
467
- const uniformDelay = (0, import_full_utils.isNumP)(data.delaySec) ? Math.max(0, Number(data.delaySec)) : void 0;
468
- const perItemDelays = (0, import_full_utils.isArr)(data.delaySec) ? data.delaySec.map((v) => Math.max(0, Number(v || 0))) : void 0;
469
- const batchSize = Math.max(1, Math.min(this.portionLength, 1e3));
470
- let idx = 0, total = 0;
471
- while (idx < data.payloads.length) {
472
- const end = Math.min(idx + batchSize, data.payloads.length);
473
- const tx = this.redis?.multi();
474
- for (let i = idx; i < end; i++) {
475
- const item = data.payloads[i];
476
- let partial;
477
- if ((0, import_full_utils.isObjFilled)(item) && Object.prototype.hasOwnProperty.call(item, "payload")) {
478
- partial = { ...item, queueName };
563
+ };
564
+ }
565
+
566
+ // src/ExecuteTasks.ts
567
+ function ExecuteTasks(Base2) {
568
+ return class extends mix(Base2, Script) {
569
+ constructor() {
570
+ super(...arguments);
571
+ this.abort = new AbortController();
572
+ this.workerExecuteLockTimeoutMs = 18e4;
573
+ this.workerCacheTaskTimeoutMs = 60;
574
+ this.approveBatchTasksCount = 2e3;
575
+ this.removeOnExecuted = false;
576
+ this.executeBatchAtOnce = false;
577
+ this.executeJobStatus = false;
578
+ this.executeJobStatusTtlSec = 300;
579
+ this.consumerHost = "host";
580
+ this.stream = "stream";
581
+ this.group = "group";
582
+ }
583
+ async onExecute(id, payload, createdAt, job, key) {
584
+ }
585
+ async onExecuted(data) {
586
+ }
587
+ async onSuccess(id, payload, createdAt, job, key) {
588
+ }
589
+ async success(id, payload, createdAt, job, key) {
590
+ if (this.executeJobStatus) {
591
+ await this.status(id, payload, createdAt, job, key);
592
+ }
593
+ await this.onSuccess(id, payload, createdAt, job, key);
594
+ }
595
+ async status(id, payload, createdAt, job, key) {
596
+ const prefix = `s:${this.stream}:`;
597
+ const { ready = 0, ok = 0 } = await this.getMany(prefix);
598
+ await this.redis.setMany([{ key: `${prefix}ready`, value: ready + 1 }, { key: `${prefix}ok`, value: ok + 1 }], this.executeJobStatusTtlSec);
599
+ }
600
+ async execute(tasks) {
601
+ const result = [];
602
+ let contended = 0, promises = [];
603
+ for (const [id, payload, createdAt, job, idemKey] of tasks) {
604
+ if (this.executeBatchAtOnce) {
605
+ promises.push((async () => {
606
+ const r = await this.executeProcess(id, payload, createdAt, job, idemKey);
607
+ if (r.id) {
608
+ result.push(id);
609
+ } else if (r.contended) {
610
+ contended++;
611
+ }
612
+ })());
479
613
  } else {
480
- partial = { queueName, payload: item };
614
+ const r = await this.executeProcess(id, payload, createdAt, job, idemKey);
615
+ if (r.id) {
616
+ result.push(id);
617
+ } else if (r.contended) {
618
+ contended++;
619
+ }
481
620
  }
482
- const task = this.buildTask(partial);
483
- const raw = this.toPayload(task);
484
- let d = 0;
485
- if ((0, import_full_utils.isNumP)(uniformDelay)) {
486
- d = uniformDelay;
487
- } else if ((0, import_full_utils.isArr)(perItemDelays)) {
488
- d = Number(perItemDelays[i] || 0);
621
+ }
622
+ try {
623
+ if (this.executeBatchAtOnce && promises.length > 0) {
624
+ await Promise.all(promises);
489
625
  }
490
- if (d > 0) {
491
- tx.zadd(delayed, now + d, raw);
492
- } else {
493
- tx.rpush(ready, raw);
626
+ await this.onExecuted(tasks);
627
+ if ((!Array.isArray(result) || !(result.length > 0)) && contended > tasks.length >> 1) {
628
+ await this.waitAbortable(15 + Math.floor(Math.random() * 35) + Math.min(250, 15 * contended + Math.floor(Math.random() * 40)));
494
629
  }
495
- total++;
630
+ } catch (err) {
496
631
  }
497
- await tx.exec();
498
- idx = end;
632
+ return result;
499
633
  }
500
- return total;
501
- }
502
- async iteration(tasks) {
503
- const tasksProcessed = await this.beforeIterationExecution(tasks);
504
- const limit = Math.max(1, Number(this.concurrency) || 1);
505
- let i = 0;
506
- while (i < tasksProcessed.length) {
507
- const slice = tasksProcessed.slice(i, i + limit);
508
- await Promise.all(slice.map((task) => this.logic(task)));
509
- i += limit;
510
- }
511
- await this.afterIterationExecution(tasksProcessed, tasksProcessed.map((t) => t.result ?? {}));
512
- }
513
- async beforeIterationExecution(data) {
514
- return data;
515
- }
516
- async afterIterationExecution(data, results) {
517
- }
518
- async beforeExecution(task) {
519
- return task;
520
- }
521
- async afterExecution(task, result) {
522
- return result;
523
- }
524
- async execute(task) {
525
- return {};
526
- }
527
- async onRetry(task) {
528
- }
529
- async onError(err, task) {
530
- }
531
- async onFail(err, task) {
532
- }
533
- async onFatal(err, task) {
534
- }
535
- async onSuccess(task, result) {
536
- }
537
- async onChainSuccess(task, result) {
538
- }
539
- async onIterationError(err, queueName) {
540
- }
541
- async logic(task) {
542
- let data = task;
543
- try {
544
- data = await this.beforeExecution(task);
545
- const before = data?.result ?? {};
546
- const after = await this.execute(data);
547
- data.result = {
548
- ...(0, import_full_utils.isObjFilled)(before) ? before : {},
549
- ...(0, import_full_utils.isObjFilled)(after) ? after : {}
550
- };
551
- await this.success(data, data.result);
552
- return await this.afterExecution(data, data.result);
553
- } catch (err) {
554
- try {
555
- await this.retry(data);
556
- } catch (err2) {
557
- await this.error(err2, data);
558
- }
559
- } finally {
560
- try {
561
- this.stopHeartbeat(data);
562
- await this.ack(data).catch(() => {
563
- });
564
- } catch {
634
+ async executeProcess(id, payload, createdAt, job, key) {
635
+ if (key) {
636
+ return await this.idempotency(id, payload, createdAt, job, key);
637
+ } else {
638
+ try {
639
+ await this.onExecute(id, payload, createdAt, job, key);
640
+ await this.success(id, payload, createdAt, job, key);
641
+ return { id };
642
+ } catch (err) {
643
+ }
565
644
  }
645
+ return {};
566
646
  }
567
- return {};
568
- }
569
- jitteredBackoffSec(attempt) {
570
- const base = Math.max(1, Number(this.retryBaseSec) || 1);
571
- const maxD = Math.max(base, Number(this.retryMaxSec) || 3600);
572
- const pow = Math.min(maxD, base * Math.pow(2, Math.max(0, attempt - 1)));
573
- const jitter = Math.floor(Math.random() * base);
574
- return Math.min(maxD, pow + jitter);
575
- }
576
- async retry(task) {
577
- if (!(0, import_full_utils.isObjFilled)(task) || !(0, import_full_utils.isStrFilled)(task.iterationId) || !(0, import_full_utils.isStrFilled)(task.id) || !(0, import_full_utils.isStrFilled)(task.queueName) || !(0, import_full_utils.isNumPZ)(task.currentAttempt) || !(0, import_full_utils.isNumPZ)(task.maxAttempts)) {
578
- await this.error(new Error("Task format error."), task);
579
- return;
580
- }
581
- const maxAttempts = task.maxAttempts ?? this.maxAttempts;
582
- try {
583
- if (task.currentAttempt < maxAttempts - 1) {
584
- const taskProcessed = { ...task, currentAttempt: task.currentAttempt + 1 };
585
- const delaySec = this.jitteredBackoffSec(taskProcessed.currentAttempt);
586
- await this.addTask(taskProcessed, delaySec);
587
- await this.onRetry(taskProcessed);
588
- return;
647
+ async approve(ids) {
648
+ if (!Array.isArray(ids) || !(ids.length > 0)) {
649
+ return 0;
589
650
  }
590
- } catch (err) {
591
- await this.fail(err, task);
592
- return;
593
- }
594
- await this.fail(new Error("The attempt limit has been reached."), task);
595
- }
596
- async iterationError(err, queueName, data) {
597
- try {
598
- await this.onIterationError(err, queueName);
599
- } catch (err2) {
600
- }
601
- for (const t of data || []) {
602
- if ((0, import_full_utils.isStrFilled)(t.id)) {
603
- this.processingRaw.delete(t.id);
651
+ const approveBatchTasksCount = Math.max(500, Math.min(4e3, this.approveBatchTasksCount));
652
+ let total = 0, i = 0;
653
+ while (i < ids.length) {
654
+ const room = Math.min(approveBatchTasksCount, ids.length - i);
655
+ const part = ids.slice(i, i + room);
656
+ const approved = await this.runScript("Approve", [this.stream], [this.group, this.removeOnExecuted ? "1" : "0", ...part], Approve);
657
+ total += Number(approved || 0);
658
+ i += room;
604
659
  }
605
- }
606
- }
607
- async error(err, task) {
608
- try {
609
- await this.addTask({
610
- ...task,
611
- queueName: [task.queueName, task.iterationId, "error", "list"].join(":"),
612
- currentAttempt: 0,
613
- payload: {
614
- ...task.payload,
615
- errorMessage: String(err?.message ?? "")
660
+ return total;
661
+ }
662
+ async idempotency(id, payload, createdAt, job, key) {
663
+ const keys = this.idempotencyKeys(key);
664
+ const allow = await this.idempotencyAllow(keys);
665
+ if (allow === 1) {
666
+ return { id };
667
+ } else if (allow === 0) {
668
+ let ttl = -2;
669
+ try {
670
+ ttl = await this.redis.pttl(keys.startKey);
671
+ } catch (err) {
616
672
  }
673
+ await this.waitAbortable(ttl);
674
+ return { contended: true };
675
+ }
676
+ if (!await this.idempotencyStart(keys)) {
677
+ return { contended: true };
678
+ }
679
+ const heartbeat = this.heartbeat(keys) || (() => {
617
680
  });
618
- await this.onError(err, task);
619
- } catch (err2) {
620
681
  try {
621
- await this.onFatal(err2, task);
622
- } catch {
682
+ await this.onExecute(id, payload, createdAt, job, key);
683
+ await this.idempotencyDone(keys);
684
+ await this.success(id, payload, createdAt, job, key);
685
+ return { id };
686
+ } catch (err) {
687
+ try {
688
+ await this.idempotencyFree(keys);
689
+ } catch (err2) {
690
+ }
691
+ } finally {
692
+ heartbeat();
623
693
  }
624
694
  }
625
- try {
626
- await this.status(task, "error");
627
- } catch {
695
+ idempotencyKeys(key) {
696
+ const prefix = `q:${this.stream.replace(/[^\w:\-]/g, "_")}:`;
697
+ const keyP = key.replace(/[^\w:\-]/g, "_");
698
+ const doneKey = `${prefix}done:${keyP}`;
699
+ const lockKey = `${prefix}lock:${keyP}`;
700
+ const startKey = `${prefix}start:${keyP}`;
701
+ const token = `${this.consumer()}:${Date.now().toString(36)}:${Math.random().toString(36).slice(2)}`;
702
+ return {
703
+ prefix,
704
+ doneKey,
705
+ lockKey,
706
+ startKey,
707
+ token
708
+ };
709
+ }
710
+ async idempotencyAllow(keys) {
711
+ const res = await this.runScript("IdempotencyAllow", [keys.doneKey, keys.lockKey, keys.startKey], [String(this.workerExecuteLockTimeoutMs), keys.token], IdempotencyAllow);
712
+ return Number(res || 0);
628
713
  }
629
- }
630
- async fail(err, task) {
631
- try {
632
- await this.addTask({
633
- ...task,
634
- queueName: [task.queueName, task.iterationId, "fail", "list"].join(":"),
635
- currentAttempt: 0,
636
- payload: {
637
- ...task.payload,
638
- errorMessage: String(err?.message ?? "")
714
+ async idempotencyStart(keys) {
715
+ const res = await this.runScript("IdempotencyStart", [keys.lockKey, keys.startKey], [keys.token, String(this.workerExecuteLockTimeoutMs)], IdempotencyStart);
716
+ return Number(res || 0) === 1;
717
+ }
718
+ async idempotencyDone(keys) {
719
+ await this.runScript("IdempotencyDone", [keys.doneKey, keys.lockKey, keys.startKey], [String(this.workerCacheTaskTimeoutMs), keys.token], IdempotencyDone);
720
+ }
721
+ async idempotencyFree(keys) {
722
+ await this.runScript("IdempotencyFree", [keys.lockKey, keys.startKey], [keys.token], IdempotencyFree);
723
+ }
724
+ async waitAbortable(ttl) {
725
+ return new Promise((resolve) => {
726
+ const signal = this.signal();
727
+ if (signal?.aborted) {
728
+ return resolve();
639
729
  }
730
+ const t = setTimeout(() => {
731
+ if (signal) {
732
+ signal.removeEventListener("abort", onAbort);
733
+ }
734
+ resolve();
735
+ }, ttl > 0 ? 25 + Math.floor(Math.random() * 50) : 5 + Math.floor(Math.random() * 15));
736
+ t.unref?.();
737
+ function onAbort() {
738
+ clearTimeout(t);
739
+ resolve();
740
+ }
741
+ signal?.addEventListener("abort", onAbort, { once: true });
640
742
  });
641
- await this.onFail(err, task);
642
- } catch (err2) {
643
- try {
644
- await this.onFatal(err2, task);
645
- } catch {
646
- }
647
743
  }
648
- try {
649
- await this.status(task, "fail");
650
- } catch {
651
- }
652
- }
653
- async success(task, result) {
654
- const taskProcessed = {
655
- ...task,
656
- progress: {
657
- ...task.progress,
658
- successAt: Date.now()
744
+ heartbeat(keys) {
745
+ if (this.workerExecuteLockTimeoutMs <= 0) {
746
+ return;
659
747
  }
660
- };
661
- try {
662
- if ((0, import_full_utils.isObjFilled)(taskProcessed.chain) && (0, import_full_utils.isArrFilled)(taskProcessed.chain.queues) && (0, import_full_utils.isNumPZ)(taskProcessed.chain.index)) {
663
- const currentIndex = taskProcessed.chain.index;
664
- const newIndex = currentIndex + 1;
665
- taskProcessed.progress.chain.push(Date.now());
666
- if (currentIndex === taskProcessed.chain.queues.length - 1) {
667
- await this.status(taskProcessed, "success");
668
- await this.onChainSuccess(taskProcessed, result);
669
- } else if (newIndex <= taskProcessed.chain.queues.length - 1) {
670
- const newQueueName = taskProcessed.chain.queues[newIndex];
671
- if ((0, import_full_utils.isStrFilled)(newQueueName)) {
672
- await this.addTask({
673
- ...taskProcessed,
674
- queueName: newQueueName,
675
- currentAttempt: 0,
676
- chain: {
677
- ...taskProcessed.chain,
678
- index: newIndex
679
- },
680
- result
681
- });
682
- } else {
683
- await this.fail(new Error("Next queue format error."), taskProcessed);
748
+ let timer, alive = true, hbFails = 0;
749
+ const workerHeartbeatTimeoutMs = Math.max(1e3, Math.floor(Math.max(5e3, this.workerExecuteLockTimeoutMs | 0) / 4));
750
+ const stop = () => {
751
+ alive = false;
752
+ if (timer) {
753
+ clearTimeout(timer);
754
+ }
755
+ };
756
+ const onAbort = () => stop();
757
+ const signal = this.signal();
758
+ signal?.addEventListener?.("abort", onAbort, { once: true });
759
+ const tick = async () => {
760
+ if (!alive) {
761
+ return;
762
+ }
763
+ try {
764
+ const r = await this.heartbeat(keys);
765
+ hbFails = r ? 0 : hbFails + 1;
766
+ if (hbFails >= 3) {
767
+ throw new Error("Heartbeat lost.");
768
+ }
769
+ } catch {
770
+ hbFails++;
771
+ if (hbFails >= 6) {
772
+ stop();
773
+ return;
684
774
  }
685
775
  }
686
- } else {
687
- await this.status(taskProcessed, "success");
688
- }
689
- await this.onSuccess(taskProcessed, result);
690
- } catch (err) {
691
- try {
692
- await this.status(taskProcessed, "fatal");
693
- } catch {
776
+ timer = setTimeout(tick, workerHeartbeatTimeoutMs).unref?.();
777
+ };
778
+ timer = setTimeout(tick, workerHeartbeatTimeoutMs).unref?.();
779
+ return () => {
780
+ signal?.removeEventListener?.("abort", onAbort);
781
+ stop();
782
+ };
783
+ }
784
+ values(value) {
785
+ const result = {};
786
+ for (let i = 0; i < value.length; i += 2) {
787
+ result[value[i]] = value[i + 1];
694
788
  }
789
+ return result;
790
+ }
791
+ payload(data) {
695
792
  try {
696
- await this.onFatal(err, taskProcessed);
697
- } catch {
793
+ return JSON.parse(data.payload);
794
+ } catch (err) {
698
795
  }
796
+ return data;
797
+ }
798
+ consumer() {
799
+ return `${String(this.consumerHost || "host")}:${process.pid}`;
699
800
  }
801
+ signal() {
802
+ return this.abort.signal;
803
+ }
804
+ };
805
+ }
806
+
807
+ // src/PowerQueues.ts
808
+ var Base = class {
809
+ };
810
+ var PowerQueues = class extends mix(Base, AddTasks, SelectTasks, ExecuteTasks) {
811
+ constructor() {
812
+ super(...arguments);
813
+ this.abort = new AbortController();
700
814
  }
701
- async status(task, category = "success") {
702
- if (!this.checkConnection()) {
703
- throw new Error("Redis connection error.");
704
- }
705
- const processedKey = this.toKeyString(task.queueName, task.iterationId, "processed");
706
- const categoryKey = this.toKeyString(task.queueName, task.iterationId, category);
707
- await this.redis?.incr(processedKey);
708
- await this.redis?.incr(categoryKey);
709
- await this.redis?.expire(processedKey, this.expireStatusSec);
710
- await this.redis?.expire(categoryKey, this.expireStatusSec);
815
+ async onSelected(data) {
816
+ return data;
711
817
  }
712
- async loop(queueName, runner) {
713
- if (!(0, import_full_utils.isStrFilled)(queueName)) {
714
- throw new Error(`Queue name is not valid: "${queueName}"; Type: "${typeof queueName}".`);
715
- }
716
- const ready = this.readyKey(queueName);
717
- const processing = this.processingKey(queueName);
718
- const processingVt = this.processingVtKey(queueName);
719
- const delayed = this.delayedKey(queueName);
720
- while (runner.running) {
721
- if (!this.checkConnection()) {
722
- await (0, import_full_utils.wait)(this.iterationTimeout);
723
- continue;
724
- }
725
- try {
726
- await this.promoteDelayed(delayed, ready);
727
- } catch {
728
- }
729
- try {
730
- await this.requeueExpired(processing, processingVt, ready);
731
- } catch {
732
- }
733
- let data = [];
818
+ async onExecute(id, payload, createdAt, job, key) {
819
+ }
820
+ async onExecuted(data) {
821
+ }
822
+ async onSuccess(id, payload, createdAt, job, key) {
823
+ }
824
+ async runQueue() {
825
+ await this.createGroup("0-0");
826
+ await this.consumerLoop();
827
+ }
828
+ async consumerLoop() {
829
+ const signal = this.signal();
830
+ while (!signal?.aborted) {
734
831
  try {
735
- data = await this.data(queueName);
736
- if (!(0, import_full_utils.isArrFilled)(data)) {
737
- await (0, import_full_utils.wait)(this.iterationTimeout);
832
+ const tasks = await this.select();
833
+ if (!Array.isArray(tasks) || !(tasks.length > 0)) {
834
+ await (0, import_full_utils.wait)(600);
738
835
  continue;
739
836
  }
740
- await this.iteration(data);
837
+ const tasksP = await this.onSelected(tasks);
838
+ const ids = await this.execute(Array.isArray(tasksP) && tasksP.length > 0 ? tasksP : tasks);
839
+ if (Array.isArray(ids) && ids.length > 0) {
840
+ await this.approve(ids);
841
+ }
741
842
  } catch (err) {
742
- await this.iterationError(err, queueName, data);
743
- await (0, import_full_utils.wait)(this.iterationTimeout);
744
- }
745
- }
746
- }
747
- async data(queueName) {
748
- const ready = this.readyKey(queueName);
749
- const processing = this.processingKey(queueName);
750
- const processingVt = this.processingVtKey(queueName);
751
- const raws = await this.reserveMany(ready, processing, processingVt, this.portionLength, this.visibilityTimeoutSec);
752
- if (!(0, import_full_utils.isArrFilled)(raws)) {
753
- return [];
754
- }
755
- const tasks = [];
756
- for (const raw of raws) {
757
- const obj = this.fromPayload(raw);
758
- if ((0, import_full_utils.isObjFilled)(obj) && (0, import_full_utils.isStrFilled)(obj.iterationId) && (0, import_full_utils.isStrFilled)(obj.id) && (0, import_full_utils.isStrFilled)(obj.queueName) && (0, import_full_utils.isNumPZ)(obj.maxAttempts) && (0, import_full_utils.isNumPZ)(obj.currentAttempt)) {
759
- const t = obj;
760
- this.processingRaw.set(t.id, raw);
761
- this.startHeartbeat(t);
762
- tasks.push(t);
763
- }
764
- }
765
- return tasks;
766
- }
767
- async ack(task) {
768
- try {
769
- const raw = this.processingRaw.get(task.id);
770
- if (!(0, import_full_utils.isStrFilled)(raw)) {
771
- return;
843
+ await (0, import_full_utils.wait)(600);
772
844
  }
773
- this.processingRaw.delete(task.id);
774
- await this.ackProcessing(this.processingKey(task.queueName), this.processingVtKey(task.queueName), raw);
775
- } catch {
776
845
  }
777
846
  }
778
847
  };
779
848
  // Annotate the CommonJS export names for ESM import in node:
780
849
  0 && (module.exports = {
781
- PowerQueue
850
+ PowerQueues
782
851
  });