power-queues 2.0.3 → 2.0.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (5) hide show
  1. package/dist/index.cjs +753 -684
  2. package/dist/index.d.cts +34 -280
  3. package/dist/index.d.ts +34 -280
  4. package/dist/index.js +752 -693
  5. package/package.json +5 -2
package/dist/index.js CHANGED
@@ -1,765 +1,824 @@
1
- // src/PowerQueue.ts
1
+ // src/PowerQueues.ts
2
+ import { wait } from "full-utils";
3
+
4
+ // src/AddTasks.ts
2
5
  import { v4 as uuid } from "uuid";
3
- import {
4
- isStrFilled,
5
- isStr,
6
- isArrFilled,
7
- isArr,
8
- isObjFilled,
9
- isNumPZ,
10
- isNumP,
11
- isFunc,
12
- wait
13
- } from "full-utils";
14
- import { PowerRedis } from "power-redis";
15
- var PowerQueue = class extends PowerRedis {
16
- constructor() {
17
- super(...arguments);
18
- this.iterationTimeout = 1e3;
19
- this.portionLength = 1e3;
20
- this.expireStatusSec = 300;
21
- this.maxAttempts = 1;
22
- this.concurrency = 32;
23
- this.visibilityTimeoutSec = 60;
24
- this.retryBaseSec = 1;
25
- this.retryMaxSec = 3600;
26
- this.runners = /* @__PURE__ */ new Map();
27
- this.processingRaw = /* @__PURE__ */ new Map();
28
- this.heartbeatTimers = /* @__PURE__ */ new Map();
29
- }
30
- nowSec() {
31
- return Math.floor(Date.now() / 1e3);
32
- }
33
- readyKey(queueName) {
34
- return this.toKeyString(queueName);
35
- }
36
- processingKey(queueName) {
37
- return this.toKeyString(queueName, "processing");
38
- }
39
- processingVtKey(queueName) {
40
- return this.toKeyString(queueName, "processing", "vt");
41
- }
42
- delayedKey(queueName) {
43
- return this.toKeyString(queueName, "delayed");
44
- }
45
- toKeyString(...parts) {
46
- return super.toKeyString("queue", ...parts);
47
- }
48
- getReserveScriptLMOVE() {
49
- return `
50
- local source = KEYS[1]
51
- local processing = KEYS[2]
52
- local vtkey = KEYS[3]
53
- local limit = tonumber(ARGV[1])
54
- local deadline = tonumber(ARGV[2])
55
- local moved = {}
56
- for i = 1, limit do
57
- local v = redis.call('LMOVE', source, processing, 'LEFT', 'RIGHT')
58
- if not v then break end
59
- table.insert(moved, v)
60
- end
61
- if #moved > 0 then
62
- for i = 1, #moved do
63
- redis.call('ZADD', vtkey, deadline, moved[i])
6
+
7
+ // src/scripts.ts
8
+ var XAddBulk = `
9
+ local UNPACK = table and table.unpack or unpack
10
+
11
+ local stream = KEYS[1]
12
+ local maxlen = tonumber(ARGV[1])
13
+ local approxFlag = tonumber(ARGV[2]) == 1
14
+ local n = tonumber(ARGV[3])
15
+ local exactFlag = tonumber(ARGV[4]) == 1
16
+ local nomkstream = tonumber(ARGV[5]) == 1
17
+ local trimLimit = tonumber(ARGV[6])
18
+ local minidWindowMs = tonumber(ARGV[7]) or 0
19
+ local minidExact = tonumber(ARGV[8]) == 1
20
+ local idx = 9
21
+ local out = {}
22
+
23
+ local common_opts = {}
24
+ local co_len = 0
25
+
26
+ if nomkstream then
27
+ co_len = co_len + 1; common_opts[co_len] = 'NOMKSTREAM'
28
+ end
29
+
30
+ if minidWindowMs > 0 then
31
+ local tm = redis.call('TIME')
32
+ local now_ms = (tonumber(tm[1]) * 1000) + math.floor(tonumber(tm[2]) / 1000)
33
+ local cutoff_ms = now_ms - minidWindowMs
34
+ if cutoff_ms < 0 then cutoff_ms = 0 end
35
+ local cutoff_id = tostring(cutoff_ms) .. '-0'
36
+
37
+ co_len = co_len + 1; common_opts[co_len] = 'MINID'
38
+ co_len = co_len + 1; common_opts[co_len] = (minidExact and '=' or '~')
39
+ co_len = co_len + 1; common_opts[co_len] = cutoff_id
40
+ if trimLimit and trimLimit > 0 then
41
+ co_len = co_len + 1; common_opts[co_len] = 'LIMIT'
42
+ co_len = co_len + 1; common_opts[co_len] = trimLimit
43
+ end
44
+ elseif maxlen and maxlen > 0 then
45
+ co_len = co_len + 1; common_opts[co_len] = 'MAXLEN'
46
+ if exactFlag then
47
+ co_len = co_len + 1; common_opts[co_len] = '='
48
+ elseif approxFlag then
49
+ co_len = co_len + 1; common_opts[co_len] = '~'
50
+ end
51
+ co_len = co_len + 1; common_opts[co_len] = maxlen
52
+ if trimLimit and trimLimit > 0 then
53
+ co_len = co_len + 1; common_opts[co_len] = 'LIMIT'
54
+ co_len = co_len + 1; common_opts[co_len] = trimLimit
55
+ end
56
+ end
57
+
58
+ for e = 1, n do
59
+ local id = ARGV[idx]; idx = idx + 1
60
+ local num_pairs = tonumber(ARGV[idx]); idx = idx + 1
61
+
62
+ local a = {}
63
+ local a_len = 0
64
+
65
+ for i = 1, co_len do a_len = a_len + 1; a[a_len] = common_opts[i] end
66
+
67
+ a_len = a_len + 1; a[a_len] = id
68
+
69
+ for j = 1, (num_pairs * 2) do
70
+ a_len = a_len + 1; a[a_len] = ARGV[idx]; idx = idx + 1
71
+ end
72
+
73
+ local addedId = redis.call('XADD', stream, UNPACK(a))
74
+ out[#out+1] = addedId or ''
75
+ end
76
+
77
+ return out
78
+ `;
79
+ var Approve = `
80
+ local stream = KEYS[1]
81
+ local group = ARGV[1]
82
+ local delFlag = tonumber(ARGV[2]) == 1
83
+
84
+ local acked = 0
85
+ local nids = #ARGV - 2
86
+ if nids > 0 then
87
+ acked = tonumber(redis.call('XACK', stream, group, unpack(ARGV, 3))) or 0
88
+ if delFlag and nids > 0 then
89
+ local ok, deln = pcall(redis.call, 'XDEL', stream, unpack(ARGV, 3))
90
+ if not ok then
91
+ deln = 0
92
+ for i = 3, #ARGV do
93
+ deln = deln + (tonumber(redis.call('XDEL', stream, ARGV[i])) or 0)
64
94
  end
65
95
  end
66
- return moved
67
- `;
68
- }
69
- getReserveScriptRPOPLPUSH() {
70
- return `
71
- local source = KEYS[1]
72
- local processing = KEYS[2]
73
- local vtkey = KEYS[3]
74
- local limit = tonumber(ARGV[1])
75
- local deadline = tonumber(ARGV[2])
76
- local moved = {}
77
- for i = 1, limit do
78
- local v = redis.call('RPOPLPUSH', source, processing)
79
- if not v then break end
80
- table.insert(moved, v)
81
- end
82
- if #moved > 0 then
83
- for i = 1, #moved do
84
- redis.call('ZADD', vtkey, deadline, moved[i])
85
- end
96
+ end
97
+ end
98
+ return acked
99
+ `;
100
+ var IdempotencyAllow = `
101
+ local doneKey = KEYS[1]
102
+ local lockKey = KEYS[2]
103
+ local startKey = KEYS[3]
104
+
105
+ if redis.call('EXISTS', doneKey) == 1 then
106
+ return 1
107
+ end
108
+
109
+ local ttl = tonumber(ARGV[1]) or 0
110
+ if ttl <= 0 then return 0 end
111
+
112
+ local ok = redis.call('SET', lockKey, ARGV[2], 'NX', 'PX', ttl)
113
+ if ok then
114
+ if startKey and startKey ~= '' then
115
+ redis.call('SET', startKey, 1, 'PX', ttl)
116
+ end
117
+ return 2
118
+ else
119
+ return 0
120
+ end
121
+ `;
122
+ var IdempotencyStart = `
123
+ local lockKey = KEYS[1]
124
+ local startKey = KEYS[2]
125
+ if redis.call('GET', lockKey) == ARGV[1] then
126
+ local ttl = tonumber(ARGV[2]) or 0
127
+ if ttl > 0 then
128
+ redis.call('SET', startKey, 1, 'PX', ttl)
129
+ redis.call('PEXPIRE', lockKey, ttl)
130
+ else
131
+ redis.call('SET', startKey, 1)
132
+ end
133
+ return 1
134
+ end
135
+ return 0
136
+ `;
137
+ var IdempotencyDone = `
138
+ local doneKey = KEYS[1]
139
+ local lockKey = KEYS[2]
140
+ local startKey = KEYS[3]
141
+ redis.call('SET', doneKey, 1)
142
+ local ttlSec = tonumber(ARGV[1]) or 0
143
+ if ttlSec > 0 then redis.call('EXPIRE', doneKey, ttlSec) end
144
+ if redis.call('GET', lockKey) == ARGV[2] then
145
+ redis.call('DEL', lockKey)
146
+ if startKey then redis.call('DEL', startKey) end
147
+ end
148
+ return 1
149
+ `;
150
+ var IdempotencyFree = `
151
+ local lockKey = KEYS[1]
152
+ local startKey = KEYS[2]
153
+ if redis.call('GET', lockKey) == ARGV[1] then
154
+ redis.call('DEL', lockKey)
155
+ if startKey then redis.call('DEL', startKey) end
156
+ return 1
157
+ end
158
+ return 0
159
+ `;
160
+ var SelectStuck = `
161
+ local stream = KEYS[1]
162
+ local group = ARGV[1]
163
+ local consumer = ARGV[2]
164
+ local pendingIdleMs = tonumber(ARGV[3])
165
+ local count = tonumber(ARGV[4]) or 0
166
+ if count < 1 then count = 1 end
167
+
168
+ local timeBudgetMs = tonumber(ARGV[5]) or 15
169
+ local t0 = redis.call('TIME')
170
+ local start_ms = (tonumber(t0[1]) * 1000) + math.floor(tonumber(t0[2]) / 1000)
171
+
172
+ local results = {}
173
+ local collected = 0
174
+ local start_id = '0-0'
175
+ local iters = 0
176
+ local max_iters = math.max(16, math.ceil(count / 100))
177
+
178
+ local function time_exceeded()
179
+ local t1 = redis.call('TIME')
180
+ local now_ms = (tonumber(t1[1]) * 1000) + math.floor(tonumber(t1[2]) / 1000)
181
+ return (now_ms - start_ms) >= timeBudgetMs
182
+ end
183
+
184
+ while (collected < count) and (iters < max_iters) do
185
+ local to_claim = count - collected
186
+ if to_claim < 1 then break end
187
+
188
+ local claim = redis.call('XAUTOCLAIM', stream, group, consumer, pendingIdleMs, start_id, 'COUNT', to_claim)
189
+ iters = iters + 1
190
+
191
+ local bucket = nil
192
+ if claim then
193
+ bucket = claim[2]
194
+ end
195
+ if bucket and #bucket > 0 then
196
+ for i = 1, #bucket do
197
+ results[#results+1] = bucket[i]
86
198
  end
87
- return moved
88
- `;
89
- }
90
- getRequeueScript() {
91
- return `
92
- -- KEYS: 1=processing, 2=processingVt, 3=ready
93
- -- ARGV: 1=now, 2=limit
94
- local processing = KEYS[1]
95
- local vt = KEYS[2]
96
- local ready = KEYS[3]
97
- local now = tonumber(ARGV[1])
98
- local limit = tonumber(ARGV[2])
99
-
100
- local members = redis.call('ZRANGEBYSCORE', vt, 0, now, 'LIMIT', 0, limit)
101
- for i=1,#members do
102
- local m = members[i]
103
- redis.call('ZREM', vt, m)
104
- redis.call('LREM', processing, 1, m)
105
- redis.call('RPUSH', ready, m)
199
+ collected = #results
200
+ end
201
+
202
+ local next_id = claim and claim[1] or start_id
203
+ if next_id == start_id then
204
+ local s, seq = string.match(start_id, '^(%d+)%-(%d+)$')
205
+ if s and seq then
206
+ start_id = s .. '-' .. tostring(tonumber(seq) + 1)
207
+ else
208
+ start_id = '0-1'
106
209
  end
107
- return #members
108
- `;
109
- }
110
- getPromoteScript() {
111
- return `
112
- -- KEYS: 1=delayed, 2=ready
113
- -- ARGV: 1=now, 2=limit
114
- local delayed = KEYS[1]
115
- local ready = KEYS[2]
116
- local now = tonumber(ARGV[1])
117
- local limit = tonumber(ARGV[2])
118
-
119
- local due = redis.call('ZRANGEBYSCORE', delayed, 0, now, 'LIMIT', 0, limit)
120
- for i=1,#due do
121
- local m = due[i]
122
- redis.call('ZREM', delayed, m)
123
- redis.call('RPUSH', ready, m)
210
+ else
211
+ start_id = next_id
212
+ end
213
+
214
+ if time_exceeded() then
215
+ break
216
+ end
217
+ end
218
+
219
+ local left = count - collected
220
+ if left > 0 then
221
+ local xr = redis.call('XREADGROUP', 'GROUP', group, consumer, 'COUNT', left, 'STREAMS', stream, '>')
222
+ if xr and xr[1] and xr[1][2] then
223
+ local entries = xr[1][2]
224
+ for i = 1, #entries do
225
+ results[#results+1] = entries[i]
124
226
  end
125
- return #due
126
- `;
127
- }
128
- async ensureReserveScript(force = false) {
129
- if (!force && (this.reserveSha || this.reserveShaRpoplpush || !isFunc(this.redis?.script))) {
130
- return;
131
- }
132
- this.reserveSha = void 0;
133
- this.reserveShaRpoplpush = void 0;
134
- try {
135
- this.reserveSha = await this.redis?.script("LOAD", this.getReserveScriptLMOVE());
136
- } catch {
137
- this.reserveShaRpoplpush = await this.redis?.script("LOAD", this.getReserveScriptRPOPLPUSH());
138
- }
139
- }
140
- async ensureRequeueScript(force = false) {
141
- if (!force && this.requeueSha) {
142
- return;
143
- }
144
- const scriptFn = this.redis?.script;
145
- if (!scriptFn) {
146
- return;
147
- }
148
- this.requeueSha = await scriptFn("LOAD", this.getRequeueScript());
149
- }
150
- async ensurePromoteScript(force = false) {
151
- if (!force && this.promoteSha) {
152
- return;
153
- }
154
- const scriptFn = this.redis?.script;
155
- if (!scriptFn) {
156
- return;
157
- }
158
- this.promoteSha = await scriptFn("LOAD", this.getPromoteScript());
159
- }
160
- async moveOneToProcessing(source, processing) {
161
- const cli = this.redis;
162
- try {
163
- if (isFunc(cli.lmove)) {
164
- const v = await cli.lmove(source, processing, "LEFT", "RIGHT");
165
- return isStr(v) ? v : null;
227
+ end
228
+ end
229
+
230
+ return results
231
+ `;
232
+
233
+ // src/mix.ts
234
+ function mix(Base2, ...mixins) {
235
+ return mixins.reduce((acc, mixin) => mixin(acc), Base2);
236
+ }
237
+
238
+ // src/Script.ts
239
+ function Script(Base2) {
240
+ return class extends Base2 {
241
+ constructor() {
242
+ super(...arguments);
243
+ this.strictCheckingConnection = ["true", "on", "yes", "y", "1"].includes(String(process.env.REDIS_STRICT_CHECK_CONNECTION ?? "").trim().toLowerCase());
244
+ this.scripts = {};
245
+ }
246
+ async runScript(name, keys, args, defaultCode) {
247
+ if (!this.scripts[name]) {
248
+ if (typeof defaultCode !== "string" || !(defaultCode.length > 0)) {
249
+ throw new Error(`Undefined script "${name}". Save it before executing.`);
250
+ }
251
+ this.saveScript(name, defaultCode);
166
252
  }
167
- } catch {
168
- }
169
- try {
170
- if (isFunc(cli.rpoplpush)) {
171
- const v = await cli.rpoplpush(source, processing);
172
- return isStr(v) ? v : null;
253
+ if (!this.scripts[name].codeReady) {
254
+ this.scripts[name].codeReady = await this.loadScript(this.scripts[name].codeBody);
173
255
  }
174
- } catch {
175
- }
176
- return null;
177
- }
178
- async evalshaWithReload(shaGetter, ensure, numKeys, keysAndArgs) {
179
- await ensure(false);
180
- const sha = shaGetter();
181
- const evalshaFn = this.redis?.evalsha;
182
- if (!sha || !evalshaFn) {
183
- throw new Error("EVALSHA not available or SHA missing");
184
- }
185
- try {
186
- return await evalshaFn(sha, numKeys, ...keysAndArgs.map(String));
187
- } catch (err) {
188
- const msg = err?.message;
189
- if (typeof msg === "string" && msg.includes("NOSCRIPT")) {
190
- await ensure(true);
191
- const sha2 = shaGetter();
192
- if (!sha2) {
193
- throw new Error("EVALSHA NOSCRIPT and reload failed (no SHA)");
256
+ try {
257
+ return await this.redis.evalsha(this.scripts[name].codeReady, keys.length, ...keys, ...args);
258
+ } catch (err) {
259
+ if (String(err?.message || "").includes("NOSCRIPT")) {
260
+ this.scripts[name].codeReady = await this.loadScript(this.scripts[name].codeBody);
261
+ return await this.redis.evalsha(this.scripts[name].codeReady, keys.length, ...keys, ...args);
194
262
  }
195
- return await evalshaFn(sha2, numKeys, ...keysAndArgs.map(String));
263
+ throw err;
196
264
  }
197
- throw err;
198
265
  }
199
- }
200
- async zaddCompatXXCH(key, score, member) {
201
- const zadd = this.redis?.zadd;
202
- try {
203
- if (zadd) {
204
- await zadd.call(this.redis, key, "XX", "CH", score, member);
205
- return;
266
+ async loadScripts(full = false) {
267
+ const scripts = full ? [
268
+ ["XAddBulk", XAddBulk],
269
+ ["Approve", Approve],
270
+ ["IdempotencyAllow", IdempotencyAllow],
271
+ ["IdempotencyStart", IdempotencyStart],
272
+ ["IdempotencyDone", IdempotencyDone],
273
+ ["IdempotencyFree", IdempotencyFree],
274
+ ["SelectStuck", SelectStuck]
275
+ ] : [
276
+ ["XAddBulk", XAddBulk]
277
+ ];
278
+ for (const [name, code] of scripts) {
279
+ await this.loadScript(this.saveScript(name, code));
206
280
  }
207
- } catch {
208
281
  }
209
- try {
210
- if (zadd) {
211
- await zadd.call(this.redis, key, "CH", "XX", score, member);
212
- return;
282
+ async loadScript(code) {
283
+ for (let i = 0; i < 3; i++) {
284
+ try {
285
+ return await this.redis.script("LOAD", code);
286
+ } catch (e) {
287
+ if (i === 2) {
288
+ throw e;
289
+ }
290
+ await new Promise((r) => setTimeout(r, 10 + Math.floor(Math.random() * 40)));
291
+ }
213
292
  }
214
- } catch {
293
+ throw new Error("Load lua script failed.");
215
294
  }
216
- try {
217
- await this.redis.zadd(key, score, member);
218
- } catch {
295
+ saveScript(name, codeBody) {
296
+ if (typeof codeBody !== "string" || !(codeBody.length > 0)) {
297
+ throw new Error("Script body is empty.");
298
+ }
299
+ this.scripts[name] = { codeBody };
300
+ return codeBody;
219
301
  }
220
- }
221
- startHeartbeat(task) {
222
- const raw = this.processingRaw.get(task.id);
223
- if (!raw) {
224
- return;
225
- }
226
- const vtKey = this.processingVtKey(task.queueName);
227
- const periodMs = Math.max(1e3, Math.floor(this.visibilityTimeoutSec * 1e3 * 0.4));
228
- const t = setInterval(() => {
229
- this.extendVisibility(vtKey, raw, this.visibilityTimeoutSec).catch(() => {
302
+ };
303
+ }
304
+
305
+ // src/AddTasks.ts
306
+ function AddTasks(Base2) {
307
+ return class extends mix(Base2, Script) {
308
+ constructor() {
309
+ super(...arguments);
310
+ this.addingBatchTasksCount = 800;
311
+ this.addingBatchKeysLimit = 1e4;
312
+ this.idemOn = true;
313
+ this.idemKey = "";
314
+ }
315
+ async addTasks(queueName, data, opts) {
316
+ if (!Array.isArray(data) || !(data.length > 0)) {
317
+ throw new Error("Tasks is not filled.");
318
+ }
319
+ if (typeof queueName !== "string" || !(queueName.length > 0)) {
320
+ throw new Error("Queue name is required.");
321
+ }
322
+ const batches = this.buildBatches(data);
323
+ const result = new Array(data.length);
324
+ const promises = [];
325
+ let cursor = 0;
326
+ for (const batch of batches) {
327
+ const start = cursor;
328
+ const end = start + batch.length;
329
+ cursor = end;
330
+ promises.push(async () => {
331
+ const partIds = await this.xaddBatch(queueName, ...this.payloadBatch(batch, opts));
332
+ for (let k = 0; k < partIds.length; k++) {
333
+ result[start + k] = partIds[k];
334
+ }
335
+ });
336
+ }
337
+ const runners = Array.from({ length: promises.length }, async () => {
338
+ while (promises.length) {
339
+ const promise = promises.shift();
340
+ if (promise) {
341
+ await promise();
342
+ }
343
+ }
230
344
  });
231
- }, periodMs);
232
- t.unref?.();
233
- this.heartbeatTimers.set(task.id, t);
234
- }
235
- stopHeartbeat(task) {
236
- const t = this.heartbeatTimers.get(task.id);
237
- if (t) {
238
- clearInterval(t);
239
- }
240
- this.heartbeatTimers.delete(task.id);
241
- }
242
- async reserveMany(source, processing, processingVt, limit = 100, visibilitySec = 60) {
243
- if (!this.checkConnection()) {
244
- throw new Error("Redis connection error.");
245
- }
246
- if (!isStrFilled(source) || !isStrFilled(processing) || !isStrFilled(processingVt)) {
247
- throw new Error("Key format error.");
248
- }
249
- if (!isNumP(limit) || !isNumP(visibilitySec)) {
250
- throw new Error("Limit/visibility format error.");
251
- }
252
- await this.ensureReserveScript();
253
- const deadline = this.nowSec() + visibilitySec;
254
- const tryEval = async () => {
255
- if (isFunc(this.redis?.evalsha)) {
256
- if (this.reserveSha) {
257
- return await this.redis?.evalsha(this.reserveSha, 3, source, processing, processingVt, String(limit), String(deadline));
345
+ await Promise.all(runners);
346
+ return result;
347
+ }
348
+ async xaddBatch(queueName, ...batches) {
349
+ return await this.runScript("XAddBulk", [queueName], batches, XAddBulk);
350
+ }
351
+ payloadBatch(data, opts) {
352
+ const maxlen = Math.max(0, Math.floor(opts?.maxlen ?? 0));
353
+ const approx = opts?.exact ? 0 : opts?.approx !== false ? 1 : 0;
354
+ const exact = opts?.exact ? 1 : 0;
355
+ const nomkstream = opts?.nomkstream ? 1 : 0;
356
+ const trimLimit = Math.max(0, Math.floor(opts?.trimLimit ?? 0));
357
+ const minidWindowMs = Math.max(0, Math.floor(opts?.minidWindowMs ?? 0));
358
+ const minidExact = opts?.minidExact ? 1 : 0;
359
+ const argv = [
360
+ String(maxlen),
361
+ String(approx),
362
+ String(data.length),
363
+ String(exact),
364
+ String(nomkstream),
365
+ String(trimLimit),
366
+ String(minidWindowMs),
367
+ String(minidExact)
368
+ ];
369
+ for (const item of data) {
370
+ const entry = item;
371
+ const id = entry.id ?? "*";
372
+ let flat;
373
+ if ("flat" in entry && Array.isArray(entry.flat) && entry.flat.length > 0) {
374
+ flat = entry.flat;
375
+ if (flat.length % 2 !== 0) {
376
+ throw new Error('Property "flat" must contain an even number of realKeysLength (field/value pairs).');
377
+ }
378
+ } else if ("payload" in entry && typeof entry.payload === "object" && Object.keys(entry.payload || {}).length > 0) {
379
+ flat = [];
380
+ for (const [k, v] of Object.entries(entry.payload)) {
381
+ flat.push(k, v);
382
+ }
383
+ } else {
384
+ throw new Error('Task must have "payload" or "flat".');
258
385
  }
259
- if (this.reserveShaRpoplpush) {
260
- return await this.redis?.evalsha(this.reserveShaRpoplpush, 3, source, processing, processingVt, String(limit), String(deadline));
386
+ const pairs = flat.length / 2;
387
+ if (pairs <= 0) {
388
+ throw new Error('Task must have "payload" or "flat".');
389
+ }
390
+ argv.push(String(id));
391
+ argv.push(String(pairs));
392
+ for (const token of flat) {
393
+ argv.push(!token ? "" : typeof token === "string" && token.length > 0 ? token : String(token));
261
394
  }
262
395
  }
263
- return null;
264
- };
265
- try {
266
- const res = await tryEval();
267
- if (isArr(res)) {
268
- return Array.from(res).map(String);
269
- }
270
- } catch (err) {
271
- if (isStr(err?.message) && String(err?.message ?? "").includes("NOSCRIPT")) {
272
- await this.ensureReserveScript(true);
273
- try {
274
- const res2 = await tryEval();
275
- if (isArr(res2)) {
276
- return Array.from(res2).map(String);
396
+ return argv;
397
+ }
398
+ buildBatches(tasks) {
399
+ const job = uuid();
400
+ const batches = [];
401
+ let batch = [], realKeysLength = 0;
402
+ for (let task of tasks) {
403
+ let entry = task;
404
+ if (this.idemOn) {
405
+ const createdAt = entry?.createdAt || Date.now();
406
+ let idemKey = entry?.idemKey || uuid();
407
+ if (typeof entry.payload === "object") {
408
+ if (this.idemKey && typeof entry.payload[this.idemKey] === "string" && entry.payload[this.idemKey].length > 0) {
409
+ idemKey = entry.payload[this.idemKey];
410
+ }
411
+ entry = {
412
+ ...entry,
413
+ payload: {
414
+ payload: JSON.stringify(entry.payload),
415
+ createdAt,
416
+ job,
417
+ idemKey
418
+ }
419
+ };
420
+ } else if (Array.isArray(entry.flat)) {
421
+ entry.flat.push("createdAt");
422
+ entry.flat.push(String(createdAt));
423
+ entry.flat.push("job");
424
+ entry.flat.push(job);
425
+ entry.flat.push("idemKey");
426
+ entry.flat.push(idemKey);
277
427
  }
278
- } catch {
279
428
  }
429
+ const reqKeysLength = this.keysLength(entry);
430
+ if (batch.length && (batch.length >= this.addingBatchTasksCount || realKeysLength + reqKeysLength > this.addingBatchKeysLimit)) {
431
+ batches.push(batch);
432
+ batch = [];
433
+ realKeysLength = 0;
434
+ }
435
+ batch.push(entry);
436
+ realKeysLength += reqKeysLength;
280
437
  }
281
- }
282
- const moved = [];
283
- for (let i = 0; i < limit; i++) {
284
- const v = await this.moveOneToProcessing(source, processing);
285
- if (!v) {
286
- break;
438
+ if (batch.length) {
439
+ batches.push(batch);
287
440
  }
288
- moved.push(v);
441
+ return batches;
289
442
  }
290
- if (moved.length) {
291
- const tx = this.redis?.multi();
292
- for (const v of moved) {
293
- tx.zadd(processingVt, deadline, v);
294
- }
295
- await tx.exec();
443
+ keysLength(task) {
444
+ return 2 + ("flat" in task && Array.isArray(task.flat) && task.flat.length ? task.flat.length : Object.keys(task).length * 2);
296
445
  }
297
- return moved;
298
- }
299
- async ackProcessing(processing, processingVt, raw) {
300
- if (!this.checkConnection()) {
301
- throw new Error("Redis connection error.");
302
- }
303
- const tx = this.redis?.multi();
304
- tx.lrem(processing, 1, raw);
305
- tx.zrem(processingVt, raw);
306
- await tx.exec();
307
- return;
308
- }
309
- async requeueExpired(processing, processingVt, ready, nowTs, chunk = 1e3) {
310
- if (!this.checkConnection()) {
311
- throw new Error("Redis connection error.");
312
- }
313
- const now = isNumP(nowTs) ? nowTs : this.nowSec();
314
- try {
315
- const moved = await this.evalshaWithReload(
316
- () => this.requeueSha,
317
- (force) => this.ensureRequeueScript(!!force),
318
- 3,
319
- [processing, processingVt, ready, String(now), String(chunk)]
320
- );
321
- return isNumP(moved) ? moved : 0;
322
- } catch {
323
- const expired = await this.redis?.zrangebyscore(processingVt, 0, now, "LIMIT", 0, chunk);
324
- if (!isArrFilled(expired)) {
325
- return 0;
326
- }
327
- const tx = this.redis?.multi();
328
- for (const raw of expired) {
329
- tx.lrem(processing, 1, raw);
330
- tx.zrem(processingVt, raw);
331
- tx.rpush(ready, raw);
446
+ };
447
+ }
448
+
449
+ // src/SelectTasks.ts
450
+ function SelectTasks(Base2) {
451
+ return class extends mix(Base2, Script) {
452
+ constructor() {
453
+ super(...arguments);
454
+ this.consumerHost = "host";
455
+ this.stream = "stream";
456
+ this.group = "group";
457
+ this.workerBatchTasksCount = 200;
458
+ this.recoveryStuckTasksTimeoutMs = 6e4;
459
+ this.workerLoopIntervalMs = 5e3;
460
+ this.workerSelectionTimeoutMs = 80;
461
+ }
462
+ async createGroup(from = "$") {
463
+ try {
464
+ await this.redis.xgroup("CREATE", this.stream, this.group, from, "MKSTREAM");
465
+ } catch (err) {
466
+ const msg = String(err?.message || "");
467
+ if (!msg.includes("BUSYGROUP")) {
468
+ throw err;
469
+ }
332
470
  }
333
- await tx.exec();
334
- return expired.length;
335
471
  }
336
- }
337
- async promoteDelayed(delayed, ready, nowTs, chunk = 1e3) {
338
- if (!this.checkConnection()) {
339
- throw new Error("Redis connection error.");
340
- }
341
- const now = isNumP(nowTs) ? nowTs : this.nowSec();
342
- try {
343
- const promoted = await this.evalshaWithReload(
344
- () => this.promoteSha,
345
- (force) => this.ensurePromoteScript(!!force),
346
- 2,
347
- [delayed, ready, String(now), String(chunk)]
348
- );
349
- return isNumP(promoted) ? promoted : 0;
350
- } catch {
351
- const due = await this.redis?.zrangebyscore(delayed, 0, now, "LIMIT", 0, chunk);
352
- if (!isArrFilled(due)) {
353
- return 0;
472
+ async select() {
473
+ let entries = await this.selectStuck();
474
+ if (!entries?.length) {
475
+ entries = await this.selectFresh();
354
476
  }
355
- const tx = this.redis?.multi();
356
- for (const raw of due) {
357
- tx.zrem(delayed, raw);
358
- tx.rpush(ready, raw);
477
+ return this.normalizeEntries(entries);
478
+ }
479
+ async selectStuck() {
480
+ try {
481
+ const res = await this.runScript("SelectStuck", [this.stream], [this.group, this.consumer(), String(this.recoveryStuckTasksTimeoutMs), String(this.workerBatchTasksCount), String(this.workerSelectionTimeoutMs)], SelectStuck);
482
+ return Array.isArray(res) ? res : [];
483
+ } catch (err) {
484
+ if (String(err?.message || "").includes("NOGROUP")) {
485
+ await this.createGroup();
486
+ }
359
487
  }
360
- await tx.exec();
361
- return due.length;
488
+ return [];
362
489
  }
363
- }
364
- async enqueue(ready, delayed, payload, delaySec) {
365
- if (!this.checkConnection()) {
366
- throw new Error("Redis connection error.");
490
+ async selectFresh() {
491
+ let entries = [];
492
+ try {
493
+ const res = await this.redis.xreadgroup(
494
+ "GROUP",
495
+ this.group,
496
+ this.consumer(),
497
+ "BLOCK",
498
+ Math.max(2, this.workerLoopIntervalMs | 0),
499
+ "COUNT",
500
+ this.workerBatchTasksCount,
501
+ "STREAMS",
502
+ this.stream,
503
+ ">"
504
+ );
505
+ if (!res?.[0]?.[1]?.length) {
506
+ return [];
507
+ }
508
+ entries = res?.[0]?.[1] ?? [];
509
+ if (!entries?.length) {
510
+ return [];
511
+ }
512
+ } catch (err) {
513
+ if (String(err?.message || "").includes("NOGROUP")) {
514
+ await this.createGroup();
515
+ }
516
+ }
517
+ return entries;
367
518
  }
368
- const raw = this.toPayload(payload);
369
- if (isNumP(delaySec) && delaySec > 0) {
370
- const score = this.nowSec() + delaySec;
371
- return await this.redis?.zadd(delayed, score, raw);
519
+ normalizeEntries(raw) {
520
+ if (!Array.isArray(raw)) {
521
+ return [];
522
+ }
523
+ return Array.from(raw || []).map((e) => {
524
+ const id = Buffer.isBuffer(e?.[0]) ? e[0].toString() : e?.[0];
525
+ const kvRaw = e?.[1] ?? [];
526
+ const kv = Array.isArray(kvRaw) ? kvRaw.map((x) => Buffer.isBuffer(x) ? x.toString() : x) : [];
527
+ return [id, kv, 0, "", ""];
528
+ }).filter(([id, kv]) => typeof id === "string" && id.length > 0 && Array.isArray(kv) && (kv.length & 1) === 0).map(([id, kv]) => {
529
+ const values = this.values(kv);
530
+ const { idemKey = "", createdAt, job, ...data } = this.payload(values);
531
+ return [id, data, createdAt, job, idemKey];
532
+ });
372
533
  }
373
- return await this.redis?.rpush(ready, raw);
374
- }
375
- async extendVisibility(processingVt, raw, visibilitySec) {
376
- const deadline = this.nowSec() + Math.max(1, visibilitySec);
377
- await this.zaddCompatXXCH(processingVt, deadline, raw);
378
- }
379
- run(queueName) {
380
- if (!isStrFilled(queueName)) {
381
- throw new Error("Queue name is not valid.");
382
- }
383
- const r = this.runners.get(queueName) ?? { running: false };
384
- if (r.running) {
385
- throw new Error(`Queue "${queueName}" already started.`);
386
- }
387
- r.running = true;
388
- this.runners.set(queueName, r);
389
- this.loop(queueName, r).catch(() => {
390
- r.running = false;
391
- });
392
- }
393
- stop(queueName) {
394
- const r = this.runners.get(queueName);
395
- if (r) {
396
- r.running = false;
397
- this.runners.delete(queueName);
534
+ consumer() {
535
+ return `${String(this.consumerHost || "host")}:${process.pid}`;
398
536
  }
399
- }
400
- buildTask(data) {
401
- if (!isObjFilled(data)) {
402
- throw new Error("Data property is not valid.");
403
- }
404
- if (!isStrFilled(data.queueName)) {
405
- throw new Error("Queue name is not valid.");
406
- }
407
- return {
408
- queueName: data.queueName,
409
- iterationId: isStrFilled(data.iterationId) ? data.iterationId : uuid(),
410
- iterationLength: Number(data.iterationLength || 0),
411
- id: isStrFilled(data.id) ? data.id : uuid(),
412
- maxAttempts: isNumPZ(data.maxAttempts) ? data.maxAttempts : this.maxAttempts,
413
- currentAttempt: isNumPZ(data.currentAttempt) ? data.currentAttempt : 0,
414
- chain: isObjFilled(data.chain) && isArrFilled(data.chain.queues) && isNumPZ(data.chain.index) ? data.chain : {
415
- queues: [],
416
- index: 0
417
- },
418
- payload: isObjFilled(data.payload) ? data.payload : {},
419
- progress: {
420
- createdAt: Date.now(),
421
- successAt: 0,
422
- errorAt: 0,
423
- failAt: 0,
424
- fatalAt: 0,
425
- retries: [],
426
- chain: [],
427
- ...isObjFilled(data.progress) ? data.progress : {}
428
- },
429
- result: isObjFilled(data.result) ? data.result : {}
430
- };
431
- }
432
- async addTask(data, delaySec) {
433
- const ready = this.readyKey(String(data.queueName));
434
- const delayed = this.delayedKey(String(data.queueName));
435
- return await this.enqueue(ready, delayed, this.buildTask(data), isNumP(delaySec) ? delaySec : 0);
436
- }
437
- async addTasks(data) {
438
- if (!this.checkConnection()) {
439
- throw new Error("Redis connection error.");
440
- }
441
- if (!isObjFilled(data) || !isStrFilled(data.queueName)) {
442
- throw new Error("Queue name is not valid.");
443
- }
444
- if (!isArrFilled(data.payloads)) {
445
- return 0;
446
- }
447
- const queueName = String(data.queueName);
448
- const ready = this.readyKey(queueName);
449
- const delayed = this.delayedKey(queueName);
450
- const now = this.nowSec();
451
- const uniformDelay = isNumP(data.delaySec) ? Math.max(0, Number(data.delaySec)) : void 0;
452
- const perItemDelays = isArr(data.delaySec) ? data.delaySec.map((v) => Math.max(0, Number(v || 0))) : void 0;
453
- const batchSize = Math.max(1, Math.min(this.portionLength, 1e3));
454
- let idx = 0, total = 0;
455
- while (idx < data.payloads.length) {
456
- const end = Math.min(idx + batchSize, data.payloads.length);
457
- const tx = this.redis?.multi();
458
- for (let i = idx; i < end; i++) {
459
- const item = data.payloads[i];
460
- let partial;
461
- if (isObjFilled(item) && Object.prototype.hasOwnProperty.call(item, "payload")) {
462
- partial = { ...item, queueName };
537
+ };
538
+ }
539
+
540
+ // src/ExecuteTasks.ts
541
+ function ExecuteTasks(Base2) {
542
+ return class extends mix(Base2, Script) {
543
+ constructor() {
544
+ super(...arguments);
545
+ this.abort = new AbortController();
546
+ this.workerExecuteLockTimeoutMs = 18e4;
547
+ this.workerCacheTaskTimeoutMs = 60;
548
+ this.approveBatchTasksCount = 2e3;
549
+ this.removeOnExecuted = false;
550
+ this.executeBatchAtOnce = false;
551
+ this.executeJobStatus = false;
552
+ this.executeJobStatusTtlSec = 300;
553
+ this.consumerHost = "host";
554
+ this.stream = "stream";
555
+ this.group = "group";
556
+ }
557
+ async onExecute(id, payload, createdAt, job, key) {
558
+ }
559
+ async onExecuted(data) {
560
+ }
561
+ async onSuccess(id, payload, createdAt, job, key) {
562
+ }
563
+ async success(id, payload, createdAt, job, key) {
564
+ if (this.executeJobStatus) {
565
+ await this.status(id, payload, createdAt, job, key);
566
+ }
567
+ await this.onSuccess(id, payload, createdAt, job, key);
568
+ }
569
+ async status(id, payload, createdAt, job, key) {
570
+ const prefix = `s:${this.stream}:`;
571
+ const { ready = 0, ok = 0 } = await this.getMany(prefix);
572
+ await this.redis.setMany([{ key: `${prefix}ready`, value: ready + 1 }, { key: `${prefix}ok`, value: ok + 1 }], this.executeJobStatusTtlSec);
573
+ }
574
+ async execute(tasks) {
575
+ const result = [];
576
+ let contended = 0, promises = [];
577
+ for (const [id, payload, createdAt, job, idemKey] of tasks) {
578
+ if (this.executeBatchAtOnce) {
579
+ promises.push((async () => {
580
+ const r = await this.executeProcess(id, payload, createdAt, job, idemKey);
581
+ if (r.id) {
582
+ result.push(id);
583
+ } else if (r.contended) {
584
+ contended++;
585
+ }
586
+ })());
463
587
  } else {
464
- partial = { queueName, payload: item };
588
+ const r = await this.executeProcess(id, payload, createdAt, job, idemKey);
589
+ if (r.id) {
590
+ result.push(id);
591
+ } else if (r.contended) {
592
+ contended++;
593
+ }
465
594
  }
466
- const task = this.buildTask(partial);
467
- const raw = this.toPayload(task);
468
- let d = 0;
469
- if (isNumP(uniformDelay)) {
470
- d = uniformDelay;
471
- } else if (isArr(perItemDelays)) {
472
- d = Number(perItemDelays[i] || 0);
595
+ }
596
+ try {
597
+ if (this.executeBatchAtOnce && promises.length > 0) {
598
+ await Promise.all(promises);
473
599
  }
474
- if (d > 0) {
475
- tx.zadd(delayed, now + d, raw);
476
- } else {
477
- tx.rpush(ready, raw);
600
+ await this.onExecuted(tasks);
601
+ if ((!Array.isArray(result) || !(result.length > 0)) && contended > tasks.length >> 1) {
602
+ await this.waitAbortable(15 + Math.floor(Math.random() * 35) + Math.min(250, 15 * contended + Math.floor(Math.random() * 40)));
478
603
  }
479
- total++;
604
+ } catch (err) {
480
605
  }
481
- await tx.exec();
482
- idx = end;
606
+ return result;
483
607
  }
484
- return total;
485
- }
486
- async iteration(tasks) {
487
- const tasksProcessed = await this.beforeIterationExecution(tasks);
488
- const limit = Math.max(1, Number(this.concurrency) || 1);
489
- let i = 0;
490
- while (i < tasksProcessed.length) {
491
- const slice = tasksProcessed.slice(i, i + limit);
492
- await Promise.all(slice.map((task) => this.logic(task)));
493
- i += limit;
494
- }
495
- await this.afterIterationExecution(tasksProcessed, tasksProcessed.map((t) => t.result ?? {}));
496
- }
497
- async beforeIterationExecution(data) {
498
- return data;
499
- }
500
- async afterIterationExecution(data, results) {
501
- }
502
- async beforeExecution(task) {
503
- return task;
504
- }
505
- async afterExecution(task, result) {
506
- return result;
507
- }
508
- async execute(task) {
509
- return {};
510
- }
511
- async onRetry(task) {
512
- }
513
- async onError(err, task) {
514
- }
515
- async onFail(err, task) {
516
- }
517
- async onFatal(err, task) {
518
- }
519
- async onSuccess(task, result) {
520
- }
521
- async onChainSuccess(task, result) {
522
- }
523
- async onIterationError(err, queueName) {
524
- }
525
- async logic(task) {
526
- let data = task;
527
- try {
528
- data = await this.beforeExecution(task);
529
- const before = data?.result ?? {};
530
- const after = await this.execute(data);
531
- data.result = {
532
- ...isObjFilled(before) ? before : {},
533
- ...isObjFilled(after) ? after : {}
534
- };
535
- await this.success(data, data.result);
536
- return await this.afterExecution(data, data.result);
537
- } catch (err) {
538
- try {
539
- await this.retry(data);
540
- } catch (err2) {
541
- await this.error(err2, data);
542
- }
543
- } finally {
544
- try {
545
- this.stopHeartbeat(data);
546
- await this.ack(data).catch(() => {
547
- });
548
- } catch {
608
+ async executeProcess(id, payload, createdAt, job, key) {
609
+ if (key) {
610
+ return await this.idempotency(id, payload, createdAt, job, key);
611
+ } else {
612
+ try {
613
+ await this.onExecute(id, payload, createdAt, job, key);
614
+ await this.success(id, payload, createdAt, job, key);
615
+ return { id };
616
+ } catch (err) {
617
+ }
549
618
  }
619
+ return {};
550
620
  }
551
- return {};
552
- }
553
- jitteredBackoffSec(attempt) {
554
- const base = Math.max(1, Number(this.retryBaseSec) || 1);
555
- const maxD = Math.max(base, Number(this.retryMaxSec) || 3600);
556
- const pow = Math.min(maxD, base * Math.pow(2, Math.max(0, attempt - 1)));
557
- const jitter = Math.floor(Math.random() * base);
558
- return Math.min(maxD, pow + jitter);
559
- }
560
- async retry(task) {
561
- if (!isObjFilled(task) || !isStrFilled(task.iterationId) || !isStrFilled(task.id) || !isStrFilled(task.queueName) || !isNumPZ(task.currentAttempt) || !isNumPZ(task.maxAttempts)) {
562
- await this.error(new Error("Task format error."), task);
563
- return;
564
- }
565
- const maxAttempts = task.maxAttempts ?? this.maxAttempts;
566
- try {
567
- if (task.currentAttempt < maxAttempts - 1) {
568
- const taskProcessed = { ...task, currentAttempt: task.currentAttempt + 1 };
569
- const delaySec = this.jitteredBackoffSec(taskProcessed.currentAttempt);
570
- await this.addTask(taskProcessed, delaySec);
571
- await this.onRetry(taskProcessed);
572
- return;
621
+ async approve(ids) {
622
+ if (!Array.isArray(ids) || !(ids.length > 0)) {
623
+ return 0;
573
624
  }
574
- } catch (err) {
575
- await this.fail(err, task);
576
- return;
577
- }
578
- await this.fail(new Error("The attempt limit has been reached."), task);
579
- }
580
- async iterationError(err, queueName, data) {
581
- try {
582
- await this.onIterationError(err, queueName);
583
- } catch (err2) {
584
- }
585
- for (const t of data || []) {
586
- if (isStrFilled(t.id)) {
587
- this.processingRaw.delete(t.id);
625
+ const approveBatchTasksCount = Math.max(500, Math.min(4e3, this.approveBatchTasksCount));
626
+ let total = 0, i = 0;
627
+ while (i < ids.length) {
628
+ const room = Math.min(approveBatchTasksCount, ids.length - i);
629
+ const part = ids.slice(i, i + room);
630
+ const approved = await this.runScript("Approve", [this.stream], [this.group, this.removeOnExecuted ? "1" : "0", ...part], Approve);
631
+ total += Number(approved || 0);
632
+ i += room;
588
633
  }
589
- }
590
- }
591
- async error(err, task) {
592
- try {
593
- await this.addTask({
594
- ...task,
595
- queueName: [task.queueName, task.iterationId, "error", "list"].join(":"),
596
- currentAttempt: 0,
597
- payload: {
598
- ...task.payload,
599
- errorMessage: String(err?.message ?? "")
634
+ return total;
635
+ }
636
+ async idempotency(id, payload, createdAt, job, key) {
637
+ const keys = this.idempotencyKeys(key);
638
+ const allow = await this.idempotencyAllow(keys);
639
+ if (allow === 1) {
640
+ return { id };
641
+ } else if (allow === 0) {
642
+ let ttl = -2;
643
+ try {
644
+ ttl = await this.redis.pttl(keys.startKey);
645
+ } catch (err) {
600
646
  }
647
+ await this.waitAbortable(ttl);
648
+ return { contended: true };
649
+ }
650
+ if (!await this.idempotencyStart(keys)) {
651
+ return { contended: true };
652
+ }
653
+ const heartbeat = this.heartbeat(keys) || (() => {
601
654
  });
602
- await this.onError(err, task);
603
- } catch (err2) {
604
655
  try {
605
- await this.onFatal(err2, task);
606
- } catch {
656
+ await this.onExecute(id, payload, createdAt, job, key);
657
+ await this.idempotencyDone(keys);
658
+ await this.success(id, payload, createdAt, job, key);
659
+ return { id };
660
+ } catch (err) {
661
+ try {
662
+ await this.idempotencyFree(keys);
663
+ } catch (err2) {
664
+ }
665
+ } finally {
666
+ heartbeat();
607
667
  }
608
668
  }
609
- try {
610
- await this.status(task, "error");
611
- } catch {
669
+ idempotencyKeys(key) {
670
+ const prefix = `q:${this.stream.replace(/[^\w:\-]/g, "_")}:`;
671
+ const keyP = key.replace(/[^\w:\-]/g, "_");
672
+ const doneKey = `${prefix}done:${keyP}`;
673
+ const lockKey = `${prefix}lock:${keyP}`;
674
+ const startKey = `${prefix}start:${keyP}`;
675
+ const token = `${this.consumer()}:${Date.now().toString(36)}:${Math.random().toString(36).slice(2)}`;
676
+ return {
677
+ prefix,
678
+ doneKey,
679
+ lockKey,
680
+ startKey,
681
+ token
682
+ };
683
+ }
684
+ async idempotencyAllow(keys) {
685
+ const res = await this.runScript("IdempotencyAllow", [keys.doneKey, keys.lockKey, keys.startKey], [String(this.workerExecuteLockTimeoutMs), keys.token], IdempotencyAllow);
686
+ return Number(res || 0);
612
687
  }
613
- }
614
- async fail(err, task) {
615
- try {
616
- await this.addTask({
617
- ...task,
618
- queueName: [task.queueName, task.iterationId, "fail", "list"].join(":"),
619
- currentAttempt: 0,
620
- payload: {
621
- ...task.payload,
622
- errorMessage: String(err?.message ?? "")
688
+ async idempotencyStart(keys) {
689
+ const res = await this.runScript("IdempotencyStart", [keys.lockKey, keys.startKey], [keys.token, String(this.workerExecuteLockTimeoutMs)], IdempotencyStart);
690
+ return Number(res || 0) === 1;
691
+ }
692
+ async idempotencyDone(keys) {
693
+ await this.runScript("IdempotencyDone", [keys.doneKey, keys.lockKey, keys.startKey], [String(this.workerCacheTaskTimeoutMs), keys.token], IdempotencyDone);
694
+ }
695
+ async idempotencyFree(keys) {
696
+ await this.runScript("IdempotencyFree", [keys.lockKey, keys.startKey], [keys.token], IdempotencyFree);
697
+ }
698
+ async waitAbortable(ttl) {
699
+ return new Promise((resolve) => {
700
+ const signal = this.signal();
701
+ if (signal?.aborted) {
702
+ return resolve();
623
703
  }
704
+ const t = setTimeout(() => {
705
+ if (signal) {
706
+ signal.removeEventListener("abort", onAbort);
707
+ }
708
+ resolve();
709
+ }, ttl > 0 ? 25 + Math.floor(Math.random() * 50) : 5 + Math.floor(Math.random() * 15));
710
+ t.unref?.();
711
+ function onAbort() {
712
+ clearTimeout(t);
713
+ resolve();
714
+ }
715
+ signal?.addEventListener("abort", onAbort, { once: true });
624
716
  });
625
- await this.onFail(err, task);
626
- } catch (err2) {
627
- try {
628
- await this.onFatal(err2, task);
629
- } catch {
630
- }
631
717
  }
632
- try {
633
- await this.status(task, "fail");
634
- } catch {
635
- }
636
- }
637
- async success(task, result) {
638
- const taskProcessed = {
639
- ...task,
640
- progress: {
641
- ...task.progress,
642
- successAt: Date.now()
718
+ heartbeat(keys) {
719
+ if (this.workerExecuteLockTimeoutMs <= 0) {
720
+ return;
643
721
  }
644
- };
645
- try {
646
- if (isObjFilled(taskProcessed.chain) && isArrFilled(taskProcessed.chain.queues) && isNumPZ(taskProcessed.chain.index)) {
647
- const currentIndex = taskProcessed.chain.index;
648
- const newIndex = currentIndex + 1;
649
- taskProcessed.progress.chain.push(Date.now());
650
- if (currentIndex === taskProcessed.chain.queues.length - 1) {
651
- await this.status(taskProcessed, "success");
652
- await this.onChainSuccess(taskProcessed, result);
653
- } else if (newIndex <= taskProcessed.chain.queues.length - 1) {
654
- const newQueueName = taskProcessed.chain.queues[newIndex];
655
- if (isStrFilled(newQueueName)) {
656
- await this.addTask({
657
- ...taskProcessed,
658
- queueName: newQueueName,
659
- currentAttempt: 0,
660
- chain: {
661
- ...taskProcessed.chain,
662
- index: newIndex
663
- },
664
- result
665
- });
666
- } else {
667
- await this.fail(new Error("Next queue format error."), taskProcessed);
722
+ let timer, alive = true, hbFails = 0;
723
+ const workerHeartbeatTimeoutMs = Math.max(1e3, Math.floor(Math.max(5e3, this.workerExecuteLockTimeoutMs | 0) / 4));
724
+ const stop = () => {
725
+ alive = false;
726
+ if (timer) {
727
+ clearTimeout(timer);
728
+ }
729
+ };
730
+ const onAbort = () => stop();
731
+ const signal = this.signal();
732
+ signal?.addEventListener?.("abort", onAbort, { once: true });
733
+ const tick = async () => {
734
+ if (!alive) {
735
+ return;
736
+ }
737
+ try {
738
+ const r = await this.heartbeat(keys);
739
+ hbFails = r ? 0 : hbFails + 1;
740
+ if (hbFails >= 3) {
741
+ throw new Error("Heartbeat lost.");
742
+ }
743
+ } catch {
744
+ hbFails++;
745
+ if (hbFails >= 6) {
746
+ stop();
747
+ return;
668
748
  }
669
749
  }
670
- } else {
671
- await this.status(taskProcessed, "success");
672
- }
673
- await this.onSuccess(taskProcessed, result);
674
- } catch (err) {
675
- try {
676
- await this.status(taskProcessed, "fatal");
677
- } catch {
750
+ timer = setTimeout(tick, workerHeartbeatTimeoutMs).unref?.();
751
+ };
752
+ timer = setTimeout(tick, workerHeartbeatTimeoutMs).unref?.();
753
+ return () => {
754
+ signal?.removeEventListener?.("abort", onAbort);
755
+ stop();
756
+ };
757
+ }
758
+ values(value) {
759
+ const result = {};
760
+ for (let i = 0; i < value.length; i += 2) {
761
+ result[value[i]] = value[i + 1];
678
762
  }
763
+ return result;
764
+ }
765
+ payload(data) {
679
766
  try {
680
- await this.onFatal(err, taskProcessed);
681
- } catch {
767
+ return JSON.parse(data.payload);
768
+ } catch (err) {
682
769
  }
770
+ return data;
771
+ }
772
+ consumer() {
773
+ return `${String(this.consumerHost || "host")}:${process.pid}`;
683
774
  }
775
+ signal() {
776
+ return this.abort.signal;
777
+ }
778
+ };
779
+ }
780
+
781
+ // src/PowerQueues.ts
782
+ var Base = class {
783
+ };
784
+ var PowerQueues = class extends mix(Base, AddTasks, SelectTasks, ExecuteTasks) {
785
+ constructor() {
786
+ super(...arguments);
787
+ this.abort = new AbortController();
684
788
  }
685
- async status(task, category = "success") {
686
- if (!this.checkConnection()) {
687
- throw new Error("Redis connection error.");
688
- }
689
- const processedKey = this.toKeyString(task.queueName, task.iterationId, "processed");
690
- const categoryKey = this.toKeyString(task.queueName, task.iterationId, category);
691
- await this.redis?.incr(processedKey);
692
- await this.redis?.incr(categoryKey);
693
- await this.redis?.expire(processedKey, this.expireStatusSec);
694
- await this.redis?.expire(categoryKey, this.expireStatusSec);
789
+ async onSelected(data) {
790
+ return data;
695
791
  }
696
- async loop(queueName, runner) {
697
- if (!isStrFilled(queueName)) {
698
- throw new Error(`Queue name is not valid: "${queueName}"; Type: "${typeof queueName}".`);
699
- }
700
- const ready = this.readyKey(queueName);
701
- const processing = this.processingKey(queueName);
702
- const processingVt = this.processingVtKey(queueName);
703
- const delayed = this.delayedKey(queueName);
704
- while (runner.running) {
705
- if (!this.checkConnection()) {
706
- await wait(this.iterationTimeout);
707
- continue;
708
- }
709
- try {
710
- await this.promoteDelayed(delayed, ready);
711
- } catch {
712
- }
713
- try {
714
- await this.requeueExpired(processing, processingVt, ready);
715
- } catch {
716
- }
717
- let data = [];
792
+ async onExecute(id, payload, createdAt, job, key) {
793
+ }
794
+ async onExecuted(data) {
795
+ }
796
+ async onSuccess(id, payload, createdAt, job, key) {
797
+ }
798
+ async runQueue() {
799
+ await this.createGroup("0-0");
800
+ await this.consumerLoop();
801
+ }
802
+ async consumerLoop() {
803
+ const signal = this.signal();
804
+ while (!signal?.aborted) {
718
805
  try {
719
- data = await this.data(queueName);
720
- if (!isArrFilled(data)) {
721
- await wait(this.iterationTimeout);
806
+ const tasks = await this.select();
807
+ if (!Array.isArray(tasks) || !(tasks.length > 0)) {
808
+ await wait(600);
722
809
  continue;
723
810
  }
724
- await this.iteration(data);
811
+ const tasksP = await this.onSelected(tasks);
812
+ const ids = await this.execute(Array.isArray(tasksP) && tasksP.length > 0 ? tasksP : tasks);
813
+ if (Array.isArray(ids) && ids.length > 0) {
814
+ await this.approve(ids);
815
+ }
725
816
  } catch (err) {
726
- await this.iterationError(err, queueName, data);
727
- await wait(this.iterationTimeout);
728
- }
729
- }
730
- }
731
- async data(queueName) {
732
- const ready = this.readyKey(queueName);
733
- const processing = this.processingKey(queueName);
734
- const processingVt = this.processingVtKey(queueName);
735
- const raws = await this.reserveMany(ready, processing, processingVt, this.portionLength, this.visibilityTimeoutSec);
736
- if (!isArrFilled(raws)) {
737
- return [];
738
- }
739
- const tasks = [];
740
- for (const raw of raws) {
741
- const obj = this.fromPayload(raw);
742
- if (isObjFilled(obj) && isStrFilled(obj.iterationId) && isStrFilled(obj.id) && isStrFilled(obj.queueName) && isNumPZ(obj.maxAttempts) && isNumPZ(obj.currentAttempt)) {
743
- const t = obj;
744
- this.processingRaw.set(t.id, raw);
745
- this.startHeartbeat(t);
746
- tasks.push(t);
747
- }
748
- }
749
- return tasks;
750
- }
751
- async ack(task) {
752
- try {
753
- const raw = this.processingRaw.get(task.id);
754
- if (!isStrFilled(raw)) {
755
- return;
817
+ await wait(600);
756
818
  }
757
- this.processingRaw.delete(task.id);
758
- await this.ackProcessing(this.processingKey(task.queueName), this.processingVtKey(task.queueName), raw);
759
- } catch {
760
819
  }
761
820
  }
762
821
  };
763
822
  export {
764
- PowerQueue
823
+ PowerQueues
765
824
  };