ts-server-lib 0.0.48
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +1 -0
- package/README.md +8 -0
- package/db/TSJournal.d.ts +108 -0
- package/db/TSJournal.js +229 -0
- package/db/TSMongo.d.ts +103 -0
- package/db/TSMongo.js +516 -0
- package/db/TSRQW.d.ts +625 -0
- package/db/TSRQW.js +1204 -0
- package/db/TSRedis.d.ts +530 -0
- package/db/TSRedis.js +1368 -0
- package/db/TSRedisTB.d.ts +80 -0
- package/db/TSRedisTB.js +178 -0
- package/package.json +85 -0
- package/ussd/TSUssdMenu.d.ts +139 -0
- package/ussd/TSUssdMenu.js +368 -0
- package/ussd/TSUssdScreen.d.ts +58 -0
- package/ussd/TSUssdScreen.js +218 -0
- package/ussd/index.d.ts +3 -0
- package/ussd/index.js +19 -0
- package/ussd/providers/AfricasTalking.d.ts +3 -0
- package/ussd/providers/AfricasTalking.js +17 -0
- package/ussd/providers/AirtelDRC.d.ts +9 -0
- package/ussd/providers/AirtelDRC.js +31 -0
- package/ussd/providers/OrangeDRC.d.ts +5 -0
- package/ussd/providers/OrangeDRC.js +213 -0
- package/ussd/providers/VodacomDRC.d.ts +9 -0
- package/ussd/providers/VodacomDRC.js +48 -0
- package/ussd/providers/_.d.ts +55 -0
- package/ussd/providers/_.js +83 -0
- package/ussd/providers/index.d.ts +13 -0
- package/ussd/providers/index.js +56 -0
- package/utils/TSFifo.d.ts +109 -0
- package/utils/TSFifo.js +145 -0
- package/utils/TSFile.d.ts +36 -0
- package/utils/TSFile.js +244 -0
- package/utils/TSHash.d.ts +19 -0
- package/utils/TSHash.js +71 -0
- package/utils/TSRequest.d.ts +248 -0
- package/utils/TSRequest.js +689 -0
- package/utils/TSStub.d.ts +159 -0
- package/utils/TSStub.js +296 -0
- package/utils/abort.d.ts +18 -0
- package/utils/abort.js +97 -0
- package/utils/mime.json +11358 -0
- package/utils/object-keys.d.ts +39 -0
- package/utils/object-keys.js +52 -0
package/db/TSRQW.js
ADDED
|
@@ -0,0 +1,1204 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
/**
|
|
3
|
+
* TSRQW — Redis Queue Worker (standalone / sentinel / cluster).
|
|
4
|
+
*
|
|
5
|
+
* R = Redis · Q = Queue · W = Worker
|
|
6
|
+
*
|
|
7
|
+
* Single class unifying queue storage (Redis Streams) + worker lifecycle
|
|
8
|
+
* (events, retry, dead-letter, cron schedule, thread pool).
|
|
9
|
+
*
|
|
10
|
+
* Backend: Redis Streams (XADD / XREADGROUP / XAUTOCLAIM / XACK).
|
|
11
|
+
* - All keys hash-tagged → zero CROSSSLOT on any topology.
|
|
12
|
+
* - No Lua scripts, no EVALSHA, no SCRIPT LOAD broadcast.
|
|
13
|
+
* - Consumer-group model: crashed consumers recovered automatically via XAUTOCLAIM.
|
|
14
|
+
* - Built-in dead-letter stream (`{ns:name}:dlq`).
|
|
15
|
+
*
|
|
16
|
+
* Performance:
|
|
17
|
+
* - send: 1 roundtrip (XADD MAXLEN ~)
|
|
18
|
+
* - receive: 1 roundtrip (XREADGROUP); XAUTOCLAIM throttled to reclaimIntervalMs
|
|
19
|
+
* - node-redis v5 auto-pipelines concurrent callers → linear throughput scaling
|
|
20
|
+
* - XAUTOCLAIM sweeps the PEL cursor-to-cursor (`nextId` fed back as the next `start`),
|
|
21
|
+
* per Redis docs — never resets to '0-0' mid-sweep, so a bounded-per-call PEL scan can
|
|
22
|
+
* never leave a static, unreachable tail of stale/repeatedly-failing entries behind
|
|
23
|
+
* while brand-new messages (a separate XREADGROUP '>' path) keep flowing fine.
|
|
24
|
+
*
|
|
25
|
+
* Events: new · ready · completed · retry · exceeded · failed · deleted · error
|
|
26
|
+
*/
|
|
27
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
28
|
+
exports.TSRQW = exports.TSRQWPool = exports.TSRQWEvents = exports.TSRQW_OFFLINE_BUFFER_MAX = void 0;
|
|
29
|
+
exports.applyWorkerDataUpdate = applyWorkerDataUpdate;
|
|
30
|
+
const events_1 = require("events");
|
|
31
|
+
const async_hooks_1 = require("async_hooks");
|
|
32
|
+
const cron_1 = require("cron");
|
|
33
|
+
const os_1 = require("os");
|
|
34
|
+
const worker_threads_1 = require("worker_threads");
|
|
35
|
+
const TSRedis_1 = require("./TSRedis");
|
|
36
|
+
const TSRedisTB_1 = require("./TSRedisTB");
|
|
37
|
+
const TSFifo_1 = require("../utils/TSFifo");
|
|
38
|
+
/**
|
|
39
|
+
* Ceiling on sends buffered while disconnected, per queue instance.
|
|
40
|
+
*
|
|
41
|
+
* Chosen against what it protects, not as a round number: these entries hold the caller's message STRING,
|
|
42
|
+
* so a few hundred thousand of them is real heap, and the buffer only grows while the broker is unreachable
|
|
43
|
+
* — precisely when the process has no way to shed it. 10,000 covers any realistic reconnect (seconds to
|
|
44
|
+
* minutes at normal send rates) while keeping the worst case bounded and observable rather than fatal.
|
|
45
|
+
* Overflow is counted by {@link TSFifo.dropped} and logged once per drain.
|
|
46
|
+
*/
|
|
47
|
+
exports.TSRQW_OFFLINE_BUFFER_MAX = 10_000;
|
|
48
|
+
/**
|
|
49
|
+
* Apply a workerData update — the ONE implementation, run identically by the pool and by every worker.
|
|
50
|
+
*
|
|
51
|
+
* ## Why it lives here and is stringified into the worker
|
|
52
|
+
*
|
|
53
|
+
* The pool keeps `this.wd` (the template every new worker is spawned from) and each worker keeps its own
|
|
54
|
+
* structured-clone. Both must apply an update the same way, and they used to do it via two hand-written copies
|
|
55
|
+
* of the same branching — one in TypeScript, one as a concatenated eval string. That is the worst possible
|
|
56
|
+
* place for a duplicate: a divergence produces workers that disagree with the pool *intermittently*, only for
|
|
57
|
+
* workers spawned after the drift, which is far harder to find than a plain bug. So the worker embeds this
|
|
58
|
+
* function's own source via `toString()`, exactly as it already does for the bootstrap callback.
|
|
59
|
+
*
|
|
60
|
+
* MUST stay self-contained — no closure over module scope, no imports, nothing from `this`. It is serialised
|
|
61
|
+
* and evaluated in another realm, so anything it does not receive as an argument does not exist there. The
|
|
62
|
+
* `__name` shim the worker preamble defines covers the tsx/esbuild wrapper.
|
|
63
|
+
*
|
|
64
|
+
* ## Semantics
|
|
65
|
+
*
|
|
66
|
+
* - `_wd_merge`: two-level merge — `target[k][sk]` gains `patch[sk]`'s fields when both are plain objects,
|
|
67
|
+
* otherwise `target[k]` is assigned outright.
|
|
68
|
+
* - `_wd_delete`: two-level prune — removes each listed field from `target[k][sk]`.
|
|
69
|
+
* - Neither present: FULL REPLACE, which is why this returns the (possibly new) target instead of mutating.
|
|
70
|
+
* A function cannot rebind its caller's variable, and the worker's `workerData` needs rebinding on replace.
|
|
71
|
+
* - With either present, remaining non-special keys are copied over as plain assignments.
|
|
72
|
+
*/
|
|
73
|
+
function applyWorkerDataUpdate(target, task) {
|
|
74
|
+
// Helpers are NESTED, not module-level, so `toString()` carries them into the worker realm with the rest of
|
|
75
|
+
// the function. Extracting them upward would compile fine and then throw at runtime in the worker only.
|
|
76
|
+
function mergeNamespaces(out, merge) {
|
|
77
|
+
for (const k in merge) {
|
|
78
|
+
const patch = merge[k];
|
|
79
|
+
const bothObjects = patch && typeof patch === 'object' && out[k] && typeof out[k] === 'object';
|
|
80
|
+
if (bothObjects) {
|
|
81
|
+
for (const sk in patch)
|
|
82
|
+
out[k][sk] = Object.assign(out[k][sk] || {}, patch[sk]);
|
|
83
|
+
}
|
|
84
|
+
else if (patch !== undefined) {
|
|
85
|
+
out[k] = patch;
|
|
86
|
+
}
|
|
87
|
+
}
|
|
88
|
+
}
|
|
89
|
+
function pruneNamespaces(out, del) {
|
|
90
|
+
for (const k in del) {
|
|
91
|
+
const nsPatch = del[k];
|
|
92
|
+
if (!nsPatch || typeof nsPatch !== 'object' || !out[k] || typeof out[k] !== 'object')
|
|
93
|
+
continue;
|
|
94
|
+
for (const sk in nsPatch) {
|
|
95
|
+
const fields = nsPatch[sk];
|
|
96
|
+
if (!Array.isArray(fields) || !out[k][sk] || typeof out[k][sk] !== 'object')
|
|
97
|
+
continue;
|
|
98
|
+
for (const field of fields)
|
|
99
|
+
delete out[k][sk][field];
|
|
100
|
+
}
|
|
101
|
+
}
|
|
102
|
+
}
|
|
103
|
+
const merge = task._wd_merge;
|
|
104
|
+
const del = task._wd_delete;
|
|
105
|
+
// Neither present = full replace. A fresh object, so the caller rebinding to the return value cannot alias
|
|
106
|
+
// the task it was handed.
|
|
107
|
+
if (!merge && !del)
|
|
108
|
+
return { ...task };
|
|
109
|
+
const out = target && typeof target === 'object' ? target : {};
|
|
110
|
+
if (merge)
|
|
111
|
+
mergeNamespaces(out, merge);
|
|
112
|
+
if (del)
|
|
113
|
+
pruneNamespaces(out, del);
|
|
114
|
+
for (const k in task) {
|
|
115
|
+
if (k !== '_wd_updated' && k !== '_wd_merge' && k !== '_wd_delete')
|
|
116
|
+
out[k] = task[k];
|
|
117
|
+
}
|
|
118
|
+
return out;
|
|
119
|
+
}
|
|
120
|
+
// ─── Events ──────────────────────────────────────────────────────────────────
|
|
121
|
+
var TSRQWEvents;
|
|
122
|
+
(function (TSRQWEvents) {
|
|
123
|
+
TSRQWEvents["new"] = "new";
|
|
124
|
+
TSRQWEvents["ready"] = "ready";
|
|
125
|
+
TSRQWEvents["deleted"] = "deleted";
|
|
126
|
+
TSRQWEvents["completed"] = "completed";
|
|
127
|
+
TSRQWEvents["retry"] = "retry";
|
|
128
|
+
TSRQWEvents["exceeded"] = "exceeded";
|
|
129
|
+
TSRQWEvents["failed"] = "failed";
|
|
130
|
+
TSRQWEvents["error"] = "error";
|
|
131
|
+
})(TSRQWEvents || (exports.TSRQWEvents = TSRQWEvents = {}));
|
|
132
|
+
// ─── Shared error-detection helpers ─────────────────────────────────────────
|
|
133
|
+
/** Compute derived snapshot fields and health from raw Streams data. */
|
|
134
|
+
function buildSnapshot(raw) {
|
|
135
|
+
const { msgs, hiddenmsgs, totalrecv, totalsentExact, lag, consumers, dlqDepth, lastDeliveredMs, oldestPendingMs, lastAddedMs } = raw;
|
|
136
|
+
const backpressure = lag + hiddenmsgs;
|
|
137
|
+
const consumptionRate = totalsentExact > 0 ? (totalrecv / totalsentExact) * 100 : 100;
|
|
138
|
+
const dlqRate = totalsentExact > 0 ? (dlqDepth / totalsentExact) * 100 : 0;
|
|
139
|
+
let health;
|
|
140
|
+
if (totalsentExact === 0 && dlqDepth === 0)
|
|
141
|
+
health = 'empty';
|
|
142
|
+
else if (oldestPendingMs > 0 && Date.now() - oldestPendingMs > 120_000)
|
|
143
|
+
health = 'stuck';
|
|
144
|
+
else if (lag > 0)
|
|
145
|
+
health = 'lagging';
|
|
146
|
+
else
|
|
147
|
+
health = 'healthy';
|
|
148
|
+
return {
|
|
149
|
+
msgs, hiddenmsgs, totalsent: msgs, totalrecv,
|
|
150
|
+
lag, consumers, dlqDepth, totalsentExact,
|
|
151
|
+
lastDeliveredMs, oldestPendingMs, lastAddedMs,
|
|
152
|
+
backpressure, consumptionRate, dlqRate, health
|
|
153
|
+
};
|
|
154
|
+
}
|
|
155
|
+
/** True for any error indicating the stream or consumer group no longer exists. */
|
|
156
|
+
function isStreamMissingError(msg) {
|
|
157
|
+
return msg.includes('NOGROUP') || msg.includes('no such key') || msg.includes('ERR no such key') || msg.includes('WRONGTYPE');
|
|
158
|
+
}
|
|
159
|
+
/** True specifically for NOGROUP (group wiped) — self-heal by resetting `ensured`. */
|
|
160
|
+
function isNoGroupError(msg) {
|
|
161
|
+
return msg.includes('NOGROUP') || msg.includes('no such key');
|
|
162
|
+
}
|
|
163
|
+
/** Read a Redis info field that may be returned as kebab-case or camelCase. */
|
|
164
|
+
function xField(obj, kebab, camel, fallback) {
|
|
165
|
+
const o = obj;
|
|
166
|
+
const v = o?.[kebab] ?? o?.[camel];
|
|
167
|
+
return (v !== undefined && v !== null ? v : fallback);
|
|
168
|
+
}
|
|
169
|
+
/** Parse the entries array out of an xReadGroup result (first stream only). */
|
|
170
|
+
function parseXReadGroupEntries(result) {
|
|
171
|
+
return result?.[0]?.messages ?? [];
|
|
172
|
+
}
|
|
173
|
+
// ─── Thread pool ──────────────────────────────────────────────────────────────
|
|
174
|
+
const kTaskInfo = Symbol('kTaskInfo');
|
|
175
|
+
const kWorkerFreedEvent = Symbol('kWorkerFreedEvent');
|
|
176
|
+
/** Set once a worker has been retired, so `error` + `exit` for the same death replace it only once. */
|
|
177
|
+
const kRetired = Symbol('kRetired');
|
|
178
|
+
class TSRQWPoolTask extends async_hooks_1.AsyncResource {
|
|
179
|
+
callback;
|
|
180
|
+
timer = null;
|
|
181
|
+
/** Retry bookkeeping — the payload is retained so a worker fault can re-run it on a fresh thread. */
|
|
182
|
+
payload = undefined;
|
|
183
|
+
attempt = 0;
|
|
184
|
+
/**
|
|
185
|
+
* A task settles EXACTLY once.
|
|
186
|
+
*
|
|
187
|
+
* Without this, a timeout followed by a late worker reply calls `done` twice, and the second
|
|
188
|
+
* `emitDestroy()` throws from inside a worker `message` handler — turning a recoverable slow parse into
|
|
189
|
+
* an uncaught exception. The promise in `work()` would ignore the second settle; `AsyncResource` does not.
|
|
190
|
+
*/
|
|
191
|
+
settled = false;
|
|
192
|
+
constructor(callback) {
|
|
193
|
+
super('TSRQWPoolTask');
|
|
194
|
+
this.callback = callback;
|
|
195
|
+
}
|
|
196
|
+
/** Arm the per-task ceiling. `unref` so a pending timeout never keeps the process alive. */
|
|
197
|
+
arm(timeoutMs, onTimeout) {
|
|
198
|
+
if (!(timeoutMs > 0))
|
|
199
|
+
return;
|
|
200
|
+
this.timer = setTimeout(onTimeout, timeoutMs);
|
|
201
|
+
this.timer.unref?.();
|
|
202
|
+
}
|
|
203
|
+
done(err, result) {
|
|
204
|
+
if (this.settled)
|
|
205
|
+
return;
|
|
206
|
+
this.settled = true;
|
|
207
|
+
if (this.timer) {
|
|
208
|
+
clearTimeout(this.timer);
|
|
209
|
+
this.timer = null;
|
|
210
|
+
}
|
|
211
|
+
this.runInAsyncScope(this.callback, null, err, result);
|
|
212
|
+
this.emitDestroy();
|
|
213
|
+
}
|
|
214
|
+
/**
|
|
215
|
+
* Retire this attempt WITHOUT settling the caller — the payload is being re-run on a fresh worker.
|
|
216
|
+
*
|
|
217
|
+
* The callback is deliberately not invoked: it belongs to the retry, not to this attempt. Still tears the
|
|
218
|
+
* AsyncResource down, or every retried task would leak an async id.
|
|
219
|
+
*/
|
|
220
|
+
abandon() {
|
|
221
|
+
if (this.settled)
|
|
222
|
+
return;
|
|
223
|
+
this.settled = true;
|
|
224
|
+
if (this.timer) {
|
|
225
|
+
clearTimeout(this.timer);
|
|
226
|
+
this.timer = null;
|
|
227
|
+
}
|
|
228
|
+
this.emitDestroy();
|
|
229
|
+
}
|
|
230
|
+
}
|
|
231
|
+
class TSRQWPool extends events_1.EventEmitter {
|
|
232
|
+
factor;
|
|
233
|
+
callback;
|
|
234
|
+
wd;
|
|
235
|
+
opts;
|
|
236
|
+
workers = [];
|
|
237
|
+
freeWorkers = [];
|
|
238
|
+
/**
|
|
239
|
+
* Tasks re-run after a worker fault (see {@link TSRQWPoolOptions.maxTaskRetries}).
|
|
240
|
+
*
|
|
241
|
+
* Exposed because a retry is invisible to the caller by design — it resolves normally — so without this a
|
|
242
|
+
* pool silently masking repeated worker deaths looks identical to a healthy one. Rising here means threads
|
|
243
|
+
* are dying under load even though nothing is failing.
|
|
244
|
+
*/
|
|
245
|
+
taskRetries = 0;
|
|
246
|
+
/**
|
|
247
|
+
* Accepted-but-unstarted tasks. An O(1) FIFO, not an array: this was drained with `shift()`, which
|
|
248
|
+
* re-indexes every remaining entry, so a pool that fell behind paid O(N) per completion — quadratic over
|
|
249
|
+
* the backlog, and worst exactly when the backlog is deepest. Read the depth via `.size`.
|
|
250
|
+
*/
|
|
251
|
+
tasks = new TSFifo_1.TSFifo();
|
|
252
|
+
closed = false;
|
|
253
|
+
constructor(factor = 1, callback, wd, opts) {
|
|
254
|
+
super();
|
|
255
|
+
this.factor = factor;
|
|
256
|
+
this.callback = callback;
|
|
257
|
+
this.wd = wd;
|
|
258
|
+
this.opts = opts;
|
|
259
|
+
// `factor` is a DIVISOR of the core count, NOT a thread count — see TSRQWPoolOptions.threads for
|
|
260
|
+
// why that distinction has already cost real memory, and prefer `threads` when you want a number.
|
|
261
|
+
const threads = Math.max(1, Math.floor(opts?.threads ?? (0, os_1.availableParallelism)() / factor));
|
|
262
|
+
for (let i = 0; i < threads; i++)
|
|
263
|
+
this._addWorker();
|
|
264
|
+
this.on(kWorkerFreedEvent, () => {
|
|
265
|
+
const item = this.tasks.shift();
|
|
266
|
+
if (item)
|
|
267
|
+
this._runTask(item.task, item.callback, item.attempt ?? 0);
|
|
268
|
+
});
|
|
269
|
+
}
|
|
270
|
+
work = (data) => {
|
|
271
|
+
if (this.closed) {
|
|
272
|
+
return Promise.reject(new Error('TSRQWPool closed'));
|
|
273
|
+
}
|
|
274
|
+
return new Promise((resolve, reject) => this._runTask(data, (err, result) => err ? reject(err) : resolve(result)));
|
|
275
|
+
};
|
|
276
|
+
/**
|
|
277
|
+
* Broadcast a workerData update to every live worker AND record it on the pool.
|
|
278
|
+
*
|
|
279
|
+
* 🔴 The pool record is the whole point. This used to only `postMessage` to `this.workers`, so an update
|
|
280
|
+
* reached the workers alive at that instant and nothing else — while new workers are spawned from `this.wd`
|
|
281
|
+
* (see {@link _addWorker}), which was never touched. The pool grows LAZILY, so any worker created after an
|
|
282
|
+
* update started with pre-update state and could never catch up.
|
|
283
|
+
*
|
|
284
|
+
* Measured in sports-service on 2026-08-10: the AMQP catalog id maps were loaded into the pool exactly once
|
|
285
|
+
* at boot (5,486 entries across `b`/`o`/`ems`), the parse pool then grew to 5 workers under load, and **100 %
|
|
286
|
+
* of odds messages arrived with unresolved wire market keys** (mean 112 per message) because the workers
|
|
287
|
+
* doing the parsing had an empty map. That forced the downstream FIG-3 fallback — a synchronous walk over
|
|
288
|
+
* every market plus two Redis round trips plus a full per-market object rebuild — on EVERY message, on the
|
|
289
|
+
* money path, for what is documented as a rare boot-window fallback. It is also self-sustaining: the reload
|
|
290
|
+
* is fingerprinted on hash length, so with a static catalog it never fires again (466 skips observed).
|
|
291
|
+
*/
|
|
292
|
+
update(data) {
|
|
293
|
+
if (this.closed)
|
|
294
|
+
return;
|
|
295
|
+
data._wd_updated = Date.now();
|
|
296
|
+
// Pool first, then live workers: if the broadcast throws, future workers still get the state, and a worker
|
|
297
|
+
// spawned DURING this call reads an already-updated `this.wd` rather than a stale one.
|
|
298
|
+
this.wd = applyWorkerDataUpdate(this.wd, data);
|
|
299
|
+
this.workers.forEach(w => w.postMessage(data));
|
|
300
|
+
}
|
|
301
|
+
close() {
|
|
302
|
+
if (this.closed)
|
|
303
|
+
return;
|
|
304
|
+
this.closed = true;
|
|
305
|
+
const err = new Error('TSRQWPool closed');
|
|
306
|
+
while (this.tasks.size > 0) {
|
|
307
|
+
this.tasks.shift().callback(err, null);
|
|
308
|
+
}
|
|
309
|
+
for (const w of this.workers) {
|
|
310
|
+
// QUEUED tasks were already rejected above, but IN-FLIGHT ones were not: closing terminated the
|
|
311
|
+
// worker and left every caller mid-parse waiting on a reply that could no longer arrive. Shutdown
|
|
312
|
+
// hung on exactly the work it was waiting to finish.
|
|
313
|
+
const task = w[kTaskInfo];
|
|
314
|
+
w[kTaskInfo] = null;
|
|
315
|
+
w[kRetired] = true;
|
|
316
|
+
if (task)
|
|
317
|
+
task.done(err, null);
|
|
318
|
+
void w.terminate();
|
|
319
|
+
}
|
|
320
|
+
this.workers = [];
|
|
321
|
+
this.freeWorkers = [];
|
|
322
|
+
}
|
|
323
|
+
_finishWorkerMessage(worker, result) {
|
|
324
|
+
const task = worker[kTaskInfo];
|
|
325
|
+
if (!task)
|
|
326
|
+
return;
|
|
327
|
+
const payload = result;
|
|
328
|
+
if (payload && typeof payload === 'object' && payload.error != null) {
|
|
329
|
+
const err = payload.error instanceof Error
|
|
330
|
+
? payload.error
|
|
331
|
+
: new Error(String(payload.error));
|
|
332
|
+
task.done(err, null);
|
|
333
|
+
}
|
|
334
|
+
else {
|
|
335
|
+
task.done(null, result);
|
|
336
|
+
}
|
|
337
|
+
worker[kTaskInfo] = null;
|
|
338
|
+
if (!this.closed) {
|
|
339
|
+
this.freeWorkers.push(worker);
|
|
340
|
+
this.emit(kWorkerFreedEvent);
|
|
341
|
+
}
|
|
342
|
+
}
|
|
343
|
+
_runTask(task, callback, attempt = 0) {
|
|
344
|
+
if (this.closed) {
|
|
345
|
+
callback(new Error('TSRQWPool closed'), null);
|
|
346
|
+
return;
|
|
347
|
+
}
|
|
348
|
+
if (this.freeWorkers.length === 0) {
|
|
349
|
+
this.tasks.push({ task, callback, attempt });
|
|
350
|
+
return;
|
|
351
|
+
}
|
|
352
|
+
const worker = this.freeWorkers.pop();
|
|
353
|
+
const info = new TSRQWPoolTask(callback);
|
|
354
|
+
// Retained so `_retireWorker` can re-run this exact payload on a replacement thread.
|
|
355
|
+
info.payload = task;
|
|
356
|
+
info.attempt = attempt;
|
|
357
|
+
worker[kTaskInfo] = info;
|
|
358
|
+
info.arm(this.opts?.taskTimeoutMs ?? 0, () => this._retireWorker(worker, new Error('TSRQWPool task timeout')));
|
|
359
|
+
// `postMessage` throws on a non-cloneable task. Left unguarded it threw out of the Promise executor in
|
|
360
|
+
// `work()` — which rejected that one caller, but leaked the worker: still holding `kTaskInfo`, already
|
|
361
|
+
// popped off `freeWorkers`, never returned. Retiring settles the task and replaces the worker instead.
|
|
362
|
+
try {
|
|
363
|
+
worker.postMessage(task);
|
|
364
|
+
}
|
|
365
|
+
catch (err) {
|
|
366
|
+
// `retriable: false` — the payload could not be cloned, and it will not clone any better on a fresh
|
|
367
|
+
// thread. Retrying here would burn the whole budget re-proving the same failure.
|
|
368
|
+
this._retireWorker(worker, err, false);
|
|
369
|
+
}
|
|
370
|
+
}
|
|
371
|
+
/**
|
|
372
|
+
* Settle a worker's in-flight task, drop the worker, and spawn a replacement — exactly once per worker.
|
|
373
|
+
*
|
|
374
|
+
* `error` and `exit` both fire for a single death (the error handler terminates, which then emits exit),
|
|
375
|
+
* so without {@link kRetired} the pool would replace one dead worker twice and grow without bound.
|
|
376
|
+
*/
|
|
377
|
+
_retireWorker(worker, err, retriable = true) {
|
|
378
|
+
if (worker[kRetired])
|
|
379
|
+
return;
|
|
380
|
+
worker[kRetired] = true;
|
|
381
|
+
const task = worker[kTaskInfo];
|
|
382
|
+
worker[kTaskInfo] = null;
|
|
383
|
+
// Decided BEFORE the worker is torn down, so the branch cannot depend on pool state that teardown
|
|
384
|
+
// mutates. `closed` blocks a retry outright: close() is already rejecting everything.
|
|
385
|
+
//
|
|
386
|
+
// Written as a nested `if` rather than one boolean expression on purpose: `!!task && … && task.attempt`
|
|
387
|
+
// compiles, but `eslint --fix` rewrites the cast to `Boolean(task)`, which is a CALL and narrows nothing,
|
|
388
|
+
// so the build then fails on `task` being possibly null. This form cannot be broken by that fix.
|
|
389
|
+
const maxRetries = this.opts?.maxTaskRetries ?? 0;
|
|
390
|
+
let willRetry = false;
|
|
391
|
+
if (task) {
|
|
392
|
+
willRetry = retriable && !this.closed && task.attempt < maxRetries;
|
|
393
|
+
// `abandon`, not `done`: the caller is not being settled, this attempt is being replaced by another.
|
|
394
|
+
if (willRetry)
|
|
395
|
+
task.abandon();
|
|
396
|
+
else
|
|
397
|
+
task.done(err, null);
|
|
398
|
+
}
|
|
399
|
+
else if (this.listenerCount('error') > 0) {
|
|
400
|
+
// Guarded: `EventEmitter.emit('error')` with no listener THROWS. An idle worker dying would then take
|
|
401
|
+
// the process down from inside an event handler — strictly worse than the failure it reports, and no
|
|
402
|
+
// caller is waiting on it. Pools that care can subscribe.
|
|
403
|
+
this.emit('error', err);
|
|
404
|
+
}
|
|
405
|
+
const fwIdx = this.freeWorkers.indexOf(worker);
|
|
406
|
+
if (fwIdx >= 0)
|
|
407
|
+
this.freeWorkers.splice(fwIdx, 1);
|
|
408
|
+
const wIdx = this.workers.indexOf(worker);
|
|
409
|
+
if (wIdx >= 0)
|
|
410
|
+
this.workers.splice(wIdx, 1);
|
|
411
|
+
void worker.terminate();
|
|
412
|
+
// `_addWorker` emits `kWorkerFreedEvent`, which drains whatever queued behind the dead worker.
|
|
413
|
+
if (!this.closed)
|
|
414
|
+
this._addWorker();
|
|
415
|
+
// Re-queued AFTER the replacement exists, so it runs immediately instead of sitting in `tasks` waiting
|
|
416
|
+
// for a `kWorkerFreedEvent` that has already fired.
|
|
417
|
+
if (task && willRetry) {
|
|
418
|
+
this.taskRetries++;
|
|
419
|
+
this._runTask(task.payload, task.callback, task.attempt + 1);
|
|
420
|
+
}
|
|
421
|
+
}
|
|
422
|
+
_addWorker() {
|
|
423
|
+
// ONE implementation, embedded by source — see applyWorkerDataUpdate. This replaced ~20 lines of
|
|
424
|
+
// concatenated eval that duplicated the pool's TypeScript branch-for-branch; the two could drift and the
|
|
425
|
+
// symptom would be workers disagreeing with the pool only after a respawn. Reassigns `workerData` because
|
|
426
|
+
// a full replace must rebind it, which a called function cannot do for its caller.
|
|
427
|
+
let ep = `if (task?._wd_updated) { workerData = (${applyWorkerDataUpdate.toString()})(workerData, task); }`;
|
|
428
|
+
ep += `else { if (workerData?.debug) { console.log('TSRQWPool:tick', threadId); } fn(task).then(data => pp.postMessage({ data, threadId })).catch(err => { try { pp.postMessage({ error: err }); } catch (_) {} }); }`;
|
|
429
|
+
let em = `import('worker_threads').then(({ threadId, isMainThread, parentPort: pp, workerData}) => { if(!isMainThread) {`;
|
|
430
|
+
// tsx/esbuild inject __name() when callback.toString() is embedded in eval workers (e.g. atfeed betradar AMQP).
|
|
431
|
+
em += `const __name=(t,n)=>t;`;
|
|
432
|
+
em += `const cb = ${this.callback.toString()};`;
|
|
433
|
+
em += `cb(workerData).then(fn => pp.on('message', task => {${ep}} )).catch(err => { pp.on('message', task => { try { pp.postMessage({ error: err }); } catch (_) {} }); });`;
|
|
434
|
+
em += `} });`;
|
|
435
|
+
if (this.wd?._debug)
|
|
436
|
+
console.log('TSRQWPool:worker', em);
|
|
437
|
+
const worker = new worker_threads_1.Worker(em, {
|
|
438
|
+
eval: true,
|
|
439
|
+
workerData: this.wd,
|
|
440
|
+
// Unset means an unbounded worker heap — see TSRQWPoolOptions.resourceLimits.
|
|
441
|
+
...(this.opts?.resourceLimits ? { resourceLimits: this.opts.resourceLimits } : {})
|
|
442
|
+
});
|
|
443
|
+
worker.on('message', result => {
|
|
444
|
+
this._finishWorkerMessage(worker, result);
|
|
445
|
+
});
|
|
446
|
+
worker.on('error', (err) => {
|
|
447
|
+
this._retireWorker(worker, err);
|
|
448
|
+
});
|
|
449
|
+
// A worker can die WITHOUT emitting `error` — `process.exit()` inside the worker, an OS kill, or an
|
|
450
|
+
// abrupt termination. That path had no handler at all, so the in-flight task was never settled and the
|
|
451
|
+
// worker was never replaced: one such death wedged the pool permanently. `exit` also fires after the
|
|
452
|
+
// `error` handler's own `terminate()`, which `kRetired` absorbs.
|
|
453
|
+
worker.on('exit', (code) => {
|
|
454
|
+
if (this.closed)
|
|
455
|
+
return;
|
|
456
|
+
this._retireWorker(worker, new Error(`TSRQWPool worker exited unexpectedly (code ${code})`));
|
|
457
|
+
});
|
|
458
|
+
this.workers.push(worker);
|
|
459
|
+
this.freeWorkers.push(worker);
|
|
460
|
+
this.emit(kWorkerFreedEvent);
|
|
461
|
+
}
|
|
462
|
+
}
|
|
463
|
+
exports.TSRQWPool = TSRQWPool;
|
|
464
|
+
// ─── TSRQW ────────────────────────────────────────────────────────────────────
|
|
465
|
+
class TSRQW extends events_1.EventEmitter {
|
|
466
|
+
redis;
|
|
467
|
+
qname;
|
|
468
|
+
rawNs;
|
|
469
|
+
ns;
|
|
470
|
+
group;
|
|
471
|
+
consumer;
|
|
472
|
+
maxLen;
|
|
473
|
+
attempts;
|
|
474
|
+
reclaimIntervalMs;
|
|
475
|
+
/**
|
|
476
|
+
* Shared cross-process consumption cap for the handler receive paths (see {@link TSRQWOptions.rateLimit}); null
|
|
477
|
+
* when unconfigured. A {@link TSRedisTBReservation} — the reusable batch/reserve/coalesce layer over `TSRedisTB`.
|
|
478
|
+
*/
|
|
479
|
+
rlReservation;
|
|
480
|
+
connected = false;
|
|
481
|
+
closed = false;
|
|
482
|
+
/**
|
|
483
|
+
* Sends accepted while disconnected, replayed on reconnect.
|
|
484
|
+
*
|
|
485
|
+
* `TSFifo`, not an array, and BOUNDED — this was the sixth instance of the drain defect this class exists
|
|
486
|
+
* to remove. It was drained with `shift()` inside `while (length > 0)`, so replaying N buffered messages
|
|
487
|
+
* cost O(N²) at exactly the worst moment: a reconnect after a long outage is when the buffer is deepest.
|
|
488
|
+
*
|
|
489
|
+
* The cap matters more than the drain. It had none, and `send()` pushes unconditionally while the
|
|
490
|
+
* connection is down, so a broker outage grew this without limit until the process died — trading a
|
|
491
|
+
* recoverable outage for an unrecoverable one. `reject-new` keeps the OLDEST work: this is a work queue,
|
|
492
|
+
* so the messages that have already waited longest are the ones a consumer is most likely still waiting
|
|
493
|
+
* on, and dropping the tail at least preserves ordering of what survives.
|
|
494
|
+
*/
|
|
495
|
+
offlineBuffer = new TSFifo_1.TSFifo({
|
|
496
|
+
maxSize: exports.TSRQW_OFFLINE_BUFFER_MAX,
|
|
497
|
+
overflow: 'reject-new'
|
|
498
|
+
});
|
|
499
|
+
delayedTimers = new Map();
|
|
500
|
+
delaySeq = 0;
|
|
501
|
+
ensured = false;
|
|
502
|
+
ensurePromise = null;
|
|
503
|
+
lastReclaimAt = 0;
|
|
504
|
+
/**
|
|
505
|
+
* XAUTOCLAIM scan cursor — persisted across calls. Redis docs: to sweep an entire PEL you
|
|
506
|
+
* must feed each call's returned `nextId` back in as the next `start` argument; resetting to
|
|
507
|
+
* '0-0' every call re-scans the same bounded prefix of the PEL forever, so any stale entries
|
|
508
|
+
* beyond that prefix (or any that keep failing and get re-claimed) are never reached even
|
|
509
|
+
* though brand-new messages (read via a separate XREADGROUP '>' path) keep flowing fine.
|
|
510
|
+
* '0-0' here means "start of PEL" — Redis itself returns '0-0' as nextId once a full sweep
|
|
511
|
+
* completes, so this naturally resets when the PEL is fully covered or empty.
|
|
512
|
+
*/
|
|
513
|
+
reclaimCursor = '0-0';
|
|
514
|
+
/** Resolves when the consumer group is ready and the first offline drain completes. */
|
|
515
|
+
initialized;
|
|
516
|
+
/** Max delivery attempts before dead-lettering (see {@link TSRQWOptions.attempts}). */
|
|
517
|
+
get maxAttempts() {
|
|
518
|
+
return this.attempts;
|
|
519
|
+
}
|
|
520
|
+
constructor(redis, options = {}) {
|
|
521
|
+
super();
|
|
522
|
+
this.redis = redis;
|
|
523
|
+
this.qname = options.name ?? 'queue';
|
|
524
|
+
this.rawNs = options.ns ?? 'tsq';
|
|
525
|
+
this.ns = this.rawNs + ':';
|
|
526
|
+
this.group = options.group ?? this.rawNs;
|
|
527
|
+
this.consumer = options.consumer ?? `${this.rawNs}-${process.pid}-${Math.random().toString(36).slice(2, 8)}`;
|
|
528
|
+
this.maxLen = options.maxLen ?? 100_000;
|
|
529
|
+
this.attempts = options.attempts ?? 3;
|
|
530
|
+
this.reclaimIntervalMs = options.reclaimIntervalMs ?? 2_000;
|
|
531
|
+
// The reservation derives its own batch from the configured rate; unset ⇒ null ⇒ no throttle.
|
|
532
|
+
this.rlReservation = options.rateLimit ? new TSRedisTB_1.TSRedisTBReservation(new TSRedisTB_1.TSRedisTB(redis, options.rateLimit)) : null;
|
|
533
|
+
this.connected = true;
|
|
534
|
+
this.initialized = this._init();
|
|
535
|
+
}
|
|
536
|
+
// ─── Key helpers ─────────────────────────────────────────────────────────────
|
|
537
|
+
static streamKey(rawNs, name) { return `{${rawNs}:${name}}:stream`; }
|
|
538
|
+
static dlqKey(rawNs, name) { return `{${rawNs}:${name}}:dlq`; }
|
|
539
|
+
_streamKey() { return TSRQW.streamKey(this.rawNs, this.qname); }
|
|
540
|
+
_dlqKey() { return TSRQW.dlqKey(this.rawNs, this.qname); }
|
|
541
|
+
/** Wrap stream commands with transient cluster retry (TRYAGAIN / CLUSTERDOWN / LOADING). */
|
|
542
|
+
_cmd(fn) {
|
|
543
|
+
return (0, TSRedis_1.executeWithClusterRetry)(fn);
|
|
544
|
+
}
|
|
545
|
+
// ─── Lifecycle ────────────────────────────────────────────────────────────────
|
|
546
|
+
async _init() {
|
|
547
|
+
await this._ensureGroup();
|
|
548
|
+
if (this.listenerCount(TSRQWEvents.ready) > 0)
|
|
549
|
+
this.emit(TSRQWEvents.ready);
|
|
550
|
+
await this._drainOffline();
|
|
551
|
+
}
|
|
552
|
+
_ensureGroup() {
|
|
553
|
+
if (this.ensured)
|
|
554
|
+
return Promise.resolve();
|
|
555
|
+
if (!this.ensurePromise) {
|
|
556
|
+
this.ensurePromise = this._createGroup().finally(() => { this.ensurePromise = null; });
|
|
557
|
+
}
|
|
558
|
+
return this.ensurePromise;
|
|
559
|
+
}
|
|
560
|
+
async _createGroup() {
|
|
561
|
+
try {
|
|
562
|
+
await this._cmd(() => this.redis.xGroupCreate(this._streamKey(), this.group, '$', { MKSTREAM: true }));
|
|
563
|
+
}
|
|
564
|
+
catch (err) {
|
|
565
|
+
const msg = err instanceof Error ? err.message : String(err);
|
|
566
|
+
if (!msg.includes('BUSYGROUP'))
|
|
567
|
+
throw err;
|
|
568
|
+
}
|
|
569
|
+
// Register in the namespace discovery index every time the group is (re)created —
|
|
570
|
+
// including NOGROUP self-heal after Redis flush. SADD is idempotent so this is safe
|
|
571
|
+
// to call repeatedly. Swallowed on error so a transient Redis hiccup cannot block startup.
|
|
572
|
+
await this.redis.sAdd(`{${this.rawNs}:QUEUES}`, this.qname).catch(() => { });
|
|
573
|
+
this.ensured = true;
|
|
574
|
+
}
|
|
575
|
+
/** Reset the ensured flag so the next receive call recreates stream+group.
|
|
576
|
+
* Call this when XREADGROUP returns NOGROUP (stream/group wiped, e.g. after Redis flush). */
|
|
577
|
+
resetEnsured() {
|
|
578
|
+
this.ensured = false;
|
|
579
|
+
}
|
|
580
|
+
/** True after {@link close} — further send/receive calls are no-ops or throw. */
|
|
581
|
+
isClosed() {
|
|
582
|
+
return this.closed;
|
|
583
|
+
}
|
|
584
|
+
/**
|
|
585
|
+
* Graceful shutdown: stop delayed sends, optionally remove this consumer from the group.
|
|
586
|
+
* Does not delete stream data — use {@link purge} for admin reset.
|
|
587
|
+
*/
|
|
588
|
+
async close(opts) {
|
|
589
|
+
if (this.closed)
|
|
590
|
+
return;
|
|
591
|
+
this.closed = true;
|
|
592
|
+
this.connected = false;
|
|
593
|
+
for (const timer of this.delayedTimers.values())
|
|
594
|
+
clearTimeout(timer);
|
|
595
|
+
this.delayedTimers.clear();
|
|
596
|
+
if (opts?.removeConsumer !== false) {
|
|
597
|
+
await this.redis.xGroupDelConsumer(this._streamKey(), this.group, this.consumer).catch(() => { });
|
|
598
|
+
}
|
|
599
|
+
}
|
|
600
|
+
_assertOpen() {
|
|
601
|
+
if (this.closed)
|
|
602
|
+
throw new Error('TSRQW closed');
|
|
603
|
+
}
|
|
604
|
+
// ─── Send — 1 roundtrip ───────────────────────────────────────────────────────
|
|
605
|
+
/**
|
|
606
|
+
* Append a message. Returns the stream entry ID for immediate sends;
|
|
607
|
+
* `delay:{token}` for delayed sends (pass token to {@link cancelDelayedSend});
|
|
608
|
+
* undefined for buffered (pre-connect) sends.
|
|
609
|
+
*/
|
|
610
|
+
async send(message, delay = 0) {
|
|
611
|
+
if (this.closed)
|
|
612
|
+
return undefined;
|
|
613
|
+
if (!this.connected) {
|
|
614
|
+
this.offlineBuffer.push({ message, delay });
|
|
615
|
+
return;
|
|
616
|
+
}
|
|
617
|
+
await this._ensureGroup();
|
|
618
|
+
if (delay > 0) {
|
|
619
|
+
const token = ++this.delaySeq;
|
|
620
|
+
const timer = setTimeout(() => {
|
|
621
|
+
this.delayedTimers.delete(token);
|
|
622
|
+
if (!this.closed)
|
|
623
|
+
void this._xadd(message);
|
|
624
|
+
}, delay * 1000);
|
|
625
|
+
timer.unref?.();
|
|
626
|
+
this.delayedTimers.set(token, timer);
|
|
627
|
+
return `delay:${token}`;
|
|
628
|
+
}
|
|
629
|
+
const id = await this._xadd(message);
|
|
630
|
+
if (this.listenerCount(TSRQWEvents.new) > 0)
|
|
631
|
+
this.emit(TSRQWEvents.new, message);
|
|
632
|
+
return id;
|
|
633
|
+
}
|
|
634
|
+
/**
|
|
635
|
+
* Cancel a pending delayed send created by {@link send} (token from `delay:{n}` return value).
|
|
636
|
+
* Returns true when a timer was found and cleared.
|
|
637
|
+
*/
|
|
638
|
+
cancelDelayedSend(token) {
|
|
639
|
+
const id = typeof token === 'string'
|
|
640
|
+
? Number(token.startsWith('delay:') ? token.slice(6) : token)
|
|
641
|
+
: token;
|
|
642
|
+
if (!Number.isFinite(id))
|
|
643
|
+
return false;
|
|
644
|
+
const timer = this.delayedTimers.get(id);
|
|
645
|
+
if (!timer)
|
|
646
|
+
return false;
|
|
647
|
+
clearTimeout(timer);
|
|
648
|
+
this.delayedTimers.delete(id);
|
|
649
|
+
return true;
|
|
650
|
+
}
|
|
651
|
+
_xadd(message) {
|
|
652
|
+
return this._cmd(() => this.redis.xAdd(this._streamKey(), '*', { p: message }, { TRIM: { strategy: 'MAXLEN', strategyModifier: '~', threshold: this.maxLen } }));
|
|
653
|
+
}
|
|
654
|
+
// ─── Receive with handler (hot path) ─────────────────────────────────────────
|
|
655
|
+
/**
|
|
656
|
+
* When a shared consumption cap is configured ({@link TSRQWOptions.rateLimit}), block until a token is free
|
|
657
|
+
* before the message reaches its handler. The message stays in the PEL during the wait (its visibility timeout
|
|
658
|
+
* still protects it against a crash), so the cap holds fleet-wide: N consumers on the same `scope` collectively
|
|
659
|
+
* never exceed `perSecond`. No-op — and never allocates — when no limiter is configured.
|
|
660
|
+
*/
|
|
661
|
+
async _acquireToken() {
|
|
662
|
+
const r = this.rlReservation;
|
|
663
|
+
if (r === null)
|
|
664
|
+
return; // no cap configured — never a blocker
|
|
665
|
+
for (;;) {
|
|
666
|
+
if (this.closed)
|
|
667
|
+
return;
|
|
668
|
+
const { admitted, retryAfterMs } = await r.acquire();
|
|
669
|
+
if (admitted)
|
|
670
|
+
return;
|
|
671
|
+
await new Promise((resolve) => setTimeout(resolve, Math.max(1, Math.min(retryAfterMs || 100, 1_000))));
|
|
672
|
+
}
|
|
673
|
+
}
|
|
674
|
+
async receiveWithStatus({ handle, visibility: vt = 30 }) {
|
|
675
|
+
if (this.closed)
|
|
676
|
+
return { status: 'idle' };
|
|
677
|
+
await this._ensureGroup();
|
|
678
|
+
const raw = (await this._tryReclaim(vt)) ?? (await this._readOne());
|
|
679
|
+
if (!raw)
|
|
680
|
+
return { status: 'idle' };
|
|
681
|
+
const { id, message, rc } = raw;
|
|
682
|
+
const meta = { id, rc, fr: this._msFromId(id) };
|
|
683
|
+
await this._acquireToken();
|
|
684
|
+
const ok = await handle(message, meta);
|
|
685
|
+
if (ok) {
|
|
686
|
+
await this._ack(id);
|
|
687
|
+
if (this.listenerCount(TSRQWEvents.completed) > 0)
|
|
688
|
+
this.emit(TSRQWEvents.completed, { message, meta });
|
|
689
|
+
if (this.listenerCount(TSRQWEvents.deleted) > 0)
|
|
690
|
+
this.emit(TSRQWEvents.deleted, { message, meta });
|
|
691
|
+
return { status: 'completed', message, meta };
|
|
692
|
+
}
|
|
693
|
+
if (rc > this.attempts) {
|
|
694
|
+
await this._moveToDeadLetter(id, message, rc);
|
|
695
|
+
await this._ack(id);
|
|
696
|
+
if (this.listenerCount(TSRQWEvents.exceeded) > 0)
|
|
697
|
+
this.emit(TSRQWEvents.exceeded, { message, meta });
|
|
698
|
+
if (this.listenerCount(TSRQWEvents.failed) > 0)
|
|
699
|
+
this.emit(TSRQWEvents.failed, { message, meta });
|
|
700
|
+
if (this.listenerCount(TSRQWEvents.deleted) > 0)
|
|
701
|
+
this.emit(TSRQWEvents.deleted, { message, meta });
|
|
702
|
+
return { status: 'failed', message, meta };
|
|
703
|
+
}
|
|
704
|
+
if (this.listenerCount(TSRQWEvents.retry) > 0)
|
|
705
|
+
this.emit(TSRQWEvents.retry, { message, meta });
|
|
706
|
+
return { status: 'retry', message, meta };
|
|
707
|
+
}
|
|
708
|
+
async receive(opts) {
|
|
709
|
+
await this.receiveWithStatus(opts);
|
|
710
|
+
}
|
|
711
|
+
/**
|
|
712
|
+
* Blocking receive — XREADGROUP with BLOCK so the connection sleeps until a message
|
|
713
|
+
* arrives, then wakes immediately. Zero polling overhead vs CronJob polling.
|
|
714
|
+
*
|
|
715
|
+
* Preferred consumer pattern for durable/recovery consumers (e.g. integration event
|
|
716
|
+
* streams). Each call blocks for up to `blockMs` ms. On timeout: returns null (no
|
|
717
|
+
* message). On message: calls `handler`, ACKs on success, leaves in PEL on failure.
|
|
718
|
+
*
|
|
719
|
+
* Run in a `while (running) { await q.receiveBlocking(handler) }` loop per channel.
|
|
720
|
+
* The loop is woken by Redis as soon as a message is written — no polling overhead.
|
|
721
|
+
*
|
|
722
|
+
* @param handler - Return true to ACK, false to leave in PEL (retry after vt).
|
|
723
|
+
* @param vt - Visibility timeout seconds.
|
|
724
|
+
* @param blockMs - Max ms to wait for a message (Redis BLOCK option). Default 2 000.
|
|
725
|
+
* Keep short (≤5s) when using a shared cluster client — BLOCK holds
|
|
726
|
+
* the master socket and prevents other commands from executing on it.
|
|
727
|
+
*/
|
|
728
|
+
async receiveBlocking(handler, vt = 30, blockMs = 2_000) {
|
|
729
|
+
if (this.closed)
|
|
730
|
+
return;
|
|
731
|
+
try {
|
|
732
|
+
await this._ensureGroup();
|
|
733
|
+
}
|
|
734
|
+
catch {
|
|
735
|
+
return; // group creation failed — caller loop will retry
|
|
736
|
+
}
|
|
737
|
+
const reclaimed = await this._tryReclaim(vt);
|
|
738
|
+
if (reclaimed) {
|
|
739
|
+
const { id, message, rc } = reclaimed;
|
|
740
|
+
const meta = { id, rc, fr: this._msFromId(id) };
|
|
741
|
+
await this._acquireToken();
|
|
742
|
+
const ok = await handler(message, meta).catch(() => false);
|
|
743
|
+
if (ok) {
|
|
744
|
+
await this._ack(id).catch(() => { });
|
|
745
|
+
return;
|
|
746
|
+
}
|
|
747
|
+
if (rc > this.attempts) {
|
|
748
|
+
await this._moveToDeadLetter(id, message, rc).catch(() => { });
|
|
749
|
+
await this._ack(id).catch(() => { });
|
|
750
|
+
}
|
|
751
|
+
return;
|
|
752
|
+
}
|
|
753
|
+
let result;
|
|
754
|
+
try {
|
|
755
|
+
result = await this._cmd(() => this.redis.xReadGroup(this.group, this.consumer, [{ key: this._streamKey(), id: '>' }], { COUNT: 1, BLOCK: blockMs }));
|
|
756
|
+
}
|
|
757
|
+
catch (err) {
|
|
758
|
+
if (isNoGroupError((0, TSRedis_1.errMsg)(err)))
|
|
759
|
+
this.ensured = false; // stream/group wiped — recreate on next call
|
|
760
|
+
return; // transient error — caller loop will retry
|
|
761
|
+
}
|
|
762
|
+
const entries = parseXReadGroupEntries(result);
|
|
763
|
+
if (!entries.length)
|
|
764
|
+
return; // BLOCK timeout — no message
|
|
765
|
+
const e = entries[0];
|
|
766
|
+
const fr = this._msFromId(e.id);
|
|
767
|
+
const message = String(e.message['p'] ?? '');
|
|
768
|
+
await this._acquireToken();
|
|
769
|
+
const ok = await handler(message, { id: e.id, rc: 1, fr }).catch(() => false);
|
|
770
|
+
if (ok) {
|
|
771
|
+
await this._ack(e.id).catch(() => { });
|
|
772
|
+
}
|
|
773
|
+
}
|
|
774
|
+
// ─── Low-level API (adaptive consumers owning their own ack cycle) ────────────
|
|
775
|
+
/** Pull one raw message without calling a handler or auto-acking. */
|
|
776
|
+
async receiveRaw(vt = 30) {
|
|
777
|
+
if (this.closed)
|
|
778
|
+
return null;
|
|
779
|
+
await this._ensureGroup();
|
|
780
|
+
const raw = (await this._tryReclaim(vt)) ?? (await this._readOne());
|
|
781
|
+
if (!raw)
|
|
782
|
+
return null;
|
|
783
|
+
const fr = this._msFromId(raw.id);
|
|
784
|
+
return { id: raw.id, message: raw.message, rc: raw.rc, fr, sent: fr };
|
|
785
|
+
}
|
|
786
|
+
/**
|
|
787
|
+
* Look up ONE queued message by its stream id WITHOUT consuming it (XRANGE id id) — a non-destructive peek so a
|
|
788
|
+
* caller can pick one specific queued entry on demand (reconcile/verify a known id) instead of draining FIFO.
|
|
789
|
+
* O(1): a direct single-entry range read on the stream's radix tree. Returns null when the id is not on the stream
|
|
790
|
+
* (already acked-and-trimmed, or never existed). Touches neither the consumer group, the PEL, nor delivery order —
|
|
791
|
+
* existing send/receive/ack behavior is completely unchanged. `rc` is 0 (a peek observes no delivery attempt).
|
|
792
|
+
*/
|
|
793
|
+
async lookupById(id) {
|
|
794
|
+
if (this.closed)
|
|
795
|
+
return null;
|
|
796
|
+
const rows = await this.redis.xRange(this._streamKey(), id, id, { COUNT: 1 });
|
|
797
|
+
const row = rows?.[0];
|
|
798
|
+
if (!row)
|
|
799
|
+
return null;
|
|
800
|
+
const fr = this._msFromId(row.id);
|
|
801
|
+
return { id: row.id, message: String(row.message?.p ?? ''), rc: 0, fr, sent: fr };
|
|
802
|
+
}
|
|
803
|
+
/**
|
|
804
|
+
* Take ONE specific queued message by id OUT of the queue: read it, then remove it — XACK (clears the group PEL if
|
|
805
|
+
* it was pending; a no-op otherwise) and XDEL (removes it from the stream) — so the normal FIFO consumers will
|
|
806
|
+
* never deliver or process it. The destructive sibling of {@link lookupById}, for picking a known entry on demand
|
|
807
|
+
* (cancel a queued item, hand one off, settle it early). O(1). Returns the message, or null if the id is not on the
|
|
808
|
+
* stream (already taken / acked-and-trimmed). Only the named entry is touched — send/receive/ack for every other
|
|
809
|
+
* message is unchanged.
|
|
810
|
+
*/
|
|
811
|
+
async claimById(id) {
|
|
812
|
+
this._assertOpen();
|
|
813
|
+
const found = await this.lookupById(id);
|
|
814
|
+
if (!found)
|
|
815
|
+
return null;
|
|
816
|
+
await this._ack(id);
|
|
817
|
+
await this.redis.xDel(this._streamKey(), id);
|
|
818
|
+
return found;
|
|
819
|
+
}
|
|
820
|
+
/** XACK a message by ID. Returns true when the PEL entry was removed. */
|
|
821
|
+
async ack(id) {
|
|
822
|
+
return (await this._ack(id)) > 0;
|
|
823
|
+
}
|
|
824
|
+
/**
|
|
825
|
+
* XACK multiple messages in one roundtrip.
|
|
826
|
+
* All IDs must belong to this stream — guaranteed by the caller holding them from receiveRaw/receiveRawBatch.
|
|
827
|
+
* Returns the number of entries removed from the PEL.
|
|
828
|
+
*/
|
|
829
|
+
async ackBatch(ids) {
|
|
830
|
+
if (ids.length === 0)
|
|
831
|
+
return 0;
|
|
832
|
+
return Number(await this.redis.xAck(this._streamKey(), this.group, ids)) || 0;
|
|
833
|
+
}
|
|
834
|
+
/**
|
|
835
|
+
* Pull up to `count` messages in one roundtrip (XREADGROUP COUNT N).
|
|
836
|
+
* Reclaim path uses a single XPENDING RANGE for all stale entries — no per-message round-trips.
|
|
837
|
+
* Returns an empty array when the queue is idle. Caller owns ack/deadLetter for each message.
|
|
838
|
+
*/
|
|
839
|
+
async receiveRawBatch(vt = 30, count = 10) {
|
|
840
|
+
if (this.closed)
|
|
841
|
+
return [];
|
|
842
|
+
try {
|
|
843
|
+
return await this._receiveRawBatchImpl(vt, count);
|
|
844
|
+
}
|
|
845
|
+
catch (err) {
|
|
846
|
+
if (isNoGroupError((0, TSRedis_1.errMsg)(err))) {
|
|
847
|
+
// Stream or consumer group was wiped (e.g. Redis flush). Reset and retry once.
|
|
848
|
+
this.ensured = false;
|
|
849
|
+
this.reclaimCursor = '0-0';
|
|
850
|
+
return this._receiveRawBatchImpl(vt, count);
|
|
851
|
+
}
|
|
852
|
+
throw err;
|
|
853
|
+
}
|
|
854
|
+
}
|
|
855
|
+
/**
|
|
856
|
+
* Build a message from a stream entry. `fr` and `sent` both derive from the id's timestamp — the two
|
|
857
|
+
* receive paths (reclaim and fresh read) previously restated this mapping independently.
|
|
858
|
+
*/
|
|
859
|
+
_toMessage(id, payload, rc) {
|
|
860
|
+
const fr = this._msFromId(id);
|
|
861
|
+
return { id, message: String(payload ?? ''), rc, fr, sent: fr };
|
|
862
|
+
}
|
|
863
|
+
/** One XPENDING RANGE covering every reclaimed id — a single roundtrip for delivery counts. */
|
|
864
|
+
async _reclaimDeliveryCounts(messages) {
|
|
865
|
+
const deliveryMap = new Map();
|
|
866
|
+
const sortedIds = messages.map(m => m.id).sort();
|
|
867
|
+
try {
|
|
868
|
+
const details = await this._cmd(() => this.redis.xPendingRange(this._streamKey(), this.group, sortedIds[0], sortedIds[sortedIds.length - 1], messages.length));
|
|
869
|
+
for (const d of details) {
|
|
870
|
+
deliveryMap.set(String(d.id ?? ''), Number(d.deliveriesCounter ?? 2));
|
|
871
|
+
}
|
|
872
|
+
}
|
|
873
|
+
catch { /* caller defaults to 2 */ }
|
|
874
|
+
return deliveryMap;
|
|
875
|
+
}
|
|
876
|
+
/** Throttled reclaim — one XAUTOCLAIM COUNT N for all stale entries. Empty until the interval elapses. */
|
|
877
|
+
async _reclaimStaleBatch(vt, count) {
|
|
878
|
+
const now = Date.now();
|
|
879
|
+
if (now - this.lastReclaimAt < this.reclaimIntervalMs)
|
|
880
|
+
return [];
|
|
881
|
+
this.lastReclaimAt = now;
|
|
882
|
+
const claimed = await this._cmd(() => this.redis.xAutoClaim(this._streamKey(), this.group, this.consumer, vt * 1000, this.reclaimCursor, { COUNT: count }));
|
|
883
|
+
this.reclaimCursor = String(claimed?.nextId ?? '0-0');
|
|
884
|
+
// XAUTOCLAIM can return null entries for PEL ids whose underlying stream entry was
|
|
885
|
+
// already trimmed/deleted (see node-redis XAutoClaimRawReply — messages: (Null | Stream)[]).
|
|
886
|
+
// Redis auto-acks these from the PEL; skip rather than crash on null.id.
|
|
887
|
+
const messages = (claimed?.messages ?? []).filter((m) => m != null);
|
|
888
|
+
if (messages.length === 0)
|
|
889
|
+
return [];
|
|
890
|
+
const deliveryMap = await this._reclaimDeliveryCounts(messages);
|
|
891
|
+
return messages.map(e => this._toMessage(e.id, e.message?.['p'], deliveryMap.get(e.id) ?? 2));
|
|
892
|
+
}
|
|
893
|
+
/** XREADGROUP COUNT (remaining) for new messages — one roundtrip. */
|
|
894
|
+
async _readNewBatch(count) {
|
|
895
|
+
if (count <= 0)
|
|
896
|
+
return [];
|
|
897
|
+
const readResult = await this._cmd(() => this.redis.xReadGroup(this.group, this.consumer, [{ key: this._streamKey(), id: '>' }], { COUNT: count }));
|
|
898
|
+
return parseXReadGroupEntries(readResult).map(e => this._toMessage(e.id, e.message['p'], 1));
|
|
899
|
+
}
|
|
900
|
+
async _receiveRawBatchImpl(vt = 30, count = 10) {
|
|
901
|
+
await this._ensureGroup();
|
|
902
|
+
// Reclaim first, then top up with fresh messages so one call never exceeds `count`.
|
|
903
|
+
const results = await this._reclaimStaleBatch(vt, count);
|
|
904
|
+
results.push(...(await this._readNewBatch(count - results.length)));
|
|
905
|
+
return results;
|
|
906
|
+
}
|
|
907
|
+
/** Move a message to the `:dlq` stream and XACK the source. */
|
|
908
|
+
async deadLetter(id, message, rc) {
|
|
909
|
+
this._assertOpen();
|
|
910
|
+
await this._moveToDeadLetter(id, message, rc);
|
|
911
|
+
await this._ack(id);
|
|
912
|
+
}
|
|
913
|
+
/**
|
|
914
|
+
* Read dead-letter entries (oldest first). Does not remove them — use {@link replayDlq}
|
|
915
|
+
* or {@link purgeDlq} for ops.
|
|
916
|
+
*/
|
|
917
|
+
async receiveDlqBatch(count = 10) {
|
|
918
|
+
const rows = await this.redis.xRange(this._dlqKey(), '-', '+', { COUNT: count });
|
|
919
|
+
if (!rows?.length)
|
|
920
|
+
return [];
|
|
921
|
+
return rows.map((row) => ({
|
|
922
|
+
id: row.id,
|
|
923
|
+
message: String(row.message?.p ?? ''),
|
|
924
|
+
srcId: String(row.message?.src ?? ''),
|
|
925
|
+
rc: Number(row.message?.rc ?? 0)
|
|
926
|
+
}));
|
|
927
|
+
}
|
|
928
|
+
/** Re-enqueue one DLQ entry onto the main stream and remove it from the DLQ. */
|
|
929
|
+
async replayDlq(dlqId) {
|
|
930
|
+
this._assertOpen();
|
|
931
|
+
const rows = await this.redis.xRange(this._dlqKey(), dlqId, dlqId, { COUNT: 1 });
|
|
932
|
+
const row = rows?.[0];
|
|
933
|
+
if (!row)
|
|
934
|
+
return undefined;
|
|
935
|
+
const id = await this._xadd(String(row.message?.p ?? ''));
|
|
936
|
+
await this.redis.xDel(this._dlqKey(), dlqId);
|
|
937
|
+
return id;
|
|
938
|
+
}
|
|
939
|
+
/** Delete all entries from the dead-letter stream. Returns number of entries removed. */
|
|
940
|
+
async purgeDlq() {
|
|
941
|
+
const len = Number(await this.redis.xLen(this._dlqKey()).catch(() => 0)) || 0;
|
|
942
|
+
if (len > 0)
|
|
943
|
+
await this.redis.del(this._dlqKey()).catch(() => { });
|
|
944
|
+
return len;
|
|
945
|
+
}
|
|
946
|
+
/**
|
|
947
|
+
* Admin reset — delete main stream + DLQ keys. Consumer group must be recreated on next use.
|
|
948
|
+
* Calls {@link resetEnsured} automatically.
|
|
949
|
+
*/
|
|
950
|
+
async purge() {
|
|
951
|
+
this._assertOpen();
|
|
952
|
+
await Promise.all([
|
|
953
|
+
this.redis.del(this._streamKey()).catch(() => { }),
|
|
954
|
+
this.redis.del(this._dlqKey()).catch(() => { })
|
|
955
|
+
]);
|
|
956
|
+
this.resetEnsured();
|
|
957
|
+
}
|
|
958
|
+
/**
|
|
959
|
+
* Approximate trim of the main stream (Redis MAXLEN ~). Does not affect the DLQ.
|
|
960
|
+
*/
|
|
961
|
+
async trimStream(approxMaxLen = 0) {
|
|
962
|
+
this._assertOpen();
|
|
963
|
+
await this.redis.xTrim(this._streamKey(), 'MAXLEN', approxMaxLen, { strategyModifier: '~' });
|
|
964
|
+
}
|
|
965
|
+
// ─── Internal primitives ─────────────────────────────────────────────────────
|
|
966
|
+
/** Handle a NOGROUP error: reset ensured flag, re-create group, return null so the caller retries. */
|
|
967
|
+
async _handleNoGroup() {
|
|
968
|
+
this.ensured = false;
|
|
969
|
+
// Stream/group was wiped and is being recreated — the old scan cursor refers to PEL
|
|
970
|
+
// state that no longer exists; restart the sweep from the beginning.
|
|
971
|
+
this.reclaimCursor = '0-0';
|
|
972
|
+
await this._ensureGroup();
|
|
973
|
+
return null;
|
|
974
|
+
}
|
|
975
|
+
/** Throttled reclaim — runs at most once per reclaimIntervalMs. Returns null when skipped or idle. */
|
|
976
|
+
async _tryReclaim(vt) {
|
|
977
|
+
const now = Date.now();
|
|
978
|
+
if (now - this.lastReclaimAt < this.reclaimIntervalMs)
|
|
979
|
+
return null;
|
|
980
|
+
this.lastReclaimAt = now;
|
|
981
|
+
return this._reclaimOne(vt);
|
|
982
|
+
}
|
|
983
|
+
async _readOne() {
|
|
984
|
+
try {
|
|
985
|
+
const result = await this._cmd(() => this.redis.xReadGroup(this.group, this.consumer, [{ key: this._streamKey(), id: '>' }], { COUNT: 1 }));
|
|
986
|
+
const entries = parseXReadGroupEntries(result);
|
|
987
|
+
if (!entries.length)
|
|
988
|
+
return null;
|
|
989
|
+
const e = entries[0];
|
|
990
|
+
return { id: e.id, message: String(e.message['p'] ?? ''), rc: 1 };
|
|
991
|
+
}
|
|
992
|
+
catch (err) {
|
|
993
|
+
if (isNoGroupError((0, TSRedis_1.errMsg)(err)))
|
|
994
|
+
return this._handleNoGroup();
|
|
995
|
+
throw err;
|
|
996
|
+
}
|
|
997
|
+
}
|
|
998
|
+
async _reclaimOne(vt) {
|
|
999
|
+
try {
|
|
1000
|
+
const result = await this._cmd(() => this.redis.xAutoClaim(this._streamKey(), this.group, this.consumer, vt * 1000, this.reclaimCursor, { COUNT: 1 }));
|
|
1001
|
+
this.reclaimCursor = String(result?.nextId ?? '0-0');
|
|
1002
|
+
const messages = result?.messages;
|
|
1003
|
+
const e = messages?.find((m) => m != null);
|
|
1004
|
+
if (!e)
|
|
1005
|
+
return null;
|
|
1006
|
+
const rc = await this._deliveryCount(e.id);
|
|
1007
|
+
return { id: e.id, message: String(e.message?.['p'] ?? ''), rc };
|
|
1008
|
+
}
|
|
1009
|
+
catch (err) {
|
|
1010
|
+
if (isNoGroupError((0, TSRedis_1.errMsg)(err)))
|
|
1011
|
+
return this._handleNoGroup();
|
|
1012
|
+
throw err;
|
|
1013
|
+
}
|
|
1014
|
+
}
|
|
1015
|
+
async _deliveryCount(id) {
|
|
1016
|
+
try {
|
|
1017
|
+
const detail = (await this._cmd(() => this.redis.xPendingRange(this._streamKey(), this.group, id, id, 1)));
|
|
1018
|
+
return Number(detail?.[0]?.deliveriesCounter ?? 2);
|
|
1019
|
+
}
|
|
1020
|
+
catch {
|
|
1021
|
+
return 2;
|
|
1022
|
+
}
|
|
1023
|
+
}
|
|
1024
|
+
_ack(id) {
|
|
1025
|
+
return this._cmd(() => this.redis.xAck(this._streamKey(), this.group, id));
|
|
1026
|
+
}
|
|
1027
|
+
_moveToDeadLetter(id, message, rc) {
|
|
1028
|
+
return this._cmd(() => this.redis.xAdd(this._dlqKey(), '*', { p: message, src: id, rc: String(rc) }, { TRIM: { strategy: 'MAXLEN', strategyModifier: '~', threshold: this.maxLen } }));
|
|
1029
|
+
}
|
|
1030
|
+
_msFromId(id) { return TSRQW._msFromId(id); }
|
|
1031
|
+
// ─── Attributes ──────────────────────────────────────────────────────────────
|
|
1032
|
+
async attributes() {
|
|
1033
|
+
try {
|
|
1034
|
+
const [len, groups] = await Promise.all([
|
|
1035
|
+
this.redis.xLen(this._streamKey()),
|
|
1036
|
+
this.redis.xInfoGroups(this._streamKey())
|
|
1037
|
+
]);
|
|
1038
|
+
const g = groups.find((x) => x.name === this.group);
|
|
1039
|
+
const totalsent = Number(len) || 0;
|
|
1040
|
+
return {
|
|
1041
|
+
msgs: totalsent,
|
|
1042
|
+
hiddenmsgs: Number(g?.pending ?? 0),
|
|
1043
|
+
totalsent,
|
|
1044
|
+
totalrecv: Number(xField(g, 'entries-read', 'entriesRead', 0))
|
|
1045
|
+
};
|
|
1046
|
+
}
|
|
1047
|
+
catch (err) {
|
|
1048
|
+
if (isStreamMissingError((0, TSRedis_1.errMsg)(err)))
|
|
1049
|
+
return null;
|
|
1050
|
+
throw err;
|
|
1051
|
+
}
|
|
1052
|
+
}
|
|
1053
|
+
/**
|
|
1054
|
+
* Rich Streams-native snapshot — for dashboards, platform stats, admin UI.
|
|
1055
|
+
* 4–5 parallel roundtrips (XLEN×2 + XINFO STREAM + XINFO GROUPS + optional XPENDING).
|
|
1056
|
+
* Do NOT call on the hot message path — use attributes() there.
|
|
1057
|
+
*/
|
|
1058
|
+
async snapshot() {
|
|
1059
|
+
try {
|
|
1060
|
+
const [len, dlqLen, info, groups] = await Promise.all([
|
|
1061
|
+
this.redis.xLen(this._streamKey()),
|
|
1062
|
+
this.redis.xLen(this._dlqKey()).catch(() => 0),
|
|
1063
|
+
this.redis.xInfoStream(this._streamKey()),
|
|
1064
|
+
this.redis.xInfoGroups(this._streamKey())
|
|
1065
|
+
]);
|
|
1066
|
+
const g = groups.find((x) => x.name === this.group);
|
|
1067
|
+
const msgs = Number(len) || 0;
|
|
1068
|
+
const dlqDepth = Number(dlqLen) || 0;
|
|
1069
|
+
const totalsentExact = Number(xField(info, 'entries-added', 'entriesAdded', msgs)) || 0;
|
|
1070
|
+
const totalrecv = Number(xField(g, 'entries-read', 'entriesRead', 0));
|
|
1071
|
+
const pendingCount = Number(g?.pending ?? 0);
|
|
1072
|
+
const lag = Number(g?.lag ?? Math.max(0, msgs - totalrecv));
|
|
1073
|
+
const lastDeliveredMs = TSRQW._msFromId(String(xField(g, 'last-delivered-id', 'lastDeliveredId', '0-0')));
|
|
1074
|
+
const lastAddedMs = TSRQW._msFromId(String(xField(info, 'last-generated-id', 'lastGeneratedId', '0-0')));
|
|
1075
|
+
// oldest pending: one extra roundtrip only when there's something in the PEL
|
|
1076
|
+
let oldestPendingMs = 0;
|
|
1077
|
+
if (pendingCount > 0) {
|
|
1078
|
+
try {
|
|
1079
|
+
const oldest = await this.redis.xPendingRange(this._streamKey(), this.group, '-', '+', 1);
|
|
1080
|
+
if (oldest?.length > 0)
|
|
1081
|
+
oldestPendingMs = TSRQW._msFromId(String(oldest[0]?.id ?? '0'));
|
|
1082
|
+
}
|
|
1083
|
+
catch { /* non-fatal — oldest stays 0 */ }
|
|
1084
|
+
}
|
|
1085
|
+
return buildSnapshot({
|
|
1086
|
+
msgs, hiddenmsgs: pendingCount, totalrecv, totalsentExact,
|
|
1087
|
+
lag, consumers: Number(g?.consumers ?? 0), dlqDepth,
|
|
1088
|
+
lastDeliveredMs, oldestPendingMs, lastAddedMs
|
|
1089
|
+
});
|
|
1090
|
+
}
|
|
1091
|
+
catch (err) {
|
|
1092
|
+
if (isStreamMissingError((0, TSRedis_1.errMsg)(err)))
|
|
1093
|
+
return null;
|
|
1094
|
+
throw err;
|
|
1095
|
+
}
|
|
1096
|
+
}
|
|
1097
|
+
// ─── Offline buffer ───────────────────────────────────────────────────────────
|
|
1098
|
+
/**
|
|
1099
|
+
* Replay buffered sends after a reconnect.
|
|
1100
|
+
*
|
|
1101
|
+
* Drains a SNAPSHOT rather than looping on the live buffer, which is a correctness fix, not a tidy-up.
|
|
1102
|
+
* `send()` re-buffers whenever `connected` is false, so the previous `while (length > 0) { shift(); send() }`
|
|
1103
|
+
* livelocked if the connection dropped mid-drain: shift removed an item, send pushed the same item back,
|
|
1104
|
+
* length never reached zero, and the loop span forever re-queueing one message without progress. Taking
|
|
1105
|
+
* the snapshot bounds the work to what was pending when the drain began.
|
|
1106
|
+
*
|
|
1107
|
+
* On a mid-drain disconnect the untried remainder is pushed back IN ORDER and the drain returns; the next
|
|
1108
|
+
* reconnect picks it up. Re-buffering can hit the cap, which is why the drop count is checked here.
|
|
1109
|
+
*/
|
|
1110
|
+
async _drainOffline() {
|
|
1111
|
+
const pending = this.offlineBuffer.drain(this.offlineBuffer.size);
|
|
1112
|
+
const droppedBefore = this.offlineBuffer.dropped;
|
|
1113
|
+
for (let i = 0; i < pending.length; i++) {
|
|
1114
|
+
if (!this.connected || this.closed) {
|
|
1115
|
+
for (let j = i; j < pending.length; j++)
|
|
1116
|
+
this.offlineBuffer.push(pending[j]);
|
|
1117
|
+
break;
|
|
1118
|
+
}
|
|
1119
|
+
await this.send(pending[i].message, pending[i].delay);
|
|
1120
|
+
}
|
|
1121
|
+
const dropped = this.offlineBuffer.dropped - droppedBefore;
|
|
1122
|
+
if (dropped > 0) {
|
|
1123
|
+
// Loud, because it is silent data loss otherwise: `send()` returns undefined for a buffered send, so
|
|
1124
|
+
// a caller cannot tell a queued message from a discarded one.
|
|
1125
|
+
console.error(`[TSRQW] offline buffer overflow — ${dropped} message(s) dropped (cap ${exports.TSRQW_OFFLINE_BUFFER_MAX}). ` +
|
|
1126
|
+
'The broker was unreachable for longer than the buffer covers.');
|
|
1127
|
+
}
|
|
1128
|
+
}
|
|
1129
|
+
// ─── Statics: schedule + thread pool ─────────────────────────────────────────
|
|
1130
|
+
/** List all queue names registered in a namespace (reads from `{ns:QUEUES}` index set). */
|
|
1131
|
+
static async listQueues(redis, ns) {
|
|
1132
|
+
return ((await redis.sMembers(`{${ns}:QUEUES}`)) ?? []).sort();
|
|
1133
|
+
}
|
|
1134
|
+
/**
|
|
1135
|
+
* Read-only aggregate snapshot across ALL consumer groups on a stream — no XGROUP CREATE.
|
|
1136
|
+
* Use from dashboards instead of `new TSRQW(...).snapshot()` to avoid creating phantom groups.
|
|
1137
|
+
*
|
|
1138
|
+
* Aggregation rules (correct for fan-out / multi-group topologies):
|
|
1139
|
+
* - lag : max across groups (worst consumer defines backpressure)
|
|
1140
|
+
* - consumers : sum across groups (total workers active)
|
|
1141
|
+
* - totalrecv : max entries-read across groups (how far the fastest consumer has reached)
|
|
1142
|
+
* - lastDeliveredMs: max last-delivered-id ms (most-recently-delivered across groups)
|
|
1143
|
+
* - dlqRate : dlqDepth / totalsentExact capped at entries-read by the most-advanced group
|
|
1144
|
+
* (avoids 100% false-positive when DLQ accumulated across stream recreations)
|
|
1145
|
+
*
|
|
1146
|
+
* Returns null when the stream does not exist.
|
|
1147
|
+
*/
|
|
1148
|
+
static async snapshotFromGroups(redis, ns, name) {
|
|
1149
|
+
try {
|
|
1150
|
+
const [len, dlqLen, info, groups] = await Promise.all([
|
|
1151
|
+
redis.xLen(TSRQW.streamKey(ns, name)),
|
|
1152
|
+
redis.xLen(TSRQW.dlqKey(ns, name)).catch(() => 0),
|
|
1153
|
+
redis.xInfoStream(TSRQW.streamKey(ns, name)),
|
|
1154
|
+
redis.xInfoGroups(TSRQW.streamKey(ns, name)).catch(() => [])
|
|
1155
|
+
]);
|
|
1156
|
+
const gs = groups;
|
|
1157
|
+
const msgs = Number(len) || 0;
|
|
1158
|
+
const dlqDepth = Number(dlqLen) || 0;
|
|
1159
|
+
const totalsentExact = Number(xField(info, 'entries-added', 'entriesAdded', msgs)) || 0;
|
|
1160
|
+
const lastAddedMs = TSRQW._msFromId(String(xField(info, 'last-generated-id', 'lastGeneratedId', '0-0')));
|
|
1161
|
+
// Exclude zero-activity groups from aggregate — phantom/producer-only groups that
|
|
1162
|
+
// never consumed would inflate lag to the full stream length.
|
|
1163
|
+
const activeGs = gs.filter(g => Number(xField(g, 'entries-read', 'entriesRead', 0)) > 0 || Number(g.pending ?? 0) > 0);
|
|
1164
|
+
const effectiveGs = activeGs.length > 0 ? activeGs : gs;
|
|
1165
|
+
let lag = 0, consumers = 0, totalrecv = 0, lastDeliveredMs = 0, hiddenmsgs = 0;
|
|
1166
|
+
for (const g of effectiveGs) {
|
|
1167
|
+
const gEntriesRead = Number(xField(g, 'entries-read', 'entriesRead', 0));
|
|
1168
|
+
lag = Math.max(lag, Number(g.lag ?? Math.max(0, msgs - gEntriesRead)));
|
|
1169
|
+
consumers += Number(g.consumers ?? 0);
|
|
1170
|
+
totalrecv = Math.max(totalrecv, gEntriesRead);
|
|
1171
|
+
lastDeliveredMs = Math.max(lastDeliveredMs, TSRQW._msFromId(String(xField(g, 'last-delivered-id', 'lastDeliveredId', '0-0'))));
|
|
1172
|
+
hiddenmsgs += Number(g.pending ?? 0);
|
|
1173
|
+
}
|
|
1174
|
+
// dlqRate against entries-read by the most-advanced group to avoid false 100% when
|
|
1175
|
+
// DLQ accumulated across stream recreations (entries-added resets, DLQ does not).
|
|
1176
|
+
const dlqBase = totalrecv > 0 ? totalrecv : totalsentExact;
|
|
1177
|
+
const snap = buildSnapshot({ msgs, hiddenmsgs, totalrecv, totalsentExact, lag, consumers, dlqDepth, lastDeliveredMs, oldestPendingMs: 0, lastAddedMs });
|
|
1178
|
+
return { ...snap, consumptionRate: totalsentExact > 0 ? (totalrecv / totalsentExact) * 100 : 100, dlqRate: dlqBase > 0 ? (dlqDepth / dlqBase) * 100 : 0 };
|
|
1179
|
+
}
|
|
1180
|
+
catch (err) {
|
|
1181
|
+
if (isStreamMissingError((0, TSRedis_1.errMsg)(err)))
|
|
1182
|
+
return null;
|
|
1183
|
+
throw err;
|
|
1184
|
+
}
|
|
1185
|
+
}
|
|
1186
|
+
static _msFromId(id) {
|
|
1187
|
+
const ms = parseInt(id.split('-')[0] ?? '0', 10);
|
|
1188
|
+
return isNaN(ms) ? 0 : ms;
|
|
1189
|
+
}
|
|
1190
|
+
static schedule({ onTick = () => { }, onComplete = null, runOnInit = false, cronTime = '*/9 * * * * *', context = null, start = false, timeZone = 'utc' }) {
|
|
1191
|
+
return new cron_1.CronJob(cronTime, onTick, onComplete, start, timeZone, context, runOnInit);
|
|
1192
|
+
}
|
|
1193
|
+
/**
|
|
1194
|
+
* Create a worker pool.
|
|
1195
|
+
*
|
|
1196
|
+
* `factor` is a DIVISOR of the core count, not a thread count — `factor: 1` (the default) means one
|
|
1197
|
+
* worker PER CORE. Pass `opts.threads` when you want an absolute number, and `opts.resourceLimits`
|
|
1198
|
+
* to bound each worker's heap. See {@link TSRQWPoolOptions}.
|
|
1199
|
+
*/
|
|
1200
|
+
static worker(cb, factor = 1, wd, opts) {
|
|
1201
|
+
return new TSRQWPool(factor, cb, wd, opts);
|
|
1202
|
+
}
|
|
1203
|
+
}
|
|
1204
|
+
exports.TSRQW = TSRQW;
|