ts-server-lib 0.0.48
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +1 -0
- package/README.md +8 -0
- package/db/TSJournal.d.ts +108 -0
- package/db/TSJournal.js +229 -0
- package/db/TSMongo.d.ts +103 -0
- package/db/TSMongo.js +516 -0
- package/db/TSRQW.d.ts +625 -0
- package/db/TSRQW.js +1204 -0
- package/db/TSRedis.d.ts +530 -0
- package/db/TSRedis.js +1368 -0
- package/db/TSRedisTB.d.ts +80 -0
- package/db/TSRedisTB.js +178 -0
- package/package.json +85 -0
- package/ussd/TSUssdMenu.d.ts +139 -0
- package/ussd/TSUssdMenu.js +368 -0
- package/ussd/TSUssdScreen.d.ts +58 -0
- package/ussd/TSUssdScreen.js +218 -0
- package/ussd/index.d.ts +3 -0
- package/ussd/index.js +19 -0
- package/ussd/providers/AfricasTalking.d.ts +3 -0
- package/ussd/providers/AfricasTalking.js +17 -0
- package/ussd/providers/AirtelDRC.d.ts +9 -0
- package/ussd/providers/AirtelDRC.js +31 -0
- package/ussd/providers/OrangeDRC.d.ts +5 -0
- package/ussd/providers/OrangeDRC.js +213 -0
- package/ussd/providers/VodacomDRC.d.ts +9 -0
- package/ussd/providers/VodacomDRC.js +48 -0
- package/ussd/providers/_.d.ts +55 -0
- package/ussd/providers/_.js +83 -0
- package/ussd/providers/index.d.ts +13 -0
- package/ussd/providers/index.js +56 -0
- package/utils/TSFifo.d.ts +109 -0
- package/utils/TSFifo.js +145 -0
- package/utils/TSFile.d.ts +36 -0
- package/utils/TSFile.js +244 -0
- package/utils/TSHash.d.ts +19 -0
- package/utils/TSHash.js +71 -0
- package/utils/TSRequest.d.ts +248 -0
- package/utils/TSRequest.js +689 -0
- package/utils/TSStub.d.ts +159 -0
- package/utils/TSStub.js +296 -0
- package/utils/abort.d.ts +18 -0
- package/utils/abort.js +97 -0
- package/utils/mime.json +11358 -0
- package/utils/object-keys.d.ts +39 -0
- package/utils/object-keys.js +52 -0
package/db/TSRQW.d.ts
ADDED
|
@@ -0,0 +1,625 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* TSRQW — Redis Queue Worker (standalone / sentinel / cluster).
|
|
3
|
+
*
|
|
4
|
+
* R = Redis · Q = Queue · W = Worker
|
|
5
|
+
*
|
|
6
|
+
* Single class unifying queue storage (Redis Streams) + worker lifecycle
|
|
7
|
+
* (events, retry, dead-letter, cron schedule, thread pool).
|
|
8
|
+
*
|
|
9
|
+
* Backend: Redis Streams (XADD / XREADGROUP / XAUTOCLAIM / XACK).
|
|
10
|
+
* - All keys hash-tagged → zero CROSSSLOT on any topology.
|
|
11
|
+
* - No Lua scripts, no EVALSHA, no SCRIPT LOAD broadcast.
|
|
12
|
+
* - Consumer-group model: crashed consumers recovered automatically via XAUTOCLAIM.
|
|
13
|
+
* - Built-in dead-letter stream (`{ns:name}:dlq`).
|
|
14
|
+
*
|
|
15
|
+
* Performance:
|
|
16
|
+
* - send: 1 roundtrip (XADD MAXLEN ~)
|
|
17
|
+
* - receive: 1 roundtrip (XREADGROUP); XAUTOCLAIM throttled to reclaimIntervalMs
|
|
18
|
+
* - node-redis v5 auto-pipelines concurrent callers → linear throughput scaling
|
|
19
|
+
* - XAUTOCLAIM sweeps the PEL cursor-to-cursor (`nextId` fed back as the next `start`),
|
|
20
|
+
* per Redis docs — never resets to '0-0' mid-sweep, so a bounded-per-call PEL scan can
|
|
21
|
+
* never leave a static, unreachable tail of stale/repeatedly-failing entries behind
|
|
22
|
+
* while brand-new messages (a separate XREADGROUP '>' path) keep flowing fine.
|
|
23
|
+
*
|
|
24
|
+
* Events: new · ready · completed · retry · exceeded · failed · deleted · error
|
|
25
|
+
*/
|
|
26
|
+
import { EventEmitter } from 'events';
|
|
27
|
+
import { AsyncResource } from 'async_hooks';
|
|
28
|
+
import { CronJob } from 'cron';
|
|
29
|
+
import { Worker } from 'worker_threads';
|
|
30
|
+
import { type TSRedisClient } from './TSRedis';
|
|
31
|
+
import { TSRedisTBReservation, type TSRedisTBOptions } from './TSRedisTB';
|
|
32
|
+
import { TSFifo } from '../utils/TSFifo';
|
|
33
|
+
/**
|
|
34
|
+
* Ceiling on sends buffered while disconnected, per queue instance.
|
|
35
|
+
*
|
|
36
|
+
* Chosen against what it protects, not as a round number: these entries hold the caller's message STRING,
|
|
37
|
+
* so a few hundred thousand of them is real heap, and the buffer only grows while the broker is unreachable
|
|
38
|
+
* — precisely when the process has no way to shed it. 10,000 covers any realistic reconnect (seconds to
|
|
39
|
+
* minutes at normal send rates) while keeping the worst case bounded and observable rather than fatal.
|
|
40
|
+
* Overflow is counted by {@link TSFifo.dropped} and logged once per drain.
|
|
41
|
+
*/
|
|
42
|
+
export declare const TSRQW_OFFLINE_BUFFER_MAX = 10000;
|
|
43
|
+
/**
|
|
44
|
+
* Apply a workerData update — the ONE implementation, run identically by the pool and by every worker.
|
|
45
|
+
*
|
|
46
|
+
* ## Why it lives here and is stringified into the worker
|
|
47
|
+
*
|
|
48
|
+
* The pool keeps `this.wd` (the template every new worker is spawned from) and each worker keeps its own
|
|
49
|
+
* structured-clone. Both must apply an update the same way, and they used to do it via two hand-written copies
|
|
50
|
+
* of the same branching — one in TypeScript, one as a concatenated eval string. That is the worst possible
|
|
51
|
+
* place for a duplicate: a divergence produces workers that disagree with the pool *intermittently*, only for
|
|
52
|
+
* workers spawned after the drift, which is far harder to find than a plain bug. So the worker embeds this
|
|
53
|
+
* function's own source via `toString()`, exactly as it already does for the bootstrap callback.
|
|
54
|
+
*
|
|
55
|
+
* MUST stay self-contained — no closure over module scope, no imports, nothing from `this`. It is serialised
|
|
56
|
+
* and evaluated in another realm, so anything it does not receive as an argument does not exist there. The
|
|
57
|
+
* `__name` shim the worker preamble defines covers the tsx/esbuild wrapper.
|
|
58
|
+
*
|
|
59
|
+
* ## Semantics
|
|
60
|
+
*
|
|
61
|
+
* - `_wd_merge`: two-level merge — `target[k][sk]` gains `patch[sk]`'s fields when both are plain objects,
|
|
62
|
+
* otherwise `target[k]` is assigned outright.
|
|
63
|
+
* - `_wd_delete`: two-level prune — removes each listed field from `target[k][sk]`.
|
|
64
|
+
* - Neither present: FULL REPLACE, which is why this returns the (possibly new) target instead of mutating.
|
|
65
|
+
* A function cannot rebind its caller's variable, and the worker's `workerData` needs rebinding on replace.
|
|
66
|
+
* - With either present, remaining non-special keys are copied over as plain assignments.
|
|
67
|
+
*/
|
|
68
|
+
export declare function applyWorkerDataUpdate(target: Record<string, any>, task: Record<string, any>): Record<string, any>;
|
|
69
|
+
export declare enum TSRQWEvents {
|
|
70
|
+
new = "new",
|
|
71
|
+
ready = "ready",
|
|
72
|
+
deleted = "deleted",
|
|
73
|
+
completed = "completed",
|
|
74
|
+
retry = "retry",
|
|
75
|
+
exceeded = "exceeded",
|
|
76
|
+
failed = "failed",
|
|
77
|
+
error = "error"
|
|
78
|
+
}
|
|
79
|
+
export type TSRQWStatus = 'idle' | 'completed' | 'retry' | 'failed';
|
|
80
|
+
export type TSRQWHandler = (message: string, meta: {
|
|
81
|
+
id: string;
|
|
82
|
+
rc: number;
|
|
83
|
+
fr: number;
|
|
84
|
+
}) => Promise<boolean | void> | boolean | void;
|
|
85
|
+
export type TSRQWCallback = (data?: unknown) => Promise<(data?: unknown) => Promise<unknown>>;
|
|
86
|
+
/** Payload for {@link TSRQWPool.update} — full replace, {@link _wd_merge} patch, or {@link _wd_delete} prune. */
|
|
87
|
+
export type TSRQWWorkerDataUpdate = Record<string, unknown> & {
|
|
88
|
+
_wd_updated?: number;
|
|
89
|
+
/**
|
|
90
|
+
* Two-level in-worker merge: for each key `k`, merges `patch[sk]` into `workerData[k][sk]`
|
|
91
|
+
* when both sides are plain objects; otherwise assigns `workerData[k] = patch`.
|
|
92
|
+
*/
|
|
93
|
+
_wd_merge?: Record<string, Record<string, unknown>>;
|
|
94
|
+
/**
|
|
95
|
+
* Two-level in-worker delete: for each key `k` and nested namespace `sk`, removes
|
|
96
|
+
* `workerData[k][sk][field]` for every `field` in the array. No-op when absent.
|
|
97
|
+
*/
|
|
98
|
+
_wd_delete?: Record<string, Record<string, string[]>>;
|
|
99
|
+
};
|
|
100
|
+
export interface TSRQWReceiveResult {
|
|
101
|
+
status: TSRQWStatus;
|
|
102
|
+
message?: string;
|
|
103
|
+
meta?: {
|
|
104
|
+
id: string;
|
|
105
|
+
rc: number;
|
|
106
|
+
fr: number;
|
|
107
|
+
};
|
|
108
|
+
}
|
|
109
|
+
export interface TSRQWAttributes {
|
|
110
|
+
msgs: number;
|
|
111
|
+
hiddenmsgs: number;
|
|
112
|
+
totalsent: number;
|
|
113
|
+
totalrecv: number;
|
|
114
|
+
}
|
|
115
|
+
/** Raw message shape returned by receiveRaw() and the adaptive consumer API. */
|
|
116
|
+
export interface TSRQWMessage {
|
|
117
|
+
id: string;
|
|
118
|
+
message: string;
|
|
119
|
+
rc: number;
|
|
120
|
+
fr: number;
|
|
121
|
+
sent: number;
|
|
122
|
+
}
|
|
123
|
+
/** Dead-letter stream entry — fields written by {@link TSRQW.deadLetter}. */
|
|
124
|
+
export interface TSRQWDlqMessage {
|
|
125
|
+
id: string;
|
|
126
|
+
message: string;
|
|
127
|
+
srcId: string;
|
|
128
|
+
rc: number;
|
|
129
|
+
}
|
|
130
|
+
/** Queue health state — derived from snapshot fields, no extra Redis calls. */
|
|
131
|
+
export type TSRQWHealth = 'empty' | 'healthy' | 'lagging' | 'stuck';
|
|
132
|
+
/**
|
|
133
|
+
* Rich Streams-native snapshot — call from dashboards / platform stats.
|
|
134
|
+
* Extends TSRQWAttributes with fields only Redis Streams can provide.
|
|
135
|
+
* NOT for the hot path — use attributes() there (2 roundtrips).
|
|
136
|
+
*/
|
|
137
|
+
export interface TSRQWSnapshot extends TSRQWAttributes {
|
|
138
|
+
/** Messages in stream not yet delivered to this group — real queue backlog. */
|
|
139
|
+
lag: number;
|
|
140
|
+
/** Active consumers in this group (0 = no worker connected). */
|
|
141
|
+
consumers: number;
|
|
142
|
+
/** Dead-letter queue depth — poison messages accumulating. */
|
|
143
|
+
dlqDepth: number;
|
|
144
|
+
/** Exact total-sent counter, unaffected by MAXLEN trim (entries-added, Redis 7.2+). */
|
|
145
|
+
totalsentExact: number;
|
|
146
|
+
/** Unix ms of last message delivered to this group (0 = never). */
|
|
147
|
+
lastDeliveredMs: number;
|
|
148
|
+
/** Unix ms of oldest in-flight (PEL) message (0 = nothing pending). */
|
|
149
|
+
oldestPendingMs: number;
|
|
150
|
+
/** Unix ms of last XADD — 0 if stream has no entries. Idle indicator. */
|
|
151
|
+
lastAddedMs: number;
|
|
152
|
+
/** lag + hiddenmsgs — total messages in the system not yet completed. */
|
|
153
|
+
backpressure: number;
|
|
154
|
+
/** (totalrecv / totalsentExact) * 100 — % of sent messages consumed. 100 = fully drained. */
|
|
155
|
+
consumptionRate: number;
|
|
156
|
+
/** (dlqDepth / totalsentExact) * 100 — % of messages dead-lettered. */
|
|
157
|
+
dlqRate: number;
|
|
158
|
+
/**
|
|
159
|
+
* empty — no messages ever sent.
|
|
160
|
+
* healthy — no lag, nothing stuck.
|
|
161
|
+
* lagging — lag > 0 (messages waiting to be delivered to this group).
|
|
162
|
+
* stuck — oldest pending entry has been in-flight > 2 min.
|
|
163
|
+
*/
|
|
164
|
+
health: TSRQWHealth;
|
|
165
|
+
}
|
|
166
|
+
export interface TSRQWOptions {
|
|
167
|
+
/** Logical queue name (default: 'queue'). */
|
|
168
|
+
name?: string;
|
|
169
|
+
/** Redis key namespace prefix (default: 'tsq'). */
|
|
170
|
+
ns?: string;
|
|
171
|
+
/** Max delivery attempts before dead-lettering (default: 3). */
|
|
172
|
+
attempts?: number;
|
|
173
|
+
/** Consumer group name (default: ns). One group per service. */
|
|
174
|
+
group?: string;
|
|
175
|
+
/** Consumer name (default: auto-generated per process). */
|
|
176
|
+
consumer?: string;
|
|
177
|
+
/** Approximate stream MAXLEN (default: 100_000). */
|
|
178
|
+
maxLen?: number;
|
|
179
|
+
/** How often to run XAUTOCLAIM for stale-consumer recovery (ms, default: 2_000). */
|
|
180
|
+
reclaimIntervalMs?: number;
|
|
181
|
+
/**
|
|
182
|
+
* Optional SHARED consumption cap for the HANDLER-based receive paths ({@link TSRQW.receive} /
|
|
183
|
+
* {@link TSRQW.receiveWithStatus} / {@link TSRQW.receiveBlocking}). When set, a token is drawn from one
|
|
184
|
+
* cross-process {@link TSRedisTB} (keyed by `scope`) before each message reaches its handler, so N consumers on
|
|
185
|
+
* the same `scope` collectively never exceed `perSecond` — the fleet-wide equivalent of a per-process limiter,
|
|
186
|
+
* the place to encode an upstream provider's per-operator cap. Tokens are claimed in batches and reserved
|
|
187
|
+
* locally (concurrent acquirers coalesce onto one round trip), so most messages are admitted with NO round trip.
|
|
188
|
+
*
|
|
189
|
+
* The LOW-LEVEL raw API ({@link TSRQW.receiveRaw} / {@link TSRQW.receiveRawBatch}) does NOT apply this — those
|
|
190
|
+
* consumers own their own ack cycle AND their own pacing (e.g. the adaptive per-driver limiter). Unset ⇒ no
|
|
191
|
+
* throttle (unchanged behavior). Fail-open: with no Redis or `perSecond <= 0` the bucket grants freely, so the
|
|
192
|
+
* queue never stalls on it — the limiter is never a blocker when unset.
|
|
193
|
+
*/
|
|
194
|
+
rateLimit?: TSRedisTBOptions;
|
|
195
|
+
}
|
|
196
|
+
declare const kTaskInfo: unique symbol;
|
|
197
|
+
/** Set once a worker has been retired, so `error` + `exit` for the same death replace it only once. */
|
|
198
|
+
declare const kRetired: unique symbol;
|
|
199
|
+
interface WorkerWithTaskInfo extends Worker {
|
|
200
|
+
[kTaskInfo]?: TSRQWPoolTask | null;
|
|
201
|
+
[kRetired]?: boolean;
|
|
202
|
+
}
|
|
203
|
+
declare class TSRQWPoolTask extends AsyncResource {
|
|
204
|
+
callback: (err: Error | null, result: unknown) => void;
|
|
205
|
+
private timer;
|
|
206
|
+
/** Retry bookkeeping — the payload is retained so a worker fault can re-run it on a fresh thread. */
|
|
207
|
+
payload: unknown;
|
|
208
|
+
attempt: number;
|
|
209
|
+
/**
|
|
210
|
+
* A task settles EXACTLY once.
|
|
211
|
+
*
|
|
212
|
+
* Without this, a timeout followed by a late worker reply calls `done` twice, and the second
|
|
213
|
+
* `emitDestroy()` throws from inside a worker `message` handler — turning a recoverable slow parse into
|
|
214
|
+
* an uncaught exception. The promise in `work()` would ignore the second settle; `AsyncResource` does not.
|
|
215
|
+
*/
|
|
216
|
+
private settled;
|
|
217
|
+
constructor(callback: (err: Error | null, result: unknown) => void);
|
|
218
|
+
/** Arm the per-task ceiling. `unref` so a pending timeout never keeps the process alive. */
|
|
219
|
+
arm(timeoutMs: number, onTimeout: () => void): void;
|
|
220
|
+
done(err: Error | null, result: unknown): void;
|
|
221
|
+
/**
|
|
222
|
+
* Retire this attempt WITHOUT settling the caller — the payload is being re-run on a fresh worker.
|
|
223
|
+
*
|
|
224
|
+
* The callback is deliberately not invoked: it belongs to the retry, not to this attempt. Still tears the
|
|
225
|
+
* AsyncResource down, or every retried task would leak an async id.
|
|
226
|
+
*/
|
|
227
|
+
abandon(): void;
|
|
228
|
+
}
|
|
229
|
+
/**
|
|
230
|
+
* Explicit pool sizing and per-worker memory ceiling.
|
|
231
|
+
*
|
|
232
|
+
* ### Why `threads` exists (a measured incident, 2026-08-05)
|
|
233
|
+
*
|
|
234
|
+
* `factor` is a DIVISOR: `threads = availableParallelism() / factor`. So `factor: 1` — which reads like
|
|
235
|
+
* "one worker" and is also the DEFAULT — means "one worker per core". On a 20-core host that is 20
|
|
236
|
+
* threads. `TSRequest`'s XML parse pool passed `1` with the comment "the pool below is size 1" and got
|
|
237
|
+
* 20 workers, each with an unbounded V8 heap, each parsing ~1 MB schedule documents: RSS climbed
|
|
238
|
+
* 1065 MB → 2015 MB while the main-thread heap stayed flat and GC'd normally.
|
|
239
|
+
*
|
|
240
|
+
* That signature is easy to misread as a leak in the wrong place, because `process.memoryUsage().heapUsed`
|
|
241
|
+
* covers the MAIN thread only — a worker's heap shows up exclusively in RSS.
|
|
242
|
+
*
|
|
243
|
+
* Pass `threads` when you want a count. `factor` stays for callers that genuinely want to scale with the
|
|
244
|
+
* host (`sports-betting`'s system-combination pool, atfeed's AMQP parse pool) and those pass a real
|
|
245
|
+
* divisor deliberately.
|
|
246
|
+
*/
|
|
247
|
+
export interface TSRQWPoolOptions {
|
|
248
|
+
/** Absolute worker count. Takes precedence over `factor`. Use this unless you mean "per N cores". */
|
|
249
|
+
threads?: number;
|
|
250
|
+
/**
|
|
251
|
+
* Per-worker V8 limits, forwarded to `new Worker`. Unset means UNBOUNDED: V8 grows the worker's old
|
|
252
|
+
* generation and does not return pages to the OS, so a long-lived pool ratchets RSS for the process
|
|
253
|
+
* lifetime.
|
|
254
|
+
*
|
|
255
|
+
* Deliberately NOT defaulted here. A blanket ceiling would silently apply to pools whose workloads
|
|
256
|
+
* were never measured (combinatorial bet-system generation can legitimately want hundreds of MB), and
|
|
257
|
+
* exceeding it terminates the worker. Opt in per pool, with a number you can justify.
|
|
258
|
+
*
|
|
259
|
+
* Exceeding the limit is survivable by design: Node emits `ERR_WORKER_OUT_OF_MEMORY` as a worker
|
|
260
|
+
* `error`, which rejects the in-flight task and replaces the worker (see `_addWorker`'s error handler).
|
|
261
|
+
* Callers that fall back on task failure — `parseXmlDocument` re-parses inline — degrade rather than
|
|
262
|
+
* break.
|
|
263
|
+
*/
|
|
264
|
+
resourceLimits?: {
|
|
265
|
+
maxOldGenerationSizeMb?: number;
|
|
266
|
+
maxYoungGenerationSizeMb?: number;
|
|
267
|
+
codeRangeSizeMb?: number;
|
|
268
|
+
stackSizeMb?: number;
|
|
269
|
+
};
|
|
270
|
+
/**
|
|
271
|
+
* Per-task ceiling in ms. Unset (or 0) keeps the original behaviour: a task waits forever.
|
|
272
|
+
*
|
|
273
|
+
* ### Why this exists (2026-08-16)
|
|
274
|
+
*
|
|
275
|
+
* A task settled only on a worker `message` or a worker `error`. Any path that produced NEITHER left the
|
|
276
|
+
* caller's promise pending forever, the worker permanently marked busy, and — because `_runTask` pops the
|
|
277
|
+
* worker off `freeWorkers` — the pool stuck at zero free workers, queueing every later task on a
|
|
278
|
+
* `kWorkerFreedEvent` that could no longer fire. Silent and unrecoverable.
|
|
279
|
+
*
|
|
280
|
+
* At least one such path is reachable from the worker entry point itself: the reply is posted as
|
|
281
|
+
* `fn(task).then(d => pp.postMessage(...)).catch(err => { try { pp.postMessage({error: err}) } catch (_) {} })`,
|
|
282
|
+
* so a non-cloneable RESULT falls into the catch, and a non-cloneable ERROR is then swallowed by the empty
|
|
283
|
+
* inner catch — nothing is ever posted back.
|
|
284
|
+
*
|
|
285
|
+
* A timeout converts that into an ordinary task failure, which callers already handle:
|
|
286
|
+
* `parseXmlDocument` re-parses inline. Set it to a value comfortably above the worst legitimate task, since
|
|
287
|
+
* firing it also RETIRES the worker — a worker that missed a deadline has unknown state.
|
|
288
|
+
*/
|
|
289
|
+
taskTimeoutMs?: number;
|
|
290
|
+
/**
|
|
291
|
+
* How many times to re-run a task whose WORKER faulted. Default 0 — the historical behaviour, where such a
|
|
292
|
+
* task is rejected and the work item is lost.
|
|
293
|
+
*
|
|
294
|
+
* ### The distinction this relies on
|
|
295
|
+
*
|
|
296
|
+
* The pool already separates two unrelated failures, and only one of them is worth retrying:
|
|
297
|
+
*
|
|
298
|
+
* - **worker fault** — the thread died, timed out, or vanished. The payload was never the problem, so the
|
|
299
|
+
* same input on a fresh worker is expected to succeed. Routed through `_retireWorker`. RETRIABLE.
|
|
300
|
+
* - **task error** — the handler rejected, or returned something that could not be cloned. The worker is
|
|
301
|
+
* alive and reports it as a `{ error }` message. Re-running it fails identically. NOT retriable, and it
|
|
302
|
+
* never reaches the retry path.
|
|
303
|
+
*
|
|
304
|
+
* ### Leave it at 0 unless the task is idempotent
|
|
305
|
+
*
|
|
306
|
+
* A retry re-executes the handler. For a pure function — XML parsing, combinatorial generation — that is
|
|
307
|
+
* strictly better than dropping the item. For anything with side effects it can double them, which is worse
|
|
308
|
+
* than the loss it prevents. That is why this is opt-in per pool rather than a default.
|
|
309
|
+
*
|
|
310
|
+
* Bounded on purpose: a task that faults every time (a payload that reliably OOMs the worker) would
|
|
311
|
+
* otherwise respawn threads forever. After the last attempt the task rejects exactly as it does today.
|
|
312
|
+
*/
|
|
313
|
+
maxTaskRetries?: number;
|
|
314
|
+
}
|
|
315
|
+
export declare class TSRQWPool extends EventEmitter {
|
|
316
|
+
factor: number;
|
|
317
|
+
callback: TSRQWCallback;
|
|
318
|
+
wd?: unknown;
|
|
319
|
+
opts?: TSRQWPoolOptions | undefined;
|
|
320
|
+
workers: WorkerWithTaskInfo[];
|
|
321
|
+
freeWorkers: WorkerWithTaskInfo[];
|
|
322
|
+
/**
|
|
323
|
+
* Tasks re-run after a worker fault (see {@link TSRQWPoolOptions.maxTaskRetries}).
|
|
324
|
+
*
|
|
325
|
+
* Exposed because a retry is invisible to the caller by design — it resolves normally — so without this a
|
|
326
|
+
* pool silently masking repeated worker deaths looks identical to a healthy one. Rising here means threads
|
|
327
|
+
* are dying under load even though nothing is failing.
|
|
328
|
+
*/
|
|
329
|
+
taskRetries: number;
|
|
330
|
+
/**
|
|
331
|
+
* Accepted-but-unstarted tasks. An O(1) FIFO, not an array: this was drained with `shift()`, which
|
|
332
|
+
* re-indexes every remaining entry, so a pool that fell behind paid O(N) per completion — quadratic over
|
|
333
|
+
* the backlog, and worst exactly when the backlog is deepest. Read the depth via `.size`.
|
|
334
|
+
*/
|
|
335
|
+
tasks: TSFifo<{
|
|
336
|
+
task: unknown;
|
|
337
|
+
callback: (err: Error | null, result: unknown) => void;
|
|
338
|
+
attempt?: number;
|
|
339
|
+
}>;
|
|
340
|
+
private closed;
|
|
341
|
+
constructor(factor: number | undefined, callback: TSRQWCallback, wd?: unknown, opts?: TSRQWPoolOptions | undefined);
|
|
342
|
+
work: (data: unknown) => Promise<unknown>;
|
|
343
|
+
/**
|
|
344
|
+
* Broadcast a workerData update to every live worker AND record it on the pool.
|
|
345
|
+
*
|
|
346
|
+
* 🔴 The pool record is the whole point. This used to only `postMessage` to `this.workers`, so an update
|
|
347
|
+
* reached the workers alive at that instant and nothing else — while new workers are spawned from `this.wd`
|
|
348
|
+
* (see {@link _addWorker}), which was never touched. The pool grows LAZILY, so any worker created after an
|
|
349
|
+
* update started with pre-update state and could never catch up.
|
|
350
|
+
*
|
|
351
|
+
* Measured in sports-service on 2026-08-10: the AMQP catalog id maps were loaded into the pool exactly once
|
|
352
|
+
* at boot (5,486 entries across `b`/`o`/`ems`), the parse pool then grew to 5 workers under load, and **100 %
|
|
353
|
+
* of odds messages arrived with unresolved wire market keys** (mean 112 per message) because the workers
|
|
354
|
+
* doing the parsing had an empty map. That forced the downstream FIG-3 fallback — a synchronous walk over
|
|
355
|
+
* every market plus two Redis round trips plus a full per-market object rebuild — on EVERY message, on the
|
|
356
|
+
* money path, for what is documented as a rare boot-window fallback. It is also self-sustaining: the reload
|
|
357
|
+
* is fingerprinted on hash length, so with a static catalog it never fires again (466 skips observed).
|
|
358
|
+
*/
|
|
359
|
+
update(data: TSRQWWorkerDataUpdate): void;
|
|
360
|
+
close(): void;
|
|
361
|
+
private _finishWorkerMessage;
|
|
362
|
+
private _runTask;
|
|
363
|
+
/**
|
|
364
|
+
* Settle a worker's in-flight task, drop the worker, and spawn a replacement — exactly once per worker.
|
|
365
|
+
*
|
|
366
|
+
* `error` and `exit` both fire for a single death (the error handler terminates, which then emits exit),
|
|
367
|
+
* so without {@link kRetired} the pool would replace one dead worker twice and grow without bound.
|
|
368
|
+
*/
|
|
369
|
+
private _retireWorker;
|
|
370
|
+
private _addWorker;
|
|
371
|
+
}
|
|
372
|
+
export declare class TSRQW extends EventEmitter {
|
|
373
|
+
protected readonly redis: TSRedisClient;
|
|
374
|
+
protected readonly qname: string;
|
|
375
|
+
protected readonly rawNs: string;
|
|
376
|
+
protected readonly ns: string;
|
|
377
|
+
protected readonly group: string;
|
|
378
|
+
protected readonly consumer: string;
|
|
379
|
+
protected readonly maxLen: number;
|
|
380
|
+
protected readonly attempts: number;
|
|
381
|
+
protected readonly reclaimIntervalMs: number;
|
|
382
|
+
/**
|
|
383
|
+
* Shared cross-process consumption cap for the handler receive paths (see {@link TSRQWOptions.rateLimit}); null
|
|
384
|
+
* when unconfigured. A {@link TSRedisTBReservation} — the reusable batch/reserve/coalesce layer over `TSRedisTB`.
|
|
385
|
+
*/
|
|
386
|
+
protected readonly rlReservation: TSRedisTBReservation | null;
|
|
387
|
+
connected: boolean;
|
|
388
|
+
private closed;
|
|
389
|
+
/**
|
|
390
|
+
* Sends accepted while disconnected, replayed on reconnect.
|
|
391
|
+
*
|
|
392
|
+
* `TSFifo`, not an array, and BOUNDED — this was the sixth instance of the drain defect this class exists
|
|
393
|
+
* to remove. It was drained with `shift()` inside `while (length > 0)`, so replaying N buffered messages
|
|
394
|
+
* cost O(N²) at exactly the worst moment: a reconnect after a long outage is when the buffer is deepest.
|
|
395
|
+
*
|
|
396
|
+
* The cap matters more than the drain. It had none, and `send()` pushes unconditionally while the
|
|
397
|
+
* connection is down, so a broker outage grew this without limit until the process died — trading a
|
|
398
|
+
* recoverable outage for an unrecoverable one. `reject-new` keeps the OLDEST work: this is a work queue,
|
|
399
|
+
* so the messages that have already waited longest are the ones a consumer is most likely still waiting
|
|
400
|
+
* on, and dropping the tail at least preserves ordering of what survives.
|
|
401
|
+
*/
|
|
402
|
+
private offlineBuffer;
|
|
403
|
+
private delayedTimers;
|
|
404
|
+
private delaySeq;
|
|
405
|
+
private ensured;
|
|
406
|
+
private ensurePromise;
|
|
407
|
+
private lastReclaimAt;
|
|
408
|
+
/**
|
|
409
|
+
* XAUTOCLAIM scan cursor — persisted across calls. Redis docs: to sweep an entire PEL you
|
|
410
|
+
* must feed each call's returned `nextId` back in as the next `start` argument; resetting to
|
|
411
|
+
* '0-0' every call re-scans the same bounded prefix of the PEL forever, so any stale entries
|
|
412
|
+
* beyond that prefix (or any that keep failing and get re-claimed) are never reached even
|
|
413
|
+
* though brand-new messages (read via a separate XREADGROUP '>' path) keep flowing fine.
|
|
414
|
+
* '0-0' here means "start of PEL" — Redis itself returns '0-0' as nextId once a full sweep
|
|
415
|
+
* completes, so this naturally resets when the PEL is fully covered or empty.
|
|
416
|
+
*/
|
|
417
|
+
private reclaimCursor;
|
|
418
|
+
/** Resolves when the consumer group is ready and the first offline drain completes. */
|
|
419
|
+
readonly initialized: Promise<void>;
|
|
420
|
+
/** Max delivery attempts before dead-lettering (see {@link TSRQWOptions.attempts}). */
|
|
421
|
+
get maxAttempts(): number;
|
|
422
|
+
constructor(redis: TSRedisClient, options?: TSRQWOptions);
|
|
423
|
+
static streamKey(rawNs: string, name: string): string;
|
|
424
|
+
static dlqKey(rawNs: string, name: string): string;
|
|
425
|
+
private _streamKey;
|
|
426
|
+
private _dlqKey;
|
|
427
|
+
/** Wrap stream commands with transient cluster retry (TRYAGAIN / CLUSTERDOWN / LOADING). */
|
|
428
|
+
private _cmd;
|
|
429
|
+
private _init;
|
|
430
|
+
private _ensureGroup;
|
|
431
|
+
private _createGroup;
|
|
432
|
+
/** Reset the ensured flag so the next receive call recreates stream+group.
|
|
433
|
+
* Call this when XREADGROUP returns NOGROUP (stream/group wiped, e.g. after Redis flush). */
|
|
434
|
+
resetEnsured(): void;
|
|
435
|
+
/** True after {@link close} — further send/receive calls are no-ops or throw. */
|
|
436
|
+
isClosed(): boolean;
|
|
437
|
+
/**
|
|
438
|
+
* Graceful shutdown: stop delayed sends, optionally remove this consumer from the group.
|
|
439
|
+
* Does not delete stream data — use {@link purge} for admin reset.
|
|
440
|
+
*/
|
|
441
|
+
close(opts?: {
|
|
442
|
+
removeConsumer?: boolean;
|
|
443
|
+
}): Promise<void>;
|
|
444
|
+
private _assertOpen;
|
|
445
|
+
/**
|
|
446
|
+
* Append a message. Returns the stream entry ID for immediate sends;
|
|
447
|
+
* `delay:{token}` for delayed sends (pass token to {@link cancelDelayedSend});
|
|
448
|
+
* undefined for buffered (pre-connect) sends.
|
|
449
|
+
*/
|
|
450
|
+
send(message: string, delay?: number): Promise<string | undefined>;
|
|
451
|
+
/**
|
|
452
|
+
* Cancel a pending delayed send created by {@link send} (token from `delay:{n}` return value).
|
|
453
|
+
* Returns true when a timer was found and cleared.
|
|
454
|
+
*/
|
|
455
|
+
cancelDelayedSend(token: number | string): boolean;
|
|
456
|
+
private _xadd;
|
|
457
|
+
/**
|
|
458
|
+
* When a shared consumption cap is configured ({@link TSRQWOptions.rateLimit}), block until a token is free
|
|
459
|
+
* before the message reaches its handler. The message stays in the PEL during the wait (its visibility timeout
|
|
460
|
+
* still protects it against a crash), so the cap holds fleet-wide: N consumers on the same `scope` collectively
|
|
461
|
+
* never exceed `perSecond`. No-op — and never allocates — when no limiter is configured.
|
|
462
|
+
*/
|
|
463
|
+
private _acquireToken;
|
|
464
|
+
receiveWithStatus({ handle, visibility: vt }: {
|
|
465
|
+
handle: TSRQWHandler;
|
|
466
|
+
visibility?: number;
|
|
467
|
+
}): Promise<TSRQWReceiveResult>;
|
|
468
|
+
receive(opts: {
|
|
469
|
+
handle: TSRQWHandler;
|
|
470
|
+
visibility?: number;
|
|
471
|
+
}): Promise<void>;
|
|
472
|
+
/**
|
|
473
|
+
* Blocking receive — XREADGROUP with BLOCK so the connection sleeps until a message
|
|
474
|
+
* arrives, then wakes immediately. Zero polling overhead vs CronJob polling.
|
|
475
|
+
*
|
|
476
|
+
* Preferred consumer pattern for durable/recovery consumers (e.g. integration event
|
|
477
|
+
* streams). Each call blocks for up to `blockMs` ms. On timeout: returns null (no
|
|
478
|
+
* message). On message: calls `handler`, ACKs on success, leaves in PEL on failure.
|
|
479
|
+
*
|
|
480
|
+
* Run in a `while (running) { await q.receiveBlocking(handler) }` loop per channel.
|
|
481
|
+
* The loop is woken by Redis as soon as a message is written — no polling overhead.
|
|
482
|
+
*
|
|
483
|
+
* @param handler - Return true to ACK, false to leave in PEL (retry after vt).
|
|
484
|
+
* @param vt - Visibility timeout seconds.
|
|
485
|
+
* @param blockMs - Max ms to wait for a message (Redis BLOCK option). Default 2 000.
|
|
486
|
+
* Keep short (≤5s) when using a shared cluster client — BLOCK holds
|
|
487
|
+
* the master socket and prevents other commands from executing on it.
|
|
488
|
+
*/
|
|
489
|
+
receiveBlocking(handler: (message: string, meta: {
|
|
490
|
+
id: string;
|
|
491
|
+
rc: number;
|
|
492
|
+
fr: number;
|
|
493
|
+
}) => Promise<boolean>, vt?: number, blockMs?: number): Promise<void>;
|
|
494
|
+
/** Pull one raw message without calling a handler or auto-acking. */
|
|
495
|
+
receiveRaw(vt?: number): Promise<TSRQWMessage | null>;
|
|
496
|
+
/**
|
|
497
|
+
* Look up ONE queued message by its stream id WITHOUT consuming it (XRANGE id id) — a non-destructive peek so a
|
|
498
|
+
* caller can pick one specific queued entry on demand (reconcile/verify a known id) instead of draining FIFO.
|
|
499
|
+
* O(1): a direct single-entry range read on the stream's radix tree. Returns null when the id is not on the stream
|
|
500
|
+
* (already acked-and-trimmed, or never existed). Touches neither the consumer group, the PEL, nor delivery order —
|
|
501
|
+
* existing send/receive/ack behavior is completely unchanged. `rc` is 0 (a peek observes no delivery attempt).
|
|
502
|
+
*/
|
|
503
|
+
lookupById(id: string): Promise<TSRQWMessage | null>;
|
|
504
|
+
/**
|
|
505
|
+
* Take ONE specific queued message by id OUT of the queue: read it, then remove it — XACK (clears the group PEL if
|
|
506
|
+
* it was pending; a no-op otherwise) and XDEL (removes it from the stream) — so the normal FIFO consumers will
|
|
507
|
+
* never deliver or process it. The destructive sibling of {@link lookupById}, for picking a known entry on demand
|
|
508
|
+
* (cancel a queued item, hand one off, settle it early). O(1). Returns the message, or null if the id is not on the
|
|
509
|
+
* stream (already taken / acked-and-trimmed). Only the named entry is touched — send/receive/ack for every other
|
|
510
|
+
* message is unchanged.
|
|
511
|
+
*/
|
|
512
|
+
claimById(id: string): Promise<TSRQWMessage | null>;
|
|
513
|
+
/** XACK a message by ID. Returns true when the PEL entry was removed. */
|
|
514
|
+
ack(id: string): Promise<boolean>;
|
|
515
|
+
/**
|
|
516
|
+
* XACK multiple messages in one roundtrip.
|
|
517
|
+
* All IDs must belong to this stream — guaranteed by the caller holding them from receiveRaw/receiveRawBatch.
|
|
518
|
+
* Returns the number of entries removed from the PEL.
|
|
519
|
+
*/
|
|
520
|
+
ackBatch(ids: string[]): Promise<number>;
|
|
521
|
+
/**
|
|
522
|
+
* Pull up to `count` messages in one roundtrip (XREADGROUP COUNT N).
|
|
523
|
+
* Reclaim path uses a single XPENDING RANGE for all stale entries — no per-message round-trips.
|
|
524
|
+
* Returns an empty array when the queue is idle. Caller owns ack/deadLetter for each message.
|
|
525
|
+
*/
|
|
526
|
+
receiveRawBatch(vt?: number, count?: number): Promise<TSRQWMessage[]>;
|
|
527
|
+
/**
|
|
528
|
+
* Build a message from a stream entry. `fr` and `sent` both derive from the id's timestamp — the two
|
|
529
|
+
* receive paths (reclaim and fresh read) previously restated this mapping independently.
|
|
530
|
+
*/
|
|
531
|
+
private _toMessage;
|
|
532
|
+
/** One XPENDING RANGE covering every reclaimed id — a single roundtrip for delivery counts. */
|
|
533
|
+
private _reclaimDeliveryCounts;
|
|
534
|
+
/** Throttled reclaim — one XAUTOCLAIM COUNT N for all stale entries. Empty until the interval elapses. */
|
|
535
|
+
private _reclaimStaleBatch;
|
|
536
|
+
/** XREADGROUP COUNT (remaining) for new messages — one roundtrip. */
|
|
537
|
+
private _readNewBatch;
|
|
538
|
+
private _receiveRawBatchImpl;
|
|
539
|
+
/** Move a message to the `:dlq` stream and XACK the source. */
|
|
540
|
+
deadLetter(id: string, message: string, rc: number): Promise<void>;
|
|
541
|
+
/**
|
|
542
|
+
* Read dead-letter entries (oldest first). Does not remove them — use {@link replayDlq}
|
|
543
|
+
* or {@link purgeDlq} for ops.
|
|
544
|
+
*/
|
|
545
|
+
receiveDlqBatch(count?: number): Promise<TSRQWDlqMessage[]>;
|
|
546
|
+
/** Re-enqueue one DLQ entry onto the main stream and remove it from the DLQ. */
|
|
547
|
+
replayDlq(dlqId: string): Promise<string | undefined>;
|
|
548
|
+
/** Delete all entries from the dead-letter stream. Returns number of entries removed. */
|
|
549
|
+
purgeDlq(): Promise<number>;
|
|
550
|
+
/**
|
|
551
|
+
* Admin reset — delete main stream + DLQ keys. Consumer group must be recreated on next use.
|
|
552
|
+
* Calls {@link resetEnsured} automatically.
|
|
553
|
+
*/
|
|
554
|
+
purge(): Promise<void>;
|
|
555
|
+
/**
|
|
556
|
+
* Approximate trim of the main stream (Redis MAXLEN ~). Does not affect the DLQ.
|
|
557
|
+
*/
|
|
558
|
+
trimStream(approxMaxLen?: number): Promise<void>;
|
|
559
|
+
/** Handle a NOGROUP error: reset ensured flag, re-create group, return null so the caller retries. */
|
|
560
|
+
private _handleNoGroup;
|
|
561
|
+
/** Throttled reclaim — runs at most once per reclaimIntervalMs. Returns null when skipped or idle. */
|
|
562
|
+
private _tryReclaim;
|
|
563
|
+
private _readOne;
|
|
564
|
+
private _reclaimOne;
|
|
565
|
+
private _deliveryCount;
|
|
566
|
+
private _ack;
|
|
567
|
+
private _moveToDeadLetter;
|
|
568
|
+
private _msFromId;
|
|
569
|
+
attributes(): Promise<TSRQWAttributes | null>;
|
|
570
|
+
/**
|
|
571
|
+
* Rich Streams-native snapshot — for dashboards, platform stats, admin UI.
|
|
572
|
+
* 4–5 parallel roundtrips (XLEN×2 + XINFO STREAM + XINFO GROUPS + optional XPENDING).
|
|
573
|
+
* Do NOT call on the hot message path — use attributes() there.
|
|
574
|
+
*/
|
|
575
|
+
snapshot(): Promise<TSRQWSnapshot | null>;
|
|
576
|
+
/**
|
|
577
|
+
* Replay buffered sends after a reconnect.
|
|
578
|
+
*
|
|
579
|
+
* Drains a SNAPSHOT rather than looping on the live buffer, which is a correctness fix, not a tidy-up.
|
|
580
|
+
* `send()` re-buffers whenever `connected` is false, so the previous `while (length > 0) { shift(); send() }`
|
|
581
|
+
* livelocked if the connection dropped mid-drain: shift removed an item, send pushed the same item back,
|
|
582
|
+
* length never reached zero, and the loop span forever re-queueing one message without progress. Taking
|
|
583
|
+
* the snapshot bounds the work to what was pending when the drain began.
|
|
584
|
+
*
|
|
585
|
+
* On a mid-drain disconnect the untried remainder is pushed back IN ORDER and the drain returns; the next
|
|
586
|
+
* reconnect picks it up. Re-buffering can hit the cap, which is why the drop count is checked here.
|
|
587
|
+
*/
|
|
588
|
+
private _drainOffline;
|
|
589
|
+
/** List all queue names registered in a namespace (reads from `{ns:QUEUES}` index set). */
|
|
590
|
+
static listQueues(redis: TSRedisClient, ns: string): Promise<string[]>;
|
|
591
|
+
/**
|
|
592
|
+
* Read-only aggregate snapshot across ALL consumer groups on a stream — no XGROUP CREATE.
|
|
593
|
+
* Use from dashboards instead of `new TSRQW(...).snapshot()` to avoid creating phantom groups.
|
|
594
|
+
*
|
|
595
|
+
* Aggregation rules (correct for fan-out / multi-group topologies):
|
|
596
|
+
* - lag : max across groups (worst consumer defines backpressure)
|
|
597
|
+
* - consumers : sum across groups (total workers active)
|
|
598
|
+
* - totalrecv : max entries-read across groups (how far the fastest consumer has reached)
|
|
599
|
+
* - lastDeliveredMs: max last-delivered-id ms (most-recently-delivered across groups)
|
|
600
|
+
* - dlqRate : dlqDepth / totalsentExact capped at entries-read by the most-advanced group
|
|
601
|
+
* (avoids 100% false-positive when DLQ accumulated across stream recreations)
|
|
602
|
+
*
|
|
603
|
+
* Returns null when the stream does not exist.
|
|
604
|
+
*/
|
|
605
|
+
static snapshotFromGroups(redis: TSRedisClient, ns: string, name: string): Promise<TSRQWSnapshot | null>;
|
|
606
|
+
static _msFromId(id: string): number;
|
|
607
|
+
static schedule({ onTick, onComplete, runOnInit, cronTime, context, start, timeZone }: {
|
|
608
|
+
onTick?: () => void;
|
|
609
|
+
onComplete?: (() => void) | null;
|
|
610
|
+
runOnInit?: boolean;
|
|
611
|
+
cronTime?: string;
|
|
612
|
+
context?: unknown;
|
|
613
|
+
start?: boolean;
|
|
614
|
+
timeZone?: string;
|
|
615
|
+
}): CronJob;
|
|
616
|
+
/**
|
|
617
|
+
* Create a worker pool.
|
|
618
|
+
*
|
|
619
|
+
* `factor` is a DIVISOR of the core count, not a thread count — `factor: 1` (the default) means one
|
|
620
|
+
* worker PER CORE. Pass `opts.threads` when you want an absolute number, and `opts.resourceLimits`
|
|
621
|
+
* to bound each worker's heap. See {@link TSRQWPoolOptions}.
|
|
622
|
+
*/
|
|
623
|
+
static worker(cb: TSRQWCallback, factor?: number, wd?: unknown, opts?: TSRQWPoolOptions): TSRQWPool;
|
|
624
|
+
}
|
|
625
|
+
export {};
|