redis-consumer 1.1.9 → 1.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +255 -116
- package/dist/client.d.ts +8 -7
- package/dist/client.js +30 -20
- package/dist/consumer.d.ts +61 -9
- package/dist/consumer.js +273 -163
- package/dist/helpers.d.ts +5 -2
- package/dist/helpers.js +59 -5
- package/dist/index.d.ts +5 -9
- package/dist/index.js +3 -22
- package/dist/producer.d.ts +4 -4
- package/dist/producer.js +2 -6
- package/dist/retry-processor.d.ts +14 -11
- package/dist/retry-processor.js +51 -22
- package/package.json +13 -8
package/README.md
CHANGED
|
@@ -3,73 +3,80 @@
|
|
|
3
3
|
Simple node package for easy use of Redis Streams functionality. This package allows for creation of a Redis consumer and producer.
|
|
4
4
|
|
|
5
5
|
- [Installation](#installation)
|
|
6
|
-
- [
|
|
7
|
-
|
|
8
|
-
- [
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
- [
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
- [
|
|
6
|
+
- [Quick start](#quick-start)
|
|
7
|
+
- [Class RedisClient](#class-redisclient)
|
|
8
|
+
- [RedisClientOptions](#redisclientoptions)
|
|
9
|
+
- [Methods](#methods)
|
|
10
|
+
- [Class RedisConsumer](#class-redisconsumer)
|
|
11
|
+
- [How messages are processed](#how-messages-are-processed)
|
|
12
|
+
- [RedisConsumerOptions](#redisconsumeroptions)
|
|
13
|
+
- [Methods](#methods-1)
|
|
14
|
+
- [StreamToListen object](#streamtolisten-object)
|
|
15
|
+
- [Class RedisProducer](#class-redisproducer)
|
|
16
|
+
- [Methods](#methods-2)
|
|
17
|
+
- [Events](#events)
|
|
18
|
+
- [Recipes](#recipes)
|
|
19
|
+
- [Dead-letter stream](#dead-letter-stream)
|
|
20
|
+
- [Graceful shutdown](#graceful-shutdown)
|
|
21
|
+
- [Performance](#performance)
|
|
22
|
+
- [TypeScript](#typescript)
|
|
23
|
+
- [Upgrading from 1.1.x](#upgrading-from-11x)
|
|
19
24
|
|
|
20
25
|
## Installation
|
|
21
26
|
|
|
22
|
-
|
|
27
|
+
Requires NodeJs 24 or newer.
|
|
28
|
+
|
|
29
|
+
This package is ESM only (`"type": "module"` in `package.json`). Use `import`. On Node 22.12+ `require()` also happens to work through Node's ESM interop, but `import` is the supported path.
|
|
30
|
+
|
|
31
|
+
The `redis` API is not re-exported. Import it from `redis` directly.
|
|
23
32
|
|
|
24
33
|
```bash
|
|
25
|
-
|
|
34
|
+
pnpm install redis-consumer
|
|
26
35
|
```
|
|
27
36
|
|
|
28
|
-
##
|
|
29
|
-
|
|
30
|
-
### Basic example
|
|
37
|
+
## Quick start
|
|
31
38
|
|
|
32
39
|
```typescript
|
|
33
40
|
import { RedisClient } from 'redis-consumer';
|
|
34
41
|
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
});
|
|
42
|
+
const client = new RedisClient({
|
|
43
|
+
groupName: 'mygroup',
|
|
44
|
+
// Must be unique per consumer within the group.
|
|
45
|
+
clientName: 'myclient1',
|
|
46
|
+
});
|
|
41
47
|
|
|
42
|
-
|
|
48
|
+
client.on('error', err => console.error('Redis client error', err));
|
|
43
49
|
|
|
44
|
-
|
|
50
|
+
await client.connect();
|
|
45
51
|
|
|
46
|
-
|
|
52
|
+
const producer = client.createProducer();
|
|
53
|
+
const consumer = client.createConsumer();
|
|
47
54
|
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
55
|
+
// Redis stream to listen to and the function that processes its messages.
|
|
56
|
+
consumer.listen({
|
|
57
|
+
name: 'mystream',
|
|
58
|
+
executable: (data, stream) => console.log('Message for stream ' + stream, data.message),
|
|
59
|
+
});
|
|
60
|
+
|
|
61
|
+
await producer.add('mystream', { firstName: 'John', lastName: 'Doe' });
|
|
53
62
|
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
})();
|
|
63
|
+
// Stops reading and closes the consumer's own connection.
|
|
64
|
+
await consumer.close();
|
|
65
|
+
await client.close();
|
|
58
66
|
```
|
|
59
67
|
|
|
60
|
-
|
|
61
|
-
unique in order for Redis to distinguish each individual client within the consumer group.
|
|
68
|
+
A more complete, runnable version is in [`example/index.ts`](example/index.ts).
|
|
62
69
|
|
|
63
|
-
|
|
70
|
+
## Class RedisClient
|
|
64
71
|
|
|
65
|
-
_Constructor_
|
|
72
|
+
_Constructor_: `new RedisClient(options)`
|
|
66
73
|
|
|
67
74
|
- `options` [RedisClientOptions](#redisclientoptions)
|
|
68
75
|
- extends the `node-redis` client constructor
|
|
69
76
|
|
|
70
77
|
The `RedisClient` is an extension of the original client from the [node-redis](https://www.npmjs.com/package/redis) package. All constructor options within the `node-redis` package are available to this class as well.
|
|
71
78
|
|
|
72
|
-
|
|
79
|
+
### Example
|
|
73
80
|
|
|
74
81
|
```typescript
|
|
75
82
|
// Connect client to Redis server with TLS enabled
|
|
@@ -85,16 +92,18 @@ const client = new RedisClient({
|
|
|
85
92
|
});
|
|
86
93
|
```
|
|
87
94
|
|
|
88
|
-
|
|
95
|
+
### RedisClientOptions
|
|
89
96
|
|
|
90
97
|
| Parameters | Description | Required |
|
|
91
98
|
| ---------- | --------------------------------------------- | -------- |
|
|
92
99
|
| groupName | Name of the consumer group | Yes |
|
|
93
100
|
| clientName | Name of the client, must be unique per client | Yes |
|
|
94
101
|
|
|
102
|
+
`groupName` and `clientName` are consumed by this package and are never forwarded to the `redis` client.
|
|
103
|
+
|
|
95
104
|
Other options can be found in the official `node-redis` github repository over [here](https://github.com/redis/node-redis/blob/master/docs/client-configuration.md).
|
|
96
105
|
|
|
97
|
-
|
|
106
|
+
### Methods
|
|
98
107
|
|
|
99
108
|
For all available methods, please look in the official `node-redis` repository over [here](https://github.com/redis/node-redis/blob/master/README.md).
|
|
100
109
|
|
|
@@ -110,108 +119,143 @@ For all available methods, please look in the official `node-redis` repository o
|
|
|
110
119
|
`streamExists(key)`
|
|
111
120
|
|
|
112
121
|
- `key` key name of the stream
|
|
113
|
-
- Returns a
|
|
122
|
+
- Returns a _Promise<boolean>_
|
|
114
123
|
|
|
115
124
|
`groupExists(key)`
|
|
116
125
|
|
|
117
126
|
- `key` name of the stream
|
|
118
|
-
- Returns a
|
|
127
|
+
- Returns a _Promise<boolean>_, `false` when the stream does not exist
|
|
119
128
|
|
|
120
129
|
`createGroup(key)`
|
|
121
130
|
|
|
122
131
|
- `key` name of the stream
|
|
123
|
-
-
|
|
132
|
+
- Creates the consumer group with `XGROUP CREATE ... $ MKSTREAM`, so the stream is created when missing and the group starts at the end of it
|
|
133
|
+
- Returns a _Promise<string>_. A group that already exists is not an error: `BUSYGROUP` is swallowed and `'OK'` is returned, so calling this repeatedly (or from several processes at once) is safe
|
|
124
134
|
|
|
125
|
-
|
|
135
|
+
## Class RedisConsumer
|
|
126
136
|
|
|
127
|
-
_Constructor_
|
|
137
|
+
_Constructor_: `client.createConsumer(options)`
|
|
128
138
|
|
|
129
139
|
- `options` [RedisConsumerOptions](#redisconsumeroptions)
|
|
130
140
|
|
|
131
|
-
The `RedisConsumer`
|
|
141
|
+
The `RedisConsumer` listens for incoming messages in one or more streams. You can pass a single stream object or an array of them, each with the stream name and the function that processes its messages. A built-in retry mechanism emits `retry-failed` once all retries were unsuccessful.
|
|
142
|
+
|
|
143
|
+
When a message is successfully processed (also in retry state), the consumer acknowledges it with `XACK` and removes it from the stream with `XDEL`. While a message is not acknowledged it stays in the consumer group's pending list.
|
|
132
144
|
|
|
133
|
-
|
|
145
|
+
### How messages are processed
|
|
134
146
|
|
|
135
|
-
|
|
147
|
+
- By default the consumer starts at id `'0-0'`, so it first re-processes the messages that are still pending **for this `clientName`** before it starts listening for new ones. Pass `id: '>'` on a stream to skip the pending list and only receive new messages.
|
|
148
|
+
- The executable is awaited. When it returns without throwing, the message is acknowledged and deleted.
|
|
149
|
+
- When it throws, `process-error` is emitted, the message is **not** acknowledged, and a retry is scheduled.
|
|
150
|
+
- Retries are attempted in memory according to `retries` and `retryTime`. The `retry` event fires before each attempt. When the last attempt fails, `retry-failed` is emitted and the message is left in the pending list until you acknowledge it yourself.
|
|
151
|
+
- Messages that are never acknowledged are redelivered the next time a consumer with the same `clientName` starts.
|
|
136
152
|
|
|
137
|
-
|
|
153
|
+
### Example
|
|
138
154
|
|
|
139
155
|
```typescript
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
156
|
+
const consumer = client.createConsumer({
|
|
157
|
+
COUNT: 3,
|
|
158
|
+
retries: 1,
|
|
159
|
+
retryTime: ['5s'],
|
|
160
|
+
});
|
|
145
161
|
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
consumer.client.on('retry-failed', (err, data) => {
|
|
159
|
-
console.error('Failed processing message in stream ' + data.stream + '. Amount of retries: ' + data.retries, data.message);
|
|
160
|
-
});
|
|
162
|
+
const streams = [
|
|
163
|
+
{
|
|
164
|
+
name: 'mystream',
|
|
165
|
+
id: '>',
|
|
166
|
+
executable: (data, stream) => console.log('Only new messages for ' + stream, data.message),
|
|
167
|
+
},
|
|
168
|
+
{
|
|
169
|
+
name: 'mysecondstream',
|
|
170
|
+
executable: (data, stream) => console.log('Message for stream ' + stream, data.message),
|
|
171
|
+
},
|
|
172
|
+
];
|
|
161
173
|
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
174
|
+
consumer.client.on('retry-failed', (err, data) => {
|
|
175
|
+
console.error('Failed processing message ' + data.message.id + ' in stream ' + data.stream, err.message);
|
|
176
|
+
});
|
|
165
177
|
|
|
166
|
-
|
|
178
|
+
consumer.client.on('process-error', (err, data) => {
|
|
179
|
+
console.error('An unexpected error occurred for stream ' + data.stream, err.message);
|
|
167
180
|
});
|
|
181
|
+
|
|
182
|
+
await consumer.listen(streams);
|
|
168
183
|
```
|
|
169
184
|
|
|
170
|
-
|
|
185
|
+
### RedisConsumerOptions
|
|
171
186
|
|
|
172
|
-
| Parameters
|
|
173
|
-
|
|
|
174
|
-
| COUNT
|
|
175
|
-
| BLOCK
|
|
176
|
-
| retries
|
|
177
|
-
| retryTime
|
|
187
|
+
| Parameters | Description | Required | Default |
|
|
188
|
+
| ------------- | ------------------------------------------------------- | -------- | -------------------- |
|
|
189
|
+
| COUNT | Number of elements to read | No | 5000 |
|
|
190
|
+
| BLOCK | Time in milliseconds to block while reading a stream | No | 0 |
|
|
191
|
+
| retries | Amount of retries for processing messages | No | 3 |
|
|
192
|
+
| retryTime | Time interval between each retry | No | ['15s', '1m', '15m'] |
|
|
193
|
+
| batchMessages | Hand the whole batch of a single read to the executable | No | false |
|
|
178
194
|
|
|
179
|
-
More information about the `BLOCK` and `COUNT` parameters can be found at the official [docs](https://redis.io/documentation) of Redis.
|
|
195
|
+
More information about the `BLOCK` and `COUNT` parameters can be found at the official [docs](https://redis.io/documentation) of Redis. `BLOCK: 0` blocks until a message arrives.
|
|
180
196
|
|
|
181
197
|
The `retryTime` is an array of time strings. Seconds, minutes and hours are supported ('s', 'm', 'h'). When there are less items in the `retryTime` array than the amount of retries, the last time string item is used.
|
|
182
198
|
|
|
183
199
|
If you want to disable the retry mechanism, select a value of 0 for `retries`.
|
|
184
200
|
|
|
185
|
-
|
|
201
|
+
When `batchMessages` is enabled, the executable receives the complete array of messages of a single read instead of being called once per message. The option applies to the whole consumer, so every stream it listens to has to use the batch form:
|
|
202
|
+
|
|
203
|
+
```typescript
|
|
204
|
+
const consumer = client.createConsumer({ batchMessages: true, COUNT: 100 });
|
|
186
205
|
|
|
187
|
-
|
|
206
|
+
await consumer.listen({
|
|
207
|
+
name: 'mystream',
|
|
208
|
+
// `messages` is an array of StreamMessageReply objects
|
|
209
|
+
executable: messages => console.log('Processing ' + messages.length + ' messages at once'),
|
|
210
|
+
});
|
|
211
|
+
```
|
|
212
|
+
|
|
213
|
+
If a batch call throws, the consumer cannot tell which message failed, so every message of that read is retried individually.
|
|
214
|
+
|
|
215
|
+
### Methods
|
|
216
|
+
|
|
217
|
+
`listen(streams, flag?)`
|
|
188
218
|
|
|
189
219
|
- `streams` [StreamToListen](#streamtolisten-object) | Array([StreamToListen](#streamtolisten-object))
|
|
220
|
+
- `flag` optional, legacy positional form of the `batchMessages` option
|
|
221
|
+
- Returns a _Promise_ that resolves once the read loop has been started
|
|
222
|
+
|
|
223
|
+
The consumer group is created automatically for every stream. Calling `listen` a second time with new streams adds them to the already running consumer instead of starting a second read loop; the new stream is picked up as soon as the current read returns. A stream that is already being listened to is left untouched.
|
|
224
|
+
|
|
225
|
+
`close()`
|
|
226
|
+
|
|
227
|
+
- Returns a _Promise_ that resolves once the read loop stopped and the consumer's connection is closed. Use it to shut down the consumer and let the process exit. Calling it more than once is safe.
|
|
228
|
+
|
|
229
|
+
`close()` also cancels the pending retries (their timers would otherwise keep the event loop alive for up to the longest `retryTime`) and flushes acknowledgements that are still queued, so the process can exit and nothing that was already processed is left in the pending list.
|
|
230
|
+
|
|
231
|
+
`flushAcks()`
|
|
232
|
+
|
|
233
|
+
- Returns a _Promise_ that resolves once the queued acknowledgements (`XACK` + `XDEL`) were sent to Redis
|
|
190
234
|
|
|
191
235
|
`addAckMessage(stream, id)`
|
|
192
236
|
|
|
193
237
|
- `stream` key name of the stream
|
|
194
238
|
- `id` id of the message
|
|
195
239
|
|
|
196
|
-
Adds the message to the
|
|
240
|
+
Adds the message to the acknowledgement queue. The read loop flushes the queue after every read, so normally you do not need `flushAcks()`. Call it when you queue an acknowledgement from outside the read loop — for example in a `retry-failed` handler — and want the message to leave the pending list right away. Flushes are serialised, so concurrent calls cannot interleave their `XACK`/`XDEL` batches.
|
|
197
241
|
|
|
198
|
-
|
|
242
|
+
### StreamToListen object
|
|
199
243
|
|
|
200
244
|
```typescript
|
|
201
245
|
{
|
|
202
246
|
name: 'mystream', // Keyname of the Redis stream
|
|
203
247
|
executable: (message, stream) => console.log(message), // Message processing function to be executed
|
|
204
|
-
id: '>' // Optional, start
|
|
248
|
+
id: '>' // Optional, start listening from this message id. Defaults to '0-0'
|
|
205
249
|
}
|
|
206
250
|
```
|
|
207
251
|
|
|
208
|
-
|
|
252
|
+
## Class RedisProducer
|
|
209
253
|
|
|
210
|
-
_Constructor_
|
|
254
|
+
_Constructor_: `client.createProducer()`
|
|
211
255
|
|
|
212
256
|
The `RedisProducer` is used to add new messages to the Redis stream.
|
|
213
257
|
|
|
214
|
-
|
|
258
|
+
### Example
|
|
215
259
|
|
|
216
260
|
```typescript
|
|
217
261
|
const message = {
|
|
@@ -220,30 +264,88 @@ const message = {
|
|
|
220
264
|
};
|
|
221
265
|
|
|
222
266
|
const producer = client.createProducer();
|
|
223
|
-
producer.add('mystream', message);
|
|
267
|
+
const id = await producer.add('mystream', message);
|
|
224
268
|
```
|
|
225
269
|
|
|
226
|
-
|
|
270
|
+
### Methods
|
|
227
271
|
|
|
228
272
|
`add(stream, message)`
|
|
229
273
|
|
|
230
274
|
- `stream` key name of the stream
|
|
231
275
|
- `message` object/message to add to the stream
|
|
276
|
+
- Returns a _Promise<string>_ resolving with the id of the added message
|
|
277
|
+
|
|
278
|
+
## Events
|
|
232
279
|
|
|
233
|
-
|
|
280
|
+
All events are emitted on `consumer.client`, not on the consumer itself.
|
|
234
281
|
|
|
235
|
-
| Event name | Description
|
|
236
|
-
| ------------- |
|
|
237
|
-
| process-error |
|
|
238
|
-
| retry |
|
|
239
|
-
| retry-failed |
|
|
282
|
+
| Event name | Signature | Description |
|
|
283
|
+
| ------------- | --------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
|
|
284
|
+
| process-error | `(err: Error, data: ProcessErrorData)` | An error occurred during execution of the stream processing function. `data` contains `stream`, `message` and `retries`. |
|
|
285
|
+
| retry | `(data: RetryMessage)` | Fired just before a retry is attempted. `data` contains `stream`, `message`, `retries` and `timestamp`. |
|
|
286
|
+
| retry-failed | `(err: Error, data: RetryFailedMessage)` | All retries for a message were unsuccessful. `data` contains `stream`, `message`, `retries` and `timestamps`. The message is still in the pending list. |
|
|
287
|
+
| listen-error | `(err: Error)` | A Redis command issued by the consumer failed. Reading is retried after a short delay, and a failed acknowledgement stays queued for the next flush. |
|
|
240
288
|
|
|
241
|
-
##
|
|
289
|
+
## Recipes
|
|
242
290
|
|
|
243
|
-
|
|
291
|
+
### Dead-letter stream
|
|
292
|
+
|
|
293
|
+
When the retries are exhausted, move the message somewhere else and acknowledge it so it does not stay in the pending list forever:
|
|
244
294
|
|
|
245
295
|
```typescript
|
|
246
|
-
|
|
296
|
+
consumer.client.on('retry-failed', async (err, data) => {
|
|
297
|
+
console.error(`Giving up on ${data.stream} ${data.message.id}: ${err.message}`);
|
|
298
|
+
|
|
299
|
+
await producer.add('dead-letters', {
|
|
300
|
+
sourceStream: data.stream,
|
|
301
|
+
sourceId: data.message.id,
|
|
302
|
+
payload: JSON.stringify(data.message.message),
|
|
303
|
+
});
|
|
304
|
+
|
|
305
|
+
consumer.addAckMessage(data.stream, data.message.id);
|
|
306
|
+
await consumer.flushAcks();
|
|
307
|
+
});
|
|
308
|
+
```
|
|
309
|
+
|
|
310
|
+
### Graceful shutdown
|
|
311
|
+
|
|
312
|
+
```typescript
|
|
313
|
+
const shutdown = async (signal: string) => {
|
|
314
|
+
console.log(`${signal} received, shutting down`);
|
|
315
|
+
await consumer.close();
|
|
316
|
+
await client.close();
|
|
317
|
+
};
|
|
318
|
+
|
|
319
|
+
process.on('SIGINT', () => void shutdown('SIGINT'));
|
|
320
|
+
process.on('SIGTERM', () => void shutdown('SIGTERM'));
|
|
321
|
+
```
|
|
322
|
+
|
|
323
|
+
## TypeScript
|
|
324
|
+
|
|
325
|
+
This package has full TypeScript support. Use `StreamProcessingFunction<T>` for a per-message executable and `StreamBatchProcessingFunction<T>` for a batched one to get your message data typed:
|
|
326
|
+
|
|
327
|
+
```typescript
|
|
328
|
+
import {
|
|
329
|
+
RedisClient,
|
|
330
|
+
StreamBatchProcessingFunction,
|
|
331
|
+
StreamMessageReply,
|
|
332
|
+
StreamsToListen,
|
|
333
|
+
} from 'redis-consumer';
|
|
334
|
+
|
|
335
|
+
// Define the interface of your message data
|
|
336
|
+
interface MyMessage {
|
|
337
|
+
firstName: string;
|
|
338
|
+
lastName: string;
|
|
339
|
+
}
|
|
340
|
+
|
|
341
|
+
const processing: StreamBatchProcessingFunction<MyMessage> = async messages => {
|
|
342
|
+
for (const data of messages) {
|
|
343
|
+
const message = data.message;
|
|
344
|
+
const fullName = message.firstName + ' ' + message.lastName; // Fully typed
|
|
345
|
+
|
|
346
|
+
console.log('Hello, my name is ' + fullName);
|
|
347
|
+
}
|
|
348
|
+
};
|
|
247
349
|
|
|
248
350
|
const client = new RedisClient({
|
|
249
351
|
groupName: 'mygroup',
|
|
@@ -258,19 +360,56 @@ const streams: StreamsToListen = [
|
|
|
258
360
|
},
|
|
259
361
|
];
|
|
260
362
|
|
|
261
|
-
const consumer = client.createConsumer();
|
|
262
|
-
consumer.listen(streams);
|
|
363
|
+
const consumer = client.createConsumer({ batchMessages: true });
|
|
364
|
+
await consumer.listen(streams);
|
|
365
|
+
```
|
|
263
366
|
|
|
264
|
-
|
|
265
|
-
interface MyMessage {
|
|
266
|
-
firstName: string;
|
|
267
|
-
lastName: string;
|
|
268
|
-
}
|
|
367
|
+
Use `StreamProcessingFunction<T>` instead when `batchMessages` is off; the executable then receives a single `StreamMessageReply<T>`.
|
|
269
368
|
|
|
270
|
-
|
|
271
|
-
const message = data.message;
|
|
272
|
-
const fullName = message.firstName + ' ' + message.lastName; // Full typing of message
|
|
369
|
+
`StreamMessageReply<T>` is the shape handed to the executable: `{ id, message, millisElapsedFromDelivery?, deliveriesCounter? }`, where `message` is `T`.
|
|
273
370
|
|
|
274
|
-
|
|
275
|
-
|
|
276
|
-
|
|
371
|
+
## Performance
|
|
372
|
+
|
|
373
|
+
`COUNT` is by far the biggest throughput lever: every read, and every acknowledgement batch, is a round trip to Redis.
|
|
374
|
+
|
|
375
|
+
Measured against a local Redis 7.0.12 with a no-op executable, 2000 messages per run, median of 3 runs:
|
|
376
|
+
|
|
377
|
+
| Consumer options | Throughput | Redis commands per message |
|
|
378
|
+
| ---------------------------------- | ----------- | -------------------------- |
|
|
379
|
+
| `COUNT: 1` | ~1.7k msg/s | ~3.0 |
|
|
380
|
+
| `COUNT: 10` | ~9.6k msg/s | ~0.30 |
|
|
381
|
+
| `COUNT: 100` | ~24k msg/s | ~0.03 |
|
|
382
|
+
| `COUNT: 1000, batchMessages: true` | ~38k msg/s | ~0.003 |
|
|
383
|
+
|
|
384
|
+
100,000 messages in a single run, same machine:
|
|
385
|
+
|
|
386
|
+
| Consumer options | Produce | Consume | Throughput | Consumer memory |
|
|
387
|
+
| ----------------------------------- | ------- | ------- | ---------- | --------------- |
|
|
388
|
+
| default (`COUNT: 5000`) | 2.4 s | 2.1 s | ~48k msg/s | ~12 MB |
|
|
389
|
+
| `COUNT: 5000, batchMessages: true` | 2.1 s | 1.9 s | ~50k msg/s | ~4 MB |
|
|
390
|
+
| `COUNT: 1000, batchMessages: true` | 2.1 s | 2.6 s | ~38k msg/s | — |
|
|
391
|
+
|
|
392
|
+
All 100,000 messages were processed exactly once — no duplicates, none missed — in every configuration.
|
|
393
|
+
|
|
394
|
+
Guidance:
|
|
395
|
+
|
|
396
|
+
- `COUNT` defaults to `5000`. Raise it for more throughput, lower it for lower latency: a read returns as soon as `COUNT` messages are available, so a small `COUNT` hands messages over sooner but costs far more round trips.
|
|
397
|
+
- `batchMessages` barely changes throughput. It avoids per-call setup in the executable, but the speed-up above comes from `COUNT`, not from batching.
|
|
398
|
+
- The memory column is the consumer's cost on top of a stream that is already populated, so with the default `COUNT: 5000` expect roughly 10 MB of headroom for in-flight messages.
|
|
399
|
+
- Acknowledgements are pipelined: the `XACK` and `XDEL` of every stream are sent in a single write, so a flush costs one round trip no matter how many streams are involved.
|
|
400
|
+
- Retries run in memory and do not block the read loop. A failed read of `COUNT` messages registers one retry per message, so a large `COUNT` with a short `retryTime` can start many concurrent retry chains at once. A failed read also emits one `process-error` per message, and arming those retries is synchronous: a 200,000-message batch that fails takes about 5 s and ~290 MB before the first retry fires, and `close()` then needs a moment to cancel them all. If your executable can fail on a whole batch, prefer a smaller `COUNT` and let several reads fill the pipeline.
|
|
401
|
+
|
|
402
|
+
The absolute numbers depend heavily on the round trip time to your Redis; over a real network the effect of `COUNT` is larger, not smaller.
|
|
403
|
+
|
|
404
|
+
## Upgrading from 1.1.x
|
|
405
|
+
|
|
406
|
+
- **NodeJs 24 or newer** is required (the `redis` package requires at least NodeJs 20).
|
|
407
|
+
- **ESM only.** The package is published as an ES module. `import` is the supported path; `require()` only works through Node's ESM interop.
|
|
408
|
+
- **The `redis` API is no longer re-exported.** `export * from 'redis'` was removed because node-redis 6 exports a runtime `RedisClient` of its own, which collided with this package's `RedisClient` and made `require()` hand out the wrong class. Import `createClient` and friends from `redis` directly.
|
|
409
|
+
- **`redis` was upgraded from 4.x to 6.x.** Commands and their replies follow the `redis` 6 API. `xPending` for example returns `{ pending, firstId, lastId, consumers }` where `redis` 4 returned `{ total, ... }`.
|
|
410
|
+
- The `@redis/client` deep imports are gone. `StreamMessageId`, `StreamMessageData` and `StreamMessageReply` are exported by this package.
|
|
411
|
+
- `producer.add()` now returns the id of the added message instead of discarding it.
|
|
412
|
+
- `consumer.listen()` can be called repeatedly and no longer starts a second read loop.
|
|
413
|
+
- `consumer.close()` was added, and the consumer no longer keeps the process alive after it.
|
|
414
|
+
- `batchMessages` was added; the positional `flag` argument of `listen()` is kept as a legacy alias.
|
|
415
|
+
- **`COUNT` now defaults to `5000` instead of `1`.** A read now returns up to 5000 messages instead of one, which is where most of the throughput comes from. Set `COUNT: 1` explicitly if you want the old latency-first behaviour, and see [Performance](#performance) for the trade-off.
|
package/dist/client.d.ts
CHANGED
|
@@ -1,9 +1,9 @@
|
|
|
1
|
-
import { RedisScripts } from 'redis';
|
|
2
|
-
import {
|
|
3
|
-
import {
|
|
4
|
-
import {
|
|
5
|
-
|
|
6
|
-
declare const InstRedisClient:
|
|
1
|
+
import type { RedisClientOptions, RedisClientType, RedisScripts } from 'redis';
|
|
2
|
+
import { RedisConsumer, ConsumerOptions, ProcessErrorData } from './consumer.js';
|
|
3
|
+
import { RedisProducer } from './producer.js';
|
|
4
|
+
import { RetryFailedMessage, RetryMessage } from './retry-processor.js';
|
|
5
|
+
type NodeRedisClientType = RedisClientType<{}, {}, {}>;
|
|
6
|
+
declare const InstRedisClient: new (options?: RedisClientOptions) => NodeRedisClientType;
|
|
7
7
|
interface AdditionalClientOptions {
|
|
8
8
|
groupName: string;
|
|
9
9
|
clientName: string;
|
|
@@ -12,6 +12,7 @@ export declare interface RedisClient<S extends RedisScripts = RedisScripts> {
|
|
|
12
12
|
on(event: 'process-error', listener: (err: Error, data: ProcessErrorData) => void): this;
|
|
13
13
|
on(event: 'retry-failed', listener: (err: Error, data: RetryFailedMessage) => void): this;
|
|
14
14
|
on(event: 'retry', listener: (data: RetryMessage) => void): this;
|
|
15
|
+
on(event: 'listen-error', listener: (err: Error) => void): this;
|
|
15
16
|
on(event: string, listener: (data: any) => void): this;
|
|
16
17
|
}
|
|
17
18
|
export declare class RedisClient<S extends RedisScripts> extends InstRedisClient {
|
|
@@ -20,7 +21,7 @@ export declare class RedisClient<S extends RedisScripts> extends InstRedisClient
|
|
|
20
21
|
constructor(options: Omit<RedisClientOptions<never, never, S>, 'modules'> & AdditionalClientOptions);
|
|
21
22
|
createConsumer<S extends RedisScripts>(options?: ConsumerOptions): RedisConsumer<S>;
|
|
22
23
|
createProducer(): RedisProducer<S>;
|
|
23
|
-
streamExists(key: string): Promise<
|
|
24
|
+
streamExists(key: string): Promise<boolean>;
|
|
24
25
|
groupExists(key: string): Promise<boolean>;
|
|
25
26
|
createGroup(key: string): Promise<string>;
|
|
26
27
|
}
|
package/dist/client.js
CHANGED
|
@@ -1,29 +1,31 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
const
|
|
10
|
-
|
|
11
|
-
class RedisClient extends InstRedisClient {
|
|
1
|
+
import { createClient } from 'redis';
|
|
2
|
+
import { RedisConsumer } from './consumer.js';
|
|
3
|
+
import { RedisProducer } from './producer.js';
|
|
4
|
+
// node-redis v5 removed the public `RedisClient.extend()` helper, so the
|
|
5
|
+
// command-augmented client class can no longer be obtained by calling it.
|
|
6
|
+
// `createClient()` returns an instance of that class (its factory wraps the
|
|
7
|
+
// subclass built by `attachConfig()`), so the class can be taken off the
|
|
8
|
+
// instance. That keeps subclassing - and `instanceof RedisClient` - working.
|
|
9
|
+
const InstRedisClient = createClient({}).constructor;
|
|
10
|
+
export class RedisClient extends InstRedisClient {
|
|
12
11
|
groupName;
|
|
13
12
|
clientName;
|
|
14
13
|
constructor(options) {
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
14
|
+
// `groupName`/`clientName` are not Redis options: node-redis v6 mutates and
|
|
15
|
+
// persists whatever it is handed, so keep them off the client options.
|
|
16
|
+
const { groupName, clientName, ...clientOptions } = options;
|
|
17
|
+
super(clientOptions);
|
|
18
|
+
this.groupName = groupName;
|
|
19
|
+
this.clientName = clientName;
|
|
18
20
|
}
|
|
19
21
|
createConsumer(options) {
|
|
20
|
-
return new
|
|
22
|
+
return new RedisConsumer(this, options);
|
|
21
23
|
}
|
|
22
24
|
createProducer() {
|
|
23
|
-
return new
|
|
25
|
+
return new RedisProducer(this);
|
|
24
26
|
}
|
|
25
27
|
async streamExists(key) {
|
|
26
|
-
return await this.exists(key);
|
|
28
|
+
return (await this.exists(key)) > 0;
|
|
27
29
|
}
|
|
28
30
|
async groupExists(key) {
|
|
29
31
|
if (!(await this.streamExists(key)))
|
|
@@ -32,8 +34,16 @@ class RedisClient extends InstRedisClient {
|
|
|
32
34
|
return groupInfo.length > 0;
|
|
33
35
|
}
|
|
34
36
|
async createGroup(key) {
|
|
35
|
-
|
|
36
|
-
|
|
37
|
+
try {
|
|
38
|
+
return await this.xGroupCreate(key, this.groupName, '$', { MKSTREAM: true });
|
|
39
|
+
}
|
|
40
|
+
catch (err) {
|
|
41
|
+
// Another consumer may have created the group in the meantime; that is
|
|
42
|
+
// the desired end state, so treat it as success.
|
|
43
|
+
if (err instanceof Error && err.message.includes('BUSYGROUP')) {
|
|
44
|
+
return 'OK';
|
|
45
|
+
}
|
|
46
|
+
throw err;
|
|
47
|
+
}
|
|
37
48
|
}
|
|
38
49
|
}
|
|
39
|
-
exports.RedisClient = RedisClient;
|