redis-consumer 1.1.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +276 -0
- package/dist/client.d.ts +27 -0
- package/dist/client.js +39 -0
- package/dist/consumer.d.ts +52 -0
- package/dist/consumer.js +162 -0
- package/dist/helpers.d.ts +2 -0
- package/dist/helpers.js +5 -0
- package/dist/index.d.ts +9 -0
- package/dist/index.js +22 -0
- package/dist/producer.d.ts +8 -0
- package/dist/producer.js +13 -0
- package/dist/retry-processor.d.ts +35 -0
- package/dist/retry-processor.js +84 -0
- package/package.json +42 -0
package/README.md
ADDED
|
@@ -0,0 +1,276 @@
|
|
|
1
|
+
# redis-streams-nodejs
|
|
2
|
+
|
|
3
|
+
Simple node package for easy use of Redis Streams functionality. This package allows for creation of a Redis consumer and producer.
|
|
4
|
+
|
|
5
|
+
- [Installation](#installation)
|
|
6
|
+
- [Usage](#usage)
|
|
7
|
+
- [Basic Example](#basic-example)
|
|
8
|
+
- [Class RedisClient](#class-redisclient)
|
|
9
|
+
- [RedisClientOptions](#redisclientoptions)
|
|
10
|
+
- [Methods](#methods)
|
|
11
|
+
- [Class RedisConsumer](#class-redisconsumer)
|
|
12
|
+
- [RedisConsumerOptions](#redisconsumeroptions)
|
|
13
|
+
- [Methods](#methods-1)
|
|
14
|
+
- [StreamToListen Object](#streamtolisten-object)
|
|
15
|
+
- [Class RedisProducer](#class-redisproducer)
|
|
16
|
+
- [Methods](#methods-2)
|
|
17
|
+
- [Events](#events)
|
|
18
|
+
- [Typescript](#typescript)
|
|
19
|
+
|
|
20
|
+
## Installation
|
|
21
|
+
|
|
22
|
+
Make sure you have NodeJs installed, then:
|
|
23
|
+
|
|
24
|
+
```bash
|
|
25
|
+
npm install redis-consumer
|
|
26
|
+
```
|
|
27
|
+
|
|
28
|
+
## Usage
|
|
29
|
+
|
|
30
|
+
### Basic example
|
|
31
|
+
|
|
32
|
+
```typescript
|
|
33
|
+
import { RedisClient } from 'redis-consumer';
|
|
34
|
+
|
|
35
|
+
(async () => {
|
|
36
|
+
// Client name must be unique per client
|
|
37
|
+
const client = new RedisClient({
|
|
38
|
+
groupName: 'mygroup',
|
|
39
|
+
clientName: 'myclient1',
|
|
40
|
+
});
|
|
41
|
+
|
|
42
|
+
client.on('error', err => console.log('Redis Client Error', err));
|
|
43
|
+
|
|
44
|
+
await client.connect();
|
|
45
|
+
|
|
46
|
+
const consumer = client.createConsumer();
|
|
47
|
+
|
|
48
|
+
// Redis stream to listen to and processable function
|
|
49
|
+
const stream = {
|
|
50
|
+
name: 'mystream',
|
|
51
|
+
executable: (data, stream) => console.log('Redis message for stream ' + stream, data),
|
|
52
|
+
};
|
|
53
|
+
|
|
54
|
+
// Listen for new messages and process them according the
|
|
55
|
+
// defined executable function
|
|
56
|
+
consumer.listen(stream);
|
|
57
|
+
})();
|
|
58
|
+
```
|
|
59
|
+
|
|
60
|
+
When creating the Redis client, make sure to define a group and client name. Note, the client name must be
|
|
61
|
+
unique in order for Redis to distinguish each individual client within the consumer group.
|
|
62
|
+
|
|
63
|
+
### Class RedisClient
|
|
64
|
+
|
|
65
|
+
_Constructor_ : `new RedisClient(options)`
|
|
66
|
+
|
|
67
|
+
- `options` [RedisClientOptions](#redisclientoptions)
|
|
68
|
+
- extends the `node-redis` client constructor
|
|
69
|
+
|
|
70
|
+
The `RedisClient` is an extension of the original client from the [node-redis](https://www.npmjs.com/package/redis) package. All constructor options within the `node-redis` package are available to this class as well.
|
|
71
|
+
|
|
72
|
+
#### Example
|
|
73
|
+
|
|
74
|
+
```typescript
|
|
75
|
+
// Connect client to Redis server with TLS enabled
|
|
76
|
+
const client = new RedisClient({
|
|
77
|
+
socket: {
|
|
78
|
+
port: 6380,
|
|
79
|
+
host: 'localhost',
|
|
80
|
+
tls: false,
|
|
81
|
+
},
|
|
82
|
+
password: 'mysupersecurepassword',
|
|
83
|
+
groupName: 'mygroup',
|
|
84
|
+
clientName: 'client1',
|
|
85
|
+
});
|
|
86
|
+
```
|
|
87
|
+
|
|
88
|
+
#### RedisClientOptions
|
|
89
|
+
|
|
90
|
+
| Parameters | Description | Required |
|
|
91
|
+
| ---------- | --------------------------------------------- | -------- |
|
|
92
|
+
| groupName | Name of the consumer group | Yes |
|
|
93
|
+
| clientName | Name of the client, must be unique per client | Yes |
|
|
94
|
+
|
|
95
|
+
Other options can be found in the official `node-redis` github repository over [here](https://github.com/redis/node-redis/blob/master/docs/client-configuration.md).
|
|
96
|
+
|
|
97
|
+
#### Methods
|
|
98
|
+
|
|
99
|
+
For all available methods, please look in the official `node-redis` repository over [here](https://github.com/redis/node-redis/blob/master/README.md).
|
|
100
|
+
|
|
101
|
+
`createConsumer(options)`
|
|
102
|
+
|
|
103
|
+
- `options` [RedisConsumerOptions](#redisconsumeroptions)
|
|
104
|
+
- Returns a _RedisConsumer_
|
|
105
|
+
|
|
106
|
+
`createProducer()`
|
|
107
|
+
|
|
108
|
+
- Returns a _RedisProducer_
|
|
109
|
+
|
|
110
|
+
`streamExists(key)`
|
|
111
|
+
|
|
112
|
+
- `key` key name of the stream
|
|
113
|
+
- Returns a _boolean_
|
|
114
|
+
|
|
115
|
+
`groupExists(key)`
|
|
116
|
+
|
|
117
|
+
- `key` name of the stream
|
|
118
|
+
- Returns a _boolean_
|
|
119
|
+
|
|
120
|
+
`createGroup(key)`
|
|
121
|
+
|
|
122
|
+
- `key` name of the stream
|
|
123
|
+
- Returns a _string_
|
|
124
|
+
|
|
125
|
+
### Class RedisConsumer
|
|
126
|
+
|
|
127
|
+
_Constructor_ : `client.createConsumer(options)`
|
|
128
|
+
|
|
129
|
+
- `options` [RedisConsumerOptions](#redisconsumeroptions)
|
|
130
|
+
|
|
131
|
+
The `RedisConsumer` is able to listen for incomming message in a stream. You can define an object or an array of objects in which you can define the name of the stream to listen for and which function should be executed for processing of the message. The consumer has a build-in retry mechanism which triggers an event `retry-failed` if all retries were unsuccessfull.
|
|
132
|
+
|
|
133
|
+
When a message is successfully processed (also in retry state), the consumer will send an acknowledgement signal to the Redis server. When the acknowlegdement is performed, the message will be removed from the pending list for that consumer group.
|
|
134
|
+
|
|
135
|
+
When the consumer starts, it will process all remaining pending messages at first before listening for new incomming messsage. However, you can overrule this behaviour by defining your own starting id.
|
|
136
|
+
|
|
137
|
+
#### Example
|
|
138
|
+
|
|
139
|
+
```typescript
|
|
140
|
+
const consumer = client.createConsumer({
|
|
141
|
+
COUNT: 3,
|
|
142
|
+
retries: 1,
|
|
143
|
+
retryTime: ['5s'],
|
|
144
|
+
});
|
|
145
|
+
|
|
146
|
+
const streams = [
|
|
147
|
+
{
|
|
148
|
+
name: 'mystream',
|
|
149
|
+
id: '>',
|
|
150
|
+
executable: (data) => console.log('Only listen to new messages', data.message)
|
|
151
|
+
},
|
|
152
|
+
{
|
|
153
|
+
name: 'myssecondstresm',
|
|
154
|
+
executable: (data, stream) => console.log('Message for stream ' + stream, data.message)
|
|
155
|
+
}
|
|
156
|
+
];
|
|
157
|
+
|
|
158
|
+
consumer.client.on('retry-failed', (err, data) => {
|
|
159
|
+
console.error('Failed processing message in stream ' + data.stream + '. Amount of retries: ' + data.retries, data.message);
|
|
160
|
+
});
|
|
161
|
+
|
|
162
|
+
consumer.client.on('process-error', (err, data) => {
|
|
163
|
+
console.error('An unexpected error occured for stream ' + data.stream, err.message);
|
|
164
|
+
});
|
|
165
|
+
|
|
166
|
+
consumer.listen(streams);
|
|
167
|
+
});
|
|
168
|
+
```
|
|
169
|
+
|
|
170
|
+
#### RedisConsumerOptions
|
|
171
|
+
|
|
172
|
+
| Parameters | Description | Required | Default |
|
|
173
|
+
| ---------- | ------------------------------------------------- | -------- | -------------------- |
|
|
174
|
+
| COUNT | Number of elements to read | No | 1 |
|
|
175
|
+
| BLOCK | Time in miliseconds to block while reading stream | No | 0 |
|
|
176
|
+
| retries | Amount of retries for processing messages | No | 3 |
|
|
177
|
+
| retryTime | Time interval between each retry | No | ['15s', '1m', '15m'] |
|
|
178
|
+
|
|
179
|
+
More information about the `BLOCK` and `COUNT` parameters can be found at the official [docs](https://redis.io/documentation) of Redis.
|
|
180
|
+
|
|
181
|
+
The `retryTime` is an array of time strings. Seconds, minutes and hours are supported ('s', 'm', 'h'). When there are less items in the `retryTime` array than the amount of retries, the last time string item is used.
|
|
182
|
+
|
|
183
|
+
If you want to disable the retry mechanism, select a value of 0 for `retries`.
|
|
184
|
+
|
|
185
|
+
#### Methods
|
|
186
|
+
|
|
187
|
+
`listen(streams)`
|
|
188
|
+
|
|
189
|
+
- `streams` [StreamToListen](#streamtolisten-object) | Array([StreamToListen](#streamtolisten-object))
|
|
190
|
+
|
|
191
|
+
`addAckMessage(stream, id)`
|
|
192
|
+
|
|
193
|
+
- `stream` key name of the stream
|
|
194
|
+
- `id` id of the message
|
|
195
|
+
|
|
196
|
+
Adds the message to the acknowlegdement list.
|
|
197
|
+
|
|
198
|
+
#### StreamToListen Object
|
|
199
|
+
|
|
200
|
+
```typescript
|
|
201
|
+
{
|
|
202
|
+
name: 'mystream', // Keyname of the Redis stream
|
|
203
|
+
executable: (message, stream) => console.log(message), // Message processing function to be executed
|
|
204
|
+
id: '>' // Optional, start listining from the message id. Defaults to '0-0'
|
|
205
|
+
}
|
|
206
|
+
```
|
|
207
|
+
|
|
208
|
+
### Class RedisProducer
|
|
209
|
+
|
|
210
|
+
_Constructor_ : `client.createProducer()`
|
|
211
|
+
|
|
212
|
+
The `RedisProducer` is used to add new messages to the Redis stream.
|
|
213
|
+
|
|
214
|
+
#### Example
|
|
215
|
+
|
|
216
|
+
```typescript
|
|
217
|
+
const message = {
|
|
218
|
+
firstName: 'John',
|
|
219
|
+
lastName: 'Doe',
|
|
220
|
+
};
|
|
221
|
+
|
|
222
|
+
const producer = client.createProducer();
|
|
223
|
+
producer.add('mystream', message);
|
|
224
|
+
```
|
|
225
|
+
|
|
226
|
+
#### Methods
|
|
227
|
+
|
|
228
|
+
`add(stream, message)`
|
|
229
|
+
|
|
230
|
+
- `stream` key name of the stream
|
|
231
|
+
- `message` object/message to add to the stream
|
|
232
|
+
|
|
233
|
+
### Events
|
|
234
|
+
|
|
235
|
+
| Event name | Description |
|
|
236
|
+
| ------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
|
|
237
|
+
| process-error | Event is triggered on the `RedisConsumer` when an error occurs during execution of the streams processing function. First argument is the error object, second argument is an object containing the `stream`, `message` and `retries`. |
|
|
238
|
+
| retry | Event is triggered on the `RedisConsumer` just before a retry is attempted. A data object with properties `stream`, `message`, `retries` and `timestamp` is forwarded to the event. |
|
|
239
|
+
| retry-failed | Event is triggered on the `RedisConsumer` when retry of the message has failed/ended. The first argument that is forwarded to the event is the `error`. The second arguments is a data object with properties `stream`, `message`, `retries` and `timestamps`. |
|
|
240
|
+
|
|
241
|
+
## Typescript
|
|
242
|
+
|
|
243
|
+
This package has full Typescript support. See the example below on how to define a processing function with typed message data.
|
|
244
|
+
|
|
245
|
+
```typescript
|
|
246
|
+
import { RedisClient, StreamsToListen, StreamMessageReply } from 'redis-consumer';
|
|
247
|
+
|
|
248
|
+
const client = new RedisClient({
|
|
249
|
+
groupName: 'mygroup',
|
|
250
|
+
clientName: 'myclient1',
|
|
251
|
+
});
|
|
252
|
+
await client.connect();
|
|
253
|
+
|
|
254
|
+
const streams: StreamsToListen = [
|
|
255
|
+
{
|
|
256
|
+
name: 'mystream',
|
|
257
|
+
executable: processing,
|
|
258
|
+
},
|
|
259
|
+
];
|
|
260
|
+
|
|
261
|
+
const consumer = client.createConsumer();
|
|
262
|
+
consumer.listen(streams);
|
|
263
|
+
|
|
264
|
+
// Define interface of your message data
|
|
265
|
+
interface MyMessage {
|
|
266
|
+
firstName: string;
|
|
267
|
+
lastName: string;
|
|
268
|
+
}
|
|
269
|
+
|
|
270
|
+
function processing(data: StreamMessageReply<MyMessage>) {
|
|
271
|
+
const message = data.message;
|
|
272
|
+
const fullName = message.firstName + ' ' + message.lastName; // Full typing of message
|
|
273
|
+
|
|
274
|
+
console.log('Hello, my name is ' + fullName);
|
|
275
|
+
}
|
|
276
|
+
```
|
package/dist/client.d.ts
ADDED
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
import { RedisScripts } from 'redis';
|
|
2
|
+
import { RedisClientOptions } from 'redis';
|
|
3
|
+
import { RedisConsumer, ConsumerOptions, ProcessErrorData } from './consumer';
|
|
4
|
+
import { RedisProducer } from './producer';
|
|
5
|
+
import { RetryFailedMessage, RetryMessage } from './retry-processor';
|
|
6
|
+
declare const InstRedisClient: import("@redis/client/dist/lib/client").InstantiableRedisClient<import("redis").RedisModules, import("redis").RedisFunctions, RedisScripts>;
|
|
7
|
+
interface AdditionalClientOptions {
|
|
8
|
+
groupName: string;
|
|
9
|
+
clientName: string;
|
|
10
|
+
}
|
|
11
|
+
export declare interface RedisClient<S extends RedisScripts = RedisScripts> {
|
|
12
|
+
on(event: 'process-error', listener: (err: Error, data: ProcessErrorData) => void): this;
|
|
13
|
+
on(event: 'retry-failed', listener: (err: Error, data: RetryFailedMessage) => void): this;
|
|
14
|
+
on(event: 'retry', listener: (data: RetryMessage) => void): this;
|
|
15
|
+
on(event: string, listener: (data: any) => void): this;
|
|
16
|
+
}
|
|
17
|
+
export declare class RedisClient<S extends RedisScripts> extends InstRedisClient {
|
|
18
|
+
readonly groupName: string;
|
|
19
|
+
readonly clientName: string;
|
|
20
|
+
constructor(options: Omit<RedisClientOptions<never, never, S>, 'modules'> & AdditionalClientOptions);
|
|
21
|
+
createConsumer<S extends RedisScripts>(options?: ConsumerOptions): RedisConsumer<S>;
|
|
22
|
+
createProducer(): RedisProducer<S>;
|
|
23
|
+
streamExists(key: string): Promise<number>;
|
|
24
|
+
groupExists(key: string): Promise<boolean>;
|
|
25
|
+
createGroup(key: string): Promise<string>;
|
|
26
|
+
}
|
|
27
|
+
export {};
|
package/dist/client.js
ADDED
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
3
|
+
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
4
|
+
};
|
|
5
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
6
|
+
exports.RedisClient = void 0;
|
|
7
|
+
const client_1 = __importDefault(require("@redis/client/dist/lib/client"));
|
|
8
|
+
const consumer_1 = require("./consumer");
|
|
9
|
+
const producer_1 = require("./producer");
|
|
10
|
+
const InstRedisClient = client_1.default.extend();
|
|
11
|
+
class RedisClient extends InstRedisClient {
|
|
12
|
+
groupName;
|
|
13
|
+
clientName;
|
|
14
|
+
constructor(options) {
|
|
15
|
+
super(options);
|
|
16
|
+
this.groupName = options.groupName;
|
|
17
|
+
this.clientName = options.clientName;
|
|
18
|
+
}
|
|
19
|
+
createConsumer(options) {
|
|
20
|
+
return new consumer_1.RedisConsumer(this, options);
|
|
21
|
+
}
|
|
22
|
+
createProducer() {
|
|
23
|
+
return new producer_1.RedisProducer(this);
|
|
24
|
+
}
|
|
25
|
+
async streamExists(key) {
|
|
26
|
+
return await this.exists(key);
|
|
27
|
+
}
|
|
28
|
+
async groupExists(key) {
|
|
29
|
+
if (!(await this.streamExists(key)))
|
|
30
|
+
return false;
|
|
31
|
+
const groupInfo = await this.xInfoGroups(key);
|
|
32
|
+
return groupInfo.length > 0;
|
|
33
|
+
}
|
|
34
|
+
async createGroup(key) {
|
|
35
|
+
const result = await this.xGroupCreate(key, this.groupName, '$', { MKSTREAM: true });
|
|
36
|
+
return result;
|
|
37
|
+
}
|
|
38
|
+
}
|
|
39
|
+
exports.RedisClient = RedisClient;
|
|
@@ -0,0 +1,52 @@
|
|
|
1
|
+
import { RedisClientType, RedisScripts } from 'redis';
|
|
2
|
+
import { StreamMessageReply as OriginalStreamMessageReply } from '@redis/client/dist/lib/commands/generic-transformers';
|
|
3
|
+
import { StreamMessageData, StreamMessageId } from '.';
|
|
4
|
+
import { RedisClient } from './client';
|
|
5
|
+
export type StreamMessageReply<T = StreamMessageData> = Omit<OriginalStreamMessageReply, 'message'> & {
|
|
6
|
+
message: T;
|
|
7
|
+
};
|
|
8
|
+
export type StreamProcessingFunction<T> = (data: StreamMessageReply<T>, stream: string) => void;
|
|
9
|
+
export interface StreamToListen {
|
|
10
|
+
name: string;
|
|
11
|
+
executable: StreamProcessingFunction<any>;
|
|
12
|
+
id?: string;
|
|
13
|
+
}
|
|
14
|
+
export type StreamsToListen = StreamToListen[];
|
|
15
|
+
export interface ConsumerOptions {
|
|
16
|
+
COUNT?: number;
|
|
17
|
+
BLOCK?: number;
|
|
18
|
+
retries?: number;
|
|
19
|
+
retryTime?: string[];
|
|
20
|
+
}
|
|
21
|
+
export interface ProcessErrorData {
|
|
22
|
+
stream: string;
|
|
23
|
+
message: OriginalStreamMessageReply;
|
|
24
|
+
retries: number;
|
|
25
|
+
}
|
|
26
|
+
export declare class RedisConsumer<S extends RedisScripts = RedisScripts> {
|
|
27
|
+
client: RedisClientType<any, any, any>;
|
|
28
|
+
private originalClient;
|
|
29
|
+
private state;
|
|
30
|
+
private retryProcessor;
|
|
31
|
+
private successfullMessages;
|
|
32
|
+
private BLOCK;
|
|
33
|
+
private COUNT;
|
|
34
|
+
private RETRIES;
|
|
35
|
+
constructor(client: RedisClient<S>, options?: ConsumerOptions);
|
|
36
|
+
set block(block: number);
|
|
37
|
+
set count(count: number);
|
|
38
|
+
set retries(retries: number);
|
|
39
|
+
get settings(): {
|
|
40
|
+
block: number;
|
|
41
|
+
count: number;
|
|
42
|
+
};
|
|
43
|
+
listen(streams: StreamToListen | StreamsToListen): Promise<void>;
|
|
44
|
+
addAckMessage(stream: string, id: StreamMessageId): void;
|
|
45
|
+
private listenForStreams;
|
|
46
|
+
private hasStreamState;
|
|
47
|
+
private getStreamState;
|
|
48
|
+
private initStreamState;
|
|
49
|
+
private readStreams;
|
|
50
|
+
private processStreamMessages;
|
|
51
|
+
private acknowlegdeMessages;
|
|
52
|
+
}
|
package/dist/consumer.js
ADDED
|
@@ -0,0 +1,162 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.RedisConsumer = void 0;
|
|
4
|
+
const retry_processor_1 = require("./retry-processor");
|
|
5
|
+
class RedisConsumer {
|
|
6
|
+
client;
|
|
7
|
+
originalClient;
|
|
8
|
+
state;
|
|
9
|
+
retryProcessor;
|
|
10
|
+
successfullMessages = new Map();
|
|
11
|
+
BLOCK;
|
|
12
|
+
COUNT;
|
|
13
|
+
RETRIES;
|
|
14
|
+
constructor(client, options = {}) {
|
|
15
|
+
this.originalClient = client;
|
|
16
|
+
this.client = client.duplicate();
|
|
17
|
+
this.state = {};
|
|
18
|
+
this.COUNT = options.COUNT ?? 1;
|
|
19
|
+
this.BLOCK = options.BLOCK ?? 0;
|
|
20
|
+
this.RETRIES = options.retries ?? 3;
|
|
21
|
+
this.retryProcessor = new retry_processor_1.RetryProcessor(this, {
|
|
22
|
+
retryTime: options.retryTime,
|
|
23
|
+
maxRetry: this.RETRIES,
|
|
24
|
+
});
|
|
25
|
+
this.client.connect();
|
|
26
|
+
}
|
|
27
|
+
set block(block) {
|
|
28
|
+
this.BLOCK = block;
|
|
29
|
+
}
|
|
30
|
+
set count(count) {
|
|
31
|
+
this.COUNT = count;
|
|
32
|
+
}
|
|
33
|
+
set retries(retries) {
|
|
34
|
+
this.RETRIES = retries;
|
|
35
|
+
}
|
|
36
|
+
get settings() {
|
|
37
|
+
return { block: this.BLOCK, count: this.COUNT };
|
|
38
|
+
}
|
|
39
|
+
async listen(streams) {
|
|
40
|
+
if (!Array.isArray(streams)) {
|
|
41
|
+
streams = [streams];
|
|
42
|
+
}
|
|
43
|
+
for (const stream of streams) {
|
|
44
|
+
const groupExists = await this.originalClient.groupExists(stream.name);
|
|
45
|
+
if (!groupExists) {
|
|
46
|
+
await this.originalClient.createGroup(stream.name);
|
|
47
|
+
}
|
|
48
|
+
if (this.hasStreamState(stream.name))
|
|
49
|
+
continue;
|
|
50
|
+
this.initStreamState(stream);
|
|
51
|
+
}
|
|
52
|
+
this.listenForStreams();
|
|
53
|
+
}
|
|
54
|
+
addAckMessage(stream, id) {
|
|
55
|
+
if (this.successfullMessages.has(stream)) {
|
|
56
|
+
const ackMessages = this.successfullMessages.get(stream);
|
|
57
|
+
ackMessages.push(id);
|
|
58
|
+
}
|
|
59
|
+
else {
|
|
60
|
+
this.successfullMessages.set(stream, [id]);
|
|
61
|
+
}
|
|
62
|
+
}
|
|
63
|
+
async listenForStreams() {
|
|
64
|
+
const state = this.state;
|
|
65
|
+
const streamsToListen = [];
|
|
66
|
+
for (const stream in state) {
|
|
67
|
+
streamsToListen.push({ key: stream, id: state[stream].nextId });
|
|
68
|
+
}
|
|
69
|
+
const streamsMessages = await this.readStreams(streamsToListen);
|
|
70
|
+
if (!streamsMessages) {
|
|
71
|
+
await this.acknowlegdeMessages();
|
|
72
|
+
this.listenForStreams();
|
|
73
|
+
return;
|
|
74
|
+
}
|
|
75
|
+
for (const streamMessages of streamsMessages) {
|
|
76
|
+
await this.processStreamMessages(streamMessages);
|
|
77
|
+
}
|
|
78
|
+
this.listenForStreams();
|
|
79
|
+
}
|
|
80
|
+
hasStreamState(name) {
|
|
81
|
+
return !!this.getStreamState(name);
|
|
82
|
+
}
|
|
83
|
+
getStreamState(name) {
|
|
84
|
+
const state = this.state[name];
|
|
85
|
+
if (state) {
|
|
86
|
+
return state;
|
|
87
|
+
}
|
|
88
|
+
else {
|
|
89
|
+
return null;
|
|
90
|
+
}
|
|
91
|
+
}
|
|
92
|
+
initStreamState(stream) {
|
|
93
|
+
const name = stream.name;
|
|
94
|
+
const executable = stream.executable;
|
|
95
|
+
let lastSuccessId;
|
|
96
|
+
if (stream.id) {
|
|
97
|
+
lastSuccessId = stream.id;
|
|
98
|
+
}
|
|
99
|
+
else {
|
|
100
|
+
lastSuccessId = '0-0';
|
|
101
|
+
}
|
|
102
|
+
let nextId = lastSuccessId;
|
|
103
|
+
this.state[name] = { nextId, lastSuccessId, executable, recovering: true };
|
|
104
|
+
}
|
|
105
|
+
async readStreams(streamsToListen) {
|
|
106
|
+
const messages = await this.client.xReadGroup(this.originalClient.groupName, this.originalClient.clientName, streamsToListen, { BLOCK: this.BLOCK, COUNT: this.COUNT });
|
|
107
|
+
if (!messages)
|
|
108
|
+
return null;
|
|
109
|
+
else
|
|
110
|
+
return messages;
|
|
111
|
+
}
|
|
112
|
+
async processStreamMessages(streamMessages) {
|
|
113
|
+
const stream = streamMessages.name;
|
|
114
|
+
const state = this.getStreamState(stream);
|
|
115
|
+
if (!state)
|
|
116
|
+
throw new Error('No state was found for stream processing of ' + stream);
|
|
117
|
+
const fnc = state.executable;
|
|
118
|
+
const messages = streamMessages.messages;
|
|
119
|
+
for (const message of messages) {
|
|
120
|
+
try {
|
|
121
|
+
await fnc(message, stream);
|
|
122
|
+
this.addAckMessage(stream, message.id);
|
|
123
|
+
}
|
|
124
|
+
catch (err) {
|
|
125
|
+
this.client.emit('process-error', err, { stream, message, retries: 0 });
|
|
126
|
+
if (this.RETRIES === 0)
|
|
127
|
+
continue;
|
|
128
|
+
if (err instanceof Error) {
|
|
129
|
+
this.retryProcessor.add(err, stream, message, fnc);
|
|
130
|
+
}
|
|
131
|
+
else {
|
|
132
|
+
const newErr = new Error(String(err));
|
|
133
|
+
this.retryProcessor.add(newErr, stream, message, fnc);
|
|
134
|
+
}
|
|
135
|
+
}
|
|
136
|
+
}
|
|
137
|
+
await this.acknowlegdeMessages();
|
|
138
|
+
const recovering = state.recovering;
|
|
139
|
+
if (recovering && messages.length === 0) {
|
|
140
|
+
state.nextId = '>';
|
|
141
|
+
state.recovering = false;
|
|
142
|
+
}
|
|
143
|
+
else if (recovering) {
|
|
144
|
+
const lastMessage = messages.slice(-1);
|
|
145
|
+
const lastId = lastMessage[0].id;
|
|
146
|
+
state.nextId = lastId;
|
|
147
|
+
}
|
|
148
|
+
return true;
|
|
149
|
+
}
|
|
150
|
+
async acknowlegdeMessages() {
|
|
151
|
+
for (const item of this.successfullMessages) {
|
|
152
|
+
const stream = item[0];
|
|
153
|
+
const ackMessages = item[1];
|
|
154
|
+
const id = await this.originalClient.xAck(stream, this.originalClient.groupName, ackMessages);
|
|
155
|
+
if (id) {
|
|
156
|
+
this.successfullMessages.delete(stream);
|
|
157
|
+
this.originalClient.xDel(stream, ackMessages);
|
|
158
|
+
}
|
|
159
|
+
}
|
|
160
|
+
}
|
|
161
|
+
}
|
|
162
|
+
exports.RedisConsumer = RedisConsumer;
|
package/dist/helpers.js
ADDED
package/dist/index.d.ts
ADDED
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
import { StreamMessageReply } from '@redis/client/dist/lib/commands/generic-transformers';
|
|
2
|
+
import { PropType } from './helpers';
|
|
3
|
+
export * from 'redis';
|
|
4
|
+
export { RedisClient } from './client';
|
|
5
|
+
export * from './consumer';
|
|
6
|
+
export * from './producer';
|
|
7
|
+
export { RetryFailedMessage } from './retry-processor';
|
|
8
|
+
export type StreamMessageId = PropType<StreamMessageReply, 'id'>;
|
|
9
|
+
export type StreamMessageData = PropType<StreamMessageReply, 'message'>;
|
package/dist/index.js
ADDED
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
|
|
3
|
+
if (k2 === undefined) k2 = k;
|
|
4
|
+
var desc = Object.getOwnPropertyDescriptor(m, k);
|
|
5
|
+
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
|
|
6
|
+
desc = { enumerable: true, get: function() { return m[k]; } };
|
|
7
|
+
}
|
|
8
|
+
Object.defineProperty(o, k2, desc);
|
|
9
|
+
}) : (function(o, m, k, k2) {
|
|
10
|
+
if (k2 === undefined) k2 = k;
|
|
11
|
+
o[k2] = m[k];
|
|
12
|
+
}));
|
|
13
|
+
var __exportStar = (this && this.__exportStar) || function(m, exports) {
|
|
14
|
+
for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
|
|
15
|
+
};
|
|
16
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
17
|
+
exports.RedisClient = void 0;
|
|
18
|
+
__exportStar(require("redis"), exports);
|
|
19
|
+
var client_1 = require("./client");
|
|
20
|
+
Object.defineProperty(exports, "RedisClient", { enumerable: true, get: function () { return client_1.RedisClient; } });
|
|
21
|
+
__exportStar(require("./consumer"), exports);
|
|
22
|
+
__exportStar(require("./producer"), exports);
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
import { RedisScripts } from 'redis';
|
|
2
|
+
import { StreamMessageData, StreamMessageId } from '.';
|
|
3
|
+
import { RedisClient } from './client';
|
|
4
|
+
export declare class RedisProducer<S extends RedisScripts = RedisScripts> {
|
|
5
|
+
private client;
|
|
6
|
+
constructor(client: RedisClient<S>);
|
|
7
|
+
add(stream: StreamMessageId, message: StreamMessageData): void;
|
|
8
|
+
}
|
package/dist/producer.js
ADDED
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.RedisProducer = void 0;
|
|
4
|
+
class RedisProducer {
|
|
5
|
+
client;
|
|
6
|
+
constructor(client) {
|
|
7
|
+
this.client = client;
|
|
8
|
+
}
|
|
9
|
+
add(stream, message) {
|
|
10
|
+
this.client.xAdd(stream, '*', message);
|
|
11
|
+
}
|
|
12
|
+
}
|
|
13
|
+
exports.RedisProducer = RedisProducer;
|
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
/// <reference types="node" />
|
|
2
|
+
import { StreamProcessingFunction, RedisConsumer } from './consumer';
|
|
3
|
+
import { StreamMessageReply } from '@redis/client/dist/lib/commands/generic-transformers';
|
|
4
|
+
import { RedisScripts } from 'redis';
|
|
5
|
+
import { EventEmitter } from 'stream';
|
|
6
|
+
interface RetryState {
|
|
7
|
+
lastError: Error;
|
|
8
|
+
timestamps: number[];
|
|
9
|
+
retries: number;
|
|
10
|
+
message: StreamMessageReply;
|
|
11
|
+
stream: string;
|
|
12
|
+
executable: StreamProcessingFunction<any>;
|
|
13
|
+
}
|
|
14
|
+
export type RetryFailedMessage = Omit<RetryState, 'executable' | 'lastError'>;
|
|
15
|
+
export type RetryMessage = Omit<RetryFailedMessage, 'timestamps' | 'lastError'> & {
|
|
16
|
+
timestamp: number;
|
|
17
|
+
};
|
|
18
|
+
interface RetryProcessorOptions {
|
|
19
|
+
maxRetry: number;
|
|
20
|
+
retryTime?: string[];
|
|
21
|
+
}
|
|
22
|
+
export declare class RetryProcessor<S extends RedisScripts = RedisScripts> extends EventEmitter {
|
|
23
|
+
private consumer;
|
|
24
|
+
private state;
|
|
25
|
+
private retryTime;
|
|
26
|
+
private maxRetry;
|
|
27
|
+
constructor(consumer: RedisConsumer<S>, options: RetryProcessorOptions);
|
|
28
|
+
add(error: Error, stream: string, message: StreamMessageReply, executable: StreamProcessingFunction<any>): void;
|
|
29
|
+
private processRetry;
|
|
30
|
+
private calcTimeoutTime;
|
|
31
|
+
private emitRetryFail;
|
|
32
|
+
private emitRetry;
|
|
33
|
+
private emitProcessFailed;
|
|
34
|
+
}
|
|
35
|
+
export {};
|
|
@@ -0,0 +1,84 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.RetryProcessor = void 0;
|
|
4
|
+
const helpers_1 = require("./helpers");
|
|
5
|
+
const stream_1 = require("stream");
|
|
6
|
+
class RetryProcessor extends stream_1.EventEmitter {
|
|
7
|
+
consumer;
|
|
8
|
+
state = new Map();
|
|
9
|
+
retryTime;
|
|
10
|
+
maxRetry;
|
|
11
|
+
constructor(consumer, options) {
|
|
12
|
+
super();
|
|
13
|
+
this.consumer = consumer;
|
|
14
|
+
this.retryTime = options.retryTime || ['15s', '1m', '15m'];
|
|
15
|
+
this.maxRetry = options.maxRetry;
|
|
16
|
+
}
|
|
17
|
+
add(error, stream, message, executable) {
|
|
18
|
+
const id = message.id;
|
|
19
|
+
if (this.state.has(id))
|
|
20
|
+
return;
|
|
21
|
+
this.state.set(id, { lastError: error, timestamps: [], retries: 0, message, stream, executable });
|
|
22
|
+
this.processRetry(id);
|
|
23
|
+
}
|
|
24
|
+
async processRetry(id) {
|
|
25
|
+
const stateObj = this.state.get(id);
|
|
26
|
+
if (stateObj.retries >= this.maxRetry) {
|
|
27
|
+
this.state.delete(id);
|
|
28
|
+
this.emitRetryFail(stateObj);
|
|
29
|
+
return;
|
|
30
|
+
}
|
|
31
|
+
stateObj.retries++;
|
|
32
|
+
const timestamp = new Date().getTime();
|
|
33
|
+
stateObj.timestamps.push(timestamp);
|
|
34
|
+
const timeoutTime = this.calcTimeoutTime(stateObj);
|
|
35
|
+
await (0, helpers_1.timeout)(timeoutTime);
|
|
36
|
+
const fnc = stateObj.executable;
|
|
37
|
+
const message = stateObj.message;
|
|
38
|
+
try {
|
|
39
|
+
this.emitRetry(stateObj);
|
|
40
|
+
await fnc(message, stateObj.stream);
|
|
41
|
+
this.consumer.addAckMessage(stateObj.stream, id);
|
|
42
|
+
this.state.delete(id);
|
|
43
|
+
}
|
|
44
|
+
catch (err) {
|
|
45
|
+
if (err instanceof Error) {
|
|
46
|
+
stateObj.lastError = err;
|
|
47
|
+
}
|
|
48
|
+
else {
|
|
49
|
+
const newErr = new Error(String(err));
|
|
50
|
+
stateObj.lastError = newErr;
|
|
51
|
+
}
|
|
52
|
+
this.emitProcessFailed(stateObj);
|
|
53
|
+
this.processRetry(id);
|
|
54
|
+
}
|
|
55
|
+
return;
|
|
56
|
+
}
|
|
57
|
+
calcTimeoutTime(stateObj) {
|
|
58
|
+
const retry = stateObj.retries;
|
|
59
|
+
const index = (retry > this.retryTime.length ? this.retryTime.length : retry) - 1;
|
|
60
|
+
const timeString = this.retryTime[index];
|
|
61
|
+
let hours = +timeString.replace(/(\d+)h/, '$1');
|
|
62
|
+
let minutes = +timeString.replace(/(\d+)m/, '$1');
|
|
63
|
+
let seconds = +timeString.replace(/(\d+)s/, '$1');
|
|
64
|
+
if (isNaN(hours))
|
|
65
|
+
hours = 0;
|
|
66
|
+
if (isNaN(minutes))
|
|
67
|
+
minutes = 0;
|
|
68
|
+
if (isNaN(seconds))
|
|
69
|
+
seconds = 0;
|
|
70
|
+
let timeoutTime = hours * 3600 * 1000 + minutes * 60 * 1000 + seconds * 1000;
|
|
71
|
+
return timeoutTime;
|
|
72
|
+
}
|
|
73
|
+
emitRetryFail({ lastError, stream, message, retries, timestamps }) {
|
|
74
|
+
this.consumer.client.emit('retry-failed', lastError, { stream, message, retries, timestamps });
|
|
75
|
+
}
|
|
76
|
+
emitRetry({ stream, message, retries, timestamps }) {
|
|
77
|
+
const timestamp = timestamps[timestamps.length - 1];
|
|
78
|
+
this.consumer.client.emit('retry', { stream, message, retries, timestamp });
|
|
79
|
+
}
|
|
80
|
+
emitProcessFailed({ lastError, stream, message, retries }) {
|
|
81
|
+
this.consumer.client.emit('process-error', lastError, { stream, message, retries });
|
|
82
|
+
}
|
|
83
|
+
}
|
|
84
|
+
exports.RetryProcessor = RetryProcessor;
|
package/package.json
ADDED
|
@@ -0,0 +1,42 @@
|
|
|
1
|
+
{
|
|
2
|
+
"name": "redis-consumer",
|
|
3
|
+
"version": "1.1.1",
|
|
4
|
+
"description": "Simple NodeJs consumer and producer for Redis Streams",
|
|
5
|
+
"main": "./dist/index.js",
|
|
6
|
+
"types": "./dist/index.d.ts",
|
|
7
|
+
"files": [
|
|
8
|
+
"dist/"
|
|
9
|
+
],
|
|
10
|
+
"scripts": {
|
|
11
|
+
"build": "tsc"
|
|
12
|
+
},
|
|
13
|
+
"repository": {
|
|
14
|
+
"type": "git",
|
|
15
|
+
"url": "git+https://github.com/Seboeb/redis-streams-nodejs.git"
|
|
16
|
+
},
|
|
17
|
+
"author": "Sebastiaan Hekner",
|
|
18
|
+
"keywords": [
|
|
19
|
+
"redis",
|
|
20
|
+
"streams",
|
|
21
|
+
"consumer",
|
|
22
|
+
"producer",
|
|
23
|
+
"messaging",
|
|
24
|
+
"pub/sub",
|
|
25
|
+
"queue",
|
|
26
|
+
"broker",
|
|
27
|
+
"typescript",
|
|
28
|
+
"nodejs"
|
|
29
|
+
],
|
|
30
|
+
"license": "ISC",
|
|
31
|
+
"bugs": {
|
|
32
|
+
"url": "https://github.com/Seboeb/redis-streams-nodejs/issues"
|
|
33
|
+
},
|
|
34
|
+
"homepage": "https://github.com/Seboeb/redis-streams-nodejs#readme",
|
|
35
|
+
"dependencies": {
|
|
36
|
+
"redis": "^4.6.10"
|
|
37
|
+
},
|
|
38
|
+
"devDependencies": {
|
|
39
|
+
"@types/node": "^18.0.0",
|
|
40
|
+
"typescript": "latest"
|
|
41
|
+
}
|
|
42
|
+
}
|