redis-smq 7.0.0-rc.6 → 7.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.codecov.yml +5 -0
- package/CHANGELOG.md +27 -0
- package/README.md +24 -22
- package/dist/index.d.ts +0 -1
- package/dist/index.js +1 -4
- package/dist/src/common/event-listeners/event-provider.d.ts +8 -0
- package/dist/src/common/event-listeners/event-provider.js +13 -0
- package/dist/src/common/redis-keys/redis-keys.js +1 -1
- package/dist/src/config/configuration.js +2 -0
- package/dist/src/config/event-listeners/event-listeners.d.ts +2 -0
- package/dist/src/config/event-listeners/event-listeners.js +14 -0
- package/dist/src/lib/base.js +0 -2
- package/dist/src/lib/consumer/consumer-heartbeat.d.ts +3 -3
- package/dist/src/lib/consumer/consumer-heartbeat.js +12 -14
- package/dist/src/lib/{broker → consumer/consumer-message-handler}/acknowledge-message.d.ts +2 -2
- package/dist/src/lib/{broker → consumer/consumer-message-handler}/acknowledge-message.js +1 -1
- package/dist/src/lib/consumer/consumer-message-handler/consume-message.d.ts +4 -2
- package/dist/src/lib/consumer/consumer-message-handler/consume-message.js +18 -17
- package/dist/src/lib/{broker → consumer/consumer-message-handler}/dead-letter-message.d.ts +2 -2
- package/dist/src/lib/{broker → consumer/consumer-message-handler}/dead-letter-message.js +1 -1
- package/dist/src/lib/{broker → consumer/consumer-message-handler}/delay-message.d.ts +2 -2
- package/dist/src/lib/{broker → consumer/consumer-message-handler}/delay-message.js +1 -1
- package/dist/src/lib/consumer/consumer-message-handler/dequeue-message.d.ts +1 -1
- package/dist/src/lib/consumer/consumer-message-handler/message-handler.d.ts +7 -4
- package/dist/src/lib/consumer/consumer-message-handler/message-handler.js +53 -30
- package/dist/src/lib/consumer/consumer-message-handler/multiplexed-message-handler/multiplexed-message-handler.js +3 -16
- package/dist/src/lib/consumer/consumer-message-handler/processing-queue.d.ts +3 -1
- package/dist/src/lib/consumer/consumer-message-handler/processing-queue.js +11 -5
- package/dist/src/lib/{broker → consumer/consumer-message-handler}/requeue-message.d.ts +2 -2
- package/dist/src/lib/{broker → consumer/consumer-message-handler}/requeue-message.js +1 -1
- package/dist/src/lib/consumer/consumer-message-handler/retry-message.d.ts +15 -0
- package/dist/src/lib/{broker → consumer/consumer-message-handler}/retry-message.js +26 -8
- package/dist/src/lib/consumer/consumer.d.ts +13 -11
- package/dist/src/lib/consumer/consumer.js +2 -2
- package/dist/src/lib/producer/producer.d.ts +3 -3
- package/dist/src/lib/producer/producer.js +8 -8
- package/dist/src/lib/{broker → producer}/schedule-message.d.ts +0 -0
- package/dist/src/lib/{broker → producer}/schedule-message.js +1 -1
- package/dist/src/workers/delay.worker.js +1 -1
- package/dist/src/workers/{heartbeat-monitor.worker.d.ts → watchdog.worker.d.ts} +5 -4
- package/dist/src/workers/watchdog.worker.js +74 -0
- package/dist/types/index.d.ts +17 -9
- package/dist/types/index.js +2 -2
- package/package.json +2 -2
- package/dist/src/lib/broker/retry-message.d.ts +0 -6
- package/dist/src/lib/consumer/offline-consumers.d.ts +0 -4
- package/dist/src/lib/consumer/offline-consumers.js +0 -18
- package/dist/src/plugins/errors/plugin-registration-not-allowed-error.d.ts +0 -4
- package/dist/src/plugins/errors/plugin-registration-not-allowed-error.js +0 -11
- package/dist/src/plugins/plugins.d.ts +0 -6
- package/dist/src/plugins/plugins.js +0 -35
- package/dist/src/workers/heartbeat-monitor.worker.js +0 -23
package/.codecov.yml
ADDED
package/CHANGELOG.md
CHANGED
|
@@ -1,5 +1,32 @@
|
|
|
1
1
|
# CHANGELOG
|
|
2
2
|
|
|
3
|
+
## 7.0.0 (2022-06-18)
|
|
4
|
+
|
|
5
|
+
* Fix typing issue (cd8bf6a)
|
|
6
|
+
* Bump up redis-smq-common to v1.0.0 (02c353d)
|
|
7
|
+
* Update callback vs promise vs async/await benchmarks (6fca49c)
|
|
8
|
+
* Update README.md (7fbdcbf)
|
|
9
|
+
|
|
10
|
+
## 7.0.0-rc.8 (2022-06-09)
|
|
11
|
+
|
|
12
|
+
* Update docs (96a45e7)
|
|
13
|
+
* Test consumers/producers event listeners (856e12d)
|
|
14
|
+
* Implement consumers/producers event listeners (6f5d603)
|
|
15
|
+
|
|
16
|
+
## 7.0.0-rc.7 (2022-06-07)
|
|
17
|
+
|
|
18
|
+
* Update redis keys prefix (a6ca852)
|
|
19
|
+
* Use codecov instead of coveralls (d8b5f83)
|
|
20
|
+
* Improve consuming-messages/test00013 (5cd823c)
|
|
21
|
+
* Improve consuming-messages/test00010 (a9a4f47)
|
|
22
|
+
* Test WatchdogWorker (73fbe9a)
|
|
23
|
+
* Clean up (aa9a51a)
|
|
24
|
+
* Improve unacknowleged messages handling, refactor (005c6a0)
|
|
25
|
+
* Fix outdated javascript examples (e31f89c)
|
|
26
|
+
* Keep a clean directory structure (642739f)
|
|
27
|
+
* Update README.md (5c011e3)
|
|
28
|
+
* Clean up tests (10ebb77)
|
|
29
|
+
|
|
3
30
|
## 7.0.0-rc.6 (2022-05-31)
|
|
4
31
|
|
|
5
32
|
* Bump up redis-smq-common to v1.0.0-rc.11 (43717f0)
|
package/README.md
CHANGED
|
@@ -7,7 +7,7 @@
|
|
|
7
7
|
|
|
8
8
|
<p>
|
|
9
9
|
<a href="https://github.com/weyoss/redis-smq/actions/workflows/tests.yml"><img src="https://github.com/weyoss/redis-smq/actions/workflows/tests.yml/badge.svg" alt="Tests" style="max-width:100%;" /></a>
|
|
10
|
-
<a href="https://
|
|
10
|
+
<a href="https://codecov.io/github/weyoss/redis-smq?branch=master" rel="nofollow"><img src="https://img.shields.io/codecov/c/github/weyoss/redis-smq" alt="Coverage Status" /></a>
|
|
11
11
|
<a href="https://npmjs.org/package/redis-smq" rel="nofollow"><img src="https://img.shields.io/npm/v/redis-smq.svg" alt="NPM version" /></a>
|
|
12
12
|
<a href="https://npmjs.org/package/redis-smq" rel="nofollow"><img src="https://img.shields.io/npm/dm/redis-smq.svg" alt="NPM downloads" /></a>
|
|
13
13
|
<a href="https://lgtm.com/projects/g/weyoss/redis-smq/context:javascript" rel="nofollow"><img src="https://img.shields.io/lgtm/grade/javascript/github/weyoss/redis-smq.svg?logo=lgtm&logoWidth=18" alt="Code quality" /></a>
|
|
@@ -19,9 +19,9 @@ RedisSMQ is a Node.js library for queuing messages (aka jobs) and processing the
|
|
|
19
19
|
|
|
20
20
|
## Features
|
|
21
21
|
|
|
22
|
-
* **[High-performance message processing](/docs/performance.md)
|
|
23
|
-
* **[Multi-Queue Producers](#producer-class) & [Multi-Queue Consumers](#consumer-class)**: Offering
|
|
24
|
-
* **[
|
|
22
|
+
* **[High-performance message processing](/docs/performance.md)**.
|
|
23
|
+
* **[Multi-Queue Producers](#producer-class) & [Multi-Queue Consumers](#consumer-class)**: Offering flexible Producer/Consumer models, with focus on simplicity and without tons of features. This can make RedisSMQ an ideal message broker for your microservices.
|
|
24
|
+
* **[Supports both at-least-once/at-most-once delivery](/docs/api/message.md#messageprototypesetretrythreshold)**: In case of failures, while delivering or processing a message, RedisSMQ can guaranty that the message will be not lost and redelivered again. When configured to do so, RedisSMQ can also ensure that the message is delivered at-most-once.
|
|
25
25
|
* **[Message Expiration](/docs/api/message.md#messageprototypesetttl)**: A message will not be delivered if it has been in a queue for longer than a given amount of time, called TTL (time-to-live).
|
|
26
26
|
* **[Message Consumption Timeout](/docs/api/message.md#messageprototypesetconsumetimeout)**: Timeout for consuming messages.
|
|
27
27
|
* **[Queue Rate Limiting](/docs/queue-rate-limiting.md)**: Allowing you to control the rate at which the messages are consumed from a given queue.
|
|
@@ -30,11 +30,10 @@ RedisSMQ is a Node.js library for queuing messages (aka jobs) and processing the
|
|
|
30
30
|
* **[Multiplexing](/docs/multiplexing.md)**: A feature which allows message handlers to use a single redis connection to dequeue and consume messages.
|
|
31
31
|
* **[HTTP API](https://github.com/weyoss/redis-smq-monitor)**: an HTTP interface is provided to interact with the MQ.
|
|
32
32
|
* **[Web UI](https://github.com/weyoss/redis-smq-monitor-client)**: RedisSMQ can be managed also from your web browser.
|
|
33
|
-
* **[Logging](https://github.com/weyoss/redis-smq-common/blob/master/docs/logs.md)**:
|
|
33
|
+
* **[Logging](https://github.com/weyoss/redis-smq-common/blob/master/docs/logs.md)**: RedisSMQ comes with a built-in JSON logger, but can also use your application logger.
|
|
34
34
|
* **[Configurable](/docs/configuration.md)**: Many options and features can be configured.
|
|
35
|
-
* **
|
|
36
|
-
* **
|
|
37
|
-
* **Highly optimized**: Strongly-typed and implemented using pure callbacks, with small memory footprint and no memory leaks. See [callbacks vs promises vs async/await benchmarks](http://bluebirdjs.com/docs/benchmarks.html).
|
|
35
|
+
* **[Supports Multiple Redis clients](/docs/configuration.md)**: Depending on your preferences, RedisSMQ can use either [node-redis v3](https://github.com/redis/node-redis/tree/v3.1.2), [node-redis v4](https://github.com/redis/node-redis), or [ioredis](https://github.com/luin/ioredis).
|
|
36
|
+
* **[Highly optimized](https://lgtm.com/projects/g/weyoss/redis-smq/context:javascript)**: Strongly-typed and implemented using pure callbacks, with small memory footprint and no memory leaks. See [Callback vs Promise vs Async/Await benchmarks](https://gist.github.com/weyoss/24f9ecbda175d943a48cb7ec38bde821).
|
|
38
37
|
|
|
39
38
|
|
|
40
39
|
### RedisSMQ Use Case: Multi-Queue Producers & Multi-Queue Consumers
|
|
@@ -53,6 +52,7 @@ RedisSMQ is a Node.js library for queuing messages (aka jobs) and processing the
|
|
|
53
52
|
1. [Message Class](#message-class)
|
|
54
53
|
2. [Producer Class](#producer-class)
|
|
55
54
|
3. [Consumer Class](#consumer-class)
|
|
55
|
+
1. [Message Acknowledgement](#message-acknowledgement)
|
|
56
56
|
2. Advanced Topics
|
|
57
57
|
1. [Scheduling Messages](/docs/scheduling-messages.md)
|
|
58
58
|
2. [Priority Queues](/docs/priority-queues.md)
|
|
@@ -133,7 +133,7 @@ See [Message Reference](/docs/api/message.md) for more details.
|
|
|
133
133
|
|
|
134
134
|
#### Producer Class
|
|
135
135
|
|
|
136
|
-
`Producer`
|
|
136
|
+
A `Producer` instance allows you to publish a message to a queue.
|
|
137
137
|
|
|
138
138
|
You can use a single `Producer` instance to produce messages, including messages with priority, to multiple queues.
|
|
139
139
|
|
|
@@ -166,19 +166,17 @@ See [Producer Reference](/docs/api/producer.md) for more details.
|
|
|
166
166
|
|
|
167
167
|
#### Consumer Class
|
|
168
168
|
|
|
169
|
-
`Consumer`
|
|
169
|
+
A `Consumer` instance can be used to receive and consume messages from one or multiple queues.
|
|
170
170
|
|
|
171
|
-
|
|
171
|
+
To consume messages from a queue, the `Consumer` class provides the [consume()](/docs/api/consumer.md#consumerprototypeconsume) method which allows you to register a `message handler`.
|
|
172
172
|
|
|
173
|
-
|
|
173
|
+
A `message handler` is a function that receives a delivered message from a given queue.
|
|
174
174
|
|
|
175
|
-
|
|
175
|
+
Message handlers can be registered at any time, before or after a consumer has been started.
|
|
176
176
|
|
|
177
|
-
|
|
177
|
+
In contrast to producers, consumers are not automatically started upon creation. To start a consumer use the [run()](/docs/api/consumer.md#consumerprototyperun) method.
|
|
178
178
|
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
To shut down and remove a given `message handler` from your consumer, use the [cancel()](/docs/api/consumer.md#consumerprototypecancel) method.
|
|
179
|
+
To stop consuming messages from a queue and to remove the associated `message handler` from your consumer, use the [cancel()](/docs/api/consumer.md#consumerprototypecancel) method.
|
|
182
180
|
|
|
183
181
|
To shut down completely your consumer and tear down all message handlers, use the [shutdown()](/docs/api/consumer.md#consumerprototypeshutdown) method.
|
|
184
182
|
|
|
@@ -202,17 +200,21 @@ consumer.consume('test_queue', messageHandler, (err) => {
|
|
|
202
200
|
consumer.run();
|
|
203
201
|
```
|
|
204
202
|
|
|
203
|
+
##### Message Acknowledgement
|
|
204
|
+
|
|
205
205
|
Once a message is received, to acknowledge it, you invoke the callback function without arguments, as shown in the example above.
|
|
206
206
|
|
|
207
|
-
Message acknowledgment informs the MQ that
|
|
207
|
+
Message acknowledgment informs the MQ that the delivered message has been successfully consumed.
|
|
208
|
+
|
|
209
|
+
If an error occurred while processing a message, you can unacknowledge it by passing in the error to the callback function.
|
|
208
210
|
|
|
209
|
-
|
|
211
|
+
By default, unacknowledged messages are re-queued and delivered again unless **message retry threshold** is exceeded.
|
|
210
212
|
|
|
211
|
-
|
|
213
|
+
Delivered messages that couldn't be processed or can not be delivered to consumers are moved to a system generated queue called **dead-letter queue (DLQ)**.
|
|
212
214
|
|
|
213
|
-
|
|
215
|
+
By default, RedisSMQ does not store acknowledged and dead-lettered messages for saving disk and memory spaces, and also to increase message processing performance.
|
|
214
216
|
|
|
215
|
-
|
|
217
|
+
If you need such feature, you can enable it from your [configuration](/docs/configuration.md) object.
|
|
216
218
|
|
|
217
219
|
See [Consumer Reference](/docs/api/consumer.md) for more details.
|
|
218
220
|
|
package/dist/index.d.ts
CHANGED
|
@@ -4,4 +4,3 @@ export { Producer } from './src/lib/producer/producer';
|
|
|
4
4
|
export { Message } from './src/lib/message/message';
|
|
5
5
|
export { MessageManager } from './src/lib/message-manager/message-manager';
|
|
6
6
|
export { QueueManager } from './src/lib/queue-manager/queue-manager';
|
|
7
|
-
export { registerProducerPlugin, registerConsumerPlugin, } from './src/plugins/plugins';
|
package/dist/index.js
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
-
exports.
|
|
3
|
+
exports.QueueManager = exports.MessageManager = exports.Message = exports.Producer = exports.Consumer = exports.events = void 0;
|
|
4
4
|
var events_1 = require("./src/common/events/events");
|
|
5
5
|
Object.defineProperty(exports, "events", { enumerable: true, get: function () { return events_1.events; } });
|
|
6
6
|
var consumer_1 = require("./src/lib/consumer/consumer");
|
|
@@ -13,7 +13,4 @@ var message_manager_1 = require("./src/lib/message-manager/message-manager");
|
|
|
13
13
|
Object.defineProperty(exports, "MessageManager", { enumerable: true, get: function () { return message_manager_1.MessageManager; } });
|
|
14
14
|
var queue_manager_1 = require("./src/lib/queue-manager/queue-manager");
|
|
15
15
|
Object.defineProperty(exports, "QueueManager", { enumerable: true, get: function () { return queue_manager_1.QueueManager; } });
|
|
16
|
-
var plugins_1 = require("./src/plugins/plugins");
|
|
17
|
-
Object.defineProperty(exports, "registerProducerPlugin", { enumerable: true, get: function () { return plugins_1.registerProducerPlugin; } });
|
|
18
|
-
Object.defineProperty(exports, "registerConsumerPlugin", { enumerable: true, get: function () { return plugins_1.registerConsumerPlugin; } });
|
|
19
16
|
//# sourceMappingURL=index.js.map
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
/// <reference types="node" />
|
|
2
|
+
import { IEventProvider } from '../../../types';
|
|
3
|
+
import { EventEmitter } from 'events';
|
|
4
|
+
export declare class EventProvider implements IEventProvider {
|
|
5
|
+
protected eventEmitter: EventEmitter;
|
|
6
|
+
constructor(eventEmitter: EventEmitter);
|
|
7
|
+
on(event: string, listener: (...args: any[]) => void): void;
|
|
8
|
+
}
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.EventProvider = void 0;
|
|
4
|
+
class EventProvider {
|
|
5
|
+
constructor(eventEmitter) {
|
|
6
|
+
this.eventEmitter = eventEmitter;
|
|
7
|
+
}
|
|
8
|
+
on(event, listener) {
|
|
9
|
+
this.eventEmitter.on(event, listener);
|
|
10
|
+
}
|
|
11
|
+
}
|
|
12
|
+
exports.EventProvider = EventProvider;
|
|
13
|
+
//# sourceMappingURL=event-provider.js.map
|
|
@@ -3,7 +3,7 @@ Object.defineProperty(exports, "__esModule", { value: true });
|
|
|
3
3
|
exports.redisKeys = void 0;
|
|
4
4
|
const redis_keys_error_1 = require("./redis-keys.error");
|
|
5
5
|
const keySegmentSeparator = ':';
|
|
6
|
-
const nsPrefix = 'redis-smq-
|
|
6
|
+
const nsPrefix = 'redis-smq-v7';
|
|
7
7
|
const globalNamespace = 'global';
|
|
8
8
|
var ERedisKey;
|
|
9
9
|
(function (ERedisKey) {
|
|
@@ -5,12 +5,14 @@ const namespace_1 = require("./namespace");
|
|
|
5
5
|
const redis_1 = require("./redis");
|
|
6
6
|
const logger_1 = require("./logger");
|
|
7
7
|
const messages_1 = require("./messages/messages");
|
|
8
|
+
const event_listeners_1 = require("./event-listeners/event-listeners");
|
|
8
9
|
function getConfiguration(config = {}) {
|
|
9
10
|
return {
|
|
10
11
|
namespace: (0, namespace_1.default)(config),
|
|
11
12
|
redis: (0, redis_1.default)(config),
|
|
12
13
|
logger: (0, logger_1.default)(config),
|
|
13
14
|
messages: (0, messages_1.default)(config),
|
|
15
|
+
eventListeners: (0, event_listeners_1.EventListeners)(config),
|
|
14
16
|
};
|
|
15
17
|
}
|
|
16
18
|
exports.getConfiguration = getConfiguration;
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.EventListeners = void 0;
|
|
4
|
+
const lodash_1 = require("lodash");
|
|
5
|
+
const defaultConfig = {
|
|
6
|
+
consumerEventListeners: [],
|
|
7
|
+
producerEventListeners: [],
|
|
8
|
+
};
|
|
9
|
+
function EventListeners(userConfig) {
|
|
10
|
+
const { eventListeners = {} } = userConfig;
|
|
11
|
+
return (0, lodash_1.merge)({}, defaultConfig, eventListeners);
|
|
12
|
+
}
|
|
13
|
+
exports.EventListeners = EventListeners;
|
|
14
|
+
//# sourceMappingURL=event-listeners.js.map
|
package/dist/src/lib/base.js
CHANGED
|
@@ -6,7 +6,6 @@ const events_1 = require("events");
|
|
|
6
6
|
const events_2 = require("../common/events/events");
|
|
7
7
|
const configuration_1 = require("../config/configuration");
|
|
8
8
|
const redis_smq_common_1 = require("redis-smq-common");
|
|
9
|
-
const plugins_1 = require("../plugins/plugins");
|
|
10
9
|
class Base extends events_1.EventEmitter {
|
|
11
10
|
constructor(config) {
|
|
12
11
|
super();
|
|
@@ -38,7 +37,6 @@ class Base extends events_1.EventEmitter {
|
|
|
38
37
|
this.config = (0, configuration_1.getConfiguration)(config);
|
|
39
38
|
this.logger = redis_smq_common_1.logger.getNamespacedLogger(this.config.logger, `${this.constructor.name.toLowerCase()}:${this.id}`);
|
|
40
39
|
this.registerEventsHandlers();
|
|
41
|
-
(0, plugins_1.disablePluginRegistration)();
|
|
42
40
|
}
|
|
43
41
|
registerEventsHandlers() {
|
|
44
42
|
this.on(events_2.events.GOING_UP, () => this.logger.info(`Going up...`));
|
|
@@ -1,9 +1,9 @@
|
|
|
1
1
|
/// <reference types="node" />
|
|
2
|
-
import {
|
|
2
|
+
import { TConsumerHeartbeat } from '../../../types';
|
|
3
3
|
import { RedisClient, Ticker } from 'redis-smq-common';
|
|
4
4
|
import { EventEmitter } from 'events';
|
|
5
5
|
import { Consumer } from './consumer';
|
|
6
|
-
import { ICallback } from 'redis-smq-common/dist/types';
|
|
6
|
+
import { ICallback, IRedisClientMulti } from 'redis-smq-common/dist/types';
|
|
7
7
|
export declare class ConsumerHeartbeat extends EventEmitter {
|
|
8
8
|
protected static readonly heartbeatTTL: number;
|
|
9
9
|
protected redisClient: RedisClient;
|
|
@@ -22,5 +22,5 @@ export declare class ConsumerHeartbeat extends EventEmitter {
|
|
|
22
22
|
}[]>): void;
|
|
23
23
|
static getValidHeartbeatIds(redisClient: RedisClient, cb: ICallback<string[]>): void;
|
|
24
24
|
static getExpiredHeartbeatIds(redisClient: RedisClient, cb: ICallback<string[]>): void;
|
|
25
|
-
static
|
|
25
|
+
static handleExpiredHeartbeatId(consumerId: string | string[], multi: IRedisClientMulti): void;
|
|
26
26
|
}
|
|
@@ -7,7 +7,6 @@ const events_1 = require("../../common/events/events");
|
|
|
7
7
|
const redis_keys_1 = require("../../common/redis-keys/redis-keys");
|
|
8
8
|
const events_2 = require("events");
|
|
9
9
|
const redis_smq_common_2 = require("redis-smq-common");
|
|
10
|
-
const offline_consumers_1 = require("./offline-consumers");
|
|
11
10
|
const cpuUsageStatsRef = {
|
|
12
11
|
cpuUsage: process.cpuUsage(),
|
|
13
12
|
time: process.hrtime(),
|
|
@@ -77,7 +76,11 @@ class ConsumerHeartbeat extends events_2.EventEmitter {
|
|
|
77
76
|
this.ticker.once(events_1.events.DOWN, cb);
|
|
78
77
|
this.ticker.quit();
|
|
79
78
|
},
|
|
80
|
-
(cb) =>
|
|
79
|
+
(cb) => {
|
|
80
|
+
const multi = this.redisClient.multi();
|
|
81
|
+
ConsumerHeartbeat.handleExpiredHeartbeatId(this.consumer.getId(), multi);
|
|
82
|
+
multi.exec((err) => cb(err));
|
|
83
|
+
},
|
|
81
84
|
(cb) => this.redisClient.halt(cb),
|
|
82
85
|
], cb);
|
|
83
86
|
}
|
|
@@ -163,18 +166,13 @@ class ConsumerHeartbeat extends events_2.EventEmitter {
|
|
|
163
166
|
cb(null, consumerIds !== null && consumerIds !== void 0 ? consumerIds : []);
|
|
164
167
|
});
|
|
165
168
|
}
|
|
166
|
-
static
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
(0, offline_consumers_1.handleOfflineConsumer)(config, multi, redisClient, consumerId, done);
|
|
174
|
-
}, () => multi.exec((err) => cb(err)));
|
|
175
|
-
}
|
|
176
|
-
else
|
|
177
|
-
cb();
|
|
169
|
+
static handleExpiredHeartbeatId(consumerId, multi) {
|
|
170
|
+
const { keyHeartbeats, keyHeartbeatConsumerWeight } = redis_keys_1.redisKeys.getMainKeys();
|
|
171
|
+
const ids = typeof consumerId === 'string' ? [consumerId] : consumerId;
|
|
172
|
+
ids.forEach((consumerId) => {
|
|
173
|
+
multi.hdel(keyHeartbeats, consumerId);
|
|
174
|
+
multi.zrem(keyHeartbeatConsumerWeight, consumerId);
|
|
175
|
+
});
|
|
178
176
|
}
|
|
179
177
|
}
|
|
180
178
|
exports.ConsumerHeartbeat = ConsumerHeartbeat;
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { RedisClient } from 'redis-smq-common';
|
|
2
|
-
import { Message } from '
|
|
2
|
+
import { Message } from '../../message/message';
|
|
3
3
|
import { ICallback } from 'redis-smq-common/dist/types';
|
|
4
|
-
import { IRequiredConfig } from '
|
|
4
|
+
import { IRequiredConfig } from '../../../../types';
|
|
5
5
|
export declare function acknowledgeMessage(config: IRequiredConfig, redisClient: RedisClient, message: Message, keyQueueProcessing: string, cb: ICallback<void>): void;
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
3
|
exports.acknowledgeMessage = void 0;
|
|
4
|
-
const redis_keys_1 = require("
|
|
4
|
+
const redis_keys_1 = require("../../../common/redis-keys/redis-keys");
|
|
5
5
|
function acknowledgeMessage(config, redisClient, message, keyQueueProcessing, cb) {
|
|
6
6
|
const queue = message.getRequiredQueue();
|
|
7
7
|
const { store, queueSize, expire } = config.messages.store.acknowledged;
|
|
@@ -2,12 +2,14 @@ import { Message } from '../../message/message';
|
|
|
2
2
|
import { EMessageUnacknowledgedCause } from '../../../../types';
|
|
3
3
|
import { MessageHandler } from './message-handler';
|
|
4
4
|
import { RedisClient } from 'redis-smq-common';
|
|
5
|
+
import { ICompatibleLogger } from 'redis-smq-common/dist/types';
|
|
5
6
|
export declare class ConsumeMessage {
|
|
6
7
|
protected keyQueueProcessing: string;
|
|
7
8
|
protected messageHandler: MessageHandler;
|
|
8
9
|
protected redisClient: RedisClient;
|
|
9
|
-
|
|
10
|
-
|
|
10
|
+
protected logger: ICompatibleLogger;
|
|
11
|
+
constructor(messageHandler: MessageHandler, redisClient: RedisClient, logger: ICompatibleLogger);
|
|
12
|
+
protected unacknowledgeMessage(msg: Message, cause: EMessageUnacknowledgedCause): void;
|
|
11
13
|
protected consumeMessage(msg: Message): void;
|
|
12
14
|
handleReceivedMessage(message: Message): void;
|
|
13
15
|
}
|
|
@@ -4,27 +4,28 @@ exports.ConsumeMessage = void 0;
|
|
|
4
4
|
const message_1 = require("../../message/message");
|
|
5
5
|
const types_1 = require("../../../../types");
|
|
6
6
|
const events_1 = require("../../../common/events/events");
|
|
7
|
-
const consumer_error_1 = require("../errors/consumer.error");
|
|
8
7
|
const redis_keys_1 = require("../../../common/redis-keys/redis-keys");
|
|
9
|
-
const
|
|
10
|
-
const
|
|
8
|
+
const redis_smq_common_1 = require("redis-smq-common");
|
|
9
|
+
const retry_message_1 = require("./retry-message");
|
|
10
|
+
const acknowledge_message_1 = require("./acknowledge-message");
|
|
11
11
|
class ConsumeMessage {
|
|
12
|
-
constructor(messageHandler, redisClient) {
|
|
12
|
+
constructor(messageHandler, redisClient, logger) {
|
|
13
13
|
this.redisClient = redisClient;
|
|
14
14
|
this.messageHandler = messageHandler;
|
|
15
15
|
const { keyQueueProcessing } = redis_keys_1.redisKeys.getQueueConsumerKeys(messageHandler.getQueue(), messageHandler.getConsumerId());
|
|
16
16
|
this.keyQueueProcessing = keyQueueProcessing;
|
|
17
|
+
this.logger = logger;
|
|
17
18
|
}
|
|
18
|
-
unacknowledgeMessage(msg, cause
|
|
19
|
-
|
|
20
|
-
}
|
|
21
|
-
(0, retry_message_1.retryMessage)(this.messageHandler.getConfig(), this.redisClient, this.keyQueueProcessing, msg, cause, (err, deadLetterCause) => {
|
|
19
|
+
unacknowledgeMessage(msg, cause) {
|
|
20
|
+
(0, retry_message_1.retryMessage)(this.messageHandler.getConfig(), this.redisClient, this.keyQueueProcessing, msg, cause, (err, retryStatus) => {
|
|
22
21
|
if (err)
|
|
23
22
|
this.messageHandler.handleError(err);
|
|
23
|
+
else if (!retryStatus)
|
|
24
|
+
this.messageHandler.handleError(new redis_smq_common_1.errors.EmptyCallbackReplyError());
|
|
24
25
|
else {
|
|
25
|
-
this.messageHandler.emit(events_1.events.MESSAGE_UNACKNOWLEDGED, msg,
|
|
26
|
-
if (
|
|
27
|
-
this.messageHandler.emit(events_1.events.MESSAGE_DEAD_LETTERED, msg
|
|
26
|
+
this.messageHandler.emit(events_1.events.MESSAGE_UNACKNOWLEDGED, msg, types_1.EMessageUnacknowledgedCause.CONSUME_ERROR);
|
|
27
|
+
if (retryStatus.status === retry_message_1.ERetryStatus.MESSAGE_DEAD_LETTERED) {
|
|
28
|
+
this.messageHandler.emit(events_1.events.MESSAGE_DEAD_LETTERED, msg);
|
|
28
29
|
}
|
|
29
30
|
}
|
|
30
31
|
});
|
|
@@ -45,8 +46,10 @@ class ConsumeMessage {
|
|
|
45
46
|
if (this.messageHandler.isRunning() && !isTimeout) {
|
|
46
47
|
if (timer)
|
|
47
48
|
clearTimeout(timer);
|
|
48
|
-
if (err)
|
|
49
|
-
this.
|
|
49
|
+
if (err) {
|
|
50
|
+
this.logger.error(err);
|
|
51
|
+
this.unacknowledgeMessage(msg, types_1.EMessageUnacknowledgedCause.UNACKNOWLEDGED);
|
|
52
|
+
}
|
|
50
53
|
else {
|
|
51
54
|
(0, acknowledge_message_1.acknowledgeMessage)(this.messageHandler.getConfig(), this.redisClient, msg, this.keyQueueProcessing, (err) => {
|
|
52
55
|
if (err)
|
|
@@ -60,10 +63,8 @@ class ConsumeMessage {
|
|
|
60
63
|
this.messageHandler.getHandler()(message_1.Message.createFromMessage(msg), onConsumed);
|
|
61
64
|
}
|
|
62
65
|
catch (error) {
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
: new consumer_error_1.ConsumerError(`An error occurred while processing message ID (${msg.getRequiredId()})`);
|
|
66
|
-
this.unacknowledgeMessage(msg, types_1.EMessageUnacknowledgedCause.CAUGHT_ERROR, err);
|
|
66
|
+
this.logger.error(error);
|
|
67
|
+
this.unacknowledgeMessage(msg, types_1.EMessageUnacknowledgedCause.CONSUME_ERROR);
|
|
67
68
|
}
|
|
68
69
|
}
|
|
69
70
|
handleReceivedMessage(message) {
|
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
import { EMessageDeadLetterCause, EMessageUnacknowledgedCause, IRequiredConfig } from '
|
|
2
|
-
import { Message } from '
|
|
1
|
+
import { EMessageDeadLetterCause, EMessageUnacknowledgedCause, IRequiredConfig } from '../../../../types';
|
|
2
|
+
import { Message } from '../../message/message';
|
|
3
3
|
import { ICallback, IRedisClientMulti } from 'redis-smq-common/dist/types';
|
|
4
4
|
import { RedisClient } from 'redis-smq-common';
|
|
5
5
|
export declare function deadLetterMessage(config: IRequiredConfig, mixed: IRedisClientMulti, message: Message, keyQueueProcessing: string, unacknowledgedCause: EMessageUnacknowledgedCause, deadLetterCause: EMessageDeadLetterCause): void;
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
3
|
exports.deadLetterMessage = void 0;
|
|
4
|
-
const redis_keys_1 = require("
|
|
4
|
+
const redis_keys_1 = require("../../../common/redis-keys/redis-keys");
|
|
5
5
|
const redis_smq_common_1 = require("redis-smq-common");
|
|
6
6
|
function deadLetterMessageTransaction(config, mixed, message, keyQueueProcessing) {
|
|
7
7
|
const queue = message.getRequiredQueue();
|
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
import { EMessageUnacknowledgedCause } from '
|
|
2
|
-
import { Message } from '
|
|
1
|
+
import { EMessageUnacknowledgedCause } from '../../../../types';
|
|
2
|
+
import { Message } from '../../message/message';
|
|
3
3
|
import { ICallback, IRedisClientMulti } from 'redis-smq-common/dist/types';
|
|
4
4
|
import { RedisClient } from 'redis-smq-common';
|
|
5
5
|
export declare function delayMessageTransaction(mixed: IRedisClientMulti, message: Message, keyQueueProcessing: string): void;
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
3
|
exports.delayMessage = exports.delayMessageTransaction = void 0;
|
|
4
|
-
const redis_keys_1 = require("
|
|
4
|
+
const redis_keys_1 = require("../../../common/redis-keys/redis-keys");
|
|
5
5
|
const redis_smq_common_1 = require("redis-smq-common");
|
|
6
6
|
function delayMessageTransaction(mixed, message, keyQueueProcessing) {
|
|
7
7
|
const queue = message.getRequiredQueue();
|
|
@@ -13,7 +13,7 @@ export declare class DequeueMessage {
|
|
|
13
13
|
protected messageHandler: MessageHandler;
|
|
14
14
|
protected priorityQueuing: boolean;
|
|
15
15
|
constructor(messageHandler: MessageHandler, redisClient: RedisClient);
|
|
16
|
-
protected dequeueMessageWithPriority(cb: ICallback<string>): void;
|
|
16
|
+
protected dequeueMessageWithPriority(cb: ICallback<string | null>): void;
|
|
17
17
|
protected waitForMessage(cb: ICallback<string | null>): void;
|
|
18
18
|
protected dequeueMessage(cb: ICallback<string | null>): void;
|
|
19
19
|
dequeue(): void;
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
/// <reference types="node" />
|
|
2
|
-
import {
|
|
2
|
+
import { IEventListener, IRequiredConfig, TConsumerMessageHandler, TQueueParams } from '../../../../types';
|
|
3
3
|
import { EventEmitter } from 'events';
|
|
4
|
+
import { TCleanUpStatus } from './processing-queue';
|
|
4
5
|
import { DequeueMessage } from './dequeue-message';
|
|
5
6
|
import { ConsumeMessage } from './consume-message';
|
|
6
7
|
import { Consumer } from '../consumer';
|
|
@@ -18,10 +19,12 @@ export declare class MessageHandler extends EventEmitter {
|
|
|
18
19
|
protected dequeueMessage: DequeueMessage;
|
|
19
20
|
protected consumeMessage: ConsumeMessage;
|
|
20
21
|
protected handler: TConsumerMessageHandler;
|
|
21
|
-
protected
|
|
22
|
+
protected eventListeners: IEventListener[];
|
|
22
23
|
constructor(consumer: Consumer, queue: TQueueParams, handler: TConsumerMessageHandler, dequeueRedisClient: RedisClient, sharedRedisClient: RedisClient, logger: ICompatibleLogger);
|
|
23
24
|
protected registerEventsHandlers(): void;
|
|
24
|
-
protected
|
|
25
|
+
protected initEventListeners(): void;
|
|
26
|
+
protected cleanUp(cb: ICallback<void>): void;
|
|
27
|
+
protected tearDownEventListeners(cb: ICallback<void>): void;
|
|
25
28
|
handleError(err: Error): void;
|
|
26
29
|
dequeue(): void;
|
|
27
30
|
run(cb: ICallback<void>): void;
|
|
@@ -32,5 +35,5 @@ export declare class MessageHandler extends EventEmitter {
|
|
|
32
35
|
getConfig(): IRequiredConfig;
|
|
33
36
|
isRunning(): boolean;
|
|
34
37
|
getHandler(): TConsumerMessageHandler;
|
|
35
|
-
static cleanUp(config: IRequiredConfig, redisClient: RedisClient, consumerId: string, queue: TQueueParams, pendingMulti: IRedisClientMulti | undefined, cb: ICallback<
|
|
38
|
+
static cleanUp(config: IRequiredConfig, redisClient: RedisClient, consumerId: string, queue: TQueueParams, pendingMulti: IRedisClientMulti | undefined, cb: ICallback<TCleanUpStatus>): void;
|
|
36
39
|
}
|