redis-smq 6.2.4 → 6.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +24 -0
- package/README.md +9 -5
- package/dist/src/system/app/consumer/consumer-frontend.d.ts +2 -2
- package/dist/src/system/app/consumer/consumer-frontend.js +8 -5
- package/dist/src/system/app/consumer/consumer-heartbeat.js +1 -3
- package/dist/src/system/app/consumer/consumer-message-handler/consume-message.d.ts +13 -0
- package/dist/src/system/app/consumer/consumer-message-handler/consume-message.js +77 -0
- package/dist/src/system/app/consumer/consumer-message-handler/dequeue-message.d.ts +24 -0
- package/dist/src/system/app/consumer/consumer-message-handler/dequeue-message.js +93 -0
- package/dist/src/system/app/consumer/consumer-message-handler/message-handler-runner.d.ts +30 -0
- package/dist/src/system/app/consumer/consumer-message-handler/message-handler-runner.js +152 -0
- package/dist/src/system/app/consumer/consumer-message-handler/message-handler.d.ts +34 -0
- package/dist/src/system/app/consumer/consumer-message-handler/message-handler.js +164 -0
- package/dist/src/system/app/consumer/consumer-message-handler/multiplexed-message-handler/multiplexed-dequeue-message.d.ts +4 -0
- package/dist/src/system/app/consumer/consumer-message-handler/multiplexed-message-handler/multiplexed-dequeue-message.js +44 -0
- package/dist/src/system/app/consumer/consumer-message-handler/multiplexed-message-handler/multiplexed-message-handler-runner.d.ts +24 -0
- package/dist/src/system/app/consumer/consumer-message-handler/multiplexed-message-handler/multiplexed-message-handler-runner.js +131 -0
- package/dist/src/system/app/consumer/consumer-message-handler/multiplexed-message-handler/multiplexed-message-handler.d.ts +9 -0
- package/dist/src/system/app/consumer/consumer-message-handler/multiplexed-message-handler/multiplexed-message-handler.js +54 -0
- package/dist/src/system/app/consumer/consumer-message-handler/processing-queue.d.ts +6 -0
- package/dist/src/system/app/consumer/consumer-message-handler/processing-queue.js +63 -0
- package/dist/src/system/app/consumer/consumer-message-rate.js +4 -4
- package/dist/src/system/app/consumer/consumer-queues.d.ts +2 -3
- package/dist/src/system/app/consumer/consumer-queues.js +8 -13
- package/dist/src/system/app/consumer/consumer.d.ts +9 -23
- package/dist/src/system/app/consumer/consumer.js +37 -207
- package/dist/src/system/app/consumer/errors/message-handler-already-exists.error.d.ts +1 -1
- package/dist/src/system/app/consumer/errors/message-handler-already-exists.error.js +2 -2
- package/dist/src/system/app/message/message.js +5 -5
- package/dist/src/system/app/producer/producer.js +22 -23
- package/dist/src/system/app/queue-manager/queue-manager.d.ts +0 -3
- package/dist/src/system/app/queue-manager/queue-manager.js +0 -11
- package/dist/src/system/common/base.d.ts +1 -1
- package/dist/src/system/common/base.js +5 -6
- package/dist/src/system/common/message-rate-writer.d.ts +1 -1
- package/dist/src/system/common/message-rate-writer.js +13 -15
- package/dist/src/system/common/message-rate.d.ts +1 -1
- package/dist/src/system/common/message-rate.js +6 -8
- package/dist/src/system/common/redis-client/redis-client.js +2 -1
- package/dist/src/system/common/redis-keys/redis-keys.js +2 -2
- package/dist/src/system/common/ticker/ticker.d.ts +2 -1
- package/dist/src/system/common/ticker/ticker.js +6 -3
- package/dist/src/system/common/worker/worker-runner/worker-runner.js +1 -1
- package/package.json +1 -1
- package/dist/src/system/app/consumer/consumer-message-handler.d.ts +0 -42
- package/dist/src/system/app/consumer/consumer-message-handler.js +0 -268
package/CHANGELOG.md
CHANGED
|
@@ -1,5 +1,29 @@
|
|
|
1
1
|
# CHANGELOG
|
|
2
2
|
|
|
3
|
+
## 6.3.0 (2022-03-15)
|
|
4
|
+
|
|
5
|
+
* Update docs (3ad938a)
|
|
6
|
+
* Use colons instead of dots for joining Redis key segments (75dbcb6)
|
|
7
|
+
* Continue testing consumer message multiplexing (17c9659)
|
|
8
|
+
* Improve multiplexing delay when dequeuing messages (a635c8a)
|
|
9
|
+
* Remove deprecated consumer.cancel(queue,priority,cb), add new tests (4a2d458)
|
|
10
|
+
* Fix test errors, clean up (09feb9c)
|
|
11
|
+
* Prefer method definition over arrow function property (5d7e664)
|
|
12
|
+
* Implement MultiplexedMessageHandlerRunner (6980cbf)
|
|
13
|
+
* Refactor MessageHandler to allow more modular structures (b079244)
|
|
14
|
+
|
|
15
|
+
## 6.2.6 (2022-03-04)
|
|
16
|
+
|
|
17
|
+
* Clean up (2837cc6)
|
|
18
|
+
* Implement MessageHandlerRunner (cfba094)
|
|
19
|
+
|
|
20
|
+
## 6.2.5 (2022-03-03)
|
|
21
|
+
|
|
22
|
+
* Update Consumer API docs (8580f4a)
|
|
23
|
+
* Do not consume messages with and without priority from the same queue (84130bf)
|
|
24
|
+
* Use default parameters when creating a Ticker instance (db9feb0)
|
|
25
|
+
* Update consumer queue list upon shutting down a message handler (14519e2)
|
|
26
|
+
|
|
3
27
|
## 6.2.4 (2022-02-23)
|
|
4
28
|
|
|
5
29
|
* Fix consuming-messages/test00015 error (c5c365a)
|
package/README.md
CHANGED
|
@@ -22,6 +22,7 @@ RedisSMQ is a Node.js library for queuing messages (aka jobs) and processing the
|
|
|
22
22
|
* **[Queue Rate Limiting](docs/queue-rate-limiting.md)**: Allowing you to control the rate at which the messages are consumed from a given queue.
|
|
23
23
|
* **[Scheduling Messages](docs/scheduling-messages.md)**: Messages can be configured to be delayed, delivered for N times with an optional period between deliveries, and to be scheduled using CRON expressions.
|
|
24
24
|
* **[Reliable Priority Queues](docs/priority-queues.md)**: Supports priority messaging.
|
|
25
|
+
* **[Multiplexing](docs/multiplexing.md)**: It's great feature which allows message handlers to use a single redis connection to dequeue and consume messages.
|
|
25
26
|
* **[HTTP API](docs/http-api.md)**: an HTTP interface is provided to interact with the MQ.
|
|
26
27
|
* **[Web UI](docs/web-ui.md)**: RedisSMQ can be managed also from your web browser.
|
|
27
28
|
* **[Logging](docs/logs.md)**: Comes with a built-in JSON logger. But you can also use your own logger instance.
|
|
@@ -51,11 +52,12 @@ RedisSMQ is a Node.js library for queuing messages (aka jobs) and processing the
|
|
|
51
52
|
1. [Scheduling Messages](docs/scheduling-messages.md)
|
|
52
53
|
2. [Priority Queues](docs/priority-queues.md)
|
|
53
54
|
3. [Queue Rate Limiting](docs/queue-rate-limiting.md)
|
|
54
|
-
4. [
|
|
55
|
-
5. [
|
|
56
|
-
6. [
|
|
57
|
-
7. [
|
|
58
|
-
8. [
|
|
55
|
+
4. [Multiplexing](docs/multiplexing.md)
|
|
56
|
+
5. [Message Manager](docs/api/message-manager.md)
|
|
57
|
+
6. [Queue Manager](docs/api/queue-manager.md)
|
|
58
|
+
7. [HTTP API](docs/http-api.md)
|
|
59
|
+
8. [Web UI](docs/web-ui.md)
|
|
60
|
+
9. [Logs](docs/logs.md)
|
|
59
61
|
5. [RedisSMQ Architecture](docs/redis-smq-architecture.md)
|
|
60
62
|
6. [Performance](#performance)
|
|
61
63
|
7. [Contributing](#contributing)
|
|
@@ -223,6 +225,8 @@ See [Consumer Reference](docs/api/consumer.md) for more details.
|
|
|
223
225
|
* [Priority Queues](docs/priority-queues.md)
|
|
224
226
|
|
|
225
227
|
* [Queue Rate Limiting](docs/queue-rate-limiting.md)
|
|
228
|
+
|
|
229
|
+
* [Multiplexing](docs/multiplexing.md)
|
|
226
230
|
|
|
227
231
|
* [Message Manager](docs/api/message-manager.md)
|
|
228
232
|
|
|
@@ -3,10 +3,10 @@ import { ICallback, TConsumerMessageHandler, TQueueParams } from '../../../../ty
|
|
|
3
3
|
import { EventEmitter } from 'events';
|
|
4
4
|
export declare class ConsumerFrontend extends EventEmitter {
|
|
5
5
|
private consumer;
|
|
6
|
-
constructor();
|
|
6
|
+
constructor(useMultiplexing?: boolean);
|
|
7
7
|
private registerEvents;
|
|
8
8
|
consume(queue: string | TQueueParams, usePriorityQueuing: boolean, messageHandler: TConsumerMessageHandler, cb: ICallback<boolean>): void;
|
|
9
|
-
cancel(queue: string | TQueueParams,
|
|
9
|
+
cancel(queue: string | TQueueParams, cb: ICallback<void>): void;
|
|
10
10
|
run(cb?: ICallback<boolean>): void;
|
|
11
11
|
shutdown(cb?: ICallback<boolean>): void;
|
|
12
12
|
isGoingUp(): boolean;
|
|
@@ -4,10 +4,11 @@ exports.ConsumerFrontend = void 0;
|
|
|
4
4
|
const events_1 = require("events");
|
|
5
5
|
const consumer_1 = require("./consumer");
|
|
6
6
|
const events_2 = require("../../common/events");
|
|
7
|
+
const queue_manager_1 = require("../queue-manager/queue-manager");
|
|
7
8
|
class ConsumerFrontend extends events_1.EventEmitter {
|
|
8
|
-
constructor() {
|
|
9
|
+
constructor(useMultiplexing = false) {
|
|
9
10
|
super();
|
|
10
|
-
this.consumer = new consumer_1.Consumer();
|
|
11
|
+
this.consumer = new consumer_1.Consumer(useMultiplexing);
|
|
11
12
|
this.registerEvents();
|
|
12
13
|
}
|
|
13
14
|
registerEvents() {
|
|
@@ -21,10 +22,12 @@ class ConsumerFrontend extends events_1.EventEmitter {
|
|
|
21
22
|
.on(events_2.events.MESSAGE_RECEIVED, (...args) => this.emit(events_2.events.MESSAGE_RECEIVED, ...args));
|
|
22
23
|
}
|
|
23
24
|
consume(queue, usePriorityQueuing, messageHandler, cb) {
|
|
24
|
-
|
|
25
|
+
const queueParams = queue_manager_1.queueManager.getQueueParams(queue);
|
|
26
|
+
return this.consumer.consume(queueParams, usePriorityQueuing, messageHandler, cb);
|
|
25
27
|
}
|
|
26
|
-
cancel(queue,
|
|
27
|
-
|
|
28
|
+
cancel(queue, cb) {
|
|
29
|
+
const queueParams = queue_manager_1.queueManager.getQueueParams(queue);
|
|
30
|
+
this.consumer.cancel(queueParams, cb);
|
|
28
31
|
}
|
|
29
32
|
run(cb) {
|
|
30
33
|
this.consumer.run(cb);
|
|
@@ -40,9 +40,7 @@ class ConsumerHeartbeat extends events_2.EventEmitter {
|
|
|
40
40
|
this.keyHeartbeats = keyHeartbeats;
|
|
41
41
|
this.keyHeartbeatTimestamps =
|
|
42
42
|
redis_keys_1.redisKeys.getMainKeys().keyHeartbeatInstanceIds;
|
|
43
|
-
this.ticker = new ticker_1.Ticker(() =>
|
|
44
|
-
this.onTick();
|
|
45
|
-
}, 1000);
|
|
43
|
+
this.ticker = new ticker_1.Ticker(() => this.onTick());
|
|
46
44
|
this.ticker.nextTick();
|
|
47
45
|
}
|
|
48
46
|
getPayload() {
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
import { Message } from '../../message/message';
|
|
2
|
+
import { EMessageUnacknowledgedCause } from '../../../../../types';
|
|
3
|
+
import { RedisClient } from '../../../common/redis-client/redis-client';
|
|
4
|
+
import { MessageHandler } from './message-handler';
|
|
5
|
+
export declare class ConsumeMessage {
|
|
6
|
+
protected keyQueueProcessing: string;
|
|
7
|
+
protected messageHandler: MessageHandler;
|
|
8
|
+
protected redisClient: RedisClient;
|
|
9
|
+
constructor(messageHandler: MessageHandler, redisClient: RedisClient);
|
|
10
|
+
protected unacknowledgeMessage(msg: Message, cause: EMessageUnacknowledgedCause, err?: Error): void;
|
|
11
|
+
protected consumeMessage(msg: Message): void;
|
|
12
|
+
handleReceivedMessage(message: Message): void;
|
|
13
|
+
}
|
|
@@ -0,0 +1,77 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.ConsumeMessage = void 0;
|
|
4
|
+
const message_1 = require("../../message/message");
|
|
5
|
+
const types_1 = require("../../../../../types");
|
|
6
|
+
const broker_1 = require("../../../common/broker/broker");
|
|
7
|
+
const events_1 = require("../../../common/events");
|
|
8
|
+
const consumer_error_1 = require("../errors/consumer.error");
|
|
9
|
+
const redis_keys_1 = require("../../../common/redis-keys/redis-keys");
|
|
10
|
+
class ConsumeMessage {
|
|
11
|
+
constructor(messageHandler, redisClient) {
|
|
12
|
+
this.redisClient = redisClient;
|
|
13
|
+
this.messageHandler = messageHandler;
|
|
14
|
+
const { keyQueueProcessing } = redis_keys_1.redisKeys.getQueueConsumerKeys(messageHandler.getQueue(), messageHandler.getConsumerId());
|
|
15
|
+
this.keyQueueProcessing = keyQueueProcessing;
|
|
16
|
+
}
|
|
17
|
+
unacknowledgeMessage(msg, cause, err) {
|
|
18
|
+
if (err) {
|
|
19
|
+
}
|
|
20
|
+
broker_1.broker.retry(this.redisClient, this.keyQueueProcessing, msg, cause, (err, deadLetterCause) => {
|
|
21
|
+
if (err)
|
|
22
|
+
this.messageHandler.handleError(err);
|
|
23
|
+
else {
|
|
24
|
+
this.messageHandler.emit(events_1.events.MESSAGE_UNACKNOWLEDGED, msg, cause);
|
|
25
|
+
if (deadLetterCause !== undefined) {
|
|
26
|
+
this.messageHandler.emit(events_1.events.MESSAGE_DEAD_LETTERED, msg, deadLetterCause);
|
|
27
|
+
}
|
|
28
|
+
}
|
|
29
|
+
});
|
|
30
|
+
}
|
|
31
|
+
consumeMessage(msg) {
|
|
32
|
+
let isTimeout = false;
|
|
33
|
+
let timer = null;
|
|
34
|
+
try {
|
|
35
|
+
const consumeTimeout = msg.getConsumeTimeout();
|
|
36
|
+
if (consumeTimeout) {
|
|
37
|
+
timer = setTimeout(() => {
|
|
38
|
+
isTimeout = true;
|
|
39
|
+
timer = null;
|
|
40
|
+
this.unacknowledgeMessage(msg, types_1.EMessageUnacknowledgedCause.TIMEOUT);
|
|
41
|
+
}, consumeTimeout);
|
|
42
|
+
}
|
|
43
|
+
const onConsumed = (err) => {
|
|
44
|
+
if (this.messageHandler.isRunning() && !isTimeout) {
|
|
45
|
+
if (timer)
|
|
46
|
+
clearTimeout(timer);
|
|
47
|
+
if (err)
|
|
48
|
+
this.unacknowledgeMessage(msg, types_1.EMessageUnacknowledgedCause.UNACKNOWLEDGED, err);
|
|
49
|
+
else {
|
|
50
|
+
broker_1.broker.acknowledgeMessage(this.redisClient, msg, this.keyQueueProcessing, (err) => {
|
|
51
|
+
if (err)
|
|
52
|
+
this.messageHandler.handleError(err);
|
|
53
|
+
else
|
|
54
|
+
this.messageHandler.emit(events_1.events.MESSAGE_ACKNOWLEDGED, msg);
|
|
55
|
+
});
|
|
56
|
+
}
|
|
57
|
+
}
|
|
58
|
+
};
|
|
59
|
+
this.messageHandler.getHandler()(message_1.Message.createFromMessage(msg), onConsumed);
|
|
60
|
+
}
|
|
61
|
+
catch (error) {
|
|
62
|
+
const err = error instanceof Error
|
|
63
|
+
? error
|
|
64
|
+
: new consumer_error_1.ConsumerError(`An error occurred while processing message ID (${msg.getRequiredId()})`);
|
|
65
|
+
this.unacknowledgeMessage(msg, types_1.EMessageUnacknowledgedCause.CAUGHT_ERROR, err);
|
|
66
|
+
}
|
|
67
|
+
}
|
|
68
|
+
handleReceivedMessage(message) {
|
|
69
|
+
if (message.getSetExpired()) {
|
|
70
|
+
this.unacknowledgeMessage(message, types_1.EMessageUnacknowledgedCause.TTL_EXPIRED);
|
|
71
|
+
}
|
|
72
|
+
else
|
|
73
|
+
this.consumeMessage(message);
|
|
74
|
+
}
|
|
75
|
+
}
|
|
76
|
+
exports.ConsumeMessage = ConsumeMessage;
|
|
77
|
+
//# sourceMappingURL=consume-message.js.map
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
import { ICallback, TQueueParams, TQueueRateLimit } from '../../../../../types';
|
|
2
|
+
import { RedisClient } from '../../../common/redis-client/redis-client';
|
|
3
|
+
import { Ticker } from '../../../common/ticker/ticker';
|
|
4
|
+
import { MessageHandler } from './message-handler';
|
|
5
|
+
export declare class DequeueMessage {
|
|
6
|
+
protected redisClient: RedisClient;
|
|
7
|
+
protected queue: TQueueParams;
|
|
8
|
+
protected consumerId: string;
|
|
9
|
+
protected keyQueuePending: string;
|
|
10
|
+
protected keyQueuePriority: string;
|
|
11
|
+
protected keyPendingMessagesWithPriority: string;
|
|
12
|
+
protected keyQueueProcessing: string;
|
|
13
|
+
protected usingPriorityQueuing: boolean;
|
|
14
|
+
protected queueRateLimit: TQueueRateLimit | null;
|
|
15
|
+
protected ticker: Ticker;
|
|
16
|
+
protected messageHandler: MessageHandler;
|
|
17
|
+
constructor(messageHandler: MessageHandler, redisClient: RedisClient);
|
|
18
|
+
protected dequeueMessageWithPriority(cb: ICallback<string>): void;
|
|
19
|
+
protected waitForMessage(cb: ICallback<string>): void;
|
|
20
|
+
protected dequeueMessage(cb: ICallback<string>): void;
|
|
21
|
+
dequeue(): void;
|
|
22
|
+
run(cb: ICallback<void>): void;
|
|
23
|
+
quit(cb: ICallback<void>): void;
|
|
24
|
+
}
|
|
@@ -0,0 +1,93 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.DequeueMessage = void 0;
|
|
4
|
+
const queue_manager_1 = require("../../queue-manager/queue-manager");
|
|
5
|
+
const redis_keys_1 = require("../../../common/redis-keys/redis-keys");
|
|
6
|
+
const ticker_1 = require("../../../common/ticker/ticker");
|
|
7
|
+
const events_1 = require("../../../common/events");
|
|
8
|
+
const consumer_queues_1 = require("../consumer-queues");
|
|
9
|
+
const processing_queue_1 = require("./processing-queue");
|
|
10
|
+
const message_1 = require("../../message/message");
|
|
11
|
+
class DequeueMessage {
|
|
12
|
+
constructor(messageHandler, redisClient) {
|
|
13
|
+
this.queueRateLimit = null;
|
|
14
|
+
this.messageHandler = messageHandler;
|
|
15
|
+
this.redisClient = redisClient;
|
|
16
|
+
this.queue = messageHandler.getQueue();
|
|
17
|
+
this.consumerId = messageHandler.getConsumerId();
|
|
18
|
+
const { keyQueuePendingPriorityMessageIds, keyQueuePending, keyQueuePendingPriorityMessages, keyQueueProcessing, } = redis_keys_1.redisKeys.getQueueConsumerKeys(this.queue, this.consumerId);
|
|
19
|
+
this.keyQueuePending = keyQueuePending;
|
|
20
|
+
this.keyPendingMessagesWithPriority = keyQueuePendingPriorityMessages;
|
|
21
|
+
this.keyQueuePriority = keyQueuePendingPriorityMessageIds;
|
|
22
|
+
this.keyQueueProcessing = keyQueueProcessing;
|
|
23
|
+
this.usingPriorityQueuing = messageHandler.isUsingPriorityQueuing();
|
|
24
|
+
this.ticker = new ticker_1.Ticker(() => this.dequeue());
|
|
25
|
+
}
|
|
26
|
+
dequeueMessageWithPriority(cb) {
|
|
27
|
+
this.redisClient.zpophgetrpush(this.keyQueuePriority, this.keyPendingMessagesWithPriority, this.keyQueueProcessing, cb);
|
|
28
|
+
}
|
|
29
|
+
waitForMessage(cb) {
|
|
30
|
+
this.redisClient.brpoplpush(this.keyQueuePending, this.keyQueueProcessing, 0, cb);
|
|
31
|
+
}
|
|
32
|
+
dequeueMessage(cb) {
|
|
33
|
+
this.redisClient.rpoplpush(this.keyQueuePending, this.keyQueueProcessing, cb);
|
|
34
|
+
}
|
|
35
|
+
dequeue() {
|
|
36
|
+
const cb = (err, reply) => {
|
|
37
|
+
if (err) {
|
|
38
|
+
this.ticker.abort();
|
|
39
|
+
this.messageHandler.handleError(err);
|
|
40
|
+
}
|
|
41
|
+
else if (typeof reply === 'string') {
|
|
42
|
+
const message = message_1.Message.createFromMessage(reply);
|
|
43
|
+
this.messageHandler.emit(events_1.events.MESSAGE_RECEIVED, message);
|
|
44
|
+
}
|
|
45
|
+
else {
|
|
46
|
+
this.ticker.nextTick();
|
|
47
|
+
}
|
|
48
|
+
};
|
|
49
|
+
const deq = () => {
|
|
50
|
+
if (this.usingPriorityQueuing)
|
|
51
|
+
this.dequeueMessageWithPriority(cb);
|
|
52
|
+
else
|
|
53
|
+
this.dequeueMessage(cb);
|
|
54
|
+
};
|
|
55
|
+
if (this.usingPriorityQueuing || this.queueRateLimit) {
|
|
56
|
+
if (this.queueRateLimit) {
|
|
57
|
+
queue_manager_1.queueManager.hasQueueRateLimitExceeded(this.redisClient, this.queue, this.queueRateLimit, (err, isExceeded) => {
|
|
58
|
+
if (err)
|
|
59
|
+
this.messageHandler.handleError(err);
|
|
60
|
+
else if (isExceeded)
|
|
61
|
+
this.ticker.nextTick();
|
|
62
|
+
else
|
|
63
|
+
deq();
|
|
64
|
+
});
|
|
65
|
+
}
|
|
66
|
+
else
|
|
67
|
+
deq();
|
|
68
|
+
}
|
|
69
|
+
else {
|
|
70
|
+
this.waitForMessage(cb);
|
|
71
|
+
}
|
|
72
|
+
}
|
|
73
|
+
run(cb) {
|
|
74
|
+
queue_manager_1.queueManager.getQueueRateLimit(this.redisClient, this.queue, (err, rateLimit) => {
|
|
75
|
+
if (err)
|
|
76
|
+
cb(err);
|
|
77
|
+
else {
|
|
78
|
+
this.queueRateLimit = rateLimit !== null && rateLimit !== void 0 ? rateLimit : null;
|
|
79
|
+
const multi = this.redisClient.multi();
|
|
80
|
+
queue_manager_1.queueManager.setUpMessageQueue(multi, this.queue);
|
|
81
|
+
consumer_queues_1.consumerQueues.addConsumer(multi, this.queue, this.consumerId);
|
|
82
|
+
processing_queue_1.processingQueue.setUpProcessingQueue(multi, this.queue, this.consumerId);
|
|
83
|
+
this.redisClient.execMulti(multi, (err) => cb(err));
|
|
84
|
+
}
|
|
85
|
+
});
|
|
86
|
+
}
|
|
87
|
+
quit(cb) {
|
|
88
|
+
this.ticker.once(events_1.events.DOWN, cb);
|
|
89
|
+
this.ticker.quit();
|
|
90
|
+
}
|
|
91
|
+
}
|
|
92
|
+
exports.DequeueMessage = DequeueMessage;
|
|
93
|
+
//# sourceMappingURL=dequeue-message.js.map
|
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
import { Consumer } from '../consumer';
|
|
2
|
+
import { MessageHandler } from './message-handler';
|
|
3
|
+
import { ICallback, ICompatibleLogger, IRequiredConfig, TConsumerMessageHandler, TConsumerMessageHandlerParams, TQueueParams } from '../../../../../types';
|
|
4
|
+
import { RedisClient } from '../../../common/redis-client/redis-client';
|
|
5
|
+
import { ConsumerMessageRate } from '../consumer-message-rate';
|
|
6
|
+
export declare class MessageHandlerRunner {
|
|
7
|
+
protected consumer: Consumer;
|
|
8
|
+
protected sharedRedisClient: RedisClient | null;
|
|
9
|
+
protected messageHandlerInstances: MessageHandler[];
|
|
10
|
+
protected messageHandlers: TConsumerMessageHandlerParams[];
|
|
11
|
+
protected config: IRequiredConfig;
|
|
12
|
+
protected logger: ICompatibleLogger;
|
|
13
|
+
constructor(consumer: Consumer);
|
|
14
|
+
protected registerMessageHandlerEvents(messageHandler: MessageHandler): void;
|
|
15
|
+
protected getMessageHandlerInstance(queue: TQueueParams): MessageHandler | undefined;
|
|
16
|
+
protected getMessageHandler(queue: TQueueParams): TConsumerMessageHandlerParams | undefined;
|
|
17
|
+
protected createMessageRateInstance(queue: TQueueParams, redisClient: RedisClient): ConsumerMessageRate;
|
|
18
|
+
protected createMessageHandlerInstance(redisClient: RedisClient, handlerParams: TConsumerMessageHandlerParams): MessageHandler;
|
|
19
|
+
protected runMessageHandler(handlerParams: TConsumerMessageHandlerParams, cb: ICallback<void>): void;
|
|
20
|
+
protected shutdownMessageHandler(messageHandler: MessageHandler, cb: ICallback<void>): void;
|
|
21
|
+
protected getSharedRedisClient(): RedisClient;
|
|
22
|
+
run(redisClient: RedisClient, cb: ICallback<void>): void;
|
|
23
|
+
shutdown(cb: ICallback<void>): void;
|
|
24
|
+
removeMessageHandler(queue: TQueueParams, cb: ICallback<void>): void;
|
|
25
|
+
addMessageHandler(queue: TQueueParams, usePriorityQueuing: boolean, messageHandler: TConsumerMessageHandler, cb: ICallback<boolean>): void;
|
|
26
|
+
getQueues(): {
|
|
27
|
+
queue: TQueueParams;
|
|
28
|
+
usingPriorityQueuing: boolean;
|
|
29
|
+
}[];
|
|
30
|
+
}
|
|
@@ -0,0 +1,152 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.MessageHandlerRunner = void 0;
|
|
4
|
+
const message_handler_1 = require("./message-handler");
|
|
5
|
+
const events_1 = require("../../../common/events");
|
|
6
|
+
const redis_client_1 = require("../../../common/redis-client/redis-client");
|
|
7
|
+
const empty_callback_reply_error_1 = require("../../../common/errors/empty-callback-reply.error");
|
|
8
|
+
const async_1 = require("../../../lib/async");
|
|
9
|
+
const configuration_1 = require("../../../common/configuration/configuration");
|
|
10
|
+
const logger_1 = require("../../../common/logger");
|
|
11
|
+
const consumer_message_rate_1 = require("../consumer-message-rate");
|
|
12
|
+
const consumer_message_rate_writer_1 = require("../consumer-message-rate-writer");
|
|
13
|
+
const message_handler_already_exists_error_1 = require("../errors/message-handler-already-exists.error");
|
|
14
|
+
const panic_error_1 = require("../../../common/errors/panic.error");
|
|
15
|
+
class MessageHandlerRunner {
|
|
16
|
+
constructor(consumer) {
|
|
17
|
+
this.sharedRedisClient = null;
|
|
18
|
+
this.messageHandlerInstances = [];
|
|
19
|
+
this.messageHandlers = [];
|
|
20
|
+
this.consumer = consumer;
|
|
21
|
+
this.config = (0, configuration_1.getConfiguration)();
|
|
22
|
+
this.logger = (0, logger_1.getNamespacedLogger)(`${consumer.constructor.name}/${consumer.getId()}/MessageHandlerRunner`);
|
|
23
|
+
}
|
|
24
|
+
registerMessageHandlerEvents(messageHandler) {
|
|
25
|
+
messageHandler.on(events_1.events.ERROR, (...args) => this.consumer.emit(events_1.events.ERROR, ...args));
|
|
26
|
+
messageHandler.on(events_1.events.IDLE, (...args) => this.consumer.emit(events_1.events.IDLE, ...args));
|
|
27
|
+
messageHandler.on(events_1.events.MESSAGE_UNACKNOWLEDGED, (...args) => this.consumer.emit(events_1.events.MESSAGE_UNACKNOWLEDGED, ...args));
|
|
28
|
+
messageHandler.on(events_1.events.MESSAGE_DEAD_LETTERED, (...args) => this.consumer.emit(events_1.events.MESSAGE_DEAD_LETTERED, ...args));
|
|
29
|
+
messageHandler.on(events_1.events.MESSAGE_ACKNOWLEDGED, (...args) => this.consumer.emit(events_1.events.MESSAGE_ACKNOWLEDGED, ...args));
|
|
30
|
+
messageHandler.on(events_1.events.MESSAGE_RECEIVED, (...args) => this.consumer.emit(events_1.events.MESSAGE_RECEIVED, ...args));
|
|
31
|
+
}
|
|
32
|
+
getMessageHandlerInstance(queue) {
|
|
33
|
+
return this.messageHandlerInstances.find((i) => {
|
|
34
|
+
const q = i.getQueue();
|
|
35
|
+
return q.name === queue.name && q.ns === queue.ns;
|
|
36
|
+
});
|
|
37
|
+
}
|
|
38
|
+
getMessageHandler(queue) {
|
|
39
|
+
return this.messageHandlers.find((i) => i.queue.name === queue.name && i.queue.ns === queue.ns);
|
|
40
|
+
}
|
|
41
|
+
createMessageRateInstance(queue, redisClient) {
|
|
42
|
+
const messageRateWriter = new consumer_message_rate_writer_1.ConsumerMessageRateWriter(redisClient, queue, this.consumer.getId());
|
|
43
|
+
return new consumer_message_rate_1.ConsumerMessageRate(messageRateWriter);
|
|
44
|
+
}
|
|
45
|
+
createMessageHandlerInstance(redisClient, handlerParams) {
|
|
46
|
+
const sharedRedisClient = this.getSharedRedisClient();
|
|
47
|
+
const { queue, usePriorityQueuing, messageHandler } = handlerParams;
|
|
48
|
+
const messageRate = this.config.monitor.enabled
|
|
49
|
+
? this.createMessageRateInstance(queue, sharedRedisClient)
|
|
50
|
+
: null;
|
|
51
|
+
const instance = new message_handler_1.MessageHandler(this.consumer.getId(), queue, messageHandler, usePriorityQueuing, redisClient, messageRate);
|
|
52
|
+
this.registerMessageHandlerEvents(instance);
|
|
53
|
+
this.messageHandlerInstances.push(instance);
|
|
54
|
+
this.logger.info(`Created a new instance (ID: ${instance.getId()}) for MessageHandler (${JSON.stringify(handlerParams)}).`);
|
|
55
|
+
return instance;
|
|
56
|
+
}
|
|
57
|
+
runMessageHandler(handlerParams, cb) {
|
|
58
|
+
redis_client_1.RedisClient.getNewInstance((err, client) => {
|
|
59
|
+
if (err)
|
|
60
|
+
cb(err);
|
|
61
|
+
else if (!client)
|
|
62
|
+
cb(new empty_callback_reply_error_1.EmptyCallbackReplyError());
|
|
63
|
+
else {
|
|
64
|
+
const handler = this.createMessageHandlerInstance(client, handlerParams);
|
|
65
|
+
handler.run(cb);
|
|
66
|
+
}
|
|
67
|
+
});
|
|
68
|
+
}
|
|
69
|
+
shutdownMessageHandler(messageHandler, cb) {
|
|
70
|
+
const queue = messageHandler.getQueue();
|
|
71
|
+
const redisClient = this.getSharedRedisClient();
|
|
72
|
+
messageHandler.shutdown(redisClient, () => {
|
|
73
|
+
this.messageHandlerInstances = this.messageHandlerInstances.filter((handler) => {
|
|
74
|
+
const q = handler.getQueue();
|
|
75
|
+
return !(q.name === queue.name && q.ns === queue.ns);
|
|
76
|
+
});
|
|
77
|
+
cb();
|
|
78
|
+
});
|
|
79
|
+
}
|
|
80
|
+
getSharedRedisClient() {
|
|
81
|
+
if (!this.sharedRedisClient) {
|
|
82
|
+
throw new panic_error_1.PanicError('Expected a non-empty value');
|
|
83
|
+
}
|
|
84
|
+
return this.sharedRedisClient;
|
|
85
|
+
}
|
|
86
|
+
run(redisClient, cb) {
|
|
87
|
+
this.sharedRedisClient = redisClient;
|
|
88
|
+
(0, async_1.each)(this.messageHandlers, (handlerParams, _, done) => {
|
|
89
|
+
this.runMessageHandler(handlerParams, done);
|
|
90
|
+
}, cb);
|
|
91
|
+
}
|
|
92
|
+
shutdown(cb) {
|
|
93
|
+
(0, async_1.each)(this.messageHandlerInstances, (handler, queue, done) => {
|
|
94
|
+
this.shutdownMessageHandler(handler, done);
|
|
95
|
+
}, (err) => {
|
|
96
|
+
if (err)
|
|
97
|
+
cb(err);
|
|
98
|
+
else {
|
|
99
|
+
this.sharedRedisClient = null;
|
|
100
|
+
cb();
|
|
101
|
+
}
|
|
102
|
+
});
|
|
103
|
+
}
|
|
104
|
+
removeMessageHandler(queue, cb) {
|
|
105
|
+
const handler = this.getMessageHandler(queue);
|
|
106
|
+
if (!handler)
|
|
107
|
+
cb();
|
|
108
|
+
else {
|
|
109
|
+
this.messageHandlers = this.messageHandlers.filter((handler) => {
|
|
110
|
+
const q = handler.queue;
|
|
111
|
+
return !(q.name === queue.name && q.ns === queue.ns);
|
|
112
|
+
});
|
|
113
|
+
this.logger.info(`Message handler with parameters (${JSON.stringify(queue)}) has been removed.`);
|
|
114
|
+
const handlerInstance = this.getMessageHandlerInstance(queue);
|
|
115
|
+
if (handlerInstance)
|
|
116
|
+
this.shutdownMessageHandler(handlerInstance, cb);
|
|
117
|
+
else
|
|
118
|
+
cb();
|
|
119
|
+
}
|
|
120
|
+
}
|
|
121
|
+
addMessageHandler(queue, usePriorityQueuing, messageHandler, cb) {
|
|
122
|
+
const handler = this.getMessageHandler(queue);
|
|
123
|
+
if (handler)
|
|
124
|
+
cb(new message_handler_already_exists_error_1.MessageHandlerAlreadyExistsError(queue));
|
|
125
|
+
else {
|
|
126
|
+
const handlerParams = {
|
|
127
|
+
queue,
|
|
128
|
+
usePriorityQueuing,
|
|
129
|
+
messageHandler,
|
|
130
|
+
};
|
|
131
|
+
this.messageHandlers.push(handlerParams);
|
|
132
|
+
this.logger.info(`Message handler with parameters (${JSON.stringify(handlerParams)}) has been registered.`);
|
|
133
|
+
if (this.consumer.isRunning())
|
|
134
|
+
this.runMessageHandler(handlerParams, (err) => {
|
|
135
|
+
if (err)
|
|
136
|
+
cb(err);
|
|
137
|
+
else
|
|
138
|
+
cb(null, true);
|
|
139
|
+
});
|
|
140
|
+
else
|
|
141
|
+
cb(null, false);
|
|
142
|
+
}
|
|
143
|
+
}
|
|
144
|
+
getQueues() {
|
|
145
|
+
return this.messageHandlers.map((i) => ({
|
|
146
|
+
queue: i.queue,
|
|
147
|
+
usingPriorityQueuing: i.usePriorityQueuing,
|
|
148
|
+
}));
|
|
149
|
+
}
|
|
150
|
+
}
|
|
151
|
+
exports.MessageHandlerRunner = MessageHandlerRunner;
|
|
152
|
+
//# sourceMappingURL=message-handler-runner.js.map
|
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
/// <reference types="node" />
|
|
2
|
+
import { ICallback, ICompatibleLogger, TConsumerMessageHandler, TQueueParams, TRedisClientMulti } from '../../../../../types';
|
|
3
|
+
import { RedisClient } from '../../../common/redis-client/redis-client';
|
|
4
|
+
import { EventEmitter } from 'events';
|
|
5
|
+
import { PowerManager } from '../../../common/power-manager/power-manager';
|
|
6
|
+
import { ConsumerMessageRate } from '../consumer-message-rate';
|
|
7
|
+
import { DequeueMessage } from './dequeue-message';
|
|
8
|
+
import { ConsumeMessage } from './consume-message';
|
|
9
|
+
export declare class MessageHandler extends EventEmitter {
|
|
10
|
+
protected id: string;
|
|
11
|
+
protected consumerId: string;
|
|
12
|
+
protected queue: TQueueParams;
|
|
13
|
+
protected redisClient: RedisClient;
|
|
14
|
+
protected powerManager: PowerManager;
|
|
15
|
+
protected messageRate: ConsumerMessageRate | null;
|
|
16
|
+
protected usingPriorityQueuing: boolean;
|
|
17
|
+
protected logger: ICompatibleLogger;
|
|
18
|
+
protected dequeueMessage: DequeueMessage;
|
|
19
|
+
protected consumeMessage: ConsumeMessage;
|
|
20
|
+
protected handler: TConsumerMessageHandler;
|
|
21
|
+
constructor(consumerId: string, queue: TQueueParams, handler: TConsumerMessageHandler, usePriorityQueuing: boolean, redisClient: RedisClient, messageRate?: ConsumerMessageRate | null);
|
|
22
|
+
protected registerEventsHandlers(): void;
|
|
23
|
+
handleError(err: Error): void;
|
|
24
|
+
dequeue(): void;
|
|
25
|
+
run(cb: ICallback<void>): void;
|
|
26
|
+
shutdown(redisClient: RedisClient, cb: ICallback<void>): void;
|
|
27
|
+
getQueue(): TQueueParams;
|
|
28
|
+
isUsingPriorityQueuing(): boolean;
|
|
29
|
+
getConsumerId(): string;
|
|
30
|
+
getId(): string;
|
|
31
|
+
isRunning(): boolean;
|
|
32
|
+
getHandler(): TConsumerMessageHandler;
|
|
33
|
+
static cleanUp(redisClient: RedisClient, consumerId: string, queue: TQueueParams, pendingMulti: TRedisClientMulti | undefined, cb: ICallback<void>): void;
|
|
34
|
+
}
|