redis-consumer 1.1.2 → 1.1.3

package/dist/client.d.ts

@@ -0,0 +1,27 @@
+import { RedisScripts } from 'redis';
+import { RedisClientOptions } from 'redis';
+import { RedisConsumer, ConsumerOptions, ProcessErrorData } from './consumer';
+import { RedisProducer } from './producer';
+import { RetryFailedMessage, RetryMessage } from './retry-processor';
+declare const InstRedisClient: import("@redis/client/dist/lib/client").InstantiableRedisClient<import("redis").RedisModules, import("redis").RedisFunctions, RedisScripts>;
+interface AdditionalClientOptions {
+    groupName: string;
+    clientName: string;
+}
+export declare interface RedisClient<S extends RedisScripts = RedisScripts> {
+    on(event: 'process-error', listener: (err: Error, data: ProcessErrorData) => void): this;
+    on(event: 'retry-failed', listener: (err: Error, data: RetryFailedMessage) => void): this;
+    on(event: 'retry', listener: (data: RetryMessage) => void): this;
+    on(event: string, listener: (data: any) => void): this;
+}
+export declare class RedisClient<S extends RedisScripts> extends InstRedisClient {
+    readonly groupName: string;
+    readonly clientName: string;
+    constructor(options: Omit<RedisClientOptions<never, never, S>, 'modules'> & AdditionalClientOptions);
+    createConsumer<S extends RedisScripts>(options?: ConsumerOptions): RedisConsumer<S>;
+    createProducer(): RedisProducer<S>;
+    streamExists(key: string): Promise<number>;
+    groupExists(key: string): Promise<boolean>;
+    createGroup(key: string): Promise<string>;
+}
+export {};

package/dist/client.js

@@ -0,0 +1,39 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.RedisClient = void 0;
+const client_1 = __importDefault(require("@redis/client/dist/lib/client"));
+const consumer_1 = require("./consumer");
+const producer_1 = require("./producer");
+const InstRedisClient = client_1.default.extend();
+class RedisClient extends InstRedisClient {
+    groupName;
+    clientName;
+    constructor(options) {
+        super(options);
+        this.groupName = options.groupName;
+        this.clientName = options.clientName;
+    }
+    createConsumer(options) {
+        return new consumer_1.RedisConsumer(this, options);
+    }
+    createProducer() {
+        return new producer_1.RedisProducer(this);
+    }
+    async streamExists(key) {
+        return await this.exists(key);
+    }
+    async groupExists(key) {
+        if (!(await this.streamExists(key)))
+            return false;
+        const groupInfo = await this.xInfoGroups(key);
+        return groupInfo.length > 0;
+    }
+    async createGroup(key) {
+        const result = await this.xGroupCreate(key, this.groupName, '$', { MKSTREAM: true });
+        return result;
+    }
+}
+exports.RedisClient = RedisClient;

package/dist/consumer.d.ts

@@ -0,0 +1,52 @@
+import { RedisClientType, RedisScripts } from 'redis';
+import { StreamMessageReply as OriginalStreamMessageReply } from '@redis/client/dist/lib/commands/generic-transformers';
+import { StreamMessageData, StreamMessageId } from '.';
+import { RedisClient } from './client';
+export type StreamMessageReply<T = StreamMessageData> = Omit<OriginalStreamMessageReply, 'message'> & {
+    message: T;
+};
+export type StreamProcessingFunction<T> = (data: StreamMessageReply<T>, stream: string) => void;
+export interface StreamToListen {
+    name: string;
+    executable: StreamProcessingFunction<any>;
+    id?: string;
+}
+export type StreamsToListen = StreamToListen[];
+export interface ConsumerOptions {
+    COUNT?: number;
+    BLOCK?: number;
+    retries?: number;
+    retryTime?: string[];
+}
+export interface ProcessErrorData {
+    stream: string;
+    message: OriginalStreamMessageReply;
+    retries: number;
+}
+export declare class RedisConsumer<S extends RedisScripts = RedisScripts> {
+    client: RedisClientType<any, any, any>;
+    private originalClient;
+    private state;
+    private retryProcessor;
+    private successfullMessages;
+    private BLOCK;
+    private COUNT;
+    private RETRIES;
+    constructor(client: RedisClient<S>, options?: ConsumerOptions);
+    set block(block: number);
+    set count(count: number);
+    set retries(retries: number);
+    get settings(): {
+        block: number;
+        count: number;
+    };
+    listen(streams: StreamToListen | StreamsToListen): Promise<void>;
+    addAckMessage(stream: string, id: StreamMessageId): void;
+    private listenForStreams;
+    private hasStreamState;
+    private getStreamState;
+    private initStreamState;
+    private readStreams;
+    private processStreamMessages;
+    private acknowlegdeMessages;
+}

package/dist/consumer.js

@@ -0,0 +1,162 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.RedisConsumer = void 0;
+const retry_processor_1 = require("./retry-processor");
+class RedisConsumer {
+    client;
+    originalClient;
+    state;
+    retryProcessor;
+    successfullMessages = new Map();
+    BLOCK;
+    COUNT;
+    RETRIES;
+    constructor(client, options = {}) {
+        this.originalClient = client;
+        this.client = client.duplicate();
+        this.state = {};
+        this.COUNT = options.COUNT ?? 1;
+        this.BLOCK = options.BLOCK ?? 0;
+        this.RETRIES = options.retries ?? 3;
+        this.retryProcessor = new retry_processor_1.RetryProcessor(this, {
+            retryTime: options.retryTime,
+            maxRetry: this.RETRIES,
+        });
+        this.client.connect();
+    }
+    set block(block) {
+        this.BLOCK = block;
+    }
+    set count(count) {
+        this.COUNT = count;
+    }
+    set retries(retries) {
+        this.RETRIES = retries;
+    }
+    get settings() {
+        return { block: this.BLOCK, count: this.COUNT };
+    }
+    async listen(streams) {
+        if (!Array.isArray(streams)) {
+            streams = [streams];
+        }
+        for (const stream of streams) {
+            const groupExists = await this.originalClient.groupExists(stream.name);
+            if (!groupExists) {
+                await this.originalClient.createGroup(stream.name);
+            }
+            if (this.hasStreamState(stream.name))
+                continue;
+            this.initStreamState(stream);
+        }
+        this.listenForStreams();
+    }
+    addAckMessage(stream, id) {
+        if (this.successfullMessages.has(stream)) {
+            const ackMessages = this.successfullMessages.get(stream);
+            ackMessages.push(id);
+        }
+        else {
+            this.successfullMessages.set(stream, [id]);
+        }
+    }
+    async listenForStreams() {
+        const state = this.state;
+        const streamsToListen = [];
+        for (const stream in state) {
+            streamsToListen.push({ key: stream, id: state[stream].nextId });
+        }
+        const streamsMessages = await this.readStreams(streamsToListen);
+        if (!streamsMessages) {
+            await this.acknowlegdeMessages();
+            this.listenForStreams();
+            return;
+        }
+        for (const streamMessages of streamsMessages) {
+            await this.processStreamMessages(streamMessages);
+        }
+        this.listenForStreams();
+    }
+    hasStreamState(name) {
+        return !!this.getStreamState(name);
+    }
+    getStreamState(name) {
+        const state = this.state[name];
+        if (state) {
+            return state;
+        }
+        else {
+            return null;
+        }
+    }
+    initStreamState(stream) {
+        const name = stream.name;
+        const executable = stream.executable;
+        let lastSuccessId;
+        if (stream.id) {
+            lastSuccessId = stream.id;
+        }
+        else {
+            lastSuccessId = '0-0';
+        }
+        let nextId = lastSuccessId;
+        this.state[name] = { nextId, lastSuccessId, executable, recovering: true };
+    }
+    async readStreams(streamsToListen) {
+        const messages = await this.client.xReadGroup(this.originalClient.groupName, this.originalClient.clientName, streamsToListen, { BLOCK: this.BLOCK, COUNT: this.COUNT });
+        if (!messages)
+            return null;
+        else
+            return messages;
+    }
+    async processStreamMessages(streamMessages) {
+        const stream = streamMessages.name;
+        const state = this.getStreamState(stream);
+        if (!state)
+            throw new Error('No state was found for stream processing of ' + stream);
+        const fnc = state.executable;
+        const messages = streamMessages.messages;
+        for (const message of messages) {
+            try {
+                await fnc(message, stream);
+                this.addAckMessage(stream, message.id);
+            }
+            catch (err) {
+                this.client.emit('process-error', err, { stream, message, retries: 0 });
+                if (this.RETRIES === 0)
+                    continue;
+                if (err instanceof Error) {
+                    this.retryProcessor.add(err, stream, message, fnc);
+                }
+                else {
+                    const newErr = new Error(String(err));
+                    this.retryProcessor.add(newErr, stream, message, fnc);
+                }
+            }
+        }
+        await this.acknowlegdeMessages();
+        const recovering = state.recovering;
+        if (recovering && messages.length === 0) {
+            state.nextId = '>';
+            state.recovering = false;
+        }
+        else if (recovering) {
+            const lastMessage = messages.slice(-1);
+            const lastId = lastMessage[0].id;
+            state.nextId = lastId;
+        }
+        return true;
+    }
+    async acknowlegdeMessages() {
+        for (const item of this.successfullMessages) {
+            const stream = item[0];
+            const ackMessages = item[1];
+            const id = await this.originalClient.xAck(stream, this.originalClient.groupName, ackMessages);
+            if (id) {
+                this.successfullMessages.delete(stream);
+                this.originalClient.xDel(stream, ackMessages);
+            }
+        }
+    }
+}
+exports.RedisConsumer = RedisConsumer;

package/dist/helpers.d.ts

@@ -0,0 +1,2 @@
+export declare const timeout: (ms: number) => Promise<unknown>;
+export type PropType<T, K extends keyof T> = T[K];

package/dist/helpers.js

@@ -0,0 +1,5 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.timeout = void 0;
+const timeout = async (ms) => new Promise(resolve => setTimeout(resolve, ms));
+exports.timeout = timeout;

package/dist/index.d.ts

@@ -0,0 +1,9 @@
+import { StreamMessageReply } from '@redis/client/dist/lib/commands/generic-transformers';
+import { PropType } from './helpers';
+export * from 'redis';
+export { RedisClient } from './client';
+export * from './consumer';
+export * from './producer';
+export { RetryFailedMessage } from './retry-processor';
+export type StreamMessageId = PropType<StreamMessageReply, 'id'>;
+export type StreamMessageData = PropType<StreamMessageReply, 'message'>;

package/dist/index.js

@@ -0,0 +1,22 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __exportStar = (this && this.__exportStar) || function(m, exports) {
+    for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.RedisClient = void 0;
+__exportStar(require("redis"), exports);
+var client_1 = require("./client");
+Object.defineProperty(exports, "RedisClient", { enumerable: true, get: function () { return client_1.RedisClient; } });
+__exportStar(require("./consumer"), exports);
+__exportStar(require("./producer"), exports);

package/dist/producer.d.ts

@@ -0,0 +1,8 @@
+import { RedisScripts } from 'redis';
+import { StreamMessageData, StreamMessageId } from '.';
+import { RedisClient } from './client';
+export declare class RedisProducer<S extends RedisScripts = RedisScripts> {
+    private client;
+    constructor(client: RedisClient<S>);
+    add(stream: StreamMessageId, message: StreamMessageData): void;
+}

package/dist/producer.js

@@ -0,0 +1,13 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.RedisProducer = void 0;
+class RedisProducer {
+    client;
+    constructor(client) {
+        this.client = client;
+    }
+    add(stream, message) {
+        this.client.xAdd(stream, '*', message);
+    }
+}
+exports.RedisProducer = RedisProducer;

package/dist/retry-processor.d.ts

@@ -0,0 +1,36 @@
+/// <reference types="node" />
+/// <reference types="node" />
+import { StreamProcessingFunction, RedisConsumer } from './consumer';
+import { StreamMessageReply } from '@redis/client/dist/lib/commands/generic-transformers';
+import { RedisScripts } from 'redis';
+import { EventEmitter } from 'stream';
+interface RetryState {
+    lastError: Error;
+    timestamps: number[];
+    retries: number;
+    message: StreamMessageReply;
+    stream: string;
+    executable: StreamProcessingFunction<any>;
+}
+export type RetryFailedMessage = Omit<RetryState, 'executable' | 'lastError'>;
+export type RetryMessage = Omit<RetryFailedMessage, 'timestamps' | 'lastError'> & {
+    timestamp: number;
+};
+interface RetryProcessorOptions {
+    maxRetry: number;
+    retryTime?: string[];
+}
+export declare class RetryProcessor<S extends RedisScripts = RedisScripts> extends EventEmitter {
+    private consumer;
+    private state;
+    private retryTime;
+    private maxRetry;
+    constructor(consumer: RedisConsumer<S>, options: RetryProcessorOptions);
+    add(error: Error, stream: string, message: StreamMessageReply, executable: StreamProcessingFunction<any>): void;
+    private processRetry;
+    private calcTimeoutTime;
+    private emitRetryFail;
+    private emitRetry;
+    private emitProcessFailed;
+}
+export {};

package/dist/retry-processor.js

@@ -0,0 +1,84 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.RetryProcessor = void 0;
+const helpers_1 = require("./helpers");
+const stream_1 = require("stream");
+class RetryProcessor extends stream_1.EventEmitter {
+    consumer;
+    state = new Map();
+    retryTime;
+    maxRetry;
+    constructor(consumer, options) {
+        super();
+        this.consumer = consumer;
+        this.retryTime = options.retryTime || ['15s', '1m', '15m'];
+        this.maxRetry = options.maxRetry;
+    }
+    add(error, stream, message, executable) {
+        const id = message.id;
+        if (this.state.has(id))
+            return;
+        this.state.set(id, { lastError: error, timestamps: [], retries: 0, message, stream, executable });
+        this.processRetry(id);
+    }
+    async processRetry(id) {
+        const stateObj = this.state.get(id);
+        if (stateObj.retries >= this.maxRetry) {
+            this.state.delete(id);
+            this.emitRetryFail(stateObj);
+            return;
+        }
+        stateObj.retries++;
+        const timestamp = new Date().getTime();
+        stateObj.timestamps.push(timestamp);
+        const timeoutTime = this.calcTimeoutTime(stateObj);
+        await (0, helpers_1.timeout)(timeoutTime);
+        const fnc = stateObj.executable;
+        const message = stateObj.message;
+        try {
+            this.emitRetry(stateObj);
+            await fnc(message, stateObj.stream);
+            this.consumer.addAckMessage(stateObj.stream, id);
+            this.state.delete(id);
+        }
+        catch (err) {
+            if (err instanceof Error) {
+                stateObj.lastError = err;
+            }
+            else {
+                const newErr = new Error(String(err));
+                stateObj.lastError = newErr;
+            }
+            this.emitProcessFailed(stateObj);
+            this.processRetry(id);
+        }
+        return;
+    }
+    calcTimeoutTime(stateObj) {
+        const retry = stateObj.retries;
+        const index = (retry > this.retryTime.length ? this.retryTime.length : retry) - 1;
+        const timeString = this.retryTime[index];
+        let hours = +timeString.replace(/(\d+)h/, '$1');
+        let minutes = +timeString.replace(/(\d+)m/, '$1');
+        let seconds = +timeString.replace(/(\d+)s/, '$1');
+        if (isNaN(hours))
+            hours = 0;
+        if (isNaN(minutes))
+            minutes = 0;
+        if (isNaN(seconds))
+            seconds = 0;
+        let timeoutTime = hours * 3600 * 1000 + minutes * 60 * 1000 + seconds * 1000;
+        return timeoutTime;
+    }
+    emitRetryFail({ lastError, stream, message, retries, timestamps }) {
+        this.consumer.client.emit('retry-failed', lastError, { stream, message, retries, timestamps });
+    }
+    emitRetry({ stream, message, retries, timestamps }) {
+        const timestamp = timestamps[timestamps.length - 1];
+        this.consumer.client.emit('retry', { stream, message, retries, timestamp });
+    }
+    emitProcessFailed({ lastError, stream, message, retries }) {
+        this.consumer.client.emit('process-error', lastError, { stream, message, retries });
+    }
+}
+exports.RetryProcessor = RetryProcessor;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "redis-consumer",
-  "version": "1.1.2",
+  "version": "1.1.3",
   "description": "Simple NodeJs consumer and producer for Redis Streams",
   "main": "./dist/index.js",
   "types": "./dist/index.d.ts",
@@ -33,7 +33,7 @@
   },
   "homepage": "https://github.com/tomasky/redis-streams-nodejs#readme",
   "dependencies": {
-    "redis": "^4.6.10"
+    "redis": "^4.6.12"
   },
   "devDependencies": {
     "@types/node": "^18.0.0",