redis-consumer 1.1.10 → 1.2.0

package/dist/consumer.d.ts

@@ -1,14 +1,29 @@
-import { RedisClientType, RedisScripts } from 'redis';
-import { StreamMessageReply as OriginalStreamMessageReply } from '@redis/client/dist/lib/commands/generic-transformers';
-import { StreamMessageData, StreamMessageId } from '.';
-import { RedisClient } from './client';
-export type StreamMessageReply<T = StreamMessageData> = Omit<OriginalStreamMessageReply, 'message'> & {
+import type { RedisArgument, RedisClientType, RedisScripts } from 'redis';
+import type { RedisClient } from './client.js';
+export type StreamMessageId = string;
+export type StreamMessageData = Record<string, RedisArgument>;
+export interface StreamMessageReply<T = StreamMessageData> {
+    id: StreamMessageId;
     message: T;
-};
+    millisElapsedFromDelivery?: number;
+    deliveriesCounter?: number;
+}
+/** Called once per message, with that single message. */
 export type StreamProcessingFunction<T> = (data: StreamMessageReply<T>, stream: string) => void;
+/** Called once per read, with the whole array of messages of that read. */
+export type StreamBatchProcessingFunction<T> = (messages: StreamMessageReply<T>[], stream: string) => void;
+/**
+ * How the consumer holds an executable.
+ *
+ * Which of the two shapes above it is actually called with depends on the
+ * `batchMessages` option, so the payload is intentionally untyped here. Annotate
+ * your own function as `StreamProcessingFunction<T>` or
+ * `StreamBatchProcessingFunction<T>` to have its parameter checked.
+ */
+export type StreamExecutable = (payload: any, stream: string) => void;
 export interface StreamToListen {
     name: string;
-    executable: StreamProcessingFunction<any>;
+    executable: StreamExecutable;
     id?: string;
 }
 export type StreamsToListen = StreamToListen[];
@@ -17,10 +32,15 @@ export interface ConsumerOptions {
     BLOCK?: number;
     retries?: number;
     retryTime?: string[];
+    /**
+     * When `true`, every read hands the whole batch of messages to the
+     * executable as a single array instead of one call per message.
+     */
+    batchMessages?: boolean;
 }
 export interface ProcessErrorData {
     stream: string;
-    message: OriginalStreamMessageReply;
+    message: StreamMessageReply;
     retries: number;
 }
 export declare class RedisConsumer<S extends RedisScripts = RedisScripts> {
@@ -32,6 +52,14 @@ export declare class RedisConsumer<S extends RedisScripts = RedisScripts> {
     private BLOCK;
     private COUNT;
     private RETRIES;
+    private batchMessages;
+    private ready;
+    private listenPromise;
+    private stopped;
+    /** Tail of the serialised acknowledgement flush chain. */
+    private ackFlush;
+    /** Aborts the read loop's error back-off so `close()` does not have to wait it out. */
+    private abort;
     constructor(client: RedisClient<S>, options?: ConsumerOptions);
     set block(block: number);
     set count(count: number);
@@ -40,13 +68,37 @@ export declare class RedisConsumer<S extends RedisScripts = RedisScripts> {
         block: number;
         count: number;
     };
-    listen(streams: StreamToListen | StreamsToListen): Promise<void>;
+    listen(streams: StreamToListen | StreamsToListen, flag?: boolean): Promise<void>;
+    /**
+     * Stops reading and closes the consumer's connection, so the process can exit.
+     */
+    close(): Promise<void>;
     addAckMessage(stream: string, id: StreamMessageId): void;
+    /**
+     * Sends the queued acknowledgements (`XACK` + `XDEL`) to Redis.
+     *
+     * Call this after {@link addAckMessage} to make the acknowledgement leave the
+     * pending list right away instead of waiting for the read loop's next flush.
+     *
+     * Flushes are serialised, so the read loop and the retry processor can both
+     * call this without interleaving their `XACK`/`XDEL` batches. Ids are only
+     * dropped once Redis accepted them, so a failed flush is retried by the next
+     * one.
+     */
+    flushAcks(): Promise<void>;
     private listenForStreams;
     private hasStreamState;
     private getStreamState;
     private initStreamState;
     private readStreams;
+    /**
+     * Runs the executable for a single read and acknowledges its messages only
+     * when processing succeeded, so a failed message stays in the pending list.
+     *
+     * In batch mode `payload` is the whole array. A failed batch call cannot tell
+     * which message failed, so every message of the read is retried on its own.
+     */
+    private handle;
     private processStreamMessages;
     private acknowlegdeMessages;
 }

package/dist/consumer.js

@@ -1,164 +1,274 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.RedisConsumer = void 0;
-const retry_processor_1 = require("./retry-processor");
-let errs = "error";
-class RedisConsumer {
-  client;
-  originalClient;
-  state;
-  retryProcessor;
-  successfullMessages = new Map();
-  BLOCK;
-  COUNT;
-  RETRIES;
-  constructor(client, options = {}) {
-    this.originalClient = client;
-    this.client = client.duplicate();
-    this.state = {};
-    this.COUNT = options.COUNT ?? 1;
-    this.BLOCK = options.BLOCK ?? 0;
-    this.RETRIES = options.retries ?? 3;
-    this.retryProcessor = new retry_processor_1.RetryProcessor(this, {
-      retryTime: options.retryTime,
-      maxRetry: this.RETRIES,
-    });
-    this.client.connect();
-  }
-  set block(block) {
-    this.BLOCK = block;
-  }
-  set count(count) {
-    this.COUNT = count;
-  }
-  set retries(retries) {
-    this.RETRIES = retries;
-  }
-  get settings() {
-    return { block: this.BLOCK, count: this.COUNT };
-  }
-  async listen(streams, flag) {
-    if (!Array.isArray(streams)) {
-      streams = [streams];
-    }
-    for (const stream of streams) {
-      const groupExists = await this.originalClient.groupExists(stream.name);
-      if (!groupExists) {
-        await this.originalClient.createGroup(stream.name);
-      }
-      if (this.hasStreamState(stream.name)) continue;
-      this.initStreamState(stream);
-    }
-    this.listenForStreams(flag);
-  }
-  addAckMessage(stream, id) {
-    if (this.successfullMessages.has(stream)) {
-      const ackMessages = this.successfullMessages.get(stream);
-      ackMessages.push(id);
-    } else {
-      this.successfullMessages.set(stream, [id]);
-    }
-  }
-  async listenForStreams(flag) {
-    const state = this.state;
-    const streamsToListen = [];
-    for (const stream in state) {
-      streamsToListen.push({ key: stream, id: state[stream].nextId });
-    }
-    const streamsMessages = await this.readStreams(streamsToListen);
-    if (!streamsMessages) {
-      await this.acknowlegdeMessages();
-      this.listenForStreams(flag);
-      return;
-    }
-    for (const streamMessages of streamsMessages) {
-      await this.processStreamMessages(streamMessages, flag);
-    }
-    this.listenForStreams(flag);
-  }
-  hasStreamState(name) {
-    return !!this.getStreamState(name);
-  }
-  getStreamState(name) {
-    return this.state[name];
-  }
-  initStreamState(stream) {
-    const name = stream.name;
-    const executable = stream.executable;
-    let lastSuccessId;
-    if (stream.id) {
-      lastSuccessId = stream.id;
-    } else {
-      lastSuccessId = "0-0";
-    }
-    let nextId = lastSuccessId;
-    this.state[name] = { nextId, lastSuccessId, executable, recovering: true };
-  }
-  async readStreams(streamsToListen) {
-    const messages = await this.client.xReadGroup(
-      this.originalClient.groupName,
-      this.originalClient.clientName,
-      streamsToListen,
-      { BLOCK: this.BLOCK, COUNT: this.COUNT },
-    );
-    return messages;
-  }
-
-  async handle(fnc, message, stream) {
-    try {
-      await fnc(message, stream);
-    } catch (err) {
-      if (errs == "error") errs = err;
-      this.client.emit("process-error", err, { stream, message, retries: 0 });
-      if (this.RETRIES === 0) return;
-      if (err instanceof Error) {
-        this.retryProcessor.add(err, stream, message, fnc);
-      } else {
-        const newErr = new Error(String(err));
-        this.retryProcessor.add(newErr, stream, message, fnc);
-      }
-    } finally {
-      if (errs && errs != "error") (console.error(errs), (errs = ""));
-    }
-  }
-  async processStreamMessages(streamMessages, flag) {
-    const stream = streamMessages.name;
-    const state = this.getStreamState(stream);
-    if (!state)
-      throw new Error("No state was found for stream processing of " + stream);
-    const fnc = state.executable;
-    const messages = streamMessages.messages;
-    if (flag) await this.handle(fnc, messages, stream);
-    for (const message of messages) {
-      if (!flag) await this.handle(fnc, message, stream);
-      this.addAckMessage(stream, message.id);
-    }
-    await this.acknowlegdeMessages();
-    const recovering = state.recovering;
-    if (recovering && messages.length === 0) {
-      state.nextId = ">";
-      state.recovering = false;
-    } else if (recovering) {
-      const lastMessage = messages.slice(-1);
-      const lastId = lastMessage[0].id;
-      state.nextId = lastId;
-    }
-    return true;
-  }
-  async acknowlegdeMessages() {
-    for (const item of this.successfullMessages) {
-      const stream = item[0];
-      const ackMessages = item[1];
-      const id = await this.originalClient.xAck(
-        stream,
-        this.originalClient.groupName,
-        ackMessages,
-      );
-      if (id) {
-        await this.originalClient.xDel(stream, ackMessages);
-        this.successfullMessages.delete(stream);
-      }
-    }
-  }
+import { RetryProcessor } from './retry-processor.js';
+import { timeout } from './helpers.js';
+/** Delay before retrying a read that failed, to avoid a hot error loop. */
+const LISTEN_ERROR_RETRY_DELAY = 1000;
+/**
+ * Wraps a batch executable so that a single retried message is delivered to it
+ * as a one-item batch, keeping retry bookkeeping per message.
+ */
+function singleMessage(fnc) {
+    return (message, stream) => fnc([message], stream);
+}
+export class RedisConsumer {
+    client;
+    originalClient;
+    state;
+    retryProcessor;
+    successfullMessages = new Map();
+    BLOCK;
+    COUNT;
+    RETRIES;
+    batchMessages;
+    ready;
+    listenPromise = null;
+    stopped = false;
+    /** Tail of the serialised acknowledgement flush chain. */
+    ackFlush = Promise.resolve();
+    /** Aborts the read loop's error back-off so `close()` does not have to wait it out. */
+    abort = new AbortController();
+    constructor(client, options = {}) {
+        this.originalClient = client;
+        this.client = client.duplicate();
+        this.state = {};
+        this.COUNT = options.COUNT ?? 5000;
+        this.BLOCK = options.BLOCK ?? 0;
+        this.RETRIES = options.retries ?? 3;
+        this.batchMessages = options.batchMessages ?? false;
+        this.retryProcessor = new RetryProcessor(this, {
+            retryTime: options.retryTime,
+            maxRetry: this.RETRIES,
+        });
+        // node-redis rejects commands issued before the connection is open, so keep
+        // the promise around and let the read loop await it before the first read.
+        this.ready = this.client.connect();
+        // Also handled here, so a consumer that is never listened on or closed
+        // cannot take the process down with an unhandled rejection.
+        this.ready.catch(() => undefined);
+    }
+    set block(block) {
+        this.BLOCK = block;
+    }
+    set count(count) {
+        this.COUNT = count;
+    }
+    set retries(retries) {
+        this.RETRIES = retries;
+        this.retryProcessor.maxRetry = retries;
+    }
+    get settings() {
+        return { block: this.BLOCK, count: this.COUNT };
+    }
+    async listen(streams, flag) {
+        // `flag` is the legacy positional form of `batchMessages`.
+        const batch = flag ?? this.batchMessages;
+        const streamList = Array.isArray(streams) ? streams : [streams];
+        for (const stream of streamList) {
+            // Checked before any Redis call, so a repeated listen() stays cheap.
+            if (this.hasStreamState(stream.name))
+                continue;
+            await this.originalClient.createGroup(stream.name);
+            this.initStreamState(stream);
+        }
+        // Nothing to read from: don't start a loop that would only fail.
+        if (Object.keys(this.state).length === 0)
+            return;
+        // A second loop would consume every stream a second time.
+        if (this.listenPromise)
+            return;
+        this.listenPromise = this.listenForStreams(batch);
+    }
+    /**
+     * Stops reading and closes the consumer's connection, so the process can exit.
+     */
+    async close() {
+        this.stopped = true;
+        // Pending retries sleep inside a timer; without this they would keep the
+        // event loop alive for up to the longest `retryTime`.
+        this.retryProcessor.stop();
+        this.abort.abort();
+        try {
+            await this.ready;
+        }
+        catch {
+            // The client never connected, so there is nothing to close.
+            return;
+        }
+        // Best effort: get acknowledgements that are still queued to Redis before
+        // the connections go away, so a message processed just before close() is not
+        // left in the pending list.
+        await this.flushAcks().catch(() => undefined);
+        // `destroy()` throws when the socket is already closed, which is the state
+        // we are aiming for anyway.
+        if (this.client.isOpen)
+            this.client.destroy();
+        await this.listenPromise;
+    }
+    addAckMessage(stream, id) {
+        const ackMessages = this.successfullMessages.get(stream);
+        if (ackMessages) {
+            ackMessages.push(id);
+        }
+        else {
+            this.successfullMessages.set(stream, [id]);
+        }
+    }
+    /**
+     * Sends the queued acknowledgements (`XACK` + `XDEL`) to Redis.
+     *
+     * Call this after {@link addAckMessage} to make the acknowledgement leave the
+     * pending list right away instead of waiting for the read loop's next flush.
+     *
+     * Flushes are serialised, so the read loop and the retry processor can both
+     * call this without interleaving their `XACK`/`XDEL` batches. Ids are only
+     * dropped once Redis accepted them, so a failed flush is retried by the next
+     * one.
+     */
+    flushAcks() {
+        const run = this.ackFlush.then(() => this.acknowlegdeMessages(), 
+        // A previous flush failed, which must not block this one.
+        () => this.acknowlegdeMessages());
+        // Keep the internal chain usable even when this flush rejects.
+        this.ackFlush = run.catch(() => undefined);
+        return run;
+    }
+    async listenForStreams(batch) {
+        try {
+            await this.ready;
+        }
+        catch (err) {
+            this.client.emit('listen-error', err);
+            return;
+        }
+        while (!this.stopped) {
+            try {
+                const streamsToListen = [];
+                for (const [key, streamState] of Object.entries(this.state)) {
+                    streamsToListen.push({ key, id: streamState.nextId });
+                }
+                const streamsMessages = await this.readStreams(streamsToListen);
+                if (streamsMessages) {
+                    for (const streamMessages of streamsMessages) {
+                        await this.processStreamMessages(streamMessages, batch);
+                    }
+                }
+                await this.flushAcks();
+            }
+            catch (err) {
+                // A deliberate close() rejects the in-flight read; that is not an error.
+                if (this.stopped)
+                    break;
+                this.client.emit('listen-error', err);
+                await timeout(LISTEN_ERROR_RETRY_DELAY, this.abort.signal);
+            }
+        }
+    }
+    hasStreamState(name) {
+        return !!this.getStreamState(name);
+    }
+    getStreamState(name) {
+        return this.state[name] ?? null;
+    }
+    initStreamState(stream) {
+        const startId = stream.id ?? '0-0';
+        this.state[stream.name] = {
+            nextId: startId,
+            executable: stream.executable,
+            // '>' means "new messages only": there is no pending list to drain first.
+            recovering: startId !== '>',
+        };
+    }
+    async readStreams(streamsToListen) {
+        const messages = await this.client.xReadGroup(this.originalClient.groupName, this.originalClient.clientName, streamsToListen, { BLOCK: this.BLOCK, COUNT: this.COUNT });
+        if (!messages)
+            return null;
+        return messages;
+    }
+    /**
+     * Runs the executable for a single read and acknowledges its messages only
+     * when processing succeeded, so a failed message stays in the pending list.
+     *
+     * In batch mode `payload` is the whole array. A failed batch call cannot tell
+     * which message failed, so every message of the read is retried on its own.
+     */
+    async handle(fnc, payload, readMessages, stream, batch) {
+        try {
+            await fnc(payload, stream);
+        }
+        catch (err) {
+            const error = err instanceof Error ? err : new Error(String(err));
+            for (const message of readMessages) {
+                this.client.emit('process-error', error, { stream, message, retries: 0 });
+                if (this.RETRIES === 0)
+                    continue;
+                this.retryProcessor.add(error, stream, message, batch ? singleMessage(fnc) : fnc);
+            }
+            return;
+        }
+        for (const message of readMessages) {
+            this.addAckMessage(stream, message.id);
+        }
+    }
+    async processStreamMessages(streamMessages, batch) {
+        const stream = streamMessages.name;
+        const state = this.getStreamState(stream);
+        if (!state)
+            throw new Error('No state was found for stream processing of ' + stream);
+        const fnc = state.executable;
+        const messages = streamMessages.messages;
+        if (batch) {
+            // A read that only served an empty pending list has nothing to hand over.
+            if (messages.length > 0) {
+                await this.handle(fnc, messages, messages, stream, true);
+            }
+        }
+        else {
+            for (const message of messages) {
+                await this.handle(fnc, message, [message], stream, false);
+            }
+        }
+        if (state.recovering) {
+            if (messages.length === 0) {
+                state.nextId = '>';
+                state.recovering = false;
+            }
+            else {
+                const lastMessage = messages[messages.length - 1];
+                if (lastMessage)
+                    state.nextId = lastMessage.id;
+            }
+        }
+    }
+    async acknowlegdeMessages() {
+        // Snapshot the entries: ids queued while this flush is awaiting belong to
+        // the next flush, not to this one.
+        const batches = [...this.successfullMessages];
+        if (batches.length === 0)
+            return;
+        // Detach before awaiting. Otherwise an `addAckMessage` for the same stream
+        // would append to the array being flushed and then be dropped by the
+        // removal below without ever reaching Redis.
+        for (const [stream] of batches)
+            this.successfullMessages.delete(stream);
+        // `XACK` and `XDEL` are independent, and node-redis pipelines commands that
+        // are in flight at the same time. Issuing all of them before awaiting costs
+        // one round trip for the whole flush instead of two per stream.
+        const inFlight = [];
+        for (const [stream, ids] of batches) {
+            // `XACK` reports ids that were already acknowledged as 0. The entry is
+            // deleted regardless, so a redelivered message cannot keep the queue
+            // growing without bound.
+            inFlight.push(this.originalClient.xAck(stream, this.originalClient.groupName, ids), this.originalClient.xDel(stream, ids));
+        }
+        try {
+            await Promise.all(inFlight);
+        }
+        catch (err) {
+            // Put the ids back so the next flush retries them.
+            for (const [stream, ids] of batches) {
+                for (const id of ids)
+                    this.addAckMessage(stream, id);
+            }
+            throw err;
+        }
+    }
 }
-exports.RedisConsumer = RedisConsumer;

package/dist/helpers.d.ts

@@ -1,2 +1,5 @@
-export declare const timeout: (ms: number) => Promise<unknown>;
-export type PropType<T, K extends keyof T> = T[K];
+/**
+ * Resolves after `ms`, or as soon as `signal` aborts. The timer is always
+ * cleared, so a pending sleep never keeps the process alive.
+ */
+export declare const timeout: (ms: number, signal?: AbortSignal) => Promise<void>;

package/dist/helpers.js

@@ -1,5 +1,59 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.timeout = void 0;
-const timeout = async (ms) => new Promise(resolve => setTimeout(resolve, ms));
-exports.timeout = timeout;
+/**
+ * Pending cancellations, one set per signal.
+ *
+ * `AbortSignal` is an `EventTarget`, and adding a listener scans the existing
+ * ones for a duplicate - so attaching one listener per sleep is O(n) per call
+ * and O(n^2) for a batch. A failed read of 20k messages arms 20k retry sleeps
+ * against the same signal, which took ~14s before a single timer fired and
+ * blocked the read loop for that whole time. Sharing one listener per signal
+ * keeps it linear.
+ */
+const cancellations = new WeakMap();
+/** Registers `cancel` to run when `signal` aborts. Returns an unregister fn. */
+function onAbort(signal, cancel) {
+    let pending = cancellations.get(signal);
+    if (!pending) {
+        pending = new Set();
+        cancellations.set(signal, pending);
+        signal.addEventListener('abort', () => {
+            const waiting = cancellations.get(signal);
+            cancellations.delete(signal);
+            if (!waiting)
+                return;
+            // Snapshot: each callback unregisters itself.
+            const callbacks = [...waiting];
+            waiting.clear();
+            for (const callback of callbacks)
+                callback();
+        }, { once: true });
+    }
+    pending.add(cancel);
+    return () => pending.delete(cancel);
+}
+/**
+ * Resolves after `ms`, or as soon as `signal` aborts. The timer is always
+ * cleared, so a pending sleep never keeps the process alive.
+ */
+export const timeout = async (ms, signal) => {
+    if (!signal) {
+        await new Promise(resolve => setTimeout(resolve, ms));
+        return;
+    }
+    if (signal.aborted)
+        return;
+    await new Promise(resolve => {
+        let settled = false;
+        const done = () => {
+            if (settled)
+                return;
+            settled = true;
+            clearTimeout(timer);
+            unregister();
+            resolve();
+        };
+        // No `await` between the `aborted` check above and this registration, so the
+        // signal cannot fire before `done` is reachable.
+        const unregister = onAbort(signal, done);
+        const timer = setTimeout(done, ms);
+    });
+};

package/dist/index.d.ts

@@ -1,9 +1,5 @@
-import { StreamMessageReply } from '@redis/client/dist/lib/commands/generic-transformers';
-import { PropType } from './helpers';
-export * from 'redis';
-export { RedisClient } from './client';
-export * from './consumer';
-export * from './producer';
-export { RetryFailedMessage } from './retry-processor';
-export type StreamMessageId = PropType<StreamMessageReply, 'id'>;
-export type StreamMessageData = PropType<StreamMessageReply, 'message'>;
+export type * from 'redis';
+export { RedisClient } from './client.js';
+export * from './consumer.js';
+export * from './producer.js';
+export type { RetryFailedMessage, RetryMessage } from './retry-processor.js';