sqs-consumer 6.0.2 → 6.2.0

package/src/consumer.ts

@@ -16,132 +16,42 @@ import {
   ReceiveMessageCommandOutput
 } from '@aws-sdk/client-sqs';
 import Debug from 'debug';
-import { EventEmitter } from 'events';
 
-import { AWSError } from './types';
+import { ConsumerOptions, TypedEventEmitter } from './types';
 import { autoBind } from './bind';
-import { SQSError, TimeoutError } from './errors';
+import {
+  SQSError,
+  TimeoutError,
+  toSQSError,
+  isConnectionError
+} from './errors';
+import { assertOptions, hasMessages } from './validation';
 
 const debug = Debug('sqs-consumer');
 
-const requiredOptions = [
-  'queueUrl',
-  // only one of handleMessage / handleMessagesBatch is required
-  'handleMessage|handleMessageBatch'
-];
-
-interface TimeoutResponse {
-  timeout: NodeJS.Timeout;
-  pending: Promise<void>;
-}
-
-function createTimeout(duration: number): TimeoutResponse[] {
-  let timeout;
-  const pending = new Promise((_, reject) => {
-    timeout = setTimeout((): void => {
-      reject(new TimeoutError());
-    }, duration);
-  });
-  return [timeout, pending];
-}
-
-function assertOptions(options: ConsumerOptions): void {
-  requiredOptions.forEach((option) => {
-    const possibilities = option.split('|');
-    if (!possibilities.find((p) => options[p])) {
-      throw new Error(
-        `Missing SQS consumer option [ ${possibilities.join(' or ')} ].`
-      );
-    }
-  });
-
-  if (options.batchSize > 10 || options.batchSize < 1) {
-    throw new Error('SQS batchSize option must be between 1 and 10.');
-  }
-
-  if (
-    options.heartbeatInterval &&
-    !(options.heartbeatInterval < options.visibilityTimeout)
-  ) {
-    throw new Error('heartbeatInterval must be less than visibilityTimeout.');
-  }
-}
-
-function isConnectionError(err: Error): boolean {
-  if (err instanceof SQSError) {
-    return (
-      err.statusCode === 403 ||
-      err.code === 'CredentialsError' ||
-      err.code === 'UnknownEndpoint'
-    );
-  }
-  return false;
-}
-
-function toSQSError(err: AWSError, message: string): SQSError {
-  const sqsError = new SQSError(message);
-  sqsError.code = err.name;
-  sqsError.statusCode = err.$metadata?.httpStatusCode;
-  sqsError.retryable = err.$retryable?.throttling;
-  sqsError.service = err.$service;
-  sqsError.fault = err.$fault;
-  sqsError.time = new Date();
-
-  return sqsError;
-}
-
-function hasMessages(response: ReceiveMessageCommandOutput): boolean {
-  return response.Messages && response.Messages.length > 0;
-}
-
-export interface ConsumerOptions {
-  queueUrl?: string;
-  attributeNames?: string[];
-  messageAttributeNames?: string[];
-  stopped?: boolean;
-  batchSize?: number;
-  visibilityTimeout?: number;
-  waitTimeSeconds?: number;
-  authenticationErrorTimeout?: number;
-  pollingWaitTimeMs?: number;
-  terminateVisibilityTimeout?: boolean;
-  heartbeatInterval?: number;
-  sqs?: SQSClient;
-  region?: string;
-  handleMessageTimeout?: number;
-  shouldDeleteMessages?: boolean;
-  handleMessage?(message: Message): Promise<void>;
-  handleMessageBatch?(messages: Message[]): Promise<void>;
-}
-
-interface Events {
-  response_processed: [];
-  empty: [];
-  message_received: [Message];
-  message_processed: [Message];
-  error: [Error, void | Message | Message[]];
-  timeout_error: [Error, Message];
-  processing_error: [Error, Message];
-  stopped: [];
-}
-
-export class Consumer extends EventEmitter {
+/**
+ * [Usage](https://bbc.github.io/sqs-consumer/index.html#usage)
+ */
+export class Consumer extends TypedEventEmitter {
+  private pollingTimeoutId: NodeJS.Timeout | undefined = undefined;
+  private heartbeatTimeoutId: NodeJS.Timeout | undefined = undefined;
+  private handleMessageTimeoutId: NodeJS.Timeout | undefined = undefined;
+  private stopped = true;
   private queueUrl: string;
-  private handleMessage: (message: Message) => Promise<void>;
-  private handleMessageBatch: (message: Message[]) => Promise<void>;
+  private handleMessage: (message: Message) => Promise<Message | void>;
+  private handleMessageBatch: (message: Message[]) => Promise<Message[] | void>;
+  private sqs: SQSClient;
   private handleMessageTimeout: number;
   private attributeNames: string[];
   private messageAttributeNames: string[];
-  private stopped: boolean;
+  private shouldDeleteMessages: boolean;
   private batchSize: number;
   private visibilityTimeout: number;
+  private terminateVisibilityTimeout: boolean;
   private waitTimeSeconds: number;
   private authenticationErrorTimeout: number;
   private pollingWaitTimeMs: number;
-  private terminateVisibilityTimeout: boolean;
   private heartbeatInterval: number;
-  private sqs: SQSClient;
-  private shouldDeleteMessages: boolean;
 
   constructor(options: ConsumerOptions) {
     super();
@@ -152,7 +62,6 @@ export class Consumer extends EventEmitter {
     this.handleMessageTimeout = options.handleMessageTimeout;
     this.attributeNames = options.attributeNames || [];
     this.messageAttributeNames = options.messageAttributeNames || [];
-    this.stopped = true;
     this.batchSize = options.batchSize || 1;
     this.visibilityTimeout = options.visibilityTimeout;
     this.terminateVisibilityTimeout =
@@ -163,89 +72,171 @@ export class Consumer extends EventEmitter {
       options.authenticationErrorTimeout ?? 10000;
     this.pollingWaitTimeMs = options.pollingWaitTimeMs ?? 0;
     this.shouldDeleteMessages = options.shouldDeleteMessages ?? true;
-
     this.sqs =
       options.sqs ||
       new SQSClient({
         region: options.region || process.env.AWS_REGION || 'eu-west-1'
       });
-
     autoBind(this);
   }
 
-  emit<T extends keyof Events>(event: T, ...args: Events[T]) {
-    return super.emit(event, ...args);
+  /**
+   * Creates a new SQS consumer.
+   */
+  public static create(options: ConsumerOptions): Consumer {
+    return new Consumer(options);
   }
 
-  on<T extends keyof Events>(
-    event: T,
-    listener: (...args: Events[T]) => void
-  ): this {
-    return super.on(event, listener);
+  /**
+   * Start polling the queue for messages.
+   */
+  public start(): void {
+    if (this.stopped) {
+      debug('Starting consumer');
+      this.stopped = false;
+      this.poll();
+    }
   }
 
-  once<T extends keyof Events>(
-    event: T,
-    listener: (...args: Events[T]) => void
-  ): this {
-    return super.once(event, listener);
+  /**
+   * Stop polling the queue for messages (pre existing requests will still be made until concluded).
+   */
+  public stop(): void {
+    if (this.stopped) {
+      debug('Consumer was already stopped');
+      return;
+    }
+
+    debug('Stopping consumer');
+    this.stopped = true;
+
+    if (this.pollingTimeoutId) {
+      clearTimeout(this.pollingTimeoutId);
+      this.pollingTimeoutId = undefined;
+    }
+
+    this.emit('stopped');
   }
 
+  /**
+   * Returns the current polling state of the consumer: `true` if it is actively polling, `false` if it is not.
+   */
   public get isRunning(): boolean {
     return !this.stopped;
   }
 
-  public static create(options: ConsumerOptions): Consumer {
-    return new Consumer(options);
+  /**
+   * Emit one of the consumer's error events depending on the error received.
+   * @param err The error object to forward on
+   * @param message The message that the error occurred on
+   */
+  private emitError(err: Error, message?: Message): void {
+    if (!message) {
+      this.emit('error', err);
+    } else if (err.name === SQSError.name) {
+      this.emit('error', err, message);
+    } else if (err instanceof TimeoutError) {
+      this.emit('timeout_error', err, message);
+    } else {
+      this.emit('processing_error', err, message);
+    }
   }
 
-  public start(): void {
+  /**
+   * Poll for new messages from SQS
+   */
+  private poll(): void {
     if (this.stopped) {
-      debug('Starting consumer');
-      this.stopped = false;
-      this.poll();
+      debug('Poll was called while consumer was stopped, cancelling poll...');
+      return;
     }
+
+    debug('Polling for messages');
+
+    let currentPollingTimeout = this.pollingWaitTimeMs;
+    this.receiveMessage({
+      QueueUrl: this.queueUrl,
+      AttributeNames: this.attributeNames,
+      MessageAttributeNames: this.messageAttributeNames,
+      MaxNumberOfMessages: this.batchSize,
+      WaitTimeSeconds: this.waitTimeSeconds,
+      VisibilityTimeout: this.visibilityTimeout
+    })
+      .then(this.handleSqsResponse)
+      .catch((err) => {
+        this.emitError(err);
+        if (isConnectionError(err)) {
+          debug('There was an authentication error. Pausing before retrying.');
+          currentPollingTimeout = this.authenticationErrorTimeout;
+        }
+        return;
+      })
+      .then(() => {
+        if (this.pollingTimeoutId) {
+          clearTimeout(this.pollingTimeoutId);
+        }
+        this.pollingTimeoutId = setTimeout(this.poll, currentPollingTimeout);
+      })
+      .catch((err) => {
+        this.emitError(err);
+      });
   }
 
-  public stop(): void {
-    debug('Stopping consumer');
-    this.stopped = true;
+  /**
+   * Send a request to SQS to retrieve messages
+   * @param params The required params to receive messages from SQS
+   */
+  private async receiveMessage(
+    params: ReceiveMessageCommandInput
+  ): Promise<ReceiveMessageCommandOutput> {
+    try {
+      return await this.sqs.send(new ReceiveMessageCommand(params));
+    } catch (err) {
+      throw toSQSError(err, `SQS receive message failed: ${err.message}`);
+    }
   }
 
+  /**
+   * Handles the response from AWS SQS, determining if we should proceed to
+   * the message handler.
+   * @param response The output from AWS SQS
+   */
   private async handleSqsResponse(
     response: ReceiveMessageCommandOutput
   ): Promise<void> {
-    debug('Received SQS response');
-    debug(response);
-
-    if (response) {
-      if (hasMessages(response)) {
-        if (this.handleMessageBatch) {
-          // prefer handling messages in batch when available
-          await this.processMessageBatch(response.Messages);
-        } else {
-          await Promise.all(response.Messages.map(this.processMessage));
-        }
-        this.emit('response_processed');
+    if (hasMessages(response)) {
+      if (this.handleMessageBatch) {
+        await this.processMessageBatch(response.Messages);
       } else {
-        this.emit('empty');
+        await Promise.all(response.Messages.map(this.processMessage));
       }
+
+      this.emit('response_processed');
+    } else if (response) {
+      this.emit('empty');
     }
   }
 
+  /**
+   * Process a message that has been received from SQS. This will execute the message
+   * handler and delete the message once complete.
+   * @param message The message that was delivered from SQS
+   */
   private async processMessage(message: Message): Promise<void> {
-    this.emit('message_received', message);
-
-    let heartbeat;
     try {
+      this.emit('message_received', message);
+
       if (this.heartbeatInterval) {
-        heartbeat = this.startHeartbeat(async () => {
-          return this.changeVisibilityTimeout(message, this.visibilityTimeout);
-        });
+        this.heartbeatTimeoutId = this.startHeartbeat(message);
+      }
+
+      const ackedMessage = await this.executeHandler(message);
+
+      if (ackedMessage?.MessageId === message.MessageId) {
+        await this.deleteMessage(message);
+
+        this.emit('message_processed', message);
       }
-      await this.executeHandler(message);
-      await this.deleteMessage(message);
-      this.emit('message_processed', message);
     } catch (err) {
       this.emitError(err, message);
 
@@ -253,63 +244,71 @@ export class Consumer extends EventEmitter {
         await this.changeVisibilityTimeout(message, 0);
       }
     } finally {
-      clearInterval(heartbeat);
+      clearInterval(this.heartbeatTimeoutId);
+      this.heartbeatTimeoutId = undefined;
     }
   }
 
-  private async receiveMessage(
-    params: ReceiveMessageCommandInput
-  ): Promise<ReceiveMessageCommandOutput> {
+  /**
+   * Process a batch of messages from the SQS queue.
+   * @param messages The messages that were delivered from SQS
+   */
+  private async processMessageBatch(messages: Message[]): Promise<void> {
     try {
-      return await this.sqs.send(new ReceiveMessageCommand(params));
-    } catch (err) {
-      throw toSQSError(err, `SQS receive message failed: ${err.message}`);
-    }
-  }
+      messages.forEach((message) => {
+        this.emit('message_received', message);
+      });
 
-  private async deleteMessage(message: Message): Promise<void> {
-    if (!this.shouldDeleteMessages) {
-      debug(
-        'Skipping message delete since shouldDeleteMessages is set to false'
-      );
-      return;
-    }
-    debug('Deleting message %s', message.MessageId);
+      if (this.heartbeatInterval) {
+        this.heartbeatTimeoutId = this.startHeartbeat(null, messages);
+      }
 
-    const deleteParams: DeleteMessageCommandInput = {
-      QueueUrl: this.queueUrl,
-      ReceiptHandle: message.ReceiptHandle
-    };
+      const ackedMessages = await this.executeBatchHandler(messages);
 
-    try {
-      await this.sqs.send(new DeleteMessageCommand(deleteParams));
-    } catch (err) {
-      throw toSQSError(err, `SQS delete message failed: ${err.message}`);
-    }
-  }
+      if (ackedMessages?.length > 0) {
+        await this.deleteMessageBatch(ackedMessages);
 
-  private async executeHandler(message: Message): Promise<void> {
-    let timeout;
-    let pending;
-    try {
-      if (this.handleMessageTimeout) {
-        [timeout, pending] = createTimeout(this.handleMessageTimeout);
-        await Promise.race([this.handleMessage(message), pending]);
-      } else {
-        await this.handleMessage(message);
+        ackedMessages.forEach((message) => {
+          this.emit('message_processed', message);
+        });
       }
     } catch (err) {
-      if (err instanceof TimeoutError) {
-        err.message = `Message handler timed out after ${this.handleMessageTimeout}ms: Operation timed out.`;
-      } else if (err instanceof Error) {
-        err.message = `Unexpected message handler failure: ${err.message}`;
+      this.emit('error', err, messages);
+
+      if (this.terminateVisibilityTimeout) {
+        await this.changeVisibilityTimeoutBatch(messages, 0);
       }
-      throw err;
     } finally {
-      clearTimeout(timeout);
+      clearInterval(this.heartbeatTimeoutId);
+      this.heartbeatTimeoutId = undefined;
     }
   }
 
+  /**
+   * Trigger a function on a set interval
+   * @param heartbeatFn The function that should be triggered
+   */
+  private startHeartbeat(
+    message?: Message,
+    messages?: Message[]
+  ): NodeJS.Timeout {
+    return setInterval(() => {
+      if (this.handleMessageBatch) {
+        return this.changeVisibilityTimeoutBatch(
+          messages,
+          this.visibilityTimeout
+        );
+      } else {
+        return this.changeVisibilityTimeout(message, this.visibilityTimeout);
+      }
+    }, this.heartbeatInterval * 1000);
+  }
+
+  /**
+   * Change the visibility timeout on a message
+   * @param message The message to change the value of
+   * @param timeout The new timeout that should be set
+   */
   private async changeVisibilityTimeout(
     message: Message,
     timeout: number
@@ -330,82 +329,113 @@ export class Consumer extends EventEmitter {
     }
   }
 
-  private emitError(err: Error, message: Message): void {
-    if (err.name === SQSError.name) {
-      this.emit('error', err, message);
-    } else if (err instanceof TimeoutError) {
-      this.emit('timeout_error', err, message);
-    } else {
-      this.emit('processing_error', err, message);
+  /**
+   * Change the visibility timeout on a batch of messages
+   * @param messages The messages to change the value of
+   * @param timeout The new timeout that should be set
+   */
+  private async changeVisibilityTimeoutBatch(
+    messages: Message[],
+    timeout: number
+  ): Promise<ChangeMessageVisibilityBatchCommandOutput> {
+    const params: ChangeMessageVisibilityBatchCommandInput = {
+      QueueUrl: this.queueUrl,
+      Entries: messages.map((message) => ({
+        Id: message.MessageId,
+        ReceiptHandle: message.ReceiptHandle,
+        VisibilityTimeout: timeout
+      }))
+    };
+    try {
+      return await this.sqs.send(
+        new ChangeMessageVisibilityBatchCommand(params)
+      );
+    } catch (err) {
+      this.emit(
+        'error',
+        toSQSError(err, `Error changing visibility timeout: ${err.message}`),
+        messages
+      );
     }
   }
 
-  private poll(): void {
-    if (this.stopped) {
-      this.emit('stopped');
-      return;
+  /**
+   * Trigger the applications handleMessage function
+   * @param message The message that was received from SQS
+   */
+  private async executeHandler(message: Message): Promise<Message> {
+    try {
+      let result;
+
+      if (this.handleMessageTimeout) {
+        const pending = new Promise((_, reject) => {
+          this.handleMessageTimeoutId = setTimeout((): void => {
+            reject(new TimeoutError());
+          }, this.handleMessageTimeout);
+        });
+        result = await Promise.race([this.handleMessage(message), pending]);
+      } else {
+        result = await this.handleMessage(message);
+      }
+
+      return result instanceof Object ? result : message;
+    } catch (err) {
+      err.message =
+        err instanceof TimeoutError
+          ? `Message handler timed out after ${this.handleMessageTimeout}ms: Operation timed out.`
+          : `Unexpected message handler failure: ${err.message}`;
+      throw err;
+    } finally {
+      if (this.handleMessageTimeoutId) {
+        clearTimeout(this.handleMessageTimeoutId);
+      }
     }
+  }
 
-    debug('Polling for messages');
-    const receiveParams: ReceiveMessageCommandInput = {
-      QueueUrl: this.queueUrl,
-      AttributeNames: this.attributeNames,
-      MessageAttributeNames: this.messageAttributeNames,
-      MaxNumberOfMessages: this.batchSize,
-      WaitTimeSeconds: this.waitTimeSeconds,
-      VisibilityTimeout: this.visibilityTimeout
-    };
+  /**
+   * Execute the application's message batch handler
+   * @param messages The messages that should be forwarded from the SQS queue
+   */
+  private async executeBatchHandler(messages: Message[]): Promise<Message[]> {
+    try {
+      const result = await this.handleMessageBatch(messages);
 
-    let currentPollingTimeout = this.pollingWaitTimeMs;
-    this.receiveMessage(receiveParams)
-      .then(this.handleSqsResponse)
-      .catch((err) => {
-        this.emit('error', err);
-        if (isConnectionError(err)) {
-          debug('There was an authentication error. Pausing before retrying.');
-          currentPollingTimeout = this.authenticationErrorTimeout;
-        }
-        return;
-      })
-      .then(() => {
-        setTimeout(this.poll, currentPollingTimeout);
-      })
-      .catch((err) => {
-        this.emit('error', err);
-      });
+      return result instanceof Object ? result : messages;
+    } catch (err) {
+      err.message = `Unexpected message handler failure: ${err.message}`;
+      throw err;
+    }
   }
 
-  private async processMessageBatch(messages: Message[]): Promise<void> {
-    messages.forEach((message) => {
-      this.emit('message_received', message);
-    });
+  /**
+   * Delete a single message from SQS
+   * @param message The message to delete from the SQS queue
+   */
+  private async deleteMessage(message: Message): Promise<void> {
+    if (!this.shouldDeleteMessages) {
+      debug(
+        'Skipping message delete since shouldDeleteMessages is set to false'
+      );
+      return;
+    }
+    debug('Deleting message %s', message.MessageId);
+
+    const deleteParams: DeleteMessageCommandInput = {
+      QueueUrl: this.queueUrl,
+      ReceiptHandle: message.ReceiptHandle
+    };
 
-    let heartbeat;
     try {
-      if (this.heartbeatInterval) {
-        heartbeat = this.startHeartbeat(async () => {
-          return this.changeVisibilityTimeoutBatch(
-            messages,
-            this.visibilityTimeout
-          );
-        });
-      }
-      await this.executeBatchHandler(messages);
-      await this.deleteMessageBatch(messages);
-      messages.forEach((message) => {
-        this.emit('message_processed', message);
-      });
+      await this.sqs.send(new DeleteMessageCommand(deleteParams));
     } catch (err) {
-      this.emit('error', err, messages);
-
-      if (this.terminateVisibilityTimeout) {
-        await this.changeVisibilityTimeoutBatch(messages, 0);
-      }
-    } finally {
-      clearInterval(heartbeat);
+      throw toSQSError(err, `SQS delete message failed: ${err.message}`);
     }
   }
 
+  /**
+   * Delete a batch of messages from the SQS queue.
+   * @param messages The messages that should be deleted from SQS
+   */
   private async deleteMessageBatch(messages: Message[]): Promise<void> {
     if (!this.shouldDeleteMessages) {
       debug(
@@ -432,44 +462,4 @@ export class Consumer extends EventEmitter {
       throw toSQSError(err, `SQS delete message failed: ${err.message}`);
     }
   }
-
-  private async executeBatchHandler(messages: Message[]): Promise<void> {
-    try {
-      await this.handleMessageBatch(messages);
-    } catch (err) {
-      err.message = `Unexpected message handler failure: ${err.message}`;
-      throw err;
-    }
-  }
-
-  private async changeVisibilityTimeoutBatch(
-    messages: Message[],
-    timeout: number
-  ): Promise<ChangeMessageVisibilityBatchCommandOutput> {
-    const params: ChangeMessageVisibilityBatchCommandInput = {
-      QueueUrl: this.queueUrl,
-      Entries: messages.map((message) => ({
-        Id: message.MessageId,
-        ReceiptHandle: message.ReceiptHandle,
-        VisibilityTimeout: timeout
-      }))
-    };
-    try {
-      return await this.sqs.send(
-        new ChangeMessageVisibilityBatchCommand(params)
-      );
-    } catch (err) {
-      this.emit(
-        'error',
-        toSQSError(err, `Error changing visibility timeout: ${err.message}`),
-        messages
-      );
-    }
-  }
-
-  private startHeartbeat(heartbeatFn: () => void): NodeJS.Timeout {
-    return setInterval(() => {
-      heartbeatFn();
-    }, this.heartbeatInterval * 1000);
-  }
 }