rocketmq-client-nodejs-bate 1.0.12

package/src/client/index.ts

@@ -0,0 +1,27 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+export * from './BaseClient';
+export * from './ClientId';
+export * from './Logger';
+export * from './UserAgent';
+export * from './RpcClient';
+export * from './RpcClientManager';
+export * from './SessionCredentials';
+export * from './Settings';
+export * from './TelemetrySession';
+export * from './UserAgent';

package/src/consumer/Assignment.ts

@@ -0,0 +1,39 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { MessageQueue } from '../route';
+
+export class Assignment {
+  readonly messageQueue: MessageQueue;
+
+  constructor(messageQueue: MessageQueue) {
+    this.messageQueue = messageQueue;
+  }
+
+  equals(other: Assignment): boolean {
+    if (this === other) return true;
+    if (!other) return false;
+    return this.messageQueue === other.messageQueue ||
+      (this.messageQueue.queueId === other.messageQueue.queueId &&
+       this.messageQueue.topic.name === other.messageQueue.topic.name &&
+       this.messageQueue.broker.name === other.messageQueue.broker.name);
+  }
+
+  toString(): string {
+    return `Assignment{messageQueue=${JSON.stringify(this.messageQueue)}}`;
+  }
+}

package/src/consumer/Assignments.ts

@@ -0,0 +1,46 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { Assignment } from './Assignment';
+
+export class Assignments {
+  readonly #assignmentList: Assignment[];
+
+  constructor(assignmentList: Assignment[]) {
+    this.#assignmentList = assignmentList;
+  }
+
+  getAssignmentList(): Assignment[] {
+    return this.#assignmentList;
+  }
+
+  equals(other?: Assignments): boolean {
+    if (this === other) return true;
+    if (!other) return false;
+    if (this.#assignmentList.length !== other.#assignmentList.length) return false;
+    for (let i = 0; i < this.#assignmentList.length; i++) {
+      if (!this.#assignmentList[i].equals(other.#assignmentList[i])) {
+        return false;
+      }
+    }
+    return true;
+  }
+
+  toString(): string {
+    return `Assignments{assignmentList=[${this.#assignmentList.map(a => a.toString()).join(', ')}]}`;
+  }
+}

package/src/consumer/ConsumeResult.ts

@@ -0,0 +1,21 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+export enum ConsumeResult {
+  SUCCESS = 'SUCCESS',
+  FAILURE = 'FAILURE',
+}

package/src/consumer/ConsumeService.ts

@@ -0,0 +1,54 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { MessageView } from '../message';
+import { ConsumeResult } from './ConsumeResult';
+import { ConsumeTask } from './ConsumeTask';
+import { MessageListener } from './MessageListener';
+import type { ProcessQueue } from './ProcessQueue';
+
+export abstract class ConsumeService {
+  protected readonly clientId: string;
+  readonly #messageListener: MessageListener;
+  #aborted = false;
+
+  constructor(clientId: string, messageListener: MessageListener) {
+    this.clientId = clientId;
+    this.#messageListener = messageListener;
+  }
+
+  abstract consume(pq: ProcessQueue, messageViews: MessageView[]): void;
+
+  async consumeMessage(messageView: MessageView, delay = 0): Promise<ConsumeResult> {
+    if (this.#aborted) {
+      return ConsumeResult.FAILURE;
+    }
+    const task = new ConsumeTask(this.clientId, this.#messageListener, messageView);
+    if (delay <= 0) {
+      return task.call();
+    }
+    await new Promise<void>(resolve => setTimeout(resolve, delay));
+    if (this.#aborted) {
+      return ConsumeResult.FAILURE;
+    }
+    return task.call();
+  }
+
+  abort() {
+    this.#aborted = true;
+  }
+}

package/src/consumer/ConsumeTask.ts

@@ -0,0 +1,40 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { MessageView } from '../message';
+import { ConsumeResult } from './ConsumeResult';
+import { MessageListener } from './MessageListener';
+
+export class ConsumeTask {
+  readonly #messageListener: MessageListener;
+  readonly #messageView: MessageView;
+
+  constructor(_clientId: string, messageListener: MessageListener, messageView: MessageView) {
+    this.#messageListener = messageListener;
+    this.#messageView = messageView;
+  }
+
+  async call(): Promise<ConsumeResult> {
+    try {
+      const result = await this.#messageListener.consume(this.#messageView);
+      return result;
+    } catch (e) {
+      // Message listener raised an exception while consuming messages
+      return ConsumeResult.FAILURE;
+    }
+  }
+}

package/src/consumer/Consumer.ts

@@ -0,0 +1,129 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { Message, Status } from '../../proto/apache/rocketmq/v2/definition_pb';
+import {
+  AckMessageRequest,
+  ChangeInvisibleDurationRequest,
+  ReceiveMessageRequest, ReceiveMessageResponse,
+} from '../../proto/apache/rocketmq/v2/service_pb';
+import { MessageView } from '../message';
+import { MessageQueue } from '../route';
+import { StatusChecker } from '../exception';
+import { BaseClient, BaseClientOptions } from '../client';
+import { createDuration, createResource } from '../util';
+import { FilterExpression } from './FilterExpression';
+
+export interface ConsumerOptions extends BaseClientOptions {
+  consumerGroup: string;
+}
+
+export abstract class Consumer extends BaseClient {
+  protected readonly consumerGroup: string;
+
+  constructor(options: ConsumerOptions) {
+    super(options);
+    this.consumerGroup = options.consumerGroup;
+  }
+
+  protected wrapReceiveMessageRequest(batchSize: number, mq: MessageQueue,
+    filterExpression: FilterExpression, invisibleDuration: number, longPollingTimeout: number) {
+    return new ReceiveMessageRequest()
+      .setGroup(createResource(this.consumerGroup))
+      .setMessageQueue(mq.toProtobuf())
+      .setFilterExpression(filterExpression.toProtobuf())
+      .setLongPollingTimeout(createDuration(longPollingTimeout))
+      .setBatchSize(batchSize)
+      .setAutoRenew(false)
+      .setInvisibleDuration(createDuration(invisibleDuration));
+  }
+
+  protected async receiveMessage(request: ReceiveMessageRequest, mq: MessageQueue, awaitDuration: number) {
+    const endpoints = mq.broker.endpoints;
+    const timeout = this.requestTimeout + awaitDuration;
+    let status: Status.AsObject | undefined;
+    const responses = await this.rpcClientManager.receiveMessage(endpoints, request, timeout);
+    const messageList: Message[] = [];
+    let transportDeliveryTimestamp: Date | undefined;
+    for (const response of responses) {
+      switch (response.getContentCase()) {
+        case ReceiveMessageResponse.ContentCase.STATUS:
+          status = response.getStatus()?.toObject();
+          break;
+        case ReceiveMessageResponse.ContentCase.MESSAGE:
+          messageList.push(response.getMessage()!);
+          break;
+        case ReceiveMessageResponse.ContentCase.DELIVERY_TIMESTAMP:
+          transportDeliveryTimestamp = response.getDeliveryTimestamp()?.toDate();
+          break;
+        default:
+          // this.logger.warn("[Bug] Not recognized content for receive message response, mq={}, " +
+          //                 "clientId={}, response={}", mq, clientId, response);
+      }
+    }
+    StatusChecker.check(status);
+    const messages = messageList.map(message => new MessageView(message, mq, transportDeliveryTimestamp));
+    return messages;
+  }
+
+  protected async ackMessage(messageView: MessageView) {
+    const endpoints = messageView.endpoints;
+    const request = new AckMessageRequest()
+      .setGroup(createResource(this.consumerGroup))
+      .setTopic(createResource(messageView.topic));
+    request.addEntries()
+      .setMessageId(messageView.messageId)
+      .setReceiptHandle(messageView.receiptHandle);
+    const res = await this.rpcClientManager.ackMessage(endpoints, request, this.requestTimeout);
+    // FIXME: handle fail ack
+    const response = res.toObject();
+    StatusChecker.check(response.status);
+    return response.entriesList;
+  }
+
+  protected async invisibleDuration(messageView: MessageView, invisibleDuration: number) {
+    const request = new ChangeInvisibleDurationRequest()
+      .setGroup(createResource(this.consumerGroup))
+      .setTopic(createResource(messageView.topic))
+      .setReceiptHandle(messageView.receiptHandle)
+      .setInvisibleDuration(createDuration(invisibleDuration))
+      .setMessageId(messageView.messageId);
+
+    const res = await this.rpcClientManager.changeInvisibleDuration(messageView.endpoints, request, this.requestTimeout);
+    const response = res.toObject();
+    StatusChecker.check(response.status);
+    return response.receiptHandle;
+  }
+
+  /**
+   * Expose public methods for ProcessQueue to access RPC operations
+   */
+  async ackMessageViaRpc(endpoints: any, request: AckMessageRequest, timeout: number) {
+    const res = await this.rpcClientManager.ackMessage(endpoints, request, timeout);
+    return res;
+  }
+
+  async changeInvisibleDurationViaRpc(endpoints: any, request: ChangeInvisibleDurationRequest, timeout: number) {
+    const res = await this.rpcClientManager.changeInvisibleDuration(endpoints, request, timeout);
+    return res;
+  }
+
+  async forwardMessageToDeadLetterQueueViaRpc(endpoints: any, request: any, timeout: number) {
+    const res = await this.rpcClientManager.forwardMessageToDeadLetterQueue(endpoints, request, timeout);
+    return res;
+  }
+}

package/src/consumer/FifoConsumeService.ts

@@ -0,0 +1,136 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { MessageView } from '../message';
+import { ConsumeService } from './ConsumeService';
+import { MessageListener } from './MessageListener';
+import type { ProcessQueue } from './ProcessQueue';
+
+export class FifoConsumeService extends ConsumeService {
+  readonly #enableAccelerator: boolean;
+  readonly #groupProcessing = new Map<string /* messageGroup */, Promise<void>>();
+
+  constructor(clientId: string, messageListener: MessageListener, enableAccelerator?: boolean) {
+    super(clientId, messageListener);
+    this.#enableAccelerator = enableAccelerator ?? false;
+  }
+
+  consume(pq: ProcessQueue, messageViews: MessageView[]): void {
+    if (this.#enableAccelerator) {
+      this.#consumeWithAccelerator(pq, messageViews);
+    } else {
+      this.#consumeIteratively(pq, messageViews, 0);
+    }
+  }
+
+  /**
+   * FIFO consume accelerator mode:
+   * - Messages with the same messageGroup are consumed sequentially
+   * - Messages with different messageGroups are consumed in parallel
+   * - Messages without messageGroup are consumed in parallel
+   */
+  #consumeWithAccelerator(pq: ProcessQueue, messageViews: MessageView[]): void {
+    // Group messages by messageGroup
+    const groupedMessages = new Map<string, MessageView[]>();
+    const ungroupedMessages: MessageView[] = [];
+
+    for (const messageView of messageViews) {
+      if (messageView.corrupted) {
+        pq.discardFifoMessage(messageView);
+        continue;
+      }
+
+      const messageGroup = messageView.messageGroup || '';
+      if (messageGroup) {
+        const group = groupedMessages.get(messageGroup) || [];
+        group.push(messageView);
+        groupedMessages.set(messageGroup, group);
+      } else {
+        ungroupedMessages.push(messageView);
+      }
+    }
+
+    // Process grouped messages (each group sequentially, groups in parallel)
+    for (const [ messageGroup, messages ] of groupedMessages.entries()) {
+      this.#processMessageGroup(pq, messageGroup, messages);
+    }
+
+    // Process ungrouped messages in parallel
+    for (const messageView of ungroupedMessages) {
+      this.consumeMessage(messageView)
+        .then(result => pq.eraseFifoMessage(messageView, result))
+        .catch(() => {
+          // Error already logged, continue with next message
+        });
+    }
+  }
+
+  /**
+   * Process messages within the same group sequentially
+   */
+  async #processMessageGroup(pq: ProcessQueue, messageGroup: string, messages: MessageView[]): Promise<void> {
+    // Check if there's already processing happening for this group
+    const existingPromise = this.#groupProcessing.get(messageGroup);
+    const processTask = (async () => {
+      // Wait for previous processing to complete if any
+      if (existingPromise) {
+        await existingPromise.catch(() => {
+          // Ignore previous errors, continue processing
+        });
+      }
+
+      // Process messages in sequence
+      for (const messageView of messages) {
+        try {
+          const result = await this.consumeMessage(messageView);
+          await pq.eraseFifoMessage(messageView, result);
+        } catch (error) {
+          console.error('Failed to process FIFO message, messageGroup=%s, messageId=%s',
+            messageGroup, messageView.messageId, error);
+          // Continue with next message even if current fails
+        }
+      }
+    })();
+
+    // Store the promise for this group
+    this.#groupProcessing.set(messageGroup, processTask);
+
+    // Clean up when done
+    processTask.finally(() => {
+      this.#groupProcessing.delete(messageGroup);
+    });
+  }
+
+  #consumeIteratively(pq: ProcessQueue, messageViews: MessageView[], index: number): void {
+    if (index >= messageViews.length) {
+      return;
+    }
+
+    const messageView = messageViews[index];
+
+    if (messageView.corrupted) {
+      pq.discardFifoMessage(messageView);
+      this.#consumeIteratively(pq, messageViews, index + 1);
+      return;
+    }
+
+    this.consumeMessage(messageView)
+      .then(result => pq.eraseFifoMessage(messageView, result))
+      .then(() => this.#consumeIteratively(pq, messageViews, index + 1))
+      .catch(() => this.#consumeIteratively(pq, messageViews, index + 1));
+  }
+}

package/src/consumer/FilterExpression.ts

@@ -0,0 +1,42 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { FilterType, FilterExpression as FilterExpressionPB } from '../../proto/apache/rocketmq/v2/definition_pb';
+
+const TAG_EXPRESSION_SUB_ALL = '*';
+
+export class FilterExpression {
+  static readonly SUB_ALL = new FilterExpression(TAG_EXPRESSION_SUB_ALL);
+
+  readonly expression: string;
+  readonly filterType: FilterType;
+
+  constructor(expression: string, filterType = FilterType.TAG) {
+    this.expression = expression;
+    this.filterType = filterType;
+  }
+
+  toProtobuf() {
+    return new FilterExpressionPB()
+      .setType(this.filterType)
+      .setExpression(this.expression);
+  }
+
+  toString() {
+    return `FilterExpression(${this.filterType},${this.expression})`;
+  }
+}

package/src/consumer/MessageListener.ts

@@ -0,0 +1,23 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { MessageView } from '../message';
+import { ConsumeResult } from './ConsumeResult';
+
+export interface MessageListener {
+  consume(messageView: MessageView): ConsumeResult | Promise<ConsumeResult>;
+}