pulsar-client 1.13.1 → 1.13.2-rc.1

package/index.d.ts

@@ -33,6 +33,7 @@ export interface ClientConfig {
   listenerName?: string;
   log?: (level: LogLevel, file: string, line: number, message: string) => void;
   logLevel?: LogLevel;
+  connectionTimeoutMs?: number;
 }
 
 export class Client {
@@ -66,6 +67,7 @@ export interface ProducerConfig {
   chunkingEnabled?: boolean;
   schema?: SchemaInfo;
   accessMode?: ProducerAccessMode;
+  batchingType?: ProducerBatchType;
 }
 
 export class Producer {
@@ -162,6 +164,7 @@ export class Message {
   getEventTimestamp(): number;
   getRedeliveryCount(): number;
   getPartitionKey(): string;
+  getOrderingKey(): string;
 }
 
 export class MessageId {
@@ -270,6 +273,10 @@ export type CompressionType =
   'ZSTD' |
   'SNAPPY';
 
+export type ProducerBatchType =
+    'DefaultBatching' |
+    'KeyBasedBatching';
+
 export type ProducerCryptoFailureAction =
   'FAIL' |
   'SEND';

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "pulsar-client",
-  "version": "1.13.1",
+  "version": "1.13.2-rc.1",
   "description": "Pulsar Node.js client",
   "main": "index.js",
   "types": "index.d.ts",
@@ -35,6 +35,7 @@
     "node": ">=12.3.0"
   },
   "devDependencies": {
+    "@definitelytyped/utils": "^0.0.168",
     "@seadub/clang-format-lint": "0.0.2",
     "@types/node": "^14.14.25",
     "clang-format": "^1.4.0",

package/pkg/linux/download-cpp-client.sh

@@ -23,14 +23,14 @@ set -e -x
 ROOT_DIR=`cd $(dirname $0) && cd ../../ && pwd`
 source $ROOT_DIR/pulsar-client-cpp.txt
 
-if [ $USER != "root" ]; then
+if [ "$USER" != "root" -a "$USER" != "" ]; then
   SUDO="sudo"
 fi
 
 # Get the flavor of Linux
 export $(cat /etc/*-release | grep "^ID=")
 UNAME_ARCH=$(uname -m)
-if [ $UNAME_ARCH == 'aarch64' ]; then
+if [ "$UNAME_ARCH" == 'aarch64' ]; then
   PLATFORM=arm64
 else
   PLATFORM=x86_64
@@ -42,18 +42,18 @@ rm -rf $ROOT_DIR/pkg/linux/tmp
 mkdir $ROOT_DIR/pkg/linux/tmp
 cd $ROOT_DIR/pkg/linux/tmp
 
-if [ $ID == 'ubuntu' -o $ID == 'debian' ]; then
+if [ "$ID" == 'ubuntu' -o "$ID" == 'debian' ]; then
   curl -L -O ${CPP_CLIENT_BASE_URL}/deb-${PLATFORM}/apache-pulsar-client-dev.deb
   $SUDO ar x apache-pulsar-client-dev.deb
   $SUDO tar -xvf data.tar.xz
   cp -r usr/* $ROOT_DIR/pkg/linux/pulsar-cpp/
 
-elif [ $ID == 'alpine' ]; then
+elif [ "$ID" == 'alpine' ]; then
   curl -L -O ${CPP_CLIENT_BASE_URL}/apk-${PLATFORM}/${UNAME_ARCH}/apache-pulsar-client-dev-${CPP_CLIENT_VERSION}-r0.apk
   $SUDO tar -xvf apache-pulsar-client-dev-${CPP_CLIENT_VERSION}-r0.apk
   cp -r usr/* $ROOT_DIR/pkg/linux/pulsar-cpp/
 
-elif [ $ID == '"centos"' -o $ID == '"rocky"' ]; then
+elif [ "$ID" == '"centos"' -o "$ID" == '"rocky"' ]; then
   curl -L -O ${CPP_CLIENT_BASE_URL}/rpm-${PLATFORM}/${UNAME_ARCH}/apache-pulsar-client-devel-${CPP_CLIENT_VERSION}-1.${UNAME_ARCH}.rpm
   $SUDO rpm -i --prefix=$ROOT_DIR/pkg/linux/pulsar-cpp apache-pulsar-client-devel-${CPP_CLIENT_VERSION}-1.${UNAME_ARCH}.rpm --nodeps --force
 

package/src/Client.cc

@@ -26,6 +26,7 @@
 #include <pulsar/c/client.h>
 #include <pulsar/c/client_configuration.h>
 #include <pulsar/c/result.h>
+#include "pulsar/ClientConfiguration.h"
 
 static const std::string CFG_SERVICE_URL = "serviceUrl";
 static const std::string CFG_AUTH = "authentication";
@@ -42,9 +43,14 @@ static const std::string CFG_STATS_INTERVAL = "statsIntervalInSeconds";
 static const std::string CFG_LOG = "log";
 static const std::string CFG_LOG_LEVEL = "logLevel";
 static const std::string CFG_LISTENER_NAME = "listenerName";
+static const std::string CFG_CONNECTION_TIMEOUT = "connectionTimeoutMs";
 
 LogCallback *Client::logCallback = nullptr;
 
+struct _pulsar_client_configuration {
+  pulsar::ClientConfiguration conf;
+};
+
 void Client::SetLogHandler(const Napi::CallbackInfo &info) {
   Napi::Env env = info.Env();
   Napi::HandleScope scope(env);
@@ -157,6 +163,13 @@ Client::Client(const Napi::CallbackInfo &info) : Napi::ObjectWrap<Client>(info)
     }
   }
 
+  if (clientConfig.Has(CFG_CONNECTION_TIMEOUT) && clientConfig.Get(CFG_CONNECTION_TIMEOUT).IsNumber()) {
+    int32_t connectionTimeoutMs = clientConfig.Get(CFG_CONNECTION_TIMEOUT).ToNumber().Int32Value();
+    if (connectionTimeoutMs > 0) {
+      cClientConfig.get()->conf.setConnectionTimeout(connectionTimeoutMs);
+    }
+  }
+
   if (clientConfig.Has(CFG_LISTENER_THREADS) && clientConfig.Get(CFG_LISTENER_THREADS).IsNumber()) {
     int32_t messageListenerThreads = clientConfig.Get(CFG_LISTENER_THREADS).ToNumber().Int32Value();
     if (messageListenerThreads > 0) {

package/src/Message.cc

@@ -45,7 +45,8 @@ Napi::Object Message::Init(Napi::Env env, Napi::Object exports) {
        InstanceMethod("getPublishTimestamp", &Message::GetPublishTimestamp),
        InstanceMethod("getEventTimestamp", &Message::GetEventTimestamp),
        InstanceMethod("getRedeliveryCount", &Message::GetRedeliveryCount),
-       InstanceMethod("getPartitionKey", &Message::GetPartitionKey)});
+       InstanceMethod("getPartitionKey", &Message::GetPartitionKey),
+       InstanceMethod("getOrderingKey", &Message::GetOrderingKey)});
 
   constructor = Napi::Persistent(func);
   constructor.SuppressDestruct();
@@ -138,6 +139,14 @@ Napi::Value Message::GetPartitionKey(const Napi::CallbackInfo &info) {
   return Napi::String::New(env, pulsar_message_get_partitionKey(this->cMessage.get()));
 }
 
+Napi::Value Message::GetOrderingKey(const Napi::CallbackInfo &info) {
+  Napi::Env env = info.Env();
+  if (!ValidateCMessage(env)) {
+    return env.Null();
+  }
+  return Napi::String::New(env, pulsar_message_get_orderingKey(this->cMessage.get()));
+}
+
 bool Message::ValidateCMessage(Napi::Env env) {
   if (this->cMessage.get()) {
     return true;

package/src/Message.h

@@ -44,6 +44,7 @@ class Message : public Napi::ObjectWrap<Message> {
   Napi::Value GetPublishTimestamp(const Napi::CallbackInfo &info);
   Napi::Value GetEventTimestamp(const Napi::CallbackInfo &info);
   Napi::Value GetPartitionKey(const Napi::CallbackInfo &info);
+  Napi::Value GetOrderingKey(const Napi::CallbackInfo &info);
   Napi::Value GetRedeliveryCount(const Napi::CallbackInfo &info);
   bool ValidateCMessage(Napi::Env env);
 

package/src/ProducerConfig.cc

@@ -19,6 +19,7 @@
 #include "SchemaInfo.h"
 #include "ProducerConfig.h"
 #include <map>
+#include "pulsar/ProducerConfiguration.h"
 
 static const std::string CFG_TOPIC = "topic";
 static const std::string CFG_PRODUCER_NAME = "producerName";
@@ -40,6 +41,11 @@ static const std::string CFG_ENCRYPTION_KEY = "encryptionKey";
 static const std::string CFG_CRYPTO_FAILURE_ACTION = "cryptoFailureAction";
 static const std::string CFG_CHUNK_ENABLED = "chunkingEnabled";
 static const std::string CFG_ACCESS_MODE = "accessMode";
+static const std::string CFG_BATCHING_TYPE = "batchingType";
+
+struct _pulsar_producer_configuration {
+  pulsar::ProducerConfiguration conf;
+};
 
 static const std::map<std::string, pulsar_partitions_routing_mode> MESSAGE_ROUTING_MODE = {
     {"UseSinglePartition", pulsar_UseSinglePartition},
@@ -71,6 +77,11 @@ static std::map<std::string, pulsar_producer_access_mode> PRODUCER_ACCESS_MODE =
     {"ExclusiveWithFencing", pulsar_ProducerAccessModeExclusiveWithFencing},
 };
 
+static std::map<std::string, pulsar::ProducerConfiguration::BatchingType> PRODUCER_BATCHING_TYPE = {
+    {"DefaultBatching", pulsar::ProducerConfiguration::DefaultBatching},
+    {"KeyBasedBatching", pulsar::ProducerConfiguration::KeyBasedBatching},
+};
+
 ProducerConfig::ProducerConfig(const Napi::Object& producerConfig) : topic("") {
   this->cProducerConfig = std::shared_ptr<pulsar_producer_configuration_t>(
       pulsar_producer_configuration_create(), pulsar_producer_configuration_free);
@@ -208,6 +219,11 @@ ProducerConfig::ProducerConfig(const Napi::Object& producerConfig) : topic("") {
     pulsar_producer_configuration_set_access_mode(this->cProducerConfig.get(),
                                                   PRODUCER_ACCESS_MODE.at(accessMode));
   }
+
+  std::string batchingType = producerConfig.Get(CFG_BATCHING_TYPE).ToString().Utf8Value();
+  if (PRODUCER_BATCHING_TYPE.count(batchingType)) {
+    this->cProducerConfig.get()->conf.setBatchingType(PRODUCER_BATCHING_TYPE.at(batchingType));
+  }
 }
 
 ProducerConfig::~ProducerConfig() {}

package/src/ReaderConfig.cc

@@ -72,7 +72,8 @@ ReaderConfig::ReaderConfig(const Napi::Object &readerConfig, pulsar_reader_liste
     std::string subscriptionRolePrefix =
         readerConfig.Get(CFG_SUBSCRIPTION_ROLE_PREFIX).ToString().Utf8Value();
     if (!subscriptionRolePrefix.empty())
-      pulsar_reader_configuration_set_reader_name(this->cReaderConfig.get(), subscriptionRolePrefix.c_str());
+      pulsar_reader_configuration_set_subscription_role_prefix(this->cReaderConfig.get(),
+                                                               subscriptionRolePrefix.c_str());
   }
 
   if (readerConfig.Has(CFG_READ_COMPACTED) && readerConfig.Get(CFG_READ_COMPACTED).IsBoolean()) {

package/tests/end_to_end.test.js

@@ -32,6 +32,7 @@ const Pulsar = require('../index');
         serviceUrl,
         tlsTrustCertsFilePath: `${__dirname}/certificate/server.crt`,
         operationTimeoutSeconds: 30,
+        connectionTimeoutMs: 20000,
         listenerName,
       });
 
@@ -1329,4 +1330,118 @@ const Pulsar = require('../index');
       await client.close();
     });
   });
+  describe('KeyBasedBatchingTest', () => {
+    let client;
+    let producer;
+    let consumer;
+    let topicName;
+
+    beforeAll(async () => {
+      client = new Pulsar.Client({
+        serviceUrl: 'pulsar://localhost:6650',
+      });
+    });
+
+    afterAll(async () => {
+      await client.close();
+    });
+
+    beforeEach(async () => {
+      topicName = `KeyBasedBatchingTest-${Date.now()}`;
+    });
+
+    afterEach(async () => {
+      if (producer) await producer.close();
+      if (consumer) await consumer.close();
+    });
+
+    const initProducer = async (maxMessages) => {
+      producer = await client.createProducer({
+        topic: topicName,
+        batchingEnabled: true,
+        batchingMaxMessages: maxMessages,
+        batchingType: 'KeyBasedBatching',
+        batchingMaxPublishDelayMs: 3600 * 1000,
+      });
+    };
+
+    const initConsumer = async () => {
+      consumer = await client.subscribe({
+        topic: topicName,
+        subscription: 'SubscriptionName',
+        subscriptionType: 'Exclusive',
+      });
+    };
+
+    const receiveAndAck = async () => {
+      const msg = await consumer.receive();
+      await consumer.acknowledge(msg);
+      return msg;
+    };
+
+    test('testSequenceId', async () => {
+      await initProducer(6);
+      await initConsumer();
+
+      // 0. Send 6 messages, use different keys and order
+      producer.send({ data: Buffer.from('0'), partitionKey: 'A' });
+      producer.send({ data: Buffer.from('1'), partitionKey: 'B' });
+      producer.send({ data: Buffer.from('2'), partitionKey: 'C' });
+      producer.send({ data: Buffer.from('3'), partitionKey: 'B' });
+      producer.send({ data: Buffer.from('4'), partitionKey: 'C' });
+      producer.send({ data: Buffer.from('5'), partitionKey: 'A' });
+      await producer.flush();
+
+      // 1. Receive all messages
+      const received = [];
+      for (let i = 0; i < 6; i += 1) {
+        const msg = await receiveAndAck();
+        received.push({
+          key: msg.getPartitionKey().toString(),
+          value: msg.getData().toString(),
+        });
+      }
+
+      // 2. Verify message order (based on key dictionary order)
+      const expected = [
+        { key: 'B', value: '1' },
+        { key: 'B', value: '3' },
+        { key: 'C', value: '2' },
+        { key: 'C', value: '4' },
+        { key: 'A', value: '0' },
+        { key: 'A', value: '5' },
+      ];
+
+      expect(received).toEqual(expected);
+    });
+
+    test('testOrderingKeyPriority', async () => {
+      await initProducer(3);
+      await initConsumer();
+
+      // 1. Send 3 messages to verify orderingKey takes precedence over partitionKey
+      producer.send({
+        data: Buffer.from('0'),
+        orderingKey: 'A',
+        partitionKey: 'B',
+      });
+      producer.send({ data: Buffer.from('2'), orderingKey: 'B' });
+      producer.send({ data: Buffer.from('1'), orderingKey: 'A' });
+      await producer.flush();
+
+      // 2. Receive messages and verify their order and keys
+      const msg1 = await receiveAndAck();
+      expect(msg1.getData().toString()).toBe('2');
+      expect(msg1.getOrderingKey().toString()).toBe('B');
+
+      const msg2 = await receiveAndAck();
+      expect(msg2.getData().toString()).toBe('0');
+      expect(msg2.getOrderingKey()).toBe('A');
+      expect(msg2.getPartitionKey()).toBe('B');
+
+      const msg3 = await receiveAndAck();
+      expect(msg3.getData().toString()).toBe('1');
+      expect(msg3.getOrderingKey().toString()).toBe('A');
+    });
+  });
 })();