pulsar-client 1.14.0 → 1.15.0

package/index.d.ts

@@ -68,6 +68,8 @@ export interface ProducerConfig {
   schema?: SchemaInfo;
   accessMode?: ProducerAccessMode;
   batchingType?: ProducerBatchType;
+  messageRouter?: MessageRouter;
+  batchingMaxAllowedSizeInBytes?: number;
 }
 
 export class Producer {
@@ -176,6 +178,23 @@ export class MessageId {
   toString(): string;
 }
 
+export interface TopicMetadata {
+  numPartitions: number;
+}
+
+/**
+ * @callback MessageRouter
+ * @description When producing messages to a partitioned topic, this router is used to select the
+ * target partition for each message. The router only works when the `messageRoutingMode` is set to
+ * `CustomPartition`. Please note that `getTopicName()` cannot be called on the `message`, otherwise
+ * the behavior will be undefined because the topic is unknown before sending the message.
+ * @param message The message to be routed.
+ * @param topicMetadata Metadata for the partitioned topic the message is being routed to.
+ * @returns {number} The index of the target partition (must be a number between 0 and
+ * topicMetadata.numPartitions - 1).
+ */
+export type MessageRouter = (message: Message, topicMetadata: TopicMetadata) => number;
+
 export interface SchemaInfo {
   schemaType: SchemaType;
   name?: string;

package/package.json

@@ -1,9 +1,12 @@
 {
   "name": "pulsar-client",
-  "version": "1.14.0",
+  "version": "1.15.0",
   "description": "Pulsar Node.js client",
   "main": "index.js",
   "types": "index.d.ts",
+  "resolutions": {
+    "minimatch": "^9.0.0"
+  },
   "directories": {
     "src": "src",
     "example": "examples"

package/src/Producer.cc

@@ -73,6 +73,7 @@ Napi::Value Producer::NewInstance(const Napi::CallbackInfo &info, std::shared_pt
         auto instanceContext = static_cast<ProducerNewInstanceContext *>(ctx);
         auto deferred = instanceContext->deferred;
         auto cClient = instanceContext->cClient;
+        auto producerConfig = instanceContext->producerConfig;
         delete instanceContext;
 
         if (result != pulsar_result_Ok) {
@@ -81,10 +82,11 @@ Napi::Value Producer::NewInstance(const Napi::CallbackInfo &info, std::shared_pt
 
         std::shared_ptr<pulsar_producer_t> cProducer(rawProducer, pulsar_producer_free);
 
-        deferred->Resolve([cProducer](const Napi::Env env) {
+        deferred->Resolve([cProducer, producerConfig](const Napi::Env env) {
           Napi::Object obj = Producer::constructor.New({});
           Producer *producer = Producer::Unwrap(obj);
           producer->SetCProducer(cProducer);
+          producer->producerConfig = producerConfig;
           return obj;
         });
       },

package/src/Producer.h

@@ -23,6 +23,8 @@
 #include <napi.h>
 #include <pulsar/c/client.h>
 #include <pulsar/c/producer.h>
+#include <memory>
+#include "ProducerConfig.h"
 
 class Producer : public Napi::ObjectWrap<Producer> {
  public:
@@ -35,6 +37,9 @@ class Producer : public Napi::ObjectWrap<Producer> {
 
  private:
   std::shared_ptr<pulsar_producer_t> cProducer;
+  // Extend the lifetime of the producer config since it's env and router function could be used when sending
+  // messages
+  std::shared_ptr<ProducerConfig> producerConfig;
   Napi::Value Send(const Napi::CallbackInfo &info);
   Napi::Value Flush(const Napi::CallbackInfo &info);
   Napi::Value Close(const Napi::CallbackInfo &info);

package/src/ProducerConfig.cc

@@ -18,8 +18,14 @@
  */
 #include "SchemaInfo.h"
 #include "ProducerConfig.h"
+#include "Message.h"
+#include <cstdio>
 #include <map>
+#include "napi-inl.h"
+#include "napi.h"
 #include "pulsar/ProducerConfiguration.h"
+#include "pulsar/c/message.h"
+#include "pulsar/c/message_router.h"
 
 static const std::string CFG_TOPIC = "topic";
 static const std::string CFG_PRODUCER_NAME = "producerName";
@@ -34,6 +40,7 @@ static const std::string CFG_COMPRESS_TYPE = "compressionType";
 static const std::string CFG_BATCH_ENABLED = "batchingEnabled";
 static const std::string CFG_BATCH_MAX_DELAY = "batchingMaxPublishDelayMs";
 static const std::string CFG_BATCH_MAX_MSG = "batchingMaxMessages";
+static const std::string CFG_BATCH_MAX_ALLOWED_SIZE_IN_BYTES = "batchingMaxAllowedSizeInBytes";
 static const std::string CFG_SCHEMA = "schema";
 static const std::string CFG_PROPS = "properties";
 static const std::string CFG_PUBLIC_KEY_PATH = "publicKeyPath";
@@ -42,6 +49,7 @@ static const std::string CFG_CRYPTO_FAILURE_ACTION = "cryptoFailureAction";
 static const std::string CFG_CHUNK_ENABLED = "chunkingEnabled";
 static const std::string CFG_ACCESS_MODE = "accessMode";
 static const std::string CFG_BATCHING_TYPE = "batchingType";
+static const std::string CFG_MESSAGE_ROUTER = "messageRouter";
 
 struct _pulsar_producer_configuration {
   pulsar::ProducerConfiguration conf;
@@ -82,6 +90,25 @@ static std::map<std::string, pulsar::ProducerConfiguration::BatchingType> PRODUC
     {"KeyBasedBatching", pulsar::ProducerConfiguration::KeyBasedBatching},
 };
 
+static int choosePartition(pulsar_message_t* msg, pulsar_topic_metadata_t* metadata, void* ctx) {
+  auto router = static_cast<Napi::FunctionReference*>(ctx);
+  const auto& env = router->Env();
+  auto jsMessage = Message::NewInstance(Napi::Object::New(env),
+                                        std::shared_ptr<pulsar_message_t>(msg, [](pulsar_message_t*) {}));
+  int numPartitions = pulsar_topic_metadata_get_num_partitions(metadata);
+
+  Napi::Object jsTopicMetadata = Napi::Object::New(env);
+  jsTopicMetadata.Set("numPartitions", Napi::Number::New(env, numPartitions));
+
+  try {
+    return router->Call({jsMessage, jsTopicMetadata}).ToNumber().Int32Value();
+  } catch (const Napi::Error& e) {
+    // TODO: how to handle the error properly? For now, return an invalid partition to fail the send
+    fprintf(stderr, "Error when calling messageRouter: %s\n", e.what());
+    return numPartitions;
+  }
+}
+
 ProducerConfig::ProducerConfig(const Napi::Object& producerConfig) : topic("") {
   this->cProducerConfig = std::shared_ptr<pulsar_producer_configuration_t>(
       pulsar_producer_configuration_create(), pulsar_producer_configuration_free);
@@ -131,8 +158,10 @@ ProducerConfig::ProducerConfig(const Napi::Object& producerConfig) : topic("") {
     pulsar_producer_configuration_set_block_if_queue_full(this->cProducerConfig.get(), blockIfQueueFull);
   }
 
+  bool useCustomPartition = false;
   if (producerConfig.Has(CFG_ROUTING_MODE) && producerConfig.Get(CFG_ROUTING_MODE).IsString()) {
     std::string messageRoutingMode = producerConfig.Get(CFG_ROUTING_MODE).ToString().Utf8Value();
+    useCustomPartition = (messageRoutingMode == "CustomPartition");
     if (MESSAGE_ROUTING_MODE.count(messageRoutingMode))
       pulsar_producer_configuration_set_partitions_routing_mode(this->cProducerConfig.get(),
                                                                 MESSAGE_ROUTING_MODE.at(messageRoutingMode));
@@ -173,6 +202,16 @@ ProducerConfig::ProducerConfig(const Napi::Object& producerConfig) : topic("") {
     }
   }
 
+  if (producerConfig.Has(CFG_BATCH_MAX_ALLOWED_SIZE_IN_BYTES) &&
+      producerConfig.Get(CFG_BATCH_MAX_ALLOWED_SIZE_IN_BYTES).IsNumber()) {
+    int64_t batchingMaxAllowedSizeInBytes =
+        producerConfig.Get(CFG_BATCH_MAX_ALLOWED_SIZE_IN_BYTES).ToNumber().Int64Value();
+    if (batchingMaxAllowedSizeInBytes > 0) {
+      pulsar_producer_configuration_set_batching_max_allowed_size_in_bytes(
+          this->cProducerConfig.get(), (unsigned long)batchingMaxAllowedSizeInBytes);
+    }
+  }
+
   if (producerConfig.Has(CFG_SCHEMA) && producerConfig.Get(CFG_SCHEMA).IsObject()) {
     SchemaInfo* schemaInfo = new SchemaInfo(producerConfig.Get(CFG_SCHEMA).ToObject());
     schemaInfo->SetProducerSchema(this->cProducerConfig);
@@ -224,6 +263,15 @@ ProducerConfig::ProducerConfig(const Napi::Object& producerConfig) : topic("") {
   if (PRODUCER_BATCHING_TYPE.count(batchingType)) {
     this->cProducerConfig.get()->conf.setBatchingType(PRODUCER_BATCHING_TYPE.at(batchingType));
   }
+
+  if (useCustomPartition && producerConfig.Has(CFG_MESSAGE_ROUTER)) {
+    auto value = producerConfig.Get(CFG_MESSAGE_ROUTER);
+    if (value.IsFunction()) {
+      messageRouter = Napi::Persistent(value.As<Napi::Function>());
+      pulsar_producer_configuration_set_message_router(this->cProducerConfig.get(), choosePartition,
+                                                       &messageRouter);
+    }
+  }
 }
 
 ProducerConfig::~ProducerConfig() {}

package/src/ProducerConfig.h

@@ -22,6 +22,11 @@
 
 #include <napi.h>
 #include <pulsar/c/producer_configuration.h>
+#include <memory>
+
+struct MessageRouterContext {
+  Napi::FunctionReference messageRouter;
+};
 
 class ProducerConfig {
  public:
@@ -33,6 +38,8 @@ class ProducerConfig {
  private:
   std::shared_ptr<pulsar_producer_configuration_t> cProducerConfig;
   std::string topic;
+  std::unique_ptr<MessageRouterContext> routerContext;
+  Napi::FunctionReference messageRouter;
 };
 
 #endif