pulsar-client 1.6.2 → 1.8.0-rc.1

package/src/ThreadSafeDeferred.cc

@@ -0,0 +1,98 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+/**
+ * MIT License
+ *
+ * Copyright (c) 2020 Michael K
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ *
+ *
+ * Adapted from https://github.com/0815fox/node-napi-threadsafe-deferred/tree/master/src
+ */
+
+#include "ThreadSafeDeferred.h"
+
+std::shared_ptr<ThreadSafeDeferred> ThreadSafeDeferred::New(const Napi::Env env) {
+  auto deferred = std::make_shared<ThreadSafeDeferred>(env);
+  deferred->self = deferred;
+  return deferred;
+}
+
+ThreadSafeDeferred::ThreadSafeDeferred(const Napi::Env env)
+    : Deferred(env),
+      fate(EFate::UNRESOLVED),
+      createValueCb(NULL),
+      errorMsg(""),
+      tsf{Napi::ThreadSafeFunction::New(env, Napi::Function::New(env, [](const Napi::CallbackInfo &info) {}),
+                                        "ThreadSafeDeferred", 0, 1, [this](Napi::Env env) {
+                                          // this access happens from another thread.
+                                          // However, no synchronization is needed as
+                                          // the other thread cannot modify this instance
+                                          // anymore after calling Resolve or Reject.
+                                          if (this->fate == EFate::RESOLVED) {
+                                            if (this->createValueCb == NULL) {
+                                              Napi::Promise::Deferred::Resolve(env.Undefined());
+                                            } else {
+                                              Napi::Promise::Deferred::Resolve(this->createValueCb(env));
+                                            }
+                                          } else {
+                                            Napi::Promise::Deferred::Reject(
+                                                Napi::Error::New(env, this->errorMsg).Value());
+                                          }
+
+                                          this->self.reset();
+                                        })} {}
+
+void ThreadSafeDeferred::Resolve() {
+  if (this->fate != EFate::UNRESOLVED) throw "Cannot resolve a promise which is not unresolved anymore.";
+  this->fate = EFate::RESOLVED;
+  this->tsf.Release();
+}
+
+void ThreadSafeDeferred::Resolve(const createValueCb_t createValueCb) {
+  if (this->fate != EFate::UNRESOLVED) throw "Cannot resolve a promise which is not unresolved anymore.";
+  this->createValueCb = createValueCb;
+  this->fate = EFate::RESOLVED;
+  this->tsf.Release();
+}
+
+void ThreadSafeDeferred::Reject(const std::string &errorMsg) {
+  if (this->fate != EFate::UNRESOLVED) throw "Cannot reject a promise which is not unresolved anymore.";
+  this->errorMsg = errorMsg;
+  this->fate = EFate::REJECTED;
+  this->tsf.Release();
+}

package/src/ThreadSafeDeferred.h

@@ -0,0 +1,85 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+/**
+ * MIT License
+ *
+ * Copyright (c) 2020 Michael K
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+#ifndef __THREADSAFE_DEFERRED_HPP
+#define __THREADSAFE_DEFERRED_HPP
+
+#include <napi.h>
+#include <functional>
+
+#define THREADSAFE_DEFERRED_RESOLVER(result) [=](const Napi::Env env) { return result; }
+
+typedef std::function<Napi::Value(const Napi::Env env)> createValueCb_t;
+
+class ThreadSafeDeferred : public Napi::Promise::Deferred {
+ private:
+  enum class EFate
+  {
+    UNRESOLVED,
+    RESOLVED,
+    REJECTED
+  };
+  EFate fate;
+  createValueCb_t createValueCb;
+  std::string errorMsg;
+  Napi::ThreadSafeFunction tsf;
+  std::shared_ptr<ThreadSafeDeferred> self;
+
+ public:
+  ThreadSafeDeferred(const Napi::Env env);
+
+  void Resolve();  // <- if only Resolve were virtual... But we can live without polymorphism here
+  void Resolve(const createValueCb_t);
+  inline void Reject() { this->Reject(""); }
+  void Reject(
+      const std::string &);  // <- if only Reject were virtual... But we can live without polymorphism here
+
+  static std::shared_ptr<ThreadSafeDeferred> New(const Napi::Env env);
+};
+
+struct ExtDeferredContext {
+  ExtDeferredContext(std::shared_ptr<ThreadSafeDeferred> deferred) : deferred(deferred){};
+  std::shared_ptr<ThreadSafeDeferred> deferred;
+};
+
+#endif /* __THREADSAFE_DEFERRED_HPP */

package/src/pulsar-binding.js

@@ -0,0 +1,26 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+const path = require('path');
+const binary = require('@mapbox/node-pre-gyp');
+
+const bindingPath = binary.find(path.resolve(path.join(__dirname, '../package.json')));
+// eslint-disable-next-line import/no-dynamic-require
+const binding = require(bindingPath);
+// eslint-disable-next-line no-multi-assign
+module.exports = exports = binding;

package/tests/conf/standalone.conf

@@ -275,3 +275,9 @@ keepAliveIntervalSeconds=30
 
 # How often broker checks for inactive topics to be deleted (topics with no subscriptions and no one connected)
 brokerServicePurgeInactiveFrequencyInSeconds=60
+
+### --- BookKeeper Configuration --- #####
+
+# The maximum netty frame size in bytes. Any message received larger than this will be rejected. The default value is 5MB.
+nettyMaxFrameSizeBytes=5253120
+

package/tests/consumer.test.js

@@ -102,7 +102,7 @@ const Pulsar = require('../index.js');
           subscription: 'sub1',
           ackTimeoutMs: 10000,
           nAckRedeliverTimeoutMs: 60000,
-        })).rejects.toThrow('Failed to create consumer: ConnectError');
+        })).rejects.toThrow('Failed to create consumer: BrokerMetadataError');
       });
 
       test('Not Exist Namespace', async () => {
@@ -111,7 +111,7 @@ const Pulsar = require('../index.js');
           subscription: 'sub1',
           ackTimeoutMs: 10000,
           nAckRedeliverTimeoutMs: 60000,
-        })).rejects.toThrow('Failed to create consumer: ConnectError');
+        })).rejects.toThrow('Failed to create consumer: BrokerMetadataError');
       });
 
       test('Not Positive NAckRedeliverTimeout', async () => {

package/tests/docker-load-test.sh

@@ -0,0 +1,35 @@
+#!/bin/bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+# NOTE: This script should only run inside a Node.js docker container.
+# See ./load-test.sh for details.
+set -ex
+
+# Create an empty directory to test
+mkdir -p /app && cd /app
+tar zxf /pulsar-client-node/pulsar-client-node.tar.gz
+
+# Use the existing Pulsar.node built in a specific container
+mkdir -p lib/binding
+cp /pulsar-client-node/build/Release/pulsar.node lib/binding/
+npm install
+
+# Test if Pulsar.node can be loaded
+node pkg/load_test.js

package/tests/end_to_end.test.js

@@ -106,7 +106,7 @@ const Pulsar = require('../index.js');
       consumer.acknowledge(msg2);
 
       await expect(consumer.receive(1000)).rejects.toThrow(
-        'Failed to received message TimeOut',
+        'Failed to receive message: TimeOut',
       );
 
       expect(results).toEqual([message, message]);
@@ -302,7 +302,7 @@ const Pulsar = require('../index.js');
       }
 
       await expect(consumer.receive(1000)).rejects.toThrow(
-        'Failed to received message TimeOut',
+        'Failed to receive message: TimeOut',
       );
 
       await producer.close();
@@ -847,5 +847,217 @@ const Pulsar = require('../index.js');
 
       await client.close();
     });
+
+    test('Consumer seek by message Id', async () => {
+      const client = new Pulsar.Client({
+        serviceUrl: 'pulsar://localhost:6650',
+        operationTimeoutSeconds: 30,
+      });
+
+      const topic = 'persistent://public/default/seek-by-msgid';
+      const producer = await client.createProducer({
+        topic,
+        sendTimeoutMs: 30000,
+        batchingEnabled: false,
+      });
+      expect(producer).not.toBeNull();
+
+      const msgIds = [];
+      for (let i = 0; i < 10; i += 1) {
+        const msg = `my-message-${i}`;
+        console.log(msg);
+        const msgId = await producer.send({
+          data: Buffer.from(msg),
+        });
+        msgIds.push(msgId);
+      }
+
+      const consumer = await client.subscribe({
+        topic,
+        subscription: 'sub',
+      });
+      expect(consumer).not.toBeNull();
+
+      await consumer.seek(msgIds[5]);
+      const msg = consumer.receive(1000);
+      console.log((await msg).getMessageId().toString());
+      expect((await msg).getData().toString()).toBe('my-message-6');
+
+      await producer.close();
+      await consumer.close();
+      await client.close();
+    });
+
+    test('Consumer seek by timestamp', async () => {
+      const client = new Pulsar.Client({
+        serviceUrl: 'pulsar://localhost:6650',
+        operationTimeoutSeconds: 30,
+      });
+
+      const topic = 'persistent://public/default/seek-by-timestamp';
+      const producer = await client.createProducer({
+        topic,
+        sendTimeoutMs: 30000,
+        batchingEnabled: false,
+      });
+      expect(producer).not.toBeNull();
+
+      for (let i = 0; i < 10; i += 1) {
+        const msg = `my-message-${i}`;
+        console.log(msg);
+        await producer.send({
+          data: Buffer.from(msg),
+        });
+      }
+
+      const consumer = await client.subscribe({
+        topic,
+        subscription: 'sub',
+      });
+      expect(consumer).not.toBeNull();
+
+      const currentTime = Date.now();
+      console.log(currentTime);
+
+      await consumer.seekTimestamp(currentTime);
+
+      console.log('End seek');
+
+      await expect(consumer.receive(1000)).rejects.toThrow('Failed to receive message: TimeOut');
+
+      await consumer.seekTimestamp(currentTime - 100000);
+
+      const msg = consumer.receive(1000);
+      console.log((await msg).getMessageId().toString());
+      expect((await msg).getData().toString()).toBe('my-message-0');
+
+      await producer.close();
+      await consumer.close();
+      await client.close();
+    });
+
+    test('Reader seek by message Id', async () => {
+      const client = new Pulsar.Client({
+        serviceUrl: 'pulsar://localhost:6650',
+        operationTimeoutSeconds: 30,
+      });
+
+      const topic = 'persistent://public/default/reader-seek-by-msgid';
+      const producer = await client.createProducer({
+        topic,
+        sendTimeoutMs: 30000,
+        batchingEnabled: false,
+      });
+      expect(producer).not.toBeNull();
+
+      const msgIds = [];
+      for (let i = 0; i < 10; i += 1) {
+        const msg = `my-message-${i}`;
+        console.log(msg);
+        const msgId = await producer.send({
+          data: Buffer.from(msg),
+        });
+        msgIds.push(msgId);
+      }
+
+      const reader = await client.createReader({
+        topic,
+        startMessageId: Pulsar.MessageId.latest(),
+      });
+      expect(reader).not.toBeNull();
+
+      await reader.seek(msgIds[5]);
+      expect(reader.hasNext()).toBe(true);
+      const msg = reader.readNext(1000);
+      console.log((await msg).getMessageId().toString());
+      expect((await msg).getData().toString()).toBe('my-message-6');
+
+      await producer.close();
+      await reader.close();
+      await client.close();
+    });
+
+    test('Reader seek by timestamp', async () => {
+      const client = new Pulsar.Client({
+        serviceUrl: 'pulsar://localhost:6650',
+        operationTimeoutSeconds: 30,
+      });
+
+      const topic = 'persistent://public/default/reader-seek-timestamp';
+      const producer = await client.createProducer({
+        topic,
+        sendTimeoutMs: 30000,
+        batchingEnabled: false,
+      });
+      expect(producer).not.toBeNull();
+
+      for (let i = 0; i < 10; i += 1) {
+        const msg = `my-message-${i}`;
+        console.log(msg);
+        await producer.send({
+          data: Buffer.from(msg),
+        });
+      }
+
+      const reader = await client.createReader({
+        topic,
+        startMessageId: Pulsar.MessageId.latest(),
+      });
+      expect(reader).not.toBeNull();
+
+      const currentTime = Date.now();
+      console.log(currentTime);
+
+      await reader.seekTimestamp(currentTime);
+
+      console.log('End seek');
+
+      expect(reader.hasNext()).toBe(false);
+
+      await reader.seekTimestamp(currentTime - 100000);
+      console.log('Seek to previous time');
+
+      expect(reader.hasNext()).toBe(true);
+      const msg = reader.readNext(1000);
+      console.log((await msg).getMessageId().toString());
+      expect((await msg).getData().toString()).toBe('my-message-0');
+
+      await producer.close();
+      await reader.close();
+      await client.close();
+    });
+
+    test('Message chunking', async () => {
+      const client = new Pulsar.Client({
+        serviceUrl: 'pulsar://localhost:6650',
+        operationTimeoutSeconds: 30,
+      });
+
+      const topic = 'persistent://public/default/message-chunking';
+      const producer = await client.createProducer({
+        topic,
+        batchingEnabled: false,
+        chunkingEnabled: true,
+      });
+
+      const consumer = await client.subscribe({
+        topic,
+        subscription: 'sub',
+        maxPendingChunkedMessage: 15,
+        autoAckOldestChunkedMessageOnQueueFull: true,
+      });
+
+      const sendMsg = Buffer.alloc(10 * 1024 * 1024);
+
+      await producer.send({
+        data: sendMsg,
+      });
+
+      const receiveMsg = await consumer.receive(3000);
+      expect(receiveMsg.getData().length).toBe(sendMsg.length);
+      await producer.close();
+      await consumer.close();
+      await client.close();
+    });
   });
 })();

package/tests/load-test.sh

@@ -0,0 +1,43 @@
+#!/bin/bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+# Test if the Pulsar.node built from ../pkg/linux/build-napi-inside-docker.sh
+# can be loaded in other Linux distributions.
+
+set -e
+
+if [[ $# -lt 2 ]]; then
+    echo "Usage $0 <node-image-name> <platform>
+
+See https://hub.docker.com/_/node for valid images"
+    exit 1
+fi
+IMAGE=$1
+PLATFORM=$2
+
+ROOT_DIR=${ROOT_DIR:-$(git rev-parse --show-toplevel)}
+cd $ROOT_DIR
+
+git archive -o pulsar-client-node.tar.gz HEAD
+
+docker run --platform $PLATFORM -v $PWD:/pulsar-client-node $IMAGE \
+    /bin/bash /pulsar-client-node/tests/docker-load-test.sh
+
+rm pulsar-client-node.tar.gz

package/tests/producer.test.js

@@ -55,7 +55,7 @@ const Pulsar = require('../index.js');
           topic: 'persistent://no-tenant/namespace/topic',
           sendTimeoutMs: 30000,
           batchingEnabled: true,
-        })).rejects.toThrow('Failed to create producer: ConnectError');
+        })).rejects.toThrow('Failed to create producer: BrokerMetadataError');
       });
 
       test('Not Exist Namespace', async () => {
@@ -63,7 +63,7 @@ const Pulsar = require('../index.js');
           topic: 'persistent://public/no-namespace/topic',
           sendTimeoutMs: 30000,
           batchingEnabled: true,
-        })).rejects.toThrow('Failed to create producer: ConnectError');
+        })).rejects.toThrow('Failed to create producer: BrokerMetadataError');
       });
 
       test('Automatic Producer Name', async () => {

package/{run-unit-tests.sh → tests/run-unit-tests.sh}

@@ -20,25 +20,16 @@
 
 set -e
 
-ROOT_DIR=$(git rev-parse --show-toplevel)
+ROOT_DIR=${ROOT_DIR:-$(git rev-parse --show-toplevel)}
 cd $ROOT_DIR
 
-# install pulsar cpp client pkg
-VERSION="${VERSION:-`cat ./pulsar-version.txt`}"
-PULSAR_PKG_DIR="/tmp/pulsar-test-pkg"
-rm -rf $PULSAR_PKG_DIR
-for pkg in apache-pulsar-client-dev.deb apache-pulsar-client.deb;do
-  curl -L --create-dir "https://archive.apache.org/dist/pulsar/pulsar-${VERSION}/DEB/${pkg}" -o $PULSAR_PKG_DIR/$pkg
-done;
-apt-get -y update
-apt-get install -y software-properties-common
-add-apt-repository ppa:ubuntu-toolchain-r/test && apt-get -y update
-apt-get -y install gcc-4.9 && apt-get upgrade -y libstdc++6
-apt install $PULSAR_PKG_DIR/apache-pulsar-client*.deb
+# download pulsar cpp client
+pkg/linux/download-cpp-client.sh
 
-./pulsar-test-service-start.sh
+cd $ROOT_DIR
+build-support/pulsar-test-service-start.sh
 npm install && npm run lint && npm run dtslint && npm run build && npm run test
 RES=$?
-./pulsar-test-service-stop.sh
+build-support/pulsar-test-service-stop.sh
 
 exit $RES

package/pulsar-version.txt

@@ -1 +0,0 @@
-2.8.0