pulsar-client 1.7.0 → 1.8.0-rc.1

package/src/Reader.h

@@ -42,6 +42,8 @@ class Reader : public Napi::ObjectWrap<Reader> {
   Napi::Value ReadNext(const Napi::CallbackInfo &info);
   Napi::Value HasNext(const Napi::CallbackInfo &info);
   Napi::Value IsConnected(const Napi::CallbackInfo &info);
+  Napi::Value Seek(const Napi::CallbackInfo &info);
+  Napi::Value SeekTimestamp(const Napi::CallbackInfo &info);
   Napi::Value Close(const Napi::CallbackInfo &info);
   void CleanupListener();
 };

package/src/pulsar-binding.js

@@ -0,0 +1,26 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+const path = require('path');
+const binary = require('@mapbox/node-pre-gyp');
+
+const bindingPath = binary.find(path.resolve(path.join(__dirname, '../package.json')));
+// eslint-disable-next-line import/no-dynamic-require
+const binding = require(bindingPath);
+// eslint-disable-next-line no-multi-assign
+module.exports = exports = binding;

package/tests/conf/standalone.conf

@@ -275,3 +275,9 @@ keepAliveIntervalSeconds=30
 
 # How often broker checks for inactive topics to be deleted (topics with no subscriptions and no one connected)
 brokerServicePurgeInactiveFrequencyInSeconds=60
+
+### --- BookKeeper Configuration --- #####
+
+# The maximum netty frame size in bytes. Any message received larger than this will be rejected. The default value is 5MB.
+nettyMaxFrameSizeBytes=5253120
+

package/tests/docker-load-test.sh

@@ -0,0 +1,35 @@
+#!/bin/bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+# NOTE: This script should only run inside a Node.js docker container.
+# See ./load-test.sh for details.
+set -ex
+
+# Create an empty directory to test
+mkdir -p /app && cd /app
+tar zxf /pulsar-client-node/pulsar-client-node.tar.gz
+
+# Use the existing Pulsar.node built in a specific container
+mkdir -p lib/binding
+cp /pulsar-client-node/build/Release/pulsar.node lib/binding/
+npm install
+
+# Test if Pulsar.node can be loaded
+node pkg/load_test.js

package/tests/end_to_end.test.js

@@ -847,5 +847,217 @@ const Pulsar = require('../index.js');
 
       await client.close();
     });
+
+    test('Consumer seek by message Id', async () => {
+      const client = new Pulsar.Client({
+        serviceUrl: 'pulsar://localhost:6650',
+        operationTimeoutSeconds: 30,
+      });
+
+      const topic = 'persistent://public/default/seek-by-msgid';
+      const producer = await client.createProducer({
+        topic,
+        sendTimeoutMs: 30000,
+        batchingEnabled: false,
+      });
+      expect(producer).not.toBeNull();
+
+      const msgIds = [];
+      for (let i = 0; i < 10; i += 1) {
+        const msg = `my-message-${i}`;
+        console.log(msg);
+        const msgId = await producer.send({
+          data: Buffer.from(msg),
+        });
+        msgIds.push(msgId);
+      }
+
+      const consumer = await client.subscribe({
+        topic,
+        subscription: 'sub',
+      });
+      expect(consumer).not.toBeNull();
+
+      await consumer.seek(msgIds[5]);
+      const msg = consumer.receive(1000);
+      console.log((await msg).getMessageId().toString());
+      expect((await msg).getData().toString()).toBe('my-message-6');
+
+      await producer.close();
+      await consumer.close();
+      await client.close();
+    });
+
+    test('Consumer seek by timestamp', async () => {
+      const client = new Pulsar.Client({
+        serviceUrl: 'pulsar://localhost:6650',
+        operationTimeoutSeconds: 30,
+      });
+
+      const topic = 'persistent://public/default/seek-by-timestamp';
+      const producer = await client.createProducer({
+        topic,
+        sendTimeoutMs: 30000,
+        batchingEnabled: false,
+      });
+      expect(producer).not.toBeNull();
+
+      for (let i = 0; i < 10; i += 1) {
+        const msg = `my-message-${i}`;
+        console.log(msg);
+        await producer.send({
+          data: Buffer.from(msg),
+        });
+      }
+
+      const consumer = await client.subscribe({
+        topic,
+        subscription: 'sub',
+      });
+      expect(consumer).not.toBeNull();
+
+      const currentTime = Date.now();
+      console.log(currentTime);
+
+      await consumer.seekTimestamp(currentTime);
+
+      console.log('End seek');
+
+      await expect(consumer.receive(1000)).rejects.toThrow('Failed to receive message: TimeOut');
+
+      await consumer.seekTimestamp(currentTime - 100000);
+
+      const msg = consumer.receive(1000);
+      console.log((await msg).getMessageId().toString());
+      expect((await msg).getData().toString()).toBe('my-message-0');
+
+      await producer.close();
+      await consumer.close();
+      await client.close();
+    });
+
+    test('Reader seek by message Id', async () => {
+      const client = new Pulsar.Client({
+        serviceUrl: 'pulsar://localhost:6650',
+        operationTimeoutSeconds: 30,
+      });
+
+      const topic = 'persistent://public/default/reader-seek-by-msgid';
+      const producer = await client.createProducer({
+        topic,
+        sendTimeoutMs: 30000,
+        batchingEnabled: false,
+      });
+      expect(producer).not.toBeNull();
+
+      const msgIds = [];
+      for (let i = 0; i < 10; i += 1) {
+        const msg = `my-message-${i}`;
+        console.log(msg);
+        const msgId = await producer.send({
+          data: Buffer.from(msg),
+        });
+        msgIds.push(msgId);
+      }
+
+      const reader = await client.createReader({
+        topic,
+        startMessageId: Pulsar.MessageId.latest(),
+      });
+      expect(reader).not.toBeNull();
+
+      await reader.seek(msgIds[5]);
+      expect(reader.hasNext()).toBe(true);
+      const msg = reader.readNext(1000);
+      console.log((await msg).getMessageId().toString());
+      expect((await msg).getData().toString()).toBe('my-message-6');
+
+      await producer.close();
+      await reader.close();
+      await client.close();
+    });
+
+    test('Reader seek by timestamp', async () => {
+      const client = new Pulsar.Client({
+        serviceUrl: 'pulsar://localhost:6650',
+        operationTimeoutSeconds: 30,
+      });
+
+      const topic = 'persistent://public/default/reader-seek-timestamp';
+      const producer = await client.createProducer({
+        topic,
+        sendTimeoutMs: 30000,
+        batchingEnabled: false,
+      });
+      expect(producer).not.toBeNull();
+
+      for (let i = 0; i < 10; i += 1) {
+        const msg = `my-message-${i}`;
+        console.log(msg);
+        await producer.send({
+          data: Buffer.from(msg),
+        });
+      }
+
+      const reader = await client.createReader({
+        topic,
+        startMessageId: Pulsar.MessageId.latest(),
+      });
+      expect(reader).not.toBeNull();
+
+      const currentTime = Date.now();
+      console.log(currentTime);
+
+      await reader.seekTimestamp(currentTime);
+
+      console.log('End seek');
+
+      expect(reader.hasNext()).toBe(false);
+
+      await reader.seekTimestamp(currentTime - 100000);
+      console.log('Seek to previous time');
+
+      expect(reader.hasNext()).toBe(true);
+      const msg = reader.readNext(1000);
+      console.log((await msg).getMessageId().toString());
+      expect((await msg).getData().toString()).toBe('my-message-0');
+
+      await producer.close();
+      await reader.close();
+      await client.close();
+    });
+
+    test('Message chunking', async () => {
+      const client = new Pulsar.Client({
+        serviceUrl: 'pulsar://localhost:6650',
+        operationTimeoutSeconds: 30,
+      });
+
+      const topic = 'persistent://public/default/message-chunking';
+      const producer = await client.createProducer({
+        topic,
+        batchingEnabled: false,
+        chunkingEnabled: true,
+      });
+
+      const consumer = await client.subscribe({
+        topic,
+        subscription: 'sub',
+        maxPendingChunkedMessage: 15,
+        autoAckOldestChunkedMessageOnQueueFull: true,
+      });
+
+      const sendMsg = Buffer.alloc(10 * 1024 * 1024);
+
+      await producer.send({
+        data: sendMsg,
+      });
+
+      const receiveMsg = await consumer.receive(3000);
+      expect(receiveMsg.getData().length).toBe(sendMsg.length);
+      await producer.close();
+      await consumer.close();
+      await client.close();
+    });
   });
 })();

package/{run-unit-tests.sh → tests/load-test.sh}

@@ -18,27 +18,26 @@
 # under the License.
 #
 
+# Test if the Pulsar.node built from ../pkg/linux/build-napi-inside-docker.sh
+# can be loaded in other Linux distributions.
+
 set -e
 
+if [[ $# -lt 2 ]]; then
+    echo "Usage $0 <node-image-name> <platform>
+
+See https://hub.docker.com/_/node for valid images"
+    exit 1
+fi
+IMAGE=$1
+PLATFORM=$2
+
 ROOT_DIR=${ROOT_DIR:-$(git rev-parse --show-toplevel)}
 cd $ROOT_DIR
 
-# install pulsar cpp client pkg
-VERSION="${VERSION:-`cat ./pulsar-version.txt`}"
-PULSAR_PKG_DIR="/tmp/pulsar-test-pkg"
-rm -rf $PULSAR_PKG_DIR
-for pkg in apache-pulsar-client-dev.deb apache-pulsar-client.deb;do
-  curl -L --create-dir "https://archive.apache.org/dist/pulsar/pulsar-${VERSION}/DEB/${pkg}" -o $PULSAR_PKG_DIR/$pkg
-done;
-apt-get -y update
-apt-get install -y software-properties-common
-add-apt-repository ppa:ubuntu-toolchain-r/test && apt-get -y update
-apt-get -y install gcc-4.9 && apt-get upgrade -y libstdc++6
-apt install $PULSAR_PKG_DIR/apache-pulsar-client*.deb
-
-./pulsar-test-service-start.sh
-npm install && npm run lint && npm run dtslint && npm run build && npm run test
-RES=$?
-./pulsar-test-service-stop.sh
-
-exit $RES
+git archive -o pulsar-client-node.tar.gz HEAD
+
+docker run --platform $PLATFORM -v $PWD:/pulsar-client-node $IMAGE \
+    /bin/bash /pulsar-client-node/tests/docker-load-test.sh
+
+rm pulsar-client-node.tar.gz

package/{docker-tests.sh → tests/run-unit-tests.sh}

@@ -18,21 +18,18 @@
 # under the License.
 #
 
+set -e
+
 ROOT_DIR=${ROOT_DIR:-$(git rev-parse --show-toplevel)}
 cd $ROOT_DIR
 
-BUILD_IMAGE_NAME="${BUILD_IMAGE_NAME:-apachepulsar/pulsar-build}"
-BUILD_IMAGE_VERSION="${BUILD_IMAGE_VERSION:-ubuntu-20.04}"
-
-IMAGE="$BUILD_IMAGE_NAME:$BUILD_IMAGE_VERSION"
-
-echo "---- Testing Pulsar node client using image $IMAGE"
+# download pulsar cpp client
+pkg/linux/download-cpp-client.sh
 
-docker pull $IMAGE
-
-TARGET_DIR=/pulsar-client-node
-DOCKER_CMD="docker run -i -e ROOT_DIR=$TARGET_DIR -v $ROOT_DIR:$TARGET_DIR $IMAGE"
+cd $ROOT_DIR
+build-support/pulsar-test-service-start.sh
+npm install && npm run lint && npm run dtslint && npm run build && npm run test
+RES=$?
+build-support/pulsar-test-service-stop.sh
 
-# Start Pulsar standalone instance
-# and execute the tests
-$DOCKER_CMD bash -c "cd /pulsar-client-node && ./run-unit-tests.sh"
+exit $RES

package/pulsar-version.txt

@@ -1 +0,0 @@
-2.10.1