pulsar-client 1.6.2 → 1.8.0-rc1

package/binding.gyp

@@ -18,19 +18,6 @@
 #
 
 {
-  'conditions': [
-    ['OS=="win"', {
-      'variables': {
-       'pulsar_cpp_dir%': '<!(echo %PULSAR_CPP_DIR%)',
-       'os_arch%': '<!(echo %OS_ARCH%)',
-      },
-    }],
-    ['OS=="mac"', {
-      'variables': {
-        'pulsar_cpp_dir': '<!(echo $PULSAR_CPP_DIR)'
-      }
-    }]
-  ],
   "targets": [
     {
       "target_name": "Pulsar",
@@ -40,64 +27,79 @@
       "include_dirs": [
         "<!@(node -p \"require('node-addon-api').include\")",
       ],
-      "defines": ["NAPI_VERSION=4", "NAPI_DISABLE_CPP_EXCEPTIONS=1"],
+      "defines": ["NAPI_VERSION=4"],
       "sources": [
         "src/addon.cc",
         "src/Message.cc",
         "src/MessageId.cc",
         "src/Authentication.cc",
         "src/Client.cc",
+        "src/SchemaInfo.cc",
         "src/Producer.cc",
         "src/ProducerConfig.cc",
         "src/Consumer.cc",
         "src/ConsumerConfig.cc",
         "src/Reader.cc",
         "src/ReaderConfig.cc",
+        "src/ThreadSafeDeferred.cc"
       ],
       'conditions': [
         ['OS=="mac"', {
-          "include_dirs": [
-            "<(pulsar_cpp_dir)/include",
+          'xcode_settings': {
+            'GCC_ENABLE_CPP_EXCEPTIONS': 'YES',
+            'GCC_ENABLE_CPP_RTTI': 'YES',
+            'MACOSX_DEPLOYMENT_TARGET': '11.0',
+            'CLANG_CXX_LANGUAGE_STANDARD': 'gnu++11',
+            'OTHER_CFLAGS': [
+                "-fPIC",
+            ]
+          },
+          "dependencies": [
+            "<!@(node -p \"require('node-addon-api').gyp\")"
           ],
-          "libraries": [
-            "<(pulsar_cpp_dir)/lib/libpulsar.dylib"
+          "include_dirs": [
+            "pkg/mac/build-pulsar/install/include"
           ],
         }],
         ['OS=="win"', {
+          "defines": [
+            "_HAS_EXCEPTIONS=1",
+            "PULSAR_STATIC"
+          ],
+          "msvs_settings": {
+            "VCCLCompilerTool": {
+              "ExceptionHandling": 1
+            },
+          },
           "include_dirs": [
-            "<(pulsar_cpp_dir)\include",
+            "pkg\\windows\\pulsar-cpp\\include",
           ],
           "libraries": [
-            "-l<(pulsar_cpp_dir)\\lib\Release\pulsar.lib"
+            "..\\pkg\\windows\\pulsar-cpp\\lib\\pulsarWithDeps.lib"
           ],
           "dependencies": [
             "<!(node -p \"require('node-addon-api').gyp\")"
-          ],
-          "copies": [
-            {
-              "destination": "<(PRODUCT_DIR)",
-              "files": [
-                "<(pulsar_cpp_dir)\\lib\Release\pulsar.dll",
-                "<(pulsar_cpp_dir)\\vcpkg_installed\\<(os_arch)\\bin\libcurl.dll",
-                "<(pulsar_cpp_dir)\\vcpkg_installed\\<(os_arch)\\bin\libprotobuf.dll",
-                "<(pulsar_cpp_dir)\\vcpkg_installed\\<(os_arch)\\bin\libssl-1_1-x64.dll",
-                "<(pulsar_cpp_dir)\\vcpkg_installed\\<(os_arch)\\bin\libcrypto-1_1-x64.dll",
-                "<(pulsar_cpp_dir)\\vcpkg_installed\\<(os_arch)\\bin\dl.dll",
-                "<(pulsar_cpp_dir)\\vcpkg_installed\\<(os_arch)\\bin\snappy.dll",
-                "<(pulsar_cpp_dir)\\vcpkg_installed\\<(os_arch)\\bin\zlib1.dll",
-                "<(pulsar_cpp_dir)\\vcpkg_installed\\<(os_arch)\\bin\zstd.dll",
-              ]
-            }
           ]
         }, {  # 'OS!="win"'
-          "libraries": [
-            "-lpulsar",
-          ],
           "dependencies": [
             "<!@(node -p \"require('node-addon-api').gyp\")"
           ],
+          "libraries": [
+             "../pkg/lib/libpulsarwithdeps.a"
+          ],
         }]
       ]
+    },
+    {
+      "target_name": "action_after_build",
+      "type": "none",
+      "dependencies": [ "<(module_name)" ],
+      "copies": [
+        {
+          "files": [ "<(PRODUCT_DIR)/<(module_name).node" ],
+          "destination": "<(module_path)"
+        }
+      ]
     }
   ]
 }

package/{pulsar-test-service-stop.sh → build-support/dep-version.py}

@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env python3
 #
 # Licensed to the Apache Software Foundation (ASF) under one
 # or more contributor license agreements.  See the NOTICE file
@@ -18,9 +18,7 @@
 # under the License.
 #
 
-set -e
+import yaml, sys
 
-PULSAR_DIR="${PULSAR_DIR:-/tmp/pulsar-test-dist}"
-cd $PULSAR_DIR
-
-bin/pulsar-daemon stop standalone
+deps = yaml.safe_load(open("dependencies.yaml"))
+print(deps[sys.argv[1]])

package/build-support/download-release-artifacts.py

@@ -0,0 +1,74 @@
+#!/usr/bin/env python3
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+import sys, json, urllib.request, os, shutil, zipfile, tempfile
+from pathlib import Path
+
+if len(sys.argv) != 3:
+    print("Usage: ")
+    print("     %s $WORKFLOW_RUN_ID $DEST_PATH" % sys.argv[0])
+    sys.exit(-1)
+
+if 'GITHUB_TOKEN' not in os.environ:
+    print('You should have a GITHUB_TOKEN environment variable')
+    sys.exit(-1)
+
+GITHUB_TOKEN = os.environ['GITHUB_TOKEN']
+
+ACCEPT_HEADER = 'application/vnd.github+json'
+LIST_URL = 'https://api.github.com/repos/apache/pulsar-client-node/actions/runs/%d/artifacts'
+
+workflow_run_id = int(sys.argv[1])
+dest_path = sys.argv[2]
+
+workflow_run_url = LIST_URL % workflow_run_id
+request = urllib.request.Request(workflow_run_url,
+                    headers={'Accept': ACCEPT_HEADER, 'Authorization': 'Bearer ' + GITHUB_TOKEN})
+with urllib.request.urlopen(request) as response:
+    data = json.loads(response.read().decode("utf-8"))
+    for artifact in data['artifacts']:
+        name = artifact['name']
+        url = artifact['archive_download_url']
+
+        print('Downloading %s from %s' % (name, url))
+        artifact_request = urllib.request.Request(url,
+                    headers={'Authorization': 'Bearer ' + GITHUB_TOKEN})
+        with urllib.request.urlopen(artifact_request) as response:
+            tmp_zip = tempfile.NamedTemporaryFile(delete=False)
+            try:
+                #
+                shutil.copyfileobj(response, tmp_zip)
+                tmp_zip.close()
+
+                dest_dir = os.path.join(dest_path, name)
+                Path(dest_dir).mkdir(parents=True, exist_ok=True)
+                with zipfile.ZipFile(tmp_zip.name, 'r') as z:
+                    z.extractall(dest_dir)
+            finally:
+                os.unlink(tmp_zip.name)
+
+    for root, dirs, files in os.walk(dest_path, topdown=False):
+        for name in files:
+            shutil.move(os.path.join(root, name), dest_path)
+        if not os.listdir(root):
+            os.rmdir(root)
+
+
+

package/build-support/generate-source-archive.sh

@@ -0,0 +1,28 @@
+#!/usr/bin/env bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+ROOT_DIR=$(git rev-parse --show-toplevel)
+
+VERSION=$(npm version | grep pulsar-client | awk -F ':' '{print $2}' | sed "s?'??g" | sed "s?,??g" | sed "s? ??g")
+
+NAME=apache-pulsar-client-node-$VERSION
+
+OUT_DIR=${1:-.}
+
+git archive --format=tar.gz --prefix ${NAME}/ -o ${OUT_DIR}/${NAME}.tar.gz HEAD

package/build-support/install-cpp-client.sh

@@ -0,0 +1,66 @@
+#!/usr/bin/env bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+set -e -x
+
+ROOT_DIR=`cd $(dirname $0) && cd .. && pwd`
+source $ROOT_DIR/pulsar-client-cpp.txt
+
+if [ $USER != "root" ]; then
+  SUDO="sudo"
+fi
+
+# Get the flavor of Linux
+export $(cat /etc/*-release | grep "^ID=")
+
+cd /tmp
+
+UNAME_ARCH=$(uname -m)
+if [ $UNAME_ARCH == 'aarch64' ]; then
+  PLATFORM=arm64
+else
+  PLATFORM=x86_64
+fi
+
+if [ $ID == 'ubuntu' -o $ID == 'debian' ]; then
+  curl -L -O ${CPP_CLIENT_BASE_URL}/deb-${PLATFORM}/apache-pulsar-client.deb
+  curl -L -O ${CPP_CLIENT_BASE_URL}/deb-${PLATFORM}/apache-pulsar-client-dev.deb
+  $SUDO apt install -y /tmp/*.deb
+
+elif [ $ID == 'alpine' ]; then
+  curl -L -O ${CPP_CLIENT_BASE_URL}/apk-${PLATFORM}/${UNAME_ARCH}/apache-pulsar-client-${CPP_CLIENT_VERSION}-r0.apk
+  curl -L -O ${CPP_CLIENT_BASE_URL}/apk-${PLATFORM}/${UNAME_ARCH}/apache-pulsar-client-dev-${CPP_CLIENT_VERSION}-r0.apk
+  $SUDO apk add --allow-untrusted /tmp/*.apk
+
+elif [ $ID == '"centos"' ]; then
+  curl -L -O ${CPP_CLIENT_BASE_URL}/rpm-${PLATFORM}/${UNAME_ARCH}/apache-pulsar-client-${CPP_CLIENT_VERSION}-1.${UNAME_ARCH}.rpm
+  curl -L -O ${CPP_CLIENT_BASE_URL}/rpm-${PLATFORM}/${UNAME_ARCH}/apache-pulsar-client-devel-${CPP_CLIENT_VERSION}-1.${UNAME_ARCH}.rpm
+  $SUDO rpm -i /tmp/*.rpm
+
+else
+  echo "Unknown Linux distribution: '$ID'"
+  exit 1
+fi
+
+mkdir -p $ROOT_DIR/pkg/lib/
+cp /usr/lib/libpulsarwithdeps.a $ROOT_DIR/pkg/lib/
+
+
+

package/{pulsar-test-service-start.sh → build-support/pulsar-test-container-start.sh}

@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 #
 # Licensed to the Apache Software Foundation (ASF) under one
 # or more contributor license agreements.  See the NOTICE file
@@ -18,28 +18,18 @@
 # under the License.
 #
 
-set -e
+set -e -x
 
-ROOT_DIR=$(git rev-parse --show-toplevel)
-cd $ROOT_DIR
-
-VERSION="${VERSION:-`cat ./pulsar-version.txt`}"
-PULSAR_DIR="${PULSAR_DIR:-/tmp/pulsar-test-dist}"
-PKG=apache-pulsar-${VERSION}-bin.tar.gz
-
-rm -rf $PULSAR_DIR
-curl -L --create-dir "https://archive.apache.org/dist/pulsar/pulsar-${VERSION}/${PKG}" -o $PULSAR_DIR/$PKG
-tar xfz $PULSAR_DIR/$PKG -C $PULSAR_DIR --strip-components 1
-
-DATA_DIR=/tmp/pulsar-test-data
-rm -rf $DATA_DIR
-mkdir -p $DATA_DIR
-
-export PULSAR_STANDALONE_CONF=$ROOT_DIR/tests/conf/standalone.conf
-$PULSAR_DIR/bin/pulsar-daemon start standalone \
+export PULSAR_STANDALONE_CONF=test-conf/standalone.conf
+# There is an issue when starting the pulsar standalone with other metadata store: https://github.com/apache/pulsar/issues/17984
+# We need to use Zookeeper here. Otherwise the `Message Chunking` test will not pass.
+# We can remove this line after the pulsar release a new version contains this fix: https://github.com/apache/pulsar/pull/18126
+export PULSAR_STANDALONE_USE_ZOOKEEPER=1
+bin/pulsar-daemon start standalone \
         --no-functions-worker --no-stream-storage \
-        --zookeeper-dir $DATA_DIR/zookeeper \
-        --bookkeeper-dir $DATA_DIR/bookkeeper
+        --bookkeeper-dir data/bookkeeper
 
 echo "-- Wait for Pulsar service to be ready"
 until curl http://localhost:8080/metrics > /dev/null 2>&1 ; do sleep 1; done
+
+echo "-- Ready to start tests"

package/build-support/pulsar-test-service-start.sh

@@ -0,0 +1,37 @@
+#!/usr/bin/env bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+set -e
+
+SRC_DIR=$(git rev-parse --show-toplevel)
+cd $SRC_DIR
+
+build-support/pulsar-test-service-stop.sh
+
+CONTAINER_ID=$(docker run -i -p 8080:8080 -p 6650:6650 -p 8443:8443 -p 6651:6651 --rm --detach apachepulsar/pulsar:2.10.2 sleep 3600)
+
+echo $CONTAINER_ID >.tests-container-id.txt
+
+docker cp $SRC_DIR/tests/conf $CONTAINER_ID:/pulsar/test-conf
+docker cp $SRC_DIR/build-support/pulsar-test-container-start.sh $CONTAINER_ID:pulsar-test-container-start.sh
+
+docker exec -i $CONTAINER_ID /pulsar-test-container-start.sh
+
+echo "-- Ready to start tests"

package/build-support/pulsar-test-service-stop.sh

@@ -0,0 +1,32 @@
+#!/usr/bin/env bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+set -e
+
+SRC_DIR=$(git rev-parse --show-toplevel)
+cd $SRC_DIR
+
+CONTAINER_ID_PATH=".tests-container-id.txt"
+
+if [ -f ${CONTAINER_ID_PATH} ]; then
+  CONTAINER_ID=$(cat $CONTAINER_ID_PATH)
+  docker kill $CONTAINER_ID || true
+  rm .tests-container-id.txt
+fi

package/{.github/workflows/nodejs.yml → build-support/sign-files.sh}

@@ -1,3 +1,4 @@
+#!/usr/bin/env bash
 #
 # Licensed to the Apache Software Foundation (ASF) under one
 # or more contributor license agreements.  See the NOTICE file
@@ -16,16 +17,16 @@
 # specific language governing permissions and limitations
 # under the License.
 #
-name: Node.js
-on: [pull_request]
-jobs:
-  build:
-    name: Build
-    runs-on: ubuntu-latest
-    steps:
-      - name: Check out code into the Node.js module directory
-        uses: actions/checkout@v2
 
-      - name: Test
-        run: |
-          ./docker-tests.sh
+set -e
+
+FILES=$*
+
+for FILE in $FILES
+do
+   echo "Signing $FILE"
+   gpg --armor --output $FILE.asc --detach-sig $FILE
+
+   # SHA-512 signature
+   shasum -a 512 $FILE > $FILE.sha512
+done

package/build-support/stage-release.sh

@@ -0,0 +1,44 @@
+#!/usr/bin/env bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+set -e -x
+
+if [ $# -neq 2 ]; then
+    echo "Usage: $0 \$DEST_PATH \$WORKFLOW_ID"
+    exit 1
+fi
+
+DEST_PATH=$(readlink -f $1)
+WORKFLOW_ID=$2
+
+pushd $(dirname "$0")
+PULSAR_NODE_PATH=$(git rev-parse --show-toplevel)
+popd
+
+mkdir -p $DEST_PATH
+
+cd $PULSAR_NODE_PATH
+
+build-support/download-release-artifacts.py $WORKFLOW_ID $DEST_PATH
+build-support/generate-source-archive.sh $DEST_PATH
+
+# Sign all files
+cd $DEST_PATH
+find . -type f | xargs $PULSAR_NODE_PATH/build-support/sign-files.sh

package/dependencies.yaml

@@ -0,0 +1,28 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+# Only used for MacOS builds
+boost: 1.80.0
+cmake: 3.24.2
+protobuf: 3.20.0
+zlib: 1.2.13
+zstd: 1.5.2
+snappy: 1.1.9
+openssl: 1.1.1s
+curl: 7.85.0