pulsar-client 1.6.2 → 1.8.0-rc1

package/.asf.yaml

@@ -43,4 +43,11 @@ github:
     # disable merge button:
     merge:   false
     # disable rebase button:
-    rebase:  false
+    rebase:  false
+
+notifications:
+  commits:      commits@pulsar.apache.org
+  issues:       commits@pulsar.apache.org
+  pullrequests: commits@pulsar.apache.org
+  discussions:  dev@pulsar.apache.org
+  jira_options: link label

package/.github/PULL_REQUEST_TEMPLATE.md

@@ -0,0 +1,65 @@
+<!--
+### Contribution Checklist
+  
+  - PR title format should be *[type][component] summary*. For details, see *[Guideline - Pulsar PR Naming Convention](https://docs.google.com/document/d/1d8Pw6ZbWk-_pCKdOmdvx9rnhPiyuxwq60_TrD68d7BA/edit#heading=h.trs9rsex3xom)*. 
+
+  - Fill out the template below to describe the changes contributed by the pull request. That will give reviewers the context they need to do the review.
+  
+  - Each pull request should address only one issue, not mix up code from multiple issues.
+  
+  - Each commit in the pull request has a meaningful commit message
+
+  - Once all items of the checklist are addressed, remove the above text and this checklist, leaving only the filled out template below.
+-->
+
+<!-- Either this PR fixes an issue, -->
+
+Fixes #<xyz>
+
+<!-- or this PR is one task of an issue -->
+
+Master Issue: #<xyz>
+
+### Motivation
+
+<!-- Explain here the context, and why you're making that change. What is the problem you're trying to solve. -->
+
+### Modifications
+
+<!-- Describe the modifications you've done. -->
+
+### Verifying this change
+
+- [ ] Make sure that the change passes the CI checks.
+
+*(Please pick either of the following options)*
+
+This change is a trivial rework / code cleanup without any test coverage.
+
+*(or)*
+
+This change is already covered by existing tests, such as *(please describe tests)*.
+
+*(or)*
+
+This change added tests and can be verified as follows:
+
+*(example:)*
+  - *Added integration tests for end-to-end deployment with large payloads (10MB)*
+  - *Extended integration test for recovery after broker failure*
+
+### Documentation
+
+<!-- DO NOT REMOVE THIS SECTION. CHECK THE PROPER BOX ONLY. -->
+
+- [ ] `doc-required` 
+(Your PR needs to update docs and you will update later)
+
+- [ ] `doc-not-needed` 
+(Please explain why)
+
+- [ ] `doc` 
+(Your PR contains doc changes)
+
+- [ ] `doc-complete`
+(Docs have been already added)

package/.github/workflows/ci-build-release-napi.yml

@@ -0,0 +1,195 @@
+name: Build release napi
+on:
+  workflow_dispatch:
+  push:
+    tags:
+      - '*'
+env:
+  FORCE_COLOR: 1
+concurrency:
+  group: ${{ github.head_ref || github.run_id }}
+  cancel-in-progress: true
+jobs:
+  macos-napi:
+    name: Build NAPI macos - Node ${{matrix.nodejs}} - ${{matrix.arch}}
+    runs-on: macos-latest
+    timeout-minutes: 3000
+
+    strategy:
+      fail-fast: false
+      matrix:
+        arch:
+          - x64
+          - arm64
+        nodejs:
+          - 18
+    steps:
+      - uses: actions/checkout@v3
+      - name: Use Node.js ${{ matrix.nodejs }}
+        uses: actions/setup-node@v3
+        with:
+          node-version: ${{ matrix.nodejs }}
+          cache: 'npm'
+
+      - name: Cache Dependencies
+        id: cache-dependencies
+        uses: actions/cache@v3
+        with:
+          path: pkg/mac/build
+          key: ${{ runner.os }}-${{ matrix.arch }}-mac-${{ hashFiles('pkg/mac/build-cpp-deps-lib.sh') }}
+
+      - name: Add arch env vars
+        run: |
+          if [ "${{ matrix.arch }}" = "x64" ]; then
+            echo "ARCH=x86_64" >> $GITHUB_ENV
+          else
+            echo "ARCH=${{ matrix.arch }}" >> $GITHUB_ENV
+          fi
+
+      - name: Build CPP dependencies lib
+        if: steps.cache-dependencies.outputs.cache-hit != 'true'
+        run: |
+          export ARCH=${{ env.ARCH }}
+          pkg/mac/build-cpp-deps-lib.sh
+
+      - name: Build CPP lib
+        if: steps.cache-pulsar.outputs.cache-hit != 'true'
+        run: |
+          export ARCH=${{ env.ARCH }}
+          pkg/mac/build-cpp-lib.sh
+
+      - name: Build Node binaries lib
+        run: |
+          npm install --ignore-scripts
+          npx node-pre-gyp configure --target_arch=${{ matrix.arch }}
+          npx node-pre-gyp build --target_arch=${{ matrix.arch }}
+
+      - name: Test loading Node binaries lib
+        if: matrix.arch == 'x64'
+        run: |
+          node pkg/load_test.js
+
+      - name: Package Node binaries lib
+        run: |
+          npx node-pre-gyp package --target_arch=${{ matrix.arch }}
+
+      - name: Upload artifacts
+        uses: actions/upload-artifact@v3
+        with:
+          name: macos-${{matrix.nodejs}}-${{matrix.arch}}
+          path: build/stage/*/*.tar.gz
+
+  linux-napi:
+    name: Build NAPI ${{matrix.image}} - Node ${{matrix.nodejs}} - ${{matrix.cpu.platform}}
+    runs-on: ubuntu-22.04
+    timeout-minutes: 3000
+
+    strategy:
+      fail-fast: false
+      matrix:
+        image:
+          - 'linux_glibc'
+          - 'linux_musl'
+        nodejs:
+          - 18
+        cpu:
+          - {arch: 'x86_64', platform: 'x86_64'}
+          - {arch: 'aarch64', platform: 'arm64'}
+
+    steps:
+      - name: checkout
+        uses: actions/checkout@v3
+
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v2
+
+      - name: Setup Docker Buildx
+        uses: docker/setup-buildx-action@v2
+
+      - name: Build Linux Docker image
+        uses: docker/build-push-action@v3
+        with:
+          context: ./pkg/${{matrix.image}}
+          load: true
+          tags: build:latest
+          platforms: linux/${{matrix.cpu.arch}}
+          build-args: |
+            PLATFORM=${{matrix.cpu.platform}}
+            ARCH=${{matrix.cpu.arch}}
+            NODE_VERSION=${{matrix.nodejs}}
+          cache-from: type=gha
+          cache-to: type=gha,mode=max
+
+      - name: Build and Test NAPI file
+        run: |
+          docker run -i -v $PWD:/pulsar-client-node build:latest \
+              /pulsar-client-node/pkg/build-napi-inside-docker.sh
+
+      - name: Upload artifacts
+        uses: actions/upload-artifact@v3
+        with:
+          name: ${{matrix.image}}-${{matrix.nodejs}}-${{matrix.cpu.platform}}
+          path: build/stage/*/*.tar.gz
+
+  windows-napi:
+    name: Build NAPI windows - Node ${{matrix.nodejs}} - ${{matrix.arch}}
+    runs-on: windows-2022
+    timeout-minutes: 3000
+
+    strategy:
+      fail-fast: false
+      matrix:
+        arch:
+          - x64
+          - x86
+        nodejs:
+          - 18
+    steps:
+      - uses: actions/checkout@v3
+      - name: Use Node.js ${{ matrix.nodejs }}
+        uses: actions/setup-node@v3
+        with:
+          node-version: ${{ matrix.nodejs }}
+          architecture: ${{ matrix.arch }}
+          cache: 'npm'
+
+      - name: Cache CPP Client
+        id: cache-dependencies
+        uses: actions/cache@v3
+        with:
+          path: pkg/windows/
+          key: ${{ runner.os }}-${{ matrix.arch }}-${{ hashFiles('pulsar-client-cpp.txt') }}
+
+      - name: Add env vars
+        shell: bash
+        run: |
+          if [ "${{ matrix.arch }}" = "x86" ]; then
+            echo "TARGET=ia32" >> $GITHUB_ENV
+          else
+            echo "TARGET=${{ matrix.arch }}" >> $GITHUB_ENV
+          fi
+
+      - name: Download CPP Client
+        if: steps.cache-dependencies.outputs.cache-hit != 'true'
+        run: |
+          pkg/windows/download-cpp-client.bat ${{ matrix.arch }}
+
+      - name: Build Node binaries lib
+        run: |
+          npm install --ignore-scripts
+          npx node-pre-gyp configure --target_arch=${{ env.TARGET }}
+          npx node-pre-gyp build --target_arch=${{ env.TARGET }}
+
+      - name: Test loading Node binaries lib
+        run: |
+          node pkg/load_test.js
+
+      - name: Package Node binaries lib
+        run: |
+          npx node-pre-gyp package --target_arch=${{ env.TARGET }}
+
+      - name: Upload artifacts
+        uses: actions/upload-artifact@v3
+        with:
+          name: windows-${{matrix.nodejs}}-${{matrix.arch}}
+          path: build/stage/*/*.tar.gz

package/.github/workflows/ci-pr-validation.yml

@@ -0,0 +1,225 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+name: PR validation
+on:
+  pull_request:
+    branches: ['master']
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  unit-tests:
+    name: Run unit tests
+    runs-on: ubuntu-22.04
+    timeout-minutes: 120
+
+    steps:
+      - uses: actions/checkout@v3
+      - name: Use Node.js 18
+        uses: actions/setup-node@v3
+        with:
+          node-version: 18
+
+      - name: Run Test
+        run: |
+          ./tests/run-unit-tests.sh
+
+  validation-yarn:
+    name: Validation use yarn install
+    runs-on: ubuntu-22.04
+    timeout-minutes: 120
+
+    steps:
+      - uses: actions/checkout@v3
+      - name: Use Node.js 18
+        uses: actions/setup-node@v3
+        with:
+          node-version: 18
+
+      - name: Use yarn install
+        run: |
+          yarn install --ignore-scripts
+
+  macos-napi:
+    name: Build NAPI macos - Node ${{matrix.nodejs}} - ${{matrix.arch}}
+    runs-on: macos-latest
+    timeout-minutes: 3000
+
+    strategy:
+      fail-fast: false
+      matrix:
+        arch:
+          - x64
+          - arm64
+        nodejs:
+          - 18
+    steps:
+      - uses: actions/checkout@v3
+      - name: Use Node.js ${{ matrix.nodejs }}
+        uses: actions/setup-node@v3
+        with:
+          node-version: ${{ matrix.nodejs }}
+          cache: 'npm'
+
+      - name: Cache Dependencies
+        id: cache-dependencies
+        uses: actions/cache@v3
+        with:
+          path: pkg/mac/build
+          key: ${{ runner.os }}-${{ matrix.arch }}-mac-${{ hashFiles('pkg/mac/build-cpp-deps-lib.sh') }}
+
+      - name: Add arch env vars
+        run: |
+          if [ "${{ matrix.arch }}" = "x64" ]; then
+            echo "ARCH=x86_64" >> $GITHUB_ENV
+          else
+            echo "ARCH=${{ matrix.arch }}" >> $GITHUB_ENV
+          fi
+
+      - name: Build CPP dependencies lib
+        if: steps.cache-dependencies.outputs.cache-hit != 'true'
+        run: |
+          export ARCH=${{ env.ARCH }}
+          pkg/mac/build-cpp-deps-lib.sh
+
+      - name: Build CPP lib
+        if: steps.cache-pulsar.outputs.cache-hit != 'true'
+        run: |
+          export ARCH=${{ env.ARCH }}
+          pkg/mac/build-cpp-lib.sh
+
+      - name: Build Node binaries lib
+        run: |
+          npm install --ignore-scripts
+          npx node-pre-gyp configure --target_arch=${{ matrix.arch }}
+          npx node-pre-gyp build --target_arch=${{ matrix.arch }}
+
+      - name: Test loading Node binaries lib
+        if: matrix.arch == 'x64'
+        run: |
+          node pkg/load_test.js
+
+      - name: Package Node binaries lib
+        run: |
+          npx node-pre-gyp package --target_arch=${{ matrix.arch }}
+
+  linux-napi:
+    name: Build NAPI ${{matrix.image}} - Node ${{matrix.nodejs}} - ${{matrix.cpu.platform}}
+    runs-on: ubuntu-22.04
+    timeout-minutes: 3000
+
+    strategy:
+      fail-fast: false
+      matrix:
+        image:
+          - 'linux_glibc'
+          - 'linux_musl'
+        nodejs:
+          - 18
+        cpu:
+          - {arch: 'x86_64', platform: 'x86_64'}
+          - {arch: 'aarch64', platform: 'arm64'}
+
+    steps:
+      - name: checkout
+        uses: actions/checkout@v3
+
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v2
+
+      - name: Setup Docker Buildx
+        uses: docker/setup-buildx-action@v2
+
+      - name: Build Linux Docker image
+        uses: docker/build-push-action@v3
+        with:
+          context: ./pkg/${{matrix.image}}
+          load: true
+          tags: build:latest
+          platforms: linux/${{matrix.cpu.arch}}
+          build-args: |
+            PLATFORM=${{matrix.cpu.platform}}
+            ARCH=${{matrix.cpu.arch}}
+            NODE_VERSION=${{matrix.nodejs}}
+          cache-from: type=gha
+          cache-to: type=gha,mode=max
+
+      - name: Build and Test NAPI file
+        run: |
+          docker run -i -v $PWD:/pulsar-client-node build:latest \
+              /pulsar-client-node/pkg/build-napi-inside-docker.sh
+
+  windows-napi:
+    name: Build NAPI windows - Node ${{matrix.nodejs}} - ${{matrix.arch}}
+    runs-on: windows-2022
+    timeout-minutes: 3000
+
+    strategy:
+      fail-fast: false
+      matrix:
+        arch:
+          - x64
+          - x86
+        nodejs:
+          - 18
+    steps:
+      - uses: actions/checkout@v3
+      - name: Use Node.js ${{ matrix.nodejs }}
+        uses: actions/setup-node@v3
+        with:
+          node-version: ${{ matrix.nodejs }}
+          architecture: ${{ matrix.arch }}
+          cache: 'npm'
+
+      - name: Cache CPP Client
+        id: cache-dependencies
+        uses: actions/cache@v3
+        with:
+          path: pkg/windows/
+          key: ${{ runner.os }}-${{ matrix.arch }}-${{ hashFiles('pulsar-client-cpp.txt') }}
+
+      - name: Add env vars
+        shell: bash
+        run: |
+          if [ "${{ matrix.arch }}" = "x86" ]; then
+            echo "TARGET=ia32" >> $GITHUB_ENV
+          else
+            echo "TARGET=${{ matrix.arch }}" >> $GITHUB_ENV
+          fi
+
+      - name: Download CPP Client
+        if: steps.cache-dependencies.outputs.cache-hit != 'true'
+        run: |
+          pkg/windows/download-cpp-client.bat ${{ matrix.arch }}
+
+      - name: Build Node binaries lib
+        run: |
+          npm install --ignore-scripts
+          npx node-pre-gyp configure --target_arch=${{ env.TARGET }}
+          npx node-pre-gyp build --target_arch=${{ env.TARGET }}
+
+      - name: Test loading Node binaries lib
+        run: |
+          node pkg/load_test.js
+
+      - name: Package Node binaries lib
+        run: |
+          npx node-pre-gyp package --target_arch=${{ env.TARGET }}

package/README.md

@@ -23,112 +23,131 @@
 
 The Pulsar Node.js client can be used to create Pulsar producers and consumers in Node.js.
 
-## Requirements
-
-Pulsar Node.js client library is based on the C++ client library. Follow the instructions for
-[C++ library](https://pulsar.apache.org/docs/en/client-libraries-cpp/) for installing the binaries through
-[RPM](https://pulsar.apache.org/docs/en/client-libraries-cpp/#rpm),
-[Deb](https://pulsar.apache.org/docs/en/client-libraries-cpp/#deb) or
-[Homebrew packages](https://pulsar.apache.org/docs/en/client-libraries-cpp/#macos).
-
-(Note: you will need to install not only the pulsar-client library but also the pulsar-client-dev library)
-
-Also, this library works only in Node.js 10.x or later because it uses the
+This library works only in Node.js 10.x or later because it uses the
 [node-addon-api](https://github.com/nodejs/node-addon-api) module to wrap the C++ library.
 
-## Compatibility
-
-Compatibility between each version of the Node.js client and the C++ client is as follows:
+## Getting Started
 
-| Node.js client | C++ client     |
-|----------------|----------------|
-| 1.0.x          | 2.3.0 or later |
-| 1.1.x          | 2.4.0 or later |
-| 1.2.x          | 2.5.0 or later |
-| 1.3.x          | 2.7.0 or later |
-| 1.4.x - 1.5.x  | 2.8.0 or later |
-
-If an incompatible version of the C++ client is installed, you may fail to build or run this library.
+> **Note**
+>
+> These instructions are only available for versions after 1.8.0. For versions previous to 1.8.0, you need to install the C++ client first. Please switch to the corresponding version branch of this repo to read the specific instructions.
+>
+> To run the examples, skip this section.
 
-## How to install
+To use the Pulsar Node.js client in your project, run:
 
-### Install on windows
+```shell
+npm install pulsar-client
+```
 
-1. Build the Pulsar C++ client on windows.
+or
 
 ```shell
-cmake \
- -A x64 \
- -DBUILD_PYTHON_WRAPPER=OFF -DBUILD_TESTS=OFF \
- -DVCPKG_TRIPLET=x64-windows \
- -DCMAKE_BUILD_TYPE=Release \
- -S .
-cmake --config Release
+yarn add pulsar-client
 ```
 
+Then you can run the following simple end-to-end example:
+
+```javascript
+const Pulsar = require('pulsar-client');
+
+(async () => {
+  // Create a client
+  const client = new Pulsar.Client({
+    serviceUrl: 'pulsar://localhost:6650'
+  });
+
+  // Create a producer
+  const producer = await client.createProducer({
+    topic: 'persistent://public/default/my-topic',
+  });
+
+  // Create a consumer
+  const consumer = await client.subscribe({
+    topic: 'persistent://public/default/my-topic',
+    subscription: 'sub1'
+  });
+
+  // Send a message
+  producer.send({
+    data: Buffer.from("hello")
+  });
+
+  // Receive the message 
+  const msg = await consumer.receive();
+  console.log(msg.getData().toString());
+  consumer.acknowledge(msg);
+
+  await producer.close();
+  await consumer.close();
+  await client.close();
+})();
+```
 
-2. Set the variable `PULSAR_CPP_DIR` with the `pulsar-client-cpp` path in a Windows command tool.
+You should find the output as:
 
-```shell
-# for example
-set PULSAR_CPP_DIR=C:\pulsar\pulsar-client-cpp
 ```
+hello
+```
+
+You can see more examples in the [examples](./examples) directory. However, since these examples might use an API that was not released yet, you need to build this module. See the next section.
 
-3. Set the variable `OS_ARCH` in a Windows command tool, `OS_ARCH` is related to the configuration of VCPKG_TRIPLET on the command line above.(Optional)
+## How to build
+
+First, clone the repository.
 
 ```shell
-set OS_ARCH=x64-windows
+git clone https://github.com/apache/pulsar-client-node.git
+cd pulsar-client-node
 ```
 
-### Install on mac
+Since this client is a [C++ addon](https://nodejs.org/api/addons.html#c-addons) that depends on the [Pulsar C++ client](https://github.com/apache/pulsar-client-cpp), you need to install the C++ client first. You need to ensure there is a C++ compiler that supports C++11 installed in your system.
 
-1. Install the Pulsar C++ client on mac.
+- Install C++ client on Linux:
 
 ```shell
-brew install libpulsar
+build-support/install-cpp-client.sh
 ```
 
-2. Get the installation path of libpulsar
+- Install C++ client on Windows:
 
 ```shell
-brew info libpulsar
+pkg\windows\download-cpp-client.bat
 ```
 
-
-2. Set the variable `PULSAR_CPP_DIR` with the `pulsar-client-cpp` path in a mac command tool.
+- Install C++ client on macOS:
 
 ```shell
-# for example
-export PULSAR_CPP_DIR=/usr/local/Cellar/libpulsar/2.9.1_1
+pkg/mac/build-cpp-deps-lib.sh
+pkg/mac/build-cpp-lib.sh
 ```
 
-
-### Install pulsar-client to your project
+After the C++ client is installed, run the following command to build this C++ addon.
 
 ```shell
-$ npm install pulsar-client
+npm install
 ```
 
-## Sample code
-
-Please refer to [examples](https://github.com/apache/pulsar-client-node/tree/master/examples).
-
-## How to build
-
-### Install dependent npm modules and build Pulsar client library:
+To verify it has been installed successfully, you can run an example like:
 
 ```shell
-$ git clone https://github.com/apache/pulsar-client-node.git
-$ cd pulsar-client-node
-$ npm install
+node examples/producer
 ```
 
-> **Note**
->
-> If you build `pulsar-client-node on` windows, you need to set the variable `PULSAR_CPP_DIR` first, then install npm (run the command `npm install`) in a Windows command-line tool.
-
-### Rebuild Pulsar client library:
+You should find the output as:
 
-```shell
-$ npm run build
 ```
+Sent message: my-message-0
+Sent message: my-message-1
+Sent message: my-message-2
+Sent message: my-message-3
+Sent message: my-message-4
+Sent message: my-message-5
+Sent message: my-message-6
+Sent message: my-message-7
+Sent message: my-message-8
+Sent message: my-message-9
+```
+
+## Documentation
+* Please see https://pulsar.apache.org/docs/client-libraries-node/ for more details about the Pulsar Node.js client.