pulsar-client 1.7.0 → 1.8.0-rc1

package/{.github/workflows/nodejs.yml → build-support/sign-files.sh}

@@ -1,3 +1,4 @@
+#!/usr/bin/env bash
 #
 # Licensed to the Apache Software Foundation (ASF) under one
 # or more contributor license agreements.  See the NOTICE file
@@ -16,16 +17,16 @@
 # specific language governing permissions and limitations
 # under the License.
 #
-name: Node.js
-on: [pull_request]
-jobs:
-  build:
-    name: Build
-    runs-on: ubuntu-latest
-    steps:
-      - name: Check out code into the Node.js module directory
-        uses: actions/checkout@v2
 
-      - name: Test
-        run: |
-          ./docker-tests.sh
+set -e
+
+FILES=$*
+
+for FILE in $FILES
+do
+   echo "Signing $FILE"
+   gpg --armor --output $FILE.asc --detach-sig $FILE
+
+   # SHA-512 signature
+   shasum -a 512 $FILE > $FILE.sha512
+done

package/build-support/stage-release.sh

@@ -0,0 +1,44 @@
+#!/usr/bin/env bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+set -e -x
+
+if [ $# -neq 2 ]; then
+    echo "Usage: $0 \$DEST_PATH \$WORKFLOW_ID"
+    exit 1
+fi
+
+DEST_PATH=$(readlink -f $1)
+WORKFLOW_ID=$2
+
+pushd $(dirname "$0")
+PULSAR_NODE_PATH=$(git rev-parse --show-toplevel)
+popd
+
+mkdir -p $DEST_PATH
+
+cd $PULSAR_NODE_PATH
+
+build-support/download-release-artifacts.py $WORKFLOW_ID $DEST_PATH
+build-support/generate-source-archive.sh $DEST_PATH
+
+# Sign all files
+cd $DEST_PATH
+find . -type f | xargs $PULSAR_NODE_PATH/build-support/sign-files.sh

package/dependencies.yaml

@@ -0,0 +1,28 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+# Only used for MacOS builds
+boost: 1.80.0
+cmake: 3.24.2
+protobuf: 3.20.0
+zlib: 1.2.13
+zstd: 1.5.2
+snappy: 1.1.9
+openssl: 1.1.1s
+curl: 7.85.0

package/docs/release-process.md

@@ -0,0 +1,242 @@
+This page contains instructions for Pulsar committers on how to perform a release.
+
+## Making the release
+
+The steps for releasing are as follows:
+1. Check and Add TypeScript Types
+2. Create the release branch
+3. Update package version and tag
+4. Sign and stage the artifacts
+5. Move master branch to next version
+6. Write release notes
+7. Run the vote
+8. Promote the release
+9. Update release notes
+10. Announce the release
+ 
+## Requirements
+If you haven't already done it, [create and publish the GPG key](https://pulsar.apache.org/contribute/releasing/create-gpg-keys) to sign the release artifacts.
+
+## Steps in detail
+
+#### 1. Check and Add TypeScript Types
+
+In order to use added features and options in TypeScript code, we need to add their type definitions.
+
+Check if there are any Pull Requests with the `types-required` label or not.
+If yes, add their type definitions (e.g. https://github.com/apache/pulsar-client-node/pull/157).
+If no, we can then proceed to next step.
+
+#### 2. Create the release branch
+
+We are going to create a branch from `master` to `branch-1.X` where the tag will be generated
+and where new fixes will be applied as part of the maintenance for the release.
+
+The branch needs only to be created when creating minor releases, and not for patch releases.
+
+Eg: When creating `v1.1.0` release, will be creating the branch `branch-1.1`,
+but for `v1.1.1` we would keep using the old `branch-1.1`.
+
+In these instructions, I'm referring to an fictitious release `1.X.0`.
+Change the release version in the examples accordingly with the real version.
+
+It is recommended to create a fresh clone of the repository to avoid any local files to interfere
+in the process:
+
+```sh
+$ git clone git@github.com:apache/pulsar-client-node.git
+$ cd pulsar-client-node
+$ git checkout -b branch-1.X origin/master
+```
+
+#### 3. Update package version and tag
+
+During the release process, we are going to initially create "candidate" tags:
+```sh
+# Bump to the release version (the suffix "-rc.0" is removed)
+$ npm version patch --no-git-tag-version
+$ git add .
+$ git commit -m 'Release v1.x.0'
+
+# Create a "candidate" tag
+$ export GPG_TTY=$(tty)
+$ git tag -u $USER@apache.org v1.X.0-rc.1 -m 'Release v1.X.0-rc.1'
+
+# Push both the branch and the tag to GitHub repo
+$ git push origin branch-1.X
+$ git push origin v1.X.0-rc.1
+```
+
+#### 4. Sign and stage the artifacts
+The src artifact need to be signed and uploaded to the dist SVN repository for staging. See [here](https://github.com/apache/pulsar/wiki/Create-GPG-keys-to-sign-release-artifacts) for how to setup gpg.
+
+When the build is pushed to the Apache repo, a Github action job will run and build all the binary artifacts.
+
+The build will be available at `https://github.com/apache/pulsar-client-node/actions`
+
+Once the workflow is completed (ETA ~20 min), the artifacts will be available for download.
+
+Record the id of the workflow as `WORKFLOW_ID`.
+
+All the artifacts need to be signed and uploaded to the dist SVN repository for staging.
+
+Before running the script, make sure that the `user@apache.org` code signing key is the default gpg signing key. One way to ensure this is to create/edit file `~/.gnupg/gpg.conf` and add a line
+
+```sh
+default-key <key fingerprint>
+```
+
+where` <key fingerprint>` should be replaced with the private key fingerprint for the `user@apache.org` key. The key fingerprint can be found in `gpg -K` output.
+
+```sh
+$ svn co https://dist.apache.org/repos/dist/dev/pulsar/pulsar-client-node pulsar-dist-dev
+$ cd pulsar-dist-dev
+
+# '-candidate-1' needs to be incremented in case of multiple iterations in getting
+#    to the final release)
+$ svn mkdir pulsar-client-node-1.X.0-candidate-1
+
+# Generate token from here: https://github.com/settings/tokens
+$ export GITHUB_TOKEN=${Your github token} 
+$ $PULSAR_PATH/build-support/stage-release.sh . $WORKFLOW_ID
+
+$ svn add *
+$ svn ci -m 'Staging artifacts and signature for Pulsar Node.js client release 1.X.0-candidate-1'
+```
+
+#### 5. Move master branch to next version
+
+We need to move master version to next iteration `X + 1`.
+
+```sh
+$ git checkout master
+$ npm version preminor --preid=rc
+```
+
+Since this needs to be merged in `master`, we need to follow the regular process
+and create a Pull Request on GitHub.
+
+#### 6. Write release notes
+
+Check the milestone in GitHub associated with the release.
+https://github.com/apache/pulsar-client-node/milestones?closed=1
+
+In the release item, add the list of most important changes that happened in the release
+and a link to the associated milestone, with the complete list of all the changes.
+https://github.com/apache/pulsar-client-node/releases
+
+#### 7. Run the vote
+
+Send an email on the Pulsar Dev mailing list:
+
+```
+To: dev@pulsar.apache.org
+Subject: [VOTE] Pulsar Node.js Client Release 1.X.0 Candidate 1
+
+Hi everyone,
+Please review and vote on the release candidate #1 for the version 1.X.0, as follows:
+[ ] +1, Approve the release
+[ ] -1, Do not approve the release (please provide specific comments)
+
+This is the first release candidate for Apache Pulsar Node.js client, version 1.X.0.
+
+It fixes the following issues:
+https://github.com/apache/pulsar-client-node/milestone/1?closed=1
+
+Please download the source files and review this release candidate:
+- Review release notes
+- Download the source package (verify shasum and asc) and follow the README.md to build and run the Pulsar Node.js client.
+
+The vote will be open for at least 72 hours. It is adopted by majority approval, with at least 3 PMC affirmative votes.
+
+Source files:
+https://dist.apache.org/repos/dist/dev/pulsar/pulsar-client-node/pulsar-client-node-1.X.0-candidate-1/
+
+Pulsar's KEYS file containing PGP keys we use to sign the release:
+https://dist.apache.org/repos/dist/dev/pulsar/KEYS
+
+SHA-512 checksum:
+5f6c7e1a096a3ae66eee71c552af89532ed86bf94da3f3d49836c080426ee5dcaabeda440a989d51772d2e67e2dca953eeee9ea83cfbc7c2a0847a0ec04b310f  pulsar-client-node-1.X.0.tar.gz
+
+The tag to be voted upon:
+v1.X.0-rc.1
+https://github.com/apache/pulsar-client-node/releases/tag/v1.X.0-rc.1
+```
+
+The vote should be open for at least 72 hours (3 days). Votes from Pulsar PMC members
+will be considered binding, while anyone else is encouraged to verify the release and
+vote as well.
+
+If the release is approved here, we can then proceed to next step.
+
+#### 8. Promote the release
+
+Create the final git tag:
+```sh
+$ git checkout branch-1.X
+$ export GPG_TTY=$(tty)
+$ git tag -u $USER@apache.org v1.X.0 -m 'Release v1.X.0'
+$ git push origin v1.X.0
+```
+
+Publish the release package:
+```sh
+# Create or verify a user account in the npm registry
+$ npm adduser
+
+Username: foobar
+Password: ********
+Email: (this IS public) foobar@apache.org
+
+# Publish the package to the npm registry
+$ npm publish
+
+# If you don't have permission to publish `pulsar-client` to the npm registry,
+# ask other committers to grant that permission.
+```
+
+Promote the artifacts on the release location (need PMC permissions):
+```sh
+$ svn mv -m 'Release Pulsar Node.js client 1.X.0' \
+  https://dist.apache.org/repos/dist/dev/pulsar/pulsar-client-node/pulsar-client-node-1.X.0-candidate-1 \
+  https://dist.apache.org/repos/dist/release/pulsar/pulsar-client-node/pulsar-client-node-1.X.0
+
+# Remove the old releases (if any)
+# We only need the latest release there, older releases are available through the Apache archive
+$ svn rm -m 'Remove the old release' \
+  https://dist.apache.org/repos/dist/release/pulsar/pulsar-client-node/pulsar-client-node-1.Y.0
+```
+
+#### 9. Update release notes
+
+Add the release notes there:
+https://github.com/apache/pulsar-client-node/releases
+
+#### 10. Announce the release
+
+Once the release artifact is available in the npm registry, we need to announce the release.
+
+Send an email on these lines:
+```
+To: dev@pulsar.apache.org, users@pulsar.apache.org, announce@apache.org
+Subject: [ANNOUNCE] Apache Pulsar Node.js client 1.X.0 released
+
+The Apache Pulsar team is proud to announce Apache Pulsar Node.js client version 1.X.0.
+
+Pulsar is a highly scalable, low latency messaging platform running on
+commodity hardware. It provides simple pub-sub semantics over topics,
+guaranteed at-least-once delivery of messages, automatic cursor management for
+subscribers, and cross-datacenter replication.
+
+For Pulsar Node.js client release details and downloads, visit:
+https://www.npmjs.com/package/pulsar-client
+
+Release Notes are at:
+https://github.com/apache/pulsar-client-node/releases
+
+We would like to thank the contributors that made the release possible.
+
+Regards,
+
+The Pulsar Team
+```

package/examples/consumer.js

@@ -17,7 +17,7 @@
  * under the License.
  */
 
-const Pulsar = require('pulsar-client');
+const Pulsar = require('../');
 
 (async () => {
   // Create a client

package/examples/consumer_listener.js

@@ -17,7 +17,7 @@
  * under the License.
  */
 
-const Pulsar = require('pulsar-client');
+const Pulsar = require('../');
 
 (async () => {
   // Create a client

package/examples/consumer_tls_auth.js

@@ -17,7 +17,7 @@
  * under the License.
  */
 
-const Pulsar = require('pulsar-client');
+const Pulsar = require('../');
 
 (async () => {
   const auth = new Pulsar.AuthenticationTls({

package/examples/custom_logger.js

@@ -0,0 +1,60 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+const Pulsar = require('..');
+
+(async () => {
+    // Create a client
+    const client = new Pulsar.Client({
+        serviceUrl: 'pulsar://localhost:6650',
+        operationTimeoutSeconds: 30,
+    });
+
+    Pulsar.Client.setLogHandler((level, file, line, message) => {
+        console.log('[%s][%s:%d] %s', Pulsar.LogLevel.toString(level), file, line, message);
+    });
+
+    // Create a producer
+    const producer = await client.createProducer({
+        topic: 'persistent://public/default/my-topic',
+        sendTimeoutMs: 30000,
+        batchingEnabled: true,
+    });
+
+    // Create a consumer
+    const consumer = await client.subscribe({
+        topic: 'persistent://public/default/my-topic',
+        subscription: 'sub1',
+        subscriptionType: 'Shared',
+        ackTimeoutMs: 10000,
+    });
+
+    const msg = `my-message`;
+    await producer.send({
+        data: Buffer.from(msg),
+    });
+
+    const receivedMsg = await consumer.receive();
+    console.log(receivedMsg.getData().toString());
+    await consumer.acknowledge(receivedMsg);
+
+    await producer.close();
+    await consumer.close();
+    await client.close();
+})();

package/examples/encryption-consumer.js

@@ -17,7 +17,7 @@
  * under the License.
  */
 
-const Pulsar = require('pulsar-client');
+const Pulsar = require('../');
 
 (async () => {
   // Create a client

package/examples/encryption-producer.js

@@ -17,7 +17,7 @@
  * under the License.
  */
 
-const Pulsar = require('pulsar-client');
+const Pulsar = require('../');
 
 (async () => {
   // Create a client

package/examples/producer.js

@@ -17,7 +17,7 @@
  * under the License.
  */
 
-const Pulsar = require('pulsar-client');
+const Pulsar = require('../');
 
 (async () => {
   // Create a client

package/examples/producer_tls_auth.js

@@ -17,7 +17,7 @@
  * under the License.
  */
 
-const Pulsar = require('pulsar-client');
+const Pulsar = require('../');
 
 (async () => {
   const auth = new Pulsar.AuthenticationTls({

package/examples/reader.js

@@ -17,7 +17,7 @@
  * under the License.
  */
 
-const Pulsar = require('pulsar-client');
+const Pulsar = require('../');
 
 (async () => {
   // Create a client

package/examples/reader_listener.js

@@ -17,7 +17,7 @@
  * under the License.
  */
 
-const Pulsar = require('pulsar-client');
+const Pulsar = require('../');
 
 (async () => {
   // Create a client

package/index.d.ts

@@ -60,6 +60,7 @@ export interface ProducerConfig {
   publicKeyPath?: string;
   encryptionKey?: string;
   cryptoFailureAction?: ProducerCryptoFailureAction;
+  chunkingEnabled?: boolean;
 }
 
 export class Producer {
@@ -88,6 +89,8 @@ export interface ConsumerConfig {
   readCompacted?: boolean;
   privateKeyPath?: string;
   cryptoFailureAction?: ConsumerCryptoFailureAction;
+  maxPendingChunkedMessage?: number;
+  autoAckOldestChunkedMessageOnQueueFull?: number;
 }
 
 export class Consumer {
@@ -98,6 +101,8 @@ export class Consumer {
   negativeAcknowledgeId(messageId: MessageId): void;
   acknowledgeCumulative(message: Message): Promise<null>;
   acknowledgeCumulativeId(messageId: MessageId): Promise<null>;
+  seek(messageId: MessageId): Promise<null>;
+  seekTimestamp(timestamp: number): Promise<null>;
   isConnected(): boolean;
   close(): Promise<null>;
   unsubscribe(): Promise<null>;
@@ -117,6 +122,8 @@ export class Reader {
   readNext(timeout?: number): Promise<Message>;
   hasNext(): boolean;
   isConnected(): boolean;
+  seek(messageId: MessageId): Promise<null>;
+  seekTimestamp(timestamp: number): Promise<null>;
   close(): Promise<null>;
 }
 

package/index.js

@@ -17,7 +17,7 @@
  * under the License.
  */
 
-const PulsarBinding = require('bindings')('Pulsar');
+const PulsarBinding = require('./src/pulsar-binding');
 const AuthenticationTls = require('./src/AuthenticationTls.js');
 const AuthenticationAthenz = require('./src/AuthenticationAthenz.js');
 const AuthenticationToken = require('./src/AuthenticationToken.js');
@@ -28,6 +28,7 @@ const LogLevel = {
   INFO: 1,
   WARN: 2,
   ERROR: 3,
+  toString: (level) => Object.keys(LogLevel).find((key) => LogLevel[key] === level),
 };
 
 const Pulsar = {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "pulsar-client",
-  "version": "1.7.0",
+  "version": "1.8.0-rc1",
   "description": "Pulsar Node.js client",
   "main": "index.js",
   "types": "index.d.ts",
@@ -9,17 +9,17 @@
     "example": "examples"
   },
   "scripts": {
-    "install": "node-gyp rebuild",
-    "clean": "node-gyp clean",
-    "configure": "node-gyp configure",
+    "install": "node-pre-gyp install --fallback-to-build",
+    "configure": "node-pre-gyp configure",
+    "build": "npm run format && node-pre-gyp build",
+    "build:debug": "npm run format && node-pre-gyp rebuild --debug",
+    "clean": "node-pre-gyp clean",
     "lint": "clang-format-lint src/*.cc src/*.h && eslint --ext .js .",
     "format": "clang-format-lint --fix src/*.cc src/*.h && eslint --fix --ext .js .",
     "dtslint": "dtslint .",
-    "build": "npm run format && node-gyp rebuild",
-    "build:debug": "npm run format && node-gyp rebuild --debug",
     "license:report": "mkdir -p report && grunt grunt-license-report",
     "license:addheader": "license-check-and-add",
-    "test": "jest --verbose"
+    "test": "jest --verbose --detectOpenHandles"
   },
   "repository": {
     "type": "git",
@@ -52,9 +52,17 @@
     "typescript": "^4.1.3"
   },
   "dependencies": {
+    "@mapbox/node-pre-gyp": "^1.0.9",
     "bindings": "^1.5.0",
     "node-addon-api": "^4.3.0"
   },
+  "binary": {
+    "module_name": "Pulsar",
+    "module_path": "./lib/binding/",
+    "host": "https://dist.apache.org/repos/dist/release/pulsar/pulsar-client-node/",
+    "remote_path": "pulsar-client-node-{version}",
+    "package_name": "napi-{platform}-{libc}-{arch}.tar.gz"
+  },
   "license-check-and-add-config": {
     "folder": ".",
     "license": "license-header.txt",

package/pkg/build-napi-inside-docker.sh

@@ -0,0 +1,31 @@
+#!/usr/bin/env bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+set -e -x
+
+cd /pulsar-client-node
+
+build-support/install-cpp-client.sh
+
+npm install --ignore-scripts
+npx node-pre-gyp configure
+npx node-pre-gyp build
+npx node-pre-gyp package
+node pkg/load_test.js

package/pkg/linux_glibc/Dockerfile

@@ -0,0 +1,33 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+ARG NODE_VERSION
+
+FROM node:${NODE_VERSION}-bullseye
+
+ARG NODE_VERSION
+
+RUN apt-get update -y && \
+     apt-get install -y \
+        curl \
+        g++ \
+        make \
+        python3
+
+CMD ["sh"]