snowpipe-streaming 1.4.0

package/README.md

@@ -0,0 +1,104 @@
+# Snowflake Streaming Ingest Node.js SDK
+
+A high-performance Node.js SDK for streaming data ingestion into Snowflake, built with Rust for optimal performance and reliability.
+
+- [Overview](#overview)
+- [Supported Platforms](#supported-platforms)
+- [Installation](#installation)
+- [Quick Start](#quick-start)
+- [Features](#features)
+- [Architecture](#architecture)
+- [Dependencies](#dependencies)
+- [Support](#support)
+- [License](#license)
+
+## Overview
+
+The Snowflake Streaming Ingest Node.js SDK provides a Node.js interface for real-time data streaming into Snowflake tables. It leverages a Rust core for high performance while providing a familiar JavaScript API for easy integration.
+
+## Supported Platforms
+
+| OS | Architecture | C Library | Notes |
+|----|-------------|-----------|-------|
+| Linux | x64 | glibc (>= 2.26) | |
+| Linux | x64 | musl | Alpine Linux, etc. |
+| Linux | arm64 | glibc (>= 2.26) | |
+| Linux | arm64 | musl | Alpine Linux, etc. |
+| macOS | arm64 | — | Apple Silicon, macOS 11.0+ |
+| Windows | x64 | MSVC | Windows 10+ |
+
+### Node.js
+- **Node.js 20** or later
+
+## Installation
+
+```bash
+npm install snowpipe-streaming
+```
+
+The correct native binary for your platform is installed automatically via `optionalDependencies`.
+
+## Quick Start
+
+```javascript
+const { createClient } = require("snowpipe-streaming");
+
+// Create a client
+const client = await createClient({
+  clientName: "my-client",
+  dbName: "MY_DB",
+  schemaName: "MY_SCHEMA",
+  pipeName: "MY_PIPE",
+  properties: {
+    account: "your_account",
+    user: "your_user",
+    private_key: "-----BEGIN PRIVATE KEY-----\n...",
+  },
+});
+
+// Open a channel
+const { channel } = await client.openChannel({ name: "my_channel" });
+
+// Insert data
+await channel.insertRows([
+  { id: 1, name: "John Doe", timestamp: "2024-01-01T00:00:00Z" },
+  { id: 2, name: "Jane Smith", timestamp: "2024-01-02T00:00:00Z" },
+]);
+
+// Wait for data to be committed to Snowflake
+await channel.waitForCommit();
+
+// Close resources
+await channel.close();
+await client.close();
+```
+
+## Features
+
+- **High Performance**: Rust-based core for optimal throughput and low latency
+- **Memory Efficient**: Minimal memory footprint with efficient data handling
+- **Automatic Retries**: Built-in retry logic for transient failures
+- **Backpressure Handling**: Intelligent backpressure management
+- **Comprehensive Logging**: Detailed logging for debugging and monitoring
+- **Cross-Platform**: Native support for Linux, macOS, and Windows
+- **Zero Dependencies**: No JavaScript runtime dependencies
+
+## Architecture
+
+The SDK uses a hybrid Rust-Node.js architecture:
+- **Rust Core**: High-performance data processing, networking, and Snowflake communication
+- **Node.js Bindings**: JavaScript API using napi-rs for seamless integration
+- **Platform Packages**: Architecture-specific native binaries distributed as npm optional dependencies
+
+## Dependencies
+
+- **None**: The SDK has zero JavaScript runtime dependencies
+- **Native Extensions**: Rust-based binary modules (automatically installed for your platform)
+
+## Support
+
+- **Documentation**: [Snowpipe Streaming Overview](https://docs.snowflake.com/en/user-guide/snowpipe-streaming/data-load-snowpipe-streaming-overview)
+
+## License
+
+This project is licensed under the Apache License 2.0.

package/package.json

@@ -0,0 +1,37 @@
+{
+  "name": "snowpipe-streaming",
+  "version": "1.4.0",
+  "description": "Snowflake Streaming Ingest SDK for Node.js",
+  "main": "src/index.js",
+  "types": "src/index.d.ts",
+  "files": [
+    "src/"
+  ],
+  "keywords": [
+    "snowflake",
+    "snowpipe",
+    "streaming",
+    "ingest"
+  ],
+  "author": {
+    "name": "Snowflake Computing, Inc.",
+    "email": "support@snowflake.com",
+    "url": "https://www.snowflake.com/"
+  },
+  "engines": {
+    "node": ">=20"
+  },
+  "license": "Apache-2.0",
+  "devDependencies": {
+    "@types/node": "^25.6.0",
+    "typedoc": "^0.28.19"
+  },
+  "optionalDependencies": {
+    "snowpipe-streaming-linux-x64-gnu": "1.4.0",
+    "snowpipe-streaming-linux-x64-musl": "1.4.0",
+    "snowpipe-streaming-linux-arm64-gnu": "1.4.0",
+    "snowpipe-streaming-linux-arm64-musl": "1.4.0",
+    "snowpipe-streaming-darwin-arm64": "1.4.0",
+    "snowpipe-streaming-win32-x64-msvc": "1.4.0"
+  }
+}

package/src/channel.js

@@ -0,0 +1,241 @@
+// Copyright 2026 Snowflake Inc.
+// SPDX-License-Identifier: Apache-2.0
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+"use strict";
+
+const { callFFI, StreamingIngestError } = require("./errors.js");
+const { ChannelStatus } = require("./channel_status.js");
+
+const WAIT_FOR_COMMIT_CHECK_INTERVAL_MS = 1000;
+
+/**
+ * Streaming ingest channel for ingesting data into a Snowflake table.
+ *
+ * Channels are created via {@link StreamingIngestClient#openChannel},
+ * not directly.
+ */
+class StreamingIngestChannel {
+  /** @param {object} nativeChannel - Internal JsChannel from FFI */
+  constructor(nativeChannel) {
+    if (!(nativeChannel && typeof nativeChannel === "object")) {
+      throw new TypeError(
+        "StreamingIngestChannel cannot be instantiated directly. " +
+          "Use client.openChannel() instead.",
+      );
+    }
+    this._native = nativeChannel;
+  }
+
+  /** @returns {string} The channel name */
+  get channelName() {
+    return this._native.channelName;
+  }
+
+  /** @returns {string} The database name */
+  get dbName() {
+    return this._native.dbName;
+  }
+
+  /** @returns {string} The schema name */
+  get schemaName() {
+    return this._native.schemaName;
+  }
+
+  /** @returns {string} The pipe name */
+  get pipeName() {
+    return this._native.pipeName;
+  }
+
+  /** @returns {boolean} True if the channel is closed */
+  get isClosed() {
+    return this._native.isClosed;
+  }
+
+  /**
+   * Get the binary input format for this channel.
+   * @returns {string} One of: "BASE64", "HEX", "UTF-8"
+   */
+  get binaryInputFormat() {
+    return this._native.binaryInputFormat;
+  }
+
+  /**
+   * Convert Buffer/Uint8Array values in a row according to binaryInputFormat.
+   * For HEX, binary values become hex-encoded strings.
+   * For UTF-8, binary values are decoded as UTF-8 strings.
+   * Only top-level values are converted; nested structures are left as-is.
+   * @param {Object} row - Row object
+   * @returns {Object} Row with binary values converted
+   * @private
+   */
+  _convertBytesInRow(row) {
+    const result = {};
+    for (const [key, value] of Object.entries(row)) {
+      if (Buffer.isBuffer(value) || value instanceof Uint8Array) {
+        const buf = Buffer.isBuffer(value) ? value : Buffer.from(value);
+        if (this.binaryInputFormat === "HEX") {
+          result[key] = buf.toString("hex");
+        } else if (this.binaryInputFormat === "UTF-8") {
+          result[key] = buf.toString("utf8");
+        } else {
+          // BASE64: convert to base64 string for JSON serialization
+          result[key] = buf.toString("base64");
+        }
+      } else {
+        result[key] = value;
+      }
+    }
+    return result;
+  }
+
+  /**
+   * Append a single row. Convenience wrapper for appendRows.
+   * @param {Object} row - Row data as key-value pairs
+   * @param {string} [offsetToken] - Offset token for this row
+   * @returns {Promise<void>}
+   */
+  async appendRow(row, offsetToken) {
+    await this.appendRows([row], offsetToken, offsetToken);
+  }
+
+  /**
+   * Append multiple rows to the channel.
+   * Rows are serialized to NDJSON (newline-delimited JSON) on the JS side.
+   * @param {Object[]} rows - Array of row objects
+   * @param {string} [startOffsetToken] - Start offset token for this batch
+   * @param {string} [endOffsetToken] - End offset token for this batch
+   * @returns {Promise<void>}
+   */
+  async appendRows(rows, startOffsetToken, endOffsetToken) {
+    if (!Array.isArray(rows)) {
+      throw new TypeError("rows must be an array");
+    }
+    // Empty arrays are handled by the Rust core, which returns a proper InvalidRequest error
+    if (rows.length === 0) {
+      // Pass empty buffer to Rust core to get proper error message
+      return callFFI(() =>
+        this._native.appendRows(Buffer.alloc(0), 0, startOffsetToken, endOffsetToken),
+      );
+    }
+    // Convert binary values according to binaryInputFormat before JSON serialization
+    const ndjson = rows.map((r) => JSON.stringify(this._convertBytesInRow(r))).join("\n");
+    const buffer = Buffer.from(ndjson, "utf8");
+    return callFFI(() =>
+      this._native.appendRows(buffer, rows.length, startOffsetToken, endOffsetToken),
+    );
+  }
+
+  /**
+   * Close the channel. Idempotent — safe to call multiple times.
+   * @param {{drop?: boolean, waitForFlush?: boolean, timeoutMs?: number}} [options]
+   * @returns {Promise<void>}
+   */
+  async close(options) {
+    const nativeOpts = options
+      ? { drop: options.drop, waitForFlush: options.waitForFlush, timeoutMs: options.timeoutMs }
+      : undefined;
+    return callFFI(() => this._native.close(nativeOpts));
+  }
+
+  /**
+   * Trigger a flush of this channel (non-blocking).
+   * @throws {StreamingIngestError} If the flush cannot be initiated
+   */
+  initiateFlush() {
+    try {
+      this._native.initiateFlush();
+    } catch (e) {
+      throw StreamingIngestError.from(e);
+    }
+  }
+
+  /**
+   * Wait for this channel to flush.
+   * @param {{timeoutMs?: number}} [options]
+   * @returns {Promise<void>}
+   */
+  async waitForFlush(options) {
+    return callFFI(() => this._native.waitForFlush(options?.timeoutMs));
+  }
+
+  /**
+   * Get the current status of this channel.
+   * @returns {Promise<ChannelStatus>}
+   */
+  async getChannelStatus() {
+    const raw = await callFFI(() => this._native.getChannelStatus());
+    return new ChannelStatus(raw);
+  }
+
+  /**
+   * Get the latest committed offset token.
+   * @returns {Promise<string|null>}
+   */
+  async getLatestCommittedOffsetToken() {
+    const status = await this.getChannelStatus();
+    return status.latestCommittedOffsetToken;
+  }
+
+  /**
+   * Wait for a specific offset token to be committed.
+   * Polls getChannelStatus() every 1 second until tokenChecker returns true.
+   * @param {function(string|null): boolean} tokenChecker - Returns true when the desired token is committed
+   * @param {{timeoutMs?: number}} [options]
+   * @returns {Promise<void>}
+   */
+  async waitForCommit(tokenChecker, options) {
+    if (typeof tokenChecker !== "function") {
+      throw new TypeError("tokenChecker must be a function");
+    }
+    const timeoutMs = options?.timeoutMs;
+    if (timeoutMs !== undefined && timeoutMs < 0) {
+      throw new RangeError("timeoutMs cannot be negative");
+    }
+
+    const startTime = Date.now();
+
+    while (true) {
+      const status = await this.getChannelStatus();
+
+      if (tokenChecker(status.latestCommittedOffsetToken)) {
+        return;
+      }
+
+      if (status.statusCode.toUpperCase() !== "SUCCESS") {
+        throw new StreamingIngestError(
+          "InvalidChannelError",
+          `Channel is invalid in the remote with status: ${status.statusCode}`,
+          409,
+          "Conflict",
+        );
+      }
+
+      if (timeoutMs !== undefined && Date.now() - startTime >= timeoutMs) {
+        throw new Error("Wait for commit timed out");
+      }
+
+      await new Promise((resolve) => setTimeout(resolve, WAIT_FOR_COMMIT_CHECK_INTERVAL_MS));
+    }
+  }
+
+  /** Support `await using channel = ...` (Node 20+) */
+  async [Symbol.asyncDispose]() {
+    if (!this.isClosed) {
+      await this.close();
+    }
+  }
+}
+
+module.exports = { StreamingIngestChannel };

package/src/channel_status.js

@@ -0,0 +1,179 @@
+// Copyright 2026 Snowflake Inc.
+// SPDX-License-Identifier: Apache-2.0
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+"use strict";
+
+/**
+ * Channel status information from Snowflake server.
+ *
+ * This class wraps the raw status data returned from the Rust FFI layer
+ * and provides a clean JavaScript API with proper getters.
+ */
+class ChannelStatus {
+  /**
+   * Create a ChannelStatus from raw FFI data.
+   * @param {Object} data - Raw status data from FFI (NodejsChannelStatus struct)
+   */
+  constructor(data) {
+    if (!data || typeof data !== "object") {
+      throw new TypeError("ChannelStatus requires a data object");
+    }
+    this._databaseName = data.databaseName;
+    this._schemaName = data.schemaName;
+    this._pipeName = data.pipeName;
+    this._channelName = data.channelName;
+    this._statusCode = data.statusCode;
+    this._latestCommittedOffsetToken = data.latestCommittedOffsetToken;
+    this._createdOnMs = data.createdOnMs;
+    this._rowsInserted = data.rowsInserted;
+    this._rowsParsed = data.rowsParsed;
+    this._rowsErrorCount = data.rowsErrorCount;
+    this._lastErrorOffsetUpperBound = data.lastErrorOffsetUpperBound;
+    this._lastErrorMessage = data.lastErrorMessage;
+    this._lastErrorTimestampMs = data.lastErrorTimestampMs;
+    this._snowflakeAvgProcessingLatencyMs = data.snowflakeAvgProcessingLatencyMs;
+    this._lastRefreshedOnMs = data.lastRefreshedOnMs;
+  }
+
+  /**
+   * Get the database name.
+   * @returns {string}
+   */
+  get databaseName() {
+    return this._databaseName;
+  }
+
+  /**
+   * Get the schema name.
+   * @returns {string}
+   */
+  get schemaName() {
+    return this._schemaName;
+  }
+
+  /**
+   * Get the pipe name.
+   * @returns {string}
+   */
+  get pipeName() {
+    return this._pipeName;
+  }
+
+  /**
+   * Get the channel name.
+   * @returns {string}
+   */
+  get channelName() {
+    return this._channelName;
+  }
+
+  /**
+   * Get the status code from Snowflake server.
+   * @returns {string}
+   */
+  get statusCode() {
+    return this._statusCode;
+  }
+
+  /**
+   * Get the latest committed offset token.
+   * @returns {string|null}
+   */
+  get latestCommittedOffsetToken() {
+    return this._latestCommittedOffsetToken;
+  }
+
+  /**
+   * Get the created on timestamp as a Date.
+   * @returns {Date}
+   */
+  get createdOn() {
+    return this._createdOnMs != null ? new Date(this._createdOnMs) : null;
+  }
+
+  /**
+   * Get the number of rows inserted.
+   * @returns {number}
+   */
+  get rowsInsertedCount() {
+    return this._rowsInserted;
+  }
+
+  /**
+   * Get the number of rows parsed.
+   * @returns {number}
+   */
+  get rowsParsedCount() {
+    return this._rowsParsed;
+  }
+
+  /**
+   * Get the number of rows with errors.
+   * @returns {number}
+   */
+  get rowsErrorCount() {
+    return this._rowsErrorCount;
+  }
+
+  /**
+   * Get the last error offset token upper bound.
+   * @returns {string|null}
+   */
+  get lastErrorOffsetTokenUpperBound() {
+    return this._lastErrorOffsetUpperBound;
+  }
+
+  /**
+   * Get the last error message.
+   * @returns {string|null}
+   */
+  get lastErrorMessage() {
+    return this._lastErrorMessage;
+  }
+
+  /**
+   * Get the last error timestamp as a Date.
+   * @returns {Date|null}
+   */
+  get lastErrorTimestamp() {
+    return this._lastErrorTimestampMs != null ? new Date(this._lastErrorTimestampMs) : null;
+  }
+
+  /**
+   * Get the Snowflake average processing latency in milliseconds.
+   * @returns {number|null}
+   */
+  get serverAvgProcessingLatencyMs() {
+    return this._snowflakeAvgProcessingLatencyMs;
+  }
+
+  /**
+   * Get the last refreshed on timestamp as a Date.
+   * @returns {Date}
+   */
+  get lastRefreshedOn() {
+    return this._lastRefreshedOnMs != null ? new Date(this._lastRefreshedOnMs) : null;
+  }
+
+  /**
+   * Returns a string representation of the channel status.
+   * @returns {string}
+   */
+  toString() {
+    return `Channel '${this._channelName}': ${this._statusCode}`;
+  }
+}
+
+module.exports = { ChannelStatus };