snowpipe-streaming 1.4.0

package/src/client.js

@@ -0,0 +1,216 @@
+// Copyright 2026 Snowflake Inc.
+// SPDX-License-Identifier: Apache-2.0
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+"use strict";
+
+const { callFFI, StreamingIngestError } = require("./errors.js");
+const { ChannelStatus } = require("./channel_status.js");
+const { StreamingIngestChannel } = require("./channel.js");
+
+/**
+ * Streaming ingest client for creating channels to ingest data into Snowflake.
+ *
+ * Use {@link createClient} to create an instance — the constructor is not public.
+ *
+ * @example
+ * const client = await createClient({
+ *   clientName: "my-client",
+ *   dbName: "MY_DB",
+ *   schemaName: "MY_SCHEMA",
+ *   pipeName: "MY_PIPE",
+ *   properties: {
+ *     account: "myaccount",
+ *     user: "myuser",
+ *     private_key: "-----BEGIN PRIVATE KEY-----\n...",
+ *   },
+ * });
+ *
+ * const { channel, status } = await client.openChannel({ name: "ch1" });
+ * // ... insert rows via channel ...
+ * await client.close();
+ */
+class StreamingIngestClient {
+  /** @param {object} nativeClient - Internal JsClient from FFI */
+  constructor(nativeClient) {
+    this._native = nativeClient;
+  }
+
+  /** @returns {string} The client name */
+  get clientName() {
+    return this._native.clientName;
+  }
+
+  /** @returns {string} The database name */
+  get dbName() {
+    return this._native.dbName;
+  }
+
+  /** @returns {string} The schema name */
+  get schemaName() {
+    return this._native.schemaName;
+  }
+
+  /** @returns {string} The pipe name */
+  get pipeName() {
+    return this._native.pipeName;
+  }
+
+  /** @returns {boolean} True if the client is closed */
+  get isClosed() {
+    return this._native.isClosed;
+  }
+
+  /**
+   * Get the binary input format for this client.
+   * @returns {string} One of: "BASE64", "HEX", "UTF-8"
+   */
+  get binaryInputFormat() {
+    return this._native.binaryInputFormat;
+  }
+
+  /**
+   * Close the client.
+   * @param {{waitForFlush?: boolean, timeoutMs?: number}} [options]
+   * @returns {Promise<void>}
+   */
+  async close(options) {
+    const nativeOpts = options
+      ? { waitForFlush: options.waitForFlush, timeoutMs: options.timeoutMs }
+      : undefined;
+    return callFFI(() => this._native.close(nativeOpts));
+  }
+
+  /**
+   * Open a channel for streaming data.
+   * @param {{name: string, offsetToken?: string}} options
+   * @returns {Promise<{channel: StreamingIngestChannel, status: ChannelStatus}>}
+   */
+  async openChannel(options) {
+    const nativeChannel = await callFFI(() =>
+      this._native.openChannel(options.name, options.offsetToken),
+    );
+    const rawStatus = nativeChannel.channelStatus;
+    return {
+      channel: new StreamingIngestChannel(nativeChannel),
+      status: rawStatus ? new ChannelStatus(rawStatus) : null,
+    };
+  }
+
+  /**
+   * Drop a channel by name (server-side removal).
+   * @param {string} channelName
+   * @returns {Promise<void>}
+   */
+  async dropChannel(channelName) {
+    return callFFI(() => this._native.dropChannel(channelName));
+  }
+
+  /**
+   * Get the status of multiple channels.
+   * @param {string[]} channelNames
+   * @returns {Promise<Object.<string, ChannelStatus>>}
+   */
+  async getChannelStatuses(channelNames) {
+    const raw = await callFFI(() => this._native.getChannelStatuses(channelNames));
+    const result = {};
+    for (const [name, data] of Object.entries(raw)) {
+      result[name] = new ChannelStatus(data);
+    }
+    return result;
+  }
+
+  /**
+   * Get the latest committed offset tokens for multiple channels.
+   * @param {string[]} channelNames
+   * @returns {Promise<Object.<string, string|null>>}
+   */
+  async getLatestCommittedOffsetTokens(channelNames) {
+    const statuses = await this.getChannelStatuses(channelNames);
+    const result = {};
+    for (const [name, status] of Object.entries(statuses)) {
+      result[name] = status.latestCommittedOffsetToken;
+    }
+    return result;
+  }
+
+  /**
+   * Trigger a flush of all channels (non-blocking).
+   * @throws {StreamingIngestError} If the flush cannot be initiated
+   */
+  initiateFlush() {
+    try {
+      this._native.initiateFlush();
+    } catch (e) {
+      throw StreamingIngestError.from(e);
+    }
+  }
+
+  /**
+   * Wait for all channels to flush.
+   * @param {{timeoutMs?: number}} [options]
+   * @returns {Promise<void>}
+   */
+  async waitForFlush(options) {
+    return callFFI(() => this._native.waitForFlush(options?.timeoutMs));
+  }
+
+  /** Support `await using client = ...` (Node 20+) */
+  async [Symbol.asyncDispose]() {
+    if (!this.isClosed) {
+      await this.close();
+    }
+  }
+}
+
+/**
+ * Create a new streaming ingest client.
+ *
+ * @param {object} native - The native addon instance (provided by index.js)
+ * @param {{clientName: string, dbName: string, schemaName: string, pipeName: string, profilePath?: string, properties?: Object.<string, string>}} options
+ * @returns {Promise<StreamingIngestClient>}
+ */
+async function createClient(native, options) {
+  // Coerce property values to strings (matching Python)
+  let properties = options.properties;
+  if (properties) {
+    properties = Object.fromEntries(Object.entries(properties).map(([k, v]) => [k, String(v)]));
+  }
+
+  // Windows workaround: SNOW-2345722
+  if (process.platform === "win32") {
+    properties = properties || {};
+    if (!("mem_throttle_by_process" in properties)) {
+      properties.mem_throttle_by_process = "false";
+    }
+  }
+
+  const nativeClient = await callFFI(() =>
+    native.createClient({
+      clientName: options.clientName,
+      dbName: options.dbName,
+      schemaName: options.schemaName,
+      pipeName: options.pipeName,
+      profilePath: options.profilePath,
+      properties,
+    }),
+  );
+
+  return new StreamingIngestClient(nativeClient);
+}
+
+module.exports = {
+  StreamingIngestClient,
+  createClient,
+};

package/src/errors.js

@@ -0,0 +1,110 @@
+// Copyright 2026 Snowflake Inc.
+// SPDX-License-Identifier: Apache-2.0
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+"use strict";
+
+/**
+ * Error class for Snowpipe Streaming SDK errors.
+ *
+ * This error is thrown when the Rust core encounters an error condition.
+ * It includes structured error information including an error code,
+ * HTTP status code, and descriptive message.
+ *
+ * @example
+ * try {
+ *   await client.openChannel({ name: "ch1" });
+ * } catch (e) {
+ *   if (e instanceof StreamingIngestError) {
+ *     console.log(e.errorCode, e.httpStatusCode, e.message);
+ *   }
+ * }
+ *
+ * @extends Error
+ */
+class StreamingIngestError extends Error {
+  /**
+   * Create a StreamingIngestError.
+   * @param {string} errorCode - The error code (e.g., "ConfigError", "ChannelNotFound")
+   * @param {string} message - The error message
+   * @param {number} httpStatusCode - HTTP status code indicating error category
+   * @param {string} httpStatusName - HTTP status name (e.g., "Bad Request")
+   */
+  constructor(errorCode, message, httpStatusCode, httpStatusName) {
+    super(message);
+    this.name = "StreamingIngestError";
+    this.errorCode = errorCode;
+    this.httpStatusCode = httpStatusCode;
+    this.httpStatusName = httpStatusName;
+  }
+
+  /**
+   * Create a StreamingIngestError from an FFI error.
+   *
+   * FFI errors from the Rust core have their message encoded as JSON with
+   * error_code, message, http_status_code, and http_status_name fields.
+   * This method parses that JSON and constructs a StreamingIngestError.
+   *
+   * @param {Error} e - The error from FFI
+   * @returns {StreamingIngestError|Error} Parsed error or original if not parseable
+   */
+  static from(e) {
+    try {
+      const data = JSON.parse(e.message);
+      if (data.error_code && data.message && data.http_status_code !== undefined) {
+        return new StreamingIngestError(
+          data.error_code,
+          data.message,
+          data.http_status_code,
+          data.http_status_name || "Unknown",
+        );
+      }
+    } catch {
+      // Not JSON, return original
+    }
+    return e;
+  }
+
+  /**
+   * Returns a string representation of the error.
+   * @returns {string} Formatted error string
+   */
+  toString() {
+    return `${this.errorCode}: ${this.message} (HTTP ${this.httpStatusCode} ${this.httpStatusName})`;
+  }
+}
+
+/**
+ * Wrap an FFI call with error conversion.
+ *
+ * Equivalent to Python's @_rethrow_ffi_errors decorator.
+ * Catches napi errors, parses the JSON message, and throws
+ * a StreamingIngestError.
+ *
+ * @param {Function} fn - The FFI function to call
+ * @returns {Promise<*>} The result of the FFI call
+ * @throws {StreamingIngestError} If the FFI call fails
+ */
+async function callFFI(fn) {
+  try {
+    return await fn();
+  } catch (e) {
+    throw StreamingIngestError.from(e);
+  }
+}
+
+module.exports = {
+  StreamingIngestError,
+  callFFI,
+};

package/src/index.d.ts

@@ -0,0 +1,351 @@
+/*
+ * Copyright 2026 Snowflake Inc.
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+//
+// TypeScript declaration file for the Snowpipe Streaming Node.js SDK.
+//
+// This file serves two purposes:
+// 1. Provides TypeScript/JavaScript users with autocomplete and type checking
+// 2. Used by TypeDoc (see docs/typedoc.json) to generate API reference HTML
+//
+// This file mirrors the public API surface defined in src/*.js.
+// The CI lint step (`npm run lint`) runs check-types-sync.js which verifies
+// that this file stays consistent with the JavaScript source.
+//
+
+/**
+ * Create a new streaming ingest client. Each client is tied to a single
+ * database, schema, and pipe. To ingest into multiple pipes, create
+ * multiple clients.
+ *
+ * Either `profilePath` or `properties` must be provided for authentication.
+ * If both are provided, `properties` takes precedence for conflicting keys.
+ *
+ * @param options - Client configuration
+ * @throws StreamingIngestError If client creation fails
+ */
+export function createClient(options: {
+  /** A unique name to identify this client instance, used for tracking and debugging purposes. */
+  clientName: string;
+  /** The name of the Snowflake database where data will be ingested. */
+  dbName: string;
+  /** The name of the schema within the database. */
+  schemaName: string;
+  /** The name of the pipe that will be used for streaming ingestion. */
+  pipeName: string;
+  /** Path to a JSON file containing connection properties and authentication information. */
+  profilePath?: string;
+  /** Connection properties and authentication information. Common properties include account, user, private_key, url. */
+  properties?: Record<string, string>;
+}): Promise<StreamingIngestClient>;
+
+/**
+ * A client that is the starting point for using the Streaming Ingest client APIs.
+ *
+ * A single client maps to exactly one account/database/schema/pipe in Snowflake;
+ * however, multiple clients can point to the same account/database/schema/pipe.
+ * Each client contains information for Snowflake authentication and authorization,
+ * and it is used to create one or more StreamingIngestChannel instances for data
+ * ingestion.
+ *
+ * Create instances using the `createClient()` function.
+ */
+export class StreamingIngestClient {
+  private constructor();
+
+  /** The client name. */
+  readonly clientName: string;
+  /** The database name. */
+  readonly dbName: string;
+  /** The schema name. */
+  readonly schemaName: string;
+  /** The pipe name. */
+  readonly pipeName: string;
+  /** Whether the client is closed. */
+  readonly isClosed: boolean;
+  /** The binary input format. One of: "BASE64", "HEX", "UTF-8". */
+  readonly binaryInputFormat: string;
+
+  /**
+   * Close the client. If waitForFlush is true, the client waits for all data in every
+   * channel to be flushed to Snowflake before shutting down.
+   *
+   * @param options - Close options
+   * @throws StreamingIngestError If closing the client fails or the timeout is reached
+   */
+  close(options?: {
+    /** Whether to wait for the flush to complete, defaults to true. */
+    waitForFlush?: boolean;
+    /** Optional timeout in milliseconds for the flush operation. */
+    timeoutMs?: number;
+  }): Promise<void>;
+
+  /**
+   * Open a channel with the given name. The channel is opened on the
+   * database/schema/pipe defined by this client. If the channel already exists,
+   * Snowflake reuses the latest persisted offset token unless a new one is provided.
+   *
+   * @param options - Channel options
+   * @throws StreamingIngestError If opening the channel fails
+   */
+  openChannel(options: {
+    /** The name of the channel to open. */
+    name: string;
+    /** Optional offset token to set on the channel. If not provided and this reopens an existing channel, the latest persisted offset token is reused. */
+    offsetToken?: string;
+  }): Promise<{
+    /** The opened channel instance. */
+    channel: StreamingIngestChannel;
+    /** The channel status at the time of opening, or null if not available. */
+    status: ChannelStatus | null;
+  }>;
+
+  /**
+   * Drop a channel by name (server-side removal).
+   *
+   * @param channelName - Name of the channel to drop
+   * @throws StreamingIngestError If dropping the channel fails
+   */
+  dropChannel(channelName: string): Promise<void>;
+
+  /**
+   * Get the status of multiple channels. Can fetch status for channels not
+   * opened by this client.
+   *
+   * @param channelNames - Names of channels to query
+   * @throws StreamingIngestError If retrieving statuses fails
+   */
+  getChannelStatuses(channelNames: string[]): Promise<Record<string, ChannelStatus>>;
+
+  /**
+   * Get the latest committed offset tokens for multiple channels.
+   *
+   * @param channelNames - Names of channels to query
+   * @throws StreamingIngestError If retrieving offset tokens fails
+   */
+  getLatestCommittedOffsetTokens(channelNames: string[]): Promise<Record<string, string | null>>;
+
+  /**
+   * Initiate a flush of the client. Causes all outstanding buffered data to be
+   * flushed to Snowflake. Data can still be accepted by the client after calling
+   * this method — this is an asynchronous call that returns after the flush is
+   * initiated for all channels.
+   *
+   * @throws StreamingIngestError If initiating the flush fails
+   */
+  initiateFlush(): void;
+
+  /**
+   * Wait for all buffered data in all channels managed by this client to be flushed
+   * to Snowflake. This method triggers a flush of all pending data across all channels
+   * and waits for the flush operations to complete.
+   *
+   * @param options - Wait options
+   * @throws StreamingIngestError If waiting for the flush fails
+   * @throws Error If the timeout is reached
+   */
+  waitForFlush(options?: {
+    /** Optional timeout in milliseconds. */
+    timeoutMs?: number;
+  }): Promise<void>;
+}
+
+/**
+ * A channel for streaming data into a Snowflake table.
+ *
+ * A channel represents a logical stream of data into a single table through a pipe.
+ * Multiple channels can be opened on the same pipe for parallel ingestion. Each channel
+ * maintains its own offset token for exactly-once delivery guarantees.
+ *
+ * Create instances using `client.openChannel()`.
+ */
+export class StreamingIngestChannel {
+  private constructor();
+
+  /** The channel name. */
+  readonly channelName: string;
+  /** The database name. */
+  readonly dbName: string;
+  /** The schema name. */
+  readonly schemaName: string;
+  /** The pipe name. */
+  readonly pipeName: string;
+  /** Whether the channel is closed. */
+  readonly isClosed: boolean;
+  /** The binary input format. One of: "BASE64", "HEX", "UTF-8". */
+  readonly binaryInputFormat: string;
+
+  /**
+   * Append a single row into the channel.
+   *
+   * The row is an object with keys as column names and values as column values.
+   * Supported value types: null, undefined, boolean, number, string, BigInt,
+   * Buffer, Uint8Array, Date, Array, and Object.
+   *
+   * @param row - Row data as column-name to value pairs
+   * @param offsetToken - Optional offset token, used to track the ingestion
+   *   progress and replay ingestion in case of failures
+   * @throws StreamingIngestError If the row appending fails
+   */
+  appendRow(row: Record<string, any>, offsetToken?: string): Promise<void>;
+
+  /**
+   * Append multiple rows to the channel.
+   *
+   * Each row is an object with keys as column names and values as column values.
+   * Supported value types: null, undefined, boolean, number, string, BigInt,
+   * Buffer, Uint8Array, Date, Array, and Object.
+   *
+   * @param rows - Array of row objects (column-name to value pairs)
+   * @param startOffsetToken - Optional start offset token of the batch
+   * @param endOffsetToken - Optional end offset token of the batch
+   * @throws TypeError If rows is not an array
+   * @throws StreamingIngestError If the rows appending fails
+   */
+  appendRows(rows: Record<string, any>[], startOffsetToken?: string, endOffsetToken?: string): Promise<void>;
+
+  /**
+   * Close the channel.
+   *
+   * @param options - Close options
+   * @throws StreamingIngestError If closing the channel fails
+   */
+  close(options?: {
+    /** Whether to drop the channel on the server, defaults to false. */
+    drop?: boolean;
+    /** Whether to wait for the flush to complete, defaults to true. */
+    waitForFlush?: boolean;
+    /** Optional timeout in milliseconds for the flush operation. */
+    timeoutMs?: number;
+  }): Promise<void>;
+
+  /**
+   * Initiate a flush of all buffered data for this channel but do not wait for
+   * the flush to complete. Calls to appendRow/appendRows are still allowed on
+   * the channel after calling this method.
+   *
+   * @throws StreamingIngestError If the flush cannot be initiated
+   */
+  initiateFlush(): void;
+
+  /**
+   * Wait for all buffered data in this channel to be flushed to Snowflake.
+   *
+   * @param options - Wait options
+   * @throws StreamingIngestError If waiting for the flush fails
+   * @throws Error If the timeout is reached
+   */
+  waitForFlush(options?: {
+    /** Optional timeout in milliseconds. */
+    timeoutMs?: number;
+  }): Promise<void>;
+
+  /**
+   * Get the current status of this channel.
+   *
+   * @throws StreamingIngestError If getting the channel status fails
+   */
+  getChannelStatus(): Promise<ChannelStatus>;
+
+  /**
+   * Get the latest committed offset token for this channel.
+   *
+   * @returns The latest committed offset token, or null if the channel is brand new
+   * @throws StreamingIngestError If getting the latest committed offset token fails
+   */
+  getLatestCommittedOffsetToken(): Promise<string | null>;
+
+  /**
+   * Wait for a specific offset token to be committed.
+   *
+   * Snowflake commits offset tokens in batches, so the latest committed offset token
+   * may jump past the one you are waiting for. The tokenChecker should handle this
+   * case by doing a range check (greater than or equal) rather than an exact match.
+   *
+   * @param tokenChecker - A callable that receives the latest committed offset token
+   *   (which may be null) and should return true when the wait condition is satisfied
+   * @param options - Wait options
+   * @throws StreamingIngestError If waiting for the commit fails
+   * @throws Error If the timeout is reached
+   * @throws TypeError If tokenChecker is not a function
+   * @throws RangeError If timeoutMs is negative
+   */
+  waitForCommit(tokenChecker: (token: string | null) => boolean, options?: {
+    /** Optional timeout in milliseconds. */
+    timeoutMs?: number;
+  }): Promise<void>;
+}
+
+/**
+ * Status information for a streaming ingest channel.
+ *
+ * Contains metadata about the channel's current state including the latest
+ * committed offset token, row counts, error information, and processing latency.
+ * Returned by `client.getChannelStatuses()` and `channel.getChannelStatus()`.
+ */
+export class ChannelStatus {
+  private constructor();
+
+  /** The database name. */
+  readonly databaseName: string;
+  /** The schema name. */
+  readonly schemaName: string;
+  /** The pipe name. */
+  readonly pipeName: string;
+  /** The channel name. */
+  readonly channelName: string;
+  /** The channel status code. */
+  readonly statusCode: string;
+  /** The latest committed offset token, or null if the channel is brand new. */
+  readonly latestCommittedOffsetToken: string | null;
+  /** When the channel was created. */
+  readonly createdOn: Date | null;
+  /** Total number of rows successfully inserted. */
+  readonly rowsInsertedCount: number;
+  /** Total number of rows parsed. */
+  readonly rowsParsedCount: number;
+  /** Total number of rows that encountered errors. */
+  readonly rowsErrorCount: number;
+  /** Upper bound offset token for the last error, or null if no errors. */
+  readonly lastErrorOffsetTokenUpperBound: string | null;
+  /** The message of the last error, or null if no errors. */
+  readonly lastErrorMessage: string | null;
+  /** Timestamp of the last error, or null if no errors. */
+  readonly lastErrorTimestamp: Date | null;
+  /** Average processing latency in milliseconds, or null if not available. */
+  readonly serverAvgProcessingLatencyMs: number | null;
+  /** When the status was last refreshed from the server. */
+  readonly lastRefreshedOn: Date | null;
+}
+
+/**
+ * Error thrown by the Snowpipe Streaming SDK.
+ *
+ * Includes structured error information: an error code, HTTP status code,
+ * and descriptive message. Use `errorCode` and `httpStatusCode` to determine
+ * the appropriate recovery action.
+ */
+export class StreamingIngestError extends Error {
+  /** The error code (for example, "ConfigError", "ChannelNotFound"). */
+  readonly errorCode: string;
+  /** HTTP status code indicating error category. */
+  readonly httpStatusCode: number;
+  /** HTTP status name (for example, "Bad Request"). */
+  readonly httpStatusName: string;
+}
+
+