snowpipe-streaming 1.7.0 → 1.8.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "snowpipe-streaming",
3
- "version": "1.7.0",
3
+ "version": "1.8.0",
4
4
  "description": "Snowflake Streaming Ingest SDK for Node.js",
5
5
  "main": "src/index.js",
6
6
  "types": "src/index.d.ts",
@@ -27,11 +27,11 @@
27
27
  "typedoc": "^0.28.19"
28
28
  },
29
29
  "optionalDependencies": {
30
- "snowpipe-streaming-linux-x64-gnu": "1.7.0",
31
- "snowpipe-streaming-linux-x64-musl": "1.7.0",
32
- "snowpipe-streaming-linux-arm64-gnu": "1.7.0",
33
- "snowpipe-streaming-linux-arm64-musl": "1.7.0",
34
- "snowpipe-streaming-darwin-arm64": "1.7.0",
35
- "snowpipe-streaming-win32-x64-msvc": "1.7.0"
30
+ "snowpipe-streaming-linux-x64-gnu": "1.8.0",
31
+ "snowpipe-streaming-linux-x64-musl": "1.8.0",
32
+ "snowpipe-streaming-linux-arm64-gnu": "1.8.0",
33
+ "snowpipe-streaming-linux-arm64-musl": "1.8.0",
34
+ "snowpipe-streaming-darwin-arm64": "1.8.0",
35
+ "snowpipe-streaming-win32-x64-msvc": "1.8.0"
36
36
  }
37
37
  }
package/src/client.js CHANGED
@@ -124,6 +124,9 @@ class StreamingIngestClient {
124
124
  // Cache the in-flight promise so concurrent callers share one open and one
125
125
  // instance. The pending-futures map is shared between the FFI callbacks and
126
126
  // the channel wrapper (the JS side owns per-message-ack promise resolution).
127
+ // No client-level callback registry to publish into (unlike Java/Python), so a
128
+ // failed open leaves nothing behind — the ack state is only reachable via the
129
+ // channel returned on success.
127
130
  this._elasticChannelPromise = (async () => {
128
131
  const pendingFutures = new Map();
129
132
  const ackState = new ElasticChannelAckState(pendingFutures);
@@ -133,7 +136,7 @@ class StreamingIngestClient {
133
136
  // Plug the native handle in before exposing the channel so the ack
134
137
  // state can drive `invalidateChannel` from JS on detected anomalies.
135
138
  ackState.setNativeChannel(nativeChannel);
136
- return new StreamingIngestElasticChannel(nativeChannel, pendingFutures);
139
+ return new StreamingIngestElasticChannel(nativeChannel, pendingFutures, ackState);
137
140
  })().catch((e) => {
138
141
  // Don't cache failures (e.g. closed client, transient open error) so a retry can re-open.
139
142
  this._elasticChannelPromise = null;
@@ -18,23 +18,44 @@
18
18
  const { callFFI, StreamingIngestError } = require("./errors.js");
19
19
  const { ChannelStatus } = require("./channel_status.js");
20
20
  const { serializeRowsToNdjson } = require("./row_serializer.js");
21
+ const { LOG_PREFIX } = require("./log.js");
22
+
23
+ // Default for the appendToken parameter, used to tell "caller omitted the argument" apart from
24
+ // "caller passed a token" — including an explicit null, which is a valid token value meaning
25
+ // "leave this append untracked". appendToken is required but nullable (matching Java's positional
26
+ // parameter and Python's default-less argument), so opting out has to be spelled out as null
27
+ // rather than achieved by leaving the argument off. Checked in _appendRowsInternal, which every
28
+ // public append funnels through: a forwarded-but-omitted argument arrives as undefined, and a
29
+ // default parameter applies to undefined however it got there.
30
+ const APPEND_TOKEN_OMITTED = Symbol("appendTokenOmitted");
21
31
 
22
32
  /**
23
33
  * Elastic channel for streaming data into a Snowflake table.
24
34
  *
25
35
  * Unlike a regular channel, an elastic channel has no offset tokens and its
26
- * lifecycle is tied to the client (no close method). appendRow/appendRows return
27
- * a Promise that resolves when Snowflake acknowledges the rows.
36
+ * lifecycle is tied to the client (no close method). Appends come in two
37
+ * flavours: fire-and-forget appendRow/appendRows return nothing, while
38
+ * appendRowWithWait/appendRowsWithWait return a Promise that resolves when
39
+ * Snowflake acknowledges the rows. All four take a required (but nullable)
40
+ * caller-supplied append token and register the append for both handlers, so
41
+ * setErrorHandler and setSuccessHandler report the append tokens of appends that
42
+ * failed or were acknowledged after their call returned. An append given a null
43
+ * token is not tracked for either handler — the WithWait variants' promise still
44
+ * settles.
28
45
  *
29
46
  * Created via {@link StreamingIngestClient#getElasticChannel}, not directly.
30
47
  */
31
48
  class StreamingIngestElasticChannel {
32
49
  /**
33
50
  * @param {object} nativeChannel - Internal JsElasticChannel from FFI
34
- * @param {Map<number, {resolve: Function, reject: Function}>} pendingFutures
35
- * Pending-promise map shared with the mark-complete callback wired into FFI.
51
+ * @param {Map<number, {resolve: Function, reject: Function, appendToken?: *}>} pendingFutures
52
+ * Pending-promise map shared with the mark-complete callback wired into FFI;
53
+ * an entry carries the caller-supplied append token of its append only when one
54
+ * was supplied.
55
+ * @param {import("./mark_complete_callback.js").ElasticChannelAckState} ackState
56
+ * Acknowledgement state that owns the customer error and success handlers.
36
57
  */
37
- constructor(nativeChannel, pendingFutures) {
58
+ constructor(nativeChannel, pendingFutures, ackState) {
38
59
  if (!(nativeChannel && typeof nativeChannel === "object")) {
39
60
  throw new TypeError(
40
61
  "StreamingIngestElasticChannel cannot be instantiated directly. " +
@@ -43,6 +64,13 @@ class StreamingIngestElasticChannel {
43
64
  }
44
65
  this._native = nativeChannel;
45
66
  this._pendingFutures = pendingFutures;
67
+ this._ackState = ackState;
68
+ // Per-channel future id counter; the wrapper allocates ids so it can register
69
+ // the promise before the row becomes ack-eligible. The single-threaded event
70
+ // loop makes the plain increment safe. The id is a JS number (a safe integer,
71
+ // exact to 2^53) marshaled to the Rust i64.
72
+ // Starts at 1; 0 is reserved as the "no future id" sentinel, matching the other SDKs.
73
+ this._nextFutureId = 1;
46
74
  }
47
75
 
48
76
  /** @returns {string} The channel name (always "ELASTIC") */
@@ -79,50 +107,149 @@ class StreamingIngestElasticChannel {
79
107
  }
80
108
 
81
109
  /**
82
- * Append a single row into the elastic channel.
110
+ * Append a single row into the elastic channel without waiting for the
111
+ * acknowledgement (fire-and-forget): returns as soon as the row is buffered.
112
+ * The append is still registered for both handlers, so its outcome is reported
113
+ * through {@link #setErrorHandler} or {@link #setSuccessHandler} (keyed by
114
+ * appendToken), just not through a promise. Use {@link #appendRowWithWait} when you need
115
+ * one to await.
116
+ *
83
117
  * @param {Object} row - Row data as column-name to value pairs
118
+ * @param {*} appendToken - Required caller-supplied opaque id for this append,
119
+ * handed back to the error handler if it fails asynchronously, or to the
120
+ * success handler once it is acknowledged. Any value is accepted, but the
121
+ * argument itself is mandatory: pass null explicitly to leave the append
122
+ * untracked for both handlers. The token is retained in memory until the
123
+ * append is acknowledged, so a large object raises the memory footprint of
124
+ * in-flight appends — prefer a small value.
125
+ * @returns {void}
126
+ * @throws {TypeError} If row is null or undefined, or appendToken is omitted
127
+ * @throws {StreamingIngestError} If the append fails synchronously
128
+ */
129
+ appendRow(row, appendToken) {
130
+ if (row == null) {
131
+ throw new TypeError("row must not be null");
132
+ }
133
+ this.appendRows([row], appendToken);
134
+ }
135
+
136
+ /**
137
+ * Append multiple rows into the elastic channel without waiting for the
138
+ * acknowledgement (fire-and-forget): returns as soon as the rows are buffered.
139
+ * The append is still registered for both handlers, so its outcome is reported
140
+ * through {@link #setErrorHandler} or {@link #setSuccessHandler} (keyed by
141
+ * appendToken), just not through a promise. Use {@link #appendRowsWithWait} when you need
142
+ * one to await.
143
+ *
144
+ * @param {Object[]} rows - Array of row objects
145
+ * @param {*} appendToken - Required caller-supplied opaque id for this batch,
146
+ * handed back to the error handler if it fails asynchronously, or to the
147
+ * success handler once it is acknowledged. Any value is accepted, but the
148
+ * argument itself is mandatory: pass null explicitly to leave the append
149
+ * untracked for both handlers. The token is retained in memory until the
150
+ * append is acknowledged, so a large object raises the memory footprint of
151
+ * in-flight appends — prefer a small value.
152
+ * @returns {void}
153
+ * @throws {TypeError} If rows is null, undefined, or not an array, or appendToken
154
+ * is omitted
155
+ * @throws {StreamingIngestError} If the append fails synchronously
156
+ */
157
+ appendRows(rows, appendToken) {
158
+ // The append is registered like any other, so its failure still reaches the
159
+ // error handler; the caller just never sees the promise. The no-op catch is
160
+ // what keeps that rejection from being reported as an unhandledRejection on
161
+ // a promise nobody is left holding. Synchronous failures (bad arguments,
162
+ // closed channel, serialization) still throw out of this call, matching the
163
+ // other SDKs' fire-and-forget variants.
164
+ this._appendRowsInternal(rows, appendToken).catch(() => {});
165
+ }
166
+
167
+ /**
168
+ * Append a single row into the elastic channel and wait for Snowflake to
169
+ * acknowledge it. If an error handler is registered it also fires on an
170
+ * asynchronous failure — dual delivery.
171
+ *
172
+ * @param {Object} row - Row data as column-name to value pairs
173
+ * @param {*} appendToken - Required caller-supplied opaque id for this append,
174
+ * handed back to the error handler if it fails asynchronously, or to the
175
+ * success handler once it is acknowledged. Any value is accepted, but the
176
+ * argument itself is mandatory: pass null explicitly to leave the append
177
+ * untracked for both handlers. The token is retained in memory until the
178
+ * append is acknowledged, so a large object raises the memory footprint of
179
+ * in-flight appends — prefer a small value.
84
180
  * @returns {Promise<void>} Resolves when Snowflake acknowledges the row
181
+ * @throws {TypeError} If row is null or undefined, or appendToken is omitted
85
182
  */
86
- async appendRow(row) {
183
+ async appendRowWithWait(row, appendToken) {
87
184
  if (row == null) {
88
185
  throw new TypeError("row must not be null");
89
186
  }
90
- return this.appendRows([row]);
187
+ return this.appendRowsWithWait([row], appendToken);
188
+ }
189
+
190
+ /**
191
+ * Append multiple rows into the elastic channel and wait for Snowflake to
192
+ * acknowledge the batch. If an error handler is registered it also fires on an
193
+ * asynchronous failure — dual delivery.
194
+ *
195
+ * @param {Object[]} rows - Array of row objects
196
+ * @param {*} appendToken - Required caller-supplied opaque id for this batch,
197
+ * handed back to the error handler if it fails asynchronously, or to the
198
+ * success handler once it is acknowledged. Any value is accepted, but the
199
+ * argument itself is mandatory: pass null explicitly to leave the append
200
+ * untracked for both handlers. The token is retained in memory until the
201
+ * append is acknowledged, so a large object raises the memory footprint of
202
+ * in-flight appends — prefer a small value.
203
+ * @returns {Promise<void>} Resolves when Snowflake acknowledges the batch
204
+ * @throws {TypeError} If rows is not an array, or appendToken is omitted
205
+ */
206
+ async appendRowsWithWait(rows, appendToken) {
207
+ return this._appendRowsInternal(rows, appendToken);
91
208
  }
92
209
 
93
210
  /**
94
- * Append multiple rows into the elastic channel.
211
+ * Shared append path behind both the fire-and-forget and the WithWait
212
+ * variants. Always registers the append for acknowledgement so the error
213
+ * handler fires on failure regardless of variant; the fire-and-forget callers
214
+ * simply discard the returned promise.
95
215
  *
96
- * The native call is synchronous: it buffers the rows and returns a future id,
97
- * which is registered in the shared pending map before returning to the event
98
- * loop. The Promise resolves later when the mark-complete callback fires with
99
- * the server acknowledgement (or rejects on error / channel invalidation).
216
+ * The wrapper allocates the future id and registers its promise in the shared
217
+ * pending map before the synchronous native append, so the ack can never
218
+ * precede registration. The Promise resolves later when the mark-complete
219
+ * callback fires with the server acknowledgement (or rejects on error).
220
+ *
221
+ * Throws rather than rejects by design: argument, serialization, and
222
+ * native-append failures leave the fire-and-forget callers at the call site
223
+ * (as in Java/Python) instead of vanishing into their discarded promise. The
224
+ * WithWait callers see them as a rejection, since those methods are `async`.
225
+ * Only a post-return failure travels on the returned promise.
100
226
  *
101
227
  * @param {Object[]} rows - Array of row objects
228
+ * @param {*} appendToken - Required (but nullable) caller-supplied opaque id for
229
+ * this batch. Omitting it — here or in the public method that forwarded to this
230
+ * one — is a TypeError; null is the way to opt out of handler tracking.
102
231
  * @returns {Promise<void>} Resolves when Snowflake acknowledges the batch
103
232
  */
104
- async appendRows(rows) {
233
+ _appendRowsInternal(rows, appendToken = APPEND_TOKEN_OMITTED) {
105
234
  if (rows == null) {
106
235
  throw new TypeError("rows must not be null");
107
236
  }
108
237
  if (!Array.isArray(rows)) {
109
238
  throw new TypeError("rows must be an array");
110
239
  }
240
+ if (appendToken === APPEND_TOKEN_OMITTED) {
241
+ throw new TypeError("appendToken is required; pass null to leave the append untracked");
242
+ }
111
243
  // Empty arrays are validated by the Rust core, which throws a proper error.
112
244
  const buffer =
113
245
  rows.length === 0 ? Buffer.alloc(0) : serializeRowsToNdjson(rows, this.binaryInputFormat);
114
- let futureId;
115
- try {
116
- futureId = Number(this._native.appendRows(buffer, rows.length));
117
- } catch (e) {
118
- throw StreamingIngestError.from(e);
119
- }
120
246
 
121
- // Build the promise out-of-line so we can stash {resolve, reject} into the
122
- // shared map synchronously — the single-threaded event loop guarantees this
123
- // happens-before any queued threadsafe-function callback runs. Map and
124
- // resolve are then visible to mark_complete_callback.js (the JS owner of
125
- // resolution) and the invalidate-all path on channel teardown.
247
+ // Allocate the id and register the promise BEFORE the native append, so the
248
+ // row cannot become ack-eligible before its promise is in the pending map.
249
+ // Building the promise out-of-line lets us stash {resolve, reject, appendToken}
250
+ // into the shared map synchronously; the single-threaded event loop guarantees
251
+ // this happens-before any queued threadsafe-function callback runs.
252
+ const futureId = this._nextFutureId++;
126
253
  let resolveFn;
127
254
  let rejectFn;
128
255
  const promise = new Promise((resolve, reject) => {
@@ -130,26 +257,128 @@ class StreamingIngestElasticChannel {
130
257
  rejectFn = reject;
131
258
  });
132
259
  const previous = this._pendingFutures.get(futureId);
133
- this._pendingFutures.set(futureId, { resolve: resolveFn, reject: rejectFn });
260
+ const entry = { resolve: resolveFn, reject: rejectFn };
261
+ // Track the append token only when the caller supplied one; a null/undefined
262
+ // token means this append is invisible to the error handler, though a WithWait
263
+ // promise still rejects. Mirrors Java's nullable PendingAppend.appendToken.
264
+ if (appendToken != null) {
265
+ entry.appendToken = appendToken;
266
+ }
267
+ this._pendingFutures.set(futureId, entry);
134
268
  if (previous !== undefined) {
135
- // Mirrors Java/Python: a duplicate future id from Rust means the SDK
136
- // and the core have drifted, which is fatal. Reject the displaced
137
- // promise, invalidate the channel, and throw to the caller so the
138
- // new promise we just stashed isn't left hanging either.
269
+ // Monotonic per-channel ids make this a should-never-happen invariant.
139
270
  const errMsg = `Duplicate future_id=${futureId} in pendingFutures`;
140
271
  const err = new StreamingIngestError("Fatal", errMsg, 500, "Internal Server Error");
141
272
  previous.reject(err);
273
+ // Displaced promise was already handed to its owner (or discarded by a
274
+ // fire-and-forget append), so its failure is asynchronous from their view;
275
+ // the current call throws instead, where the caller still holds its own
276
+ // appendToken. An untracked displaced append has no token to report. No
277
+ // request id: this is our own duplicate-id invariant, not a Snowflake response.
278
+ if (previous.appendToken != null) {
279
+ this._ackState.fireErrorHandler([previous.appendToken], err, null, null);
280
+ }
142
281
  try {
143
282
  this._native.invalidateChannel(errMsg);
144
283
  } catch (invErr) {
145
284
  // eslint-disable-next-line no-console
146
- console.error("Elastic channel invalidate_channel FFI call failed:", invErr);
285
+ console.error(`${LOG_PREFIX} Elastic channel invalidate_channel FFI call failed:`, invErr);
147
286
  }
148
287
  throw err;
149
288
  }
289
+
290
+ try {
291
+ this._native.appendRows(buffer, rows.length, futureId);
292
+ } catch (e) {
293
+ // Never enqueued (not ack-eligible), so no ack can arrive; drop the promise.
294
+ this._pendingFutures.delete(futureId);
295
+ throw StreamingIngestError.from(e);
296
+ }
150
297
  return promise;
151
298
  }
152
299
 
300
+ /**
301
+ * Register (or replace) a handler invoked when appends fail *asynchronously*,
302
+ * i.e. after the append call already returned. Fires once per failure event
303
+ * with a single {@link ErrorDetail} bundling the append tokens that failed
304
+ * together (a server-ack batch, or every in-flight append on invalidation) and
305
+ * their common error, so callers need not await each promise. Both append
306
+ * flavours are covered: it is the only failure signal for the fire-and-forget
307
+ * ones, and an additional one for the WithWait ones, whose promise still
308
+ * rejects. Only appends given an append token appear, and tokens are opaque and
309
+ * not deduplicated. Synchronous failures (closed channel, serialization, bad
310
+ * arguments) throw from the append call and never reach the handler. A handler
311
+ * that throws is caught and logged.
312
+ *
313
+ * The handler runs inline on the tick that drains a server acknowledgement. It
314
+ * must therefore return quickly and do no heavy or blocking work: no synchronous
315
+ * blocking calls, no busy loops, no long CPU-bound work. Do only cheap
316
+ * bookkeeping inline (e.g. record the failed tokens) and hand any real work to a
317
+ * worker thread or your own queue — blocking here blocks the event loop, which
318
+ * stalls acknowledgements (and append/flush completion) for every channel on this
319
+ * client and the rest of your application with them. The SDK logs a warning when
320
+ * a handler runs slowly.
321
+ *
322
+ * @param {(detail: import("./error_detail.js").ErrorDetail) => void} handler
323
+ * @throws {TypeError} If handler is not a function, or declares more than one
324
+ * parameter (it is called with a single {@link ErrorDetail})
325
+ */
326
+ setErrorHandler(handler) {
327
+ if (typeof handler !== "function") {
328
+ throw new TypeError("handler must be a function");
329
+ }
330
+ // Reject the (appendTokens, error) two-argument shape: such a handler would
331
+ // silently see `undefined` as its second argument. `> 1` rather than `!== 1`
332
+ // keeps zero-arg, rest-parameter (length 0) and default-parameter handlers
333
+ // valid.
334
+ if (handler.length > 1) {
335
+ throw new TypeError(
336
+ `handler must take a single ErrorDetail argument, but declares ${handler.length} parameters`,
337
+ );
338
+ }
339
+ this._ackState.setErrorHandler(handler);
340
+ }
341
+
342
+ /**
343
+ * Register (or replace) a handler invoked when appends are acknowledged by
344
+ * Snowflake. Fires once per successful ack batch with a single
345
+ * {@link SuccessDetail} bundling the append tokens acknowledged together, so
346
+ * callers need not await each promise. Both append flavours are covered: it is
347
+ * the only success signal for the fire-and-forget ones, and an additional one
348
+ * for the WithWait ones, whose promise still resolves. Purely opt-in — without a
349
+ * registered handler nothing is collected. Only appends given an append token
350
+ * appear, and tokens are opaque and not deduplicated. A handler that throws is
351
+ * caught and logged.
352
+ *
353
+ * Like the error handler, this runs inline on the tick that drains a server
354
+ * acknowledgement. It must therefore return quickly and do no heavy or blocking
355
+ * work: no synchronous blocking calls, no busy loops, no long CPU-bound work. Do
356
+ * only cheap bookkeeping inline (e.g. record the tokens) and hand any real work to
357
+ * a worker thread or your own queue — blocking here blocks the event loop, which
358
+ * stalls acknowledgements (and append/flush completion) for every channel on this
359
+ * client and the rest of your application with them. The SDK logs a warning when a
360
+ * handler runs slowly.
361
+ *
362
+ * @param {(detail: import("./success_detail.js").SuccessDetail) => void} handler
363
+ * @throws {TypeError} If handler is not a function, or declares more than one
364
+ * parameter (it is called with a single {@link SuccessDetail})
365
+ */
366
+ setSuccessHandler(handler) {
367
+ if (typeof handler !== "function") {
368
+ throw new TypeError("handler must be a function");
369
+ }
370
+ // Same arity guard as setErrorHandler: a multi-parameter handler would
371
+ // silently see `undefined` beyond the first argument. `> 1` rather than
372
+ // `!== 1` keeps zero-arg, rest-parameter (length 0) and default-parameter
373
+ // handlers valid.
374
+ if (handler.length > 1) {
375
+ throw new TypeError(
376
+ `handler must take a single SuccessDetail argument, but declares ${handler.length} parameters`,
377
+ );
378
+ }
379
+ this._ackState.setSuccessHandler(handler);
380
+ }
381
+
153
382
  /**
154
383
  * Trigger a flush of this channel (non-blocking).
155
384
  * @throws {StreamingIngestError} If the flush cannot be initiated
@@ -162,6 +391,15 @@ class StreamingIngestElasticChannel {
162
391
  }
163
392
  }
164
393
 
394
+ /**
395
+ * Wait for this channel to flush.
396
+ * @param {{timeoutMs?: number}} [options]
397
+ * @returns {Promise<void>}
398
+ */
399
+ async waitForFlush(options) {
400
+ return callFFI(() => this._native.waitForFlush(options?.timeoutMs));
401
+ }
402
+
165
403
  /**
166
404
  * Get the current status of this channel.
167
405
  * @returns {Promise<ChannelStatus>}
@@ -0,0 +1,95 @@
1
+ // Copyright 2026 Snowflake Inc.
2
+ // SPDX-License-Identifier: Apache-2.0
3
+ //
4
+ // Licensed under the Apache License, Version 2.0 (the "License");
5
+ // you may not use this file except in compliance with the License.
6
+ // You may obtain a copy of the License at
7
+ //
8
+ // http://www.apache.org/licenses/LICENSE-2.0
9
+ //
10
+ // Unless required by applicable law or agreed to in writing, software
11
+ // distributed under the License is distributed on an "AS IS" BASIS,
12
+ // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13
+ // See the License for the specific language governing permissions and
14
+ // limitations under the License.
15
+
16
+ "use strict";
17
+
18
+ /**
19
+ * The payload handed to an elastic-channel error handler (registered via
20
+ * {@link StreamingIngestElasticChannel#setErrorHandler}) when appends fail
21
+ * asynchronously.
22
+ *
23
+ * Delivered once per failure event, carrying the caller-supplied append tokens
24
+ * of every append that failed together under a common error — a whole
25
+ * server-ack error batch, or every tracked in-flight append on channel
26
+ * invalidation.
27
+ *
28
+ * Instances are created by the SDK and deeply frozen — both the object and its
29
+ * `appendTokens` array — so a handler cannot mutate what a sibling handler or a
30
+ * later log statement will read. Handlers only read this type, so new fields can
31
+ * be added over time without breaking them. Port of Java's `ErrorDetail`
32
+ * (immutable `Iterable`) and Python's frozen `ErrorDetail` dataclass (immutable
33
+ * tuple).
34
+ */
35
+ class ErrorDetail {
36
+ /**
37
+ * @param {unknown[]} appendTokens - The opaque, caller-supplied append tokens of
38
+ * the appends that failed with `error`. Never empty; a token reused across
39
+ * appends appears once per failed append (not deduplicated). Frozen here, so
40
+ * callers must pass an array they are handing over.
41
+ * @param {import("./errors.js").StreamingIngestError} error - The error that
42
+ * rejected the failing appends.
43
+ * @param {string | null} requestId - The Snowflake request id of the server
44
+ * response that failed these appends, for support escalation. Null when the
45
+ * failure did not come from a server response (channel invalidation, or a
46
+ * local protocol inconsistency detected by the SDK).
47
+ * @param {number} retryCount - How many retries the failing request had already
48
+ * made: 0 when the first attempt failed, N when the SDK retried N times before
49
+ * giving up.
50
+ *
51
+ * **Use this to decide whether these rows may be duplicated in the table.** A retry
52
+ * re-sends the same rowset, and the SDK retries whenever it cannot tell that the
53
+ * previous attempt failed -- a timeout or a 5xx may mean the rows were never
54
+ * processed, or that they were processed and only the response was lost. `0` means
55
+ * the SDK sent this rowset once and introduced no duplicate; `> 0` means it sent it
56
+ * more than once and an earlier attempt may already have been applied, so the rows
57
+ * may appear more than once. Reconcile or de-duplicate downstream if you need
58
+ * exactly-once. The signal is conservative: it counts every re-send, including an
59
+ * authentication refresh, which is rejected before the rows are processed and so
60
+ * cannot have duplicated anything -- `> 0` means "a duplicate is possible", not
61
+ * "a duplicate happened". `0` means the SDK introduced no duplicate of its own, not
62
+ * that the rows are absent from the table.
63
+ *
64
+ * **Whenever `requestId` is null, neither field tells you whether the rows
65
+ * arrived.** A null id means the SDK failed this append itself rather than relaying
66
+ * a server response — channel or client invalidation, a close or drop, or an
67
+ * internal fault — and all of those fail every append still in flight, locally and
68
+ * immediately. That includes appends whose request had already gone to Snowflake and
69
+ * may still be applied after this handler runs. The accompanying `retryCount` of 0
70
+ * says only that the SDK sent the rowset once; it is not a statement that Snowflake
71
+ * rejected it. The SDK cannot resolve this for you — the append has to be failed
72
+ * when the channel goes away, and the in-flight request's outcome is not known at
73
+ * that point — so reconcile against the table if you need to know. (Some null-id
74
+ * failures never reached the network at all, such as a payload the SDK could not
75
+ * serialize, so a null id spans both "definitely not applied" and "possibly
76
+ * applied", which is why it cannot answer the question.)
77
+ *
78
+ * Read it only when `requestId` is non-null: a failure with no originating request
79
+ * reports 0 too. With the request id this names the exact attempt server-side — the
80
+ * SDK sends both as the `requestId` and `retryCount` query params on every request.
81
+ */
82
+ constructor(appendTokens, error, requestId, retryCount) {
83
+ // Freeze the array as well as the instance: `Object.freeze(this)` alone would
84
+ // still let a handler push into or reorder appendTokens. Only the array
85
+ // itself is frozen, not the tokens inside it — those are the caller's own
86
+ // opaque objects and are handed back untouched.
87
+ this.appendTokens = Object.freeze(appendTokens);
88
+ this.error = error;
89
+ this.requestId = requestId;
90
+ this.retryCount = retryCount;
91
+ Object.freeze(this);
92
+ }
93
+ }
94
+
95
+ module.exports = { ErrorDetail };
package/src/errors.js CHANGED
@@ -36,17 +36,30 @@
36
36
  class StreamingIngestError extends Error {
37
37
  /**
38
38
  * Create a StreamingIngestError.
39
+ *
40
+ * `message` (and therefore `stack` / `console.error`) is the decorated form
41
+ * `{errorCode}: {detail} (HTTP {code} {name})`. Pass the Rust/SDK detail as
42
+ * `message`; read it back from {@link StreamingIngestError#detail}.
43
+ *
39
44
  * @param {string} errorCode - The error code (e.g., "ConfigError", "ChannelNotFound")
40
- * @param {string} message - The error message
45
+ * @param {string} message - The undecorated error detail
41
46
  * @param {number} httpStatusCode - HTTP status code indicating error category
42
47
  * @param {string} httpStatusName - HTTP status name (e.g., "Bad Request")
43
48
  */
44
49
  constructor(errorCode, message, httpStatusCode, httpStatusName) {
45
- super(message);
50
+ // V8 bakes `name` + `message` into `stack` at construct time, and
51
+ // `console.error` / pino print that stack — not `toString()`. Put the
52
+ // decorated text on `message` so logging cannot drop error-code / HTTP status.
53
+ const statusName = httpStatusName != null ? httpStatusName : "Unknown";
54
+ const detail = message != null ? String(message) : "";
55
+ const decorated = `${errorCode}: ${detail} (HTTP ${httpStatusCode} ${statusName})`;
56
+ super(decorated);
46
57
  this.name = "StreamingIngestError";
58
+ this.message = decorated;
47
59
  this.errorCode = errorCode;
48
60
  this.httpStatusCode = httpStatusCode;
49
- this.httpStatusName = httpStatusName;
61
+ this.httpStatusName = statusName;
62
+ this.detail = detail;
50
63
  }
51
64
 
52
65
  /**
@@ -67,7 +80,7 @@ class StreamingIngestError extends Error {
67
80
  data.error_code,
68
81
  data.message,
69
82
  data.http_status_code,
70
- data.http_status_name || "Unknown",
83
+ data.http_status_name,
71
84
  );
72
85
  }
73
86
  } catch {
@@ -78,10 +91,10 @@ class StreamingIngestError extends Error {
78
91
 
79
92
  /**
80
93
  * Returns a string representation of the error.
81
- * @returns {string} Formatted error string
94
+ * @returns {string} Decorated error string, without the `StreamingIngestError:` prefix
82
95
  */
83
96
  toString() {
84
- return `${this.errorCode}: ${this.message} (HTTP ${this.httpStatusCode} ${this.httpStatusName})`;
97
+ return this.message;
85
98
  }
86
99
  }
87
100
 
package/src/index.d.ts CHANGED
@@ -327,13 +327,135 @@ export class StreamingIngestChannel {
327
327
  }): Promise<void>;
328
328
  }
329
329
 
330
+ /**
331
+ * The payload handed to an elastic-channel error handler (registered via
332
+ * `StreamingIngestElasticChannel.setErrorHandler`) when appends fail
333
+ * asynchronously.
334
+ *
335
+ * Delivered once per failure event, carrying the caller-supplied append tokens of
336
+ * every append that failed together under a common `error` — a whole server-ack
337
+ * error batch, or every tracked in-flight append on channel invalidation.
338
+ *
339
+ * Instances are created by the SDK and frozen; handlers only read this type, so
340
+ * new fields can be added over time without breaking them.
341
+ */
342
+ export class ErrorDetail {
343
+ private constructor();
344
+
345
+ /**
346
+ * The opaque, caller-supplied append tokens of the appends that failed with
347
+ * `error`. Never empty; a token reused across appends appears once per failed
348
+ * append (not deduplicated). Frozen at construction — the array itself cannot
349
+ * be modified, though the tokens inside it are the caller's own objects.
350
+ */
351
+ readonly appendTokens: readonly unknown[];
352
+ /** The error that rejected the failing appends. */
353
+ readonly error: StreamingIngestError;
354
+ /**
355
+ * The Snowflake request id of the server response that failed these appends,
356
+ * for support escalation. Null when the failure did not come from a server
357
+ * response — channel invalidation, or a protocol inconsistency the SDK detected
358
+ * on its own.
359
+ */
360
+ readonly requestId: string | null;
361
+ /**
362
+ * How many retries the failing request had already made: 0 when the first
363
+ * attempt failed, N when the SDK retried N times before giving up.
364
+ *
365
+ * **Use this to decide whether these rows may be duplicated in the table.** A retry
366
+ * re-sends the same rowset, and the SDK retries whenever it cannot tell that the
367
+ * previous attempt failed — a timeout or a 5xx may mean the rows were never processed,
368
+ * or that they were processed and only the response was lost. `0` means the SDK sent
369
+ * this rowset once and introduced no duplicate; `> 0` means it sent it more than once
370
+ * and an earlier attempt may already have been applied, so the rows may appear more
371
+ * than once. Reconcile or de-duplicate downstream if you need exactly-once.
372
+ *
373
+ * The signal is conservative: it counts every re-send, including an authentication
374
+ * refresh, which is rejected before the rows are processed and so cannot have
375
+ * duplicated anything. `> 0` means "a duplicate is possible", not "a duplicate
376
+ * happened". `0` means the SDK introduced no duplicate of its own, not that the rows
377
+ * are absent from the table.
378
+ *
379
+ * **Whenever `requestId` is null, neither field tells you whether the rows arrived.** A
380
+ * null id means the SDK failed this append itself rather than relaying a server response
381
+ * — channel or client invalidation, a close or drop, or an internal fault — and all of
382
+ * those fail every append still in flight, locally and immediately. That includes appends
383
+ * whose request had already gone to Snowflake and may still be applied after this handler
384
+ * runs. The accompanying `retryCount` of 0 says only that the SDK sent the rowset once;
385
+ * it is not a statement that Snowflake rejected it. Reconcile against the table if you
386
+ * need to know whether those rows landed.
387
+ *
388
+ * Read it only when `requestId` is non-null: a failure with no originating request
389
+ * reports 0 too. With the request id this names the exact attempt server-side — the
390
+ * SDK sends both as query params on every request.
391
+ */
392
+ readonly retryCount: number;
393
+ }
394
+
395
+ /**
396
+ * The payload handed to an elastic-channel success handler (registered via
397
+ * `StreamingIngestElasticChannel.setSuccessHandler`) when appends are
398
+ * acknowledged by Snowflake.
399
+ *
400
+ * Delivered once per successful acknowledgement, carrying the caller-supplied
401
+ * append tokens of every append acked together in that server-ack batch. There is
402
+ * no error field — unlike `ErrorDetail`, there is nothing to explain.
403
+ *
404
+ * Instances are created by the SDK and frozen; handlers only read this type, so
405
+ * new fields can be added over time without breaking them.
406
+ */
407
+ export class SuccessDetail {
408
+ private constructor();
409
+
410
+ /**
411
+ * The opaque, caller-supplied append tokens of the appends that were
412
+ * acknowledged together. Never empty; a token reused across appends appears once
413
+ * per acknowledged append (not deduplicated). Frozen at construction — the array
414
+ * itself cannot be modified, though the tokens inside it are the caller's own
415
+ * objects.
416
+ */
417
+ readonly appendTokens: readonly unknown[];
418
+ /**
419
+ * The Snowflake request id of the server response that acknowledged these
420
+ * appends, for support escalation.
421
+ */
422
+ readonly requestId: string;
423
+ /**
424
+ * How many retries that request took: 0 when the first attempt was acknowledged,
425
+ * N when the SDK retried N times before Snowflake accepted it.
426
+ *
427
+ * **Use this to decide whether these rows may be duplicated in the table.** A retry
428
+ * re-sends the same rowset, and the SDK retries whenever it cannot tell that the
429
+ * previous attempt failed — a timeout or a 5xx may mean the rows were never processed,
430
+ * or that they were processed and only the response was lost. `0` means the SDK sent
431
+ * this rowset once and introduced no duplicate; `> 0` means it sent it more than once
432
+ * and an earlier attempt may already have been applied, so the rows may appear more
433
+ * than once. Reconcile or de-duplicate downstream if you need exactly-once.
434
+ *
435
+ * The signal is conservative: it counts every re-send, including an authentication
436
+ * refresh, which is rejected before the rows are processed and so cannot have
437
+ * duplicated anything. `> 0` means "a duplicate is possible", not "a duplicate
438
+ * happened".
439
+ *
440
+ * Read it only when `requestId` is non-null: an acknowledgement with no originating
441
+ * request reports 0 too. A steadily non-zero count also means the SDK is absorbing
442
+ * retries on your behalf.
443
+ */
444
+ readonly retryCount: number;
445
+ }
446
+
330
447
  /**
331
448
  * An elastic channel for streaming data into a Snowflake table.
332
449
  *
333
450
  * Unlike a regular channel, an elastic channel has no offset tokens and its
334
451
  * lifecycle is tied to the client — there is no close method, and the same
335
452
  * instance is returned on repeated calls to `client.getElasticChannel()`.
336
- * `appendRow`/`appendRows` resolve when Snowflake acknowledges the rows.
453
+ * Appends come in two flavours: fire-and-forget `appendRow`/`appendRows` return
454
+ * nothing, while `appendRowWithWait`/`appendRowsWithWait` return a promise that
455
+ * resolves when Snowflake acknowledges the rows. Every append must carry an opaque
456
+ * `appendToken` — required, but nullable — handed back to the handler registered via
457
+ * `setErrorHandler` when that append fails asynchronously, or to `setSuccessHandler`
458
+ * when it is acknowledged; pass `null` to opt that append out of both handlers.
337
459
  *
338
460
  * Create instances using `client.getElasticChannel()`.
339
461
  */
@@ -353,24 +475,119 @@ export class StreamingIngestElasticChannel {
353
475
  /** The binary input format. One of: "BASE64", "HEX", "UTF-8". */
354
476
  readonly binaryInputFormat: string;
355
477
 
478
+ /**
479
+ * Append a single row into the elastic channel without waiting for the
480
+ * acknowledgement (fire-and-forget). The append is still registered for the
481
+ * error handler, so an asynchronous failure is reported through
482
+ * `setErrorHandler` (keyed by `appendToken`), just not through a promise. Use
483
+ * `appendRowWithWait` when you need one to await.
484
+ *
485
+ * @param row - Row data as column-name to value pairs
486
+ * @param appendToken - Required caller-supplied opaque id for this append, handed
487
+ * to the `setErrorHandler` handler on asynchronous failure, or to the
488
+ * `setSuccessHandler` handler on acknowledgement. Not sent to Snowflake; any
489
+ * value (including `""`) is valid. The argument is mandatory: pass `null`
490
+ * explicitly to leave the append untracked for both handlers. The token is
491
+ * retained in memory until the append is acknowledged, so a large object raises
492
+ * the memory footprint of in-flight appends — prefer a small value.
493
+ * @throws TypeError If row is null or undefined
494
+ * @throws StreamingIngestError If the row appending fails synchronously
495
+ */
496
+ appendRow(row: Record<string, any>, appendToken: unknown): void;
497
+
498
+ /**
499
+ * Append multiple rows into the elastic channel without waiting for the
500
+ * acknowledgement (fire-and-forget). The append is still registered for the
501
+ * error handler, so an asynchronous failure is reported through
502
+ * `setErrorHandler` (keyed by `appendToken`), just not through a promise. Use
503
+ * `appendRowsWithWait` when you need one to await.
504
+ *
505
+ * @param rows - Array of row objects (column-name to value pairs)
506
+ * @param appendToken - Required caller-supplied opaque id for this batch, handed
507
+ * to the `setErrorHandler` handler on asynchronous failure, or to the
508
+ * `setSuccessHandler` handler on acknowledgement. Not sent to Snowflake; any
509
+ * value (including `""`) is valid. The argument is mandatory: pass `null`
510
+ * explicitly to leave the append untracked for both handlers. The token is
511
+ * retained in memory until the append is acknowledged, so a large object raises
512
+ * the memory footprint of in-flight appends — prefer a small value.
513
+ * @throws TypeError If rows is not an array
514
+ * @throws StreamingIngestError If the rows appending fails synchronously
515
+ */
516
+ appendRows(rows: Record<string, any>[], appendToken: unknown): void;
517
+
356
518
  /**
357
519
  * Append a single row into the elastic channel. The returned promise resolves
358
- * when Snowflake acknowledges the row.
520
+ * when Snowflake acknowledges the row. If an error handler is registered it
521
+ * also fires on an asynchronous failure — dual delivery.
359
522
  *
360
523
  * @param row - Row data as column-name to value pairs
524
+ * @param appendToken - Required caller-supplied opaque id for this append, handed
525
+ * to the `setErrorHandler` handler on asynchronous failure, or to the
526
+ * `setSuccessHandler` handler on acknowledgement. Not sent to Snowflake; any
527
+ * value (including `""`) is valid. The argument is mandatory: pass `null`
528
+ * explicitly to leave the append untracked for both handlers. The token is
529
+ * retained in memory until the append is acknowledged, so a large object raises
530
+ * the memory footprint of in-flight appends — prefer a small value.
361
531
  * @throws StreamingIngestError If the row appending fails
362
532
  */
363
- appendRow(row: Record<string, any>): Promise<void>;
533
+ appendRowWithWait(row: Record<string, any>, appendToken: unknown): Promise<void>;
364
534
 
365
535
  /**
366
536
  * Append multiple rows into the elastic channel. The returned promise resolves
367
- * when Snowflake acknowledges the batch.
537
+ * when Snowflake acknowledges the batch. If an error handler is registered it
538
+ * also fires on an asynchronous failure — dual delivery.
368
539
  *
369
540
  * @param rows - Array of row objects (column-name to value pairs)
541
+ * @param appendToken - Required caller-supplied opaque id for this batch, handed
542
+ * to the `setErrorHandler` handler on asynchronous failure, or to the
543
+ * `setSuccessHandler` handler on acknowledgement. Not sent to Snowflake; any
544
+ * value (including `""`) is valid. The argument is mandatory: pass `null`
545
+ * explicitly to leave the append untracked for both handlers. The token is
546
+ * retained in memory until the append is acknowledged, so a large object raises
547
+ * the memory footprint of in-flight appends — prefer a small value.
370
548
  * @throws TypeError If rows is not an array
371
549
  * @throws StreamingIngestError If the rows appending fails
372
550
  */
373
- appendRows(rows: Record<string, any>[]): Promise<void>;
551
+ appendRowsWithWait(rows: Record<string, any>[], appendToken: unknown): Promise<void>;
552
+
553
+ /**
554
+ * Register (or replace) a handler invoked when appends fail *asynchronously*,
555
+ * i.e. after the append call already returned. Fires once per failure event
556
+ * with a single {@link ErrorDetail} bundling the `appendToken`s that failed
557
+ * together (a server-ack batch, or every in-flight append on invalidation) and
558
+ * their common error, so callers need not await each promise. Both append
559
+ * flavours are covered: it is the only failure signal for the fire-and-forget
560
+ * ones, and an additional one for the `WithWait` ones, whose promise still
561
+ * rejects. Only appends given an `appendToken` appear, and tokens are opaque and
562
+ * not deduplicated. Synchronous failures (closed channel, serialization, bad
563
+ * arguments) throw from the append call and never reach the handler. A handler
564
+ * that throws is caught and logged.
565
+ *
566
+ * @param handler - Called with one {@link ErrorDetail} per failure event
567
+ * @throws TypeError If handler is not a function, or declares more than one
568
+ * parameter (it is called with a single {@link ErrorDetail})
569
+ */
570
+ setErrorHandler(handler: (detail: ErrorDetail) => void): void;
571
+
572
+ /**
573
+ * Register (or replace) a handler invoked when appends are acknowledged by
574
+ * Snowflake. Fires once per successful ack batch with a single
575
+ * {@link SuccessDetail} bundling the `appendToken`s acknowledged together, so
576
+ * callers need not await each promise. Both append flavours are covered: it is
577
+ * the only success signal for the fire-and-forget ones, and an additional one for
578
+ * the `WithWait` ones, whose promise still resolves. Purely opt-in — without a
579
+ * registered handler nothing is collected. Only appends given an `appendToken`
580
+ * appear, and tokens are opaque and not deduplicated. A handler that throws is
581
+ * caught and logged.
582
+ *
583
+ * Like the error handler, this runs on the acknowledgement path, so a slow
584
+ * handler delays subsequent acknowledgements — keep it short.
585
+ *
586
+ * @param handler - Called with one {@link SuccessDetail} per successful ack batch
587
+ * @throws TypeError If handler is not a function, or declares more than one
588
+ * parameter (it is called with a single {@link SuccessDetail})
589
+ */
590
+ setSuccessHandler(handler: (detail: SuccessDetail) => void): void;
374
591
 
375
592
  /**
376
593
  * Initiate a flush of all buffered data for this channel but do not wait for
@@ -380,6 +597,18 @@ export class StreamingIngestElasticChannel {
380
597
  */
381
598
  initiateFlush(): void;
382
599
 
600
+ /**
601
+ * Wait for all buffered data in this channel to be flushed to Snowflake.
602
+ *
603
+ * @param options - Wait options
604
+ * @throws StreamingIngestError If waiting for the flush fails
605
+ * @throws Error If the timeout is reached
606
+ */
607
+ waitForFlush(options?: {
608
+ /** Optional timeout in milliseconds. */
609
+ timeoutMs?: number;
610
+ }): Promise<void>;
611
+
383
612
  /**
384
613
  * Get the current status of this channel.
385
614
  *
@@ -440,8 +669,16 @@ export class ChannelStatus {
440
669
  export class StreamingIngestError extends Error {
441
670
  /** The error code (for example, "ConfigError", "ChannelNotFound"). */
442
671
  readonly errorCode: string;
443
- /** The error message. Inherited from `Error`; redeclared for autodoc visibility. */
672
+ /**
673
+ * Decorated error message: `{errorCode}: {detail} (HTTP {code} {name})`.
674
+ * Inherited from `Error`; this is what `stack` / `console.error` print.
675
+ */
444
676
  readonly message: string;
677
+ /**
678
+ * Original detail passed to the constructor, without error-code / HTTP-status
679
+ * decoration. Analog of Java `SFException#getDetailMessage()`.
680
+ */
681
+ readonly detail: string;
445
682
  /** HTTP status code indicating error category. */
446
683
  readonly httpStatusCode: number;
447
684
  /** HTTP status name (for example, "Bad Request"). */
package/src/index.js CHANGED
@@ -27,6 +27,8 @@ const {
27
27
  } = require("./client.js");
28
28
  const { StreamingIngestChannel } = require("./channel.js");
29
29
  const { StreamingIngestElasticChannel } = require("./elastic_channel.js");
30
+ const { ErrorDetail } = require("./error_detail.js");
31
+ const { SuccessDetail } = require("./success_detail.js");
30
32
 
31
33
  // Load native addon and bootstrap the Rust runtime
32
34
  const native = loadNativeAddon();
@@ -52,6 +54,8 @@ module.exports = {
52
54
  StreamingIngestElasticChannel,
53
55
  StreamingIngestError,
54
56
  StreamingIngestErrorCode,
57
+ ErrorDetail,
58
+ SuccessDetail,
55
59
  ChannelStatus,
56
60
  // E2E testing only - available when compiled with e2e_test feature
57
61
  registerMockRoute: native.registerMockRoute,
package/src/loader.js CHANGED
@@ -20,7 +20,7 @@
20
20
  const path = require("node:path");
21
21
  const fs = require("node:fs");
22
22
 
23
- const LOG_PREFIX = "[snowpipe-streaming]";
23
+ const { LOG_PREFIX } = require("./log.js");
24
24
 
25
25
  // Verbose loader breadcrumbs are gated on SS_LOG_LEVEL=debug|trace, matching
26
26
  // the env var the Rust core and Java/Python SDKs already use (see
package/src/log.js ADDED
@@ -0,0 +1,27 @@
1
+ // Copyright 2026 Snowflake Inc.
2
+ // SPDX-License-Identifier: Apache-2.0
3
+ //
4
+ // Licensed under the Apache License, Version 2.0 (the "License");
5
+ // you may not use this file except in compliance with the License.
6
+ // You may obtain a copy of the License at
7
+ //
8
+ // http://www.apache.org/licenses/LICENSE-2.0
9
+ //
10
+ // Unless required by applicable law or agreed to in writing, software
11
+ // distributed under the License is distributed on an "AS IS" BASIS,
12
+ // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13
+ // See the License for the specific language governing permissions and
14
+ // limitations under the License.
15
+
16
+ "use strict";
17
+
18
+ /**
19
+ * Prefix for every line this SDK writes to stderr, so SDK output is
20
+ * attributable in a customer's log stream. Used by the native-addon loader and
21
+ * by the elastic channel's acknowledgement path, which are the only places the
22
+ * SDK logs on its own (everything else surfaces as a thrown or rejected
23
+ * StreamingIngestError).
24
+ */
25
+ const LOG_PREFIX = "[snowpipe-streaming]";
26
+
27
+ module.exports = { LOG_PREFIX };
@@ -32,6 +32,14 @@
32
32
  // directly — same end state, driven from JS.
33
33
 
34
34
  const { StreamingIngestError } = require("./errors.js");
35
+ const { ErrorDetail } = require("./error_detail.js");
36
+ const { SuccessDetail } = require("./success_detail.js");
37
+ const { LOG_PREFIX } = require("./log.js");
38
+
39
+ // A handler runs inline on the acknowledgement tick, so blocking it stalls the whole event loop —
40
+ // every channel on this client, and the rest of the application. Warn past this many milliseconds.
41
+ // Timing uses the global `performance` (Node >= 16; this package requires >= 20), like `Buffer`.
42
+ const SLOW_HANDLER_WARN_THRESHOLD_MILLIS = 200;
35
43
 
36
44
  function buildError(success, errorCode, message, httpStatusCode, httpStatusName) {
37
45
  if (success) {
@@ -52,12 +60,23 @@ function buildError(success, errorCode, message, httpStatusCode, httpStatusName)
52
60
  * for JS-driven invalidation. The native reference is plugged in after the
53
61
  * channel is opened (chicken-and-egg: these callbacks are passed *into* the
54
62
  * open call, so they exist before the native handle does).
63
+ *
64
+ * Also owns the customer error handler, invoked once per failure event with a
65
+ * single {@link ErrorDetail} carrying the append tokens of every append that
66
+ * failed under a common error, and the customer success handler, invoked once per
67
+ * successful server-ack batch with a single {@link SuccessDetail} carrying that
68
+ * batch's append tokens. Tokens are opaque and optional, so only entries
69
+ * that carry one are reported. This class is elastic-only, so the append token
70
+ * rides on the pending-map value; Java/Python need a parallel token map because
71
+ * their callback also serves regular channels.
55
72
  */
56
73
  class ElasticChannelAckState {
57
74
  constructor(pendingFutures) {
58
75
  this._pendingFutures = pendingFutures;
59
76
  this._invalidated = false;
60
77
  this._nativeChannel = null;
78
+ this._errorHandler = null;
79
+ this._successHandler = null;
61
80
  this.markComplete = this._markComplete.bind(this);
62
81
  this.invalidateAll = this._invalidateAll.bind(this);
63
82
  }
@@ -67,8 +86,149 @@ class ElasticChannelAckState {
67
86
  this._nativeChannel = nativeChannel;
68
87
  }
69
88
 
89
+ /**
90
+ * Register (or replace) the customer error handler. The argument is validated
91
+ * by `StreamingIngestElasticChannel.setErrorHandler`.
92
+ */
93
+ setErrorHandler(handler) {
94
+ this._errorHandler = handler;
95
+ }
96
+
97
+ /**
98
+ * Register (or replace) the customer success handler. The argument is validated
99
+ * by `StreamingIngestElasticChannel.setSuccessHandler`.
100
+ */
101
+ setSuccessHandler(handler) {
102
+ this._successHandler = handler;
103
+ }
104
+
105
+ /**
106
+ * Invoke the error handler once for a batch of append tokens that failed with a
107
+ * common error. Both are bundled into the single {@link ErrorDetail} the handler
108
+ * takes (mirroring Java's and Python's `ErrorDetail`), so new fields can be
109
+ * added later without changing the handler signature. No-op without a handler
110
+ * or on an empty batch. A throwing handler is caught and logged so it can't
111
+ * break the acknowledgement path.
112
+ *
113
+ * @param {unknown[]} appendTokens - Opaque append tokens of the appends that failed together.
114
+ * @param {StreamingIngestError} error - The error that failed them.
115
+ * @param {string | null} requestId - Snowflake request id of the failing response, or null.
116
+ * @param {number} retryCount - How many retries that request took; 0 with a null id, where it
117
+ * carries no meaning.
118
+ */
119
+ fireErrorHandler(appendTokens, error, requestId, retryCount) {
120
+ if (this._errorHandler === null || appendTokens == null || appendTokens.length === 0) {
121
+ return;
122
+ }
123
+ const start = performance.now();
124
+ try {
125
+ this._errorHandler(new ErrorDetail(appendTokens, error, requestId, retryCount));
126
+ } catch (handlerErr) {
127
+ // eslint-disable-next-line no-console
128
+ console.error(
129
+ `${LOG_PREFIX} Elastic channel error handler threw for ` +
130
+ `appendTokens=[${appendTokens.join(", ")}]; ignoring.`,
131
+ handlerErr,
132
+ );
133
+ } finally {
134
+ this._logHandlerDuration("error", start, appendTokens.length);
135
+ }
136
+ }
137
+
138
+ /**
139
+ * Warn when a handler invocation crossed {@link SLOW_HANDLER_WARN_THRESHOLD_MILLIS}.
140
+ * It runs inline on the event-loop tick that drains the Rust acknowledgement, so
141
+ * blocking it stalls this client's acknowledgements and everything else on the
142
+ * loop. Only the slow case is logged — a per-invocation line would fire once per
143
+ * acknowledgement batch, effectively once per append on a busy pipe.
144
+ *
145
+ * @param {string} handlerKind - "error" or "success", for the log message.
146
+ * @param {number} start - `performance.now()` captured before invoking the handler.
147
+ * @param {number} appendTokenCount - how many append tokens the invocation carried.
148
+ */
149
+ _logHandlerDuration(handlerKind, start, appendTokenCount) {
150
+ const elapsedMillis = performance.now() - start;
151
+ if (elapsedMillis >= SLOW_HANDLER_WARN_THRESHOLD_MILLIS) {
152
+ // eslint-disable-next-line no-console
153
+ console.warn(
154
+ `${LOG_PREFIX} Elastic channel ${handlerKind} handler took ` +
155
+ `${elapsedMillis.toFixed(1)} ms for ${appendTokenCount} appendToken(s); it is blocking ` +
156
+ `the event loop — offload heavy work to a worker thread or your own queue.`,
157
+ );
158
+ }
159
+ }
160
+
161
+ /**
162
+ * Invoke the success handler once for a batch of append tokens acknowledged
163
+ * together. They are bundled into the single {@link SuccessDetail} the handler
164
+ * takes, so new fields can be added later without changing the handler
165
+ * signature. No-op without a handler or on an empty batch. A throwing handler is
166
+ * caught and logged so it can't break the acknowledgement path.
167
+ *
168
+ * @param {unknown[]} appendTokens - Opaque append tokens of the appends acknowledged together.
169
+ * @param {string | null} requestId - Snowflake request id of the acking response, or null.
170
+ * @param {number} retryCount - How many retries that request took.
171
+ */
172
+ fireSuccessHandler(appendTokens, requestId, retryCount) {
173
+ if (requestId === null || requestId === undefined) {
174
+ // SuccessDetail.requestId is declared non-null: an acknowledgement can only come from a
175
+ // request Snowflake answered. Reaching here means the FFI bridge dropped the field. Log it
176
+ // rather than withhold the customer's notification over a support-escalation value.
177
+ // eslint-disable-next-line no-console
178
+ console.error(
179
+ `${LOG_PREFIX} Elastic channel success ack carried no requestId for appendTokens=` +
180
+ `${appendTokens}; this is an SDK defect, the acknowledgement itself is still reported.`,
181
+ );
182
+ }
183
+ if (this._successHandler === null || appendTokens.length === 0) {
184
+ return;
185
+ }
186
+ const start = performance.now();
187
+ try {
188
+ this._successHandler(new SuccessDetail(appendTokens, requestId, retryCount));
189
+ } catch (handlerErr) {
190
+ // eslint-disable-next-line no-console
191
+ console.error(
192
+ `${LOG_PREFIX} Elastic channel success handler threw for ` +
193
+ `appendTokens=[${appendTokens.join(", ")}]; ignoring.`,
194
+ handlerErr,
195
+ );
196
+ } finally {
197
+ this._logHandlerDuration("success", start, appendTokens.length);
198
+ }
199
+ }
200
+
201
+ /**
202
+ * Reject and clear every pending promise, returning the append tokens of the
203
+ * appends that carried one so the handler can be fired once the whole loop is
204
+ * done. Untracked appends (no append token) reject without being reported.
205
+ */
206
+ _rejectAllPending(error) {
207
+ const appendTokens = [];
208
+ for (const pending of this._pendingFutures.values()) {
209
+ pending.reject(error);
210
+ if (pending.appendToken != null) {
211
+ appendTokens.push(pending.appendToken);
212
+ }
213
+ }
214
+ this._pendingFutures.clear();
215
+ return appendTokens;
216
+ }
217
+
70
218
  _markComplete(payload) {
71
- const { futureIdRanges, success, errorCode, message, httpStatusCode, httpStatusName } = payload;
219
+ const {
220
+ futureIdRanges,
221
+ success,
222
+ errorCode,
223
+ message,
224
+ httpStatusCode,
225
+ httpStatusName,
226
+ // requestId defaults to null (not undefined) so a missing native field still honors the
227
+ // declared `string | null` contract. retryCount defaults to 0: it is a plain number, and a
228
+ // null requestId is what marks the pair absent.
229
+ requestId = null,
230
+ retryCount = 0,
231
+ } = payload;
72
232
 
73
233
  if (futureIdRanges.length % 2 !== 0) {
74
234
  const errMsg = `future_id_ranges has odd length ${futureIdRanges.length}, expected even (flattened [min, max] pairs)`;
@@ -81,6 +241,8 @@ class ElasticChannelAckState {
81
241
  const error = buildError(success, errorCode, message, httpStatusCode, httpStatusName);
82
242
 
83
243
  const missingIds = [];
244
+ const errorTokensToNotify = [];
245
+ const successTokensToNotify = [];
84
246
  for (let i = 0; i < futureIdRanges.length; i += 2) {
85
247
  const min = Number(futureIdRanges[i]);
86
248
  const max = Number(futureIdRanges[i + 1]);
@@ -93,12 +255,27 @@ class ElasticChannelAckState {
93
255
  this._pendingFutures.delete(fid);
94
256
  if (error) {
95
257
  pending.reject(error);
258
+ // Only appends the caller gave an append token are surfaced to the handler.
259
+ if (pending.appendToken != null) {
260
+ errorTokensToNotify.push(pending.appendToken);
261
+ }
96
262
  } else {
97
263
  pending.resolve();
264
+ // Same rule on the success side: an untracked append has nothing to report.
265
+ if (pending.appendToken != null) {
266
+ successTokensToNotify.push(pending.appendToken);
267
+ }
98
268
  }
99
269
  }
100
270
  }
101
271
 
272
+ // Fire once after the resolve/reject loop, so a slow or throwing handler can't
273
+ // delay or skip a sibling append. Mirrors Java firing outside the ack lock.
274
+ // A batch is wholly successful or wholly failed, so only one of these fires.
275
+ // Both carry the request id, and retry count, of the response that drove this ack.
276
+ this.fireErrorHandler(errorTokensToNotify, error, requestId, retryCount);
277
+ this.fireSuccessHandler(successTokensToNotify, requestId, retryCount);
278
+
102
279
  // `_invalidated` gate matches Java/Python: after invalidateAll has run,
103
280
  // the map is empty and further mark-complete callbacks for in-flight
104
281
  // batches are expected to find nothing — don't escalate that to fatal.
@@ -120,10 +297,11 @@ class ElasticChannelAckState {
120
297
  payload.httpStatusCode,
121
298
  payload.httpStatusName,
122
299
  ) || new StreamingIngestError("Fatal", "Elastic channel invalidated", 409, "Conflict");
123
- for (const pending of this._pendingFutures.values()) {
124
- pending.reject(error);
125
- }
126
- this._pendingFutures.clear();
300
+ // No request id: invalidation fails every in-flight append at once and is not
301
+ // attributable to the one server response any of them was waiting on. Same in-flight race as
302
+ // the close/drop path below: an append whose request was already on the wire is failed here and
303
+ // that request may still be applied, which the ErrorDetail contract covers via the null id.
304
+ this.fireErrorHandler(this._rejectAllPending(error), error, null, 0);
127
305
  }
128
306
 
129
307
  /**
@@ -132,25 +310,32 @@ class ElasticChannelAckState {
132
310
  * via `_invalidated`. Promises are rejected first so the user observation
133
311
  * doesn't depend on the FFI call succeeding; the native invalidate is
134
312
  * best-effort — Rust will fire `invalidateAll` on its next informer
135
- * tick, which finds an empty map and is a no-op.
313
+ * tick, which finds an empty map and is a no-op. The error handler fires last,
314
+ * after both the rejections and the FFI call, with no request id — the error is
315
+ * ours, not a Snowflake response.
136
316
  */
137
317
  _invalidateLocally(error) {
138
318
  if (this._invalidated) {
139
319
  return;
140
320
  }
141
321
  this._invalidated = true;
142
- for (const pending of this._pendingFutures.values()) {
143
- pending.reject(error);
144
- }
145
- this._pendingFutures.clear();
322
+ const errorTokensToNotify = this._rejectAllPending(error);
146
323
  if (this._nativeChannel) {
147
324
  try {
148
- this._nativeChannel.invalidateChannel(error.message);
325
+ this._nativeChannel.invalidateChannel(error.detail);
149
326
  } catch (invErr) {
150
327
  // eslint-disable-next-line no-console
151
- console.error("Elastic channel invalidate_channel FFI call failed:", invErr);
328
+ console.error(`${LOG_PREFIX} Elastic channel invalidate_channel FFI call failed:`, invErr);
152
329
  }
153
330
  }
331
+ // Races with an in-flight request, and reports the pessimistic side of it. An append whose
332
+ // request is already on the wire is failed here locally; if it then lands, the rows are in the
333
+ // table even though the customer was told the append failed. The handler reports a null
334
+ // requestId and retryCount 0, which per the ErrorDetail contract reads as "the SDK sent this
335
+ // once, so it introduced no duplicate" -- true, but not a claim that the rows are absent.
336
+ // Nothing here can close the race: the promise has to be settled at close time and the
337
+ // request's outcome is not knowable yet. A caller needing certainty must reconcile downstream.
338
+ this.fireErrorHandler(errorTokensToNotify, error, null, 0);
154
339
  }
155
340
  }
156
341
 
@@ -0,0 +1,73 @@
1
+ // Copyright 2026 Snowflake Inc.
2
+ // SPDX-License-Identifier: Apache-2.0
3
+ //
4
+ // Licensed under the Apache License, Version 2.0 (the "License");
5
+ // you may not use this file except in compliance with the License.
6
+ // You may obtain a copy of the License at
7
+ //
8
+ // http://www.apache.org/licenses/LICENSE-2.0
9
+ //
10
+ // Unless required by applicable law or agreed to in writing, software
11
+ // distributed under the License is distributed on an "AS IS" BASIS,
12
+ // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13
+ // See the License for the specific language governing permissions and
14
+ // limitations under the License.
15
+
16
+ "use strict";
17
+
18
+ /**
19
+ * The payload handed to an elastic-channel success handler (registered via
20
+ * {@link StreamingIngestElasticChannel#setSuccessHandler}) when appends are
21
+ * acknowledged by Snowflake.
22
+ *
23
+ * Delivered once per successful acknowledgement, carrying the caller-supplied
24
+ * append tokens of every append that was acked together in that server-ack
25
+ * batch. There is no error field — the counterpart to {@link ErrorDetail}, whose
26
+ * `error` explains a failure, has nothing to explain.
27
+ *
28
+ * Instances are created by the SDK and deeply frozen — both the object and its
29
+ * `appendTokens` array — so a handler cannot mutate what a sibling handler or a
30
+ * later log statement will read. Handlers only read this type, so new fields can
31
+ * be added over time without breaking them.
32
+ */
33
+ class SuccessDetail {
34
+ /**
35
+ * @param {unknown[]} appendTokens - The opaque, caller-supplied append tokens of
36
+ * the appends that were acknowledged together. Never empty; a token reused
37
+ * across appends appears once per acknowledged append (not deduplicated).
38
+ * Frozen here, so callers must pass an array they are handing over.
39
+ * @param {string} requestId - The Snowflake request id of the server response that
40
+ * acknowledged these appends, for support escalation.
41
+ * @param {number} retryCount - How many retries that request took: 0 when the
42
+ * first attempt was acknowledged, N when the SDK retried N times before
43
+ * Snowflake accepted it.
44
+ *
45
+ * **Use this to decide whether these rows may be duplicated in the table.** A retry
46
+ * re-sends the same rowset, and the SDK retries whenever it cannot tell that the
47
+ * previous attempt failed -- a timeout or a 5xx may mean the rows were never
48
+ * processed, or that they were processed and only the response was lost. `0` means
49
+ * the SDK sent this rowset once and introduced no duplicate; `> 0` means it sent it
50
+ * more than once and an earlier attempt may already have been applied, so the rows
51
+ * may appear more than once. Reconcile or de-duplicate downstream if you need
52
+ * exactly-once. The signal is conservative: it counts every re-send, including an
53
+ * authentication refresh, which is rejected before the rows are processed and so
54
+ * cannot have duplicated anything -- `> 0` means "a duplicate is possible", not
55
+ * "a duplicate happened".
56
+ *
57
+ * Read it only when `requestId` is non-null: an acknowledgement with no originating
58
+ * request reports 0 too. A steadily non-zero count also means the SDK is absorbing
59
+ * retries on your behalf.
60
+ */
61
+ constructor(appendTokens, requestId, retryCount) {
62
+ // Freeze the array as well as the instance: `Object.freeze(this)` alone would
63
+ // still let a handler push into or reorder appendTokens. Only the array
64
+ // itself is frozen, not the tokens inside it — those are the caller's own
65
+ // opaque objects and are handed back untouched.
66
+ this.appendTokens = Object.freeze(appendTokens);
67
+ this.requestId = requestId;
68
+ this.retryCount = retryCount;
69
+ Object.freeze(this);
70
+ }
71
+ }
72
+
73
+ module.exports = { SuccessDetail };