snowpipe-streaming 1.7.1 → 1.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +7 -7
- package/src/client.js +4 -1
- package/src/elastic_channel.js +238 -15
- package/src/error_detail.js +95 -0
- package/src/errors.js +19 -6
- package/src/index.d.ts +231 -6
- package/src/index.js +4 -0
- package/src/loader.js +1 -1
- package/src/log.js +27 -0
- package/src/mark_complete_callback.js +197 -12
- package/src/success_detail.js +73 -0
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "snowpipe-streaming",
|
|
3
|
-
"version": "1.
|
|
3
|
+
"version": "1.8.0",
|
|
4
4
|
"description": "Snowflake Streaming Ingest SDK for Node.js",
|
|
5
5
|
"main": "src/index.js",
|
|
6
6
|
"types": "src/index.d.ts",
|
|
@@ -27,11 +27,11 @@
|
|
|
27
27
|
"typedoc": "^0.28.19"
|
|
28
28
|
},
|
|
29
29
|
"optionalDependencies": {
|
|
30
|
-
"snowpipe-streaming-linux-x64-gnu": "1.
|
|
31
|
-
"snowpipe-streaming-linux-x64-musl": "1.
|
|
32
|
-
"snowpipe-streaming-linux-arm64-gnu": "1.
|
|
33
|
-
"snowpipe-streaming-linux-arm64-musl": "1.
|
|
34
|
-
"snowpipe-streaming-darwin-arm64": "1.
|
|
35
|
-
"snowpipe-streaming-win32-x64-msvc": "1.
|
|
30
|
+
"snowpipe-streaming-linux-x64-gnu": "1.8.0",
|
|
31
|
+
"snowpipe-streaming-linux-x64-musl": "1.8.0",
|
|
32
|
+
"snowpipe-streaming-linux-arm64-gnu": "1.8.0",
|
|
33
|
+
"snowpipe-streaming-linux-arm64-musl": "1.8.0",
|
|
34
|
+
"snowpipe-streaming-darwin-arm64": "1.8.0",
|
|
35
|
+
"snowpipe-streaming-win32-x64-msvc": "1.8.0"
|
|
36
36
|
}
|
|
37
37
|
}
|
package/src/client.js
CHANGED
|
@@ -124,6 +124,9 @@ class StreamingIngestClient {
|
|
|
124
124
|
// Cache the in-flight promise so concurrent callers share one open and one
|
|
125
125
|
// instance. The pending-futures map is shared between the FFI callbacks and
|
|
126
126
|
// the channel wrapper (the JS side owns per-message-ack promise resolution).
|
|
127
|
+
// No client-level callback registry to publish into (unlike Java/Python), so a
|
|
128
|
+
// failed open leaves nothing behind — the ack state is only reachable via the
|
|
129
|
+
// channel returned on success.
|
|
127
130
|
this._elasticChannelPromise = (async () => {
|
|
128
131
|
const pendingFutures = new Map();
|
|
129
132
|
const ackState = new ElasticChannelAckState(pendingFutures);
|
|
@@ -133,7 +136,7 @@ class StreamingIngestClient {
|
|
|
133
136
|
// Plug the native handle in before exposing the channel so the ack
|
|
134
137
|
// state can drive `invalidateChannel` from JS on detected anomalies.
|
|
135
138
|
ackState.setNativeChannel(nativeChannel);
|
|
136
|
-
return new StreamingIngestElasticChannel(nativeChannel, pendingFutures);
|
|
139
|
+
return new StreamingIngestElasticChannel(nativeChannel, pendingFutures, ackState);
|
|
137
140
|
})().catch((e) => {
|
|
138
141
|
// Don't cache failures (e.g. closed client, transient open error) so a retry can re-open.
|
|
139
142
|
this._elasticChannelPromise = null;
|
package/src/elastic_channel.js
CHANGED
|
@@ -18,23 +18,44 @@
|
|
|
18
18
|
const { callFFI, StreamingIngestError } = require("./errors.js");
|
|
19
19
|
const { ChannelStatus } = require("./channel_status.js");
|
|
20
20
|
const { serializeRowsToNdjson } = require("./row_serializer.js");
|
|
21
|
+
const { LOG_PREFIX } = require("./log.js");
|
|
22
|
+
|
|
23
|
+
// Default for the appendToken parameter, used to tell "caller omitted the argument" apart from
|
|
24
|
+
// "caller passed a token" — including an explicit null, which is a valid token value meaning
|
|
25
|
+
// "leave this append untracked". appendToken is required but nullable (matching Java's positional
|
|
26
|
+
// parameter and Python's default-less argument), so opting out has to be spelled out as null
|
|
27
|
+
// rather than achieved by leaving the argument off. Checked in _appendRowsInternal, which every
|
|
28
|
+
// public append funnels through: a forwarded-but-omitted argument arrives as undefined, and a
|
|
29
|
+
// default parameter applies to undefined however it got there.
|
|
30
|
+
const APPEND_TOKEN_OMITTED = Symbol("appendTokenOmitted");
|
|
21
31
|
|
|
22
32
|
/**
|
|
23
33
|
* Elastic channel for streaming data into a Snowflake table.
|
|
24
34
|
*
|
|
25
35
|
* Unlike a regular channel, an elastic channel has no offset tokens and its
|
|
26
|
-
* lifecycle is tied to the client (no close method).
|
|
27
|
-
*
|
|
36
|
+
* lifecycle is tied to the client (no close method). Appends come in two
|
|
37
|
+
* flavours: fire-and-forget appendRow/appendRows return nothing, while
|
|
38
|
+
* appendRowWithWait/appendRowsWithWait return a Promise that resolves when
|
|
39
|
+
* Snowflake acknowledges the rows. All four take a required (but nullable)
|
|
40
|
+
* caller-supplied append token and register the append for both handlers, so
|
|
41
|
+
* setErrorHandler and setSuccessHandler report the append tokens of appends that
|
|
42
|
+
* failed or were acknowledged after their call returned. An append given a null
|
|
43
|
+
* token is not tracked for either handler — the WithWait variants' promise still
|
|
44
|
+
* settles.
|
|
28
45
|
*
|
|
29
46
|
* Created via {@link StreamingIngestClient#getElasticChannel}, not directly.
|
|
30
47
|
*/
|
|
31
48
|
class StreamingIngestElasticChannel {
|
|
32
49
|
/**
|
|
33
50
|
* @param {object} nativeChannel - Internal JsElasticChannel from FFI
|
|
34
|
-
* @param {Map<number, {resolve: Function, reject: Function}>} pendingFutures
|
|
35
|
-
* Pending-promise map shared with the mark-complete callback wired into FFI
|
|
51
|
+
* @param {Map<number, {resolve: Function, reject: Function, appendToken?: *}>} pendingFutures
|
|
52
|
+
* Pending-promise map shared with the mark-complete callback wired into FFI;
|
|
53
|
+
* an entry carries the caller-supplied append token of its append only when one
|
|
54
|
+
* was supplied.
|
|
55
|
+
* @param {import("./mark_complete_callback.js").ElasticChannelAckState} ackState
|
|
56
|
+
* Acknowledgement state that owns the customer error and success handlers.
|
|
36
57
|
*/
|
|
37
|
-
constructor(nativeChannel, pendingFutures) {
|
|
58
|
+
constructor(nativeChannel, pendingFutures, ackState) {
|
|
38
59
|
if (!(nativeChannel && typeof nativeChannel === "object")) {
|
|
39
60
|
throw new TypeError(
|
|
40
61
|
"StreamingIngestElasticChannel cannot be instantiated directly. " +
|
|
@@ -43,6 +64,7 @@ class StreamingIngestElasticChannel {
|
|
|
43
64
|
}
|
|
44
65
|
this._native = nativeChannel;
|
|
45
66
|
this._pendingFutures = pendingFutures;
|
|
67
|
+
this._ackState = ackState;
|
|
46
68
|
// Per-channel future id counter; the wrapper allocates ids so it can register
|
|
47
69
|
// the promise before the row becomes ack-eligible. The single-threaded event
|
|
48
70
|
// loop makes the plain increment safe. The id is a JS number (a safe integer,
|
|
@@ -85,44 +107,148 @@ class StreamingIngestElasticChannel {
|
|
|
85
107
|
}
|
|
86
108
|
|
|
87
109
|
/**
|
|
88
|
-
* Append a single row into the elastic channel
|
|
110
|
+
* Append a single row into the elastic channel without waiting for the
|
|
111
|
+
* acknowledgement (fire-and-forget): returns as soon as the row is buffered.
|
|
112
|
+
* The append is still registered for both handlers, so its outcome is reported
|
|
113
|
+
* through {@link #setErrorHandler} or {@link #setSuccessHandler} (keyed by
|
|
114
|
+
* appendToken), just not through a promise. Use {@link #appendRowWithWait} when you need
|
|
115
|
+
* one to await.
|
|
116
|
+
*
|
|
117
|
+
* @param {Object} row - Row data as column-name to value pairs
|
|
118
|
+
* @param {*} appendToken - Required caller-supplied opaque id for this append,
|
|
119
|
+
* handed back to the error handler if it fails asynchronously, or to the
|
|
120
|
+
* success handler once it is acknowledged. Any value is accepted, but the
|
|
121
|
+
* argument itself is mandatory: pass null explicitly to leave the append
|
|
122
|
+
* untracked for both handlers. The token is retained in memory until the
|
|
123
|
+
* append is acknowledged, so a large object raises the memory footprint of
|
|
124
|
+
* in-flight appends — prefer a small value.
|
|
125
|
+
* @returns {void}
|
|
126
|
+
* @throws {TypeError} If row is null or undefined, or appendToken is omitted
|
|
127
|
+
* @throws {StreamingIngestError} If the append fails synchronously
|
|
128
|
+
*/
|
|
129
|
+
appendRow(row, appendToken) {
|
|
130
|
+
if (row == null) {
|
|
131
|
+
throw new TypeError("row must not be null");
|
|
132
|
+
}
|
|
133
|
+
this.appendRows([row], appendToken);
|
|
134
|
+
}
|
|
135
|
+
|
|
136
|
+
/**
|
|
137
|
+
* Append multiple rows into the elastic channel without waiting for the
|
|
138
|
+
* acknowledgement (fire-and-forget): returns as soon as the rows are buffered.
|
|
139
|
+
* The append is still registered for both handlers, so its outcome is reported
|
|
140
|
+
* through {@link #setErrorHandler} or {@link #setSuccessHandler} (keyed by
|
|
141
|
+
* appendToken), just not through a promise. Use {@link #appendRowsWithWait} when you need
|
|
142
|
+
* one to await.
|
|
143
|
+
*
|
|
144
|
+
* @param {Object[]} rows - Array of row objects
|
|
145
|
+
* @param {*} appendToken - Required caller-supplied opaque id for this batch,
|
|
146
|
+
* handed back to the error handler if it fails asynchronously, or to the
|
|
147
|
+
* success handler once it is acknowledged. Any value is accepted, but the
|
|
148
|
+
* argument itself is mandatory: pass null explicitly to leave the append
|
|
149
|
+
* untracked for both handlers. The token is retained in memory until the
|
|
150
|
+
* append is acknowledged, so a large object raises the memory footprint of
|
|
151
|
+
* in-flight appends — prefer a small value.
|
|
152
|
+
* @returns {void}
|
|
153
|
+
* @throws {TypeError} If rows is null, undefined, or not an array, or appendToken
|
|
154
|
+
* is omitted
|
|
155
|
+
* @throws {StreamingIngestError} If the append fails synchronously
|
|
156
|
+
*/
|
|
157
|
+
appendRows(rows, appendToken) {
|
|
158
|
+
// The append is registered like any other, so its failure still reaches the
|
|
159
|
+
// error handler; the caller just never sees the promise. The no-op catch is
|
|
160
|
+
// what keeps that rejection from being reported as an unhandledRejection on
|
|
161
|
+
// a promise nobody is left holding. Synchronous failures (bad arguments,
|
|
162
|
+
// closed channel, serialization) still throw out of this call, matching the
|
|
163
|
+
// other SDKs' fire-and-forget variants.
|
|
164
|
+
this._appendRowsInternal(rows, appendToken).catch(() => {});
|
|
165
|
+
}
|
|
166
|
+
|
|
167
|
+
/**
|
|
168
|
+
* Append a single row into the elastic channel and wait for Snowflake to
|
|
169
|
+
* acknowledge it. If an error handler is registered it also fires on an
|
|
170
|
+
* asynchronous failure — dual delivery.
|
|
171
|
+
*
|
|
89
172
|
* @param {Object} row - Row data as column-name to value pairs
|
|
173
|
+
* @param {*} appendToken - Required caller-supplied opaque id for this append,
|
|
174
|
+
* handed back to the error handler if it fails asynchronously, or to the
|
|
175
|
+
* success handler once it is acknowledged. Any value is accepted, but the
|
|
176
|
+
* argument itself is mandatory: pass null explicitly to leave the append
|
|
177
|
+
* untracked for both handlers. The token is retained in memory until the
|
|
178
|
+
* append is acknowledged, so a large object raises the memory footprint of
|
|
179
|
+
* in-flight appends — prefer a small value.
|
|
90
180
|
* @returns {Promise<void>} Resolves when Snowflake acknowledges the row
|
|
181
|
+
* @throws {TypeError} If row is null or undefined, or appendToken is omitted
|
|
91
182
|
*/
|
|
92
|
-
async
|
|
183
|
+
async appendRowWithWait(row, appendToken) {
|
|
93
184
|
if (row == null) {
|
|
94
185
|
throw new TypeError("row must not be null");
|
|
95
186
|
}
|
|
96
|
-
return this.
|
|
187
|
+
return this.appendRowsWithWait([row], appendToken);
|
|
188
|
+
}
|
|
189
|
+
|
|
190
|
+
/**
|
|
191
|
+
* Append multiple rows into the elastic channel and wait for Snowflake to
|
|
192
|
+
* acknowledge the batch. If an error handler is registered it also fires on an
|
|
193
|
+
* asynchronous failure — dual delivery.
|
|
194
|
+
*
|
|
195
|
+
* @param {Object[]} rows - Array of row objects
|
|
196
|
+
* @param {*} appendToken - Required caller-supplied opaque id for this batch,
|
|
197
|
+
* handed back to the error handler if it fails asynchronously, or to the
|
|
198
|
+
* success handler once it is acknowledged. Any value is accepted, but the
|
|
199
|
+
* argument itself is mandatory: pass null explicitly to leave the append
|
|
200
|
+
* untracked for both handlers. The token is retained in memory until the
|
|
201
|
+
* append is acknowledged, so a large object raises the memory footprint of
|
|
202
|
+
* in-flight appends — prefer a small value.
|
|
203
|
+
* @returns {Promise<void>} Resolves when Snowflake acknowledges the batch
|
|
204
|
+
* @throws {TypeError} If rows is not an array, or appendToken is omitted
|
|
205
|
+
*/
|
|
206
|
+
async appendRowsWithWait(rows, appendToken) {
|
|
207
|
+
return this._appendRowsInternal(rows, appendToken);
|
|
97
208
|
}
|
|
98
209
|
|
|
99
210
|
/**
|
|
100
|
-
*
|
|
211
|
+
* Shared append path behind both the fire-and-forget and the WithWait
|
|
212
|
+
* variants. Always registers the append for acknowledgement so the error
|
|
213
|
+
* handler fires on failure regardless of variant; the fire-and-forget callers
|
|
214
|
+
* simply discard the returned promise.
|
|
101
215
|
*
|
|
102
216
|
* The wrapper allocates the future id and registers its promise in the shared
|
|
103
217
|
* pending map before the synchronous native append, so the ack can never
|
|
104
218
|
* precede registration. The Promise resolves later when the mark-complete
|
|
105
219
|
* callback fires with the server acknowledgement (or rejects on error).
|
|
106
220
|
*
|
|
221
|
+
* Throws rather than rejects by design: argument, serialization, and
|
|
222
|
+
* native-append failures leave the fire-and-forget callers at the call site
|
|
223
|
+
* (as in Java/Python) instead of vanishing into their discarded promise. The
|
|
224
|
+
* WithWait callers see them as a rejection, since those methods are `async`.
|
|
225
|
+
* Only a post-return failure travels on the returned promise.
|
|
226
|
+
*
|
|
107
227
|
* @param {Object[]} rows - Array of row objects
|
|
228
|
+
* @param {*} appendToken - Required (but nullable) caller-supplied opaque id for
|
|
229
|
+
* this batch. Omitting it — here or in the public method that forwarded to this
|
|
230
|
+
* one — is a TypeError; null is the way to opt out of handler tracking.
|
|
108
231
|
* @returns {Promise<void>} Resolves when Snowflake acknowledges the batch
|
|
109
232
|
*/
|
|
110
|
-
|
|
233
|
+
_appendRowsInternal(rows, appendToken = APPEND_TOKEN_OMITTED) {
|
|
111
234
|
if (rows == null) {
|
|
112
235
|
throw new TypeError("rows must not be null");
|
|
113
236
|
}
|
|
114
237
|
if (!Array.isArray(rows)) {
|
|
115
238
|
throw new TypeError("rows must be an array");
|
|
116
239
|
}
|
|
240
|
+
if (appendToken === APPEND_TOKEN_OMITTED) {
|
|
241
|
+
throw new TypeError("appendToken is required; pass null to leave the append untracked");
|
|
242
|
+
}
|
|
117
243
|
// Empty arrays are validated by the Rust core, which throws a proper error.
|
|
118
244
|
const buffer =
|
|
119
245
|
rows.length === 0 ? Buffer.alloc(0) : serializeRowsToNdjson(rows, this.binaryInputFormat);
|
|
120
246
|
|
|
121
247
|
// Allocate the id and register the promise BEFORE the native append, so the
|
|
122
248
|
// row cannot become ack-eligible before its promise is in the pending map.
|
|
123
|
-
// Building the promise out-of-line lets us stash {resolve, reject}
|
|
124
|
-
// shared map synchronously; the single-threaded event loop guarantees
|
|
125
|
-
// happens-before any queued threadsafe-function callback runs.
|
|
249
|
+
// Building the promise out-of-line lets us stash {resolve, reject, appendToken}
|
|
250
|
+
// into the shared map synchronously; the single-threaded event loop guarantees
|
|
251
|
+
// this happens-before any queued threadsafe-function callback runs.
|
|
126
252
|
const futureId = this._nextFutureId++;
|
|
127
253
|
let resolveFn;
|
|
128
254
|
let rejectFn;
|
|
@@ -131,17 +257,32 @@ class StreamingIngestElasticChannel {
|
|
|
131
257
|
rejectFn = reject;
|
|
132
258
|
});
|
|
133
259
|
const previous = this._pendingFutures.get(futureId);
|
|
134
|
-
|
|
260
|
+
const entry = { resolve: resolveFn, reject: rejectFn };
|
|
261
|
+
// Track the append token only when the caller supplied one; a null/undefined
|
|
262
|
+
// token means this append is invisible to the error handler, though a WithWait
|
|
263
|
+
// promise still rejects. Mirrors Java's nullable PendingAppend.appendToken.
|
|
264
|
+
if (appendToken != null) {
|
|
265
|
+
entry.appendToken = appendToken;
|
|
266
|
+
}
|
|
267
|
+
this._pendingFutures.set(futureId, entry);
|
|
135
268
|
if (previous !== undefined) {
|
|
136
269
|
// Monotonic per-channel ids make this a should-never-happen invariant.
|
|
137
270
|
const errMsg = `Duplicate future_id=${futureId} in pendingFutures`;
|
|
138
271
|
const err = new StreamingIngestError("Fatal", errMsg, 500, "Internal Server Error");
|
|
139
272
|
previous.reject(err);
|
|
273
|
+
// Displaced promise was already handed to its owner (or discarded by a
|
|
274
|
+
// fire-and-forget append), so its failure is asynchronous from their view;
|
|
275
|
+
// the current call throws instead, where the caller still holds its own
|
|
276
|
+
// appendToken. An untracked displaced append has no token to report. No
|
|
277
|
+
// request id: this is our own duplicate-id invariant, not a Snowflake response.
|
|
278
|
+
if (previous.appendToken != null) {
|
|
279
|
+
this._ackState.fireErrorHandler([previous.appendToken], err, null, null);
|
|
280
|
+
}
|
|
140
281
|
try {
|
|
141
282
|
this._native.invalidateChannel(errMsg);
|
|
142
283
|
} catch (invErr) {
|
|
143
284
|
// eslint-disable-next-line no-console
|
|
144
|
-
console.error(
|
|
285
|
+
console.error(`${LOG_PREFIX} Elastic channel invalidate_channel FFI call failed:`, invErr);
|
|
145
286
|
}
|
|
146
287
|
throw err;
|
|
147
288
|
}
|
|
@@ -156,6 +297,88 @@ class StreamingIngestElasticChannel {
|
|
|
156
297
|
return promise;
|
|
157
298
|
}
|
|
158
299
|
|
|
300
|
+
/**
|
|
301
|
+
* Register (or replace) a handler invoked when appends fail *asynchronously*,
|
|
302
|
+
* i.e. after the append call already returned. Fires once per failure event
|
|
303
|
+
* with a single {@link ErrorDetail} bundling the append tokens that failed
|
|
304
|
+
* together (a server-ack batch, or every in-flight append on invalidation) and
|
|
305
|
+
* their common error, so callers need not await each promise. Both append
|
|
306
|
+
* flavours are covered: it is the only failure signal for the fire-and-forget
|
|
307
|
+
* ones, and an additional one for the WithWait ones, whose promise still
|
|
308
|
+
* rejects. Only appends given an append token appear, and tokens are opaque and
|
|
309
|
+
* not deduplicated. Synchronous failures (closed channel, serialization, bad
|
|
310
|
+
* arguments) throw from the append call and never reach the handler. A handler
|
|
311
|
+
* that throws is caught and logged.
|
|
312
|
+
*
|
|
313
|
+
* The handler runs inline on the tick that drains a server acknowledgement. It
|
|
314
|
+
* must therefore return quickly and do no heavy or blocking work: no synchronous
|
|
315
|
+
* blocking calls, no busy loops, no long CPU-bound work. Do only cheap
|
|
316
|
+
* bookkeeping inline (e.g. record the failed tokens) and hand any real work to a
|
|
317
|
+
* worker thread or your own queue — blocking here blocks the event loop, which
|
|
318
|
+
* stalls acknowledgements (and append/flush completion) for every channel on this
|
|
319
|
+
* client and the rest of your application with them. The SDK logs a warning when
|
|
320
|
+
* a handler runs slowly.
|
|
321
|
+
*
|
|
322
|
+
* @param {(detail: import("./error_detail.js").ErrorDetail) => void} handler
|
|
323
|
+
* @throws {TypeError} If handler is not a function, or declares more than one
|
|
324
|
+
* parameter (it is called with a single {@link ErrorDetail})
|
|
325
|
+
*/
|
|
326
|
+
setErrorHandler(handler) {
|
|
327
|
+
if (typeof handler !== "function") {
|
|
328
|
+
throw new TypeError("handler must be a function");
|
|
329
|
+
}
|
|
330
|
+
// Reject the (appendTokens, error) two-argument shape: such a handler would
|
|
331
|
+
// silently see `undefined` as its second argument. `> 1` rather than `!== 1`
|
|
332
|
+
// keeps zero-arg, rest-parameter (length 0) and default-parameter handlers
|
|
333
|
+
// valid.
|
|
334
|
+
if (handler.length > 1) {
|
|
335
|
+
throw new TypeError(
|
|
336
|
+
`handler must take a single ErrorDetail argument, but declares ${handler.length} parameters`,
|
|
337
|
+
);
|
|
338
|
+
}
|
|
339
|
+
this._ackState.setErrorHandler(handler);
|
|
340
|
+
}
|
|
341
|
+
|
|
342
|
+
/**
|
|
343
|
+
* Register (or replace) a handler invoked when appends are acknowledged by
|
|
344
|
+
* Snowflake. Fires once per successful ack batch with a single
|
|
345
|
+
* {@link SuccessDetail} bundling the append tokens acknowledged together, so
|
|
346
|
+
* callers need not await each promise. Both append flavours are covered: it is
|
|
347
|
+
* the only success signal for the fire-and-forget ones, and an additional one
|
|
348
|
+
* for the WithWait ones, whose promise still resolves. Purely opt-in — without a
|
|
349
|
+
* registered handler nothing is collected. Only appends given an append token
|
|
350
|
+
* appear, and tokens are opaque and not deduplicated. A handler that throws is
|
|
351
|
+
* caught and logged.
|
|
352
|
+
*
|
|
353
|
+
* Like the error handler, this runs inline on the tick that drains a server
|
|
354
|
+
* acknowledgement. It must therefore return quickly and do no heavy or blocking
|
|
355
|
+
* work: no synchronous blocking calls, no busy loops, no long CPU-bound work. Do
|
|
356
|
+
* only cheap bookkeeping inline (e.g. record the tokens) and hand any real work to
|
|
357
|
+
* a worker thread or your own queue — blocking here blocks the event loop, which
|
|
358
|
+
* stalls acknowledgements (and append/flush completion) for every channel on this
|
|
359
|
+
* client and the rest of your application with them. The SDK logs a warning when a
|
|
360
|
+
* handler runs slowly.
|
|
361
|
+
*
|
|
362
|
+
* @param {(detail: import("./success_detail.js").SuccessDetail) => void} handler
|
|
363
|
+
* @throws {TypeError} If handler is not a function, or declares more than one
|
|
364
|
+
* parameter (it is called with a single {@link SuccessDetail})
|
|
365
|
+
*/
|
|
366
|
+
setSuccessHandler(handler) {
|
|
367
|
+
if (typeof handler !== "function") {
|
|
368
|
+
throw new TypeError("handler must be a function");
|
|
369
|
+
}
|
|
370
|
+
// Same arity guard as setErrorHandler: a multi-parameter handler would
|
|
371
|
+
// silently see `undefined` beyond the first argument. `> 1` rather than
|
|
372
|
+
// `!== 1` keeps zero-arg, rest-parameter (length 0) and default-parameter
|
|
373
|
+
// handlers valid.
|
|
374
|
+
if (handler.length > 1) {
|
|
375
|
+
throw new TypeError(
|
|
376
|
+
`handler must take a single SuccessDetail argument, but declares ${handler.length} parameters`,
|
|
377
|
+
);
|
|
378
|
+
}
|
|
379
|
+
this._ackState.setSuccessHandler(handler);
|
|
380
|
+
}
|
|
381
|
+
|
|
159
382
|
/**
|
|
160
383
|
* Trigger a flush of this channel (non-blocking).
|
|
161
384
|
* @throws {StreamingIngestError} If the flush cannot be initiated
|
|
@@ -0,0 +1,95 @@
|
|
|
1
|
+
// Copyright 2026 Snowflake Inc.
|
|
2
|
+
// SPDX-License-Identifier: Apache-2.0
|
|
3
|
+
//
|
|
4
|
+
// Licensed under the Apache License, Version 2.0 (the "License");
|
|
5
|
+
// you may not use this file except in compliance with the License.
|
|
6
|
+
// You may obtain a copy of the License at
|
|
7
|
+
//
|
|
8
|
+
// http://www.apache.org/licenses/LICENSE-2.0
|
|
9
|
+
//
|
|
10
|
+
// Unless required by applicable law or agreed to in writing, software
|
|
11
|
+
// distributed under the License is distributed on an "AS IS" BASIS,
|
|
12
|
+
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
13
|
+
// See the License for the specific language governing permissions and
|
|
14
|
+
// limitations under the License.
|
|
15
|
+
|
|
16
|
+
"use strict";
|
|
17
|
+
|
|
18
|
+
/**
|
|
19
|
+
* The payload handed to an elastic-channel error handler (registered via
|
|
20
|
+
* {@link StreamingIngestElasticChannel#setErrorHandler}) when appends fail
|
|
21
|
+
* asynchronously.
|
|
22
|
+
*
|
|
23
|
+
* Delivered once per failure event, carrying the caller-supplied append tokens
|
|
24
|
+
* of every append that failed together under a common error — a whole
|
|
25
|
+
* server-ack error batch, or every tracked in-flight append on channel
|
|
26
|
+
* invalidation.
|
|
27
|
+
*
|
|
28
|
+
* Instances are created by the SDK and deeply frozen — both the object and its
|
|
29
|
+
* `appendTokens` array — so a handler cannot mutate what a sibling handler or a
|
|
30
|
+
* later log statement will read. Handlers only read this type, so new fields can
|
|
31
|
+
* be added over time without breaking them. Port of Java's `ErrorDetail`
|
|
32
|
+
* (immutable `Iterable`) and Python's frozen `ErrorDetail` dataclass (immutable
|
|
33
|
+
* tuple).
|
|
34
|
+
*/
|
|
35
|
+
class ErrorDetail {
|
|
36
|
+
/**
|
|
37
|
+
* @param {unknown[]} appendTokens - The opaque, caller-supplied append tokens of
|
|
38
|
+
* the appends that failed with `error`. Never empty; a token reused across
|
|
39
|
+
* appends appears once per failed append (not deduplicated). Frozen here, so
|
|
40
|
+
* callers must pass an array they are handing over.
|
|
41
|
+
* @param {import("./errors.js").StreamingIngestError} error - The error that
|
|
42
|
+
* rejected the failing appends.
|
|
43
|
+
* @param {string | null} requestId - The Snowflake request id of the server
|
|
44
|
+
* response that failed these appends, for support escalation. Null when the
|
|
45
|
+
* failure did not come from a server response (channel invalidation, or a
|
|
46
|
+
* local protocol inconsistency detected by the SDK).
|
|
47
|
+
* @param {number} retryCount - How many retries the failing request had already
|
|
48
|
+
* made: 0 when the first attempt failed, N when the SDK retried N times before
|
|
49
|
+
* giving up.
|
|
50
|
+
*
|
|
51
|
+
* **Use this to decide whether these rows may be duplicated in the table.** A retry
|
|
52
|
+
* re-sends the same rowset, and the SDK retries whenever it cannot tell that the
|
|
53
|
+
* previous attempt failed -- a timeout or a 5xx may mean the rows were never
|
|
54
|
+
* processed, or that they were processed and only the response was lost. `0` means
|
|
55
|
+
* the SDK sent this rowset once and introduced no duplicate; `> 0` means it sent it
|
|
56
|
+
* more than once and an earlier attempt may already have been applied, so the rows
|
|
57
|
+
* may appear more than once. Reconcile or de-duplicate downstream if you need
|
|
58
|
+
* exactly-once. The signal is conservative: it counts every re-send, including an
|
|
59
|
+
* authentication refresh, which is rejected before the rows are processed and so
|
|
60
|
+
* cannot have duplicated anything -- `> 0` means "a duplicate is possible", not
|
|
61
|
+
* "a duplicate happened". `0` means the SDK introduced no duplicate of its own, not
|
|
62
|
+
* that the rows are absent from the table.
|
|
63
|
+
*
|
|
64
|
+
* **Whenever `requestId` is null, neither field tells you whether the rows
|
|
65
|
+
* arrived.** A null id means the SDK failed this append itself rather than relaying
|
|
66
|
+
* a server response — channel or client invalidation, a close or drop, or an
|
|
67
|
+
* internal fault — and all of those fail every append still in flight, locally and
|
|
68
|
+
* immediately. That includes appends whose request had already gone to Snowflake and
|
|
69
|
+
* may still be applied after this handler runs. The accompanying `retryCount` of 0
|
|
70
|
+
* says only that the SDK sent the rowset once; it is not a statement that Snowflake
|
|
71
|
+
* rejected it. The SDK cannot resolve this for you — the append has to be failed
|
|
72
|
+
* when the channel goes away, and the in-flight request's outcome is not known at
|
|
73
|
+
* that point — so reconcile against the table if you need to know. (Some null-id
|
|
74
|
+
* failures never reached the network at all, such as a payload the SDK could not
|
|
75
|
+
* serialize, so a null id spans both "definitely not applied" and "possibly
|
|
76
|
+
* applied", which is why it cannot answer the question.)
|
|
77
|
+
*
|
|
78
|
+
* Read it only when `requestId` is non-null: a failure with no originating request
|
|
79
|
+
* reports 0 too. With the request id this names the exact attempt server-side — the
|
|
80
|
+
* SDK sends both as the `requestId` and `retryCount` query params on every request.
|
|
81
|
+
*/
|
|
82
|
+
constructor(appendTokens, error, requestId, retryCount) {
|
|
83
|
+
// Freeze the array as well as the instance: `Object.freeze(this)` alone would
|
|
84
|
+
// still let a handler push into or reorder appendTokens. Only the array
|
|
85
|
+
// itself is frozen, not the tokens inside it — those are the caller's own
|
|
86
|
+
// opaque objects and are handed back untouched.
|
|
87
|
+
this.appendTokens = Object.freeze(appendTokens);
|
|
88
|
+
this.error = error;
|
|
89
|
+
this.requestId = requestId;
|
|
90
|
+
this.retryCount = retryCount;
|
|
91
|
+
Object.freeze(this);
|
|
92
|
+
}
|
|
93
|
+
}
|
|
94
|
+
|
|
95
|
+
module.exports = { ErrorDetail };
|
package/src/errors.js
CHANGED
|
@@ -36,17 +36,30 @@
|
|
|
36
36
|
class StreamingIngestError extends Error {
|
|
37
37
|
/**
|
|
38
38
|
* Create a StreamingIngestError.
|
|
39
|
+
*
|
|
40
|
+
* `message` (and therefore `stack` / `console.error`) is the decorated form
|
|
41
|
+
* `{errorCode}: {detail} (HTTP {code} {name})`. Pass the Rust/SDK detail as
|
|
42
|
+
* `message`; read it back from {@link StreamingIngestError#detail}.
|
|
43
|
+
*
|
|
39
44
|
* @param {string} errorCode - The error code (e.g., "ConfigError", "ChannelNotFound")
|
|
40
|
-
* @param {string} message - The error
|
|
45
|
+
* @param {string} message - The undecorated error detail
|
|
41
46
|
* @param {number} httpStatusCode - HTTP status code indicating error category
|
|
42
47
|
* @param {string} httpStatusName - HTTP status name (e.g., "Bad Request")
|
|
43
48
|
*/
|
|
44
49
|
constructor(errorCode, message, httpStatusCode, httpStatusName) {
|
|
45
|
-
|
|
50
|
+
// V8 bakes `name` + `message` into `stack` at construct time, and
|
|
51
|
+
// `console.error` / pino print that stack — not `toString()`. Put the
|
|
52
|
+
// decorated text on `message` so logging cannot drop error-code / HTTP status.
|
|
53
|
+
const statusName = httpStatusName != null ? httpStatusName : "Unknown";
|
|
54
|
+
const detail = message != null ? String(message) : "";
|
|
55
|
+
const decorated = `${errorCode}: ${detail} (HTTP ${httpStatusCode} ${statusName})`;
|
|
56
|
+
super(decorated);
|
|
46
57
|
this.name = "StreamingIngestError";
|
|
58
|
+
this.message = decorated;
|
|
47
59
|
this.errorCode = errorCode;
|
|
48
60
|
this.httpStatusCode = httpStatusCode;
|
|
49
|
-
this.httpStatusName =
|
|
61
|
+
this.httpStatusName = statusName;
|
|
62
|
+
this.detail = detail;
|
|
50
63
|
}
|
|
51
64
|
|
|
52
65
|
/**
|
|
@@ -67,7 +80,7 @@ class StreamingIngestError extends Error {
|
|
|
67
80
|
data.error_code,
|
|
68
81
|
data.message,
|
|
69
82
|
data.http_status_code,
|
|
70
|
-
data.http_status_name
|
|
83
|
+
data.http_status_name,
|
|
71
84
|
);
|
|
72
85
|
}
|
|
73
86
|
} catch {
|
|
@@ -78,10 +91,10 @@ class StreamingIngestError extends Error {
|
|
|
78
91
|
|
|
79
92
|
/**
|
|
80
93
|
* Returns a string representation of the error.
|
|
81
|
-
* @returns {string}
|
|
94
|
+
* @returns {string} Decorated error string, without the `StreamingIngestError:` prefix
|
|
82
95
|
*/
|
|
83
96
|
toString() {
|
|
84
|
-
return
|
|
97
|
+
return this.message;
|
|
85
98
|
}
|
|
86
99
|
}
|
|
87
100
|
|
package/src/index.d.ts
CHANGED
|
@@ -327,13 +327,135 @@ export class StreamingIngestChannel {
|
|
|
327
327
|
}): Promise<void>;
|
|
328
328
|
}
|
|
329
329
|
|
|
330
|
+
/**
|
|
331
|
+
* The payload handed to an elastic-channel error handler (registered via
|
|
332
|
+
* `StreamingIngestElasticChannel.setErrorHandler`) when appends fail
|
|
333
|
+
* asynchronously.
|
|
334
|
+
*
|
|
335
|
+
* Delivered once per failure event, carrying the caller-supplied append tokens of
|
|
336
|
+
* every append that failed together under a common `error` — a whole server-ack
|
|
337
|
+
* error batch, or every tracked in-flight append on channel invalidation.
|
|
338
|
+
*
|
|
339
|
+
* Instances are created by the SDK and frozen; handlers only read this type, so
|
|
340
|
+
* new fields can be added over time without breaking them.
|
|
341
|
+
*/
|
|
342
|
+
export class ErrorDetail {
|
|
343
|
+
private constructor();
|
|
344
|
+
|
|
345
|
+
/**
|
|
346
|
+
* The opaque, caller-supplied append tokens of the appends that failed with
|
|
347
|
+
* `error`. Never empty; a token reused across appends appears once per failed
|
|
348
|
+
* append (not deduplicated). Frozen at construction — the array itself cannot
|
|
349
|
+
* be modified, though the tokens inside it are the caller's own objects.
|
|
350
|
+
*/
|
|
351
|
+
readonly appendTokens: readonly unknown[];
|
|
352
|
+
/** The error that rejected the failing appends. */
|
|
353
|
+
readonly error: StreamingIngestError;
|
|
354
|
+
/**
|
|
355
|
+
* The Snowflake request id of the server response that failed these appends,
|
|
356
|
+
* for support escalation. Null when the failure did not come from a server
|
|
357
|
+
* response — channel invalidation, or a protocol inconsistency the SDK detected
|
|
358
|
+
* on its own.
|
|
359
|
+
*/
|
|
360
|
+
readonly requestId: string | null;
|
|
361
|
+
/**
|
|
362
|
+
* How many retries the failing request had already made: 0 when the first
|
|
363
|
+
* attempt failed, N when the SDK retried N times before giving up.
|
|
364
|
+
*
|
|
365
|
+
* **Use this to decide whether these rows may be duplicated in the table.** A retry
|
|
366
|
+
* re-sends the same rowset, and the SDK retries whenever it cannot tell that the
|
|
367
|
+
* previous attempt failed — a timeout or a 5xx may mean the rows were never processed,
|
|
368
|
+
* or that they were processed and only the response was lost. `0` means the SDK sent
|
|
369
|
+
* this rowset once and introduced no duplicate; `> 0` means it sent it more than once
|
|
370
|
+
* and an earlier attempt may already have been applied, so the rows may appear more
|
|
371
|
+
* than once. Reconcile or de-duplicate downstream if you need exactly-once.
|
|
372
|
+
*
|
|
373
|
+
* The signal is conservative: it counts every re-send, including an authentication
|
|
374
|
+
* refresh, which is rejected before the rows are processed and so cannot have
|
|
375
|
+
* duplicated anything. `> 0` means "a duplicate is possible", not "a duplicate
|
|
376
|
+
* happened". `0` means the SDK introduced no duplicate of its own, not that the rows
|
|
377
|
+
* are absent from the table.
|
|
378
|
+
*
|
|
379
|
+
* **Whenever `requestId` is null, neither field tells you whether the rows arrived.** A
|
|
380
|
+
* null id means the SDK failed this append itself rather than relaying a server response
|
|
381
|
+
* — channel or client invalidation, a close or drop, or an internal fault — and all of
|
|
382
|
+
* those fail every append still in flight, locally and immediately. That includes appends
|
|
383
|
+
* whose request had already gone to Snowflake and may still be applied after this handler
|
|
384
|
+
* runs. The accompanying `retryCount` of 0 says only that the SDK sent the rowset once;
|
|
385
|
+
* it is not a statement that Snowflake rejected it. Reconcile against the table if you
|
|
386
|
+
* need to know whether those rows landed.
|
|
387
|
+
*
|
|
388
|
+
* Read it only when `requestId` is non-null: a failure with no originating request
|
|
389
|
+
* reports 0 too. With the request id this names the exact attempt server-side — the
|
|
390
|
+
* SDK sends both as query params on every request.
|
|
391
|
+
*/
|
|
392
|
+
readonly retryCount: number;
|
|
393
|
+
}
|
|
394
|
+
|
|
395
|
+
/**
|
|
396
|
+
* The payload handed to an elastic-channel success handler (registered via
|
|
397
|
+
* `StreamingIngestElasticChannel.setSuccessHandler`) when appends are
|
|
398
|
+
* acknowledged by Snowflake.
|
|
399
|
+
*
|
|
400
|
+
* Delivered once per successful acknowledgement, carrying the caller-supplied
|
|
401
|
+
* append tokens of every append acked together in that server-ack batch. There is
|
|
402
|
+
* no error field — unlike `ErrorDetail`, there is nothing to explain.
|
|
403
|
+
*
|
|
404
|
+
* Instances are created by the SDK and frozen; handlers only read this type, so
|
|
405
|
+
* new fields can be added over time without breaking them.
|
|
406
|
+
*/
|
|
407
|
+
export class SuccessDetail {
|
|
408
|
+
private constructor();
|
|
409
|
+
|
|
410
|
+
/**
|
|
411
|
+
* The opaque, caller-supplied append tokens of the appends that were
|
|
412
|
+
* acknowledged together. Never empty; a token reused across appends appears once
|
|
413
|
+
* per acknowledged append (not deduplicated). Frozen at construction — the array
|
|
414
|
+
* itself cannot be modified, though the tokens inside it are the caller's own
|
|
415
|
+
* objects.
|
|
416
|
+
*/
|
|
417
|
+
readonly appendTokens: readonly unknown[];
|
|
418
|
+
/**
|
|
419
|
+
* The Snowflake request id of the server response that acknowledged these
|
|
420
|
+
* appends, for support escalation.
|
|
421
|
+
*/
|
|
422
|
+
readonly requestId: string;
|
|
423
|
+
/**
|
|
424
|
+
* How many retries that request took: 0 when the first attempt was acknowledged,
|
|
425
|
+
* N when the SDK retried N times before Snowflake accepted it.
|
|
426
|
+
*
|
|
427
|
+
* **Use this to decide whether these rows may be duplicated in the table.** A retry
|
|
428
|
+
* re-sends the same rowset, and the SDK retries whenever it cannot tell that the
|
|
429
|
+
* previous attempt failed — a timeout or a 5xx may mean the rows were never processed,
|
|
430
|
+
* or that they were processed and only the response was lost. `0` means the SDK sent
|
|
431
|
+
* this rowset once and introduced no duplicate; `> 0` means it sent it more than once
|
|
432
|
+
* and an earlier attempt may already have been applied, so the rows may appear more
|
|
433
|
+
* than once. Reconcile or de-duplicate downstream if you need exactly-once.
|
|
434
|
+
*
|
|
435
|
+
* The signal is conservative: it counts every re-send, including an authentication
|
|
436
|
+
* refresh, which is rejected before the rows are processed and so cannot have
|
|
437
|
+
* duplicated anything. `> 0` means "a duplicate is possible", not "a duplicate
|
|
438
|
+
* happened".
|
|
439
|
+
*
|
|
440
|
+
* Read it only when `requestId` is non-null: an acknowledgement with no originating
|
|
441
|
+
* request reports 0 too. A steadily non-zero count also means the SDK is absorbing
|
|
442
|
+
* retries on your behalf.
|
|
443
|
+
*/
|
|
444
|
+
readonly retryCount: number;
|
|
445
|
+
}
|
|
446
|
+
|
|
330
447
|
/**
|
|
331
448
|
* An elastic channel for streaming data into a Snowflake table.
|
|
332
449
|
*
|
|
333
450
|
* Unlike a regular channel, an elastic channel has no offset tokens and its
|
|
334
451
|
* lifecycle is tied to the client — there is no close method, and the same
|
|
335
452
|
* instance is returned on repeated calls to `client.getElasticChannel()`.
|
|
336
|
-
* `appendRow`/`appendRows`
|
|
453
|
+
* Appends come in two flavours: fire-and-forget `appendRow`/`appendRows` return
|
|
454
|
+
* nothing, while `appendRowWithWait`/`appendRowsWithWait` return a promise that
|
|
455
|
+
* resolves when Snowflake acknowledges the rows. Every append must carry an opaque
|
|
456
|
+
* `appendToken` — required, but nullable — handed back to the handler registered via
|
|
457
|
+
* `setErrorHandler` when that append fails asynchronously, or to `setSuccessHandler`
|
|
458
|
+
* when it is acknowledged; pass `null` to opt that append out of both handlers.
|
|
337
459
|
*
|
|
338
460
|
* Create instances using `client.getElasticChannel()`.
|
|
339
461
|
*/
|
|
@@ -353,24 +475,119 @@ export class StreamingIngestElasticChannel {
|
|
|
353
475
|
/** The binary input format. One of: "BASE64", "HEX", "UTF-8". */
|
|
354
476
|
readonly binaryInputFormat: string;
|
|
355
477
|
|
|
478
|
+
/**
|
|
479
|
+
* Append a single row into the elastic channel without waiting for the
|
|
480
|
+
* acknowledgement (fire-and-forget). The append is still registered for the
|
|
481
|
+
* error handler, so an asynchronous failure is reported through
|
|
482
|
+
* `setErrorHandler` (keyed by `appendToken`), just not through a promise. Use
|
|
483
|
+
* `appendRowWithWait` when you need one to await.
|
|
484
|
+
*
|
|
485
|
+
* @param row - Row data as column-name to value pairs
|
|
486
|
+
* @param appendToken - Required caller-supplied opaque id for this append, handed
|
|
487
|
+
* to the `setErrorHandler` handler on asynchronous failure, or to the
|
|
488
|
+
* `setSuccessHandler` handler on acknowledgement. Not sent to Snowflake; any
|
|
489
|
+
* value (including `""`) is valid. The argument is mandatory: pass `null`
|
|
490
|
+
* explicitly to leave the append untracked for both handlers. The token is
|
|
491
|
+
* retained in memory until the append is acknowledged, so a large object raises
|
|
492
|
+
* the memory footprint of in-flight appends — prefer a small value.
|
|
493
|
+
* @throws TypeError If row is null or undefined
|
|
494
|
+
* @throws StreamingIngestError If the row appending fails synchronously
|
|
495
|
+
*/
|
|
496
|
+
appendRow(row: Record<string, any>, appendToken: unknown): void;
|
|
497
|
+
|
|
498
|
+
/**
|
|
499
|
+
* Append multiple rows into the elastic channel without waiting for the
|
|
500
|
+
* acknowledgement (fire-and-forget). The append is still registered for the
|
|
501
|
+
* error handler, so an asynchronous failure is reported through
|
|
502
|
+
* `setErrorHandler` (keyed by `appendToken`), just not through a promise. Use
|
|
503
|
+
* `appendRowsWithWait` when you need one to await.
|
|
504
|
+
*
|
|
505
|
+
* @param rows - Array of row objects (column-name to value pairs)
|
|
506
|
+
* @param appendToken - Required caller-supplied opaque id for this batch, handed
|
|
507
|
+
* to the `setErrorHandler` handler on asynchronous failure, or to the
|
|
508
|
+
* `setSuccessHandler` handler on acknowledgement. Not sent to Snowflake; any
|
|
509
|
+
* value (including `""`) is valid. The argument is mandatory: pass `null`
|
|
510
|
+
* explicitly to leave the append untracked for both handlers. The token is
|
|
511
|
+
* retained in memory until the append is acknowledged, so a large object raises
|
|
512
|
+
* the memory footprint of in-flight appends — prefer a small value.
|
|
513
|
+
* @throws TypeError If rows is not an array
|
|
514
|
+
* @throws StreamingIngestError If the rows appending fails synchronously
|
|
515
|
+
*/
|
|
516
|
+
appendRows(rows: Record<string, any>[], appendToken: unknown): void;
|
|
517
|
+
|
|
356
518
|
/**
|
|
357
519
|
* Append a single row into the elastic channel. The returned promise resolves
|
|
358
|
-
* when Snowflake acknowledges the row.
|
|
520
|
+
* when Snowflake acknowledges the row. If an error handler is registered it
|
|
521
|
+
* also fires on an asynchronous failure — dual delivery.
|
|
359
522
|
*
|
|
360
523
|
* @param row - Row data as column-name to value pairs
|
|
524
|
+
* @param appendToken - Required caller-supplied opaque id for this append, handed
|
|
525
|
+
* to the `setErrorHandler` handler on asynchronous failure, or to the
|
|
526
|
+
* `setSuccessHandler` handler on acknowledgement. Not sent to Snowflake; any
|
|
527
|
+
* value (including `""`) is valid. The argument is mandatory: pass `null`
|
|
528
|
+
* explicitly to leave the append untracked for both handlers. The token is
|
|
529
|
+
* retained in memory until the append is acknowledged, so a large object raises
|
|
530
|
+
* the memory footprint of in-flight appends — prefer a small value.
|
|
361
531
|
* @throws StreamingIngestError If the row appending fails
|
|
362
532
|
*/
|
|
363
|
-
|
|
533
|
+
appendRowWithWait(row: Record<string, any>, appendToken: unknown): Promise<void>;
|
|
364
534
|
|
|
365
535
|
/**
|
|
366
536
|
* Append multiple rows into the elastic channel. The returned promise resolves
|
|
367
|
-
* when Snowflake acknowledges the batch.
|
|
537
|
+
* when Snowflake acknowledges the batch. If an error handler is registered it
|
|
538
|
+
* also fires on an asynchronous failure — dual delivery.
|
|
368
539
|
*
|
|
369
540
|
* @param rows - Array of row objects (column-name to value pairs)
|
|
541
|
+
* @param appendToken - Required caller-supplied opaque id for this batch, handed
|
|
542
|
+
* to the `setErrorHandler` handler on asynchronous failure, or to the
|
|
543
|
+
* `setSuccessHandler` handler on acknowledgement. Not sent to Snowflake; any
|
|
544
|
+
* value (including `""`) is valid. The argument is mandatory: pass `null`
|
|
545
|
+
* explicitly to leave the append untracked for both handlers. The token is
|
|
546
|
+
* retained in memory until the append is acknowledged, so a large object raises
|
|
547
|
+
* the memory footprint of in-flight appends — prefer a small value.
|
|
370
548
|
* @throws TypeError If rows is not an array
|
|
371
549
|
* @throws StreamingIngestError If the rows appending fails
|
|
372
550
|
*/
|
|
373
|
-
|
|
551
|
+
appendRowsWithWait(rows: Record<string, any>[], appendToken: unknown): Promise<void>;
|
|
552
|
+
|
|
553
|
+
/**
|
|
554
|
+
* Register (or replace) a handler invoked when appends fail *asynchronously*,
|
|
555
|
+
* i.e. after the append call already returned. Fires once per failure event
|
|
556
|
+
* with a single {@link ErrorDetail} bundling the `appendToken`s that failed
|
|
557
|
+
* together (a server-ack batch, or every in-flight append on invalidation) and
|
|
558
|
+
* their common error, so callers need not await each promise. Both append
|
|
559
|
+
* flavours are covered: it is the only failure signal for the fire-and-forget
|
|
560
|
+
* ones, and an additional one for the `WithWait` ones, whose promise still
|
|
561
|
+
* rejects. Only appends given an `appendToken` appear, and tokens are opaque and
|
|
562
|
+
* not deduplicated. Synchronous failures (closed channel, serialization, bad
|
|
563
|
+
* arguments) throw from the append call and never reach the handler. A handler
|
|
564
|
+
* that throws is caught and logged.
|
|
565
|
+
*
|
|
566
|
+
* @param handler - Called with one {@link ErrorDetail} per failure event
|
|
567
|
+
* @throws TypeError If handler is not a function, or declares more than one
|
|
568
|
+
* parameter (it is called with a single {@link ErrorDetail})
|
|
569
|
+
*/
|
|
570
|
+
setErrorHandler(handler: (detail: ErrorDetail) => void): void;
|
|
571
|
+
|
|
572
|
+
/**
|
|
573
|
+
* Register (or replace) a handler invoked when appends are acknowledged by
|
|
574
|
+
* Snowflake. Fires once per successful ack batch with a single
|
|
575
|
+
* {@link SuccessDetail} bundling the `appendToken`s acknowledged together, so
|
|
576
|
+
* callers need not await each promise. Both append flavours are covered: it is
|
|
577
|
+
* the only success signal for the fire-and-forget ones, and an additional one for
|
|
578
|
+
* the `WithWait` ones, whose promise still resolves. Purely opt-in — without a
|
|
579
|
+
* registered handler nothing is collected. Only appends given an `appendToken`
|
|
580
|
+
* appear, and tokens are opaque and not deduplicated. A handler that throws is
|
|
581
|
+
* caught and logged.
|
|
582
|
+
*
|
|
583
|
+
* Like the error handler, this runs on the acknowledgement path, so a slow
|
|
584
|
+
* handler delays subsequent acknowledgements — keep it short.
|
|
585
|
+
*
|
|
586
|
+
* @param handler - Called with one {@link SuccessDetail} per successful ack batch
|
|
587
|
+
* @throws TypeError If handler is not a function, or declares more than one
|
|
588
|
+
* parameter (it is called with a single {@link SuccessDetail})
|
|
589
|
+
*/
|
|
590
|
+
setSuccessHandler(handler: (detail: SuccessDetail) => void): void;
|
|
374
591
|
|
|
375
592
|
/**
|
|
376
593
|
* Initiate a flush of all buffered data for this channel but do not wait for
|
|
@@ -452,8 +669,16 @@ export class ChannelStatus {
|
|
|
452
669
|
export class StreamingIngestError extends Error {
|
|
453
670
|
/** The error code (for example, "ConfigError", "ChannelNotFound"). */
|
|
454
671
|
readonly errorCode: string;
|
|
455
|
-
/**
|
|
672
|
+
/**
|
|
673
|
+
* Decorated error message: `{errorCode}: {detail} (HTTP {code} {name})`.
|
|
674
|
+
* Inherited from `Error`; this is what `stack` / `console.error` print.
|
|
675
|
+
*/
|
|
456
676
|
readonly message: string;
|
|
677
|
+
/**
|
|
678
|
+
* Original detail passed to the constructor, without error-code / HTTP-status
|
|
679
|
+
* decoration. Analog of Java `SFException#getDetailMessage()`.
|
|
680
|
+
*/
|
|
681
|
+
readonly detail: string;
|
|
457
682
|
/** HTTP status code indicating error category. */
|
|
458
683
|
readonly httpStatusCode: number;
|
|
459
684
|
/** HTTP status name (for example, "Bad Request"). */
|
package/src/index.js
CHANGED
|
@@ -27,6 +27,8 @@ const {
|
|
|
27
27
|
} = require("./client.js");
|
|
28
28
|
const { StreamingIngestChannel } = require("./channel.js");
|
|
29
29
|
const { StreamingIngestElasticChannel } = require("./elastic_channel.js");
|
|
30
|
+
const { ErrorDetail } = require("./error_detail.js");
|
|
31
|
+
const { SuccessDetail } = require("./success_detail.js");
|
|
30
32
|
|
|
31
33
|
// Load native addon and bootstrap the Rust runtime
|
|
32
34
|
const native = loadNativeAddon();
|
|
@@ -52,6 +54,8 @@ module.exports = {
|
|
|
52
54
|
StreamingIngestElasticChannel,
|
|
53
55
|
StreamingIngestError,
|
|
54
56
|
StreamingIngestErrorCode,
|
|
57
|
+
ErrorDetail,
|
|
58
|
+
SuccessDetail,
|
|
55
59
|
ChannelStatus,
|
|
56
60
|
// E2E testing only - available when compiled with e2e_test feature
|
|
57
61
|
registerMockRoute: native.registerMockRoute,
|
package/src/loader.js
CHANGED
|
@@ -20,7 +20,7 @@
|
|
|
20
20
|
const path = require("node:path");
|
|
21
21
|
const fs = require("node:fs");
|
|
22
22
|
|
|
23
|
-
const LOG_PREFIX = "
|
|
23
|
+
const { LOG_PREFIX } = require("./log.js");
|
|
24
24
|
|
|
25
25
|
// Verbose loader breadcrumbs are gated on SS_LOG_LEVEL=debug|trace, matching
|
|
26
26
|
// the env var the Rust core and Java/Python SDKs already use (see
|
package/src/log.js
ADDED
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
// Copyright 2026 Snowflake Inc.
|
|
2
|
+
// SPDX-License-Identifier: Apache-2.0
|
|
3
|
+
//
|
|
4
|
+
// Licensed under the Apache License, Version 2.0 (the "License");
|
|
5
|
+
// you may not use this file except in compliance with the License.
|
|
6
|
+
// You may obtain a copy of the License at
|
|
7
|
+
//
|
|
8
|
+
// http://www.apache.org/licenses/LICENSE-2.0
|
|
9
|
+
//
|
|
10
|
+
// Unless required by applicable law or agreed to in writing, software
|
|
11
|
+
// distributed under the License is distributed on an "AS IS" BASIS,
|
|
12
|
+
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
13
|
+
// See the License for the specific language governing permissions and
|
|
14
|
+
// limitations under the License.
|
|
15
|
+
|
|
16
|
+
"use strict";
|
|
17
|
+
|
|
18
|
+
/**
|
|
19
|
+
* Prefix for every line this SDK writes to stderr, so SDK output is
|
|
20
|
+
* attributable in a customer's log stream. Used by the native-addon loader and
|
|
21
|
+
* by the elastic channel's acknowledgement path, which are the only places the
|
|
22
|
+
* SDK logs on its own (everything else surfaces as a thrown or rejected
|
|
23
|
+
* StreamingIngestError).
|
|
24
|
+
*/
|
|
25
|
+
const LOG_PREFIX = "[snowpipe-streaming]";
|
|
26
|
+
|
|
27
|
+
module.exports = { LOG_PREFIX };
|
|
@@ -32,6 +32,14 @@
|
|
|
32
32
|
// directly — same end state, driven from JS.
|
|
33
33
|
|
|
34
34
|
const { StreamingIngestError } = require("./errors.js");
|
|
35
|
+
const { ErrorDetail } = require("./error_detail.js");
|
|
36
|
+
const { SuccessDetail } = require("./success_detail.js");
|
|
37
|
+
const { LOG_PREFIX } = require("./log.js");
|
|
38
|
+
|
|
39
|
+
// A handler runs inline on the acknowledgement tick, so blocking it stalls the whole event loop —
|
|
40
|
+
// every channel on this client, and the rest of the application. Warn past this many milliseconds.
|
|
41
|
+
// Timing uses the global `performance` (Node >= 16; this package requires >= 20), like `Buffer`.
|
|
42
|
+
const SLOW_HANDLER_WARN_THRESHOLD_MILLIS = 200;
|
|
35
43
|
|
|
36
44
|
function buildError(success, errorCode, message, httpStatusCode, httpStatusName) {
|
|
37
45
|
if (success) {
|
|
@@ -52,12 +60,23 @@ function buildError(success, errorCode, message, httpStatusCode, httpStatusName)
|
|
|
52
60
|
* for JS-driven invalidation. The native reference is plugged in after the
|
|
53
61
|
* channel is opened (chicken-and-egg: these callbacks are passed *into* the
|
|
54
62
|
* open call, so they exist before the native handle does).
|
|
63
|
+
*
|
|
64
|
+
* Also owns the customer error handler, invoked once per failure event with a
|
|
65
|
+
* single {@link ErrorDetail} carrying the append tokens of every append that
|
|
66
|
+
* failed under a common error, and the customer success handler, invoked once per
|
|
67
|
+
* successful server-ack batch with a single {@link SuccessDetail} carrying that
|
|
68
|
+
* batch's append tokens. Tokens are opaque and optional, so only entries
|
|
69
|
+
* that carry one are reported. This class is elastic-only, so the append token
|
|
70
|
+
* rides on the pending-map value; Java/Python need a parallel token map because
|
|
71
|
+
* their callback also serves regular channels.
|
|
55
72
|
*/
|
|
56
73
|
class ElasticChannelAckState {
|
|
57
74
|
constructor(pendingFutures) {
|
|
58
75
|
this._pendingFutures = pendingFutures;
|
|
59
76
|
this._invalidated = false;
|
|
60
77
|
this._nativeChannel = null;
|
|
78
|
+
this._errorHandler = null;
|
|
79
|
+
this._successHandler = null;
|
|
61
80
|
this.markComplete = this._markComplete.bind(this);
|
|
62
81
|
this.invalidateAll = this._invalidateAll.bind(this);
|
|
63
82
|
}
|
|
@@ -67,8 +86,149 @@ class ElasticChannelAckState {
|
|
|
67
86
|
this._nativeChannel = nativeChannel;
|
|
68
87
|
}
|
|
69
88
|
|
|
89
|
+
/**
|
|
90
|
+
* Register (or replace) the customer error handler. The argument is validated
|
|
91
|
+
* by `StreamingIngestElasticChannel.setErrorHandler`.
|
|
92
|
+
*/
|
|
93
|
+
setErrorHandler(handler) {
|
|
94
|
+
this._errorHandler = handler;
|
|
95
|
+
}
|
|
96
|
+
|
|
97
|
+
/**
|
|
98
|
+
* Register (or replace) the customer success handler. The argument is validated
|
|
99
|
+
* by `StreamingIngestElasticChannel.setSuccessHandler`.
|
|
100
|
+
*/
|
|
101
|
+
setSuccessHandler(handler) {
|
|
102
|
+
this._successHandler = handler;
|
|
103
|
+
}
|
|
104
|
+
|
|
105
|
+
/**
|
|
106
|
+
* Invoke the error handler once for a batch of append tokens that failed with a
|
|
107
|
+
* common error. Both are bundled into the single {@link ErrorDetail} the handler
|
|
108
|
+
* takes (mirroring Java's and Python's `ErrorDetail`), so new fields can be
|
|
109
|
+
* added later without changing the handler signature. No-op without a handler
|
|
110
|
+
* or on an empty batch. A throwing handler is caught and logged so it can't
|
|
111
|
+
* break the acknowledgement path.
|
|
112
|
+
*
|
|
113
|
+
* @param {unknown[]} appendTokens - Opaque append tokens of the appends that failed together.
|
|
114
|
+
* @param {StreamingIngestError} error - The error that failed them.
|
|
115
|
+
* @param {string | null} requestId - Snowflake request id of the failing response, or null.
|
|
116
|
+
* @param {number} retryCount - How many retries that request took; 0 with a null id, where it
|
|
117
|
+
* carries no meaning.
|
|
118
|
+
*/
|
|
119
|
+
fireErrorHandler(appendTokens, error, requestId, retryCount) {
|
|
120
|
+
if (this._errorHandler === null || appendTokens == null || appendTokens.length === 0) {
|
|
121
|
+
return;
|
|
122
|
+
}
|
|
123
|
+
const start = performance.now();
|
|
124
|
+
try {
|
|
125
|
+
this._errorHandler(new ErrorDetail(appendTokens, error, requestId, retryCount));
|
|
126
|
+
} catch (handlerErr) {
|
|
127
|
+
// eslint-disable-next-line no-console
|
|
128
|
+
console.error(
|
|
129
|
+
`${LOG_PREFIX} Elastic channel error handler threw for ` +
|
|
130
|
+
`appendTokens=[${appendTokens.join(", ")}]; ignoring.`,
|
|
131
|
+
handlerErr,
|
|
132
|
+
);
|
|
133
|
+
} finally {
|
|
134
|
+
this._logHandlerDuration("error", start, appendTokens.length);
|
|
135
|
+
}
|
|
136
|
+
}
|
|
137
|
+
|
|
138
|
+
/**
|
|
139
|
+
* Warn when a handler invocation crossed {@link SLOW_HANDLER_WARN_THRESHOLD_MILLIS}.
|
|
140
|
+
* It runs inline on the event-loop tick that drains the Rust acknowledgement, so
|
|
141
|
+
* blocking it stalls this client's acknowledgements and everything else on the
|
|
142
|
+
* loop. Only the slow case is logged — a per-invocation line would fire once per
|
|
143
|
+
* acknowledgement batch, effectively once per append on a busy pipe.
|
|
144
|
+
*
|
|
145
|
+
* @param {string} handlerKind - "error" or "success", for the log message.
|
|
146
|
+
* @param {number} start - `performance.now()` captured before invoking the handler.
|
|
147
|
+
* @param {number} appendTokenCount - how many append tokens the invocation carried.
|
|
148
|
+
*/
|
|
149
|
+
_logHandlerDuration(handlerKind, start, appendTokenCount) {
|
|
150
|
+
const elapsedMillis = performance.now() - start;
|
|
151
|
+
if (elapsedMillis >= SLOW_HANDLER_WARN_THRESHOLD_MILLIS) {
|
|
152
|
+
// eslint-disable-next-line no-console
|
|
153
|
+
console.warn(
|
|
154
|
+
`${LOG_PREFIX} Elastic channel ${handlerKind} handler took ` +
|
|
155
|
+
`${elapsedMillis.toFixed(1)} ms for ${appendTokenCount} appendToken(s); it is blocking ` +
|
|
156
|
+
`the event loop — offload heavy work to a worker thread or your own queue.`,
|
|
157
|
+
);
|
|
158
|
+
}
|
|
159
|
+
}
|
|
160
|
+
|
|
161
|
+
/**
|
|
162
|
+
* Invoke the success handler once for a batch of append tokens acknowledged
|
|
163
|
+
* together. They are bundled into the single {@link SuccessDetail} the handler
|
|
164
|
+
* takes, so new fields can be added later without changing the handler
|
|
165
|
+
* signature. No-op without a handler or on an empty batch. A throwing handler is
|
|
166
|
+
* caught and logged so it can't break the acknowledgement path.
|
|
167
|
+
*
|
|
168
|
+
* @param {unknown[]} appendTokens - Opaque append tokens of the appends acknowledged together.
|
|
169
|
+
* @param {string | null} requestId - Snowflake request id of the acking response, or null.
|
|
170
|
+
* @param {number} retryCount - How many retries that request took.
|
|
171
|
+
*/
|
|
172
|
+
fireSuccessHandler(appendTokens, requestId, retryCount) {
|
|
173
|
+
if (requestId === null || requestId === undefined) {
|
|
174
|
+
// SuccessDetail.requestId is declared non-null: an acknowledgement can only come from a
|
|
175
|
+
// request Snowflake answered. Reaching here means the FFI bridge dropped the field. Log it
|
|
176
|
+
// rather than withhold the customer's notification over a support-escalation value.
|
|
177
|
+
// eslint-disable-next-line no-console
|
|
178
|
+
console.error(
|
|
179
|
+
`${LOG_PREFIX} Elastic channel success ack carried no requestId for appendTokens=` +
|
|
180
|
+
`${appendTokens}; this is an SDK defect, the acknowledgement itself is still reported.`,
|
|
181
|
+
);
|
|
182
|
+
}
|
|
183
|
+
if (this._successHandler === null || appendTokens.length === 0) {
|
|
184
|
+
return;
|
|
185
|
+
}
|
|
186
|
+
const start = performance.now();
|
|
187
|
+
try {
|
|
188
|
+
this._successHandler(new SuccessDetail(appendTokens, requestId, retryCount));
|
|
189
|
+
} catch (handlerErr) {
|
|
190
|
+
// eslint-disable-next-line no-console
|
|
191
|
+
console.error(
|
|
192
|
+
`${LOG_PREFIX} Elastic channel success handler threw for ` +
|
|
193
|
+
`appendTokens=[${appendTokens.join(", ")}]; ignoring.`,
|
|
194
|
+
handlerErr,
|
|
195
|
+
);
|
|
196
|
+
} finally {
|
|
197
|
+
this._logHandlerDuration("success", start, appendTokens.length);
|
|
198
|
+
}
|
|
199
|
+
}
|
|
200
|
+
|
|
201
|
+
/**
|
|
202
|
+
* Reject and clear every pending promise, returning the append tokens of the
|
|
203
|
+
* appends that carried one so the handler can be fired once the whole loop is
|
|
204
|
+
* done. Untracked appends (no append token) reject without being reported.
|
|
205
|
+
*/
|
|
206
|
+
_rejectAllPending(error) {
|
|
207
|
+
const appendTokens = [];
|
|
208
|
+
for (const pending of this._pendingFutures.values()) {
|
|
209
|
+
pending.reject(error);
|
|
210
|
+
if (pending.appendToken != null) {
|
|
211
|
+
appendTokens.push(pending.appendToken);
|
|
212
|
+
}
|
|
213
|
+
}
|
|
214
|
+
this._pendingFutures.clear();
|
|
215
|
+
return appendTokens;
|
|
216
|
+
}
|
|
217
|
+
|
|
70
218
|
_markComplete(payload) {
|
|
71
|
-
const {
|
|
219
|
+
const {
|
|
220
|
+
futureIdRanges,
|
|
221
|
+
success,
|
|
222
|
+
errorCode,
|
|
223
|
+
message,
|
|
224
|
+
httpStatusCode,
|
|
225
|
+
httpStatusName,
|
|
226
|
+
// requestId defaults to null (not undefined) so a missing native field still honors the
|
|
227
|
+
// declared `string | null` contract. retryCount defaults to 0: it is a plain number, and a
|
|
228
|
+
// null requestId is what marks the pair absent.
|
|
229
|
+
requestId = null,
|
|
230
|
+
retryCount = 0,
|
|
231
|
+
} = payload;
|
|
72
232
|
|
|
73
233
|
if (futureIdRanges.length % 2 !== 0) {
|
|
74
234
|
const errMsg = `future_id_ranges has odd length ${futureIdRanges.length}, expected even (flattened [min, max] pairs)`;
|
|
@@ -81,6 +241,8 @@ class ElasticChannelAckState {
|
|
|
81
241
|
const error = buildError(success, errorCode, message, httpStatusCode, httpStatusName);
|
|
82
242
|
|
|
83
243
|
const missingIds = [];
|
|
244
|
+
const errorTokensToNotify = [];
|
|
245
|
+
const successTokensToNotify = [];
|
|
84
246
|
for (let i = 0; i < futureIdRanges.length; i += 2) {
|
|
85
247
|
const min = Number(futureIdRanges[i]);
|
|
86
248
|
const max = Number(futureIdRanges[i + 1]);
|
|
@@ -93,12 +255,27 @@ class ElasticChannelAckState {
|
|
|
93
255
|
this._pendingFutures.delete(fid);
|
|
94
256
|
if (error) {
|
|
95
257
|
pending.reject(error);
|
|
258
|
+
// Only appends the caller gave an append token are surfaced to the handler.
|
|
259
|
+
if (pending.appendToken != null) {
|
|
260
|
+
errorTokensToNotify.push(pending.appendToken);
|
|
261
|
+
}
|
|
96
262
|
} else {
|
|
97
263
|
pending.resolve();
|
|
264
|
+
// Same rule on the success side: an untracked append has nothing to report.
|
|
265
|
+
if (pending.appendToken != null) {
|
|
266
|
+
successTokensToNotify.push(pending.appendToken);
|
|
267
|
+
}
|
|
98
268
|
}
|
|
99
269
|
}
|
|
100
270
|
}
|
|
101
271
|
|
|
272
|
+
// Fire once after the resolve/reject loop, so a slow or throwing handler can't
|
|
273
|
+
// delay or skip a sibling append. Mirrors Java firing outside the ack lock.
|
|
274
|
+
// A batch is wholly successful or wholly failed, so only one of these fires.
|
|
275
|
+
// Both carry the request id, and retry count, of the response that drove this ack.
|
|
276
|
+
this.fireErrorHandler(errorTokensToNotify, error, requestId, retryCount);
|
|
277
|
+
this.fireSuccessHandler(successTokensToNotify, requestId, retryCount);
|
|
278
|
+
|
|
102
279
|
// `_invalidated` gate matches Java/Python: after invalidateAll has run,
|
|
103
280
|
// the map is empty and further mark-complete callbacks for in-flight
|
|
104
281
|
// batches are expected to find nothing — don't escalate that to fatal.
|
|
@@ -120,10 +297,11 @@ class ElasticChannelAckState {
|
|
|
120
297
|
payload.httpStatusCode,
|
|
121
298
|
payload.httpStatusName,
|
|
122
299
|
) || new StreamingIngestError("Fatal", "Elastic channel invalidated", 409, "Conflict");
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
300
|
+
// No request id: invalidation fails every in-flight append at once and is not
|
|
301
|
+
// attributable to the one server response any of them was waiting on. Same in-flight race as
|
|
302
|
+
// the close/drop path below: an append whose request was already on the wire is failed here and
|
|
303
|
+
// that request may still be applied, which the ErrorDetail contract covers via the null id.
|
|
304
|
+
this.fireErrorHandler(this._rejectAllPending(error), error, null, 0);
|
|
127
305
|
}
|
|
128
306
|
|
|
129
307
|
/**
|
|
@@ -132,25 +310,32 @@ class ElasticChannelAckState {
|
|
|
132
310
|
* via `_invalidated`. Promises are rejected first so the user observation
|
|
133
311
|
* doesn't depend on the FFI call succeeding; the native invalidate is
|
|
134
312
|
* best-effort — Rust will fire `invalidateAll` on its next informer
|
|
135
|
-
* tick, which finds an empty map and is a no-op.
|
|
313
|
+
* tick, which finds an empty map and is a no-op. The error handler fires last,
|
|
314
|
+
* after both the rejections and the FFI call, with no request id — the error is
|
|
315
|
+
* ours, not a Snowflake response.
|
|
136
316
|
*/
|
|
137
317
|
_invalidateLocally(error) {
|
|
138
318
|
if (this._invalidated) {
|
|
139
319
|
return;
|
|
140
320
|
}
|
|
141
321
|
this._invalidated = true;
|
|
142
|
-
|
|
143
|
-
pending.reject(error);
|
|
144
|
-
}
|
|
145
|
-
this._pendingFutures.clear();
|
|
322
|
+
const errorTokensToNotify = this._rejectAllPending(error);
|
|
146
323
|
if (this._nativeChannel) {
|
|
147
324
|
try {
|
|
148
|
-
this._nativeChannel.invalidateChannel(error.
|
|
325
|
+
this._nativeChannel.invalidateChannel(error.detail);
|
|
149
326
|
} catch (invErr) {
|
|
150
327
|
// eslint-disable-next-line no-console
|
|
151
|
-
console.error(
|
|
328
|
+
console.error(`${LOG_PREFIX} Elastic channel invalidate_channel FFI call failed:`, invErr);
|
|
152
329
|
}
|
|
153
330
|
}
|
|
331
|
+
// Races with an in-flight request, and reports the pessimistic side of it. An append whose
|
|
332
|
+
// request is already on the wire is failed here locally; if it then lands, the rows are in the
|
|
333
|
+
// table even though the customer was told the append failed. The handler reports a null
|
|
334
|
+
// requestId and retryCount 0, which per the ErrorDetail contract reads as "the SDK sent this
|
|
335
|
+
// once, so it introduced no duplicate" -- true, but not a claim that the rows are absent.
|
|
336
|
+
// Nothing here can close the race: the promise has to be settled at close time and the
|
|
337
|
+
// request's outcome is not knowable yet. A caller needing certainty must reconcile downstream.
|
|
338
|
+
this.fireErrorHandler(errorTokensToNotify, error, null, 0);
|
|
154
339
|
}
|
|
155
340
|
}
|
|
156
341
|
|
|
@@ -0,0 +1,73 @@
|
|
|
1
|
+
// Copyright 2026 Snowflake Inc.
|
|
2
|
+
// SPDX-License-Identifier: Apache-2.0
|
|
3
|
+
//
|
|
4
|
+
// Licensed under the Apache License, Version 2.0 (the "License");
|
|
5
|
+
// you may not use this file except in compliance with the License.
|
|
6
|
+
// You may obtain a copy of the License at
|
|
7
|
+
//
|
|
8
|
+
// http://www.apache.org/licenses/LICENSE-2.0
|
|
9
|
+
//
|
|
10
|
+
// Unless required by applicable law or agreed to in writing, software
|
|
11
|
+
// distributed under the License is distributed on an "AS IS" BASIS,
|
|
12
|
+
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
13
|
+
// See the License for the specific language governing permissions and
|
|
14
|
+
// limitations under the License.
|
|
15
|
+
|
|
16
|
+
"use strict";
|
|
17
|
+
|
|
18
|
+
/**
|
|
19
|
+
* The payload handed to an elastic-channel success handler (registered via
|
|
20
|
+
* {@link StreamingIngestElasticChannel#setSuccessHandler}) when appends are
|
|
21
|
+
* acknowledged by Snowflake.
|
|
22
|
+
*
|
|
23
|
+
* Delivered once per successful acknowledgement, carrying the caller-supplied
|
|
24
|
+
* append tokens of every append that was acked together in that server-ack
|
|
25
|
+
* batch. There is no error field — the counterpart to {@link ErrorDetail}, whose
|
|
26
|
+
* `error` explains a failure, has nothing to explain.
|
|
27
|
+
*
|
|
28
|
+
* Instances are created by the SDK and deeply frozen — both the object and its
|
|
29
|
+
* `appendTokens` array — so a handler cannot mutate what a sibling handler or a
|
|
30
|
+
* later log statement will read. Handlers only read this type, so new fields can
|
|
31
|
+
* be added over time without breaking them.
|
|
32
|
+
*/
|
|
33
|
+
class SuccessDetail {
|
|
34
|
+
/**
|
|
35
|
+
* @param {unknown[]} appendTokens - The opaque, caller-supplied append tokens of
|
|
36
|
+
* the appends that were acknowledged together. Never empty; a token reused
|
|
37
|
+
* across appends appears once per acknowledged append (not deduplicated).
|
|
38
|
+
* Frozen here, so callers must pass an array they are handing over.
|
|
39
|
+
* @param {string} requestId - The Snowflake request id of the server response that
|
|
40
|
+
* acknowledged these appends, for support escalation.
|
|
41
|
+
* @param {number} retryCount - How many retries that request took: 0 when the
|
|
42
|
+
* first attempt was acknowledged, N when the SDK retried N times before
|
|
43
|
+
* Snowflake accepted it.
|
|
44
|
+
*
|
|
45
|
+
* **Use this to decide whether these rows may be duplicated in the table.** A retry
|
|
46
|
+
* re-sends the same rowset, and the SDK retries whenever it cannot tell that the
|
|
47
|
+
* previous attempt failed -- a timeout or a 5xx may mean the rows were never
|
|
48
|
+
* processed, or that they were processed and only the response was lost. `0` means
|
|
49
|
+
* the SDK sent this rowset once and introduced no duplicate; `> 0` means it sent it
|
|
50
|
+
* more than once and an earlier attempt may already have been applied, so the rows
|
|
51
|
+
* may appear more than once. Reconcile or de-duplicate downstream if you need
|
|
52
|
+
* exactly-once. The signal is conservative: it counts every re-send, including an
|
|
53
|
+
* authentication refresh, which is rejected before the rows are processed and so
|
|
54
|
+
* cannot have duplicated anything -- `> 0` means "a duplicate is possible", not
|
|
55
|
+
* "a duplicate happened".
|
|
56
|
+
*
|
|
57
|
+
* Read it only when `requestId` is non-null: an acknowledgement with no originating
|
|
58
|
+
* request reports 0 too. A steadily non-zero count also means the SDK is absorbing
|
|
59
|
+
* retries on your behalf.
|
|
60
|
+
*/
|
|
61
|
+
constructor(appendTokens, requestId, retryCount) {
|
|
62
|
+
// Freeze the array as well as the instance: `Object.freeze(this)` alone would
|
|
63
|
+
// still let a handler push into or reorder appendTokens. Only the array
|
|
64
|
+
// itself is frozen, not the tokens inside it — those are the caller's own
|
|
65
|
+
// opaque objects and are handed back untouched.
|
|
66
|
+
this.appendTokens = Object.freeze(appendTokens);
|
|
67
|
+
this.requestId = requestId;
|
|
68
|
+
this.retryCount = retryCount;
|
|
69
|
+
Object.freeze(this);
|
|
70
|
+
}
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
module.exports = { SuccessDetail };
|