postgrejs 3.3.0 → 3.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +55 -1
- package/README.md +68 -7
- package/connection/connection.d.ts +134 -0
- package/connection/connection.js +213 -2
- package/connection/cursor.d.ts +8 -0
- package/connection/cursor.js +17 -0
- package/connection/intl-connection.d.ts +118 -3
- package/connection/intl-connection.js +615 -31
- package/connection/pool.d.ts +16 -0
- package/connection/pool.js +24 -0
- package/connection/prepared-statement.d.ts +64 -1
- package/connection/prepared-statement.js +95 -5
- package/data-types/float4-type.js +2 -1
- package/data-types/float8-type.js +2 -1
- package/data-types/int2-type.js +2 -1
- package/data-types/int4-type.js +2 -1
- package/data-types/int8-type.js +2 -1
- package/data-types/numeric-type.js +7 -0
- package/data-types/oid-type.js +2 -1
- package/index.d.ts +3 -0
- package/index.js +3 -0
- package/interfaces/batch-result.d.ts +42 -0
- package/interfaces/batch-result.js +1 -0
- package/interfaces/copy-from-rows-options.d.ts +58 -0
- package/interfaces/copy-from-rows-options.js +1 -0
- package/interfaces/database-connection-params.d.ts +25 -0
- package/interfaces/pipeline-request.d.ts +16 -0
- package/interfaces/pipeline-request.js +1 -0
- package/interfaces/query-options.d.ts +6 -0
- package/package.json +5 -2
- package/protocol/database-error.d.ts +16 -0
- package/protocol/database-error.js +15 -0
- package/protocol/pg-socket.d.ts +76 -0
- package/protocol/pg-socket.js +130 -12
- package/util/assert-integer.d.ts +43 -0
- package/util/assert-integer.js +58 -0
- package/util/copy-from-rows.d.ts +56 -0
- package/util/copy-from-rows.js +95 -0
- package/util/encode-copy-binary.d.ts +48 -0
- package/util/encode-copy-binary.js +112 -0
|
@@ -1,5 +1,20 @@
|
|
|
1
1
|
import { Protocol } from './protocol.js';
|
|
2
2
|
export class DatabaseError extends Error {
|
|
3
|
+
/**
|
|
4
|
+
* Which entry of a multi-statement submission the server rejected -
|
|
5
|
+
* PreparedStatement.executeBatch()'s parameter sets, or the statements
|
|
6
|
+
* given to Connection.pipeline(). Entries after this index never ran:
|
|
7
|
+
* PostgreSQL discards everything between an error and the single Sync
|
|
8
|
+
* that closes such a submission.
|
|
9
|
+
*/
|
|
10
|
+
failedIndex;
|
|
11
|
+
/**
|
|
12
|
+
* The sets that had already completed when the batch failed, in
|
|
13
|
+
* submission order - only set by PreparedStatement.executeBatch(). They
|
|
14
|
+
* did run, but the batch shares one implicit transaction, so unless an
|
|
15
|
+
* explicit transaction was already open they are rolled back with it.
|
|
16
|
+
*/
|
|
17
|
+
batchResults;
|
|
3
18
|
severity;
|
|
4
19
|
code;
|
|
5
20
|
detail;
|
package/protocol/pg-socket.d.ts
CHANGED
|
@@ -93,6 +93,37 @@ export declare class PgSocket extends SafeEventEmitter {
|
|
|
93
93
|
bind: Frontend.BindMessageArgs;
|
|
94
94
|
describe: Frontend.DescribeMessageArgs;
|
|
95
95
|
execute: Frontend.ExecuteMessageArgs;
|
|
96
|
+
before?: string;
|
|
97
|
+
after?: string;
|
|
98
|
+
}, cb: CaptureCallback): Promise<any>;
|
|
99
|
+
/**
|
|
100
|
+
* Parse + Bind + Describe(portal) + Execute per statement, then a SINGLE
|
|
101
|
+
* Sync for the lot - the wire shape behind Connection.pipeline().
|
|
102
|
+
*
|
|
103
|
+
* The same framing as sendBatchBindExecuteMessages(), and the same
|
|
104
|
+
* consequences, except that each statement carries its own Parse rather
|
|
105
|
+
* than reusing one prepared statement: the statements share one implicit
|
|
106
|
+
* transaction, and PostgreSQL discards everything between an error and
|
|
107
|
+
* the Sync, so a rejected statement stops the ones behind it.
|
|
108
|
+
*
|
|
109
|
+
* Describe is not optional here even though nothing caches its answer.
|
|
110
|
+
* Without it the server sends DataRows for a row-returning statement with
|
|
111
|
+
* no RowDescription in front of them, leaving the column names and types
|
|
112
|
+
* unknown - the reason the one-shot path in sendExtendedQueryMessages()
|
|
113
|
+
* includes it too. It costs about a quarter of a millisecond across
|
|
114
|
+
* twenty statements, measured.
|
|
115
|
+
*
|
|
116
|
+
* Every Execute is unlimited (fetchCount 0): a portal that suspended
|
|
117
|
+
* mid-pipeline would answer PortalSuspended instead of CommandComplete
|
|
118
|
+
* and desynchronise the positional statement-to-result mapping.
|
|
119
|
+
*/
|
|
120
|
+
sendPipelineMessages(args: {
|
|
121
|
+
statements: {
|
|
122
|
+
parse?: Frontend.ParseMessageArgs;
|
|
123
|
+
bind: Frontend.BindMessageArgs;
|
|
124
|
+
describe?: Frontend.DescribeMessageArgs;
|
|
125
|
+
execute: Frontend.ExecuteMessageArgs;
|
|
126
|
+
}[];
|
|
96
127
|
}, cb: CaptureCallback): Promise<any>;
|
|
97
128
|
/**
|
|
98
129
|
* Parse + Describe(statement) + Sync as a single round trip - used by
|
|
@@ -117,6 +148,33 @@ export declare class PgSocket extends SafeEventEmitter {
|
|
|
117
148
|
sendBindExecuteMessages(args: {
|
|
118
149
|
bind: Frontend.BindMessageArgs;
|
|
119
150
|
execute: Frontend.ExecuteMessageArgs;
|
|
151
|
+
before?: string;
|
|
152
|
+
after?: string;
|
|
153
|
+
}, cb: CaptureCallback): Promise<any>;
|
|
154
|
+
/**
|
|
155
|
+
* One Bind+Execute pair per parameter set against the same prepared
|
|
156
|
+
* statement, then a SINGLE Sync for the lot - the wire shape behind
|
|
157
|
+
* PreparedStatement.executeBatch().
|
|
158
|
+
*
|
|
159
|
+
* The single Sync is the whole point, and it is also what gives the
|
|
160
|
+
* batch its semantics. Sending Bind/Execute/Sync per set (what N
|
|
161
|
+
* separate execute() calls do, even pipelined) makes the server finish
|
|
162
|
+
* an implicit transaction and answer ReadyForQuery every time; one Sync
|
|
163
|
+
* at the end collapses that to one, which is where the bulk of the
|
|
164
|
+
* speedup comes from. It also means the sets share one implicit
|
|
165
|
+
* transaction, and that PostgreSQL discards everything between an error
|
|
166
|
+
* and the Sync - so a set that fails stops the ones behind it from
|
|
167
|
+
* running at all. Both of those are inherent to this framing rather
|
|
168
|
+
* than choices made above it.
|
|
169
|
+
*
|
|
170
|
+
* Every Execute here must be unlimited (fetchCount 0): a portal that
|
|
171
|
+
* suspends mid-batch would answer PortalSuspended instead of
|
|
172
|
+
* CommandComplete and desynchronise the positional set-to-result
|
|
173
|
+
* mapping the caller relies on.
|
|
174
|
+
*/
|
|
175
|
+
sendBatchBindExecuteMessages(args: {
|
|
176
|
+
binds: Frontend.BindMessageArgs[];
|
|
177
|
+
execute: Frontend.ExecuteMessageArgs;
|
|
120
178
|
}, cb: CaptureCallback): Promise<any>;
|
|
121
179
|
/**
|
|
122
180
|
* Bind + Describe(portal) + Flush as a single round trip (NOT Sync - see
|
|
@@ -193,6 +251,24 @@ export declare class PgSocket extends SafeEventEmitter {
|
|
|
193
251
|
* "start listening" step to race or misuse.
|
|
194
252
|
*/
|
|
195
253
|
private _sendAndCapture;
|
|
254
|
+
/**
|
|
255
|
+
* Parse + Bind + Execute for a parameterless utility statement, appended
|
|
256
|
+
* to a message list that is about to be closed by one Sync - how
|
|
257
|
+
* sendBindExecuteMessages() and sendExtendedQueryMessages() carry their
|
|
258
|
+
* `before`/`after` SQL (today, SAVEPOINT and RELEASE) in the same round
|
|
259
|
+
* trip as the statement itself rather than as a round trip each.
|
|
260
|
+
*
|
|
261
|
+
* Everything here goes through the unnamed statement and the unnamed
|
|
262
|
+
* portal. That is safe in this order: the caller's own Bind names a
|
|
263
|
+
* prepared statement rather than the unnamed one, and PostgreSQL
|
|
264
|
+
* processes the messages in sequence, so each Bind only ever discards an
|
|
265
|
+
* unnamed portal whose Execute has already run.
|
|
266
|
+
*
|
|
267
|
+
* No Describe: neither statement returns rows, so there is no
|
|
268
|
+
* RowDescription worth asking for. Result formats come from the caller's
|
|
269
|
+
* own Bind args, which likewise cost nothing against no columns.
|
|
270
|
+
*/
|
|
271
|
+
private _pushUtilityStatement;
|
|
196
272
|
protected _removeListeners(): void;
|
|
197
273
|
protected _reset(): void;
|
|
198
274
|
protected _handleConnect(): void;
|
package/protocol/pg-socket.js
CHANGED
|
@@ -238,15 +238,67 @@ export class PgSocket extends SafeEventEmitter {
|
|
|
238
238
|
* statement/portal at the next Parse/Bind referencing the unnamed name.
|
|
239
239
|
*/
|
|
240
240
|
sendExtendedQueryMessages(args, cb) {
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
|
|
241
|
+
if (!args.before && !args.after) {
|
|
242
|
+
const data = [
|
|
243
|
+
this._frontend.getParseMessage(args.parse),
|
|
244
|
+
this._frontend.getBindMessage(args.bind),
|
|
245
|
+
this._frontend.getDescribeMessage(args.describe),
|
|
246
|
+
this._frontend.getExecuteMessage(args.execute),
|
|
247
|
+
this._frontend.getSyncMessage(),
|
|
248
|
+
];
|
|
249
|
+
return this._sendAndCapture(data, cb, 'sendExtendedQueryMessages', args);
|
|
250
|
+
}
|
|
251
|
+
const data = [];
|
|
252
|
+
if (args.before)
|
|
253
|
+
this._pushUtilityStatement(data, args.before, args.bind);
|
|
254
|
+
data.push(this._frontend.getParseMessage(args.parse), this._frontend.getBindMessage(args.bind), this._frontend.getDescribeMessage(args.describe), this._frontend.getExecuteMessage(args.execute));
|
|
255
|
+
if (args.after)
|
|
256
|
+
this._pushUtilityStatement(data, args.after, args.bind);
|
|
257
|
+
data.push(this._frontend.getSyncMessage());
|
|
248
258
|
return this._sendAndCapture(data, cb, 'sendExtendedQueryMessages', args);
|
|
249
259
|
}
|
|
260
|
+
/**
|
|
261
|
+
* Parse + Bind + Describe(portal) + Execute per statement, then a SINGLE
|
|
262
|
+
* Sync for the lot - the wire shape behind Connection.pipeline().
|
|
263
|
+
*
|
|
264
|
+
* The same framing as sendBatchBindExecuteMessages(), and the same
|
|
265
|
+
* consequences, except that each statement carries its own Parse rather
|
|
266
|
+
* than reusing one prepared statement: the statements share one implicit
|
|
267
|
+
* transaction, and PostgreSQL discards everything between an error and
|
|
268
|
+
* the Sync, so a rejected statement stops the ones behind it.
|
|
269
|
+
*
|
|
270
|
+
* Describe is not optional here even though nothing caches its answer.
|
|
271
|
+
* Without it the server sends DataRows for a row-returning statement with
|
|
272
|
+
* no RowDescription in front of them, leaving the column names and types
|
|
273
|
+
* unknown - the reason the one-shot path in sendExtendedQueryMessages()
|
|
274
|
+
* includes it too. It costs about a quarter of a millisecond across
|
|
275
|
+
* twenty statements, measured.
|
|
276
|
+
*
|
|
277
|
+
* Every Execute is unlimited (fetchCount 0): a portal that suspended
|
|
278
|
+
* mid-pipeline would answer PortalSuspended instead of CommandComplete
|
|
279
|
+
* and desynchronise the positional statement-to-result mapping.
|
|
280
|
+
*/
|
|
281
|
+
sendPipelineMessages(args, cb) {
|
|
282
|
+
const l = args.statements.length;
|
|
283
|
+
const data = [];
|
|
284
|
+
let i;
|
|
285
|
+
let st;
|
|
286
|
+
for (i = 0; i < l; i++) {
|
|
287
|
+
st = args.statements[i];
|
|
288
|
+
// Parse and Describe are both optional: a statement bound to one the
|
|
289
|
+
// connection has already prepared needs neither, since the server
|
|
290
|
+
// still holds the plan and IntlConnection still holds the
|
|
291
|
+
// RowDescription that came with it.
|
|
292
|
+
if (st.parse)
|
|
293
|
+
data.push(this._frontend.getParseMessage(st.parse));
|
|
294
|
+
data.push(this._frontend.getBindMessage(st.bind));
|
|
295
|
+
if (st.describe)
|
|
296
|
+
data.push(this._frontend.getDescribeMessage(st.describe));
|
|
297
|
+
data.push(this._frontend.getExecuteMessage(st.execute));
|
|
298
|
+
}
|
|
299
|
+
data.push(this._frontend.getSyncMessage());
|
|
300
|
+
return this._sendAndCapture(data, cb, 'sendPipelineMessages', args);
|
|
301
|
+
}
|
|
250
302
|
/**
|
|
251
303
|
* Parse + Describe(statement) + Sync as a single round trip - used by
|
|
252
304
|
* PreparedStatement.prepare() instead of a separate awaited Parse then a
|
|
@@ -272,13 +324,56 @@ export class PgSocket extends SafeEventEmitter {
|
|
|
272
324
|
* auto-clears an unnamed portal at the next Bind naming it.
|
|
273
325
|
*/
|
|
274
326
|
sendBindExecuteMessages(args, cb) {
|
|
275
|
-
|
|
276
|
-
|
|
277
|
-
|
|
278
|
-
|
|
279
|
-
|
|
327
|
+
if (!args.before && !args.after) {
|
|
328
|
+
const data = [
|
|
329
|
+
this._frontend.getBindMessage(args.bind),
|
|
330
|
+
this._frontend.getExecuteMessage(args.execute),
|
|
331
|
+
this._frontend.getSyncMessage(),
|
|
332
|
+
];
|
|
333
|
+
return this._sendAndCapture(data, cb, 'sendBindExecuteMessages', args);
|
|
334
|
+
}
|
|
335
|
+
const data = [];
|
|
336
|
+
if (args.before)
|
|
337
|
+
this._pushUtilityStatement(data, args.before, args.bind);
|
|
338
|
+
data.push(this._frontend.getBindMessage(args.bind), this._frontend.getExecuteMessage(args.execute));
|
|
339
|
+
if (args.after)
|
|
340
|
+
this._pushUtilityStatement(data, args.after, args.bind);
|
|
341
|
+
data.push(this._frontend.getSyncMessage());
|
|
280
342
|
return this._sendAndCapture(data, cb, 'sendBindExecuteMessages', args);
|
|
281
343
|
}
|
|
344
|
+
/**
|
|
345
|
+
* One Bind+Execute pair per parameter set against the same prepared
|
|
346
|
+
* statement, then a SINGLE Sync for the lot - the wire shape behind
|
|
347
|
+
* PreparedStatement.executeBatch().
|
|
348
|
+
*
|
|
349
|
+
* The single Sync is the whole point, and it is also what gives the
|
|
350
|
+
* batch its semantics. Sending Bind/Execute/Sync per set (what N
|
|
351
|
+
* separate execute() calls do, even pipelined) makes the server finish
|
|
352
|
+
* an implicit transaction and answer ReadyForQuery every time; one Sync
|
|
353
|
+
* at the end collapses that to one, which is where the bulk of the
|
|
354
|
+
* speedup comes from. It also means the sets share one implicit
|
|
355
|
+
* transaction, and that PostgreSQL discards everything between an error
|
|
356
|
+
* and the Sync - so a set that fails stops the ones behind it from
|
|
357
|
+
* running at all. Both of those are inherent to this framing rather
|
|
358
|
+
* than choices made above it.
|
|
359
|
+
*
|
|
360
|
+
* Every Execute here must be unlimited (fetchCount 0): a portal that
|
|
361
|
+
* suspends mid-batch would answer PortalSuspended instead of
|
|
362
|
+
* CommandComplete and desynchronise the positional set-to-result
|
|
363
|
+
* mapping the caller relies on.
|
|
364
|
+
*/
|
|
365
|
+
sendBatchBindExecuteMessages(args, cb) {
|
|
366
|
+
const binds = args.binds;
|
|
367
|
+
const l = binds.length;
|
|
368
|
+
const data = new Array(l * 2 + 1);
|
|
369
|
+
let i;
|
|
370
|
+
for (i = 0; i < l; i++) {
|
|
371
|
+
data[i * 2] = this._frontend.getBindMessage(binds[i]);
|
|
372
|
+
data[i * 2 + 1] = this._frontend.getExecuteMessage(args.execute);
|
|
373
|
+
}
|
|
374
|
+
data[l * 2] = this._frontend.getSyncMessage();
|
|
375
|
+
return this._sendAndCapture(data, cb, 'sendBatchBindExecuteMessages', args);
|
|
376
|
+
}
|
|
282
377
|
/**
|
|
283
378
|
* Bind + Describe(portal) + Flush as a single round trip (NOT Sync - see
|
|
284
379
|
* Portal.bindAndRetrieveFields()'s doc comment for why a Sync here would
|
|
@@ -405,6 +500,29 @@ export class PgSocket extends SafeEventEmitter {
|
|
|
405
500
|
}
|
|
406
501
|
});
|
|
407
502
|
}
|
|
503
|
+
/**
|
|
504
|
+
* Parse + Bind + Execute for a parameterless utility statement, appended
|
|
505
|
+
* to a message list that is about to be closed by one Sync - how
|
|
506
|
+
* sendBindExecuteMessages() and sendExtendedQueryMessages() carry their
|
|
507
|
+
* `before`/`after` SQL (today, SAVEPOINT and RELEASE) in the same round
|
|
508
|
+
* trip as the statement itself rather than as a round trip each.
|
|
509
|
+
*
|
|
510
|
+
* Everything here goes through the unnamed statement and the unnamed
|
|
511
|
+
* portal. That is safe in this order: the caller's own Bind names a
|
|
512
|
+
* prepared statement rather than the unnamed one, and PostgreSQL
|
|
513
|
+
* processes the messages in sequence, so each Bind only ever discards an
|
|
514
|
+
* unnamed portal whose Execute has already run.
|
|
515
|
+
*
|
|
516
|
+
* No Describe: neither statement returns rows, so there is no
|
|
517
|
+
* RowDescription worth asking for. Result formats come from the caller's
|
|
518
|
+
* own Bind args, which likewise cost nothing against no columns.
|
|
519
|
+
*/
|
|
520
|
+
_pushUtilityStatement(data, sql, bind) {
|
|
521
|
+
data.push(this._frontend.getParseMessage({ sql }), this._frontend.getBindMessage({
|
|
522
|
+
typeMap: bind.typeMap,
|
|
523
|
+
queryOptions: bind.queryOptions,
|
|
524
|
+
}), this._frontend.getExecuteMessage({}));
|
|
525
|
+
}
|
|
408
526
|
_removeListeners() {
|
|
409
527
|
if (!this._socket)
|
|
410
528
|
return;
|
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Returns `v` coerced to an integer, or throws when it cannot be one.
|
|
3
|
+
*
|
|
4
|
+
* `fastParseInt()` already answers NaN for a value with no integer reading
|
|
5
|
+
* - `'abc'`, `{}`, `[]`, `true` - but `Buffer.writeInt32BE(NaN)` then
|
|
6
|
+
* stores a silent zero, so the answer is lost exactly where it matters.
|
|
7
|
+
* Nothing downstream can tell that apart from a real zero afterwards: the
|
|
8
|
+
* row inserts, the copy succeeds, and the wrong number is in the table.
|
|
9
|
+
*
|
|
10
|
+
* Floats are deliberately not rejected: truncating 3.7 to 3 is the
|
|
11
|
+
* long-standing behaviour of these encoders and is usually what a caller
|
|
12
|
+
* writing into an integer column means. Only "this is not a number at all"
|
|
13
|
+
* is refused.
|
|
14
|
+
*
|
|
15
|
+
* Unlike the integer types, float4/float8/numeric leave NaN alone - it is a
|
|
16
|
+
* value PostgreSQL stores and returns as NaN, distinct from NULL, on every
|
|
17
|
+
* supported server version.
|
|
18
|
+
*/
|
|
19
|
+
export declare function assertInteger(v: number, typeName: string): number;
|
|
20
|
+
/**
|
|
21
|
+
* Returns `parsed` when a value that was NOT already a number coerced to
|
|
22
|
+
* one, and throws when it did not.
|
|
23
|
+
*
|
|
24
|
+
* Kept apart from assertInteger() because the two families disagree about
|
|
25
|
+
* NaN on purpose. float4/float8/numeric all store NaN as a value distinct
|
|
26
|
+
* from NULL, so a caller passing the number NaN means it and it never
|
|
27
|
+
* reaches here - the encoders take `typeof v === 'number'` straight
|
|
28
|
+
* through. What does reach here is `'abc'`, `{}` or `true`, whose coercion
|
|
29
|
+
* produced NaN because they have no numeric reading at all; writing those
|
|
30
|
+
* as NaN would be indistinguishable afterwards from a NaN that was meant.
|
|
31
|
+
*/
|
|
32
|
+
export declare function assertCoercedNumber(parsed: number, typeName: string): number;
|
|
33
|
+
/**
|
|
34
|
+
* Refuses a value that only has an integer reading through a coercion
|
|
35
|
+
* nobody asked for.
|
|
36
|
+
*
|
|
37
|
+
* int8 goes to `BigInt` rather than through `fastParseInt()`, and BigInt is
|
|
38
|
+
* happy to convert things that are not numbers at all: `BigInt([])` is 0n
|
|
39
|
+
* because an empty array stringifies to "", and `BigInt(true)` is 1n. A
|
|
40
|
+
* bad value in a bigint column would land as a plausible-looking 0 or 1,
|
|
41
|
+
* so only the types that genuinely carry an integer are let through.
|
|
42
|
+
*/
|
|
43
|
+
export declare function assertBigIntSource(v: any, typeName: string): bigint | number;
|
|
@@ -0,0 +1,58 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Returns `v` coerced to an integer, or throws when it cannot be one.
|
|
3
|
+
*
|
|
4
|
+
* `fastParseInt()` already answers NaN for a value with no integer reading
|
|
5
|
+
* - `'abc'`, `{}`, `[]`, `true` - but `Buffer.writeInt32BE(NaN)` then
|
|
6
|
+
* stores a silent zero, so the answer is lost exactly where it matters.
|
|
7
|
+
* Nothing downstream can tell that apart from a real zero afterwards: the
|
|
8
|
+
* row inserts, the copy succeeds, and the wrong number is in the table.
|
|
9
|
+
*
|
|
10
|
+
* Floats are deliberately not rejected: truncating 3.7 to 3 is the
|
|
11
|
+
* long-standing behaviour of these encoders and is usually what a caller
|
|
12
|
+
* writing into an integer column means. Only "this is not a number at all"
|
|
13
|
+
* is refused.
|
|
14
|
+
*
|
|
15
|
+
* Unlike the integer types, float4/float8/numeric leave NaN alone - it is a
|
|
16
|
+
* value PostgreSQL stores and returns as NaN, distinct from NULL, on every
|
|
17
|
+
* supported server version.
|
|
18
|
+
*/
|
|
19
|
+
export function assertInteger(v, typeName) {
|
|
20
|
+
if (Number.isNaN(v)) {
|
|
21
|
+
throw new TypeError(`Cannot encode value as ${typeName}: it has no integer value`);
|
|
22
|
+
}
|
|
23
|
+
return v;
|
|
24
|
+
}
|
|
25
|
+
/**
|
|
26
|
+
* Returns `parsed` when a value that was NOT already a number coerced to
|
|
27
|
+
* one, and throws when it did not.
|
|
28
|
+
*
|
|
29
|
+
* Kept apart from assertInteger() because the two families disagree about
|
|
30
|
+
* NaN on purpose. float4/float8/numeric all store NaN as a value distinct
|
|
31
|
+
* from NULL, so a caller passing the number NaN means it and it never
|
|
32
|
+
* reaches here - the encoders take `typeof v === 'number'` straight
|
|
33
|
+
* through. What does reach here is `'abc'`, `{}` or `true`, whose coercion
|
|
34
|
+
* produced NaN because they have no numeric reading at all; writing those
|
|
35
|
+
* as NaN would be indistinguishable afterwards from a NaN that was meant.
|
|
36
|
+
*/
|
|
37
|
+
export function assertCoercedNumber(parsed, typeName) {
|
|
38
|
+
if (Number.isNaN(parsed)) {
|
|
39
|
+
throw new TypeError(`Cannot encode value as ${typeName}: it has no numeric value`);
|
|
40
|
+
}
|
|
41
|
+
return parsed;
|
|
42
|
+
}
|
|
43
|
+
/**
|
|
44
|
+
* Refuses a value that only has an integer reading through a coercion
|
|
45
|
+
* nobody asked for.
|
|
46
|
+
*
|
|
47
|
+
* int8 goes to `BigInt` rather than through `fastParseInt()`, and BigInt is
|
|
48
|
+
* happy to convert things that are not numbers at all: `BigInt([])` is 0n
|
|
49
|
+
* because an empty array stringifies to "", and `BigInt(true)` is 1n. A
|
|
50
|
+
* bad value in a bigint column would land as a plausible-looking 0 or 1,
|
|
51
|
+
* so only the types that genuinely carry an integer are let through.
|
|
52
|
+
*/
|
|
53
|
+
export function assertBigIntSource(v, typeName) {
|
|
54
|
+
const t = typeof v;
|
|
55
|
+
if (t === 'bigint' || t === 'number' || t === 'string')
|
|
56
|
+
return v;
|
|
57
|
+
throw new TypeError(`Cannot encode value as ${typeName}: it has no integer value`);
|
|
58
|
+
}
|
|
@@ -0,0 +1,56 @@
|
|
|
1
|
+
import type { Readable } from 'node:stream';
|
|
2
|
+
import type { CopyFromRowsOptions } from '../interfaces/copy-from-rows-options.js';
|
|
3
|
+
import type { OID } from '../types.js';
|
|
4
|
+
/** Default bytes accumulated before a CopyData message goes out. */
|
|
5
|
+
export declare const DEFAULT_COPY_CHUNK_SIZE: number;
|
|
6
|
+
export type CopyRow = any[] | Record<string, any>;
|
|
7
|
+
export type CopyRowSource = Iterable<CopyRow> | AsyncIterable<CopyRow> | Readable;
|
|
8
|
+
/**
|
|
9
|
+
* Quotes a possibly schema-qualified name. A single dot separates schema
|
|
10
|
+
* from table, which is what PostgreSQL's own client tools assume for a name
|
|
11
|
+
* given as one string; a table whose name genuinely contains a dot has to
|
|
12
|
+
* be reached through a view or a plain (unqualified) name instead.
|
|
13
|
+
*/
|
|
14
|
+
export declare function quoteQualifiedName(name: string): string;
|
|
15
|
+
/**
|
|
16
|
+
* The statement whose RowDescription tells us the destination columns'
|
|
17
|
+
* type OIDs. `where false` keeps the server from producing rows for a
|
|
18
|
+
* Describe that is never followed by an Execute, and letting PostgreSQL
|
|
19
|
+
* resolve the name is the point: schemas, quoting, search_path and views
|
|
20
|
+
* all behave exactly as they will for the COPY itself, which parsing the
|
|
21
|
+
* name here could only approximate.
|
|
22
|
+
*/
|
|
23
|
+
export declare function buildProbeSql(table: string, columns?: string[]): string;
|
|
24
|
+
export declare function buildCopySql(table: string, columns?: string[]): string;
|
|
25
|
+
export interface CopyFromRowsContext {
|
|
26
|
+
columns: string[];
|
|
27
|
+
dataTypeIds: OID[];
|
|
28
|
+
options: CopyFromRowsOptions;
|
|
29
|
+
/**
|
|
30
|
+
* Sends one CopyData message, resolving once the socket has taken it -
|
|
31
|
+
* immediately when it had room, and only after it drains when it did
|
|
32
|
+
* not. Awaiting this is what makes backpressure reach the source: the
|
|
33
|
+
* `for await` below cannot ask for another row until it settles.
|
|
34
|
+
*/
|
|
35
|
+
write: (chunk: Buffer) => Promise<void>;
|
|
36
|
+
}
|
|
37
|
+
/**
|
|
38
|
+
* Encodes `source` into binary COPY chunks and hands them to `write`,
|
|
39
|
+
* accumulating rows until a chunk is worth sending.
|
|
40
|
+
*
|
|
41
|
+
* The chunk size is the difference between this being fast and being
|
|
42
|
+
* pointless: one CopyData per row costs a message header and a socket write
|
|
43
|
+
* each time, which measured 1208ms against 129ms at 64KB for the same
|
|
44
|
+
* 200,000 rows. Sizes from 64KB to 256KB all land near the floor, so the
|
|
45
|
+
* default sits inside that range rather than at either edge.
|
|
46
|
+
*
|
|
47
|
+
* Pulls from `source` rather than being pushed into, so an async iterable -
|
|
48
|
+
* which every Readable is - only produces the next row once the previous
|
|
49
|
+
* chunk has been accepted, making backpressure the caller's own loop
|
|
50
|
+
* pausing rather than an unbounded queue here.
|
|
51
|
+
*/
|
|
52
|
+
export declare function writeCopyBinaryRows(source: CopyRowSource, ctx: CopyFromRowsContext): Promise<{
|
|
53
|
+
rowCount: number;
|
|
54
|
+
skippedRows: number;
|
|
55
|
+
nulledValues: number;
|
|
56
|
+
}>;
|
|
@@ -0,0 +1,95 @@
|
|
|
1
|
+
import { GlobalTypeMap } from '../data-type-map.js';
|
|
2
|
+
import { SmartBuffer } from '../protocol/smart-buffer.js';
|
|
3
|
+
import { assertCopyBinaryEncodable, COPY_BINARY_HEADER, COPY_BINARY_TRAILER, encodeCopyBinaryRow, } from './encode-copy-binary.js';
|
|
4
|
+
import { escapeIdentifier } from './escape-identifier.js';
|
|
5
|
+
/** Default bytes accumulated before a CopyData message goes out. */
|
|
6
|
+
export const DEFAULT_COPY_CHUNK_SIZE = 128 * 1024;
|
|
7
|
+
/**
|
|
8
|
+
* Quotes a possibly schema-qualified name. A single dot separates schema
|
|
9
|
+
* from table, which is what PostgreSQL's own client tools assume for a name
|
|
10
|
+
* given as one string; a table whose name genuinely contains a dot has to
|
|
11
|
+
* be reached through a view or a plain (unqualified) name instead.
|
|
12
|
+
*/
|
|
13
|
+
export function quoteQualifiedName(name) {
|
|
14
|
+
const dot = name.indexOf('.');
|
|
15
|
+
return dot < 0
|
|
16
|
+
? escapeIdentifier(name)
|
|
17
|
+
: escapeIdentifier(name.slice(0, dot)) +
|
|
18
|
+
'.' +
|
|
19
|
+
escapeIdentifier(name.slice(dot + 1));
|
|
20
|
+
}
|
|
21
|
+
/**
|
|
22
|
+
* The statement whose RowDescription tells us the destination columns'
|
|
23
|
+
* type OIDs. `where false` keeps the server from producing rows for a
|
|
24
|
+
* Describe that is never followed by an Execute, and letting PostgreSQL
|
|
25
|
+
* resolve the name is the point: schemas, quoting, search_path and views
|
|
26
|
+
* all behave exactly as they will for the COPY itself, which parsing the
|
|
27
|
+
* name here could only approximate.
|
|
28
|
+
*/
|
|
29
|
+
export function buildProbeSql(table, columns) {
|
|
30
|
+
const cols = columns?.length ? columns.map(escapeIdentifier).join(', ') : '*';
|
|
31
|
+
return `select ${cols} from ${quoteQualifiedName(table)} where false`;
|
|
32
|
+
}
|
|
33
|
+
export function buildCopySql(table, columns) {
|
|
34
|
+
const cols = columns?.length
|
|
35
|
+
? ' (' + columns.map(escapeIdentifier).join(', ') + ')'
|
|
36
|
+
: '';
|
|
37
|
+
return (`copy ${quoteQualifiedName(table)}${cols} ` +
|
|
38
|
+
`from stdin with (format binary)`);
|
|
39
|
+
}
|
|
40
|
+
/** Positional array, or an object read by column name. */
|
|
41
|
+
function toValues(row, columns) {
|
|
42
|
+
if (Array.isArray(row))
|
|
43
|
+
return row;
|
|
44
|
+
const l = columns.length;
|
|
45
|
+
const out = new Array(l);
|
|
46
|
+
let i;
|
|
47
|
+
for (i = 0; i < l; i++)
|
|
48
|
+
out[i] = row[columns[i]];
|
|
49
|
+
return out;
|
|
50
|
+
}
|
|
51
|
+
/**
|
|
52
|
+
* Encodes `source` into binary COPY chunks and hands them to `write`,
|
|
53
|
+
* accumulating rows until a chunk is worth sending.
|
|
54
|
+
*
|
|
55
|
+
* The chunk size is the difference between this being fast and being
|
|
56
|
+
* pointless: one CopyData per row costs a message header and a socket write
|
|
57
|
+
* each time, which measured 1208ms against 129ms at 64KB for the same
|
|
58
|
+
* 200,000 rows. Sizes from 64KB to 256KB all land near the floor, so the
|
|
59
|
+
* default sits inside that range rather than at either edge.
|
|
60
|
+
*
|
|
61
|
+
* Pulls from `source` rather than being pushed into, so an async iterable -
|
|
62
|
+
* which every Readable is - only produces the next row once the previous
|
|
63
|
+
* chunk has been accepted, making backpressure the caller's own loop
|
|
64
|
+
* pausing rather than an unbounded queue here.
|
|
65
|
+
*/
|
|
66
|
+
export async function writeCopyBinaryRows(source, ctx) {
|
|
67
|
+
const { columns, dataTypeIds, options } = ctx;
|
|
68
|
+
const typeMap = options.typeMap || GlobalTypeMap;
|
|
69
|
+
assertCopyBinaryEncodable(dataTypeIds, columns, typeMap);
|
|
70
|
+
const chunkSize = options.chunkSize || DEFAULT_COPY_CHUNK_SIZE;
|
|
71
|
+
const io = new SmartBuffer();
|
|
72
|
+
io.start();
|
|
73
|
+
io.writeBytes(COPY_BINARY_HEADER);
|
|
74
|
+
const onInvalidValue = options.onInvalidValue || 'throw';
|
|
75
|
+
let rowCount = 0;
|
|
76
|
+
let skippedRows = 0;
|
|
77
|
+
const stats = { nulled: 0 };
|
|
78
|
+
let index = 0;
|
|
79
|
+
for await (const row of source) {
|
|
80
|
+
const written = encodeCopyBinaryRow(io, toValues(row, columns), dataTypeIds, columns, typeMap, options, index, onInvalidValue, stats);
|
|
81
|
+
index++;
|
|
82
|
+
if (!written) {
|
|
83
|
+
skippedRows++;
|
|
84
|
+
continue;
|
|
85
|
+
}
|
|
86
|
+
rowCount++;
|
|
87
|
+
if (io.size >= chunkSize) {
|
|
88
|
+
await ctx.write(io.flush());
|
|
89
|
+
io.start();
|
|
90
|
+
}
|
|
91
|
+
}
|
|
92
|
+
io.writeBytes(COPY_BINARY_TRAILER);
|
|
93
|
+
await ctx.write(io.flush());
|
|
94
|
+
return { rowCount, skippedRows, nulledValues: stats.nulled };
|
|
95
|
+
}
|
|
@@ -0,0 +1,48 @@
|
|
|
1
|
+
import type { DataTypeMap } from '../data-type-map.js';
|
|
2
|
+
import type { DataMappingOptions } from '../interfaces/data-mapping-options.js';
|
|
3
|
+
import type { SmartBuffer } from '../protocol/smart-buffer.js';
|
|
4
|
+
import type { OID } from '../types.js';
|
|
5
|
+
/**
|
|
6
|
+
* `PGCOPY\n\377\r\n\0` followed by two int32 words: a flags field and the
|
|
7
|
+
* length of a header extension area. Bit 16 of the flags marks a stream
|
|
8
|
+
* that carries an OID per row - an obsolete variant PostgreSQL still reads
|
|
9
|
+
* but nothing should write - so both words are zero here.
|
|
10
|
+
*/
|
|
11
|
+
export declare const COPY_BINARY_HEADER: Buffer<ArrayBuffer>;
|
|
12
|
+
/** A field count of -1 where a row would start, ending the stream. */
|
|
13
|
+
export declare const COPY_BINARY_TRAILER: Buffer<ArrayBuffer>;
|
|
14
|
+
/**
|
|
15
|
+
* Appends one row to `io` in PostgreSQL's binary COPY format: an int16
|
|
16
|
+
* field count, then per field an int32 byte length followed by that many
|
|
17
|
+
* bytes, with -1 standing in for NULL.
|
|
18
|
+
*
|
|
19
|
+
* Each value is written by its own type's `encodeBinary`, which is the same
|
|
20
|
+
* function a binary query parameter goes through (see Frontend's Bind) -
|
|
21
|
+
* both are the type's `send` representation, so there is one encoder per
|
|
22
|
+
* type rather than a COPY-specific set.
|
|
23
|
+
*
|
|
24
|
+
* Binary COPY performs no conversion on the server: the bytes have to be
|
|
25
|
+
* exactly what the destination column's type expects. `dataTypeIds` must
|
|
26
|
+
* therefore be the column's own OIDs, which is why the caller obtains them
|
|
27
|
+
* from the server rather than inferring them from the JavaScript values.
|
|
28
|
+
*
|
|
29
|
+
* @param rowIndex Only used to say which row failed, since a value that
|
|
30
|
+
* cannot be encoded otherwise surfaces from the server as an opaque
|
|
31
|
+
* "incorrect binary data format" naming neither row nor column.
|
|
32
|
+
* @returns Whether the row was written - false only under `'skip'`, where
|
|
33
|
+
* the partial row has already been unwound.
|
|
34
|
+
*/
|
|
35
|
+
export declare function encodeCopyBinaryRow(io: SmartBuffer, values: any[], dataTypeIds: OID[], columnNames: string[], typeMap: DataTypeMap, options: DataMappingOptions, rowIndex: number, onInvalidValue?: 'throw' | 'null' | 'skip',
|
|
36
|
+
/** Incremented per value replaced under `'null'`; counted here rather
|
|
37
|
+
* than inferred by the caller, which cannot see inside a written row. */
|
|
38
|
+
stats?: {
|
|
39
|
+
nulled: number;
|
|
40
|
+
}): boolean;
|
|
41
|
+
/**
|
|
42
|
+
* Fails before a single byte goes out when any column's type cannot be
|
|
43
|
+
* written in binary. Worth doing up front: once the server has answered
|
|
44
|
+
* CopyInResponse it is waiting for a complete stream, so discovering an
|
|
45
|
+
* unencodable column at row 900,000 means aborting a copy that was already
|
|
46
|
+
* most of the way done.
|
|
47
|
+
*/
|
|
48
|
+
export declare function assertCopyBinaryEncodable(dataTypeIds: OID[], columnNames: string[], typeMap: DataTypeMap): void;
|