postgrejs 3.1.3 → 3.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +79 -1
- package/README.md +79 -13
- package/connection/connection.d.ts +97 -0
- package/connection/connection.js +141 -2
- package/connection/cursor.d.ts +2 -1
- package/connection/cursor.js +6 -6
- package/connection/intl-connection.d.ts +113 -3
- package/connection/intl-connection.js +530 -45
- package/connection/prepared-statement.d.ts +64 -1
- package/connection/prepared-statement.js +97 -6
- package/data-types/bool-type.js +0 -1
- package/data-types/box-type.js +0 -1
- package/data-types/bytea-type.js +5 -3
- package/data-types/char-type.js +2 -2
- package/data-types/circle-type.js +0 -1
- package/data-types/date-type.js +1 -2
- package/data-types/float4-type.js +4 -5
- package/data-types/float8-type.js +4 -6
- package/data-types/int2-type.js +2 -2
- package/data-types/int2-vector-type.js +3 -2
- package/data-types/int4-type.js +2 -2
- package/data-types/int8-type.js +2 -2
- package/data-types/json-type.js +2 -2
- package/data-types/jsonb-type.js +6 -5
- package/data-types/lseg-type.js +0 -1
- package/data-types/numeric-type.js +9 -4
- package/data-types/oid-type.js +2 -2
- package/data-types/oid-vector-type.js +3 -2
- package/data-types/point-type.js +0 -1
- package/data-types/time-type.js +1 -2
- package/data-types/timestamp-type.js +13 -3
- package/data-types/timestamptz-type.js +27 -10
- package/data-types/uuid-type.js +1 -2
- package/data-types/varchar-type.js +2 -2
- package/index.d.ts +4 -0
- package/index.js +4 -0
- package/interfaces/batch-result.d.ts +42 -0
- package/interfaces/batch-result.js +1 -0
- package/interfaces/command-result.d.ts +3 -2
- package/interfaces/copy-from-rows-options.d.ts +58 -0
- package/interfaces/copy-from-rows-options.js +1 -0
- package/interfaces/data-type.d.ts +0 -1
- package/interfaces/database-connection-params.d.ts +25 -0
- package/interfaces/pipeline-request.d.ts +16 -0
- package/interfaces/pipeline-request.js +1 -0
- package/interfaces/query-options.d.ts +18 -0
- package/interfaces/script-execute-options.d.ts +12 -0
- package/package.json +18 -3
- package/protocol/backend.js +6 -6
- package/protocol/buffer-reader.d.ts +7 -12
- package/protocol/buffer-reader.js +11 -58
- package/protocol/database-error.d.ts +16 -0
- package/protocol/database-error.js +15 -0
- package/protocol/frontend.js +86 -96
- package/protocol/pg-socket.d.ts +82 -33
- package/protocol/pg-socket.js +143 -86
- package/protocol/pgoutput.js +2 -2
- package/protocol/smart-buffer.d.ts +11 -29
- package/protocol/smart-buffer.js +19 -162
- package/types.d.ts +13 -1
- package/util/assert-integer.d.ts +43 -0
- package/util/assert-integer.js +58 -0
- package/util/bigint-methods.d.ts +0 -1
- package/util/bigint-methods.js +0 -22
- package/util/copy-from-rows.d.ts +56 -0
- package/util/copy-from-rows.js +95 -0
- package/util/decode-binaryarray.d.ts +1 -1
- package/util/decode-binaryarray.js +7 -7
- package/util/encode-binaryarray.js +6 -6
- package/util/encode-copy-binary.d.ts +48 -0
- package/util/encode-copy-binary.js +112 -0
- package/util/fast-parsefloat.d.ts +15 -0
- package/util/fast-parsefloat.js +141 -0
- package/util/get-parsers.js +4 -8
- package/util/parse-datetime.js +7 -0
- package/util/parse-pg-timestamp.d.ts +31 -0
- package/util/parse-pg-timestamp.js +199 -0
- package/util/parse-row.js +120 -0
- package/util/row-decoder.d.ts +49 -0
- package/util/row-decoder.js +63 -0
package/util/bigint-methods.d.ts
CHANGED
package/util/bigint-methods.js
CHANGED
|
@@ -1,6 +1,5 @@
|
|
|
1
1
|
/* eslint-disable */
|
|
2
2
|
const big0 = BigInt(0);
|
|
3
|
-
const beAnd = BigInt('0xffffffff');
|
|
4
3
|
const big32 = BigInt(32);
|
|
5
4
|
// https://github.com/nodejs/node/blob/v13.9.0/lib/internal/buffer.js
|
|
6
5
|
export function readBigInt64BE(buf, offset = 0) {
|
|
@@ -18,24 +17,3 @@ export function readBigInt64BE(buf, offset = 0) {
|
|
|
18
17
|
buf[++offset] * 2 ** 8 +
|
|
19
18
|
last));
|
|
20
19
|
}
|
|
21
|
-
export function writeBigUInt64BE(buf, value, offset = 0) {
|
|
22
|
-
// @ts-ignore
|
|
23
|
-
let lo = Number(value & beAnd);
|
|
24
|
-
buf[offset + 7] = lo;
|
|
25
|
-
lo = lo >> 8;
|
|
26
|
-
buf[offset + 6] = lo;
|
|
27
|
-
lo = lo >> 8;
|
|
28
|
-
buf[offset + 5] = lo;
|
|
29
|
-
lo = lo >> 8;
|
|
30
|
-
buf[offset + 4] = lo;
|
|
31
|
-
// @ts-ignore
|
|
32
|
-
let hi = Number((value >> big32) & beAnd);
|
|
33
|
-
buf[offset + 3] = hi;
|
|
34
|
-
hi = hi >> 8;
|
|
35
|
-
buf[offset + 2] = hi;
|
|
36
|
-
hi = hi >> 8;
|
|
37
|
-
buf[offset + 1] = hi;
|
|
38
|
-
hi = hi >> 8;
|
|
39
|
-
buf[offset] = hi;
|
|
40
|
-
return offset + 8;
|
|
41
|
-
}
|
|
@@ -0,0 +1,56 @@
|
|
|
1
|
+
import type { Readable } from 'node:stream';
|
|
2
|
+
import type { CopyFromRowsOptions } from '../interfaces/copy-from-rows-options.js';
|
|
3
|
+
import type { OID } from '../types.js';
|
|
4
|
+
/** Default bytes accumulated before a CopyData message goes out. */
|
|
5
|
+
export declare const DEFAULT_COPY_CHUNK_SIZE: number;
|
|
6
|
+
export type CopyRow = any[] | Record<string, any>;
|
|
7
|
+
export type CopyRowSource = Iterable<CopyRow> | AsyncIterable<CopyRow> | Readable;
|
|
8
|
+
/**
|
|
9
|
+
* Quotes a possibly schema-qualified name. A single dot separates schema
|
|
10
|
+
* from table, which is what PostgreSQL's own client tools assume for a name
|
|
11
|
+
* given as one string; a table whose name genuinely contains a dot has to
|
|
12
|
+
* be reached through a view or a plain (unqualified) name instead.
|
|
13
|
+
*/
|
|
14
|
+
export declare function quoteQualifiedName(name: string): string;
|
|
15
|
+
/**
|
|
16
|
+
* The statement whose RowDescription tells us the destination columns'
|
|
17
|
+
* type OIDs. `where false` keeps the server from producing rows for a
|
|
18
|
+
* Describe that is never followed by an Execute, and letting PostgreSQL
|
|
19
|
+
* resolve the name is the point: schemas, quoting, search_path and views
|
|
20
|
+
* all behave exactly as they will for the COPY itself, which parsing the
|
|
21
|
+
* name here could only approximate.
|
|
22
|
+
*/
|
|
23
|
+
export declare function buildProbeSql(table: string, columns?: string[]): string;
|
|
24
|
+
export declare function buildCopySql(table: string, columns?: string[]): string;
|
|
25
|
+
export interface CopyFromRowsContext {
|
|
26
|
+
columns: string[];
|
|
27
|
+
dataTypeIds: OID[];
|
|
28
|
+
options: CopyFromRowsOptions;
|
|
29
|
+
/**
|
|
30
|
+
* Sends one CopyData message, resolving once the socket has taken it -
|
|
31
|
+
* immediately when it had room, and only after it drains when it did
|
|
32
|
+
* not. Awaiting this is what makes backpressure reach the source: the
|
|
33
|
+
* `for await` below cannot ask for another row until it settles.
|
|
34
|
+
*/
|
|
35
|
+
write: (chunk: Buffer) => Promise<void>;
|
|
36
|
+
}
|
|
37
|
+
/**
|
|
38
|
+
* Encodes `source` into binary COPY chunks and hands them to `write`,
|
|
39
|
+
* accumulating rows until a chunk is worth sending.
|
|
40
|
+
*
|
|
41
|
+
* The chunk size is the difference between this being fast and being
|
|
42
|
+
* pointless: one CopyData per row costs a message header and a socket write
|
|
43
|
+
* each time, which measured 1208ms against 129ms at 64KB for the same
|
|
44
|
+
* 200,000 rows. Sizes from 64KB to 256KB all land near the floor, so the
|
|
45
|
+
* default sits inside that range rather than at either edge.
|
|
46
|
+
*
|
|
47
|
+
* Pulls from `source` rather than being pushed into, so an async iterable -
|
|
48
|
+
* which every Readable is - only produces the next row once the previous
|
|
49
|
+
* chunk has been accepted, making backpressure the caller's own loop
|
|
50
|
+
* pausing rather than an unbounded queue here.
|
|
51
|
+
*/
|
|
52
|
+
export declare function writeCopyBinaryRows(source: CopyRowSource, ctx: CopyFromRowsContext): Promise<{
|
|
53
|
+
rowCount: number;
|
|
54
|
+
skippedRows: number;
|
|
55
|
+
nulledValues: number;
|
|
56
|
+
}>;
|
|
@@ -0,0 +1,95 @@
|
|
|
1
|
+
import { GlobalTypeMap } from '../data-type-map.js';
|
|
2
|
+
import { SmartBuffer } from '../protocol/smart-buffer.js';
|
|
3
|
+
import { assertCopyBinaryEncodable, COPY_BINARY_HEADER, COPY_BINARY_TRAILER, encodeCopyBinaryRow, } from './encode-copy-binary.js';
|
|
4
|
+
import { escapeIdentifier } from './escape-identifier.js';
|
|
5
|
+
/** Default bytes accumulated before a CopyData message goes out. */
|
|
6
|
+
export const DEFAULT_COPY_CHUNK_SIZE = 128 * 1024;
|
|
7
|
+
/**
|
|
8
|
+
* Quotes a possibly schema-qualified name. A single dot separates schema
|
|
9
|
+
* from table, which is what PostgreSQL's own client tools assume for a name
|
|
10
|
+
* given as one string; a table whose name genuinely contains a dot has to
|
|
11
|
+
* be reached through a view or a plain (unqualified) name instead.
|
|
12
|
+
*/
|
|
13
|
+
export function quoteQualifiedName(name) {
|
|
14
|
+
const dot = name.indexOf('.');
|
|
15
|
+
return dot < 0
|
|
16
|
+
? escapeIdentifier(name)
|
|
17
|
+
: escapeIdentifier(name.slice(0, dot)) +
|
|
18
|
+
'.' +
|
|
19
|
+
escapeIdentifier(name.slice(dot + 1));
|
|
20
|
+
}
|
|
21
|
+
/**
|
|
22
|
+
* The statement whose RowDescription tells us the destination columns'
|
|
23
|
+
* type OIDs. `where false` keeps the server from producing rows for a
|
|
24
|
+
* Describe that is never followed by an Execute, and letting PostgreSQL
|
|
25
|
+
* resolve the name is the point: schemas, quoting, search_path and views
|
|
26
|
+
* all behave exactly as they will for the COPY itself, which parsing the
|
|
27
|
+
* name here could only approximate.
|
|
28
|
+
*/
|
|
29
|
+
export function buildProbeSql(table, columns) {
|
|
30
|
+
const cols = columns?.length ? columns.map(escapeIdentifier).join(', ') : '*';
|
|
31
|
+
return `select ${cols} from ${quoteQualifiedName(table)} where false`;
|
|
32
|
+
}
|
|
33
|
+
export function buildCopySql(table, columns) {
|
|
34
|
+
const cols = columns?.length
|
|
35
|
+
? ' (' + columns.map(escapeIdentifier).join(', ') + ')'
|
|
36
|
+
: '';
|
|
37
|
+
return (`copy ${quoteQualifiedName(table)}${cols} ` +
|
|
38
|
+
`from stdin with (format binary)`);
|
|
39
|
+
}
|
|
40
|
+
/** Positional array, or an object read by column name. */
|
|
41
|
+
function toValues(row, columns) {
|
|
42
|
+
if (Array.isArray(row))
|
|
43
|
+
return row;
|
|
44
|
+
const l = columns.length;
|
|
45
|
+
const out = new Array(l);
|
|
46
|
+
let i;
|
|
47
|
+
for (i = 0; i < l; i++)
|
|
48
|
+
out[i] = row[columns[i]];
|
|
49
|
+
return out;
|
|
50
|
+
}
|
|
51
|
+
/**
|
|
52
|
+
* Encodes `source` into binary COPY chunks and hands them to `write`,
|
|
53
|
+
* accumulating rows until a chunk is worth sending.
|
|
54
|
+
*
|
|
55
|
+
* The chunk size is the difference between this being fast and being
|
|
56
|
+
* pointless: one CopyData per row costs a message header and a socket write
|
|
57
|
+
* each time, which measured 1208ms against 129ms at 64KB for the same
|
|
58
|
+
* 200,000 rows. Sizes from 64KB to 256KB all land near the floor, so the
|
|
59
|
+
* default sits inside that range rather than at either edge.
|
|
60
|
+
*
|
|
61
|
+
* Pulls from `source` rather than being pushed into, so an async iterable -
|
|
62
|
+
* which every Readable is - only produces the next row once the previous
|
|
63
|
+
* chunk has been accepted, making backpressure the caller's own loop
|
|
64
|
+
* pausing rather than an unbounded queue here.
|
|
65
|
+
*/
|
|
66
|
+
export async function writeCopyBinaryRows(source, ctx) {
|
|
67
|
+
const { columns, dataTypeIds, options } = ctx;
|
|
68
|
+
const typeMap = options.typeMap || GlobalTypeMap;
|
|
69
|
+
assertCopyBinaryEncodable(dataTypeIds, columns, typeMap);
|
|
70
|
+
const chunkSize = options.chunkSize || DEFAULT_COPY_CHUNK_SIZE;
|
|
71
|
+
const io = new SmartBuffer();
|
|
72
|
+
io.start();
|
|
73
|
+
io.writeBytes(COPY_BINARY_HEADER);
|
|
74
|
+
const onInvalidValue = options.onInvalidValue || 'throw';
|
|
75
|
+
let rowCount = 0;
|
|
76
|
+
let skippedRows = 0;
|
|
77
|
+
const stats = { nulled: 0 };
|
|
78
|
+
let index = 0;
|
|
79
|
+
for await (const row of source) {
|
|
80
|
+
const written = encodeCopyBinaryRow(io, toValues(row, columns), dataTypeIds, columns, typeMap, options, index, onInvalidValue, stats);
|
|
81
|
+
index++;
|
|
82
|
+
if (!written) {
|
|
83
|
+
skippedRows++;
|
|
84
|
+
continue;
|
|
85
|
+
}
|
|
86
|
+
rowCount++;
|
|
87
|
+
if (io.size >= chunkSize) {
|
|
88
|
+
await ctx.write(io.flush());
|
|
89
|
+
io.start();
|
|
90
|
+
}
|
|
91
|
+
}
|
|
92
|
+
io.writeBytes(COPY_BINARY_TRAILER);
|
|
93
|
+
await ctx.write(io.flush());
|
|
94
|
+
return { rowCount, skippedRows, nulledValues: stats.nulled };
|
|
95
|
+
}
|
|
@@ -1,3 +1,3 @@
|
|
|
1
1
|
import type { DataMappingOptions } from '../interfaces/data-mapping-options.js';
|
|
2
2
|
import type { DecodeBinaryFunction, Nullable } from '../types.js';
|
|
3
|
-
export declare function decodeBinaryArray<T = any>(buf: Buffer, offset: number, decoder: DecodeBinaryFunction, options?: DataMappingOptions
|
|
3
|
+
export declare function decodeBinaryArray<T = any>(buf: Buffer, offset: number, decoder: DecodeBinaryFunction, options?: DataMappingOptions): Nullable<T[]>;
|
|
@@ -1,9 +1,9 @@
|
|
|
1
1
|
import { BufferReader } from '../protocol/buffer-reader.js';
|
|
2
|
-
export function decodeBinaryArray(buf, offset, decoder, options = {}
|
|
2
|
+
export function decodeBinaryArray(buf, offset, decoder, options = {}) {
|
|
3
3
|
if (!buf.length)
|
|
4
4
|
return null;
|
|
5
5
|
const io = new BufferReader(buf);
|
|
6
|
-
io.
|
|
6
|
+
io.position = offset;
|
|
7
7
|
const ndims = io.readInt32BE();
|
|
8
8
|
io.readInt32BE(); // hasNulls
|
|
9
9
|
const elementOID = io.readInt32BE(); // element oid
|
|
@@ -26,12 +26,12 @@ export function decodeBinaryArray(buf, offset, decoder, options = {}, fixedBinar
|
|
|
26
26
|
len = io.readInt32BE();
|
|
27
27
|
if (len === -1)
|
|
28
28
|
target[i] = null;
|
|
29
|
-
else if (fixedBinarySize != null) {
|
|
30
|
-
target[i] = decoder(buf, io.offset, elementOptions);
|
|
31
|
-
io.offset += len;
|
|
32
|
-
}
|
|
33
29
|
else {
|
|
34
|
-
|
|
30
|
+
// Every element carries its own length on the wire, so the decoder
|
|
31
|
+
// can be pointed straight at it - no bounded slice per element,
|
|
32
|
+
// whether or not the element type has a fixed binary size.
|
|
33
|
+
target[i] = decoder(buf, io.position, len, elementOptions);
|
|
34
|
+
io.position += len;
|
|
35
35
|
}
|
|
36
36
|
}
|
|
37
37
|
return target;
|
|
@@ -5,10 +5,10 @@ export function encodeBinaryArray(io, value, itemOid, options, encode, encodeCal
|
|
|
5
5
|
itemOid = itemOid || DataTypeOIDs.varchar;
|
|
6
6
|
const dim = encodeCalculateDimFn(value);
|
|
7
7
|
const ndims = dim.length;
|
|
8
|
-
const zeroOffset = io.
|
|
9
|
-
io.writeInt32BE(ndims) // Number of dimensions
|
|
10
|
-
|
|
11
|
-
|
|
8
|
+
const zeroOffset = io.position;
|
|
9
|
+
io.writeInt32BE(ndims); // Number of dimensions
|
|
10
|
+
io.writeInt32BE(0); // reserved for has-null flag
|
|
11
|
+
io.writeInt32BE(itemOid);
|
|
12
12
|
let d;
|
|
13
13
|
for (d = 0; d < ndims; d++) {
|
|
14
14
|
io.writeInt32BE(dim[d]); // Number of items in dimension
|
|
@@ -33,10 +33,10 @@ export function encodeBinaryArray(io, value, itemOid, options, encode, encodeCal
|
|
|
33
33
|
continue;
|
|
34
34
|
}
|
|
35
35
|
io.writeInt32BE(0); // reserved for data len
|
|
36
|
-
pos = io.
|
|
36
|
+
pos = io.position;
|
|
37
37
|
encode(io, arr[i], options);
|
|
38
38
|
// Update item data size
|
|
39
|
-
io.buffer.writeInt32BE(io.
|
|
39
|
+
io.buffer.writeInt32BE(io.size - pos, pos - 4);
|
|
40
40
|
}
|
|
41
41
|
};
|
|
42
42
|
writeDim(value, 0);
|
|
@@ -0,0 +1,48 @@
|
|
|
1
|
+
import type { DataTypeMap } from '../data-type-map.js';
|
|
2
|
+
import type { DataMappingOptions } from '../interfaces/data-mapping-options.js';
|
|
3
|
+
import type { SmartBuffer } from '../protocol/smart-buffer.js';
|
|
4
|
+
import type { OID } from '../types.js';
|
|
5
|
+
/**
|
|
6
|
+
* `PGCOPY\n\377\r\n\0` followed by two int32 words: a flags field and the
|
|
7
|
+
* length of a header extension area. Bit 16 of the flags marks a stream
|
|
8
|
+
* that carries an OID per row - an obsolete variant PostgreSQL still reads
|
|
9
|
+
* but nothing should write - so both words are zero here.
|
|
10
|
+
*/
|
|
11
|
+
export declare const COPY_BINARY_HEADER: Buffer<ArrayBuffer>;
|
|
12
|
+
/** A field count of -1 where a row would start, ending the stream. */
|
|
13
|
+
export declare const COPY_BINARY_TRAILER: Buffer<ArrayBuffer>;
|
|
14
|
+
/**
|
|
15
|
+
* Appends one row to `io` in PostgreSQL's binary COPY format: an int16
|
|
16
|
+
* field count, then per field an int32 byte length followed by that many
|
|
17
|
+
* bytes, with -1 standing in for NULL.
|
|
18
|
+
*
|
|
19
|
+
* Each value is written by its own type's `encodeBinary`, which is the same
|
|
20
|
+
* function a binary query parameter goes through (see Frontend's Bind) -
|
|
21
|
+
* both are the type's `send` representation, so there is one encoder per
|
|
22
|
+
* type rather than a COPY-specific set.
|
|
23
|
+
*
|
|
24
|
+
* Binary COPY performs no conversion on the server: the bytes have to be
|
|
25
|
+
* exactly what the destination column's type expects. `dataTypeIds` must
|
|
26
|
+
* therefore be the column's own OIDs, which is why the caller obtains them
|
|
27
|
+
* from the server rather than inferring them from the JavaScript values.
|
|
28
|
+
*
|
|
29
|
+
* @param rowIndex Only used to say which row failed, since a value that
|
|
30
|
+
* cannot be encoded otherwise surfaces from the server as an opaque
|
|
31
|
+
* "incorrect binary data format" naming neither row nor column.
|
|
32
|
+
* @returns Whether the row was written - false only under `'skip'`, where
|
|
33
|
+
* the partial row has already been unwound.
|
|
34
|
+
*/
|
|
35
|
+
export declare function encodeCopyBinaryRow(io: SmartBuffer, values: any[], dataTypeIds: OID[], columnNames: string[], typeMap: DataTypeMap, options: DataMappingOptions, rowIndex: number, onInvalidValue?: 'throw' | 'null' | 'skip',
|
|
36
|
+
/** Incremented per value replaced under `'null'`; counted here rather
|
|
37
|
+
* than inferred by the caller, which cannot see inside a written row. */
|
|
38
|
+
stats?: {
|
|
39
|
+
nulled: number;
|
|
40
|
+
}): boolean;
|
|
41
|
+
/**
|
|
42
|
+
* Fails before a single byte goes out when any column's type cannot be
|
|
43
|
+
* written in binary. Worth doing up front: once the server has answered
|
|
44
|
+
* CopyInResponse it is waiting for a complete stream, so discovering an
|
|
45
|
+
* unencodable column at row 900,000 means aborting a copy that was already
|
|
46
|
+
* most of the way done.
|
|
47
|
+
*/
|
|
48
|
+
export declare function assertCopyBinaryEncodable(dataTypeIds: OID[], columnNames: string[], typeMap: DataTypeMap): void;
|
|
@@ -0,0 +1,112 @@
|
|
|
1
|
+
import { encodeBinaryArray } from './encode-binaryarray.js';
|
|
2
|
+
/**
|
|
3
|
+
* `PGCOPY\n\377\r\n\0` followed by two int32 words: a flags field and the
|
|
4
|
+
* length of a header extension area. Bit 16 of the flags marks a stream
|
|
5
|
+
* that carries an OID per row - an obsolete variant PostgreSQL still reads
|
|
6
|
+
* but nothing should write - so both words are zero here.
|
|
7
|
+
*/
|
|
8
|
+
export const COPY_BINARY_HEADER = Buffer.from([
|
|
9
|
+
0x50, 0x47, 0x43, 0x4f, 0x50, 0x59, 0x0a, 0xff, 0x0d, 0x0a, 0x00, 0, 0, 0, 0,
|
|
10
|
+
0, 0, 0, 0,
|
|
11
|
+
]);
|
|
12
|
+
/** A field count of -1 where a row would start, ending the stream. */
|
|
13
|
+
export const COPY_BINARY_TRAILER = Buffer.from([0xff, 0xff]);
|
|
14
|
+
/**
|
|
15
|
+
* Appends one row to `io` in PostgreSQL's binary COPY format: an int16
|
|
16
|
+
* field count, then per field an int32 byte length followed by that many
|
|
17
|
+
* bytes, with -1 standing in for NULL.
|
|
18
|
+
*
|
|
19
|
+
* Each value is written by its own type's `encodeBinary`, which is the same
|
|
20
|
+
* function a binary query parameter goes through (see Frontend's Bind) -
|
|
21
|
+
* both are the type's `send` representation, so there is one encoder per
|
|
22
|
+
* type rather than a COPY-specific set.
|
|
23
|
+
*
|
|
24
|
+
* Binary COPY performs no conversion on the server: the bytes have to be
|
|
25
|
+
* exactly what the destination column's type expects. `dataTypeIds` must
|
|
26
|
+
* therefore be the column's own OIDs, which is why the caller obtains them
|
|
27
|
+
* from the server rather than inferring them from the JavaScript values.
|
|
28
|
+
*
|
|
29
|
+
* @param rowIndex Only used to say which row failed, since a value that
|
|
30
|
+
* cannot be encoded otherwise surfaces from the server as an opaque
|
|
31
|
+
* "incorrect binary data format" naming neither row nor column.
|
|
32
|
+
* @returns Whether the row was written - false only under `'skip'`, where
|
|
33
|
+
* the partial row has already been unwound.
|
|
34
|
+
*/
|
|
35
|
+
export function encodeCopyBinaryRow(io, values, dataTypeIds, columnNames, typeMap, options, rowIndex, onInvalidValue = 'throw',
|
|
36
|
+
/** Incremented per value replaced under `'null'`; counted here rather
|
|
37
|
+
* than inferred by the caller, which cannot see inside a written row. */
|
|
38
|
+
stats) {
|
|
39
|
+
const rowStart = io.size;
|
|
40
|
+
const l = dataTypeIds.length;
|
|
41
|
+
io.writeInt16BE(l);
|
|
42
|
+
let i;
|
|
43
|
+
let v;
|
|
44
|
+
for (i = 0; i < l; i++) {
|
|
45
|
+
v = values[i];
|
|
46
|
+
if (v === null || v === undefined) {
|
|
47
|
+
io.writeInt32BE(-1);
|
|
48
|
+
continue;
|
|
49
|
+
}
|
|
50
|
+
const dt = typeMap.get(dataTypeIds[i]);
|
|
51
|
+
/* c8 ignore next 5 - unreachable: assertCopyBinaryEncodable() rejects
|
|
52
|
+
every column whose type has no encodeBinary before any row is
|
|
53
|
+
encoded, so this is a guard against a type map mutated mid-copy. */
|
|
54
|
+
if (!dt?.encodeBinary) {
|
|
55
|
+
throw new Error(`Column "${columnNames[i]}" has no binary encoder (type OID ${dataTypeIds[i]})`);
|
|
56
|
+
}
|
|
57
|
+
if (dt.encodeAsNull?.(v, options)) {
|
|
58
|
+
io.writeInt32BE(-1);
|
|
59
|
+
continue;
|
|
60
|
+
}
|
|
61
|
+
io.writeInt32BE(0); // placeholder, backfilled below
|
|
62
|
+
const dataOffset = io.position;
|
|
63
|
+
try {
|
|
64
|
+
if (dt.elementsOID) {
|
|
65
|
+
encodeBinaryArray(io, Array.isArray(v) ? v : [v], dt.elementsOID, options, dt.encodeBinary, dt.encodeCalculateDim);
|
|
66
|
+
}
|
|
67
|
+
else {
|
|
68
|
+
dt.encodeBinary(io, v, options);
|
|
69
|
+
}
|
|
70
|
+
}
|
|
71
|
+
catch (e) {
|
|
72
|
+
if (onInvalidValue === 'skip') {
|
|
73
|
+
// Unwind the partial row so the stream never carries half of it.
|
|
74
|
+
io.setSize(rowStart);
|
|
75
|
+
io.position = rowStart;
|
|
76
|
+
return false;
|
|
77
|
+
}
|
|
78
|
+
if (onInvalidValue === 'null') {
|
|
79
|
+
io.setSize(dataOffset);
|
|
80
|
+
io.position = dataOffset - 4;
|
|
81
|
+
io.writeInt32BE(-1);
|
|
82
|
+
if (stats)
|
|
83
|
+
stats.nulled++;
|
|
84
|
+
continue;
|
|
85
|
+
}
|
|
86
|
+
throw new Error(`Cannot encode row ${rowIndex}, column "${columnNames[i]}" (${dt.name}): ${e.message}`, { cause: e });
|
|
87
|
+
}
|
|
88
|
+
io.buffer.writeInt32BE(io.size - dataOffset, dataOffset - 4);
|
|
89
|
+
}
|
|
90
|
+
return true;
|
|
91
|
+
}
|
|
92
|
+
/**
|
|
93
|
+
* Fails before a single byte goes out when any column's type cannot be
|
|
94
|
+
* written in binary. Worth doing up front: once the server has answered
|
|
95
|
+
* CopyInResponse it is waiting for a complete stream, so discovering an
|
|
96
|
+
* unencodable column at row 900,000 means aborting a copy that was already
|
|
97
|
+
* most of the way done.
|
|
98
|
+
*/
|
|
99
|
+
export function assertCopyBinaryEncodable(dataTypeIds, columnNames, typeMap) {
|
|
100
|
+
const missing = [];
|
|
101
|
+
const l = dataTypeIds.length;
|
|
102
|
+
let i;
|
|
103
|
+
for (i = 0; i < l; i++) {
|
|
104
|
+
const dt = typeMap.get(dataTypeIds[i]);
|
|
105
|
+
if (!dt?.encodeBinary)
|
|
106
|
+
missing.push(`"${columnNames[i]}" (OID ${dataTypeIds[i]})`);
|
|
107
|
+
}
|
|
108
|
+
if (missing.length) {
|
|
109
|
+
throw new Error(`Binary COPY cannot encode ${missing.join(', ')} - no binary encoder is ` +
|
|
110
|
+
`registered for that type. Use copyFrom() with a text format instead.`);
|
|
111
|
+
}
|
|
112
|
+
}
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Decodes a decimal number straight out of `buf`, without materialising the
|
|
3
|
+
* intermediate string `parseFloat(buf.toString(...))` needs - measured at
|
|
4
|
+
* over half the cost of decoding a float column.
|
|
5
|
+
*
|
|
6
|
+
* Only takes that shortcut when the result is provably identical to what
|
|
7
|
+
* parseFloat() would return: when the mantissa's digits fit exactly in a
|
|
8
|
+
* double and the decimal exponent is small enough that its power of ten
|
|
9
|
+
* does too, a single multiply (or divide) of two exact values is by
|
|
10
|
+
* definition correctly rounded. Everything else - more digits than that,
|
|
11
|
+
* a large exponent, `NaN`/`Infinity`, a leading `+`, whitespace, anything
|
|
12
|
+
* malformed - falls through to parseFloat() itself, so those values are
|
|
13
|
+
* decoded exactly as before.
|
|
14
|
+
*/
|
|
15
|
+
export declare function fastParseFloatBuffer(buf: Buffer, offset?: number, len?: number): number;
|
|
@@ -0,0 +1,141 @@
|
|
|
1
|
+
// 10^0 .. 10^22 - every one of these is exactly representable as a double,
|
|
2
|
+
// which is what makes the single multiply/divide below correctly rounded.
|
|
3
|
+
// 10^23 is the first that isn't, so the table stops here deliberately.
|
|
4
|
+
const POW10 = [
|
|
5
|
+
1, 1e1, 1e2, 1e3, 1e4, 1e5, 1e6, 1e7, 1e8, 1e9, 1e10, 1e11, 1e12, 1e13, 1e14,
|
|
6
|
+
1e15, 1e16, 1e17, 1e18, 1e19, 1e20, 1e21, 1e22,
|
|
7
|
+
];
|
|
8
|
+
const MAX_EXACT_EXPONENT = 22;
|
|
9
|
+
// 10^15 - 1 still fits well inside 2^53, so a mantissa of at most this many
|
|
10
|
+
// digits is carried exactly.
|
|
11
|
+
const MAX_EXACT_DIGITS = 15;
|
|
12
|
+
/**
|
|
13
|
+
* Decodes a decimal number straight out of `buf`, without materialising the
|
|
14
|
+
* intermediate string `parseFloat(buf.toString(...))` needs - measured at
|
|
15
|
+
* over half the cost of decoding a float column.
|
|
16
|
+
*
|
|
17
|
+
* Only takes that shortcut when the result is provably identical to what
|
|
18
|
+
* parseFloat() would return: when the mantissa's digits fit exactly in a
|
|
19
|
+
* double and the decimal exponent is small enough that its power of ten
|
|
20
|
+
* does too, a single multiply (or divide) of two exact values is by
|
|
21
|
+
* definition correctly rounded. Everything else - more digits than that,
|
|
22
|
+
* a large exponent, `NaN`/`Infinity`, a leading `+`, whitespace, anything
|
|
23
|
+
* malformed - falls through to parseFloat() itself, so those values are
|
|
24
|
+
* decoded exactly as before.
|
|
25
|
+
*/
|
|
26
|
+
export function fastParseFloatBuffer(buf, offset = 0, len = buf.length - offset) {
|
|
27
|
+
const end = offset + len;
|
|
28
|
+
// The longest value the exact path can possibly take is a sign, 15
|
|
29
|
+
// mantissa digits, a decimal point and a two-digit exponent with its own
|
|
30
|
+
// sign. Anything longer is going to parseFloat() regardless, so check
|
|
31
|
+
// that up front rather than after scanning digits that get discarded.
|
|
32
|
+
if (len > 21)
|
|
33
|
+
return parseFloat(buf.toString('latin1', offset, end));
|
|
34
|
+
let i = offset;
|
|
35
|
+
let negative = false;
|
|
36
|
+
if (i < end && buf[i] === 45 /* - */) {
|
|
37
|
+
negative = true;
|
|
38
|
+
i++;
|
|
39
|
+
}
|
|
40
|
+
let mantissa = 0;
|
|
41
|
+
let digits = 0;
|
|
42
|
+
let fractionDigits = 0;
|
|
43
|
+
let seenDot = false;
|
|
44
|
+
let c;
|
|
45
|
+
while (i < end) {
|
|
46
|
+
c = buf[i];
|
|
47
|
+
if (c >= 48 && c <= 57) {
|
|
48
|
+
// Bailing out the moment the mantissa outgrows what a double holds
|
|
49
|
+
// exactly, rather than after the loop: a long numeric's digits would
|
|
50
|
+
// otherwise all be scanned before being thrown away.
|
|
51
|
+
if (++digits > MAX_EXACT_DIGITS)
|
|
52
|
+
return parseFloat(buf.toString('latin1', offset, end));
|
|
53
|
+
mantissa = mantissa * 10 + (c - 48);
|
|
54
|
+
if (seenDot)
|
|
55
|
+
fractionDigits++;
|
|
56
|
+
}
|
|
57
|
+
else if (c === 46 /* . */ && !seenDot) {
|
|
58
|
+
seenDot = true;
|
|
59
|
+
}
|
|
60
|
+
else
|
|
61
|
+
break;
|
|
62
|
+
i++;
|
|
63
|
+
}
|
|
64
|
+
if (!digits)
|
|
65
|
+
return parseFloat(buf.toString('latin1', offset, end));
|
|
66
|
+
let exponent = 0;
|
|
67
|
+
if (i < end && ((c = buf[i]) === 101 /* e */ || c === 69) /* E */) {
|
|
68
|
+
i++;
|
|
69
|
+
let expNegative = false;
|
|
70
|
+
if (i < end) {
|
|
71
|
+
c = buf[i];
|
|
72
|
+
if (c === 45 /* - */) {
|
|
73
|
+
expNegative = true;
|
|
74
|
+
i++;
|
|
75
|
+
}
|
|
76
|
+
else if (c === 43 /* + */)
|
|
77
|
+
i++;
|
|
78
|
+
}
|
|
79
|
+
let expDigits = 0;
|
|
80
|
+
let value = 0;
|
|
81
|
+
while (i < end) {
|
|
82
|
+
c = buf[i];
|
|
83
|
+
if (c < 48 || c > 57)
|
|
84
|
+
break;
|
|
85
|
+
value = value * 10 + (c - 48);
|
|
86
|
+
// Pinned at a value far outside the exact range instead of being
|
|
87
|
+
// allowed to grow without bound. It must never be *truncated* toward
|
|
88
|
+
// a plausible one: an exponent quietly cut from 234 to 23 would be
|
|
89
|
+
// taken as exact and decode to a wrong number rather than falling
|
|
90
|
+
// back. Since the fractional digits that offset it number at most
|
|
91
|
+
// MAX_EXACT_DIGITS, nothing this large can come back into range.
|
|
92
|
+
if (value > 1000)
|
|
93
|
+
value = 1000;
|
|
94
|
+
expDigits++;
|
|
95
|
+
i++;
|
|
96
|
+
}
|
|
97
|
+
if (!expDigits)
|
|
98
|
+
return parseFloat(buf.toString('latin1', offset, end));
|
|
99
|
+
exponent = expNegative ? -value : value;
|
|
100
|
+
}
|
|
101
|
+
// The `| 0` is not redundant: without it this is -0 whenever there is
|
|
102
|
+
// nothing to subtract (any value with no fractional digits, e.g. "42").
|
|
103
|
+
// A -0 index can't be a small integer, so POW10[exponent] below misses
|
|
104
|
+
// V8's fast element access and takes a generic property lookup instead -
|
|
105
|
+
// and the site stays generic afterwards, slowing down every later value
|
|
106
|
+
// too, not just the one that poisoned it.
|
|
107
|
+
exponent = (exponent - fractionDigits) | 0;
|
|
108
|
+
// Trailing characters mean this wasn't a plain decimal number after all.
|
|
109
|
+
if (i !== end)
|
|
110
|
+
return parseFloat(buf.toString('latin1', offset, end));
|
|
111
|
+
let n;
|
|
112
|
+
if (exponent >= 0) {
|
|
113
|
+
if (exponent > MAX_EXACT_EXPONENT) {
|
|
114
|
+
// Too large a power of ten to be exact on its own, but the surplus
|
|
115
|
+
// can often be folded into the mantissa instead: if the mantissa is
|
|
116
|
+
// small enough that scaling it up stays a whole number a double
|
|
117
|
+
// holds exactly, both operands of the multiply below are still
|
|
118
|
+
// exact, so its single rounding is still the correct one. This is
|
|
119
|
+
// what keeps values like 1.2345678e+30 (a mantissa of only eight
|
|
120
|
+
// digits) on the exact path.
|
|
121
|
+
const shift = exponent - MAX_EXACT_EXPONENT;
|
|
122
|
+
if (shift > MAX_EXACT_EXPONENT)
|
|
123
|
+
return parseFloat(buf.toString('latin1', offset, end));
|
|
124
|
+
const scaled = mantissa * POW10[shift];
|
|
125
|
+
if (scaled > Number.MAX_SAFE_INTEGER)
|
|
126
|
+
return parseFloat(buf.toString('latin1', offset, end));
|
|
127
|
+
mantissa = scaled;
|
|
128
|
+
exponent = MAX_EXACT_EXPONENT;
|
|
129
|
+
}
|
|
130
|
+
n = mantissa * POW10[exponent];
|
|
131
|
+
}
|
|
132
|
+
else {
|
|
133
|
+
if (exponent < -MAX_EXACT_EXPONENT)
|
|
134
|
+
return parseFloat(buf.toString('latin1', offset, end));
|
|
135
|
+
n = mantissa / POW10[-exponent];
|
|
136
|
+
}
|
|
137
|
+
// `0 * -1` is -0, which is the value PostgreSQL's own "-0" has to decode
|
|
138
|
+
// to - so the sign is applied by multiplication rather than negation of
|
|
139
|
+
// a separately built positive result.
|
|
140
|
+
return negative ? n * -1 : n;
|
|
141
|
+
}
|
package/util/get-parsers.js
CHANGED
|
@@ -17,16 +17,12 @@ export function getParsers(typeMap, fields) {
|
|
|
17
17
|
const decode = dataTypeReg.decodeBinary;
|
|
18
18
|
if (decode) {
|
|
19
19
|
if (isArray) {
|
|
20
|
-
parsers[i] = (data, offset, len, options) => decodeBinaryArray(data, offset, decode, options
|
|
21
|
-
}
|
|
22
|
-
else if (dataTypeReg.fixedBinarySize != null) {
|
|
23
|
-
const fixedSize = dataTypeReg.fixedBinarySize;
|
|
24
|
-
parsers[i] = (data, offset, len, options) => len === fixedSize
|
|
25
|
-
? decode(data, offset, options)
|
|
26
|
-
: decode(data.subarray(offset, offset + len), 0, options);
|
|
20
|
+
parsers[i] = (data, offset, len, options) => decodeBinaryArray(data, offset, decode, options);
|
|
27
21
|
}
|
|
28
22
|
else {
|
|
29
|
-
|
|
23
|
+
// Straight through: decodeBinary is told where its value starts
|
|
24
|
+
// and how long it is, so there is nothing to slice first.
|
|
25
|
+
parsers[i] = decode;
|
|
30
26
|
}
|
|
31
27
|
}
|
|
32
28
|
}
|
package/util/parse-datetime.js
CHANGED
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
// noinspection RegExpUnnecessaryNonCapturingGroup
|
|
2
2
|
import { fastParseInt } from './fast-parseint.js';
|
|
3
|
+
import { parsePgTimestampString } from './parse-pg-timestamp.js';
|
|
3
4
|
// noinspection RegExpUnnecessaryNonCapturingGroup
|
|
4
5
|
const TIMESTAMP_PATTERN = /^(\d{4})-?(0[1-9]|1[012])?-?([123]0|[012][1-9]|31)?(?:[T ]?([01][0-9]|2[0-3]):?([0-5][0-9]):?([0-5][0-9])?(?:\.(\d+))?(?:(Z)|(?:([+-])([01]?[0-9]|2[0-3]):?([0-5][0-9])?))?)?$/;
|
|
5
6
|
function noMatch(str) {
|
|
@@ -45,6 +46,9 @@ export function parseDateTime(str, utc) {
|
|
|
45
46
|
return Infinity;
|
|
46
47
|
if (str === '-infinity')
|
|
47
48
|
return -Infinity;
|
|
49
|
+
const fast = parsePgTimestampString(str);
|
|
50
|
+
if (fast)
|
|
51
|
+
return fast;
|
|
48
52
|
const d = new Date(str);
|
|
49
53
|
if (!isNaN(d.getTime()))
|
|
50
54
|
return d;
|
|
@@ -73,6 +77,9 @@ export function parseDateTimeTz(str, utc) {
|
|
|
73
77
|
return Infinity;
|
|
74
78
|
if (str === '-infinity')
|
|
75
79
|
return -Infinity;
|
|
80
|
+
const fast = parsePgTimestampString(str);
|
|
81
|
+
if (fast)
|
|
82
|
+
return fast;
|
|
76
83
|
const native = new Date(str);
|
|
77
84
|
if (!isNaN(native.getTime()))
|
|
78
85
|
return native;
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Decodes the exact shape PostgreSQL's own ISO output uses -
|
|
3
|
+
* `YYYY-MM-DD HH:MM:SS[.f+][(+|-)HH[:MM]]` - straight out of the wire
|
|
4
|
+
* bytes, instead of building a string and handing that to the engine's
|
|
5
|
+
* general-purpose date parser. `new Date(str)` costs ~150ns per value and
|
|
6
|
+
* dominates decode time on timestamp-heavy result sets; even the string
|
|
7
|
+
* this replaces costs ~43ns to materialise before any parsing starts.
|
|
8
|
+
*
|
|
9
|
+
* Returns undefined for anything that isn't *exactly* that shape, so every
|
|
10
|
+
* other input (compact forms, `T` separators, a `BC` suffix, years outside
|
|
11
|
+
* 1000-9999, LMT-style ±HH:MM:SS offsets) falls through to the original
|
|
12
|
+
* `new Date()`/regex path and keeps its existing result - including the
|
|
13
|
+
* engine's own quirks, which callers may already depend on:
|
|
14
|
+
* `0001-01-01` parsing as year 2001, and `-04:56:02` offsets being
|
|
15
|
+
* rejected outright.
|
|
16
|
+
*
|
|
17
|
+
* A missing offset means local time, matching what `new Date(str)` does
|
|
18
|
+
* with a space-separated, offset-less timestamp.
|
|
19
|
+
*/
|
|
20
|
+
export declare function parsePgTimestampBuffer(buf: Buffer, offset: number, end: number): Date | undefined;
|
|
21
|
+
/**
|
|
22
|
+
* The same scan for a value that is already a string - the array-element
|
|
23
|
+
* and string-parameter paths, which never see the wire bytes.
|
|
24
|
+
*
|
|
25
|
+
* Copying into a scratch buffer first still beats scanning the string
|
|
26
|
+
* directly through a second, parallel implementation: the copy costs about
|
|
27
|
+
* half of what materialising the string cost in the first place, and there
|
|
28
|
+
* is only one scanner to keep correct. Safe to share one buffer because
|
|
29
|
+
* the copy and the scan that reads it are a single synchronous step.
|
|
30
|
+
*/
|
|
31
|
+
export declare function parsePgTimestampString(str: string): Date | undefined;
|