prostgles-server 4.1.130 → 4.1.132
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/DboBuilder/QueryStreamer.d.ts +3 -1
- package/dist/DboBuilder/QueryStreamer.d.ts.map +1 -1
- package/dist/DboBuilder/QueryStreamer.js +45 -25
- package/dist/DboBuilder/QueryStreamer.js.map +1 -1
- package/lib/DboBuilder/QueryStreamer.ts +49 -27
- package/package.json +1 -1
- package/tests/client_only_queries.ts +18 -0
- package/tests/server/package-lock.json +1 -1
|
@@ -19,8 +19,10 @@ export declare class QueryStreamer {
|
|
|
19
19
|
db: DB;
|
|
20
20
|
dboBuilder: DboBuilder;
|
|
21
21
|
socketQueries: Record<string, Record<string, StreamedQuery>>;
|
|
22
|
+
adminClient: pg.Client;
|
|
22
23
|
constructor(dboBuilder: DboBuilder);
|
|
23
|
-
getConnection: (onError: ((err: any) => void) | undefined) => pg.Client;
|
|
24
|
+
getConnection: (onError: ((err: any) => void) | undefined, extraOptions?: pg.ClientConfig) => pg.Client;
|
|
25
|
+
onDisconnect: (socketId: string) => void;
|
|
24
26
|
create: (query: ClientStreamedRequest) => Promise<SocketSQLStreamServer>;
|
|
25
27
|
}
|
|
26
28
|
export {};
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"QueryStreamer.d.ts","sourceRoot":"","sources":["../../lib/DboBuilder/QueryStreamer.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,EAAE,EAAE,MAAM,kBAAkB,CAAC;AACtC,OAAO,KAAK,EAAE,MAAM,IAAI,CAAC;AACzB,OAAO,EAAY,UAAU,EAAyB,qBAAqB,
|
|
1
|
+
{"version":3,"file":"QueryStreamer.d.ts","sourceRoot":"","sources":["../../lib/DboBuilder/QueryStreamer.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,EAAE,EAAE,MAAM,kBAAkB,CAAC;AACtC,OAAO,KAAK,EAAE,MAAM,IAAI,CAAC;AACzB,OAAO,EAAY,UAAU,EAAyB,qBAAqB,EAAY,MAAM,iBAAiB,CAAC;AAC/G,OAAO,EAAE,YAAY,EAAE,MAAM,mBAAmB,CAAC;AAGjD,OAAO,EAAE,UAAU,EAAE,MAAM,cAAc,CAAC;AAC1C,OAAO,eAAe,MAAM,iBAAiB,CAAC;AAM9C,KAAK,qBAAqB,GAAG;IAC3B,MAAM,EAAE,YAAY,CAAC;IACrB,KAAK,EAAE,MAAM,CAAC;IACd,OAAO,EAAE,UAAU,GAAG,SAAS,CAAC;IAChC,iBAAiB,CAAC,EAAE,OAAO,CAAC;CAE7B,CAAA;AACD,KAAK,aAAa,GAAG,qBAAqB,GAAG;IAC3C,MAAM,EAAE,eAAe,GAAG,SAAS,CAAC;IACpC,MAAM,EAAE,EAAE,CAAC,MAAM,GAAG,SAAS,CAAC;IAC9B,OAAO,EAAE,CAAC,CAAC,KAAK,EAAE,GAAG,KAAK,IAAI,CAAC,CAAC;CACjC,CAAA;AAWD,qBAAa,aAAa;IACxB,EAAE,EAAE,EAAE,CAAC;IACP,UAAU,EAAE,UAAU,CAAC;IACvB,aAAa,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,aAAa,CAAC,CAAC,CAAM;IAClE,WAAW,EAAE,EAAE,CAAC,MAAM,CAAC;gBACX,UAAU,EAAE,UAAU;IAclC,aAAa,mBAAoB,GAAG,KAAK,IAAI,8BAA8B,GAAG,YAAY,eAOzF;IAED,YAAY,aAAc,MAAM,UAO/B;IAED,MAAM,UAAiB,qBAAqB,KAAG,QAAQ,qBAAqB,CAAC,CAoJ5E;CACF"}
|
|
@@ -18,18 +18,37 @@ class QueryStreamer {
|
|
|
18
18
|
db;
|
|
19
19
|
dboBuilder;
|
|
20
20
|
socketQueries = {};
|
|
21
|
+
adminClient;
|
|
21
22
|
constructor(dboBuilder) {
|
|
22
23
|
this.dboBuilder = dboBuilder;
|
|
23
24
|
this.db = dboBuilder.db;
|
|
25
|
+
const setAdminClient = () => {
|
|
26
|
+
this.adminClient = this.getConnection(undefined, { keepAlive: true });
|
|
27
|
+
this.adminClient.connect();
|
|
28
|
+
};
|
|
29
|
+
this.adminClient = this.getConnection((error) => {
|
|
30
|
+
console.log("Admin client error. Reconnecting...", error);
|
|
31
|
+
setAdminClient();
|
|
32
|
+
}, { keepAlive: true });
|
|
33
|
+
this.adminClient.connect();
|
|
24
34
|
}
|
|
25
|
-
getConnection = (onError) => {
|
|
35
|
+
getConnection = (onError, extraOptions) => {
|
|
26
36
|
const connectionInfo = typeof this.db.$cn === "string" ? { connectionString: this.db.$cn } : this.db.$cn;
|
|
27
|
-
const client = new pg.Client(connectionInfo);
|
|
37
|
+
const client = new pg.Client({ ...connectionInfo, ...extraOptions });
|
|
28
38
|
client.on("error", (err) => {
|
|
29
39
|
onError?.(err);
|
|
30
40
|
});
|
|
31
41
|
return client;
|
|
32
42
|
};
|
|
43
|
+
onDisconnect = (socketId) => {
|
|
44
|
+
const socketQueries = this.socketQueries[socketId];
|
|
45
|
+
if (!socketQueries)
|
|
46
|
+
return;
|
|
47
|
+
Object.values(socketQueries).forEach(({ client }) => {
|
|
48
|
+
client?.end();
|
|
49
|
+
});
|
|
50
|
+
delete this.socketQueries[socketId];
|
|
51
|
+
};
|
|
33
52
|
create = async (query) => {
|
|
34
53
|
const { socket, persistConnection } = query;
|
|
35
54
|
const socketId = socket.id;
|
|
@@ -67,22 +86,20 @@ class QueryStreamer {
|
|
|
67
86
|
let finished = false;
|
|
68
87
|
const batchSize = 10000;
|
|
69
88
|
let stream;
|
|
70
|
-
|
|
71
|
-
const emit = (type, stream) => {
|
|
72
|
-
const result = stream?._result;
|
|
89
|
+
const emit = (type, streamResult) => {
|
|
73
90
|
const ended = type === "ended";
|
|
74
91
|
if (finished)
|
|
75
92
|
return;
|
|
76
93
|
finished = finished || ended;
|
|
77
|
-
if (!
|
|
94
|
+
if (!streamResult?.fields)
|
|
78
95
|
throw "No fields";
|
|
79
|
-
const fields = runSQL_1.getDetailedFieldInfo.bind(this.dboBuilder)(
|
|
80
|
-
const packet = { type: "data", rows: batchRows, fields, ended, processId: processID };
|
|
96
|
+
const fields = runSQL_1.getDetailedFieldInfo.bind(this.dboBuilder)(streamResult.fields);
|
|
97
|
+
const packet = { type: "data", rows: batchRows, fields, info: streamResult, ended, processId: processID };
|
|
81
98
|
socket.emit(channel, packet);
|
|
82
99
|
if (ended) {
|
|
83
|
-
if (!
|
|
100
|
+
if (!streamResult)
|
|
84
101
|
throw "No result info";
|
|
85
|
-
runSQL_1.watchSchemaFallback.bind(this.dboBuilder)({ queryWithoutRLS: query.query, command:
|
|
102
|
+
runSQL_1.watchSchemaFallback.bind(this.dboBuilder)({ queryWithoutRLS: query.query, command: streamResult.command });
|
|
86
103
|
}
|
|
87
104
|
};
|
|
88
105
|
const client = this.getConnection(err => {
|
|
@@ -91,29 +108,39 @@ class QueryStreamer {
|
|
|
91
108
|
});
|
|
92
109
|
try {
|
|
93
110
|
await client.connect();
|
|
94
|
-
poolClient = client;
|
|
95
111
|
processID = client.processID;
|
|
96
|
-
const queryStream = new QueryStream(query.query,
|
|
112
|
+
const queryStream = new QueryStream(query.query, undefined, { batchSize: 1e6, highWaterMark: 1e6, rowMode: "array" });
|
|
97
113
|
stream = client.query(queryStream);
|
|
98
|
-
this.socketQueries[socketId][id].client =
|
|
114
|
+
this.socketQueries[socketId][id].client = client;
|
|
99
115
|
this.socketQueries[socketId][id].stream = stream;
|
|
116
|
+
const getStreamResult = () => (0, prostgles_types_1.omitKeys)(stream._result, ["rows"]);
|
|
100
117
|
stream.on('data', async (data) => {
|
|
101
118
|
batchRows.push(data);
|
|
102
119
|
if (options?.streamLimit) {
|
|
103
120
|
if (batchRows.length >= options.streamLimit) {
|
|
104
|
-
emit("ended",
|
|
121
|
+
emit("ended", getStreamResult());
|
|
105
122
|
}
|
|
106
123
|
}
|
|
107
124
|
if (batchRows.length >= batchSize) {
|
|
108
|
-
emit("rows",
|
|
125
|
+
emit("rows", getStreamResult());
|
|
109
126
|
batchRows = [];
|
|
110
127
|
}
|
|
111
128
|
});
|
|
112
129
|
stream.on('error', error => {
|
|
113
|
-
|
|
130
|
+
if (error.message === "cannot insert multiple commands into a prepared statement") {
|
|
131
|
+
this.dboBuilder.dbo.sql(query.query, {}, { returnType: "arrayMode", hasParams: false }).then(res => {
|
|
132
|
+
batchRows.push(res.rows);
|
|
133
|
+
emit("ended", res);
|
|
134
|
+
}).catch(newError => {
|
|
135
|
+
socketQuery.onError(newError);
|
|
136
|
+
});
|
|
137
|
+
}
|
|
138
|
+
else {
|
|
139
|
+
socketQuery.onError(error);
|
|
140
|
+
}
|
|
114
141
|
});
|
|
115
142
|
stream.on('end', () => {
|
|
116
|
-
emit("ended",
|
|
143
|
+
emit("ended", getStreamResult());
|
|
117
144
|
// release the client when the stream is finished AND connection is not persisted
|
|
118
145
|
if (!options?.persistStreamConnection) {
|
|
119
146
|
delete this.socketQueries[socketId]?.[id];
|
|
@@ -130,13 +157,9 @@ class QueryStreamer {
|
|
|
130
157
|
const { stream, client: queryClient } = this.socketQueries[socketId]?.[id] ?? {};
|
|
131
158
|
if (!stream || !queryClient)
|
|
132
159
|
return;
|
|
133
|
-
const client = this.getConnection(undefined);
|
|
134
160
|
try {
|
|
135
|
-
await client.connect();
|
|
136
|
-
if (!client)
|
|
137
|
-
return cb(null, "No client");
|
|
138
161
|
const stopFunction = opts?.terminate ? "pg_terminate_backend" : "pg_cancel_backend";
|
|
139
|
-
const rows = await
|
|
162
|
+
const rows = await this.adminClient.query(`SELECT ${stopFunction}(pid), pid, state, query FROM pg_stat_activity WHERE pid = $1 AND query = $2`, [processID, query.query]);
|
|
140
163
|
socket.removeAllListeners(unsubChannel);
|
|
141
164
|
socket.removeAllListeners(channel);
|
|
142
165
|
cb({ processID, info: rows.rows[0] });
|
|
@@ -144,9 +167,6 @@ class QueryStreamer {
|
|
|
144
167
|
catch (error) {
|
|
145
168
|
cb(null, error);
|
|
146
169
|
}
|
|
147
|
-
finally {
|
|
148
|
-
await client.end();
|
|
149
|
-
}
|
|
150
170
|
};
|
|
151
171
|
socket.removeAllListeners(unsubChannel);
|
|
152
172
|
socket.once(unsubChannel, stop);
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"QueryStreamer.js","sourceRoot":"","sources":["../../lib/DboBuilder/QueryStreamer.ts"],"names":[],"mappings":";;;AACA,yBAAyB;AACzB,
|
|
1
|
+
{"version":3,"file":"QueryStreamer.js","sourceRoot":"","sources":["../../lib/DboBuilder/QueryStreamer.ts"],"names":[],"mappings":";;;AACA,yBAAyB;AACzB,qDAA+G;AAE/G,uDAAwE;AACxE,qCAAqE;AAKrE,MAAM,WAAW,GAA2B,OAAO,CAAC,iBAAiB,CAAC,CAAC;AAgBvE,MAAM,cAAc,GAA2B,EAAE,CAAC;AAClD,MAAM,mBAAmB,GAAG,CAAC,QAAgB,EAAE,EAAE;IAC/C,MAAM,OAAO,GAAG,QAAQ,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;IACrC,MAAM,MAAM,GAAG,cAAc,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;IAC5C,MAAM,KAAK,GAAG,MAAM,GAAG,CAAC,CAAC;IACzB,cAAc,CAAC,OAAO,CAAC,GAAG,KAAK,CAAC;IAChC,OAAO,KAAK,CAAC;AACf,CAAC,CAAC;AAEF,MAAa,aAAa;IACxB,EAAE,CAAK;IACP,UAAU,CAAa;IACvB,aAAa,GAAkD,EAAE,CAAC;IAClE,WAAW,CAAY;IACvB,YAAY,UAAsB;QAChC,IAAI,CAAC,UAAU,GAAG,UAAU,CAAC;QAC7B,IAAI,CAAC,EAAE,GAAG,UAAU,CAAC,EAAE,CAAC;QACxB,MAAM,cAAc,GAAG,GAAG,EAAE;YAC1B,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC,aAAa,CAAC,SAAS,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;YACtE,IAAI,CAAC,WAAW,CAAC,OAAO,EAAE,CAAC;QAC7B,CAAC,CAAA;QACD,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC,aAAa,CAAC,CAAC,KAAK,EAAE,EAAE;YAC9C,OAAO,CAAC,GAAG,CAAC,qCAAqC,EAAE,KAAK,CAAC,CAAC;YAC1D,cAAc,EAAE,CAAC;QACnB,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QACxB,IAAI,CAAC,WAAW,CAAC,OAAO,EAAE,CAAC;IAC7B,CAAC;IAED,aAAa,GAAG,CAAC,OAAyC,EAAE,YAA8B,EAAE,EAAE;QAC5F,MAAM,cAAc,GAAG,OAAO,IAAI,CAAC,EAAE,CAAC,GAAG,KAAK,QAAQ,CAAA,CAAC,CAAC,EAAE,gBAAgB,EAAE,IAAI,CAAC,EAAE,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,GAAU,CAAC;QAC/G,MAAM,MAAM,GAAG,IAAI,EAAE,CAAC,MAAM,CAAC,EAAE,GAAG,cAAc,EAAE,GAAG,YAAY,EAAE,CAAC,CAAC;QACrE,MAAM,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,GAAG,EAAE,EAAE;YACzB,OAAO,EAAE,CAAC,GAAG,CAAC,CAAC;QACjB,CAAC,CAAC,CAAC;QACH,OAAO,MAAM,CAAC;IAChB,CAAC,CAAA;IAED,YAAY,GAAG,CAAC,QAAgB,EAAE,EAAE;QAClC,MAAM,aAAa,GAAG,IAAI,CAAC,aAAa,CAAC,QAAQ,CAAC,CAAC;QACnD,IAAG,CAAC,aAAa;YAAE,OAAO;QAC1B,MAAM,CAAC,MAAM,CAAC,aAAa,CAAC,CAAC,OAAO,CAAC,CAAC,EAAE,MAAM,EAAE,EAAE,EAAE;YAClD,MAAM,EAAE,GAAG,EAAE,CAAC;QAChB,CAAC,CAAC,CAAC;QACH,OAAO,IAAI,CAAC,aAAa,CAAC,QAAQ,CAAC,CAAC;IACtC,CAAC,CAAA;IAED,MAAM,GAAG,KAAK,EAAE,KAA4B,EAAkC,EAAE;QAC9E,MAAM,EAAE,MAAM,EAAE,iBAAiB,EAAE,GAAG,KAAK,CAAC;QAC5C,MAAM,QAAQ,GAAG,MAAM,CAAC,EAAE,CAAC;QAC3B,MAAM,EAAE,GAAG,mBAAmB,CAAC,QAAQ,CAAC,CAAC;QACzC,MAAM,OAAO,GAAG,GAAG,0BAAQ,CAAC,UAAU,KAAK,QAAQ,IAAI,EAAE,EAAE,CAAC;QAC5D,MAAM,YAAY,GAAG,GAAG,OAAO,cAAc,CAAC;QAC9C,IAAG,IAAI,CAAC,aAAa,CAAC,EAAE,CAAC,IAAI,CAAC,iBAAiB,EAAC;YAC9C,MAAM,4BAA4B,EAAE,QAAQ,CAAC;SAC9C;QAED,IAAI,CAAC,aAAa,CAAC,QAAQ,CAAC,KAAK,EAAE,CAAC;QACpC,IAAI,OAAO,GAAG,KAAK,CAAC;QACpB,MAAM,WAAW,GAAG;YAClB,GAAG,KAAK;YACR,MAAM,EAAE,SAAS;YACjB,MAAM,EAAE,SAAS;YACjB,OAAO,EAAE,CAAC,QAAa,EAAE,EAAE;gBACzB,IAAG,OAAO;oBAAE,OAAO;gBACnB,OAAO,GAAG,IAAI,CAAC;gBAEf,MAAM,iBAAiB,GAAG,IAAA,qDAAmC,EAAC,QAAQ,EAAE,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,CAAC;gBACzF,gFAAgF;gBAChF,MAAM,KAAK,GAAG,EAAE,GAAG,iBAAiB,EAAE,KAAK,EAAE,KAAK,CAAC,KAAK,EAAE,CAAC;gBAC3D,MAAM,CAAC,IAAI,CAAC,OAAO,EAAE,EAAE,IAAI,EAAE,OAAO,EAAE,KAAK,EAAkC,CAAC,CAAC;YACjF,CAAC;SACF,CAAC;QACF,IAAI,CAAC,aAAa,CAAC,QAAQ,CAAE,CAAC,EAAE,CAAC,KAAK,WAAW,CAAC;QAClD,MAAM,EAAE,OAAO,EAAE,GAAG,KAAK,CAAA;QACzB,IAAI,SAAS,GAAG,CAAC,CAAC,CAAC;QAEnB,MAAM,WAAW,GAAG,KAAK,IAAI,EAAE;YAC7B,MAAM,WAAW,GAAG,IAAI,CAAC,aAAa,CAAC,QAAQ,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC;YACvD,IAAG,CAAC,WAAW,EAAC;gBACd,MAAM,wBAAwB,CAAC;aAChC;YACD,IAAI,SAAS,GAAU,EAAE,CAAC;YAC1B,IAAI,QAAQ,GAAG,KAAK,CAAC;YACrB,MAAM,SAAS,GAAG,KAAK,CAAC;YACxB,IAAI,MAAuB,CAAC;YAE5B,MAAM,IAAI,GAAG,CAAC,IAAsB,EAAE,YAA8B,EAAE,EAAE;gBACtE,MAAM,KAAK,GAAG,IAAI,KAAK,OAAO,CAAC;gBAC/B,IAAG,QAAQ;oBAAE,OAAO;gBACpB,QAAQ,GAAG,QAAQ,IAAI,KAAK,CAAC;gBAC7B,IAAG,CAAC,YAAY,EAAE,MAAM;oBAAE,MAAM,WAAW,CAAC;gBAC5C,MAAM,MAAM,GAAG,6BAAoB,CAAC,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC,YAAY,CAAC,MAAa,CAAC,CAAC;gBACtF,MAAM,MAAM,GAA0B,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,SAAS,EAAE,MAAM,EAAE,IAAI,EAAE,YAAY,EAAE,KAAK,EAAE,SAAS,EAAE,SAAS,EAAE,CAAC;gBACjI,MAAM,CAAC,IAAI,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;gBAC7B,IAAG,KAAK,EAAC;oBACP,IAAG,CAAC,YAAY;wBAAE,MAAM,gBAAgB,CAAC;oBACzC,4BAAmB,CAAC,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC,EAAE,eAAe,EAAE,KAAK,CAAC,KAAK,EAAE,OAAO,EAAE,YAAY,CAAC,OAAO,EAAE,CAAC,CAAC;iBAC5G;YACH,CAAC,CAAA;YACD,MAAM,MAAM,GAAG,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,EAAE;gBACtC,WAAW,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;gBACzB,MAAM,CAAC,GAAG,EAAE,CAAC;YACf,CAAC,CAAC,CAAC;YACH,IAAI;gBACF,MAAM,MAAM,CAAC,OAAO,EAAE,CAAC;gBACvB,SAAS,GAAI,MAAc,CAAC,SAAS,CAAA;gBACrC,MAAM,WAAW,GAAG,IAAI,WAAW,CAAC,KAAK,CAAC,KAAK,EAAE,SAAS,EAAE,EAAE,SAAS,EAAE,GAAG,EAAE,aAAa,EAAE,GAAG,EAAE,OAAO,EAAE,OAAO,EAAE,CAAC,CAAC;gBACtH,MAAM,GAAG,MAAM,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC;gBACnC,IAAI,CAAC,aAAa,CAAC,QAAQ,CAAE,CAAC,EAAE,CAAE,CAAC,MAAM,GAAG,MAAM,CAAC;gBACnD,IAAI,CAAC,aAAa,CAAC,QAAQ,CAAE,CAAC,EAAE,CAAE,CAAC,MAAM,GAAG,MAAM,CAAC;gBACnD,MAAM,eAAe,GAAG,GAAG,EAAE,CAAC,IAAA,0BAAQ,EAAC,MAAM,CAAC,OAAO,EAAE,CAAC,MAAM,CAAC,CAAS,CAAC;gBACzE,MAAM,CAAC,EAAE,CAAC,MAAM,EAAE,KAAK,EAAE,IAAI,EAAE,EAAE;oBAC/B,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;oBACrB,IAAG,OAAO,EAAE,WAAW,EAAE;wBACvB,IAAG,SAAS,CAAC,MAAM,IAAI,OAAO,CAAC,WAAW,EAAC;4BACzC,IAAI,CAAC,OAAO,EAAE,eAAe,EAAE,CAAC,CAAC;yBAClC;qBACF;oBACD,IAAI,SAAS,CAAC,MAAM,IAAI,SAAS,EAAE;wBACjC,IAAI,CAAC,MAAM,EAAE,eAAe,EAAE,CAAC,CAAC;wBAChC,SAAS,GAAG,EAAE,CAAC;qBAChB;gBACH,CAAC,CAAC,CAAC;gBACH,MAAM,CAAC,EAAE,CAAC,OAAO,EAAE,KAAK,CAAC,EAAE;oBACzB,IAAG,KAAK,CAAC,OAAO,KAAK,2DAA2D,EAAE;wBAChF,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,GAAI,CAAC,KAAK,CAAC,KAAK,EAAE,EAAE,EAAE,EAAE,UAAU,EAAE,WAAW,EAAE,SAAS,EAAE,KAAK,EAAE,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE;4BAClG,SAAS,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;4BACzB,IAAI,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC;wBACrB,CAAC,CAAC,CAAC,KAAK,CAAC,QAAQ,CAAC,EAAE;4BAClB,WAAW,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC;wBAChC,CAAC,CAAC,CAAC;qBACJ;yBAAM;wBACL,WAAW,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;qBAC5B;gBACH,CAAC,CAAC,CAAC;gBACH,MAAM,CAAC,EAAE,CAAC,KAAK,EAAE,GAAG,EAAE;oBACpB,IAAI,CAAC,OAAO,EAAE,eAAe,EAAE,CAAC,CAAC;oBACjC,iFAAiF;oBACjF,IAAG,CAAC,OAAO,EAAE,uBAAuB,EAAC;wBACnC,OAAO,IAAI,CAAC,aAAa,CAAC,QAAQ,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC;wBAC1C,MAAM,CAAC,GAAG,EAAE,CAAC;qBACd;gBACH,CAAC,CAAC,CAAC;aACJ;YAAC,OAAM,GAAG,EAAC;gBACV,WAAW,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;gBACzB,MAAM,MAAM,CAAC,GAAG,EAAE,CAAC;aACpB;QACH,CAAC,CAAA;QAED,MAAM,IAAI,GAAG,KAAK,EAAE,IAA0C,EAAE,EAAiB,EAAE,EAAE;YACnF,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,WAAW,EAAE,GAAG,IAAI,CAAC,aAAa,CAAC,QAAQ,CAAC,EAAE,CAAC,EAAE,CAAC,IAAI,EAAE,CAAC;YACjF,IAAG,CAAC,MAAM,IAAI,CAAC,WAAW;gBAAE,OAAO;YACnC,IAAI;gBACF,MAAM,YAAY,GAAG,IAAI,EAAE,SAAS,CAAA,CAAC,CAAC,sBAAsB,CAAC,CAAC,CAAC,mBAAmB,CAAC;gBACnF,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,WAAW,CAAC,KAAK,CAAC,UAAU,YAAY,8EAA8E,EAAE,CAAC,SAAS,EAAE,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC;gBAC1K,MAAM,CAAC,kBAAkB,CAAC,YAAY,CAAC,CAAC;gBACxC,MAAM,CAAC,kBAAkB,CAAC,OAAO,CAAC,CAAC;gBACnC,EAAE,CAAC,EAAE,SAAS,EAAE,IAAI,EAAE,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;aACvC;YAAC,OAAO,KAAK,EAAC;gBACb,EAAE,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC;aACjB;QACH,CAAC,CAAA;QAED,MAAM,CAAC,kBAAkB,CAAC,YAAY,CAAC,CAAC;QACxC,MAAM,CAAC,IAAI,CAAC,YAAY,EAAE,IAAI,CAAC,CAAC;QAEhC,IAAI,OAAO,GAAG,KAAK,CAAC;QACpB,MAAM,CAAC,kBAAkB,CAAC,OAAO,CAAC,CAAC;QACnC,MAAM,CAAC,EAAE,CAAC,OAAO,EAAE,KAAK,EAAE,KAAiD,EAAE,EAAiB,EAAE,EAAE;YAChG,IAAG,OAAO,EAAC;gBACT,OAAO;gBACP,OAAO,EAAE,CAAC,IAAI,EAAE,iBAAiB,CAAC,CAAC;aACpC;YACD,OAAO,GAAG,IAAI,CAAC;YACf,IAAI;gBACF,MAAM,WAAW,EAAE,CAAC;gBACpB,EAAE,EAAE,CAAC;aACN;YAAC,OAAM,GAAG,EAAC;gBACV,EAAE,CAAC,IAAI,EAAE,GAAG,IAAI,sBAAsB,CAAC,CAAC;aACzC;QACH,CAAC,CAAC,CAAC;QAEH,qEAAqE;QACrE,UAAU,CAAC,GAAG,EAAE;YACd,IAAG,OAAO;gBAAE,OAAO;YACnB,IAAI,CAAC,EAAE,EAAE,GAAG,EAAE;gBACZ,SAAS;YACX,CAAC,CAAC,CAAC;QACL,CAAC,EAAE,GAAG,CAAC,CAAC;QAER,OAAO;YACL,OAAO;YACP,YAAY;SACb,CAAA;IACH,CAAC,CAAA;CACF;AA1LD,sCA0LC"}
|
|
@@ -1,12 +1,13 @@
|
|
|
1
1
|
import { DB } from "../initProstgles";
|
|
2
2
|
import * as pg from "pg";
|
|
3
|
-
import { CHANNELS, SQLOptions, SocketSQLStreamPacket, SocketSQLStreamServer } from "prostgles-types";
|
|
3
|
+
import { CHANNELS, SQLOptions, SocketSQLStreamPacket, SocketSQLStreamServer, omitKeys } from "prostgles-types";
|
|
4
4
|
import { PRGLIOSocket } from "./DboBuilderTypes";
|
|
5
5
|
import { getSerializedClientErrorFromPGError } from "./dboBuilderUtils";
|
|
6
6
|
import { getDetailedFieldInfo, watchSchemaFallback } from "./runSQL";
|
|
7
7
|
import { DboBuilder } from "./DboBuilder";
|
|
8
8
|
import QueryStreamType from 'pg-query-stream';
|
|
9
9
|
import { BasicCallback } from "../PubSubManager/PubSubManager";
|
|
10
|
+
import e from "express";
|
|
10
11
|
const QueryStream: typeof QueryStreamType = require('pg-query-stream');
|
|
11
12
|
|
|
12
13
|
|
|
@@ -36,20 +37,39 @@ export class QueryStreamer {
|
|
|
36
37
|
db: DB;
|
|
37
38
|
dboBuilder: DboBuilder;
|
|
38
39
|
socketQueries: Record<string, Record<string, StreamedQuery>> = {};
|
|
40
|
+
adminClient: pg.Client;
|
|
39
41
|
constructor(dboBuilder: DboBuilder) {
|
|
40
42
|
this.dboBuilder = dboBuilder;
|
|
41
43
|
this.db = dboBuilder.db;
|
|
44
|
+
const setAdminClient = () => {
|
|
45
|
+
this.adminClient = this.getConnection(undefined, { keepAlive: true });
|
|
46
|
+
this.adminClient.connect();
|
|
47
|
+
}
|
|
48
|
+
this.adminClient = this.getConnection((error) => {
|
|
49
|
+
console.log("Admin client error. Reconnecting...", error);
|
|
50
|
+
setAdminClient();
|
|
51
|
+
}, { keepAlive: true });
|
|
52
|
+
this.adminClient.connect();
|
|
42
53
|
}
|
|
43
54
|
|
|
44
|
-
getConnection = (onError: ((err: any) => void) | undefined) => {
|
|
55
|
+
getConnection = (onError: ((err: any) => void) | undefined, extraOptions?: pg.ClientConfig) => {
|
|
45
56
|
const connectionInfo = typeof this.db.$cn === "string"? { connectionString: this.db.$cn } : this.db.$cn as any;
|
|
46
|
-
const client = new pg.Client(connectionInfo);
|
|
57
|
+
const client = new pg.Client({ ...connectionInfo, ...extraOptions });
|
|
47
58
|
client.on("error", (err) => {
|
|
48
59
|
onError?.(err);
|
|
49
60
|
});
|
|
50
61
|
return client;
|
|
51
62
|
}
|
|
52
63
|
|
|
64
|
+
onDisconnect = (socketId: string) => {
|
|
65
|
+
const socketQueries = this.socketQueries[socketId];
|
|
66
|
+
if(!socketQueries) return;
|
|
67
|
+
Object.values(socketQueries).forEach(({ client }) => {
|
|
68
|
+
client?.end();
|
|
69
|
+
});
|
|
70
|
+
delete this.socketQueries[socketId];
|
|
71
|
+
}
|
|
72
|
+
|
|
53
73
|
create = async (query: ClientStreamedRequest): Promise<SocketSQLStreamServer> => {
|
|
54
74
|
const { socket, persistConnection } = query;
|
|
55
75
|
const socketId = socket.id;
|
|
@@ -88,20 +108,19 @@ export class QueryStreamer {
|
|
|
88
108
|
let batchRows: any[] = [];
|
|
89
109
|
let finished = false;
|
|
90
110
|
const batchSize = 10000;
|
|
91
|
-
let stream: QueryStreamType;
|
|
92
|
-
|
|
93
|
-
const emit = (type: "rows" | "ended",
|
|
94
|
-
const result = stream?._result as { command: string; fields: any[] } | undefined;
|
|
111
|
+
let stream: QueryStreamType;
|
|
112
|
+
type Info = { command: string; fields: any[]; rowCount: number; duration: number; }
|
|
113
|
+
const emit = (type: "rows" | "ended", streamResult: Info | undefined) => {
|
|
95
114
|
const ended = type === "ended";
|
|
96
115
|
if(finished) return;
|
|
97
116
|
finished = finished || ended;
|
|
98
|
-
if(!
|
|
99
|
-
const fields = getDetailedFieldInfo.bind(this.dboBuilder)(
|
|
100
|
-
const packet: SocketSQLStreamPacket = { type: "data", rows: batchRows, fields, ended, processId: processID };
|
|
117
|
+
if(!streamResult?.fields) throw "No fields";
|
|
118
|
+
const fields = getDetailedFieldInfo.bind(this.dboBuilder)(streamResult.fields as any);
|
|
119
|
+
const packet: SocketSQLStreamPacket = { type: "data", rows: batchRows, fields, info: streamResult, ended, processId: processID };
|
|
101
120
|
socket.emit(channel, packet);
|
|
102
121
|
if(ended){
|
|
103
|
-
if(!
|
|
104
|
-
watchSchemaFallback.bind(this.dboBuilder)({ queryWithoutRLS: query.query, command:
|
|
122
|
+
if(!streamResult) throw "No result info";
|
|
123
|
+
watchSchemaFallback.bind(this.dboBuilder)({ queryWithoutRLS: query.query, command: streamResult.command });
|
|
105
124
|
}
|
|
106
125
|
}
|
|
107
126
|
const client = this.getConnection(err => {
|
|
@@ -109,31 +128,39 @@ export class QueryStreamer {
|
|
|
109
128
|
client.end();
|
|
110
129
|
});
|
|
111
130
|
try {
|
|
112
|
-
await client.connect();
|
|
113
|
-
poolClient = client;
|
|
131
|
+
await client.connect();
|
|
114
132
|
processID = (client as any).processID
|
|
115
|
-
const queryStream = new QueryStream(query.query,
|
|
133
|
+
const queryStream = new QueryStream(query.query, undefined, { batchSize: 1e6, highWaterMark: 1e6, rowMode: "array" });
|
|
116
134
|
stream = client.query(queryStream);
|
|
117
|
-
this.socketQueries[socketId]![id]!.client =
|
|
135
|
+
this.socketQueries[socketId]![id]!.client = client;
|
|
118
136
|
this.socketQueries[socketId]![id]!.stream = stream;
|
|
137
|
+
const getStreamResult = () => omitKeys(stream._result, ["rows"]) as Info;
|
|
119
138
|
stream.on('data', async (data) => {
|
|
120
139
|
batchRows.push(data);
|
|
121
140
|
if(options?.streamLimit) {
|
|
122
141
|
if(batchRows.length >= options.streamLimit){
|
|
123
|
-
emit("ended",
|
|
142
|
+
emit("ended", getStreamResult());
|
|
124
143
|
}
|
|
125
144
|
}
|
|
126
145
|
if (batchRows.length >= batchSize) {
|
|
127
|
-
emit("rows",
|
|
146
|
+
emit("rows", getStreamResult());
|
|
128
147
|
batchRows = [];
|
|
129
148
|
}
|
|
130
149
|
});
|
|
131
150
|
stream.on('error', error => {
|
|
132
|
-
|
|
151
|
+
if(error.message === "cannot insert multiple commands into a prepared statement") {
|
|
152
|
+
this.dboBuilder.dbo.sql!(query.query, {}, { returnType: "arrayMode", hasParams: false }).then(res => {
|
|
153
|
+
batchRows.push(res.rows);
|
|
154
|
+
emit("ended", res);
|
|
155
|
+
}).catch(newError => {
|
|
156
|
+
socketQuery.onError(newError);
|
|
157
|
+
});
|
|
158
|
+
} else {
|
|
159
|
+
socketQuery.onError(error);
|
|
160
|
+
}
|
|
133
161
|
});
|
|
134
|
-
|
|
135
162
|
stream.on('end', () => {
|
|
136
|
-
emit("ended",
|
|
163
|
+
emit("ended", getStreamResult());
|
|
137
164
|
// release the client when the stream is finished AND connection is not persisted
|
|
138
165
|
if(!options?.persistStreamConnection){
|
|
139
166
|
delete this.socketQueries[socketId]?.[id];
|
|
@@ -149,19 +176,14 @@ export class QueryStreamer {
|
|
|
149
176
|
const stop = async (opts: { terminate?: boolean; } | undefined, cb: BasicCallback) => {
|
|
150
177
|
const { stream, client: queryClient } = this.socketQueries[socketId]?.[id] ?? {};
|
|
151
178
|
if(!stream || !queryClient) return;
|
|
152
|
-
const client = this.getConnection(undefined);
|
|
153
179
|
try {
|
|
154
|
-
await client.connect();
|
|
155
|
-
if (!client) return cb(null, "No client");
|
|
156
180
|
const stopFunction = opts?.terminate? "pg_terminate_backend" : "pg_cancel_backend";
|
|
157
|
-
const rows = await
|
|
181
|
+
const rows = await this.adminClient.query(`SELECT ${stopFunction}(pid), pid, state, query FROM pg_stat_activity WHERE pid = $1 AND query = $2`, [processID, query.query]);
|
|
158
182
|
socket.removeAllListeners(unsubChannel);
|
|
159
183
|
socket.removeAllListeners(channel);
|
|
160
184
|
cb({ processID, info: rows.rows[0] });
|
|
161
185
|
} catch (error){
|
|
162
186
|
cb(null, error);
|
|
163
|
-
} finally {
|
|
164
|
-
await client.end();
|
|
165
187
|
}
|
|
166
188
|
}
|
|
167
189
|
|
package/package.json
CHANGED
|
@@ -162,6 +162,24 @@ export default async function client_only(db: DBHandlerClient, auth: Auth, log:
|
|
|
162
162
|
};
|
|
163
163
|
await res.start(listener);
|
|
164
164
|
});
|
|
165
|
+
await tryRunP("SQL Stream works for multiple statements", async (resolve, reject) => {
|
|
166
|
+
const res = await db.sql!("SELECT * FROM planes; SELECT 1 as a", {}, { returnType: "stream" });
|
|
167
|
+
const listener = async (packet: SocketSQLStreamPacket) => {
|
|
168
|
+
if(packet.type === "error"){
|
|
169
|
+
reject(packet.error);
|
|
170
|
+
} else {
|
|
171
|
+
assert.equal(packet.type, "data");
|
|
172
|
+
assert.equal(packet.ended, true);
|
|
173
|
+
assert.equal(packet.rows.length, 1);
|
|
174
|
+
const normalSql = await db.sql!("SELECT 1 as a", {});
|
|
175
|
+
await db.sql!("DELETE FROM planes", {});
|
|
176
|
+
assert.deepStrictEqual(packet.fields, normalSql.fields);
|
|
177
|
+
assert.equal(packet.fields.length > 0, true);
|
|
178
|
+
resolve("ok");
|
|
179
|
+
}
|
|
180
|
+
};
|
|
181
|
+
await res.start(listener);
|
|
182
|
+
});
|
|
165
183
|
|
|
166
184
|
// await tryRunP("SQL Stream ensure the connection is never released (same pg_backend_pid is the same for subsequent) queries when using persistConnectionId", async (resolve, reject) => {
|
|
167
185
|
// const res = await db.sql!("SELECT pg_backend_pid()", {}, { returnType: "stream", persistConnectionId: true });
|