prostgles-server 4.1.129 → 4.1.131
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/DboBuilder/QueryStreamer.d.ts +3 -1
- package/dist/DboBuilder/QueryStreamer.d.ts.map +1 -1
- package/dist/DboBuilder/QueryStreamer.js +34 -28
- package/dist/DboBuilder/QueryStreamer.js.map +1 -1
- package/lib/DboBuilder/QueryStreamer.ts +35 -27
- package/package.json +2 -2
- package/tests/client/package-lock.json +15 -15
- package/tests/client/package.json +1 -1
- package/tests/client_only_queries.ts +4 -4
- package/tests/server/package-lock.json +3 -3
|
@@ -19,8 +19,10 @@ export declare class QueryStreamer {
|
|
|
19
19
|
db: DB;
|
|
20
20
|
dboBuilder: DboBuilder;
|
|
21
21
|
socketQueries: Record<string, Record<string, StreamedQuery>>;
|
|
22
|
+
adminClient: pg.Client;
|
|
22
23
|
constructor(dboBuilder: DboBuilder);
|
|
23
|
-
getConnection: (onError: ((err: any) => void) | undefined) => pg.Client;
|
|
24
|
+
getConnection: (onError: ((err: any) => void) | undefined, extraOptions?: pg.ClientConfig) => pg.Client;
|
|
25
|
+
onDisconnect: (socketId: string) => void;
|
|
24
26
|
create: (query: ClientStreamedRequest) => Promise<SocketSQLStreamServer>;
|
|
25
27
|
}
|
|
26
28
|
export {};
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"QueryStreamer.d.ts","sourceRoot":"","sources":["../../lib/DboBuilder/QueryStreamer.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,EAAE,EAAE,MAAM,kBAAkB,CAAC;AACtC,OAAO,KAAK,EAAE,MAAM,IAAI,CAAC;AACzB,OAAO,EAAY,UAAU,EAAyB,qBAAqB,EAAE,MAAM,iBAAiB,CAAC;AACrG,OAAO,EAAE,YAAY,EAAE,MAAM,mBAAmB,CAAC;AAGjD,OAAO,EAAE,UAAU,EAAE,MAAM,cAAc,CAAC;AAC1C,OAAO,eAAe,MAAM,iBAAiB,CAAC;AAK9C,KAAK,qBAAqB,GAAG;IAC3B,MAAM,EAAE,YAAY,CAAC;IACrB,KAAK,EAAE,MAAM,CAAC;IACd,OAAO,EAAE,UAAU,GAAG,SAAS,CAAC;IAChC,iBAAiB,CAAC,EAAE,OAAO,CAAC;CAE7B,CAAA;AACD,KAAK,aAAa,GAAG,qBAAqB,GAAG;IAC3C,MAAM,EAAE,eAAe,GAAG,SAAS,CAAC;IACpC,MAAM,EAAE,EAAE,CAAC,MAAM,GAAG,SAAS,CAAC;IAC9B,OAAO,EAAE,CAAC,CAAC,KAAK,EAAE,GAAG,KAAK,IAAI,CAAC,CAAC;CACjC,CAAA;AAWD,qBAAa,aAAa;IACxB,EAAE,EAAE,EAAE,CAAC;IACP,UAAU,EAAE,UAAU,CAAC;IACvB,aAAa,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,aAAa,CAAC,CAAC,CAAM;
|
|
1
|
+
{"version":3,"file":"QueryStreamer.d.ts","sourceRoot":"","sources":["../../lib/DboBuilder/QueryStreamer.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,EAAE,EAAE,MAAM,kBAAkB,CAAC;AACtC,OAAO,KAAK,EAAE,MAAM,IAAI,CAAC;AACzB,OAAO,EAAY,UAAU,EAAyB,qBAAqB,EAAE,MAAM,iBAAiB,CAAC;AACrG,OAAO,EAAE,YAAY,EAAE,MAAM,mBAAmB,CAAC;AAGjD,OAAO,EAAE,UAAU,EAAE,MAAM,cAAc,CAAC;AAC1C,OAAO,eAAe,MAAM,iBAAiB,CAAC;AAK9C,KAAK,qBAAqB,GAAG;IAC3B,MAAM,EAAE,YAAY,CAAC;IACrB,KAAK,EAAE,MAAM,CAAC;IACd,OAAO,EAAE,UAAU,GAAG,SAAS,CAAC;IAChC,iBAAiB,CAAC,EAAE,OAAO,CAAC;CAE7B,CAAA;AACD,KAAK,aAAa,GAAG,qBAAqB,GAAG;IAC3C,MAAM,EAAE,eAAe,GAAG,SAAS,CAAC;IACpC,MAAM,EAAE,EAAE,CAAC,MAAM,GAAG,SAAS,CAAC;IAC9B,OAAO,EAAE,CAAC,CAAC,KAAK,EAAE,GAAG,KAAK,IAAI,CAAC,CAAC;CACjC,CAAA;AAWD,qBAAa,aAAa;IACxB,EAAE,EAAE,EAAE,CAAC;IACP,UAAU,EAAE,UAAU,CAAC;IACvB,aAAa,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,aAAa,CAAC,CAAC,CAAM;IAClE,WAAW,EAAE,EAAE,CAAC,MAAM,CAAC;gBACX,UAAU,EAAE,UAAU;IAclC,aAAa,mBAAoB,GAAG,KAAK,IAAI,8BAA8B,GAAG,YAAY,eAOzF;IAED,YAAY,aAAc,MAAM,UAO/B;IAED,MAAM,UAAiB,qBAAqB,KAAG,QAAQ,qBAAqB,CAAC,CA0I5E;CACF"}
|
|
@@ -18,18 +18,37 @@ class QueryStreamer {
|
|
|
18
18
|
db;
|
|
19
19
|
dboBuilder;
|
|
20
20
|
socketQueries = {};
|
|
21
|
+
adminClient;
|
|
21
22
|
constructor(dboBuilder) {
|
|
22
23
|
this.dboBuilder = dboBuilder;
|
|
23
24
|
this.db = dboBuilder.db;
|
|
25
|
+
const setAdminClient = () => {
|
|
26
|
+
this.adminClient = this.getConnection(undefined, { keepAlive: true });
|
|
27
|
+
this.adminClient.connect();
|
|
28
|
+
};
|
|
29
|
+
this.adminClient = this.getConnection((error) => {
|
|
30
|
+
console.log("Admin client error. Reconnecting...", error);
|
|
31
|
+
setAdminClient();
|
|
32
|
+
}, { keepAlive: true });
|
|
33
|
+
this.adminClient.connect();
|
|
24
34
|
}
|
|
25
|
-
getConnection = (onError) => {
|
|
35
|
+
getConnection = (onError, extraOptions) => {
|
|
26
36
|
const connectionInfo = typeof this.db.$cn === "string" ? { connectionString: this.db.$cn } : this.db.$cn;
|
|
27
|
-
const client = new pg.Client(connectionInfo);
|
|
37
|
+
const client = new pg.Client({ ...connectionInfo, ...extraOptions });
|
|
28
38
|
client.on("error", (err) => {
|
|
29
39
|
onError?.(err);
|
|
30
40
|
});
|
|
31
41
|
return client;
|
|
32
42
|
};
|
|
43
|
+
onDisconnect = (socketId) => {
|
|
44
|
+
const socketQueries = this.socketQueries[socketId];
|
|
45
|
+
if (!socketQueries)
|
|
46
|
+
return;
|
|
47
|
+
Object.values(socketQueries).forEach(({ client }) => {
|
|
48
|
+
client?.end();
|
|
49
|
+
});
|
|
50
|
+
delete this.socketQueries[socketId];
|
|
51
|
+
};
|
|
33
52
|
create = async (query) => {
|
|
34
53
|
const { socket, persistConnection } = query;
|
|
35
54
|
const socketId = socket.id;
|
|
@@ -63,35 +82,26 @@ class QueryStreamer {
|
|
|
63
82
|
if (!socketQuery) {
|
|
64
83
|
throw "socket query not found";
|
|
65
84
|
}
|
|
66
|
-
let emittedPackets = 0;
|
|
67
85
|
let batchRows = [];
|
|
68
86
|
let finished = false;
|
|
69
87
|
const batchSize = 10000;
|
|
70
88
|
let stream;
|
|
71
|
-
let poolClient;
|
|
72
89
|
const emit = (type, stream) => {
|
|
73
90
|
const result = stream?._result;
|
|
74
|
-
let packet;
|
|
75
91
|
const ended = type === "ended";
|
|
76
92
|
if (finished)
|
|
77
93
|
return;
|
|
78
94
|
finished = finished || ended;
|
|
79
|
-
if (!
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
packet = { type: "start", rows: batchRows, fields, ended, processId: processID };
|
|
84
|
-
}
|
|
85
|
-
else {
|
|
86
|
-
packet = { type: "rows", rows: batchRows, ended };
|
|
87
|
-
}
|
|
95
|
+
if (!result?.fields)
|
|
96
|
+
throw "No fields";
|
|
97
|
+
const fields = runSQL_1.getDetailedFieldInfo.bind(this.dboBuilder)(result.fields);
|
|
98
|
+
const packet = { type: "data", rows: batchRows, fields, ended, processId: processID };
|
|
88
99
|
socket.emit(channel, packet);
|
|
89
100
|
if (ended) {
|
|
90
101
|
if (!result)
|
|
91
102
|
throw "No result info";
|
|
92
103
|
runSQL_1.watchSchemaFallback.bind(this.dboBuilder)({ queryWithoutRLS: query.query, command: result.command });
|
|
93
104
|
}
|
|
94
|
-
emittedPackets++;
|
|
95
105
|
};
|
|
96
106
|
const client = this.getConnection(err => {
|
|
97
107
|
socketQuery.onError(err);
|
|
@@ -99,11 +109,10 @@ class QueryStreamer {
|
|
|
99
109
|
});
|
|
100
110
|
try {
|
|
101
111
|
await client.connect();
|
|
102
|
-
poolClient = client;
|
|
103
112
|
processID = client.processID;
|
|
104
113
|
const queryStream = new QueryStream(query.query, [], { batchSize: 1e6, highWaterMark: 1e6, rowMode: "array" });
|
|
105
114
|
stream = client.query(queryStream);
|
|
106
|
-
this.socketQueries[socketId][id].client =
|
|
115
|
+
this.socketQueries[socketId][id].client = client;
|
|
107
116
|
this.socketQueries[socketId][id].stream = stream;
|
|
108
117
|
stream.on('data', async (data) => {
|
|
109
118
|
batchRows.push(data);
|
|
@@ -135,16 +144,12 @@ class QueryStreamer {
|
|
|
135
144
|
}
|
|
136
145
|
};
|
|
137
146
|
const stop = async (opts, cb) => {
|
|
138
|
-
const { stream, client:
|
|
139
|
-
if (!stream || !
|
|
147
|
+
const { stream, client: queryClient } = this.socketQueries[socketId]?.[id] ?? {};
|
|
148
|
+
if (!stream || !queryClient)
|
|
140
149
|
return;
|
|
141
|
-
const client = this.getConnection(undefined);
|
|
142
150
|
try {
|
|
143
|
-
await client.connect();
|
|
144
|
-
if (!client)
|
|
145
|
-
return cb(null, "No client");
|
|
146
151
|
const stopFunction = opts?.terminate ? "pg_terminate_backend" : "pg_cancel_backend";
|
|
147
|
-
const rows = await
|
|
152
|
+
const rows = await this.adminClient.query(`SELECT ${stopFunction}(pid), pid, state, query FROM pg_stat_activity WHERE pid = $1 AND query = $2`, [processID, query.query]);
|
|
148
153
|
socket.removeAllListeners(unsubChannel);
|
|
149
154
|
socket.removeAllListeners(channel);
|
|
150
155
|
cb({ processID, info: rows.rows[0] });
|
|
@@ -152,15 +157,16 @@ class QueryStreamer {
|
|
|
152
157
|
catch (error) {
|
|
153
158
|
cb(null, error);
|
|
154
159
|
}
|
|
155
|
-
finally {
|
|
156
|
-
await client.end();
|
|
157
|
-
}
|
|
158
160
|
};
|
|
159
161
|
socket.removeAllListeners(unsubChannel);
|
|
160
162
|
socket.once(unsubChannel, stop);
|
|
161
163
|
let started = false;
|
|
162
164
|
socket.removeAllListeners(channel);
|
|
163
|
-
socket.
|
|
165
|
+
socket.on(channel, async (_data, cb) => {
|
|
166
|
+
if (started) {
|
|
167
|
+
// TODO
|
|
168
|
+
return cb(null, "Already started");
|
|
169
|
+
}
|
|
164
170
|
started = true;
|
|
165
171
|
try {
|
|
166
172
|
await startStream();
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"QueryStreamer.js","sourceRoot":"","sources":["../../lib/DboBuilder/QueryStreamer.ts"],"names":[],"mappings":";;;AACA,yBAAyB;AACzB,qDAAqG;AAErG,uDAAwE;AACxE,qCAAqE;AAIrE,MAAM,WAAW,GAA2B,OAAO,CAAC,iBAAiB,CAAC,CAAC;AAgBvE,MAAM,cAAc,GAA2B,EAAE,CAAC;AAClD,MAAM,mBAAmB,GAAG,CAAC,QAAgB,EAAE,EAAE;IAC/C,MAAM,OAAO,GAAG,QAAQ,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;IACrC,MAAM,MAAM,GAAG,cAAc,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;IAC5C,MAAM,KAAK,GAAG,MAAM,GAAG,CAAC,CAAC;IACzB,cAAc,CAAC,OAAO,CAAC,GAAG,KAAK,CAAC;IAChC,OAAO,KAAK,CAAC;AACf,CAAC,CAAC;AAEF,MAAa,aAAa;IACxB,EAAE,CAAK;IACP,UAAU,CAAa;IACvB,aAAa,GAAkD,EAAE,CAAC;IAClE,YAAY,UAAsB;QAChC,IAAI,CAAC,UAAU,GAAG,UAAU,CAAC;QAC7B,IAAI,CAAC,EAAE,GAAG,UAAU,CAAC,EAAE,CAAC;
|
|
1
|
+
{"version":3,"file":"QueryStreamer.js","sourceRoot":"","sources":["../../lib/DboBuilder/QueryStreamer.ts"],"names":[],"mappings":";;;AACA,yBAAyB;AACzB,qDAAqG;AAErG,uDAAwE;AACxE,qCAAqE;AAIrE,MAAM,WAAW,GAA2B,OAAO,CAAC,iBAAiB,CAAC,CAAC;AAgBvE,MAAM,cAAc,GAA2B,EAAE,CAAC;AAClD,MAAM,mBAAmB,GAAG,CAAC,QAAgB,EAAE,EAAE;IAC/C,MAAM,OAAO,GAAG,QAAQ,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;IACrC,MAAM,MAAM,GAAG,cAAc,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;IAC5C,MAAM,KAAK,GAAG,MAAM,GAAG,CAAC,CAAC;IACzB,cAAc,CAAC,OAAO,CAAC,GAAG,KAAK,CAAC;IAChC,OAAO,KAAK,CAAC;AACf,CAAC,CAAC;AAEF,MAAa,aAAa;IACxB,EAAE,CAAK;IACP,UAAU,CAAa;IACvB,aAAa,GAAkD,EAAE,CAAC;IAClE,WAAW,CAAY;IACvB,YAAY,UAAsB;QAChC,IAAI,CAAC,UAAU,GAAG,UAAU,CAAC;QAC7B,IAAI,CAAC,EAAE,GAAG,UAAU,CAAC,EAAE,CAAC;QACxB,MAAM,cAAc,GAAG,GAAG,EAAE;YAC1B,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC,aAAa,CAAC,SAAS,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;YACtE,IAAI,CAAC,WAAW,CAAC,OAAO,EAAE,CAAC;QAC7B,CAAC,CAAA;QACD,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC,aAAa,CAAC,CAAC,KAAK,EAAE,EAAE;YAC9C,OAAO,CAAC,GAAG,CAAC,qCAAqC,EAAE,KAAK,CAAC,CAAC;YAC1D,cAAc,EAAE,CAAC;QACnB,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QACxB,IAAI,CAAC,WAAW,CAAC,OAAO,EAAE,CAAC;IAC7B,CAAC;IAED,aAAa,GAAG,CAAC,OAAyC,EAAE,YAA8B,EAAE,EAAE;QAC5F,MAAM,cAAc,GAAG,OAAO,IAAI,CAAC,EAAE,CAAC,GAAG,KAAK,QAAQ,CAAA,CAAC,CAAC,EAAE,gBAAgB,EAAE,IAAI,CAAC,EAAE,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,GAAU,CAAC;QAC/G,MAAM,MAAM,GAAG,IAAI,EAAE,CAAC,MAAM,CAAC,EAAE,GAAG,cAAc,EAAE,GAAG,YAAY,EAAE,CAAC,CAAC;QACrE,MAAM,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,GAAG,EAAE,EAAE;YACzB,OAAO,EAAE,CAAC,GAAG,CAAC,CAAC;QACjB,CAAC,CAAC,CAAC;QACH,OAAO,MAAM,CAAC;IAChB,CAAC,CAAA;IAED,YAAY,GAAG,CAAC,QAAgB,EAAE,EAAE;QAClC,MAAM,aAAa,GAAG,IAAI,CAAC,aAAa,CAAC,QAAQ,CAAC,CAAC;QACnD,IAAG,CAAC,aAAa;YAAE,OAAO;QAC1B,MAAM,CAAC,MAAM,CAAC,aAAa,CAAC,CAAC,OAAO,CAAC,CAAC,EAAE,MAAM,EAAE,EAAE,EAAE;YAClD,MAAM,EAAE,GAAG,EAAE,CAAC;QAChB,CAAC,CAAC,CAAC;QACH,OAAO,IAAI,CAAC,aAAa,CAAC,QAAQ,CAAC,CAAC;IACtC,CAAC,CAAA;IAED,MAAM,GAAG,KAAK,EAAE,KAA4B,EAAkC,EAAE;QAC9E,MAAM,EAAE,MAAM,EAAE,iBAAiB,EAAE,GAAG,KAAK,CAAC;QAC5C,MAAM,QAAQ,GAAG,MAAM,CAAC,EAAE,CAAC;QAC3B,MAAM,EAAE,GAAG,mBAAmB,CAAC,QAAQ,CAAC,CAAC;QACzC,MAAM,OAAO,GAAG,GAAG,0BAAQ,CAAC,UAAU,KAAK,QAAQ,IAAI,EAAE,EAAE,CAAC;QAC5D,MAAM,YAAY,GAAG,GAAG,OAAO,cAAc,CAAC;QAC9C,IAAG,IAAI,CAAC,aAAa,CAAC,EAAE,CAAC,IAAI,CAAC,iBAAiB,EAAC;YAC9C,MAAM,4BAA4B,EAAE,QAAQ,CAAC;SAC9C;QAED,IAAI,CAAC,aAAa,CAAC,QAAQ,CAAC,KAAK,EAAE,CAAC;QACpC,IAAI,OAAO,GAAG,KAAK,CAAC;QACpB,MAAM,WAAW,GAAG;YAClB,GAAG,KAAK;YACR,MAAM,EAAE,SAAS;YACjB,MAAM,EAAE,SAAS;YACjB,OAAO,EAAE,CAAC,QAAa,EAAE,EAAE;gBACzB,IAAG,OAAO;oBAAE,OAAO;gBACnB,OAAO,GAAG,IAAI,CAAC;gBAEf,MAAM,iBAAiB,GAAG,IAAA,qDAAmC,EAAC,QAAQ,EAAE,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,CAAC;gBACzF,gFAAgF;gBAChF,MAAM,KAAK,GAAG,EAAE,GAAG,iBAAiB,EAAE,KAAK,EAAE,KAAK,CAAC,KAAK,EAAE,CAAC;gBAC3D,MAAM,CAAC,IAAI,CAAC,OAAO,EAAE,EAAE,IAAI,EAAE,OAAO,EAAE,KAAK,EAAkC,CAAC,CAAC;YACjF,CAAC;SACF,CAAC;QACF,IAAI,CAAC,aAAa,CAAC,QAAQ,CAAE,CAAC,EAAE,CAAC,KAAK,WAAW,CAAC;QAClD,MAAM,EAAE,OAAO,EAAE,GAAG,KAAK,CAAA;QACzB,IAAI,SAAS,GAAG,CAAC,CAAC,CAAC;QAEnB,MAAM,WAAW,GAAG,KAAK,IAAI,EAAE;YAC7B,MAAM,WAAW,GAAG,IAAI,CAAC,aAAa,CAAC,QAAQ,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC;YACvD,IAAG,CAAC,WAAW,EAAC;gBACd,MAAM,wBAAwB,CAAC;aAChC;YACD,IAAI,SAAS,GAAU,EAAE,CAAC;YAC1B,IAAI,QAAQ,GAAG,KAAK,CAAC;YACrB,MAAM,SAAS,GAAG,KAAK,CAAC;YACxB,IAAI,MAAuB,CAAC;YAC5B,MAAM,IAAI,GAAG,CAAC,IAAsB,EAAE,MAAmC,EAAE,EAAE;gBAC3E,MAAM,MAAM,GAAG,MAAM,EAAE,OAAyD,CAAC;gBACjF,MAAM,KAAK,GAAG,IAAI,KAAK,OAAO,CAAC;gBAC/B,IAAG,QAAQ;oBAAE,OAAO;gBACpB,QAAQ,GAAG,QAAQ,IAAI,KAAK,CAAC;gBAC7B,IAAG,CAAC,MAAM,EAAE,MAAM;oBAAE,MAAM,WAAW,CAAC;gBACtC,MAAM,MAAM,GAAG,6BAAoB,CAAC,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC,MAAM,CAAC,MAAa,CAAC,CAAC;gBAChF,MAAM,MAAM,GAA0B,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,SAAS,EAAE,MAAM,EAAE,KAAK,EAAE,SAAS,EAAE,SAAS,EAAE,CAAC;gBAC7G,MAAM,CAAC,IAAI,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;gBAC7B,IAAG,KAAK,EAAC;oBACP,IAAG,CAAC,MAAM;wBAAE,MAAM,gBAAgB,CAAC;oBACnC,4BAAmB,CAAC,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC,EAAE,eAAe,EAAE,KAAK,CAAC,KAAK,EAAE,OAAO,EAAE,MAAM,CAAC,OAAO,EAAE,CAAC,CAAC;iBACtG;YACH,CAAC,CAAA;YACD,MAAM,MAAM,GAAG,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,EAAE;gBACtC,WAAW,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;gBACzB,MAAM,CAAC,GAAG,EAAE,CAAC;YACf,CAAC,CAAC,CAAC;YACH,IAAI;gBACF,MAAM,MAAM,CAAC,OAAO,EAAE,CAAC;gBACvB,SAAS,GAAI,MAAc,CAAC,SAAS,CAAA;gBACrC,MAAM,WAAW,GAAG,IAAI,WAAW,CAAC,KAAK,CAAC,KAAK,EAAE,EAAE,EAAE,EAAE,SAAS,EAAE,GAAG,EAAE,aAAa,EAAE,GAAG,EAAE,OAAO,EAAE,OAAO,EAAE,CAAC,CAAC;gBAC/G,MAAM,GAAG,MAAM,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC;gBACnC,IAAI,CAAC,aAAa,CAAC,QAAQ,CAAE,CAAC,EAAE,CAAE,CAAC,MAAM,GAAG,MAAM,CAAC;gBACnD,IAAI,CAAC,aAAa,CAAC,QAAQ,CAAE,CAAC,EAAE,CAAE,CAAC,MAAM,GAAG,MAAM,CAAC;gBACnD,MAAM,CAAC,EAAE,CAAC,MAAM,EAAE,KAAK,EAAE,IAAI,EAAE,EAAE;oBAC/B,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;oBACrB,IAAG,OAAO,EAAE,WAAW,EAAE;wBACvB,IAAG,SAAS,CAAC,MAAM,IAAI,OAAO,CAAC,WAAW,EAAC;4BACzC,IAAI,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;yBACvB;qBACF;oBACD,IAAI,SAAS,CAAC,MAAM,IAAI,SAAS,EAAE;wBACjC,IAAI,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;wBACrB,SAAS,GAAG,EAAE,CAAC;qBAChB;gBACH,CAAC,CAAC,CAAC;gBACH,MAAM,CAAC,EAAE,CAAC,OAAO,EAAE,KAAK,CAAC,EAAE;oBACzB,WAAW,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;gBAC7B,CAAC,CAAC,CAAC;gBACH,MAAM,CAAC,EAAE,CAAC,KAAK,EAAE,GAAG,EAAE;oBACpB,IAAI,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;oBACtB,iFAAiF;oBACjF,IAAG,CAAC,OAAO,EAAE,uBAAuB,EAAC;wBACnC,OAAO,IAAI,CAAC,aAAa,CAAC,QAAQ,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC;wBAC1C,MAAM,CAAC,GAAG,EAAE,CAAC;qBACd;gBACH,CAAC,CAAC,CAAC;aACJ;YAAC,OAAM,GAAG,EAAC;gBACV,WAAW,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;gBACzB,MAAM,MAAM,CAAC,GAAG,EAAE,CAAC;aACpB;QACH,CAAC,CAAA;QAED,MAAM,IAAI,GAAG,KAAK,EAAE,IAA0C,EAAE,EAAiB,EAAE,EAAE;YACnF,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,WAAW,EAAE,GAAG,IAAI,CAAC,aAAa,CAAC,QAAQ,CAAC,EAAE,CAAC,EAAE,CAAC,IAAI,EAAE,CAAC;YACjF,IAAG,CAAC,MAAM,IAAI,CAAC,WAAW;gBAAE,OAAO;YACnC,IAAI;gBACF,MAAM,YAAY,GAAG,IAAI,EAAE,SAAS,CAAA,CAAC,CAAC,sBAAsB,CAAC,CAAC,CAAC,mBAAmB,CAAC;gBACnF,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,WAAW,CAAC,KAAK,CAAC,UAAU,YAAY,8EAA8E,EAAE,CAAC,SAAS,EAAE,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC;gBAC1K,MAAM,CAAC,kBAAkB,CAAC,YAAY,CAAC,CAAC;gBACxC,MAAM,CAAC,kBAAkB,CAAC,OAAO,CAAC,CAAC;gBACnC,EAAE,CAAC,EAAE,SAAS,EAAE,IAAI,EAAE,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;aACvC;YAAC,OAAO,KAAK,EAAC;gBACb,EAAE,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC;aACjB;QACH,CAAC,CAAA;QAED,MAAM,CAAC,kBAAkB,CAAC,YAAY,CAAC,CAAC;QACxC,MAAM,CAAC,IAAI,CAAC,YAAY,EAAE,IAAI,CAAC,CAAC;QAEhC,IAAI,OAAO,GAAG,KAAK,CAAC;QACpB,MAAM,CAAC,kBAAkB,CAAC,OAAO,CAAC,CAAC;QACnC,MAAM,CAAC,EAAE,CAAC,OAAO,EAAE,KAAK,EAAE,KAAiD,EAAE,EAAiB,EAAE,EAAE;YAChG,IAAG,OAAO,EAAC;gBACT,OAAO;gBACP,OAAO,EAAE,CAAC,IAAI,EAAE,iBAAiB,CAAC,CAAC;aACpC;YACD,OAAO,GAAG,IAAI,CAAC;YACf,IAAI;gBACF,MAAM,WAAW,EAAE,CAAC;gBACpB,EAAE,EAAE,CAAC;aACN;YAAC,OAAM,GAAG,EAAC;gBACV,EAAE,CAAC,IAAI,EAAE,GAAG,IAAI,sBAAsB,CAAC,CAAC;aACzC;QACH,CAAC,CAAC,CAAC;QAEH,qEAAqE;QACrE,UAAU,CAAC,GAAG,EAAE;YACd,IAAG,OAAO;gBAAE,OAAO;YACnB,IAAI,CAAC,EAAE,EAAE,GAAG,EAAE;gBACZ,SAAS;YACX,CAAC,CAAC,CAAC;QACL,CAAC,EAAE,GAAG,CAAC,CAAC;QAER,OAAO;YACL,OAAO;YACP,YAAY;SACb,CAAA;IACH,CAAC,CAAA;CACF;AAhLD,sCAgLC"}
|
|
@@ -36,20 +36,39 @@ export class QueryStreamer {
|
|
|
36
36
|
db: DB;
|
|
37
37
|
dboBuilder: DboBuilder;
|
|
38
38
|
socketQueries: Record<string, Record<string, StreamedQuery>> = {};
|
|
39
|
+
adminClient: pg.Client;
|
|
39
40
|
constructor(dboBuilder: DboBuilder) {
|
|
40
41
|
this.dboBuilder = dboBuilder;
|
|
41
42
|
this.db = dboBuilder.db;
|
|
43
|
+
const setAdminClient = () => {
|
|
44
|
+
this.adminClient = this.getConnection(undefined, { keepAlive: true });
|
|
45
|
+
this.adminClient.connect();
|
|
46
|
+
}
|
|
47
|
+
this.adminClient = this.getConnection((error) => {
|
|
48
|
+
console.log("Admin client error. Reconnecting...", error);
|
|
49
|
+
setAdminClient();
|
|
50
|
+
}, { keepAlive: true });
|
|
51
|
+
this.adminClient.connect();
|
|
42
52
|
}
|
|
43
53
|
|
|
44
|
-
getConnection = (onError: ((err: any) => void) | undefined) => {
|
|
54
|
+
getConnection = (onError: ((err: any) => void) | undefined, extraOptions?: pg.ClientConfig) => {
|
|
45
55
|
const connectionInfo = typeof this.db.$cn === "string"? { connectionString: this.db.$cn } : this.db.$cn as any;
|
|
46
|
-
const client = new pg.Client(connectionInfo);
|
|
56
|
+
const client = new pg.Client({ ...connectionInfo, ...extraOptions });
|
|
47
57
|
client.on("error", (err) => {
|
|
48
58
|
onError?.(err);
|
|
49
59
|
});
|
|
50
60
|
return client;
|
|
51
61
|
}
|
|
52
62
|
|
|
63
|
+
onDisconnect = (socketId: string) => {
|
|
64
|
+
const socketQueries = this.socketQueries[socketId];
|
|
65
|
+
if(!socketQueries) return;
|
|
66
|
+
Object.values(socketQueries).forEach(({ client }) => {
|
|
67
|
+
client?.end();
|
|
68
|
+
});
|
|
69
|
+
delete this.socketQueries[socketId];
|
|
70
|
+
}
|
|
71
|
+
|
|
53
72
|
create = async (query: ClientStreamedRequest): Promise<SocketSQLStreamServer> => {
|
|
54
73
|
const { socket, persistConnection } = query;
|
|
55
74
|
const socketId = socket.id;
|
|
@@ -85,43 +104,34 @@ export class QueryStreamer {
|
|
|
85
104
|
if(!socketQuery){
|
|
86
105
|
throw "socket query not found";
|
|
87
106
|
}
|
|
88
|
-
let emittedPackets = 0;
|
|
89
107
|
let batchRows: any[] = [];
|
|
90
108
|
let finished = false;
|
|
91
109
|
const batchSize = 10000;
|
|
92
|
-
let stream: QueryStreamType;
|
|
93
|
-
let poolClient: pg.Client;
|
|
110
|
+
let stream: QueryStreamType;
|
|
94
111
|
const emit = (type: "rows" | "ended", stream: QueryStreamType | undefined) => {
|
|
95
112
|
const result = stream?._result as { command: string; fields: any[] } | undefined;
|
|
96
|
-
let packet: SocketSQLStreamPacket | undefined;
|
|
97
113
|
const ended = type === "ended";
|
|
98
114
|
if(finished) return;
|
|
99
115
|
finished = finished || ended;
|
|
100
|
-
if
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
packet = { type: "start", rows: batchRows, fields, ended, processId: processID };
|
|
104
|
-
} else {
|
|
105
|
-
packet = { type: "rows", rows: batchRows, ended };
|
|
106
|
-
}
|
|
116
|
+
if(!result?.fields) throw "No fields";
|
|
117
|
+
const fields = getDetailedFieldInfo.bind(this.dboBuilder)(result.fields as any);
|
|
118
|
+
const packet: SocketSQLStreamPacket = { type: "data", rows: batchRows, fields, ended, processId: processID };
|
|
107
119
|
socket.emit(channel, packet);
|
|
108
120
|
if(ended){
|
|
109
121
|
if(!result) throw "No result info";
|
|
110
122
|
watchSchemaFallback.bind(this.dboBuilder)({ queryWithoutRLS: query.query, command: result.command });
|
|
111
123
|
}
|
|
112
|
-
emittedPackets++;
|
|
113
124
|
}
|
|
114
125
|
const client = this.getConnection(err => {
|
|
115
126
|
socketQuery.onError(err);
|
|
116
127
|
client.end();
|
|
117
128
|
});
|
|
118
129
|
try {
|
|
119
|
-
await client.connect();
|
|
120
|
-
poolClient = client;
|
|
130
|
+
await client.connect();
|
|
121
131
|
processID = (client as any).processID
|
|
122
132
|
const queryStream = new QueryStream(query.query, [], { batchSize: 1e6, highWaterMark: 1e6, rowMode: "array" });
|
|
123
133
|
stream = client.query(queryStream);
|
|
124
|
-
this.socketQueries[socketId]![id]!.client =
|
|
134
|
+
this.socketQueries[socketId]![id]!.client = client;
|
|
125
135
|
this.socketQueries[socketId]![id]!.stream = stream;
|
|
126
136
|
stream.on('data', async (data) => {
|
|
127
137
|
batchRows.push(data);
|
|
@@ -138,7 +148,6 @@ export class QueryStreamer {
|
|
|
138
148
|
stream.on('error', error => {
|
|
139
149
|
socketQuery.onError(error);
|
|
140
150
|
});
|
|
141
|
-
|
|
142
151
|
stream.on('end', () => {
|
|
143
152
|
emit("ended", stream);
|
|
144
153
|
// release the client when the stream is finished AND connection is not persisted
|
|
@@ -154,21 +163,16 @@ export class QueryStreamer {
|
|
|
154
163
|
}
|
|
155
164
|
|
|
156
165
|
const stop = async (opts: { terminate?: boolean; } | undefined, cb: BasicCallback) => {
|
|
157
|
-
const { stream, client:
|
|
158
|
-
if(!stream || !
|
|
159
|
-
const client = this.getConnection(undefined);
|
|
166
|
+
const { stream, client: queryClient } = this.socketQueries[socketId]?.[id] ?? {};
|
|
167
|
+
if(!stream || !queryClient) return;
|
|
160
168
|
try {
|
|
161
|
-
await client.connect();
|
|
162
|
-
if (!client) return cb(null, "No client");
|
|
163
169
|
const stopFunction = opts?.terminate? "pg_terminate_backend" : "pg_cancel_backend";
|
|
164
|
-
const rows = await
|
|
170
|
+
const rows = await this.adminClient.query(`SELECT ${stopFunction}(pid), pid, state, query FROM pg_stat_activity WHERE pid = $1 AND query = $2`, [processID, query.query]);
|
|
165
171
|
socket.removeAllListeners(unsubChannel);
|
|
166
172
|
socket.removeAllListeners(channel);
|
|
167
173
|
cb({ processID, info: rows.rows[0] });
|
|
168
174
|
} catch (error){
|
|
169
175
|
cb(null, error);
|
|
170
|
-
} finally {
|
|
171
|
-
await client.end();
|
|
172
176
|
}
|
|
173
177
|
}
|
|
174
178
|
|
|
@@ -177,7 +181,11 @@ export class QueryStreamer {
|
|
|
177
181
|
|
|
178
182
|
let started = false;
|
|
179
183
|
socket.removeAllListeners(channel);
|
|
180
|
-
socket.
|
|
184
|
+
socket.on(channel, async (_data: { query: string; params: any } | undefined, cb: BasicCallback) => {
|
|
185
|
+
if(started){
|
|
186
|
+
// TODO
|
|
187
|
+
return cb(null, "Already started");
|
|
188
|
+
}
|
|
181
189
|
started = true;
|
|
182
190
|
try {
|
|
183
191
|
await startStream();
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "prostgles-server",
|
|
3
|
-
"version": "4.1.
|
|
3
|
+
"version": "4.1.131",
|
|
4
4
|
"description": "",
|
|
5
5
|
"main": "dist/index.js",
|
|
6
6
|
"types": "dist/index.d.ts",
|
|
@@ -46,7 +46,7 @@
|
|
|
46
46
|
"pg-promise": "^11.5.4",
|
|
47
47
|
"pg-query-stream": "^4.5.3",
|
|
48
48
|
"prostgles-client": "^4.0.53",
|
|
49
|
-
"prostgles-types": "^4.0.
|
|
49
|
+
"prostgles-types": "^4.0.67"
|
|
50
50
|
},
|
|
51
51
|
"devDependencies": {
|
|
52
52
|
"@types/bluebird": "^3.5.36",
|
|
@@ -10,7 +10,7 @@
|
|
|
10
10
|
"license": "ISC",
|
|
11
11
|
"dependencies": {
|
|
12
12
|
"@types/node": "^14.14.16",
|
|
13
|
-
"prostgles-client": "^4.0.
|
|
13
|
+
"prostgles-client": "^4.0.62",
|
|
14
14
|
"prostgles-types": "^4.0.51",
|
|
15
15
|
"socket.io-client": "^4.7.1"
|
|
16
16
|
},
|
|
@@ -75,11 +75,11 @@
|
|
|
75
75
|
"integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w=="
|
|
76
76
|
},
|
|
77
77
|
"node_modules/prostgles-client": {
|
|
78
|
-
"version": "4.0.
|
|
79
|
-
"resolved": "https://registry.npmjs.org/prostgles-client/-/prostgles-client-4.0.
|
|
80
|
-
"integrity": "sha512-
|
|
78
|
+
"version": "4.0.62",
|
|
79
|
+
"resolved": "https://registry.npmjs.org/prostgles-client/-/prostgles-client-4.0.62.tgz",
|
|
80
|
+
"integrity": "sha512-0tSymKQWkf5hShxyw3kbBzs0wVqwzFYYRqWWTV8zCJGuCLfoKCqhUU6BY257ZAuhRBmMClTa430JR1PccXVYvw==",
|
|
81
81
|
"dependencies": {
|
|
82
|
-
"prostgles-types": "^4.0.
|
|
82
|
+
"prostgles-types": "^4.0.67"
|
|
83
83
|
},
|
|
84
84
|
"peerDependencies": {
|
|
85
85
|
"react": "^16.8.0 || ^17.0.0 || ^18.0.0"
|
|
@@ -91,9 +91,9 @@
|
|
|
91
91
|
}
|
|
92
92
|
},
|
|
93
93
|
"node_modules/prostgles-types": {
|
|
94
|
-
"version": "4.0.
|
|
95
|
-
"resolved": "https://registry.npmjs.org/prostgles-types/-/prostgles-types-4.0.
|
|
96
|
-
"integrity": "sha512-
|
|
94
|
+
"version": "4.0.67",
|
|
95
|
+
"resolved": "https://registry.npmjs.org/prostgles-types/-/prostgles-types-4.0.67.tgz",
|
|
96
|
+
"integrity": "sha512-KOZ+q3apiwmn+97PAGYpdH6EZwOSpA4OmzZX6fzbzGLG2spE1ghrakQIBGa7HqwuuIoejQInKWbRCs4Arm1n6A==",
|
|
97
97
|
"dependencies": {
|
|
98
98
|
"json-schema": "^0.4.0"
|
|
99
99
|
}
|
|
@@ -213,17 +213,17 @@
|
|
|
213
213
|
"integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w=="
|
|
214
214
|
},
|
|
215
215
|
"prostgles-client": {
|
|
216
|
-
"version": "4.0.
|
|
217
|
-
"resolved": "https://registry.npmjs.org/prostgles-client/-/prostgles-client-4.0.
|
|
218
|
-
"integrity": "sha512-
|
|
216
|
+
"version": "4.0.62",
|
|
217
|
+
"resolved": "https://registry.npmjs.org/prostgles-client/-/prostgles-client-4.0.62.tgz",
|
|
218
|
+
"integrity": "sha512-0tSymKQWkf5hShxyw3kbBzs0wVqwzFYYRqWWTV8zCJGuCLfoKCqhUU6BY257ZAuhRBmMClTa430JR1PccXVYvw==",
|
|
219
219
|
"requires": {
|
|
220
|
-
"prostgles-types": "^4.0.
|
|
220
|
+
"prostgles-types": "^4.0.67"
|
|
221
221
|
}
|
|
222
222
|
},
|
|
223
223
|
"prostgles-types": {
|
|
224
|
-
"version": "4.0.
|
|
225
|
-
"resolved": "https://registry.npmjs.org/prostgles-types/-/prostgles-types-4.0.
|
|
226
|
-
"integrity": "sha512-
|
|
224
|
+
"version": "4.0.67",
|
|
225
|
+
"resolved": "https://registry.npmjs.org/prostgles-types/-/prostgles-types-4.0.67.tgz",
|
|
226
|
+
"integrity": "sha512-KOZ+q3apiwmn+97PAGYpdH6EZwOSpA4OmzZX6fzbzGLG2spE1ghrakQIBGa7HqwuuIoejQInKWbRCs4Arm1n6A==",
|
|
227
227
|
"requires": {
|
|
228
228
|
"json-schema": "^0.4.0"
|
|
229
229
|
}
|
|
@@ -17,7 +17,7 @@ export default async function client_only(db: DBHandlerClient, auth: Auth, log:
|
|
|
17
17
|
assert.equal(packet.error.message, "canceling statement due to user request");
|
|
18
18
|
resolve("ok");
|
|
19
19
|
} else {
|
|
20
|
-
assert.equal(packet.type, "
|
|
20
|
+
assert.equal(packet.type, "data");
|
|
21
21
|
assert.equal(packet.ended, true);
|
|
22
22
|
assert.deepStrictEqual(packet.rows, []);
|
|
23
23
|
reject("ok");
|
|
@@ -79,7 +79,7 @@ export default async function client_only(db: DBHandlerClient, auth: Auth, log:
|
|
|
79
79
|
const res = await db.sql!("SELECT ${val} as val", { val }, { returnType: "stream" });
|
|
80
80
|
const listener = async (packet: SocketSQLStreamPacket) => {
|
|
81
81
|
try {
|
|
82
|
-
assert.equal(packet.type, "
|
|
82
|
+
assert.equal(packet.type, "data");
|
|
83
83
|
assert.equal(packet.ended, true);
|
|
84
84
|
assert.deepStrictEqual(packet.rows, [[val]]);
|
|
85
85
|
resolve(1);
|
|
@@ -123,7 +123,7 @@ export default async function client_only(db: DBHandlerClient, auth: Auth, log:
|
|
|
123
123
|
if(packet.type === "error"){
|
|
124
124
|
reject(packet.error);
|
|
125
125
|
} else {
|
|
126
|
-
assert.equal(packet.type, "
|
|
126
|
+
assert.equal(packet.type, "data");
|
|
127
127
|
assert.equal(packet.ended, true);
|
|
128
128
|
assert.equal(packet.rows.length, 10);
|
|
129
129
|
|
|
@@ -150,7 +150,7 @@ export default async function client_only(db: DBHandlerClient, auth: Auth, log:
|
|
|
150
150
|
if(packet.type === "error"){
|
|
151
151
|
reject(packet.error);
|
|
152
152
|
} else {
|
|
153
|
-
assert.equal(packet.type, "
|
|
153
|
+
assert.equal(packet.type, "data");
|
|
154
154
|
assert.equal(packet.ended, true);
|
|
155
155
|
assert.equal(packet.rows.length, 1);
|
|
156
156
|
const normalSql = await db.sql!("SELECT * FROM planes LIMIT 1", {});
|
|
@@ -21,7 +21,7 @@
|
|
|
21
21
|
},
|
|
22
22
|
"../..": {
|
|
23
23
|
"name": "prostgles-server",
|
|
24
|
-
"version": "4.1.
|
|
24
|
+
"version": "4.1.130",
|
|
25
25
|
"license": "MIT",
|
|
26
26
|
"dependencies": {
|
|
27
27
|
"@types/express": "^4.17.13",
|
|
@@ -35,7 +35,7 @@
|
|
|
35
35
|
"pg-promise": "^11.5.4",
|
|
36
36
|
"pg-query-stream": "^4.5.3",
|
|
37
37
|
"prostgles-client": "^4.0.53",
|
|
38
|
-
"prostgles-types": "^4.0.
|
|
38
|
+
"prostgles-types": "^4.0.67"
|
|
39
39
|
},
|
|
40
40
|
"devDependencies": {
|
|
41
41
|
"@types/bluebird": "^3.5.36",
|
|
@@ -1546,7 +1546,7 @@
|
|
|
1546
1546
|
"pg-promise": "^11.5.4",
|
|
1547
1547
|
"pg-query-stream": "^4.5.3",
|
|
1548
1548
|
"prostgles-client": "^4.0.53",
|
|
1549
|
-
"prostgles-types": "^4.0.
|
|
1549
|
+
"prostgles-types": "^4.0.67",
|
|
1550
1550
|
"typescript": "^5.2.2"
|
|
1551
1551
|
}
|
|
1552
1552
|
},
|