prostgles-server 4.1.127 → 4.1.128
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/DboBuilder/QueryStreamer.d.ts +1 -0
- package/dist/DboBuilder/QueryStreamer.d.ts.map +1 -1
- package/dist/DboBuilder/QueryStreamer.js +37 -19
- package/dist/DboBuilder/QueryStreamer.js.map +1 -1
- package/lib/DboBuilder/QueryStreamer.ts +40 -20
- package/package.json +2 -2
- package/tests/client/index.ts +11 -7
- package/tests/client/package-lock.json +15 -15
- package/tests/client/package.json +2 -2
- package/tests/client_only_queries.ts +28 -0
- package/tests/server/index.ts +3 -0
- package/tests/server/package-lock.json +3 -3
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"QueryStreamer.d.ts","sourceRoot":"","sources":["../../lib/DboBuilder/QueryStreamer.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,EAAE,EAAE,MAAM,kBAAkB,CAAC;AACtC,OAAO,KAAK,EAAE,MAAM,IAAI,CAAC;AACzB,OAAO,
|
|
1
|
+
{"version":3,"file":"QueryStreamer.d.ts","sourceRoot":"","sources":["../../lib/DboBuilder/QueryStreamer.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,EAAE,EAAE,MAAM,kBAAkB,CAAC;AACtC,OAAO,KAAK,EAAE,MAAM,IAAI,CAAC;AACzB,OAAO,EAAY,UAAU,EAAyB,qBAAqB,EAAE,MAAM,iBAAiB,CAAC;AACrG,OAAO,EAAE,YAAY,EAAE,MAAM,mBAAmB,CAAC;AAGjD,OAAO,EAAE,UAAU,EAAE,MAAM,cAAc,CAAC;AAC1C,OAAO,eAAe,MAAM,iBAAiB,CAAC;AAK9C,KAAK,qBAAqB,GAAG;IAC3B,MAAM,EAAE,YAAY,CAAC;IACrB,KAAK,EAAE,MAAM,CAAC;IACd,OAAO,EAAE,UAAU,GAAG,SAAS,CAAC;IAChC,iBAAiB,CAAC,EAAE,OAAO,CAAC;CAE7B,CAAA;AACD,KAAK,aAAa,GAAG,qBAAqB,GAAG;IAC3C,MAAM,EAAE,eAAe,GAAG,SAAS,CAAC;IACpC,UAAU,EAAE,EAAE,CAAC,UAAU,GAAG,SAAS,CAAC;IACtC,OAAO,EAAE,CAAC,CAAC,KAAK,EAAE,GAAG,KAAK,IAAI,CAAC,CAAC;CACjC,CAAA;AAWD,qBAAa,aAAa;IACxB,EAAE,EAAE,EAAE,CAAC;IACP,UAAU,EAAE,UAAU,CAAC;IACvB,aAAa,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,aAAa,CAAC,CAAC,CAAM;IAClE,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC;IACd,SAAS,EAAE,EAAE,CAAC,IAAI,CAAC;gBACP,UAAU,EAAE,UAAU;IAmBlC,MAAM,UAAiB,qBAAqB,KAAG,QAAQ,qBAAqB,CAAC,CAyJ5E;CACF"}
|
|
@@ -23,8 +23,20 @@ class QueryStreamer {
|
|
|
23
23
|
constructor(dboBuilder) {
|
|
24
24
|
this.dboBuilder = dboBuilder;
|
|
25
25
|
this.db = dboBuilder.db;
|
|
26
|
-
|
|
27
|
-
|
|
26
|
+
const connectionInfo = typeof this.db.$cn === "string" ? { connectionString: this.db.$cn } : this.db.$cn;
|
|
27
|
+
const onPoolError = (err, _client) => {
|
|
28
|
+
console.error(err.message);
|
|
29
|
+
};
|
|
30
|
+
this.pool = new pg.Pool({ ...connectionInfo, max: 50 }).on("error", (error) => {
|
|
31
|
+
// if(error.message !== "Connection terminated") return;
|
|
32
|
+
Object.entries(this.socketQueries).forEach(([socketId, queries]) => {
|
|
33
|
+
Object.entries(queries).forEach(([id, query]) => {
|
|
34
|
+
query.onError?.({ message: error.message });
|
|
35
|
+
delete this.socketQueries[socketId]?.[id];
|
|
36
|
+
});
|
|
37
|
+
});
|
|
38
|
+
});
|
|
39
|
+
this.adminPool = new pg.Pool(connectionInfo).on("error", onPoolError);
|
|
28
40
|
}
|
|
29
41
|
create = async (query) => {
|
|
30
42
|
const { socket, persistConnection } = query;
|
|
@@ -36,11 +48,22 @@ class QueryStreamer {
|
|
|
36
48
|
throw `Must stop existing query ${id} first`;
|
|
37
49
|
}
|
|
38
50
|
this.socketQueries[socketId] ??= {};
|
|
39
|
-
|
|
51
|
+
let errored = false;
|
|
52
|
+
const socketQuery = {
|
|
40
53
|
...query,
|
|
41
54
|
poolClient: undefined,
|
|
42
55
|
stream: undefined,
|
|
56
|
+
onError: (rawError) => {
|
|
57
|
+
if (errored)
|
|
58
|
+
return;
|
|
59
|
+
errored = true;
|
|
60
|
+
const errorWithoutQuery = (0, dboBuilderUtils_1.getSerializedClientErrorFromPGError)(rawError, { type: "sql" });
|
|
61
|
+
// For some reason query is not present on the error object from sql stream mode
|
|
62
|
+
const error = { ...errorWithoutQuery, query: query.query };
|
|
63
|
+
socket.emit(channel, { type: "error", error });
|
|
64
|
+
},
|
|
43
65
|
};
|
|
66
|
+
this.socketQueries[socketId][id] ??= socketQuery;
|
|
44
67
|
const { options } = query;
|
|
45
68
|
let processID = -1;
|
|
46
69
|
const startStream = async () => {
|
|
@@ -55,20 +78,14 @@ class QueryStreamer {
|
|
|
55
78
|
const batchSize = 10000;
|
|
56
79
|
let stream;
|
|
57
80
|
let poolClient;
|
|
58
|
-
const emit = (type,
|
|
81
|
+
const emit = (type, stream) => {
|
|
59
82
|
const result = stream?._result;
|
|
60
83
|
let packet;
|
|
61
84
|
const ended = type === "ended";
|
|
62
85
|
if (finished)
|
|
63
86
|
return;
|
|
64
87
|
finished = finished || ended;
|
|
65
|
-
if (
|
|
66
|
-
const errorWithoutQuery = (0, dboBuilderUtils_1.getSerializedClientErrorFromPGError)(rawError, { type: "sql" });
|
|
67
|
-
// For some reason query is not present on the error object from sql stream mode
|
|
68
|
-
const error = { ...errorWithoutQuery, query: query.query };
|
|
69
|
-
packet = { type: "error", error };
|
|
70
|
-
}
|
|
71
|
-
else if (!emittedPackets) {
|
|
88
|
+
if (!emittedPackets) {
|
|
72
89
|
if (!result?.fields)
|
|
73
90
|
throw "No fields";
|
|
74
91
|
const fields = runSQL_1.getDetailedFieldInfo.bind(this.dboBuilder)(result.fields);
|
|
@@ -101,19 +118,19 @@ class QueryStreamer {
|
|
|
101
118
|
batchRows.push(data);
|
|
102
119
|
if (options?.streamLimit) {
|
|
103
120
|
if (batchRows.length >= options.streamLimit) {
|
|
104
|
-
emit("ended",
|
|
121
|
+
emit("ended", stream);
|
|
105
122
|
}
|
|
106
123
|
}
|
|
107
124
|
if (batchRows.length >= batchSize) {
|
|
108
|
-
emit("rows",
|
|
125
|
+
emit("rows", stream);
|
|
109
126
|
batchRows = [];
|
|
110
127
|
}
|
|
111
128
|
});
|
|
112
129
|
stream.on('error', error => {
|
|
113
|
-
|
|
130
|
+
socketQuery.onError(error);
|
|
114
131
|
});
|
|
115
132
|
stream.on('end', () => {
|
|
116
|
-
emit("ended",
|
|
133
|
+
emit("ended", stream);
|
|
117
134
|
// release the client when the stream is finished
|
|
118
135
|
if (!options?.persistStreamConnection) {
|
|
119
136
|
delete this.socketQueries[socketId]?.[id];
|
|
@@ -123,11 +140,11 @@ class QueryStreamer {
|
|
|
123
140
|
});
|
|
124
141
|
}
|
|
125
142
|
catch (err) {
|
|
126
|
-
|
|
143
|
+
socketQuery.onError(err);
|
|
127
144
|
}
|
|
128
145
|
})();
|
|
129
146
|
};
|
|
130
|
-
const stop = (
|
|
147
|
+
const stop = (opts, cb) => {
|
|
131
148
|
// Must kill query if not ended
|
|
132
149
|
const { stream, poolClient } = this.socketQueries[socketId]?.[id] ?? {};
|
|
133
150
|
if (!stream || !poolClient)
|
|
@@ -137,7 +154,8 @@ class QueryStreamer {
|
|
|
137
154
|
return cb(null, err);
|
|
138
155
|
if (!client)
|
|
139
156
|
return cb(null, "No client");
|
|
140
|
-
|
|
157
|
+
const stopFunction = opts?.terminate ? "pg_terminate_backend" : "pg_cancel_backend";
|
|
158
|
+
client.query(`SELECT ${stopFunction}(pid) FROM pg_stat_activity WHERE pid = $1`, [processID], (err) => {
|
|
141
159
|
if (err) {
|
|
142
160
|
cb(null, err);
|
|
143
161
|
console.error(err);
|
|
@@ -168,7 +186,7 @@ class QueryStreamer {
|
|
|
168
186
|
setTimeout(() => {
|
|
169
187
|
if (started)
|
|
170
188
|
return;
|
|
171
|
-
stop(
|
|
189
|
+
stop({}, () => {
|
|
172
190
|
// empty
|
|
173
191
|
});
|
|
174
192
|
}, 5e3);
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"QueryStreamer.js","sourceRoot":"","sources":["../../lib/DboBuilder/QueryStreamer.ts"],"names":[],"mappings":";;;AACA,yBAAyB;AACzB,
|
|
1
|
+
{"version":3,"file":"QueryStreamer.js","sourceRoot":"","sources":["../../lib/DboBuilder/QueryStreamer.ts"],"names":[],"mappings":";;;AACA,yBAAyB;AACzB,qDAAqG;AAErG,uDAAwE;AACxE,qCAAqE;AAIrE,MAAM,WAAW,GAA2B,OAAO,CAAC,iBAAiB,CAAC,CAAC;AAgBvE,MAAM,cAAc,GAA2B,EAAE,CAAC;AAClD,MAAM,mBAAmB,GAAG,CAAC,QAAgB,EAAE,EAAE;IAC/C,MAAM,OAAO,GAAG,QAAQ,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;IACrC,MAAM,MAAM,GAAG,cAAc,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;IAC5C,MAAM,KAAK,GAAG,MAAM,GAAG,CAAC,CAAC;IACzB,cAAc,CAAC,OAAO,CAAC,GAAG,KAAK,CAAC;IAChC,OAAO,KAAK,CAAC;AACf,CAAC,CAAC;AAEF,MAAa,aAAa;IACxB,EAAE,CAAK;IACP,UAAU,CAAa;IACvB,aAAa,GAAkD,EAAE,CAAC;IAClE,IAAI,CAAU;IACd,SAAS,CAAU;IACnB,YAAY,UAAsB;QAChC,IAAI,CAAC,UAAU,GAAG,UAAU,CAAC;QAC7B,IAAI,CAAC,EAAE,GAAG,UAAU,CAAC,EAAE,CAAC;QACxB,MAAM,cAAc,GAAG,OAAO,IAAI,CAAC,EAAE,CAAC,GAAG,KAAK,QAAQ,CAAA,CAAC,CAAC,EAAE,gBAAgB,EAAE,IAAI,CAAC,EAAE,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,GAAU,CAAC;QAC/G,MAAM,WAAW,GAAG,CAAC,GAAU,EAAE,OAAsB,EAAE,EAAE;YACzD,OAAO,CAAC,KAAK,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;QAC7B,CAAC,CAAA;QACD,IAAI,CAAC,IAAI,GAAG,IAAI,EAAE,CAAC,IAAI,CAAC,EAAE,GAAG,cAAc,EAAE,GAAG,EAAE,EAAE,EAAE,CAAC,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,KAAK,EAAE,EAAE;YAC5E,wDAAwD;YACxD,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,QAAQ,EAAE,OAAO,CAAC,EAAE,EAAE;gBACjE,MAAM,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,KAAK,CAAC,EAAE,EAAE;oBAC9C,KAAK,CAAC,OAAO,EAAE,CAAC,EAAE,OAAO,EAAE,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC;oBAC5C,OAAO,IAAI,CAAC,aAAa,CAAC,QAAQ,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC;gBAC5C,CAAC,CAAC,CAAC;YACL,CAAC,CAAC,CAAC;QACL,CAAC,CAAC,CAAC;QACH,IAAI,CAAC,SAAS,GAAG,IAAI,EAAE,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC,EAAE,CAAC,OAAO,EAAE,WAAW,CAAC,CAAC;IACxE,CAAC;IAED,MAAM,GAAG,KAAK,EAAE,KAA4B,EAAkC,EAAE;QAC9E,MAAM,EAAE,MAAM,EAAE,iBAAiB,EAAE,GAAG,KAAK,CAAC;QAC5C,MAAM,QAAQ,GAAG,MAAM,CAAC,EAAE,CAAC;QAC3B,MAAM,EAAE,GAAG,mBAAmB,CAAC,QAAQ,CAAC,CAAC;QACzC,MAAM,OAAO,GAAG,GAAG,0BAAQ,CAAC,UAAU,KAAK,QAAQ,IAAI,EAAE,EAAE,CAAC;QAC5D,MAAM,YAAY,GAAG,GAAG,OAAO,cAAc,CAAC;QAC9C,IAAG,IAAI,CAAC,aAAa,CAAC,EAAE,CAAC,IAAI,CAAC,iBAAiB,EAAC;YAC9C,MAAM,4BAA4B,EAAE,QAAQ,CAAC;SAC9C;QAED,IAAI,CAAC,aAAa,CAAC,QAAQ,CAAC,KAAK,EAAE,CAAC;QACpC,IAAI,OAAO,GAAG,KAAK,CAAC;QACpB,MAAM,WAAW,GAAG;YAClB,GAAG,KAAK;YACR,UAAU,EAAE,SAAS;YACrB,MAAM,EAAE,SAAS;YACjB,OAAO,EAAE,CAAC,QAAa,EAAE,EAAE;gBACzB,IAAG,OAAO;oBAAE,OAAO;gBACnB,OAAO,GAAG,IAAI,CAAC;gBAEf,MAAM,iBAAiB,GAAG,IAAA,qDAAmC,EAAC,QAAQ,EAAE,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,CAAC;gBACzF,gFAAgF;gBAChF,MAAM,KAAK,GAAG,EAAE,GAAG,iBAAiB,EAAE,KAAK,EAAE,KAAK,CAAC,KAAK,EAAE,CAAC;gBAC3D,MAAM,CAAC,IAAI,CAAC,OAAO,EAAE,EAAE,IAAI,EAAE,OAAO,EAAE,KAAK,EAAkC,CAAC,CAAC;YACjF,CAAC;SACF,CAAC;QACF,IAAI,CAAC,aAAa,CAAC,QAAQ,CAAE,CAAC,EAAE,CAAC,KAAK,WAAW,CAAC;QAClD,MAAM,EAAE,OAAO,EAAE,GAAG,KAAK,CAAA;QACzB,IAAI,SAAS,GAAG,CAAC,CAAC,CAAC;QACnB,MAAM,WAAW,GAAG,KAAK,IAAI,EAAE;YAC7B,MAAM,WAAW,GAAG,IAAI,CAAC,aAAa,CAAC,QAAQ,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC;YACvD,IAAG,CAAC,WAAW,EAAC;gBACd,MAAM,wBAAwB,CAAC;aAChC;YACD,CAAC,KAAK,IAAI,EAAE;gBACV,IAAI,cAAc,GAAG,CAAC,CAAC;gBACvB,IAAI,SAAS,GAAU,EAAE,CAAC;gBAC1B,IAAI,QAAQ,GAAG,KAAK,CAAC;gBACrB,MAAM,SAAS,GAAG,KAAK,CAAC;gBACxB,IAAI,MAAuB,CAAC;gBAC5B,IAAI,UAAyB,CAAC;gBAC9B,MAAM,IAAI,GAAG,CAAC,IAAsB,EAAE,MAAmC,EAAE,EAAE;oBAC3E,MAAM,MAAM,GAAG,MAAM,EAAE,OAAyD,CAAC;oBACjF,IAAI,MAAyC,CAAC;oBAC9C,MAAM,KAAK,GAAG,IAAI,KAAK,OAAO,CAAC;oBAC/B,IAAG,QAAQ;wBAAE,OAAO;oBACpB,QAAQ,GAAG,QAAQ,IAAI,KAAK,CAAC;oBAC7B,IAAI,CAAC,cAAc,EAAE;wBACnB,IAAG,CAAC,MAAM,EAAE,MAAM;4BAAE,MAAM,WAAW,CAAC;wBACtC,MAAM,MAAM,GAAG,6BAAoB,CAAC,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC,MAAM,CAAC,MAAa,CAAC,CAAC;wBAChF,MAAM,GAAG,EAAE,IAAI,EAAE,OAAO,EAAE,IAAI,EAAE,SAAS,EAAE,MAAM,EAAE,KAAK,EAAE,SAAS,EAAE,SAAS,EAAE,CAAC;qBAClF;yBAAM;wBACL,MAAM,GAAG,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,SAAS,EAAE,KAAK,EAAE,CAAC;qBACnD;oBACD,MAAM,CAAC,IAAI,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;oBAC7B,IAAG,KAAK,EAAC;wBACP,IAAG,CAAC,MAAM;4BAAE,MAAM,gBAAgB,CAAC;wBACnC,4BAAmB,CAAC,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC,EAAE,eAAe,EAAE,KAAK,CAAC,KAAK,EAAE,OAAO,EAAE,MAAM,CAAC,OAAO,EAAE,CAAC,CAAC;qBACtG;oBACD,cAAc,EAAE,CAAC;gBACnB,CAAC,CAAA;gBAED,IAAI;oBAEF,MAAM,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,GAAG,EAAE,MAAM,EAAE,IAAI,EAAE,EAAE;wBAC5C,IAAI,GAAG;4BAAE,MAAM,GAAG,CAAC;wBACnB,IAAI,CAAC,MAAM;4BAAE,MAAM,WAAW,CAAC;wBAC/B,UAAU,GAAG,MAAM,CAAC;wBACpB,SAAS,GAAI,MAAc,CAAC,SAAS,CAAA;wBACrC,MAAM,WAAW,GAAG,IAAI,WAAW,CAAC,KAAK,CAAC,KAAK,EAAE,EAAE,EAAE,EAAE,SAAS,EAAE,OAAO,EAAE,OAAO,EAAE,CAAC,CAAC;wBACtF,MAAM,GAAG,MAAM,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC;wBACnC,IAAI,CAAC,aAAa,CAAC,QAAQ,CAAE,CAAC,EAAE,CAAE,CAAC,UAAU,GAAG,UAAU,CAAC;wBAC3D,IAAI,CAAC,aAAa,CAAC,QAAQ,CAAE,CAAC,EAAE,CAAE,CAAC,MAAM,GAAG,MAAM,CAAC;wBACnD,MAAM,CAAC,EAAE,CAAC,MAAM,EAAE,KAAK,EAAE,IAAI,EAAE,EAAE;4BAC/B,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;4BACrB,IAAG,OAAO,EAAE,WAAW,EAAE;gCACvB,IAAG,SAAS,CAAC,MAAM,IAAI,OAAO,CAAC,WAAW,EAAC;oCACzC,IAAI,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;iCACvB;6BACF;4BACD,IAAI,SAAS,CAAC,MAAM,IAAI,SAAS,EAAE;gCACjC,IAAI,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;gCACrB,SAAS,GAAG,EAAE,CAAC;6BAChB;wBACH,CAAC,CAAC,CAAC;wBACH,MAAM,CAAC,EAAE,CAAC,OAAO,EAAE,KAAK,CAAC,EAAE;4BACzB,WAAW,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;wBAC7B,CAAC,CAAC,CAAC;wBAEH,MAAM,CAAC,EAAE,CAAC,KAAK,EAAE,GAAG,EAAE;4BACpB,IAAI,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;4BACtB,iDAAiD;4BACjD,IAAG,CAAC,OAAO,EAAE,uBAAuB,EAAC;gCACnC,OAAO,IAAI,CAAC,aAAa,CAAC,QAAQ,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC;gCAC1C,IAAI,EAAE,CAAC;6BACR;wBACH,CAAC,CAAC,CAAA;oBACJ,CAAC,CAAC,CAAC;iBACJ;gBAAC,OAAM,GAAG,EAAC;oBACV,WAAW,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;iBAC1B;YACH,CAAC,CAAC,EAAE,CAAC;QACP,CAAC,CAAA;QAED,MAAM,IAAI,GAAG,CAAC,IAA0C,EAAE,EAAiB,EAAE,EAAE;YAC7E,+BAA+B;YAC/B,MAAM,EAAE,MAAM,EAAE,UAAU,EAAE,GAAG,IAAI,CAAC,aAAa,CAAC,QAAQ,CAAC,EAAE,CAAC,EAAE,CAAC,IAAI,EAAE,CAAC;YACxE,IAAG,CAAC,MAAM,IAAI,CAAC,UAAU;gBAAE,OAAO;YAClC,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,KAAK,EAAE,GAAG,EAAE,MAAM,EAAE,IAAI,EAAE,EAAE;gBACjD,IAAI,GAAG;oBAAE,OAAO,EAAE,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC;gBAC9B,IAAI,CAAC,MAAM;oBAAE,OAAO,EAAE,CAAC,IAAI,EAAE,WAAW,CAAC,CAAC;gBAC1C,MAAM,YAAY,GAAG,IAAI,EAAE,SAAS,CAAA,CAAC,CAAC,sBAAsB,CAAC,CAAC,CAAC,mBAAmB,CAAC;gBACnF,MAAM,CAAC,KAAK,CAAC,UAAU,YAAY,4CAA4C,EAAE,CAAC,SAAS,CAAC,EAAE,CAAC,GAAG,EAAE,EAAE;oBACpG,IAAG,GAAG,EAAE;wBACN,EAAE,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC;wBACd,OAAO,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;qBACpB;oBACD,OAAO,IAAI,CAAC,aAAa,CAAC,QAAQ,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC;oBAC1C,UAAU,CAAC,OAAO,EAAE,CAAC;oBACrB,MAAM,CAAC,kBAAkB,CAAC,YAAY,CAAC,CAAC;oBACxC,MAAM,CAAC,kBAAkB,CAAC,OAAO,CAAC,CAAC;oBACnC,IAAI,EAAE,CAAC;gBACT,CAAC,CAAC,CAAC;YACL,CAAC,CAAC,CAAC;QACL,CAAC,CAAA;QAED,MAAM,CAAC,kBAAkB,CAAC,YAAY,CAAC,CAAC;QACxC,MAAM,CAAC,IAAI,CAAC,YAAY,EAAE,IAAI,CAAC,CAAC;QAEhC,IAAI,OAAO,GAAG,KAAK,CAAC;QACpB,MAAM,CAAC,kBAAkB,CAAC,OAAO,CAAC,CAAC;QACnC,MAAM,CAAC,IAAI,CAAC,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,EAAE,EAAE,EAAE;YACvC,OAAO,GAAG,IAAI,CAAC;YACf,IAAI;gBACF,MAAM,WAAW,EAAE,CAAC;gBACpB,EAAE,EAAE,CAAC;aACN;YAAC,OAAM,GAAG,EAAC;gBACV,EAAE,CAAC,IAAI,EAAE,GAAG,IAAI,sBAAsB,CAAC,CAAC;aACzC;QACH,CAAC,CAAC,CAAC;QAEH,qEAAqE;QACrE,UAAU,CAAC,GAAG,EAAE;YACd,IAAG,OAAO;gBAAE,OAAO;YACnB,IAAI,CAAC,EAAE,EAAE,GAAG,EAAE;gBACZ,SAAS;YACX,CAAC,CAAC,CAAC;QACL,CAAC,EAAE,GAAG,CAAC,CAAC;QAER,OAAO;YACL,OAAO;YACP,YAAY;SACb,CAAA;IACH,CAAC,CAAA;CACF;AAnLD,sCAmLC"}
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { DB } from "../initProstgles";
|
|
2
2
|
import * as pg from "pg";
|
|
3
|
-
import {
|
|
3
|
+
import { CHANNELS, SQLOptions, SocketSQLStreamPacket, SocketSQLStreamServer } from "prostgles-types";
|
|
4
4
|
import { PRGLIOSocket } from "./DboBuilderTypes";
|
|
5
5
|
import { getSerializedClientErrorFromPGError } from "./dboBuilderUtils";
|
|
6
6
|
import { getDetailedFieldInfo, watchSchemaFallback } from "./runSQL";
|
|
@@ -20,6 +20,7 @@ type ClientStreamedRequest = {
|
|
|
20
20
|
type StreamedQuery = ClientStreamedRequest & {
|
|
21
21
|
stream: QueryStreamType | undefined;
|
|
22
22
|
poolClient: pg.PoolClient | undefined;
|
|
23
|
+
onError: ((error: any) => void);
|
|
23
24
|
}
|
|
24
25
|
|
|
25
26
|
const shortSocketIds: Record<string, number> = {};
|
|
@@ -40,8 +41,20 @@ export class QueryStreamer {
|
|
|
40
41
|
constructor(dboBuilder: DboBuilder) {
|
|
41
42
|
this.dboBuilder = dboBuilder;
|
|
42
43
|
this.db = dboBuilder.db;
|
|
43
|
-
|
|
44
|
-
|
|
44
|
+
const connectionInfo = typeof this.db.$cn === "string"? { connectionString: this.db.$cn } : this.db.$cn as any;
|
|
45
|
+
const onPoolError = (err: Error, _client: pg.PoolClient) => {
|
|
46
|
+
console.error(err.message);
|
|
47
|
+
}
|
|
48
|
+
this.pool = new pg.Pool({ ...connectionInfo, max: 50 }).on("error", (error) => {
|
|
49
|
+
// if(error.message !== "Connection terminated") return;
|
|
50
|
+
Object.entries(this.socketQueries).forEach(([socketId, queries]) => {
|
|
51
|
+
Object.entries(queries).forEach(([id, query]) => {
|
|
52
|
+
query.onError?.({ message: error.message });
|
|
53
|
+
delete this.socketQueries[socketId]?.[id];
|
|
54
|
+
});
|
|
55
|
+
});
|
|
56
|
+
});
|
|
57
|
+
this.adminPool = new pg.Pool(connectionInfo).on("error", onPoolError);
|
|
45
58
|
}
|
|
46
59
|
|
|
47
60
|
create = async (query: ClientStreamedRequest): Promise<SocketSQLStreamServer> => {
|
|
@@ -54,12 +67,23 @@ export class QueryStreamer {
|
|
|
54
67
|
throw `Must stop existing query ${id} first`;
|
|
55
68
|
}
|
|
56
69
|
|
|
57
|
-
this.socketQueries[socketId] ??= {}
|
|
58
|
-
|
|
70
|
+
this.socketQueries[socketId] ??= {};
|
|
71
|
+
let errored = false;
|
|
72
|
+
const socketQuery = {
|
|
59
73
|
...query,
|
|
60
74
|
poolClient: undefined,
|
|
61
75
|
stream: undefined,
|
|
76
|
+
onError: (rawError: any) => {
|
|
77
|
+
if(errored) return;
|
|
78
|
+
errored = true;
|
|
79
|
+
|
|
80
|
+
const errorWithoutQuery = getSerializedClientErrorFromPGError(rawError, { type: "sql" });
|
|
81
|
+
// For some reason query is not present on the error object from sql stream mode
|
|
82
|
+
const error = { ...errorWithoutQuery, query: query.query };
|
|
83
|
+
socket.emit(channel, { type: "error", error } satisfies SocketSQLStreamPacket);
|
|
84
|
+
},
|
|
62
85
|
};
|
|
86
|
+
this.socketQueries[socketId]![id] ??= socketQuery;
|
|
63
87
|
const { options } = query
|
|
64
88
|
let processID = -1;
|
|
65
89
|
const startStream = async () => {
|
|
@@ -74,18 +98,13 @@ export class QueryStreamer {
|
|
|
74
98
|
const batchSize = 10000;
|
|
75
99
|
let stream: QueryStreamType;
|
|
76
100
|
let poolClient: pg.PoolClient;
|
|
77
|
-
const emit = (type: "rows" | "
|
|
101
|
+
const emit = (type: "rows" | "ended", stream: QueryStreamType | undefined) => {
|
|
78
102
|
const result = stream?._result as { command: string; fields: any[] } | undefined;
|
|
79
103
|
let packet: SocketSQLStreamPacket | undefined;
|
|
80
104
|
const ended = type === "ended";
|
|
81
105
|
if(finished) return;
|
|
82
106
|
finished = finished || ended;
|
|
83
|
-
if(
|
|
84
|
-
const errorWithoutQuery = getSerializedClientErrorFromPGError(rawError, { type: "sql" });
|
|
85
|
-
// For some reason query is not present on the error object from sql stream mode
|
|
86
|
-
const error = { ...errorWithoutQuery, query: query.query };
|
|
87
|
-
packet = { type: "error", error };
|
|
88
|
-
} else if (!emittedPackets) {
|
|
107
|
+
if (!emittedPackets) {
|
|
89
108
|
if(!result?.fields) throw "No fields";
|
|
90
109
|
const fields = getDetailedFieldInfo.bind(this.dboBuilder)(result.fields as any);
|
|
91
110
|
packet = { type: "start", rows: batchRows, fields, ended, processId: processID };
|
|
@@ -115,20 +134,20 @@ export class QueryStreamer {
|
|
|
115
134
|
batchRows.push(data);
|
|
116
135
|
if(options?.streamLimit) {
|
|
117
136
|
if(batchRows.length >= options.streamLimit){
|
|
118
|
-
emit("ended",
|
|
137
|
+
emit("ended", stream);
|
|
119
138
|
}
|
|
120
139
|
}
|
|
121
140
|
if (batchRows.length >= batchSize) {
|
|
122
|
-
emit("rows",
|
|
141
|
+
emit("rows", stream);
|
|
123
142
|
batchRows = [];
|
|
124
143
|
}
|
|
125
144
|
});
|
|
126
145
|
stream.on('error', error => {
|
|
127
|
-
|
|
146
|
+
socketQuery.onError(error);
|
|
128
147
|
});
|
|
129
148
|
|
|
130
149
|
stream.on('end', () => {
|
|
131
|
-
emit("ended",
|
|
150
|
+
emit("ended", stream);
|
|
132
151
|
// release the client when the stream is finished
|
|
133
152
|
if(!options?.persistStreamConnection){
|
|
134
153
|
delete this.socketQueries[socketId]?.[id];
|
|
@@ -137,19 +156,20 @@ export class QueryStreamer {
|
|
|
137
156
|
})
|
|
138
157
|
});
|
|
139
158
|
} catch(err){
|
|
140
|
-
|
|
159
|
+
socketQuery.onError(err);
|
|
141
160
|
}
|
|
142
161
|
})();
|
|
143
162
|
}
|
|
144
163
|
|
|
145
|
-
const stop = (
|
|
164
|
+
const stop = (opts: { terminate?: boolean; } | undefined, cb: BasicCallback) => {
|
|
146
165
|
// Must kill query if not ended
|
|
147
166
|
const { stream, poolClient } = this.socketQueries[socketId]?.[id] ?? {};
|
|
148
167
|
if(!stream || !poolClient) return;
|
|
149
168
|
this.adminPool.connect(async (err, client, done) => {
|
|
150
169
|
if (err) return cb(null, err);
|
|
151
170
|
if (!client) return cb(null, "No client");
|
|
152
|
-
|
|
171
|
+
const stopFunction = opts?.terminate? "pg_terminate_backend" : "pg_cancel_backend";
|
|
172
|
+
client.query(`SELECT ${stopFunction}(pid) FROM pg_stat_activity WHERE pid = $1`, [processID], (err) => {
|
|
153
173
|
if(err) {
|
|
154
174
|
cb(null, err);
|
|
155
175
|
console.error(err);
|
|
@@ -181,7 +201,7 @@ export class QueryStreamer {
|
|
|
181
201
|
/** If not started in 5 seconds then assume this will never happen */
|
|
182
202
|
setTimeout(() => {
|
|
183
203
|
if(started) return;
|
|
184
|
-
stop(
|
|
204
|
+
stop({}, () => {
|
|
185
205
|
// empty
|
|
186
206
|
});
|
|
187
207
|
}, 5e3);
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "prostgles-server",
|
|
3
|
-
"version": "4.1.
|
|
3
|
+
"version": "4.1.128",
|
|
4
4
|
"description": "",
|
|
5
5
|
"main": "dist/index.js",
|
|
6
6
|
"types": "dist/index.d.ts",
|
|
@@ -46,7 +46,7 @@
|
|
|
46
46
|
"pg-promise": "^11.5.4",
|
|
47
47
|
"pg-query-stream": "^4.5.3",
|
|
48
48
|
"prostgles-client": "^4.0.53",
|
|
49
|
-
"prostgles-types": "^4.0.
|
|
49
|
+
"prostgles-types": "^4.0.66"
|
|
50
50
|
},
|
|
51
51
|
"devDependencies": {
|
|
52
52
|
"@types/bluebird": "^3.5.36",
|
package/tests/client/index.ts
CHANGED
|
@@ -92,13 +92,17 @@ try {
|
|
|
92
92
|
onReady: async (db, methods, tableSchema, auth, isReconnect) => {
|
|
93
93
|
log(`TEST_NAME: ${TEST_NAME} Started`)
|
|
94
94
|
try {
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
95
|
+
if(typeof window !== "undefined"){
|
|
96
|
+
const onLog = (...args: any[]) => {
|
|
97
|
+
socket.emit("log", args.map(v => typeof v === "object"? JSON.stringify(v) : v).join(" "));
|
|
98
|
+
}
|
|
99
|
+
window.onerror = function myErrorHandler(errorMsg, url, lineNumber) {
|
|
100
|
+
console.error("Error occured: " + errorMsg);
|
|
101
|
+
stopTest({ err: errorMsg });
|
|
102
|
+
return false;
|
|
103
|
+
}
|
|
104
|
+
console.log = onLog;
|
|
105
|
+
}
|
|
102
106
|
await test.onRun(db, methods, tableSchema, auth, isReconnect);
|
|
103
107
|
|
|
104
108
|
stopTest();
|
|
@@ -10,7 +10,7 @@
|
|
|
10
10
|
"license": "ISC",
|
|
11
11
|
"dependencies": {
|
|
12
12
|
"@types/node": "^14.14.16",
|
|
13
|
-
"prostgles-client": "^4.0.
|
|
13
|
+
"prostgles-client": "^4.0.61",
|
|
14
14
|
"prostgles-types": "^4.0.51",
|
|
15
15
|
"socket.io-client": "^4.7.1"
|
|
16
16
|
},
|
|
@@ -75,11 +75,11 @@
|
|
|
75
75
|
"integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w=="
|
|
76
76
|
},
|
|
77
77
|
"node_modules/prostgles-client": {
|
|
78
|
-
"version": "4.0.
|
|
79
|
-
"resolved": "https://registry.npmjs.org/prostgles-client/-/prostgles-client-4.0.
|
|
80
|
-
"integrity": "sha512-
|
|
78
|
+
"version": "4.0.61",
|
|
79
|
+
"resolved": "https://registry.npmjs.org/prostgles-client/-/prostgles-client-4.0.61.tgz",
|
|
80
|
+
"integrity": "sha512-AEgiO5erJnDde0o5hSd1ztKGRWPK2OVCKeVYCbEDGHYIcZzvBA1jRwxpreGaqENZDrWix5LJHyTTTZuLr+LhCw==",
|
|
81
81
|
"dependencies": {
|
|
82
|
-
"prostgles-types": "^4.0.
|
|
82
|
+
"prostgles-types": "^4.0.66"
|
|
83
83
|
},
|
|
84
84
|
"peerDependencies": {
|
|
85
85
|
"react": "^16.8.0 || ^17.0.0 || ^18.0.0"
|
|
@@ -91,9 +91,9 @@
|
|
|
91
91
|
}
|
|
92
92
|
},
|
|
93
93
|
"node_modules/prostgles-types": {
|
|
94
|
-
"version": "4.0.
|
|
95
|
-
"resolved": "https://registry.npmjs.org/prostgles-types/-/prostgles-types-4.0.
|
|
96
|
-
"integrity": "sha512-
|
|
94
|
+
"version": "4.0.66",
|
|
95
|
+
"resolved": "https://registry.npmjs.org/prostgles-types/-/prostgles-types-4.0.66.tgz",
|
|
96
|
+
"integrity": "sha512-nDMoWmXmHPBFRK1KJaf7ttfy1VTiB072CtxZoY6LExPPKVCjnJ5PBafIEG1N6opa1WFaU0ez4WTm9Ykc4AOmAQ==",
|
|
97
97
|
"dependencies": {
|
|
98
98
|
"json-schema": "^0.4.0"
|
|
99
99
|
}
|
|
@@ -213,17 +213,17 @@
|
|
|
213
213
|
"integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w=="
|
|
214
214
|
},
|
|
215
215
|
"prostgles-client": {
|
|
216
|
-
"version": "4.0.
|
|
217
|
-
"resolved": "https://registry.npmjs.org/prostgles-client/-/prostgles-client-4.0.
|
|
218
|
-
"integrity": "sha512-
|
|
216
|
+
"version": "4.0.61",
|
|
217
|
+
"resolved": "https://registry.npmjs.org/prostgles-client/-/prostgles-client-4.0.61.tgz",
|
|
218
|
+
"integrity": "sha512-AEgiO5erJnDde0o5hSd1ztKGRWPK2OVCKeVYCbEDGHYIcZzvBA1jRwxpreGaqENZDrWix5LJHyTTTZuLr+LhCw==",
|
|
219
219
|
"requires": {
|
|
220
|
-
"prostgles-types": "^4.0.
|
|
220
|
+
"prostgles-types": "^4.0.66"
|
|
221
221
|
}
|
|
222
222
|
},
|
|
223
223
|
"prostgles-types": {
|
|
224
|
-
"version": "4.0.
|
|
225
|
-
"resolved": "https://registry.npmjs.org/prostgles-types/-/prostgles-types-4.0.
|
|
226
|
-
"integrity": "sha512-
|
|
224
|
+
"version": "4.0.66",
|
|
225
|
+
"resolved": "https://registry.npmjs.org/prostgles-types/-/prostgles-types-4.0.66.tgz",
|
|
226
|
+
"integrity": "sha512-nDMoWmXmHPBFRK1KJaf7ttfy1VTiB072CtxZoY6LExPPKVCjnJ5PBafIEG1N6opa1WFaU0ez4WTm9Ykc4AOmAQ==",
|
|
227
227
|
"requires": {
|
|
228
228
|
"json-schema": "^0.4.0"
|
|
229
229
|
}
|
|
@@ -7,13 +7,13 @@
|
|
|
7
7
|
"start": "tsc-watch --onSuccess \"node dist/client/index.js\"",
|
|
8
8
|
"dev": "NOSTOP=true npm start",
|
|
9
9
|
"build": "rm -rf ./node_modules/* && rm -rf ./dist/* && npm i && tsc",
|
|
10
|
-
"test": "node
|
|
10
|
+
"test": "node dist/client/index.js"
|
|
11
11
|
},
|
|
12
12
|
"author": "",
|
|
13
13
|
"license": "ISC",
|
|
14
14
|
"dependencies": {
|
|
15
15
|
"@types/node": "^14.14.16",
|
|
16
|
-
"prostgles-client": "^4.0.
|
|
16
|
+
"prostgles-client": "^4.0.61",
|
|
17
17
|
"prostgles-types": "^4.0.51",
|
|
18
18
|
"socket.io-client": "^4.7.1"
|
|
19
19
|
},
|
|
@@ -134,6 +134,34 @@ export default async function client_only(db: DBHandlerClient, auth: Auth, log:
|
|
|
134
134
|
startHandler.stop().catch(reject);
|
|
135
135
|
}, 1000);
|
|
136
136
|
});
|
|
137
|
+
// await tryRunP("SQL Stream stop with terminate kills the query", async (resolve, reject) => {
|
|
138
|
+
// const totalRows = 5e6;
|
|
139
|
+
// const query = `SELECT * FROM generate_series(1, ${totalRows})`;
|
|
140
|
+
// const res = await db.sql!(query, {}, { returnType: "stream" });
|
|
141
|
+
// const rowsReceived: any[] = [];
|
|
142
|
+
// const listener = async (packet: SocketSQLStreamPacket) => {
|
|
143
|
+
// if(packet.type === "error"){
|
|
144
|
+
// const queryState = await db.sql!("SELECT * FROM pg_stat_activity WHERE query = $1", [query], { returnType: "rows" });
|
|
145
|
+
// assert.equal(queryState.length, 1);
|
|
146
|
+
// assert.equal(queryState[0].state, "idle");
|
|
147
|
+
// assert.equal(packet.error.message, "canceling statement due to user request");
|
|
148
|
+
// resolve("ok");
|
|
149
|
+
// } else {
|
|
150
|
+
// try {
|
|
151
|
+
// rowsReceived.push(...packet.rows);
|
|
152
|
+
// console.log(rowsReceived.length)
|
|
153
|
+
// assert.equal(packet.ended, false);
|
|
154
|
+
// assert.equal(rowsReceived.length < totalRows, true);
|
|
155
|
+
// } catch(error){
|
|
156
|
+
// reject(error);
|
|
157
|
+
// }
|
|
158
|
+
// }
|
|
159
|
+
// };
|
|
160
|
+
// const startHandler = await res.start(listener);
|
|
161
|
+
// setTimeout(() => {
|
|
162
|
+
// startHandler.stop(true).catch(reject);
|
|
163
|
+
// }, 22);
|
|
164
|
+
// });
|
|
137
165
|
|
|
138
166
|
// await tryRunP("SQL Stream ensure the connection is never released (same pg_backend_pid is the same for subsequent) queries when using persistConnectionId", async (resolve, reject) => {
|
|
139
167
|
// const res = await db.sql!("SELECT pg_backend_pid()", {}, { returnType: "stream", persistConnectionId: true });
|
package/tests/server/index.ts
CHANGED
|
@@ -115,6 +115,9 @@ prostgles<DBSchemaGenerated>({
|
|
|
115
115
|
if(clientTest){
|
|
116
116
|
log("Client connected -> console does not work. use log function. socket.id:", socket.id);
|
|
117
117
|
socket.emit("start-test", { server_id: Math.random() });
|
|
118
|
+
socket.on("log", async (data, cb) => {
|
|
119
|
+
console.log("Client log ", data);
|
|
120
|
+
});
|
|
118
121
|
socket.on("stop-test", async (err, cb) => {
|
|
119
122
|
cb();
|
|
120
123
|
console.log("Client test " + (!err? "successful" : "failed"));
|
|
@@ -21,7 +21,7 @@
|
|
|
21
21
|
},
|
|
22
22
|
"../..": {
|
|
23
23
|
"name": "prostgles-server",
|
|
24
|
-
"version": "4.1.
|
|
24
|
+
"version": "4.1.127",
|
|
25
25
|
"license": "MIT",
|
|
26
26
|
"dependencies": {
|
|
27
27
|
"@types/express": "^4.17.13",
|
|
@@ -35,7 +35,7 @@
|
|
|
35
35
|
"pg-promise": "^11.5.4",
|
|
36
36
|
"pg-query-stream": "^4.5.3",
|
|
37
37
|
"prostgles-client": "^4.0.53",
|
|
38
|
-
"prostgles-types": "^4.0.
|
|
38
|
+
"prostgles-types": "^4.0.66"
|
|
39
39
|
},
|
|
40
40
|
"devDependencies": {
|
|
41
41
|
"@types/bluebird": "^3.5.36",
|
|
@@ -1546,7 +1546,7 @@
|
|
|
1546
1546
|
"pg-promise": "^11.5.4",
|
|
1547
1547
|
"pg-query-stream": "^4.5.3",
|
|
1548
1548
|
"prostgles-client": "^4.0.53",
|
|
1549
|
-
"prostgles-types": "^4.0.
|
|
1549
|
+
"prostgles-types": "^4.0.66",
|
|
1550
1550
|
"typescript": "^5.2.2"
|
|
1551
1551
|
}
|
|
1552
1552
|
},
|