prostgles-server 4.1.131 → 4.1.132
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/DboBuilder/QueryStreamer.d.ts.map +1 -1
- package/dist/DboBuilder/QueryStreamer.js +22 -12
- package/dist/DboBuilder/QueryStreamer.js.map +1 -1
- package/lib/DboBuilder/QueryStreamer.ts +24 -13
- package/package.json +1 -1
- package/tests/client_only_queries.ts +18 -0
- package/tests/server/package-lock.json +1 -1
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"QueryStreamer.d.ts","sourceRoot":"","sources":["../../lib/DboBuilder/QueryStreamer.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,EAAE,EAAE,MAAM,kBAAkB,CAAC;AACtC,OAAO,KAAK,EAAE,MAAM,IAAI,CAAC;AACzB,OAAO,EAAY,UAAU,EAAyB,qBAAqB,
|
|
1
|
+
{"version":3,"file":"QueryStreamer.d.ts","sourceRoot":"","sources":["../../lib/DboBuilder/QueryStreamer.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,EAAE,EAAE,MAAM,kBAAkB,CAAC;AACtC,OAAO,KAAK,EAAE,MAAM,IAAI,CAAC;AACzB,OAAO,EAAY,UAAU,EAAyB,qBAAqB,EAAY,MAAM,iBAAiB,CAAC;AAC/G,OAAO,EAAE,YAAY,EAAE,MAAM,mBAAmB,CAAC;AAGjD,OAAO,EAAE,UAAU,EAAE,MAAM,cAAc,CAAC;AAC1C,OAAO,eAAe,MAAM,iBAAiB,CAAC;AAM9C,KAAK,qBAAqB,GAAG;IAC3B,MAAM,EAAE,YAAY,CAAC;IACrB,KAAK,EAAE,MAAM,CAAC;IACd,OAAO,EAAE,UAAU,GAAG,SAAS,CAAC;IAChC,iBAAiB,CAAC,EAAE,OAAO,CAAC;CAE7B,CAAA;AACD,KAAK,aAAa,GAAG,qBAAqB,GAAG;IAC3C,MAAM,EAAE,eAAe,GAAG,SAAS,CAAC;IACpC,MAAM,EAAE,EAAE,CAAC,MAAM,GAAG,SAAS,CAAC;IAC9B,OAAO,EAAE,CAAC,CAAC,KAAK,EAAE,GAAG,KAAK,IAAI,CAAC,CAAC;CACjC,CAAA;AAWD,qBAAa,aAAa;IACxB,EAAE,EAAE,EAAE,CAAC;IACP,UAAU,EAAE,UAAU,CAAC;IACvB,aAAa,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,aAAa,CAAC,CAAC,CAAM;IAClE,WAAW,EAAE,EAAE,CAAC,MAAM,CAAC;gBACX,UAAU,EAAE,UAAU;IAclC,aAAa,mBAAoB,GAAG,KAAK,IAAI,8BAA8B,GAAG,YAAY,eAOzF;IAED,YAAY,aAAc,MAAM,UAO/B;IAED,MAAM,UAAiB,qBAAqB,KAAG,QAAQ,qBAAqB,CAAC,CAoJ5E;CACF"}
|
|
@@ -86,21 +86,20 @@ class QueryStreamer {
|
|
|
86
86
|
let finished = false;
|
|
87
87
|
const batchSize = 10000;
|
|
88
88
|
let stream;
|
|
89
|
-
const emit = (type,
|
|
90
|
-
const result = stream?._result;
|
|
89
|
+
const emit = (type, streamResult) => {
|
|
91
90
|
const ended = type === "ended";
|
|
92
91
|
if (finished)
|
|
93
92
|
return;
|
|
94
93
|
finished = finished || ended;
|
|
95
|
-
if (!
|
|
94
|
+
if (!streamResult?.fields)
|
|
96
95
|
throw "No fields";
|
|
97
|
-
const fields = runSQL_1.getDetailedFieldInfo.bind(this.dboBuilder)(
|
|
98
|
-
const packet = { type: "data", rows: batchRows, fields, ended, processId: processID };
|
|
96
|
+
const fields = runSQL_1.getDetailedFieldInfo.bind(this.dboBuilder)(streamResult.fields);
|
|
97
|
+
const packet = { type: "data", rows: batchRows, fields, info: streamResult, ended, processId: processID };
|
|
99
98
|
socket.emit(channel, packet);
|
|
100
99
|
if (ended) {
|
|
101
|
-
if (!
|
|
100
|
+
if (!streamResult)
|
|
102
101
|
throw "No result info";
|
|
103
|
-
runSQL_1.watchSchemaFallback.bind(this.dboBuilder)({ queryWithoutRLS: query.query, command:
|
|
102
|
+
runSQL_1.watchSchemaFallback.bind(this.dboBuilder)({ queryWithoutRLS: query.query, command: streamResult.command });
|
|
104
103
|
}
|
|
105
104
|
};
|
|
106
105
|
const client = this.getConnection(err => {
|
|
@@ -110,27 +109,38 @@ class QueryStreamer {
|
|
|
110
109
|
try {
|
|
111
110
|
await client.connect();
|
|
112
111
|
processID = client.processID;
|
|
113
|
-
const queryStream = new QueryStream(query.query,
|
|
112
|
+
const queryStream = new QueryStream(query.query, undefined, { batchSize: 1e6, highWaterMark: 1e6, rowMode: "array" });
|
|
114
113
|
stream = client.query(queryStream);
|
|
115
114
|
this.socketQueries[socketId][id].client = client;
|
|
116
115
|
this.socketQueries[socketId][id].stream = stream;
|
|
116
|
+
const getStreamResult = () => (0, prostgles_types_1.omitKeys)(stream._result, ["rows"]);
|
|
117
117
|
stream.on('data', async (data) => {
|
|
118
118
|
batchRows.push(data);
|
|
119
119
|
if (options?.streamLimit) {
|
|
120
120
|
if (batchRows.length >= options.streamLimit) {
|
|
121
|
-
emit("ended",
|
|
121
|
+
emit("ended", getStreamResult());
|
|
122
122
|
}
|
|
123
123
|
}
|
|
124
124
|
if (batchRows.length >= batchSize) {
|
|
125
|
-
emit("rows",
|
|
125
|
+
emit("rows", getStreamResult());
|
|
126
126
|
batchRows = [];
|
|
127
127
|
}
|
|
128
128
|
});
|
|
129
129
|
stream.on('error', error => {
|
|
130
|
-
|
|
130
|
+
if (error.message === "cannot insert multiple commands into a prepared statement") {
|
|
131
|
+
this.dboBuilder.dbo.sql(query.query, {}, { returnType: "arrayMode", hasParams: false }).then(res => {
|
|
132
|
+
batchRows.push(res.rows);
|
|
133
|
+
emit("ended", res);
|
|
134
|
+
}).catch(newError => {
|
|
135
|
+
socketQuery.onError(newError);
|
|
136
|
+
});
|
|
137
|
+
}
|
|
138
|
+
else {
|
|
139
|
+
socketQuery.onError(error);
|
|
140
|
+
}
|
|
131
141
|
});
|
|
132
142
|
stream.on('end', () => {
|
|
133
|
-
emit("ended",
|
|
143
|
+
emit("ended", getStreamResult());
|
|
134
144
|
// release the client when the stream is finished AND connection is not persisted
|
|
135
145
|
if (!options?.persistStreamConnection) {
|
|
136
146
|
delete this.socketQueries[socketId]?.[id];
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"QueryStreamer.js","sourceRoot":"","sources":["../../lib/DboBuilder/QueryStreamer.ts"],"names":[],"mappings":";;;AACA,yBAAyB;AACzB,
|
|
1
|
+
{"version":3,"file":"QueryStreamer.js","sourceRoot":"","sources":["../../lib/DboBuilder/QueryStreamer.ts"],"names":[],"mappings":";;;AACA,yBAAyB;AACzB,qDAA+G;AAE/G,uDAAwE;AACxE,qCAAqE;AAKrE,MAAM,WAAW,GAA2B,OAAO,CAAC,iBAAiB,CAAC,CAAC;AAgBvE,MAAM,cAAc,GAA2B,EAAE,CAAC;AAClD,MAAM,mBAAmB,GAAG,CAAC,QAAgB,EAAE,EAAE;IAC/C,MAAM,OAAO,GAAG,QAAQ,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;IACrC,MAAM,MAAM,GAAG,cAAc,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;IAC5C,MAAM,KAAK,GAAG,MAAM,GAAG,CAAC,CAAC;IACzB,cAAc,CAAC,OAAO,CAAC,GAAG,KAAK,CAAC;IAChC,OAAO,KAAK,CAAC;AACf,CAAC,CAAC;AAEF,MAAa,aAAa;IACxB,EAAE,CAAK;IACP,UAAU,CAAa;IACvB,aAAa,GAAkD,EAAE,CAAC;IAClE,WAAW,CAAY;IACvB,YAAY,UAAsB;QAChC,IAAI,CAAC,UAAU,GAAG,UAAU,CAAC;QAC7B,IAAI,CAAC,EAAE,GAAG,UAAU,CAAC,EAAE,CAAC;QACxB,MAAM,cAAc,GAAG,GAAG,EAAE;YAC1B,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC,aAAa,CAAC,SAAS,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;YACtE,IAAI,CAAC,WAAW,CAAC,OAAO,EAAE,CAAC;QAC7B,CAAC,CAAA;QACD,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC,aAAa,CAAC,CAAC,KAAK,EAAE,EAAE;YAC9C,OAAO,CAAC,GAAG,CAAC,qCAAqC,EAAE,KAAK,CAAC,CAAC;YAC1D,cAAc,EAAE,CAAC;QACnB,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QACxB,IAAI,CAAC,WAAW,CAAC,OAAO,EAAE,CAAC;IAC7B,CAAC;IAED,aAAa,GAAG,CAAC,OAAyC,EAAE,YAA8B,EAAE,EAAE;QAC5F,MAAM,cAAc,GAAG,OAAO,IAAI,CAAC,EAAE,CAAC,GAAG,KAAK,QAAQ,CAAA,CAAC,CAAC,EAAE,gBAAgB,EAAE,IAAI,CAAC,EAAE,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,GAAU,CAAC;QAC/G,MAAM,MAAM,GAAG,IAAI,EAAE,CAAC,MAAM,CAAC,EAAE,GAAG,cAAc,EAAE,GAAG,YAAY,EAAE,CAAC,CAAC;QACrE,MAAM,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,GAAG,EAAE,EAAE;YACzB,OAAO,EAAE,CAAC,GAAG,CAAC,CAAC;QACjB,CAAC,CAAC,CAAC;QACH,OAAO,MAAM,CAAC;IAChB,CAAC,CAAA;IAED,YAAY,GAAG,CAAC,QAAgB,EAAE,EAAE;QAClC,MAAM,aAAa,GAAG,IAAI,CAAC,aAAa,CAAC,QAAQ,CAAC,CAAC;QACnD,IAAG,CAAC,aAAa;YAAE,OAAO;QAC1B,MAAM,CAAC,MAAM,CAAC,aAAa,CAAC,CAAC,OAAO,CAAC,CAAC,EAAE,MAAM,EAAE,EAAE,EAAE;YAClD,MAAM,EAAE,GAAG,EAAE,CAAC;QAChB,CAAC,CAAC,CAAC;QACH,OAAO,IAAI,CAAC,aAAa,CAAC,QAAQ,CAAC,CAAC;IACtC,CAAC,CAAA;IAED,MAAM,GAAG,KAAK,EAAE,KAA4B,EAAkC,EAAE;QAC9E,MAAM,EAAE,MAAM,EAAE,iBAAiB,EAAE,GAAG,KAAK,CAAC;QAC5C,MAAM,QAAQ,GAAG,MAAM,CAAC,EAAE,CAAC;QAC3B,MAAM,EAAE,GAAG,mBAAmB,CAAC,QAAQ,CAAC,CAAC;QACzC,MAAM,OAAO,GAAG,GAAG,0BAAQ,CAAC,UAAU,KAAK,QAAQ,IAAI,EAAE,EAAE,CAAC;QAC5D,MAAM,YAAY,GAAG,GAAG,OAAO,cAAc,CAAC;QAC9C,IAAG,IAAI,CAAC,aAAa,CAAC,EAAE,CAAC,IAAI,CAAC,iBAAiB,EAAC;YAC9C,MAAM,4BAA4B,EAAE,QAAQ,CAAC;SAC9C;QAED,IAAI,CAAC,aAAa,CAAC,QAAQ,CAAC,KAAK,EAAE,CAAC;QACpC,IAAI,OAAO,GAAG,KAAK,CAAC;QACpB,MAAM,WAAW,GAAG;YAClB,GAAG,KAAK;YACR,MAAM,EAAE,SAAS;YACjB,MAAM,EAAE,SAAS;YACjB,OAAO,EAAE,CAAC,QAAa,EAAE,EAAE;gBACzB,IAAG,OAAO;oBAAE,OAAO;gBACnB,OAAO,GAAG,IAAI,CAAC;gBAEf,MAAM,iBAAiB,GAAG,IAAA,qDAAmC,EAAC,QAAQ,EAAE,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,CAAC;gBACzF,gFAAgF;gBAChF,MAAM,KAAK,GAAG,EAAE,GAAG,iBAAiB,EAAE,KAAK,EAAE,KAAK,CAAC,KAAK,EAAE,CAAC;gBAC3D,MAAM,CAAC,IAAI,CAAC,OAAO,EAAE,EAAE,IAAI,EAAE,OAAO,EAAE,KAAK,EAAkC,CAAC,CAAC;YACjF,CAAC;SACF,CAAC;QACF,IAAI,CAAC,aAAa,CAAC,QAAQ,CAAE,CAAC,EAAE,CAAC,KAAK,WAAW,CAAC;QAClD,MAAM,EAAE,OAAO,EAAE,GAAG,KAAK,CAAA;QACzB,IAAI,SAAS,GAAG,CAAC,CAAC,CAAC;QAEnB,MAAM,WAAW,GAAG,KAAK,IAAI,EAAE;YAC7B,MAAM,WAAW,GAAG,IAAI,CAAC,aAAa,CAAC,QAAQ,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC;YACvD,IAAG,CAAC,WAAW,EAAC;gBACd,MAAM,wBAAwB,CAAC;aAChC;YACD,IAAI,SAAS,GAAU,EAAE,CAAC;YAC1B,IAAI,QAAQ,GAAG,KAAK,CAAC;YACrB,MAAM,SAAS,GAAG,KAAK,CAAC;YACxB,IAAI,MAAuB,CAAC;YAE5B,MAAM,IAAI,GAAG,CAAC,IAAsB,EAAE,YAA8B,EAAE,EAAE;gBACtE,MAAM,KAAK,GAAG,IAAI,KAAK,OAAO,CAAC;gBAC/B,IAAG,QAAQ;oBAAE,OAAO;gBACpB,QAAQ,GAAG,QAAQ,IAAI,KAAK,CAAC;gBAC7B,IAAG,CAAC,YAAY,EAAE,MAAM;oBAAE,MAAM,WAAW,CAAC;gBAC5C,MAAM,MAAM,GAAG,6BAAoB,CAAC,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC,YAAY,CAAC,MAAa,CAAC,CAAC;gBACtF,MAAM,MAAM,GAA0B,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,SAAS,EAAE,MAAM,EAAE,IAAI,EAAE,YAAY,EAAE,KAAK,EAAE,SAAS,EAAE,SAAS,EAAE,CAAC;gBACjI,MAAM,CAAC,IAAI,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;gBAC7B,IAAG,KAAK,EAAC;oBACP,IAAG,CAAC,YAAY;wBAAE,MAAM,gBAAgB,CAAC;oBACzC,4BAAmB,CAAC,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC,EAAE,eAAe,EAAE,KAAK,CAAC,KAAK,EAAE,OAAO,EAAE,YAAY,CAAC,OAAO,EAAE,CAAC,CAAC;iBAC5G;YACH,CAAC,CAAA;YACD,MAAM,MAAM,GAAG,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,EAAE;gBACtC,WAAW,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;gBACzB,MAAM,CAAC,GAAG,EAAE,CAAC;YACf,CAAC,CAAC,CAAC;YACH,IAAI;gBACF,MAAM,MAAM,CAAC,OAAO,EAAE,CAAC;gBACvB,SAAS,GAAI,MAAc,CAAC,SAAS,CAAA;gBACrC,MAAM,WAAW,GAAG,IAAI,WAAW,CAAC,KAAK,CAAC,KAAK,EAAE,SAAS,EAAE,EAAE,SAAS,EAAE,GAAG,EAAE,aAAa,EAAE,GAAG,EAAE,OAAO,EAAE,OAAO,EAAE,CAAC,CAAC;gBACtH,MAAM,GAAG,MAAM,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC;gBACnC,IAAI,CAAC,aAAa,CAAC,QAAQ,CAAE,CAAC,EAAE,CAAE,CAAC,MAAM,GAAG,MAAM,CAAC;gBACnD,IAAI,CAAC,aAAa,CAAC,QAAQ,CAAE,CAAC,EAAE,CAAE,CAAC,MAAM,GAAG,MAAM,CAAC;gBACnD,MAAM,eAAe,GAAG,GAAG,EAAE,CAAC,IAAA,0BAAQ,EAAC,MAAM,CAAC,OAAO,EAAE,CAAC,MAAM,CAAC,CAAS,CAAC;gBACzE,MAAM,CAAC,EAAE,CAAC,MAAM,EAAE,KAAK,EAAE,IAAI,EAAE,EAAE;oBAC/B,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;oBACrB,IAAG,OAAO,EAAE,WAAW,EAAE;wBACvB,IAAG,SAAS,CAAC,MAAM,IAAI,OAAO,CAAC,WAAW,EAAC;4BACzC,IAAI,CAAC,OAAO,EAAE,eAAe,EAAE,CAAC,CAAC;yBAClC;qBACF;oBACD,IAAI,SAAS,CAAC,MAAM,IAAI,SAAS,EAAE;wBACjC,IAAI,CAAC,MAAM,EAAE,eAAe,EAAE,CAAC,CAAC;wBAChC,SAAS,GAAG,EAAE,CAAC;qBAChB;gBACH,CAAC,CAAC,CAAC;gBACH,MAAM,CAAC,EAAE,CAAC,OAAO,EAAE,KAAK,CAAC,EAAE;oBACzB,IAAG,KAAK,CAAC,OAAO,KAAK,2DAA2D,EAAE;wBAChF,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,GAAI,CAAC,KAAK,CAAC,KAAK,EAAE,EAAE,EAAE,EAAE,UAAU,EAAE,WAAW,EAAE,SAAS,EAAE,KAAK,EAAE,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE;4BAClG,SAAS,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;4BACzB,IAAI,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC;wBACrB,CAAC,CAAC,CAAC,KAAK,CAAC,QAAQ,CAAC,EAAE;4BAClB,WAAW,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC;wBAChC,CAAC,CAAC,CAAC;qBACJ;yBAAM;wBACL,WAAW,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;qBAC5B;gBACH,CAAC,CAAC,CAAC;gBACH,MAAM,CAAC,EAAE,CAAC,KAAK,EAAE,GAAG,EAAE;oBACpB,IAAI,CAAC,OAAO,EAAE,eAAe,EAAE,CAAC,CAAC;oBACjC,iFAAiF;oBACjF,IAAG,CAAC,OAAO,EAAE,uBAAuB,EAAC;wBACnC,OAAO,IAAI,CAAC,aAAa,CAAC,QAAQ,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC;wBAC1C,MAAM,CAAC,GAAG,EAAE,CAAC;qBACd;gBACH,CAAC,CAAC,CAAC;aACJ;YAAC,OAAM,GAAG,EAAC;gBACV,WAAW,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;gBACzB,MAAM,MAAM,CAAC,GAAG,EAAE,CAAC;aACpB;QACH,CAAC,CAAA;QAED,MAAM,IAAI,GAAG,KAAK,EAAE,IAA0C,EAAE,EAAiB,EAAE,EAAE;YACnF,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,WAAW,EAAE,GAAG,IAAI,CAAC,aAAa,CAAC,QAAQ,CAAC,EAAE,CAAC,EAAE,CAAC,IAAI,EAAE,CAAC;YACjF,IAAG,CAAC,MAAM,IAAI,CAAC,WAAW;gBAAE,OAAO;YACnC,IAAI;gBACF,MAAM,YAAY,GAAG,IAAI,EAAE,SAAS,CAAA,CAAC,CAAC,sBAAsB,CAAC,CAAC,CAAC,mBAAmB,CAAC;gBACnF,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,WAAW,CAAC,KAAK,CAAC,UAAU,YAAY,8EAA8E,EAAE,CAAC,SAAS,EAAE,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC;gBAC1K,MAAM,CAAC,kBAAkB,CAAC,YAAY,CAAC,CAAC;gBACxC,MAAM,CAAC,kBAAkB,CAAC,OAAO,CAAC,CAAC;gBACnC,EAAE,CAAC,EAAE,SAAS,EAAE,IAAI,EAAE,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;aACvC;YAAC,OAAO,KAAK,EAAC;gBACb,EAAE,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC;aACjB;QACH,CAAC,CAAA;QAED,MAAM,CAAC,kBAAkB,CAAC,YAAY,CAAC,CAAC;QACxC,MAAM,CAAC,IAAI,CAAC,YAAY,EAAE,IAAI,CAAC,CAAC;QAEhC,IAAI,OAAO,GAAG,KAAK,CAAC;QACpB,MAAM,CAAC,kBAAkB,CAAC,OAAO,CAAC,CAAC;QACnC,MAAM,CAAC,EAAE,CAAC,OAAO,EAAE,KAAK,EAAE,KAAiD,EAAE,EAAiB,EAAE,EAAE;YAChG,IAAG,OAAO,EAAC;gBACT,OAAO;gBACP,OAAO,EAAE,CAAC,IAAI,EAAE,iBAAiB,CAAC,CAAC;aACpC;YACD,OAAO,GAAG,IAAI,CAAC;YACf,IAAI;gBACF,MAAM,WAAW,EAAE,CAAC;gBACpB,EAAE,EAAE,CAAC;aACN;YAAC,OAAM,GAAG,EAAC;gBACV,EAAE,CAAC,IAAI,EAAE,GAAG,IAAI,sBAAsB,CAAC,CAAC;aACzC;QACH,CAAC,CAAC,CAAC;QAEH,qEAAqE;QACrE,UAAU,CAAC,GAAG,EAAE;YACd,IAAG,OAAO;gBAAE,OAAO;YACnB,IAAI,CAAC,EAAE,EAAE,GAAG,EAAE;gBACZ,SAAS;YACX,CAAC,CAAC,CAAC;QACL,CAAC,EAAE,GAAG,CAAC,CAAC;QAER,OAAO;YACL,OAAO;YACP,YAAY;SACb,CAAA;IACH,CAAC,CAAA;CACF;AA1LD,sCA0LC"}
|
|
@@ -1,12 +1,13 @@
|
|
|
1
1
|
import { DB } from "../initProstgles";
|
|
2
2
|
import * as pg from "pg";
|
|
3
|
-
import { CHANNELS, SQLOptions, SocketSQLStreamPacket, SocketSQLStreamServer } from "prostgles-types";
|
|
3
|
+
import { CHANNELS, SQLOptions, SocketSQLStreamPacket, SocketSQLStreamServer, omitKeys } from "prostgles-types";
|
|
4
4
|
import { PRGLIOSocket } from "./DboBuilderTypes";
|
|
5
5
|
import { getSerializedClientErrorFromPGError } from "./dboBuilderUtils";
|
|
6
6
|
import { getDetailedFieldInfo, watchSchemaFallback } from "./runSQL";
|
|
7
7
|
import { DboBuilder } from "./DboBuilder";
|
|
8
8
|
import QueryStreamType from 'pg-query-stream';
|
|
9
9
|
import { BasicCallback } from "../PubSubManager/PubSubManager";
|
|
10
|
+
import e from "express";
|
|
10
11
|
const QueryStream: typeof QueryStreamType = require('pg-query-stream');
|
|
11
12
|
|
|
12
13
|
|
|
@@ -108,18 +109,18 @@ export class QueryStreamer {
|
|
|
108
109
|
let finished = false;
|
|
109
110
|
const batchSize = 10000;
|
|
110
111
|
let stream: QueryStreamType;
|
|
111
|
-
|
|
112
|
-
|
|
112
|
+
type Info = { command: string; fields: any[]; rowCount: number; duration: number; }
|
|
113
|
+
const emit = (type: "rows" | "ended", streamResult: Info | undefined) => {
|
|
113
114
|
const ended = type === "ended";
|
|
114
115
|
if(finished) return;
|
|
115
116
|
finished = finished || ended;
|
|
116
|
-
if(!
|
|
117
|
-
const fields = getDetailedFieldInfo.bind(this.dboBuilder)(
|
|
118
|
-
const packet: SocketSQLStreamPacket = { type: "data", rows: batchRows, fields, ended, processId: processID };
|
|
117
|
+
if(!streamResult?.fields) throw "No fields";
|
|
118
|
+
const fields = getDetailedFieldInfo.bind(this.dboBuilder)(streamResult.fields as any);
|
|
119
|
+
const packet: SocketSQLStreamPacket = { type: "data", rows: batchRows, fields, info: streamResult, ended, processId: processID };
|
|
119
120
|
socket.emit(channel, packet);
|
|
120
121
|
if(ended){
|
|
121
|
-
if(!
|
|
122
|
-
watchSchemaFallback.bind(this.dboBuilder)({ queryWithoutRLS: query.query, command:
|
|
122
|
+
if(!streamResult) throw "No result info";
|
|
123
|
+
watchSchemaFallback.bind(this.dboBuilder)({ queryWithoutRLS: query.query, command: streamResult.command });
|
|
123
124
|
}
|
|
124
125
|
}
|
|
125
126
|
const client = this.getConnection(err => {
|
|
@@ -129,27 +130,37 @@ export class QueryStreamer {
|
|
|
129
130
|
try {
|
|
130
131
|
await client.connect();
|
|
131
132
|
processID = (client as any).processID
|
|
132
|
-
const queryStream = new QueryStream(query.query,
|
|
133
|
+
const queryStream = new QueryStream(query.query, undefined, { batchSize: 1e6, highWaterMark: 1e6, rowMode: "array" });
|
|
133
134
|
stream = client.query(queryStream);
|
|
134
135
|
this.socketQueries[socketId]![id]!.client = client;
|
|
135
136
|
this.socketQueries[socketId]![id]!.stream = stream;
|
|
137
|
+
const getStreamResult = () => omitKeys(stream._result, ["rows"]) as Info;
|
|
136
138
|
stream.on('data', async (data) => {
|
|
137
139
|
batchRows.push(data);
|
|
138
140
|
if(options?.streamLimit) {
|
|
139
141
|
if(batchRows.length >= options.streamLimit){
|
|
140
|
-
emit("ended",
|
|
142
|
+
emit("ended", getStreamResult());
|
|
141
143
|
}
|
|
142
144
|
}
|
|
143
145
|
if (batchRows.length >= batchSize) {
|
|
144
|
-
emit("rows",
|
|
146
|
+
emit("rows", getStreamResult());
|
|
145
147
|
batchRows = [];
|
|
146
148
|
}
|
|
147
149
|
});
|
|
148
150
|
stream.on('error', error => {
|
|
149
|
-
|
|
151
|
+
if(error.message === "cannot insert multiple commands into a prepared statement") {
|
|
152
|
+
this.dboBuilder.dbo.sql!(query.query, {}, { returnType: "arrayMode", hasParams: false }).then(res => {
|
|
153
|
+
batchRows.push(res.rows);
|
|
154
|
+
emit("ended", res);
|
|
155
|
+
}).catch(newError => {
|
|
156
|
+
socketQuery.onError(newError);
|
|
157
|
+
});
|
|
158
|
+
} else {
|
|
159
|
+
socketQuery.onError(error);
|
|
160
|
+
}
|
|
150
161
|
});
|
|
151
162
|
stream.on('end', () => {
|
|
152
|
-
emit("ended",
|
|
163
|
+
emit("ended", getStreamResult());
|
|
153
164
|
// release the client when the stream is finished AND connection is not persisted
|
|
154
165
|
if(!options?.persistStreamConnection){
|
|
155
166
|
delete this.socketQueries[socketId]?.[id];
|
package/package.json
CHANGED
|
@@ -162,6 +162,24 @@ export default async function client_only(db: DBHandlerClient, auth: Auth, log:
|
|
|
162
162
|
};
|
|
163
163
|
await res.start(listener);
|
|
164
164
|
});
|
|
165
|
+
await tryRunP("SQL Stream works for multiple statements", async (resolve, reject) => {
|
|
166
|
+
const res = await db.sql!("SELECT * FROM planes; SELECT 1 as a", {}, { returnType: "stream" });
|
|
167
|
+
const listener = async (packet: SocketSQLStreamPacket) => {
|
|
168
|
+
if(packet.type === "error"){
|
|
169
|
+
reject(packet.error);
|
|
170
|
+
} else {
|
|
171
|
+
assert.equal(packet.type, "data");
|
|
172
|
+
assert.equal(packet.ended, true);
|
|
173
|
+
assert.equal(packet.rows.length, 1);
|
|
174
|
+
const normalSql = await db.sql!("SELECT 1 as a", {});
|
|
175
|
+
await db.sql!("DELETE FROM planes", {});
|
|
176
|
+
assert.deepStrictEqual(packet.fields, normalSql.fields);
|
|
177
|
+
assert.equal(packet.fields.length > 0, true);
|
|
178
|
+
resolve("ok");
|
|
179
|
+
}
|
|
180
|
+
};
|
|
181
|
+
await res.start(listener);
|
|
182
|
+
});
|
|
165
183
|
|
|
166
184
|
// await tryRunP("SQL Stream ensure the connection is never released (same pg_backend_pid is the same for subsequent) queries when using persistConnectionId", async (resolve, reject) => {
|
|
167
185
|
// const res = await db.sql!("SELECT pg_backend_pid()", {}, { returnType: "stream", persistConnectionId: true });
|