pgserve 1.1.5 → 1.1.7-rc.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.husky/pre-commit +2 -2
- package/README.md +3 -3
- package/bun.lock +8 -8
- package/package.json +5 -5
- package/src/cluster.js +80 -50
- package/src/postgres.js +132 -1
- package/src/router.js +79 -59
- package/tests/backpressure.test.js +167 -0
package/.husky/pre-commit
CHANGED
|
@@ -1,2 +1,2 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
1
|
+
bunx lint-staged
|
|
2
|
+
bun run deadcode
|
package/README.md
CHANGED
|
@@ -5,7 +5,7 @@
|
|
|
5
5
|
<p>
|
|
6
6
|
<a href="https://www.npmjs.com/package/pgserve"><img src="https://img.shields.io/npm/v/pgserve?style=flat-square&color=00D9FF" alt="npm version"></a>
|
|
7
7
|
<img src="https://img.shields.io/badge/node-%3E%3D18-green?style=flat-square" alt="Node.js">
|
|
8
|
-
<img src="https://img.shields.io/badge/PostgreSQL-
|
|
8
|
+
<img src="https://img.shields.io/badge/PostgreSQL-18-blue?style=flat-square" alt="PostgreSQL">
|
|
9
9
|
<a href="LICENSE"><img src="https://img.shields.io/badge/license-MIT-green?style=flat-square" alt="License"></a>
|
|
10
10
|
<a href="https://discord.gg/xcW8c7fF3R"><img src="https://img.shields.io/discord/1095114867012292758?style=flat-square&color=00D9FF&label=discord" alt="Discord"></a>
|
|
11
11
|
</p>
|
|
@@ -41,7 +41,7 @@ psql postgresql://localhost:8432/myapp
|
|
|
41
41
|
|
|
42
42
|
<table>
|
|
43
43
|
<tr>
|
|
44
|
-
<td><b>Real PostgreSQL
|
|
44
|
+
<td><b>Real PostgreSQL 18</b></td>
|
|
45
45
|
<td>Native binaries, not WASM — full compatibility, extensions support</td>
|
|
46
46
|
</tr>
|
|
47
47
|
<tr>
|
|
@@ -450,7 +450,7 @@ CREATE EXTENSION IF NOT EXISTS vector;
|
|
|
450
450
|
</tr>
|
|
451
451
|
</table>
|
|
452
452
|
|
|
453
|
-
> <b>Methodology:</b> Recall@k measured against brute-force ground truth (industry standard). PostgreSQL baseline is Docker <code>pgvector/pgvector:
|
|
453
|
+
> <b>Methodology:</b> Recall@k measured against brute-force ground truth (industry standard). PostgreSQL baseline is Docker <code>pgvector/pgvector:pg18</code>. RAM mode available on Linux and WSL2.
|
|
454
454
|
>
|
|
455
455
|
> Run benchmarks yourself: <code>bun tests/benchmarks/runner.js --include-vector</code>
|
|
456
456
|
|
package/bun.lock
CHANGED
|
@@ -17,23 +17,23 @@
|
|
|
17
17
|
"pg": "^8.16.3",
|
|
18
18
|
},
|
|
19
19
|
"optionalDependencies": {
|
|
20
|
-
"@embedded-postgres/darwin-arm64": "
|
|
21
|
-
"@embedded-postgres/darwin-x64": "
|
|
22
|
-
"@embedded-postgres/linux-x64": "
|
|
23
|
-
"@embedded-postgres/windows-x64": "
|
|
20
|
+
"@embedded-postgres/darwin-arm64": "18.2.0-beta.16",
|
|
21
|
+
"@embedded-postgres/darwin-x64": "18.2.0-beta.16",
|
|
22
|
+
"@embedded-postgres/linux-x64": "18.2.0-beta.16",
|
|
23
|
+
"@embedded-postgres/windows-x64": "18.2.0-beta.16",
|
|
24
24
|
},
|
|
25
25
|
},
|
|
26
26
|
},
|
|
27
27
|
"packages": {
|
|
28
28
|
"@electric-sql/pglite": ["@electric-sql/pglite@0.2.17", "", {}, "sha512-qEpKRT2oUaWDH6tjRxLHjdzMqRUGYDnGZlKrnL4dJ77JVMcP2Hpo3NYnOSPKdZdeec57B6QPprCUFg0picx5Pw=="],
|
|
29
29
|
|
|
30
|
-
"@embedded-postgres/darwin-arm64": ["@embedded-postgres/darwin-arm64@
|
|
30
|
+
"@embedded-postgres/darwin-arm64": ["@embedded-postgres/darwin-arm64@18.2.0-beta.16", "", { "os": "darwin", "cpu": "arm64" }, "sha512-wnswaF+uDvGeitqajJ8v8xOG4ttFrzixElwKNe2MIxBXSLWPV3xhi6tBY0Sjw8Lmiu6UG9vNLFZSjHPrIeokBg=="],
|
|
31
31
|
|
|
32
|
-
"@embedded-postgres/darwin-x64": ["@embedded-postgres/darwin-x64@
|
|
32
|
+
"@embedded-postgres/darwin-x64": ["@embedded-postgres/darwin-x64@18.2.0-beta.16", "", { "os": "darwin", "cpu": "x64" }, "sha512-u9WtTPxRuO0uOny5IniXHSDaLmtOujwzDoExIV/jFT0Fu8SzpX7wdoPbsSPBLgyQWdr/nPA77K9QI4r6P1/fKA=="],
|
|
33
33
|
|
|
34
|
-
"@embedded-postgres/linux-x64": ["@embedded-postgres/linux-x64@
|
|
34
|
+
"@embedded-postgres/linux-x64": ["@embedded-postgres/linux-x64@18.2.0-beta.16", "", { "os": "linux", "cpu": "x64" }, "sha512-BIt485ioL8/AwDgw37IcdraOfRgHNDOtGM6Hh63vnNaUAG4Z0qtJd5zXS5fr2wZTEsYHyC5PC60k7zkCRZXSzg=="],
|
|
35
35
|
|
|
36
|
-
"@embedded-postgres/windows-x64": ["@embedded-postgres/windows-x64@
|
|
36
|
+
"@embedded-postgres/windows-x64": ["@embedded-postgres/windows-x64@18.2.0-beta.16", "", { "os": "win32", "cpu": "x64" }, "sha512-Sj6GhCZrvtMwchATEtWuEmexEBWpRNMHPTUHsqPuyDrHX/XgKfpIxz2/AMHa4sp7SZ0JOHGouH8AFIVsWQrQsQ=="],
|
|
37
37
|
|
|
38
38
|
"@emnapi/core": ["@emnapi/core@1.7.1", "", { "dependencies": { "@emnapi/wasi-threads": "1.1.0", "tslib": "^2.4.0" } }, "sha512-o1uhUASyo921r2XtHYOHy7gdkGLge8ghBEQHMWmyJFoXlpU58kIrhhN3w26lpQb6dspetweapMn2CSNwQ8I4wg=="],
|
|
39
39
|
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "pgserve",
|
|
3
|
-
"version": "1.1.
|
|
3
|
+
"version": "1.1.7-rc.2",
|
|
4
4
|
"description": "Embedded PostgreSQL server with true concurrent connections - zero config, auto-provision databases",
|
|
5
5
|
"main": "src/index.js",
|
|
6
6
|
"type": "module",
|
|
@@ -41,10 +41,10 @@
|
|
|
41
41
|
},
|
|
42
42
|
"homepage": "https://github.com/namastexlabs/pgserve#readme",
|
|
43
43
|
"optionalDependencies": {
|
|
44
|
-
"@embedded-postgres/darwin-arm64": "
|
|
45
|
-
"@embedded-postgres/darwin-x64": "
|
|
46
|
-
"@embedded-postgres/linux-x64": "
|
|
47
|
-
"@embedded-postgres/windows-x64": "
|
|
44
|
+
"@embedded-postgres/darwin-arm64": "18.2.0-beta.16",
|
|
45
|
+
"@embedded-postgres/darwin-x64": "18.2.0-beta.16",
|
|
46
|
+
"@embedded-postgres/linux-x64": "18.2.0-beta.16",
|
|
47
|
+
"@embedded-postgres/windows-x64": "18.2.0-beta.16"
|
|
48
48
|
},
|
|
49
49
|
"devDependencies": {
|
|
50
50
|
"@electric-sql/pglite": "^0.2.17",
|
package/src/cluster.js
CHANGED
|
@@ -23,6 +23,17 @@ const SSL_REQUEST_CODE = 80877103;
|
|
|
23
23
|
const GSSAPI_REQUEST_CODE = 80877104;
|
|
24
24
|
const CANCEL_REQUEST_CODE = 80877102;
|
|
25
25
|
|
|
26
|
+
/**
|
|
27
|
+
* Attempt to write a pending buffer to a target socket.
|
|
28
|
+
* Returns remaining unwritten bytes, or null if fully flushed.
|
|
29
|
+
*/
|
|
30
|
+
function flushPending(target, pending) {
|
|
31
|
+
const written = target.write(pending);
|
|
32
|
+
if (written === pending.byteLength) return null;
|
|
33
|
+
if (written === 0) return pending;
|
|
34
|
+
return pending.subarray(written);
|
|
35
|
+
}
|
|
36
|
+
|
|
26
37
|
// Stats collection constants
|
|
27
38
|
const WORKER_STATS_TIMEOUT_MS = 10000; // Worker stats older than this are considered stale
|
|
28
39
|
const WORKER_STATS_REPORT_INTERVAL_MS = 4000; // How often workers report stats to primary
|
|
@@ -100,8 +111,14 @@ class ClusterRouter extends EventEmitter {
|
|
|
100
111
|
},
|
|
101
112
|
drain(socket) {
|
|
102
113
|
const state = router.socketState.get(socket);
|
|
103
|
-
if (state
|
|
104
|
-
|
|
114
|
+
if (!state) return;
|
|
115
|
+
// Flush any pending PG→Client data
|
|
116
|
+
if (state.pendingToClient) {
|
|
117
|
+
state.pendingToClient = flushPending(socket, state.pendingToClient);
|
|
118
|
+
}
|
|
119
|
+
// If fully flushed, resume reading from PostgreSQL
|
|
120
|
+
if (!state.pendingToClient && state.pgSocket) {
|
|
121
|
+
state.pgSocket.resume();
|
|
105
122
|
}
|
|
106
123
|
}
|
|
107
124
|
}
|
|
@@ -182,7 +199,9 @@ class ClusterRouter extends EventEmitter {
|
|
|
182
199
|
buffer: null,
|
|
183
200
|
pgSocket: null,
|
|
184
201
|
dbName: null,
|
|
185
|
-
handshakeComplete: false
|
|
202
|
+
handshakeComplete: false,
|
|
203
|
+
pendingToPg: null,
|
|
204
|
+
pendingToClient: null
|
|
186
205
|
});
|
|
187
206
|
this.connections.add(socket);
|
|
188
207
|
this.connectionStats.totalConnected++;
|
|
@@ -197,7 +216,17 @@ class ClusterRouter extends EventEmitter {
|
|
|
197
216
|
|
|
198
217
|
// If handshake complete, forward to PostgreSQL
|
|
199
218
|
if (state.handshakeComplete && state.pgSocket) {
|
|
200
|
-
|
|
219
|
+
// If there's already pending data, append to it
|
|
220
|
+
if (state.pendingToPg) {
|
|
221
|
+
state.pendingToPg = Buffer.concat([state.pendingToPg, data]);
|
|
222
|
+
return;
|
|
223
|
+
}
|
|
224
|
+
const written = state.pgSocket.write(data);
|
|
225
|
+
if (written < data.byteLength) {
|
|
226
|
+
// Partial write — buffer remainder and pause client
|
|
227
|
+
state.pendingToPg = written === 0 ? Buffer.from(data) : Buffer.from(data.subarray(written));
|
|
228
|
+
socket.pause();
|
|
229
|
+
}
|
|
201
230
|
return;
|
|
202
231
|
}
|
|
203
232
|
|
|
@@ -250,50 +279,47 @@ class ClusterRouter extends EventEmitter {
|
|
|
250
279
|
|
|
251
280
|
const router = this;
|
|
252
281
|
|
|
253
|
-
//
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
},
|
|
261
|
-
open(pgSocket) {
|
|
262
|
-
pgSocket.write(startupMessage);
|
|
263
|
-
state.handshakeComplete = true;
|
|
264
|
-
},
|
|
265
|
-
close(_pgSocket) {
|
|
266
|
-
socket.end();
|
|
267
|
-
},
|
|
268
|
-
error(_pgSocket, error) {
|
|
269
|
-
router.logger.error({ dbName, err: error }, 'PostgreSQL socket error');
|
|
270
|
-
socket.end();
|
|
271
|
-
},
|
|
272
|
-
drain(_pgSocket) {}
|
|
282
|
+
// Shared handler for pgSocket (used by both unix and TCP paths)
|
|
283
|
+
const pgHandler = {
|
|
284
|
+
data(_pgSocket, pgData) {
|
|
285
|
+
// Forward PostgreSQL response to client with backpressure
|
|
286
|
+
if (state.pendingToClient) {
|
|
287
|
+
state.pendingToClient = Buffer.concat([state.pendingToClient, pgData]);
|
|
288
|
+
return;
|
|
273
289
|
}
|
|
274
|
-
|
|
275
|
-
|
|
276
|
-
|
|
277
|
-
|
|
278
|
-
port: this.pgPort,
|
|
279
|
-
socket: {
|
|
280
|
-
data(_pgSocket, pgData) {
|
|
281
|
-
socket.write(pgData);
|
|
282
|
-
},
|
|
283
|
-
open(pgSocket) {
|
|
284
|
-
pgSocket.write(startupMessage);
|
|
285
|
-
state.handshakeComplete = true;
|
|
286
|
-
},
|
|
287
|
-
close(_pgSocket) {
|
|
288
|
-
socket.end();
|
|
289
|
-
},
|
|
290
|
-
error(_pgSocket, error) {
|
|
291
|
-
router.logger.error({ dbName, err: error }, 'PostgreSQL socket error');
|
|
292
|
-
socket.end();
|
|
293
|
-
},
|
|
294
|
-
drain(_pgSocket) {}
|
|
290
|
+
const written = socket.write(pgData);
|
|
291
|
+
if (written < pgData.byteLength) {
|
|
292
|
+
state.pendingToClient = written === 0 ? Buffer.from(pgData) : Buffer.from(pgData.subarray(written));
|
|
293
|
+
_pgSocket.pause();
|
|
295
294
|
}
|
|
296
|
-
}
|
|
295
|
+
},
|
|
296
|
+
open(pgSocket) {
|
|
297
|
+
pgSocket.write(startupMessage);
|
|
298
|
+
state.handshakeComplete = true;
|
|
299
|
+
},
|
|
300
|
+
close(_pgSocket) {
|
|
301
|
+
socket.end();
|
|
302
|
+
},
|
|
303
|
+
error(_pgSocket, error) {
|
|
304
|
+
router.logger.error({ dbName, err: error }, 'PostgreSQL socket error');
|
|
305
|
+
socket.end();
|
|
306
|
+
},
|
|
307
|
+
drain(_pgSocket) {
|
|
308
|
+
// Flush any pending Client→PG data
|
|
309
|
+
if (state.pendingToPg) {
|
|
310
|
+
state.pendingToPg = flushPending(_pgSocket, state.pendingToPg);
|
|
311
|
+
}
|
|
312
|
+
// If fully flushed, resume reading from client
|
|
313
|
+
if (!state.pendingToPg) {
|
|
314
|
+
socket.resume();
|
|
315
|
+
}
|
|
316
|
+
}
|
|
317
|
+
};
|
|
318
|
+
|
|
319
|
+
if (this.pgSocketPath) {
|
|
320
|
+
state.pgSocket = await Bun.connect({ unix: this.pgSocketPath, socket: pgHandler });
|
|
321
|
+
} else {
|
|
322
|
+
state.pgSocket = await Bun.connect({ hostname: '127.0.0.1', port: this.pgPort, socket: pgHandler });
|
|
297
323
|
}
|
|
298
324
|
} catch (error) {
|
|
299
325
|
this.logger.error({ dbName, err: error }, 'Connection error');
|
|
@@ -306,8 +332,10 @@ class ClusterRouter extends EventEmitter {
|
|
|
306
332
|
*/
|
|
307
333
|
handleSocketClose(socket) {
|
|
308
334
|
const state = this.socketState.get(socket);
|
|
309
|
-
if (state
|
|
310
|
-
state.
|
|
335
|
+
if (state) {
|
|
336
|
+
state.pendingToPg = null;
|
|
337
|
+
state.pendingToClient = null;
|
|
338
|
+
if (state.pgSocket) state.pgSocket.end();
|
|
311
339
|
}
|
|
312
340
|
this.connections.delete(socket);
|
|
313
341
|
this.socketState.delete(socket);
|
|
@@ -334,8 +362,10 @@ class ClusterRouter extends EventEmitter {
|
|
|
334
362
|
if (error.code !== 'ECONNRESET') {
|
|
335
363
|
this.logger.error({ err: error, dbName: state?.dbName }, 'Socket error');
|
|
336
364
|
}
|
|
337
|
-
if (state
|
|
338
|
-
state.
|
|
365
|
+
if (state) {
|
|
366
|
+
state.pendingToPg = null;
|
|
367
|
+
state.pendingToClient = null;
|
|
368
|
+
if (state.pgSocket) state.pgSocket.end();
|
|
339
369
|
}
|
|
340
370
|
this.connections.delete(socket);
|
|
341
371
|
this.socketState.delete(socket);
|
package/src/postgres.js
CHANGED
|
@@ -66,7 +66,7 @@ async function downloadPostgresBinaries() {
|
|
|
66
66
|
|
|
67
67
|
const platformKey = getPlatformKey();
|
|
68
68
|
const pkgName = `@embedded-postgres/${platformKey}`;
|
|
69
|
-
const pkgVersion = '
|
|
69
|
+
const pkgVersion = '18.2.0-beta.16';
|
|
70
70
|
|
|
71
71
|
console.log(`[pgserve] PostgreSQL binaries not found.`);
|
|
72
72
|
console.log(`[pgserve] Downloading ${pkgName}@${pkgVersion}...`);
|
|
@@ -509,6 +509,12 @@ export class PostgresManager {
|
|
|
509
509
|
persistent: this.persistent
|
|
510
510
|
}, 'PostgreSQL started successfully');
|
|
511
511
|
|
|
512
|
+
// Pre-install pgvector extension files if enabled
|
|
513
|
+
// This ensures vector.so + vector.control are ready before any CREATE EXTENSION call
|
|
514
|
+
if (this.enablePgvector) {
|
|
515
|
+
await this.ensurePgvectorFiles();
|
|
516
|
+
}
|
|
517
|
+
|
|
512
518
|
return this;
|
|
513
519
|
}
|
|
514
520
|
|
|
@@ -917,12 +923,137 @@ export class PostgresManager {
|
|
|
917
923
|
}
|
|
918
924
|
}
|
|
919
925
|
|
|
926
|
+
/**
|
|
927
|
+
* Ensure pgvector extension files are installed in the PG binary dirs.
|
|
928
|
+
* Downloads prebuilt vector.so from apt.postgresql.org on first use (cached).
|
|
929
|
+
* Patches vector.control to use absolute module_pathname.
|
|
930
|
+
*
|
|
931
|
+
* Linux only — .deb extraction requires dpkg-deb or ar+tar.
|
|
932
|
+
* Serialized via _pgvectorInstallPromise to prevent concurrent races.
|
|
933
|
+
*/
|
|
934
|
+
async ensurePgvectorFiles() {
|
|
935
|
+
// Serialize: only one install runs at a time
|
|
936
|
+
if (this._pgvectorInstallPromise) {
|
|
937
|
+
return this._pgvectorInstallPromise;
|
|
938
|
+
}
|
|
939
|
+
this._pgvectorInstallPromise = this._doEnsurePgvectorFiles();
|
|
940
|
+
try {
|
|
941
|
+
await this._pgvectorInstallPromise;
|
|
942
|
+
} finally {
|
|
943
|
+
this._pgvectorInstallPromise = null;
|
|
944
|
+
}
|
|
945
|
+
}
|
|
946
|
+
|
|
947
|
+
async _doEnsurePgvectorFiles() {
|
|
948
|
+
if (!this.binaries?.libDir) return;
|
|
949
|
+
|
|
950
|
+
// Linux only — .deb packages are Linux-specific
|
|
951
|
+
if (os.platform() !== 'linux') {
|
|
952
|
+
this.logger.info('pgvector auto-install is Linux-only. On macOS, install via: brew install pgvector');
|
|
953
|
+
return;
|
|
954
|
+
}
|
|
955
|
+
|
|
956
|
+
const libDir = this.binaries.libDir;
|
|
957
|
+
const binDir = this.binaries.binDir;
|
|
958
|
+
const extDir = path.join(path.dirname(binDir), 'share', 'postgresql', 'extension');
|
|
959
|
+
const vectorSo = path.join(libDir, 'vector.so');
|
|
960
|
+
const vectorControl = path.join(extDir, 'vector.control');
|
|
961
|
+
|
|
962
|
+
// Already installed
|
|
963
|
+
if (fs.existsSync(vectorSo) && fs.existsSync(vectorControl)) return;
|
|
964
|
+
|
|
965
|
+
this.logger.info('pgvector extension files not found — downloading prebuilt binary...');
|
|
966
|
+
|
|
967
|
+
try {
|
|
968
|
+
// Detect PG major version from the postgres binary
|
|
969
|
+
const { execSync } = await import('node:child_process');
|
|
970
|
+
const pgVersion = execSync(`${this.binaries.postgres} --version`, { encoding: 'utf-8' }).trim();
|
|
971
|
+
const majorMatch = pgVersion.match(/PostgreSQL (\d+)/);
|
|
972
|
+
const pgMajor = majorMatch ? majorMatch[1] : '17';
|
|
973
|
+
|
|
974
|
+
// Detect architecture — fail explicitly on unsupported platforms
|
|
975
|
+
const nodeArch = os.arch();
|
|
976
|
+
let arch;
|
|
977
|
+
if (nodeArch === 'x64') arch = 'amd64';
|
|
978
|
+
else if (nodeArch === 'arm64') arch = 'arm64';
|
|
979
|
+
else {
|
|
980
|
+
this.logger.warn({ arch: nodeArch }, 'Unsupported architecture for pgvector auto-install. Supported: x64, arm64');
|
|
981
|
+
return;
|
|
982
|
+
}
|
|
983
|
+
|
|
984
|
+
// Download prebuilt pgvector .deb from apt.postgresql.org (HTTPS)
|
|
985
|
+
// Version 0.8.1-2 — update when new releases ship
|
|
986
|
+
const debUrl = `https://apt.postgresql.org/pub/repos/apt/pool/main/p/pgvector/postgresql-${pgMajor}-pgvector_0.8.1-2.pgdg%2B1_${arch}.deb`;
|
|
987
|
+
this.logger.info({ url: debUrl }, 'Downloading pgvector...');
|
|
988
|
+
|
|
989
|
+
const res = await fetch(debUrl);
|
|
990
|
+
if (!res.ok) throw new Error(`Download failed: ${res.status}`);
|
|
991
|
+
|
|
992
|
+
const buffer = Buffer.from(await res.arrayBuffer());
|
|
993
|
+
|
|
994
|
+
// Extract .deb (it's an ar archive containing data.tar.xz)
|
|
995
|
+
const tmpDir = path.join(os.tmpdir(), `pgserve-pgvector-${process.pid}-${Date.now()}`);
|
|
996
|
+
fs.mkdirSync(tmpDir, { recursive: true });
|
|
997
|
+
const debPath = path.join(tmpDir, 'pgvector.deb');
|
|
998
|
+
fs.writeFileSync(debPath, buffer);
|
|
999
|
+
|
|
1000
|
+
// Use dpkg-deb or ar to extract
|
|
1001
|
+
try {
|
|
1002
|
+
execSync(`dpkg-deb -x ${debPath} ${tmpDir}/extracted`, { stdio: 'pipe' });
|
|
1003
|
+
} catch {
|
|
1004
|
+
// Fallback: try ar + tar
|
|
1005
|
+
fs.mkdirSync(path.join(tmpDir, 'extracted'), { recursive: true });
|
|
1006
|
+
execSync(`cd ${tmpDir} && ar x pgvector.deb && tar xf data.tar.* -C ${tmpDir}/extracted 2>/dev/null || tar xf data.tar.xz -C ${tmpDir}/extracted`, { stdio: 'pipe' });
|
|
1007
|
+
}
|
|
1008
|
+
|
|
1009
|
+
// Copy .so file
|
|
1010
|
+
const soSrc = path.join(tmpDir, 'extracted', 'usr', 'lib', 'postgresql', pgMajor, 'lib', 'vector.so');
|
|
1011
|
+
if (fs.existsSync(soSrc)) {
|
|
1012
|
+
fs.copyFileSync(soSrc, vectorSo);
|
|
1013
|
+
this.logger.info({ path: vectorSo }, 'Installed vector.so');
|
|
1014
|
+
}
|
|
1015
|
+
|
|
1016
|
+
// Copy extension SQL + control files
|
|
1017
|
+
const extSrc = path.join(tmpDir, 'extracted', 'usr', 'share', 'postgresql', pgMajor, 'extension');
|
|
1018
|
+
if (fs.existsSync(extSrc)) {
|
|
1019
|
+
fs.mkdirSync(extDir, { recursive: true });
|
|
1020
|
+
for (const f of fs.readdirSync(extSrc)) {
|
|
1021
|
+
if (f.startsWith('vector')) {
|
|
1022
|
+
fs.copyFileSync(path.join(extSrc, f), path.join(extDir, f));
|
|
1023
|
+
}
|
|
1024
|
+
}
|
|
1025
|
+
this.logger.info({ path: extDir }, 'Installed vector extension SQL files');
|
|
1026
|
+
}
|
|
1027
|
+
|
|
1028
|
+
// Patch vector.control to use absolute module_pathname
|
|
1029
|
+
// (embedded PG's $libdir doesn't match the compiled-in path)
|
|
1030
|
+
if (fs.existsSync(vectorControl)) {
|
|
1031
|
+
let control = fs.readFileSync(vectorControl, 'utf-8');
|
|
1032
|
+
control = control.replace(
|
|
1033
|
+
/module_pathname\s*=\s*'\$libdir\/vector'/,
|
|
1034
|
+
`module_pathname = '${vectorSo.replace('.so', '')}'`
|
|
1035
|
+
);
|
|
1036
|
+
fs.writeFileSync(vectorControl, control);
|
|
1037
|
+
this.logger.info('Patched vector.control with absolute module path');
|
|
1038
|
+
}
|
|
1039
|
+
|
|
1040
|
+
// Cleanup
|
|
1041
|
+
fs.rmSync(tmpDir, { recursive: true, force: true });
|
|
1042
|
+
this.logger.info('pgvector extension installed successfully');
|
|
1043
|
+
} catch (error) {
|
|
1044
|
+
this.logger.warn({ err: error.message }, 'Failed to install pgvector extension files (non-fatal)');
|
|
1045
|
+
}
|
|
1046
|
+
}
|
|
1047
|
+
|
|
920
1048
|
/**
|
|
921
1049
|
* Enable pgvector extension on a database
|
|
922
1050
|
* Creates a temporary connection to the specific database to run CREATE EXTENSION
|
|
923
1051
|
* @param {string} dbName - Database name to enable pgvector on
|
|
924
1052
|
*/
|
|
925
1053
|
async enablePgvectorExtension(dbName) {
|
|
1054
|
+
// Ensure extension files are installed first
|
|
1055
|
+
await this.ensurePgvectorFiles();
|
|
1056
|
+
|
|
926
1057
|
const { SQL } = await import('bun');
|
|
927
1058
|
let dbPool = null;
|
|
928
1059
|
|
package/src/router.js
CHANGED
|
@@ -28,6 +28,17 @@ const SSL_REQUEST_CODE = 80877103;
|
|
|
28
28
|
const GSSAPI_REQUEST_CODE = 80877104;
|
|
29
29
|
const CANCEL_REQUEST_CODE = 80877102;
|
|
30
30
|
|
|
31
|
+
/**
|
|
32
|
+
* Attempt to write a pending buffer to a target socket.
|
|
33
|
+
* Returns remaining unwritten bytes, or null if fully flushed.
|
|
34
|
+
*/
|
|
35
|
+
function flushPending(target, pending) {
|
|
36
|
+
const written = target.write(pending);
|
|
37
|
+
if (written === pending.byteLength) return null;
|
|
38
|
+
if (written === 0) return pending;
|
|
39
|
+
return pending.subarray(written);
|
|
40
|
+
}
|
|
41
|
+
|
|
31
42
|
/**
|
|
32
43
|
* Multi-Tenant Router Server
|
|
33
44
|
*/
|
|
@@ -175,12 +186,17 @@ export class MultiTenantRouter extends EventEmitter {
|
|
|
175
186
|
error(socket, error) {
|
|
176
187
|
router.handleSocketError(socket, error);
|
|
177
188
|
},
|
|
178
|
-
// Called when socket is ready to receive more data
|
|
189
|
+
// Called when client socket is ready to receive more data
|
|
179
190
|
drain(socket) {
|
|
180
|
-
// Resume reading from PostgreSQL if backpressure cleared
|
|
181
191
|
const state = router.socketState.get(socket);
|
|
182
|
-
if (state
|
|
183
|
-
|
|
192
|
+
if (!state) return;
|
|
193
|
+
// Flush any pending PG→Client data
|
|
194
|
+
if (state.pendingToClient) {
|
|
195
|
+
state.pendingToClient = flushPending(socket, state.pendingToClient);
|
|
196
|
+
}
|
|
197
|
+
// If fully flushed, resume reading from PostgreSQL
|
|
198
|
+
if (!state.pendingToClient && state.pgSocket) {
|
|
199
|
+
state.pgSocket.resume();
|
|
184
200
|
}
|
|
185
201
|
}
|
|
186
202
|
}
|
|
@@ -214,7 +230,9 @@ export class MultiTenantRouter extends EventEmitter {
|
|
|
214
230
|
buffer: null,
|
|
215
231
|
pgSocket: null,
|
|
216
232
|
dbName: null,
|
|
217
|
-
handshakeComplete: false
|
|
233
|
+
handshakeComplete: false,
|
|
234
|
+
pendingToPg: null,
|
|
235
|
+
pendingToClient: null
|
|
218
236
|
});
|
|
219
237
|
|
|
220
238
|
// Track connection
|
|
@@ -231,9 +249,16 @@ export class MultiTenantRouter extends EventEmitter {
|
|
|
231
249
|
|
|
232
250
|
// If handshake complete, forward to PostgreSQL
|
|
233
251
|
if (state.handshakeComplete && state.pgSocket) {
|
|
252
|
+
// If there's already pending data, append to it
|
|
253
|
+
if (state.pendingToPg) {
|
|
254
|
+
state.pendingToPg = Buffer.concat([state.pendingToPg, data]);
|
|
255
|
+
return;
|
|
256
|
+
}
|
|
234
257
|
const written = state.pgSocket.write(data);
|
|
235
|
-
if (written
|
|
236
|
-
//
|
|
258
|
+
if (written < data.byteLength) {
|
|
259
|
+
// Partial write — buffer remainder and pause client
|
|
260
|
+
state.pendingToPg = written === 0 ? Buffer.from(data) : Buffer.from(data.subarray(written));
|
|
261
|
+
socket.pause();
|
|
237
262
|
}
|
|
238
263
|
return;
|
|
239
264
|
}
|
|
@@ -300,56 +325,47 @@ export class MultiTenantRouter extends EventEmitter {
|
|
|
300
325
|
const socketPath = this.pgManager.getSocketPath();
|
|
301
326
|
const router = this;
|
|
302
327
|
|
|
303
|
-
|
|
304
|
-
|
|
305
|
-
|
|
306
|
-
|
|
307
|
-
|
|
308
|
-
|
|
309
|
-
|
|
310
|
-
socket.write(pgData);
|
|
311
|
-
},
|
|
312
|
-
open(pgSocket) {
|
|
313
|
-
// Send buffered startup message to PostgreSQL
|
|
314
|
-
pgSocket.write(startupMessage);
|
|
315
|
-
state.handshakeComplete = true;
|
|
316
|
-
},
|
|
317
|
-
close(_pgSocket) {
|
|
318
|
-
// PostgreSQL closed - close client
|
|
319
|
-
socket.end();
|
|
320
|
-
},
|
|
321
|
-
error(_pgSocket, error) {
|
|
322
|
-
router.logger.error({ dbName, err: error }, 'PostgreSQL socket error');
|
|
323
|
-
socket.end();
|
|
324
|
-
},
|
|
325
|
-
drain(_pgSocket) {
|
|
326
|
-
// Ready for more data
|
|
327
|
-
}
|
|
328
|
+
// Shared handler for pgSocket (used by both unix and TCP paths)
|
|
329
|
+
const pgHandler = {
|
|
330
|
+
data(_pgSocket, pgData) {
|
|
331
|
+
// Forward PostgreSQL response to client with backpressure
|
|
332
|
+
if (state.pendingToClient) {
|
|
333
|
+
state.pendingToClient = Buffer.concat([state.pendingToClient, pgData]);
|
|
334
|
+
return;
|
|
328
335
|
}
|
|
329
|
-
|
|
330
|
-
|
|
331
|
-
|
|
332
|
-
|
|
333
|
-
|
|
334
|
-
|
|
335
|
-
|
|
336
|
-
|
|
337
|
-
|
|
338
|
-
|
|
339
|
-
|
|
340
|
-
|
|
341
|
-
|
|
342
|
-
|
|
343
|
-
|
|
344
|
-
|
|
345
|
-
|
|
346
|
-
|
|
347
|
-
|
|
348
|
-
|
|
349
|
-
|
|
350
|
-
|
|
336
|
+
const written = socket.write(pgData);
|
|
337
|
+
if (written < pgData.byteLength) {
|
|
338
|
+
state.pendingToClient = written === 0 ? Buffer.from(pgData) : Buffer.from(pgData.subarray(written));
|
|
339
|
+
_pgSocket.pause();
|
|
340
|
+
}
|
|
341
|
+
},
|
|
342
|
+
open(pgSocket) {
|
|
343
|
+
pgSocket.write(startupMessage);
|
|
344
|
+
state.handshakeComplete = true;
|
|
345
|
+
},
|
|
346
|
+
close(_pgSocket) {
|
|
347
|
+
socket.end();
|
|
348
|
+
},
|
|
349
|
+
error(_pgSocket, error) {
|
|
350
|
+
router.logger.error({ dbName, err: error }, 'PostgreSQL socket error');
|
|
351
|
+
socket.end();
|
|
352
|
+
},
|
|
353
|
+
drain(_pgSocket) {
|
|
354
|
+
// Flush any pending Client→PG data
|
|
355
|
+
if (state.pendingToPg) {
|
|
356
|
+
state.pendingToPg = flushPending(_pgSocket, state.pendingToPg);
|
|
357
|
+
}
|
|
358
|
+
// If fully flushed, resume reading from client
|
|
359
|
+
if (!state.pendingToPg) {
|
|
360
|
+
socket.resume();
|
|
351
361
|
}
|
|
352
|
-
}
|
|
362
|
+
}
|
|
363
|
+
};
|
|
364
|
+
|
|
365
|
+
if (socketPath) {
|
|
366
|
+
state.pgSocket = await Bun.connect({ unix: socketPath, socket: pgHandler });
|
|
367
|
+
} else {
|
|
368
|
+
state.pgSocket = await Bun.connect({ hostname: '127.0.0.1', port: this.pgPort, socket: pgHandler });
|
|
353
369
|
}
|
|
354
370
|
|
|
355
371
|
this.emit('connection', { dbName, socket });
|
|
@@ -365,8 +381,10 @@ export class MultiTenantRouter extends EventEmitter {
|
|
|
365
381
|
*/
|
|
366
382
|
handleSocketClose(socket) {
|
|
367
383
|
const state = this.socketState.get(socket);
|
|
368
|
-
if (state
|
|
369
|
-
state.
|
|
384
|
+
if (state) {
|
|
385
|
+
state.pendingToPg = null;
|
|
386
|
+
state.pendingToClient = null;
|
|
387
|
+
if (state.pgSocket) state.pgSocket.end();
|
|
370
388
|
}
|
|
371
389
|
this.connections.delete(socket);
|
|
372
390
|
this.socketState.delete(socket);
|
|
@@ -381,8 +399,10 @@ export class MultiTenantRouter extends EventEmitter {
|
|
|
381
399
|
if (error.code !== 'ECONNRESET') {
|
|
382
400
|
this.logger.error({ err: error, dbName: state?.dbName }, 'Socket error');
|
|
383
401
|
}
|
|
384
|
-
if (state
|
|
385
|
-
state.
|
|
402
|
+
if (state) {
|
|
403
|
+
state.pendingToPg = null;
|
|
404
|
+
state.pendingToClient = null;
|
|
405
|
+
if (state.pgSocket) state.pgSocket.end();
|
|
386
406
|
}
|
|
387
407
|
this.connections.delete(socket);
|
|
388
408
|
this.socketState.delete(socket);
|
|
@@ -0,0 +1,167 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Backpressure / Large Message Regression Tests
|
|
3
|
+
*
|
|
4
|
+
* Reproduces the deadlock from issue #14: TCP proxy drops bytes when
|
|
5
|
+
* socket buffers are full, causing PostgreSQL to wait forever for the
|
|
6
|
+
* remainder of a truncated wire protocol message.
|
|
7
|
+
*/
|
|
8
|
+
|
|
9
|
+
import { startMultiTenantServer } from '../src/index.js';
|
|
10
|
+
import pg from 'pg';
|
|
11
|
+
import { test, expect } from 'bun:test';
|
|
12
|
+
import fs from 'fs';
|
|
13
|
+
|
|
14
|
+
const { Client } = pg;
|
|
15
|
+
|
|
16
|
+
const TEST_PORT = 15433;
|
|
17
|
+
const testDataDir = './test-data-backpressure';
|
|
18
|
+
|
|
19
|
+
function cleanup() {
|
|
20
|
+
if (fs.existsSync(testDataDir)) {
|
|
21
|
+
fs.rmSync(testDataDir, { recursive: true, force: true });
|
|
22
|
+
}
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
/** Create a connected pg.Client */
|
|
26
|
+
async function connect(dbName) {
|
|
27
|
+
const client = new Client({
|
|
28
|
+
host: '127.0.0.1',
|
|
29
|
+
port: TEST_PORT,
|
|
30
|
+
database: dbName,
|
|
31
|
+
user: 'postgres',
|
|
32
|
+
password: 'postgres',
|
|
33
|
+
});
|
|
34
|
+
await client.connect();
|
|
35
|
+
return client;
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
test('Large INSERT (~360KB payload) does not deadlock', async () => {
|
|
39
|
+
cleanup();
|
|
40
|
+
const router = await startMultiTenantServer({
|
|
41
|
+
port: TEST_PORT,
|
|
42
|
+
baseDir: testDataDir,
|
|
43
|
+
logLevel: 'warn',
|
|
44
|
+
});
|
|
45
|
+
|
|
46
|
+
let client;
|
|
47
|
+
try {
|
|
48
|
+
client = await connect('bp_insert');
|
|
49
|
+
await client.query('CREATE TABLE big (id SERIAL PRIMARY KEY, payload TEXT)');
|
|
50
|
+
|
|
51
|
+
// ~360KB of text — exceeds typical socket buffer size
|
|
52
|
+
const bigPayload = 'x'.repeat(360_000);
|
|
53
|
+
await client.query('INSERT INTO big (payload) VALUES ($1)', [bigPayload]);
|
|
54
|
+
|
|
55
|
+
const res = await client.query('SELECT length(payload) AS len FROM big');
|
|
56
|
+
expect(Number(res.rows[0].len)).toBe(360000);
|
|
57
|
+
} finally {
|
|
58
|
+
if (client) await client.end().catch(() => {});
|
|
59
|
+
await router.stop();
|
|
60
|
+
cleanup();
|
|
61
|
+
}
|
|
62
|
+
}, 30_000);
|
|
63
|
+
|
|
64
|
+
test('Large SELECT result (500KB+) does not deadlock', async () => {
|
|
65
|
+
cleanup();
|
|
66
|
+
const router = await startMultiTenantServer({
|
|
67
|
+
port: TEST_PORT,
|
|
68
|
+
baseDir: testDataDir,
|
|
69
|
+
logLevel: 'warn',
|
|
70
|
+
});
|
|
71
|
+
|
|
72
|
+
let client;
|
|
73
|
+
try {
|
|
74
|
+
client = await connect('bp_select');
|
|
75
|
+
await client.query('CREATE TABLE chunks (id SERIAL PRIMARY KEY, data TEXT)');
|
|
76
|
+
|
|
77
|
+
// Insert many rows that sum to >500KB
|
|
78
|
+
const chunkSize = 10_000;
|
|
79
|
+
const numChunks = 60; // 60 * 10KB = 600KB total
|
|
80
|
+
const chunk = 'y'.repeat(chunkSize);
|
|
81
|
+
|
|
82
|
+
for (let i = 0; i < numChunks; i++) {
|
|
83
|
+
await client.query('INSERT INTO chunks (data) VALUES ($1)', [chunk]);
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
// Fetch all rows in a single result set (PG→Client backpressure)
|
|
87
|
+
const res = await client.query('SELECT * FROM chunks');
|
|
88
|
+
expect(res.rows.length).toBe(numChunks);
|
|
89
|
+
expect(res.rows[0].data.length).toBe(chunkSize);
|
|
90
|
+
} finally {
|
|
91
|
+
if (client) await client.end().catch(() => {});
|
|
92
|
+
await router.stop();
|
|
93
|
+
cleanup();
|
|
94
|
+
}
|
|
95
|
+
}, 30_000);
|
|
96
|
+
|
|
97
|
+
test('Large single query with multi-value INSERT (500KB+)', async () => {
|
|
98
|
+
cleanup();
|
|
99
|
+
const router = await startMultiTenantServer({
|
|
100
|
+
port: TEST_PORT,
|
|
101
|
+
baseDir: testDataDir,
|
|
102
|
+
logLevel: 'warn',
|
|
103
|
+
});
|
|
104
|
+
|
|
105
|
+
let client;
|
|
106
|
+
try {
|
|
107
|
+
client = await connect('bp_multivalue');
|
|
108
|
+
await client.query('CREATE TABLE items (id INT, val TEXT)');
|
|
109
|
+
|
|
110
|
+
// Build a single INSERT with many value tuples to produce a large wire message
|
|
111
|
+
const rowCount = 500;
|
|
112
|
+
const rowValue = 'z'.repeat(1_000); // 1KB per row → ~500KB total
|
|
113
|
+
const values = [];
|
|
114
|
+
const params = [];
|
|
115
|
+
for (let i = 0; i < rowCount; i++) {
|
|
116
|
+
values.push(`($${i * 2 + 1}, $${i * 2 + 2})`);
|
|
117
|
+
params.push(i, rowValue);
|
|
118
|
+
}
|
|
119
|
+
|
|
120
|
+
const sql = `INSERT INTO items (id, val) VALUES ${values.join(', ')}`;
|
|
121
|
+
await client.query(sql, params);
|
|
122
|
+
|
|
123
|
+
const res = await client.query('SELECT count(*)::int AS cnt FROM items');
|
|
124
|
+
expect(res.rows[0].cnt).toBe(rowCount);
|
|
125
|
+
} finally {
|
|
126
|
+
if (client) await client.end().catch(() => {});
|
|
127
|
+
await router.stop();
|
|
128
|
+
cleanup();
|
|
129
|
+
}
|
|
130
|
+
}, 30_000);
|
|
131
|
+
|
|
132
|
+
test('Concurrent large operations (5 clients x 300KB)', async () => {
|
|
133
|
+
cleanup();
|
|
134
|
+
const router = await startMultiTenantServer({
|
|
135
|
+
port: TEST_PORT,
|
|
136
|
+
baseDir: testDataDir,
|
|
137
|
+
logLevel: 'warn',
|
|
138
|
+
});
|
|
139
|
+
|
|
140
|
+
try {
|
|
141
|
+
const numClients = 5;
|
|
142
|
+
const payloadSize = 300_000;
|
|
143
|
+
const payload = 'c'.repeat(payloadSize);
|
|
144
|
+
|
|
145
|
+
// Run all clients concurrently
|
|
146
|
+
const results = await Promise.all(
|
|
147
|
+
Array.from({ length: numClients }, async (_, i) => {
|
|
148
|
+
const dbName = `bp_concurrent_${i}`;
|
|
149
|
+
const client = await connect(dbName);
|
|
150
|
+
await client.query('CREATE TABLE stress (id SERIAL PRIMARY KEY, data TEXT)');
|
|
151
|
+
await client.query('INSERT INTO stress (data) VALUES ($1)', [payload]);
|
|
152
|
+
|
|
153
|
+
const res = await client.query('SELECT length(data) AS len FROM stress');
|
|
154
|
+
await client.end();
|
|
155
|
+
return parseInt(res.rows[0].len, 10);
|
|
156
|
+
})
|
|
157
|
+
);
|
|
158
|
+
|
|
159
|
+
// All clients should have successfully stored the full payload
|
|
160
|
+
for (const len of results) {
|
|
161
|
+
expect(len).toBe(payloadSize);
|
|
162
|
+
}
|
|
163
|
+
} finally {
|
|
164
|
+
await router.stop();
|
|
165
|
+
cleanup();
|
|
166
|
+
}
|
|
167
|
+
}, 60_000);
|