pgserve 2.0.0 → 2.0.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -2,15 +2,14 @@
2
2
 
3
3
  /**
4
4
  * Benchmark Runner
5
- * Compares SQLite, PGlite, PostgreSQL Server, and pgserve performance
5
+ * Compares SQLite, PostgreSQL, and pgserve performance
6
6
  *
7
7
  * 100% Bun-native: Uses bun:sqlite instead of better-sqlite3
8
8
  */
9
9
 
10
10
  import { Database } from 'bun:sqlite';
11
- import { PGlite } from '@electric-sql/pglite';
12
- import { vector } from '@electric-sql/pglite/vector';
13
11
  import { startMultiTenantServer } from '../../src/index.js';
12
+ import { spawn } from 'child_process';
14
13
  import fs from 'fs';
15
14
  import path from 'path';
16
15
  import os from 'os';
@@ -18,6 +17,9 @@ import pg from 'pg';
18
17
  import { loadEmbeddings, generateQueryVectors, formatPgVector, getEmbeddingsPath, getGroundTruth, calculateRecall } from './vector-generator.js';
19
18
 
20
19
  const { Pool } = pg;
20
+ const LEGACY_PGSERVE_VERSION = '1.2.0';
21
+ const LEGACY_PGSERVE_SPEC = `pgserve@${LEGACY_PGSERVE_VERSION}`;
22
+ const NPX_BIN = process.platform === 'win32' ? 'npx.cmd' : 'npx';
21
23
 
22
24
  // ============================================================================
23
25
  // ANSI Colors and Visual Utilities (stress-test style)
@@ -43,7 +45,7 @@ function banner() {
43
45
  ${C.cyan}${C.bold}╔════════════════════════════════════════════════════════════════╗
44
46
  ║ pgserve UNIFIED BENCHMARK SUITE ║
45
47
  ║ ║
46
- ║ Comparing: SQLite │ PGlitePostgreSQL │ pgserve
48
+ ║ Comparing: SQLite │ PostgreSQLpgserve 1.2.0 │ pgserve v2
47
49
  ╚════════════════════════════════════════════════════════════════╝${C.reset}
48
50
  `);
49
51
  }
@@ -104,10 +106,10 @@ ${C.cyan}${C.bold}
104
106
  // Note: recallCount tracks only SEARCH scenarios (INSERT has 'N/A' recall)
105
107
  const engines = {
106
108
  sqlite: { name: 'SQLite', crudQps: 0, vecQps: 0, vecRecall: 0, p50: 0, p99: 0, errors: 0, count: 0, vecCount: 0, recallCount: 0 },
107
- pglite: { name: 'PGlite', crudQps: 0, vecQps: 0, vecRecall: 0, p50: 0, p99: 0, errors: 0, count: 0, vecCount: 0, recallCount: 0 },
108
109
  postgres: { name: 'PostgreSQL', crudQps: 0, vecQps: 0, vecRecall: 0, p50: 0, p99: 0, errors: 0, count: 0, vecCount: 0, recallCount: 0, skipped: false },
109
- pgserve: { name: 'pgserve (disk)', crudQps: 0, vecQps: 0, vecRecall: 0, p50: 0, p99: 0, errors: 0, count: 0, vecCount: 0, recallCount: 0 },
110
- pgserveRam: { name: 'pgserve (RAM)', crudQps: 0, vecQps: 0, vecRecall: 0, p50: 0, p99: 0, errors: 0, count: 0, vecCount: 0, recallCount: 0 },
110
+ pgserveV1: { name: 'pgserve 1.2.0', crudQps: 0, vecQps: 0, vecRecall: 0, p50: 0, p99: 0, errors: 0, count: 0, vecCount: 0, recallCount: 0 },
111
+ pgserve: { name: 'pgserve v2', crudQps: 0, vecQps: 0, vecRecall: 0, p50: 0, p99: 0, errors: 0, count: 0, vecCount: 0, recallCount: 0 },
112
+ pgserveRam: { name: 'pgserve v2 RAM', crudQps: 0, vecQps: 0, vecRecall: 0, p50: 0, p99: 0, errors: 0, count: 0, vecCount: 0, recallCount: 0 },
111
113
  };
112
114
 
113
115
  // Aggregate CRUD results
@@ -176,7 +178,7 @@ ${C.cyan}${C.bold}
176
178
  }
177
179
 
178
180
  // Print each engine row
179
- const engineOrder = ['sqlite', 'pglite', 'postgres', 'pgserve'];
181
+ const engineOrder = ['sqlite', 'postgres', 'pgserveV1', 'pgserve'];
180
182
  if (canUseRam) engineOrder.push('pgserveRam');
181
183
 
182
184
  let maxScore = 0;
@@ -221,7 +223,7 @@ ${C.magenta}${C.bold}╔══════════════════
221
223
  `);
222
224
  }
223
225
 
224
- // Global error handlers (suppress expected PGlite WASM ExitStatus errors)
226
+ // Global error handlers (suppress expected PostgreSQL WASM ExitStatus errors)
225
227
  process.on('unhandledRejection', (reason, promise) => {
226
228
  if (reason && reason.name === 'ExitStatus') return;
227
229
  console.error('Unhandled Promise Rejection:', reason);
@@ -370,54 +372,85 @@ class Metrics {
370
372
  }
371
373
  }
372
374
 
373
- /**
374
- * SQLite Benchmark
375
- */
376
- async function benchmarkSQLite(scenario) {
377
- console.log(' 🔸 Running SQLite benchmark...');
375
+ function skippedCrud(reason, errors = 1) {
376
+ return { throughput: 0, p50: 0, p99: 0, errors, lockTimeouts: 0, totalOps: 0, skipped: true, reason };
377
+ }
378
378
 
379
- const dbPath = path.join(RESULTS_DIR, 'sqlite-bench.db');
380
- if (fs.existsSync(dbPath)) fs.unlinkSync(dbPath);
379
+ function skippedVector(reason, errors = 1) {
380
+ return { throughput: 0, recall: 'N/A', p50: 0, p99: 0, errors, totalOps: 0, skipped: true, reason };
381
+ }
381
382
 
382
- const db = new Database(dbPath);
383
+ async function openPgPool({ port, database, max = 20, timeoutMs = 30_000 }) {
384
+ const pool = new Pool({
385
+ host: '127.0.0.1',
386
+ port,
387
+ database,
388
+ user: 'postgres',
389
+ password: 'postgres',
390
+ max,
391
+ connectionTimeoutMillis: 1000
392
+ });
383
393
 
384
- // Setup schema
385
- db.exec(`
386
- CREATE TABLE IF NOT EXISTS messages (
387
- id INTEGER PRIMARY KEY AUTOINCREMENT,
394
+ const deadline = Date.now() + timeoutMs;
395
+ let lastError;
396
+ while (Date.now() < deadline) {
397
+ try {
398
+ await pool.query('SELECT 1');
399
+ return pool;
400
+ } catch (error) {
401
+ lastError = error;
402
+ await new Promise(resolve => setTimeout(resolve, 500));
403
+ }
404
+ }
405
+
406
+ await pool.end().catch(() => {});
407
+ throw lastError || new Error(`PostgreSQL did not become ready on port ${port}`);
408
+ }
409
+
410
+ async function runCrudScenarioOnPool(pool, scenario) {
411
+ await pool.query(`
412
+ DROP TABLE IF EXISTS bench_messages;
413
+ CREATE TABLE bench_messages (
414
+ id SERIAL PRIMARY KEY,
388
415
  content TEXT,
389
- timestamp INTEGER
416
+ timestamp BIGINT
390
417
  )
391
418
  `);
392
419
 
393
420
  const metrics = new Metrics();
394
421
  metrics.start();
395
422
 
396
- // Run operations
397
423
  for (const op of scenario.operations) {
398
424
  if (op.type === 'INSERT') {
399
425
  const concurrent = op.concurrent || 1;
400
426
  const perThread = Math.floor(op.count / concurrent);
401
427
 
428
+ const promises = [];
402
429
  for (let i = 0; i < concurrent; i++) {
403
- for (let j = 0; j < perThread; j++) {
404
- const start = Date.now();
405
- try {
406
- db.prepare('INSERT INTO messages (content, timestamp) VALUES (?, ?)').run(
407
- `Message ${i}-${j}`,
408
- Date.now()
409
- );
410
- metrics.addLatency(Date.now() - start);
411
- } catch (error) {
412
- metrics.addError(error);
413
- }
414
- }
430
+ promises.push(
431
+ (async () => {
432
+ for (let j = 0; j < perThread; j++) {
433
+ const start = Date.now();
434
+ try {
435
+ await pool.query(
436
+ 'INSERT INTO bench_messages (content, timestamp) VALUES ($1, $2)',
437
+ [`Message ${i}-${j}`, Date.now()]
438
+ );
439
+ metrics.addLatency(Date.now() - start);
440
+ } catch (error) {
441
+ metrics.addError(error);
442
+ }
443
+ }
444
+ })()
445
+ );
415
446
  }
447
+
448
+ await Promise.all(promises);
416
449
  } else if (op.type === 'SELECT') {
417
450
  for (let i = 0; i < op.count; i++) {
418
451
  const start = Date.now();
419
452
  try {
420
- db.prepare('SELECT * FROM messages LIMIT 10').all();
453
+ await pool.query('SELECT * FROM bench_messages LIMIT 10');
421
454
  metrics.addLatency(Date.now() - start);
422
455
  } catch (error) {
423
456
  metrics.addError(error);
@@ -427,9 +460,9 @@ async function benchmarkSQLite(scenario) {
427
460
  for (let i = 0; i < op.count; i++) {
428
461
  const start = Date.now();
429
462
  try {
430
- db.prepare('UPDATE messages SET content = ? WHERE id = ?').run(
431
- `Updated ${i}`,
432
- (i % 100) + 1
463
+ await pool.query(
464
+ 'UPDATE bench_messages SET content = $1 WHERE id = $2',
465
+ [`Updated ${i}`, (i % 100) + 1]
433
466
  );
434
467
  metrics.addLatency(Date.now() - start);
435
468
  } catch (error) {
@@ -440,57 +473,220 @@ async function benchmarkSQLite(scenario) {
440
473
  }
441
474
 
442
475
  metrics.end();
443
- db.close();
444
-
476
+ await pool.query('DROP TABLE IF EXISTS bench_messages');
445
477
  return metrics.getReport();
446
478
  }
447
479
 
480
+ async function runVectorScenarioOnPool(pool, scenario, embeddings, queryVectors, groundTruth) {
481
+ await pool.query('CREATE EXTENSION IF NOT EXISTS vector');
482
+
483
+ await pool.query(`
484
+ DROP TABLE IF EXISTS embeddings;
485
+ CREATE TABLE embeddings (
486
+ id INTEGER PRIMARY KEY,
487
+ vector vector(${scenario.dimension})
488
+ )
489
+ `);
490
+
491
+ if (scenario.type === 'INSERT') {
492
+ console.log(` Inserting ${scenario.insertCount} vectors (measured)...`);
493
+ const latencies = [];
494
+
495
+ for (let i = 0; i < scenario.insertCount; i++) {
496
+ const vec = formatPgVector(embeddings.vectors[i]);
497
+ const start = performance.now();
498
+ await pool.query('INSERT INTO embeddings (id, vector) VALUES ($1, $2::vector)', [i + 1, vec]);
499
+ latencies.push(performance.now() - start);
500
+ }
501
+
502
+ const totalTime = latencies.reduce((a, b) => a + b, 0);
503
+ const qps = Math.round((scenario.insertCount / totalTime) * 1000);
504
+ latencies.sort((a, b) => a - b);
505
+ const p50 = latencies[Math.floor(latencies.length * 0.5)]?.toFixed(1) || 0;
506
+ const p99 = latencies[Math.floor(latencies.length * 0.99)]?.toFixed(1) || 0;
507
+
508
+ await pool.query('DROP TABLE IF EXISTS embeddings');
509
+
510
+ return {
511
+ throughput: qps,
512
+ recall: 'N/A',
513
+ p50: parseFloat(p50),
514
+ p99: parseFloat(p99),
515
+ errors: 0,
516
+ totalOps: scenario.insertCount,
517
+ skipped: false
518
+ };
519
+ }
520
+
521
+ console.log(' Inserting vectors...');
522
+ for (let i = 0; i < embeddings.vectors.length; i++) {
523
+ const vec = formatPgVector(embeddings.vectors[i]);
524
+ await pool.query('INSERT INTO embeddings (id, vector) VALUES ($1, $2::vector)', [i + 1, vec]);
525
+ }
526
+
527
+ console.log(' Warming up...');
528
+ for (let i = 0; i < scenario.warmupQueries; i++) {
529
+ const queryVec = formatPgVector(queryVectors[i % queryVectors.length]);
530
+ await pool.query('SELECT id FROM embeddings ORDER BY vector <-> $1::vector LIMIT $2', [queryVec, scenario.k]);
531
+ }
532
+
533
+ console.log(' Running measured queries...');
534
+ const latencies = [];
535
+ const approximateResults = [];
536
+
537
+ for (let i = 0; i < scenario.queryCount; i++) {
538
+ const queryVec = formatPgVector(queryVectors[i]);
539
+ const start = performance.now();
540
+ const result = await pool.query(
541
+ 'SELECT id FROM embeddings ORDER BY vector <-> $1::vector LIMIT $2',
542
+ [queryVec, scenario.k]
543
+ );
544
+ latencies.push(performance.now() - start);
545
+ approximateResults.push(result.rows.map(r => r.id));
546
+ }
547
+
548
+ const { recall } = calculateRecall(approximateResults, groundTruth, scenario.k);
549
+ const totalTime = latencies.reduce((a, b) => a + b, 0);
550
+ const qps = Math.round((scenario.queryCount / totalTime) * 1000);
551
+ latencies.sort((a, b) => a - b);
552
+ const p50 = latencies[Math.floor(latencies.length * 0.5)]?.toFixed(1) || 0;
553
+ const p99 = latencies[Math.floor(latencies.length * 0.99)]?.toFixed(1) || 0;
554
+
555
+ await pool.query('DROP TABLE IF EXISTS embeddings');
556
+
557
+ return {
558
+ throughput: qps,
559
+ recall: (recall * 100).toFixed(1),
560
+ p50: parseFloat(p50),
561
+ p99: parseFloat(p99),
562
+ errors: 0,
563
+ totalOps: scenario.queryCount,
564
+ skipped: false
565
+ };
566
+ }
567
+
568
+ async function startLegacyPgserve({ port, enablePgvector = false }) {
569
+ const dataDir = path.join(RESULTS_DIR, `pgserve-${LEGACY_PGSERVE_VERSION}-port-${port}`);
570
+ fs.rmSync(dataDir, { recursive: true, force: true });
571
+
572
+ const args = [
573
+ '-y',
574
+ LEGACY_PGSERVE_SPEC,
575
+ '--port',
576
+ String(port),
577
+ '--host',
578
+ '127.0.0.1',
579
+ '--data',
580
+ dataDir,
581
+ '--log',
582
+ 'error',
583
+ '--no-stats',
584
+ '--no-cluster',
585
+ ];
586
+ if (enablePgvector) args.push('--pgvector');
587
+
588
+ const tail = [];
589
+ const child = spawn(NPX_BIN, args, {
590
+ stdio: ['ignore', 'pipe', 'pipe'],
591
+ env: { ...process.env, NO_COLOR: '1' },
592
+ });
593
+
594
+ const append = (chunk) => {
595
+ tail.push(String(chunk));
596
+ while (tail.join('').length > 4000) tail.shift();
597
+ };
598
+ child.stdout.on('data', append);
599
+ child.stderr.on('data', append);
600
+
601
+ let exited = false;
602
+ child.once('exit', () => {
603
+ exited = true;
604
+ });
605
+
606
+ try {
607
+ const pool = await openPgPool({ port, database: 'bench_test', timeoutMs: 60_000 });
608
+ await pool.end();
609
+ } catch (error) {
610
+ await stopChildProcess(child);
611
+ const output = tail.join('').trim();
612
+ const detail = output ? `; output: ${output}` : '';
613
+ throw new Error(`${LEGACY_PGSERVE_SPEC} failed to become ready: ${error.message}${detail}`);
614
+ }
615
+
616
+ return {
617
+ async stop() {
618
+ if (!exited) await stopChildProcess(child);
619
+ fs.rmSync(dataDir, { recursive: true, force: true });
620
+ },
621
+ };
622
+ }
623
+
624
+ async function stopChildProcess(child) {
625
+ if (child.exitCode !== null || child.signalCode !== null) return;
626
+
627
+ child.kill('SIGTERM');
628
+ const exited = await new Promise((resolve) => {
629
+ const timer = setTimeout(() => resolve(false), 3000);
630
+ child.once('exit', () => {
631
+ clearTimeout(timer);
632
+ resolve(true);
633
+ });
634
+ });
635
+
636
+ if (!exited) {
637
+ child.kill('SIGKILL');
638
+ await new Promise((resolve) => child.once('exit', resolve));
639
+ }
640
+ }
641
+
448
642
  /**
449
- * PGlite Benchmark (in-process WASM PostgreSQL)
643
+ * SQLite Benchmark
450
644
  */
451
- async function benchmarkPGlite(scenario) {
452
- console.log(' 🔹 Running PGlite benchmark...');
645
+ async function benchmarkSQLite(scenario) {
646
+ console.log(' 🔸 Running SQLite benchmark...');
453
647
 
454
- const dataDir = path.join(RESULTS_DIR, 'pglite-bench');
455
- if (fs.existsSync(dataDir)) {
456
- fs.rmSync(dataDir, { recursive: true });
457
- }
648
+ const dbPath = path.join(RESULTS_DIR, 'sqlite-bench.db');
649
+ if (fs.existsSync(dbPath)) fs.unlinkSync(dbPath);
458
650
 
459
- const db = new PGlite(dataDir);
651
+ const db = new Database(dbPath);
460
652
 
461
653
  // Setup schema
462
- await db.exec(`
654
+ db.exec(`
463
655
  CREATE TABLE IF NOT EXISTS messages (
464
- id SERIAL PRIMARY KEY,
656
+ id INTEGER PRIMARY KEY AUTOINCREMENT,
465
657
  content TEXT,
466
- timestamp BIGINT
658
+ timestamp INTEGER
467
659
  )
468
660
  `);
469
661
 
470
662
  const metrics = new Metrics();
471
663
  metrics.start();
472
664
 
473
- // Run operations (PGlite is single-threaded, so concurrent = sequential)
665
+ // Run operations
474
666
  for (const op of scenario.operations) {
475
667
  if (op.type === 'INSERT') {
476
- const total = op.count;
477
- for (let i = 0; i < total; i++) {
478
- const start = Date.now();
479
- try {
480
- await db.query(
481
- 'INSERT INTO messages (content, timestamp) VALUES ($1, $2)',
482
- [`Message ${i}`, Date.now()]
483
- );
484
- metrics.addLatency(Date.now() - start);
485
- } catch (error) {
486
- metrics.addError(error);
668
+ const concurrent = op.concurrent || 1;
669
+ const perThread = Math.floor(op.count / concurrent);
670
+
671
+ for (let i = 0; i < concurrent; i++) {
672
+ for (let j = 0; j < perThread; j++) {
673
+ const start = Date.now();
674
+ try {
675
+ db.prepare('INSERT INTO messages (content, timestamp) VALUES (?, ?)').run(
676
+ `Message ${i}-${j}`,
677
+ Date.now()
678
+ );
679
+ metrics.addLatency(Date.now() - start);
680
+ } catch (error) {
681
+ metrics.addError(error);
682
+ }
487
683
  }
488
684
  }
489
685
  } else if (op.type === 'SELECT') {
490
686
  for (let i = 0; i < op.count; i++) {
491
687
  const start = Date.now();
492
688
  try {
493
- await db.query('SELECT * FROM messages LIMIT 10');
689
+ db.prepare('SELECT * FROM messages LIMIT 10').all();
494
690
  metrics.addLatency(Date.now() - start);
495
691
  } catch (error) {
496
692
  metrics.addError(error);
@@ -500,9 +696,9 @@ async function benchmarkPGlite(scenario) {
500
696
  for (let i = 0; i < op.count; i++) {
501
697
  const start = Date.now();
502
698
  try {
503
- await db.query(
504
- 'UPDATE messages SET content = $1 WHERE id = $2',
505
- [`Updated ${i}`, (i % 100) + 1]
699
+ db.prepare('UPDATE messages SET content = ? WHERE id = ?').run(
700
+ `Updated ${i}`,
701
+ (i % 100) + 1
506
702
  );
507
703
  metrics.addLatency(Date.now() - start);
508
704
  } catch (error) {
@@ -513,12 +709,7 @@ async function benchmarkPGlite(scenario) {
513
709
  }
514
710
 
515
711
  metrics.end();
516
-
517
- try {
518
- await db.close();
519
- } catch (e) {
520
- // Ignore ExitStatus errors from WASM cleanup
521
- }
712
+ db.close();
522
713
 
523
714
  return metrics.getReport();
524
715
  }
@@ -529,92 +720,36 @@ async function benchmarkPGlite(scenario) {
529
720
  async function benchmarkPostgreSQL(scenario) {
530
721
  console.log(' 🔷 Running PostgreSQL Server benchmark...');
531
722
 
532
- const pool = new Pool({
533
- ...POSTGRES_CONFIG,
534
- max: 20
535
- });
536
-
723
+ let pool;
537
724
  try {
538
- // Test connection first
539
- await pool.query('SELECT 1');
540
-
541
- // Setup schema
542
- await pool.query(`
543
- DROP TABLE IF EXISTS bench_messages;
544
- CREATE TABLE bench_messages (
545
- id SERIAL PRIMARY KEY,
546
- content TEXT,
547
- timestamp BIGINT
548
- )
549
- `);
550
-
551
- const metrics = new Metrics();
552
- metrics.start();
553
-
554
- // Run operations
555
- for (const op of scenario.operations) {
556
- if (op.type === 'INSERT') {
557
- const concurrent = op.concurrent || 1;
558
- const perThread = Math.floor(op.count / concurrent);
559
-
560
- const promises = [];
561
- for (let i = 0; i < concurrent; i++) {
562
- promises.push(
563
- (async () => {
564
- for (let j = 0; j < perThread; j++) {
565
- const start = Date.now();
566
- try {
567
- await pool.query(
568
- 'INSERT INTO bench_messages (content, timestamp) VALUES ($1, $2)',
569
- [`Message ${i}-${j}`, Date.now()]
570
- );
571
- metrics.addLatency(Date.now() - start);
572
- } catch (error) {
573
- metrics.addError(error);
574
- }
575
- }
576
- })()
577
- );
578
- }
579
-
580
- await Promise.all(promises);
581
- } else if (op.type === 'SELECT') {
582
- for (let i = 0; i < op.count; i++) {
583
- const start = Date.now();
584
- try {
585
- await pool.query('SELECT * FROM bench_messages LIMIT 10');
586
- metrics.addLatency(Date.now() - start);
587
- } catch (error) {
588
- metrics.addError(error);
589
- }
590
- }
591
- } else if (op.type === 'UPDATE') {
592
- for (let i = 0; i < op.count; i++) {
593
- const start = Date.now();
594
- try {
595
- await pool.query(
596
- 'UPDATE bench_messages SET content = $1 WHERE id = $2',
597
- [`Updated ${i}`, (i % 100) + 1]
598
- );
599
- metrics.addLatency(Date.now() - start);
600
- } catch (error) {
601
- metrics.addError(error);
602
- }
603
- }
604
- }
605
- }
606
-
607
- metrics.end();
725
+ pool = await openPgPool({ ...POSTGRES_CONFIG });
726
+ return await runCrudScenarioOnPool(pool, scenario);
727
+ } catch (error) {
728
+ console.error(' PostgreSQL benchmark skipped:', error.message);
729
+ return skippedCrud(error.message, 0);
730
+ } finally {
731
+ await pool?.end().catch(() => {});
732
+ }
733
+ }
608
734
 
609
- // Cleanup
610
- await pool.query('DROP TABLE IF EXISTS bench_messages');
611
- await pool.end();
735
+ /**
736
+ * pgserve 1.2.0 Benchmark (published npm package)
737
+ */
738
+ async function benchmarkPgserveV1(scenario) {
739
+ console.log(` 🧭 Running ${LEGACY_PGSERVE_SPEC} benchmark...`);
612
740
 
613
- return metrics.getReport();
741
+ let legacy;
742
+ let pool;
743
+ try {
744
+ legacy = await startLegacyPgserve({ port: 18431 });
745
+ pool = await openPgPool({ port: 18431, database: 'bench_test' });
746
+ return await runCrudScenarioOnPool(pool, scenario);
614
747
  } catch (error) {
615
- console.error(' PostgreSQL benchmark skipped:', error.message);
616
- await pool.end().catch(() => {});
617
- return { throughput: 0, p50: 0, p99: 0, errors: 0, lockTimeouts: 0, totalOps: 0, skipped: true };
748
+ console.error(` ${LEGACY_PGSERVE_SPEC} benchmark skipped:`, error.message);
749
+ return skippedCrud(error.message);
750
+ } finally {
751
+ await pool?.end().catch(() => {});
752
+ await legacy?.stop().catch(() => {});
618
753
  }
619
754
  }
620
755
 
@@ -625,127 +760,27 @@ async function benchmarkPostgreSQL(scenario) {
625
760
  */
626
761
  async function benchmarkPgserve(scenario, useRam = false) {
627
762
  const mode = useRam ? 'RAM' : 'disk';
628
- console.log(` 🚀 Running pgserve (${mode}) benchmark...`);
763
+ console.log(` 🚀 Running pgserve v2 (${mode}) benchmark...`);
629
764
 
630
765
  let server;
766
+ let pool;
631
767
  try {
632
768
  // Start pgserve in memory mode (optionally with RAM storage)
769
+ const port = useRam ? 18433 : 18432;
633
770
  server = await startMultiTenantServer({
634
- port: useRam ? 18433 : 18432,
771
+ port,
635
772
  logLevel: 'error',
636
773
  useRam
637
774
  });
638
775
 
639
- // Wait for server to be fully ready
640
- await new Promise(resolve => setTimeout(resolve, 2000));
641
-
642
- const port = useRam ? 18433 : 18432;
643
- const pool = new Pool({
644
- host: 'localhost',
645
- port,
646
- database: 'bench_test',
647
- user: 'postgres',
648
- password: 'postgres',
649
- max: 20,
650
- connectionTimeoutMillis: 30000
651
- });
652
-
653
- // Wait for connection with retries
654
- let connected = false;
655
- for (let i = 0; i < 10; i++) {
656
- try {
657
- await pool.query('SELECT 1');
658
- connected = true;
659
- break;
660
- } catch (error) {
661
- if (i === 9) throw error;
662
- await new Promise(resolve => setTimeout(resolve, 1000));
663
- }
664
- }
665
-
666
- if (!connected) {
667
- throw new Error('Failed to connect to pgserve');
668
- }
669
-
670
- // Setup schema
671
- await pool.query(`
672
- DROP TABLE IF EXISTS bench_messages;
673
- CREATE TABLE bench_messages (
674
- id SERIAL PRIMARY KEY,
675
- content TEXT,
676
- timestamp BIGINT
677
- )
678
- `);
679
-
680
- const metrics = new Metrics();
681
- metrics.start();
682
-
683
- // Run operations (TRUE concurrent - pgserve handles this natively)
684
- for (const op of scenario.operations) {
685
- if (op.type === 'INSERT') {
686
- const concurrent = op.concurrent || 1;
687
- const perThread = Math.floor(op.count / concurrent);
688
-
689
- const promises = [];
690
- for (let i = 0; i < concurrent; i++) {
691
- promises.push(
692
- (async () => {
693
- for (let j = 0; j < perThread; j++) {
694
- const start = Date.now();
695
- try {
696
- await pool.query(
697
- 'INSERT INTO bench_messages (content, timestamp) VALUES ($1, $2)',
698
- [`Message ${i}-${j}`, Date.now()]
699
- );
700
- metrics.addLatency(Date.now() - start);
701
- } catch (error) {
702
- metrics.addError(error);
703
- }
704
- }
705
- })()
706
- );
707
- }
708
-
709
- await Promise.all(promises);
710
- } else if (op.type === 'SELECT') {
711
- for (let i = 0; i < op.count; i++) {
712
- const start = Date.now();
713
- try {
714
- await pool.query('SELECT * FROM bench_messages LIMIT 10');
715
- metrics.addLatency(Date.now() - start);
716
- } catch (error) {
717
- metrics.addError(error);
718
- }
719
- }
720
- } else if (op.type === 'UPDATE') {
721
- for (let i = 0; i < op.count; i++) {
722
- const start = Date.now();
723
- try {
724
- await pool.query(
725
- 'UPDATE bench_messages SET content = $1 WHERE id = $2',
726
- [`Updated ${i}`, (i % 100) + 1]
727
- );
728
- metrics.addLatency(Date.now() - start);
729
- } catch (error) {
730
- metrics.addError(error);
731
- }
732
- }
733
- }
734
- }
735
-
736
- metrics.end();
737
-
738
- // Cleanup
739
- await pool.end();
740
- await server.stop();
741
-
742
- return metrics.getReport();
776
+ pool = await openPgPool({ port, database: 'bench_test' });
777
+ return await runCrudScenarioOnPool(pool, scenario);
743
778
  } catch (error) {
744
- console.error(` pgserve (${mode}) benchmark failed:`, error.message);
745
- if (server) {
746
- try { await server.stop(); } catch (e) {}
747
- }
748
- return { throughput: 0, p50: 0, p99: 0, errors: 1, lockTimeouts: 0, totalOps: 0, skipped: true };
779
+ console.error(` pgserve v2 (${mode}) benchmark failed:`, error.message);
780
+ return skippedCrud(error.message);
781
+ } finally {
782
+ await pool?.end().catch(() => {});
783
+ await server?.stop().catch(() => {});
749
784
  }
750
785
  }
751
786
 
@@ -754,247 +789,42 @@ async function benchmarkPgserve(scenario, useRam = false) {
754
789
  // ============================================================================
755
790
 
756
791
  /**
757
- * PGlite Vector Benchmark
792
+ * PostgreSQL Server Vector Benchmark
758
793
  * Supports both INSERT and SEARCH scenarios
759
794
  */
760
- async function benchmarkPGliteVector(scenario, embeddings, queryVectors, groundTruth) {
761
- console.log(' 🔹 Running PGlite vector benchmark...');
762
-
763
- const dataDir = path.join(RESULTS_DIR, 'pglite-vector-bench');
764
- if (fs.existsSync(dataDir)) {
765
- fs.rmSync(dataDir, { recursive: true });
766
- }
795
+ async function benchmarkPostgreSQLVector(scenario, embeddings, queryVectors, groundTruth) {
796
+ console.log(' 🔷 Running PostgreSQL vector benchmark...');
767
797
 
798
+ let pool;
768
799
  try {
769
- // Create PGlite with pgvector extension
770
- const db = new PGlite(dataDir, { extensions: { vector } });
771
- await db.exec('CREATE EXTENSION IF NOT EXISTS vector');
772
-
773
- // Setup schema
774
- await db.exec(`
775
- DROP TABLE IF EXISTS embeddings;
776
- CREATE TABLE embeddings (
777
- id INTEGER PRIMARY KEY,
778
- vector vector(${scenario.dimension})
779
- )
780
- `);
781
-
782
- // INSERT scenario - measure insert speed
783
- if (scenario.type === 'INSERT') {
784
- console.log(` Inserting ${scenario.insertCount} vectors (measured)...`);
785
- const latencies = [];
786
-
787
- for (let i = 0; i < scenario.insertCount; i++) {
788
- const vec = formatPgVector(embeddings.vectors[i]);
789
- const start = performance.now();
790
- await db.query('INSERT INTO embeddings (id, vector) VALUES ($1, $2::vector)', [i + 1, vec]);
791
- latencies.push(performance.now() - start);
792
- }
793
-
794
- const totalTime = latencies.reduce((a, b) => a + b, 0);
795
- const qps = Math.round((scenario.insertCount / totalTime) * 1000);
796
- latencies.sort((a, b) => a - b);
797
- const p50 = latencies[Math.floor(latencies.length * 0.5)]?.toFixed(1) || 0;
798
- const p99 = latencies[Math.floor(latencies.length * 0.99)]?.toFixed(1) || 0;
799
-
800
- try { await db.close(); } catch { /* ignore */ }
801
-
802
- return {
803
- throughput: qps,
804
- recall: 'N/A',
805
- p50: parseFloat(p50),
806
- p99: parseFloat(p99),
807
- errors: 0,
808
- totalOps: scenario.insertCount,
809
- skipped: false
810
- };
811
- }
812
-
813
- // SEARCH scenario - measure recall and QPS
814
- // Phase 1: Insert all vectors (not measured)
815
- console.log(' Inserting vectors...');
816
- for (let i = 0; i < embeddings.vectors.length; i++) {
817
- const vec = formatPgVector(embeddings.vectors[i]);
818
- await db.query('INSERT INTO embeddings (id, vector) VALUES ($1, $2::vector)', [i + 1, vec]);
819
- }
820
-
821
- // Phase 2: Warm-up queries (not measured)
822
- console.log(' Warming up...');
823
- for (let i = 0; i < scenario.warmupQueries; i++) {
824
- const queryVec = formatPgVector(queryVectors[i % queryVectors.length]);
825
- await db.query(`SELECT id FROM embeddings ORDER BY vector <-> $1::vector LIMIT $2`, [queryVec, scenario.k]);
826
- }
827
-
828
- // Phase 3: Measured queries with recall tracking
829
- console.log(' Running measured queries...');
830
- const latencies = [];
831
- const approximateResults = [];
832
-
833
- for (let i = 0; i < scenario.queryCount; i++) {
834
- const queryVec = formatPgVector(queryVectors[i]);
835
- const start = performance.now();
836
- const result = await db.query(
837
- `SELECT id FROM embeddings ORDER BY vector <-> $1::vector LIMIT $2`,
838
- [queryVec, scenario.k]
839
- );
840
- latencies.push(performance.now() - start);
841
-
842
- // Collect IDs for recall calculation
843
- approximateResults.push(result.rows.map(r => r.id));
844
- }
845
-
846
- // Calculate recall
847
- const { recall } = calculateRecall(approximateResults, groundTruth, scenario.k);
848
-
849
- // Calculate metrics
850
- const totalTime = latencies.reduce((a, b) => a + b, 0);
851
- const qps = Math.round((scenario.queryCount / totalTime) * 1000);
852
- latencies.sort((a, b) => a - b);
853
- const p50 = latencies[Math.floor(latencies.length * 0.5)]?.toFixed(1) || 0;
854
- const p99 = latencies[Math.floor(latencies.length * 0.99)]?.toFixed(1) || 0;
855
-
856
- try { await db.close(); } catch { /* ignore */ }
857
-
858
- return {
859
- throughput: qps,
860
- recall: (recall * 100).toFixed(1), // as percentage
861
- p50: parseFloat(p50),
862
- p99: parseFloat(p99),
863
- errors: 0,
864
- totalOps: scenario.queryCount,
865
- skipped: false
866
- };
800
+ pool = await openPgPool({ ...POSTGRES_CONFIG });
801
+ return await runVectorScenarioOnPool(pool, scenario, embeddings, queryVectors, groundTruth);
867
802
  } catch (error) {
868
- console.error(' PGlite vector benchmark failed:', error.message);
869
- return { throughput: 0, recall: 0, p50: 0, p99: 0, errors: 1, totalOps: 0, skipped: true };
803
+ console.error(' PostgreSQL vector benchmark skipped:', error.message);
804
+ return skippedVector(error.message, 0);
805
+ } finally {
806
+ await pool?.end().catch(() => {});
870
807
  }
871
808
  }
872
809
 
873
810
  /**
874
- * PostgreSQL Server Vector Benchmark
875
- * Supports both INSERT and SEARCH scenarios
811
+ * pgserve 1.2.0 Vector Benchmark (published npm package)
876
812
  */
877
- async function benchmarkPostgreSQLVector(scenario, embeddings, queryVectors, groundTruth) {
878
- console.log(' 🔷 Running PostgreSQL vector benchmark...');
879
-
880
- const pool = new Pool({
881
- ...POSTGRES_CONFIG,
882
- max: 20
883
- });
813
+ async function benchmarkPgserveV1Vector(scenario, embeddings, queryVectors, groundTruth) {
814
+ console.log(` 🧭 Running ${LEGACY_PGSERVE_SPEC} vector benchmark...`);
884
815
 
816
+ let legacy;
817
+ let pool;
885
818
  try {
886
- // Test connection and check for pgvector
887
- await pool.query('SELECT 1');
888
-
889
- // Check if pgvector is available
890
- try {
891
- await pool.query('CREATE EXTENSION IF NOT EXISTS vector');
892
- } catch (e) {
893
- console.error(' pgvector extension not available. Use pgvector/pgvector:pg17 Docker image.');
894
- await pool.end();
895
- return { throughput: 0, recall: 'N/A', p50: 0, p99: 0, errors: 1, totalOps: 0, skipped: true };
896
- }
897
-
898
- // Setup schema
899
- await pool.query(`
900
- DROP TABLE IF EXISTS embeddings;
901
- CREATE TABLE embeddings (
902
- id INTEGER PRIMARY KEY,
903
- vector vector(${scenario.dimension})
904
- )
905
- `);
906
-
907
- // INSERT scenario - measure insert speed
908
- if (scenario.type === 'INSERT') {
909
- console.log(` Inserting ${scenario.insertCount} vectors (measured)...`);
910
- const latencies = [];
911
-
912
- for (let i = 0; i < scenario.insertCount; i++) {
913
- const vec = formatPgVector(embeddings.vectors[i]);
914
- const start = performance.now();
915
- await pool.query('INSERT INTO embeddings (id, vector) VALUES ($1, $2::vector)', [i + 1, vec]);
916
- latencies.push(performance.now() - start);
917
- }
918
-
919
- const totalTime = latencies.reduce((a, b) => a + b, 0);
920
- const qps = Math.round((scenario.insertCount / totalTime) * 1000);
921
- latencies.sort((a, b) => a - b);
922
- const p50 = latencies[Math.floor(latencies.length * 0.5)]?.toFixed(1) || 0;
923
- const p99 = latencies[Math.floor(latencies.length * 0.99)]?.toFixed(1) || 0;
924
-
925
- await pool.query('DROP TABLE IF EXISTS embeddings');
926
- await pool.end();
927
-
928
- return {
929
- throughput: qps,
930
- recall: 'N/A',
931
- p50: parseFloat(p50),
932
- p99: parseFloat(p99),
933
- errors: 0,
934
- totalOps: scenario.insertCount,
935
- skipped: false
936
- };
937
- }
938
-
939
- // SEARCH scenario - Phase 1: Insert all vectors (not measured)
940
- console.log(' Inserting vectors...');
941
- for (let i = 0; i < embeddings.vectors.length; i++) {
942
- const vec = formatPgVector(embeddings.vectors[i]);
943
- await pool.query('INSERT INTO embeddings (id, vector) VALUES ($1, $2::vector)', [i + 1, vec]);
944
- }
945
-
946
- // Phase 2: Warm-up queries (not measured)
947
- console.log(' Warming up...');
948
- for (let i = 0; i < scenario.warmupQueries; i++) {
949
- const queryVec = formatPgVector(queryVectors[i % queryVectors.length]);
950
- await pool.query(`SELECT id FROM embeddings ORDER BY vector <-> $1::vector LIMIT $2`, [queryVec, scenario.k]);
951
- }
952
-
953
- // Phase 3: Measured queries with recall tracking
954
- console.log(' Running measured queries...');
955
- const latencies = [];
956
- const approximateResults = [];
957
-
958
- for (let i = 0; i < scenario.queryCount; i++) {
959
- const queryVec = formatPgVector(queryVectors[i]);
960
- const start = performance.now();
961
- const result = await pool.query(
962
- `SELECT id FROM embeddings ORDER BY vector <-> $1::vector LIMIT $2`,
963
- [queryVec, scenario.k]
964
- );
965
- latencies.push(performance.now() - start);
966
-
967
- // Collect IDs for recall calculation
968
- approximateResults.push(result.rows.map(r => r.id));
969
- }
970
-
971
- // Calculate recall
972
- const { recall } = calculateRecall(approximateResults, groundTruth, scenario.k);
973
-
974
- // Calculate metrics
975
- const totalTime = latencies.reduce((a, b) => a + b, 0);
976
- const qps = Math.round((scenario.queryCount / totalTime) * 1000);
977
- latencies.sort((a, b) => a - b);
978
- const p50 = latencies[Math.floor(latencies.length * 0.5)]?.toFixed(1) || 0;
979
- const p99 = latencies[Math.floor(latencies.length * 0.99)]?.toFixed(1) || 0;
980
-
981
- // Cleanup
982
- await pool.query('DROP TABLE IF EXISTS embeddings');
983
- await pool.end();
984
-
985
- return {
986
- throughput: qps,
987
- recall: (recall * 100).toFixed(1), // as percentage
988
- p50: parseFloat(p50),
989
- p99: parseFloat(p99),
990
- errors: 0,
991
- totalOps: scenario.queryCount,
992
- skipped: false
993
- };
819
+ legacy = await startLegacyPgserve({ port: 18434, enablePgvector: true });
820
+ pool = await openPgPool({ port: 18434, database: 'vector_bench' });
821
+ return await runVectorScenarioOnPool(pool, scenario, embeddings, queryVectors, groundTruth);
994
822
  } catch (error) {
995
- console.error(' PostgreSQL vector benchmark skipped:', error.message);
996
- await pool.end().catch(() => {});
997
- return { throughput: 0, recall: 'N/A', p50: 0, p99: 0, errors: 0, totalOps: 0, skipped: true };
823
+ console.error(` ${LEGACY_PGSERVE_SPEC} vector benchmark skipped:`, error.message);
824
+ return skippedVector(error.message);
825
+ } finally {
826
+ await pool?.end().catch(() => {});
827
+ await legacy?.stop().catch(() => {});
998
828
  }
999
829
  }
1000
830
 
@@ -1004,162 +834,28 @@ async function benchmarkPostgreSQLVector(scenario, embeddings, queryVectors, gro
1004
834
  */
1005
835
  async function benchmarkPgserveVector(scenario, embeddings, queryVectors, groundTruth, useRam = false) {
1006
836
  const mode = useRam ? 'RAM' : 'disk';
1007
- console.log(` 🚀 Running pgserve (${mode}) vector benchmark...`);
837
+ console.log(` 🚀 Running pgserve v2 (${mode}) vector benchmark...`);
1008
838
 
1009
839
  let server;
840
+ let pool;
1010
841
  try {
1011
842
  // Start pgserve (use different ports for vector benchmarks to avoid conflicts)
1012
- const port = useRam ? 18435 : 18434;
843
+ const port = useRam ? 18436 : 18435;
1013
844
  server = await startMultiTenantServer({
1014
845
  port,
1015
846
  logLevel: 'error',
1016
- useRam
847
+ useRam,
848
+ enablePgvector: true
1017
849
  });
1018
850
 
1019
- // Wait for server to be fully ready
1020
- await new Promise(resolve => setTimeout(resolve, 2000));
1021
-
1022
- const pool = new Pool({
1023
- host: 'localhost',
1024
- port,
1025
- database: 'vector_bench',
1026
- user: 'postgres',
1027
- password: 'postgres',
1028
- max: 20,
1029
- connectionTimeoutMillis: 30000
1030
- });
1031
-
1032
- // Wait for connection with retries
1033
- let connected = false;
1034
- for (let i = 0; i < 10; i++) {
1035
- try {
1036
- await pool.query('SELECT 1');
1037
- connected = true;
1038
- break;
1039
- } catch (error) {
1040
- if (i === 9) throw error;
1041
- await new Promise(resolve => setTimeout(resolve, 1000));
1042
- }
1043
- }
1044
-
1045
- if (!connected) {
1046
- throw new Error('Failed to connect to pgserve');
1047
- }
1048
-
1049
- // Enable pgvector extension
1050
- try {
1051
- await pool.query('CREATE EXTENSION IF NOT EXISTS vector');
1052
- } catch (e) {
1053
- console.error(` pgvector extension not available in pgserve. Install pgvector files to ~/.pgserve/bin/<platform>/`);
1054
- await pool.end();
1055
- await server.stop();
1056
- return { throughput: 0, recall: 'N/A', p50: 0, p99: 0, errors: 0, totalOps: 0, skipped: true, reason: 'pgvector not installed' };
1057
- }
1058
-
1059
- // Setup schema
1060
- await pool.query(`
1061
- DROP TABLE IF EXISTS embeddings;
1062
- CREATE TABLE embeddings (
1063
- id INTEGER PRIMARY KEY,
1064
- vector vector(${scenario.dimension})
1065
- )
1066
- `);
1067
-
1068
- // INSERT scenario - measure insert speed
1069
- if (scenario.type === 'INSERT') {
1070
- console.log(` Inserting ${scenario.insertCount} vectors (measured)...`);
1071
- const latencies = [];
1072
-
1073
- for (let i = 0; i < scenario.insertCount; i++) {
1074
- const vec = formatPgVector(embeddings.vectors[i]);
1075
- const start = performance.now();
1076
- await pool.query('INSERT INTO embeddings (id, vector) VALUES ($1, $2::vector)', [i + 1, vec]);
1077
- latencies.push(performance.now() - start);
1078
- }
1079
-
1080
- const totalTime = latencies.reduce((a, b) => a + b, 0);
1081
- const qps = Math.round((scenario.insertCount / totalTime) * 1000);
1082
- latencies.sort((a, b) => a - b);
1083
- const p50 = latencies[Math.floor(latencies.length * 0.5)]?.toFixed(1) || 0;
1084
- const p99 = latencies[Math.floor(latencies.length * 0.99)]?.toFixed(1) || 0;
1085
-
1086
- await pool.query('DROP TABLE IF EXISTS embeddings');
1087
- await pool.end();
1088
- await server.stop();
1089
-
1090
- return {
1091
- throughput: qps,
1092
- recall: 'N/A',
1093
- p50: parseFloat(p50),
1094
- p99: parseFloat(p99),
1095
- errors: 0,
1096
- totalOps: scenario.insertCount,
1097
- skipped: false
1098
- };
1099
- }
1100
-
1101
- // SEARCH scenario - Phase 1: Insert all vectors (not measured)
1102
- console.log(' Inserting vectors...');
1103
- for (let i = 0; i < embeddings.vectors.length; i++) {
1104
- const vec = formatPgVector(embeddings.vectors[i]);
1105
- await pool.query('INSERT INTO embeddings (id, vector) VALUES ($1, $2::vector)', [i + 1, vec]);
1106
- }
1107
-
1108
- // Phase 2: Warm-up queries (not measured)
1109
- console.log(' Warming up...');
1110
- for (let i = 0; i < scenario.warmupQueries; i++) {
1111
- const queryVec = formatPgVector(queryVectors[i % queryVectors.length]);
1112
- await pool.query(`SELECT id FROM embeddings ORDER BY vector <-> $1::vector LIMIT $2`, [queryVec, scenario.k]);
1113
- }
1114
-
1115
- // Phase 3: Measured queries with recall tracking
1116
- console.log(' Running measured queries...');
1117
- const latencies = [];
1118
- const approximateResults = [];
1119
-
1120
- for (let i = 0; i < scenario.queryCount; i++) {
1121
- const queryVec = formatPgVector(queryVectors[i]);
1122
- const start = performance.now();
1123
- const result = await pool.query(
1124
- `SELECT id FROM embeddings ORDER BY vector <-> $1::vector LIMIT $2`,
1125
- [queryVec, scenario.k]
1126
- );
1127
- latencies.push(performance.now() - start);
1128
-
1129
- // Collect IDs for recall calculation
1130
- approximateResults.push(result.rows.map(r => r.id));
1131
- }
1132
-
1133
- // Calculate recall
1134
- const { recall } = calculateRecall(approximateResults, groundTruth, scenario.k);
1135
-
1136
- // Calculate metrics
1137
- const totalTime = latencies.reduce((a, b) => a + b, 0);
1138
- const qps = Math.round((scenario.queryCount / totalTime) * 1000);
1139
- latencies.sort((a, b) => a - b);
1140
- const p50 = latencies[Math.floor(latencies.length * 0.5)]?.toFixed(1) || 0;
1141
- const p99 = latencies[Math.floor(latencies.length * 0.99)]?.toFixed(1) || 0;
1142
-
1143
- // Cleanup
1144
- await pool.query('DROP TABLE IF EXISTS embeddings');
1145
- await pool.end();
1146
- await server.stop();
1147
-
1148
- return {
1149
- throughput: qps,
1150
- recall: (recall * 100).toFixed(1), // as percentage
1151
- p50: parseFloat(p50),
1152
- p99: parseFloat(p99),
1153
- errors: 0,
1154
- totalOps: scenario.queryCount,
1155
- skipped: false
1156
- };
851
+ pool = await openPgPool({ port, database: 'vector_bench' });
852
+ return await runVectorScenarioOnPool(pool, scenario, embeddings, queryVectors, groundTruth);
1157
853
  } catch (error) {
1158
- console.error(` pgserve (${mode}) vector benchmark failed:`, error.message);
1159
- if (server) {
1160
- try { await server.stop(); } catch { /* ignore */ }
1161
- }
1162
- return { throughput: 0, recall: 'N/A', p50: 0, p99: 0, errors: 0, totalOps: 0, skipped: true, reason: error.message };
854
+ console.error(` pgserve v2 (${mode}) vector benchmark failed:`, error.message);
855
+ return skippedVector(error.message, 0);
856
+ } finally {
857
+ await pool?.end().catch(() => {});
858
+ await server?.stop().catch(() => {});
1163
859
  }
1164
860
  }
1165
861
 
@@ -1188,89 +884,66 @@ function generateReport(results, vectorResults = []) {
1188
884
  md += 'psql postgresql://localhost:5432/mydb\n';
1189
885
  md += '```\n\n';
1190
886
 
887
+ const metricValue = (data, key) => {
888
+ if (!data || data.skipped) return 'N/A';
889
+ const value = data[key];
890
+ return value === undefined || value === null ? 'N/A' : value;
891
+ };
892
+ const metricNumber = (data, key) => {
893
+ if (!data || data.skipped) return null;
894
+ const value = Number.parseFloat(data[key]);
895
+ return Number.isFinite(value) ? value : null;
896
+ };
897
+ const winnerName = (rows, key, direction) => {
898
+ let winner = null;
899
+ for (const row of rows) {
900
+ const value = metricNumber(row.data, key);
901
+ if (value === null) continue;
902
+ if (!winner || (direction === 'max' ? value > winner.value : value < winner.value)) {
903
+ winner = { name: row.name, value };
904
+ }
905
+ }
906
+ return winner?.name || 'N/A';
907
+ };
908
+ const pctDelta = (current, baseline) => {
909
+ if (!current || !baseline || current.skipped || baseline.skipped || baseline.throughput <= 0) return null;
910
+ return ((current.throughput / baseline.throughput - 1) * 100).toFixed(1);
911
+ };
912
+ const renderMetricTable = (rows, metrics) => {
913
+ let table = `| Metric | ${rows.map(r => r.name).join(' | ')} | Winner |\n`;
914
+ table += `| --- | ${rows.map(() => '---:').join(' | ')} | --- |\n`;
915
+ for (const metric of metrics) {
916
+ table += `| ${metric.label} | ${rows.map(r => metric.format(metricValue(r.data, metric.key))).join(' | ')} | ${winnerName(rows, metric.key, metric.direction)} |\n`;
917
+ }
918
+ return `${table}\n`;
919
+ };
920
+ const plain = (value) => String(value);
921
+ const percent = (value) => value === 'N/A' ? value : `${value}%`;
922
+
1191
923
  for (const scenario of results) {
1192
924
  md += `## ${scenario.name}\n\n`;
1193
925
  md += `${scenario.description}\n\n`;
1194
926
 
1195
- const { sqlite, pglite, postgres, pgserve, pgserveRam } = scenario;
1196
- const hasRam = pgserveRam && !pgserveRam.skipped;
1197
-
1198
- if (hasRam) {
1199
- // Extended table with RAM column
1200
- md += '```\n';
1201
- md += '┌─────────────────┬──────────┬──────────┬──────────┬──────────┬─────────────┬─────────────┐\n';
1202
- md += '│ Metric │ SQLite │ PGlite │ PostgreSQL│ pgserve pgserve RAM Winner │\n';
1203
- md += '├─────────────────┼──────────┼──────────┼──────────┼──────────┼─────────────┼─────────────┤\n';
1204
-
1205
- // Find winners (include RAM)
1206
- const throughputs = { sqlite: sqlite.throughput, pglite: pglite.throughput, postgres: postgres.throughput, pgserve: pgserve.throughput, pgserveRam: pgserveRam.throughput };
1207
- const p50s = { sqlite: sqlite.p50, pglite: pglite.p50, postgres: postgres.p50, pgserve: pgserve.p50, pgserveRam: pgserveRam.p50 };
1208
- const p99s = { sqlite: sqlite.p99, pglite: pglite.p99, postgres: postgres.p99, pgserve: pgserve.p99, pgserveRam: pgserveRam.p99 };
1209
- const errors = { sqlite: sqlite.errors, pglite: pglite.errors, postgres: postgres.errors, pgserve: pgserve.errors, pgserveRam: pgserveRam.errors };
1210
-
1211
- const getMaxKey = (obj) => Object.entries(obj).reduce((a, b) => a[1] > b[1] ? a : b)[0];
1212
- const getMinKey = (obj) => Object.entries(obj).filter(([k,v]) => v > 0 || k === 'sqlite').reduce((a, b) => a[1] < b[1] ? a : b)[0];
1213
- const getMinErrorKey = (obj) => Object.entries(obj).reduce((a, b) => a[1] <= b[1] ? a : b)[0];
1214
-
1215
- const nameMap = { sqlite: 'SQLite', pglite: 'PGlite', postgres: 'PostgreSQL', pgserve: 'pgserve', pgserveRam: 'pgserve RAM' };
1216
-
1217
- const pad = (s, n) => String(s).padEnd(n);
1218
-
1219
- md += `│ Throughput (qps)│ ${pad(sqlite.throughput, 8)} │ ${pad(pglite.throughput, 8)} │ ${pad(postgres.throughput, 9)} │ ${pad(pgserve.throughput, 8)} │ ${pad(pgserveRam.throughput, 11)} │ ${pad(nameMap[getMaxKey(throughputs)], 11)} │\n`;
1220
- md += `│ P50 latency (ms)│ ${pad(sqlite.p50, 8)} │ ${pad(pglite.p50, 8)} │ ${pad(postgres.p50, 9)} │ ${pad(pgserve.p50, 8)} │ ${pad(pgserveRam.p50, 11)} │ ${pad(nameMap[getMinKey(p50s)], 11)} │\n`;
1221
- md += `│ P99 latency (ms)│ ${pad(sqlite.p99, 8)} │ ${pad(pglite.p99, 8)} │ ${pad(postgres.p99, 9)} │ ${pad(pgserve.p99, 8)} │ ${pad(pgserveRam.p99, 11)} │ ${pad(nameMap[getMinKey(p99s)], 11)} │\n`;
1222
- md += `│ Errors │ ${pad(sqlite.errors, 8)} │ ${pad(pglite.errors, 8)} │ ${pad(postgres.errors, 9)} │ ${pad(pgserve.errors, 8)} │ ${pad(pgserveRam.errors, 11)} │ ${pad(nameMap[getMinErrorKey(errors)], 11)} │\n`;
1223
- md += '└─────────────────┴──────────┴──────────┴──────────┴──────────┴─────────────┴─────────────┘\n';
1224
- md += '```\n\n';
1225
-
1226
- // Analysis with RAM comparison
1227
- const winner = nameMap[getMaxKey(throughputs)];
1228
- if (winner === 'pgserve RAM') {
1229
- const vsDisk = pgserve.throughput > 0 ? ((pgserveRam.throughput / pgserve.throughput - 1) * 100).toFixed(1) : 'N/A';
1230
- const vsPGlite = pglite.throughput > 0 ? ((pgserveRam.throughput / pglite.throughput - 1) * 100).toFixed(1) : 'N/A';
1231
- md += `**pgserve RAM wins!** ${vsDisk}% faster than disk mode, ${vsPGlite}% faster than PGlite.\n\n`;
1232
- } else if (winner === 'pgserve') {
1233
- const vsPGlite = pglite.throughput > 0 ? ((pgserve.throughput / pglite.throughput - 1) * 100).toFixed(1) : 'N/A';
1234
- md += `**pgserve wins!** ${vsPGlite}% faster than PGlite for concurrent workloads.\n\n`;
1235
- } else {
1236
- md += `**${winner} wins** this scenario.\n\n`;
1237
- }
1238
- } else {
1239
- // Original table without RAM column
1240
- md += '```\n';
1241
- md += '┌─────────────────┬──────────┬──────────┬──────────┬──────────┬──────────┐\n';
1242
- md += '│ Metric │ SQLite │ PGlite │ PostgreSQL│ pgserve │ Winner │\n';
1243
- md += '├─────────────────┼──────────┼──────────┼──────────┼──────────┼──────────┤\n';
1244
-
1245
- // Find winners
1246
- const throughputs = { sqlite: sqlite.throughput, pglite: pglite.throughput, postgres: postgres.throughput, pgserve: pgserve.throughput };
1247
- const p50s = { sqlite: sqlite.p50, pglite: pglite.p50, postgres: postgres.p50, pgserve: pgserve.p50 };
1248
- const p99s = { sqlite: sqlite.p99, pglite: pglite.p99, postgres: postgres.p99, pgserve: pgserve.p99 };
1249
- const errors = { sqlite: sqlite.errors, pglite: pglite.errors, postgres: postgres.errors, pgserve: pgserve.errors };
1250
-
1251
- const getMaxKey = (obj) => Object.entries(obj).reduce((a, b) => a[1] > b[1] ? a : b)[0];
1252
- const getMinKey = (obj) => Object.entries(obj).filter(([k,v]) => v > 0 || k === 'sqlite').reduce((a, b) => a[1] < b[1] ? a : b)[0];
1253
- const getMinErrorKey = (obj) => Object.entries(obj).reduce((a, b) => a[1] <= b[1] ? a : b)[0];
1254
-
1255
- const nameMap = { sqlite: 'SQLite', pglite: 'PGlite', postgres: 'PostgreSQL', pgserve: 'pgserve' };
1256
-
1257
- const pad = (s, n) => String(s).padEnd(n);
1258
-
1259
- md += `│ Throughput (qps)│ ${pad(sqlite.throughput, 8)} │ ${pad(pglite.throughput, 8)} │ ${pad(postgres.throughput, 9)} │ ${pad(pgserve.throughput, 8)} │ ${pad(nameMap[getMaxKey(throughputs)], 8)} │\n`;
1260
- md += `│ P50 latency (ms)│ ${pad(sqlite.p50, 8)} │ ${pad(pglite.p50, 8)} │ ${pad(postgres.p50, 9)} │ ${pad(pgserve.p50, 8)} │ ${pad(nameMap[getMinKey(p50s)], 8)} │\n`;
1261
- md += `│ P99 latency (ms)│ ${pad(sqlite.p99, 8)} │ ${pad(pglite.p99, 8)} │ ${pad(postgres.p99, 9)} │ ${pad(pgserve.p99, 8)} │ ${pad(nameMap[getMinKey(p99s)], 8)} │\n`;
1262
- md += `│ Errors │ ${pad(sqlite.errors, 8)} │ ${pad(pglite.errors, 8)} │ ${pad(postgres.errors, 9)} │ ${pad(pgserve.errors, 8)} │ ${pad(nameMap[getMinErrorKey(errors)], 8)} │\n`;
1263
- md += '└─────────────────┴──────────┴──────────┴──────────┴──────────┴──────────┘\n';
1264
- md += '```\n\n';
1265
-
1266
- // Analysis
1267
- const winner = nameMap[getMaxKey(throughputs)];
1268
- if (winner === 'pgserve') {
1269
- const vsPGlite = pglite.throughput > 0 ? ((pgserve.throughput / pglite.throughput - 1) * 100).toFixed(1) : 'N/A';
1270
- md += `**pgserve wins!** ${vsPGlite}% faster than PGlite for concurrent workloads.\n\n`;
1271
- } else {
1272
- md += `**${winner} wins** this scenario.\n\n`;
1273
- }
927
+ const rows = [
928
+ { name: 'SQLite', data: scenario.sqlite },
929
+ { name: 'PostgreSQL', data: scenario.postgres },
930
+ { name: 'pgserve 1.2.0', data: scenario.pgserveV1 },
931
+ { name: 'pgserve v2', data: scenario.pgserve },
932
+ ];
933
+ if (scenario.pgserveRam && !scenario.pgserveRam.skipped) {
934
+ rows.push({ name: 'pgserve v2 RAM', data: scenario.pgserveRam });
935
+ }
936
+
937
+ md += renderMetricTable(rows, [
938
+ { label: 'Throughput (qps)', key: 'throughput', direction: 'max', format: plain },
939
+ { label: 'P50 latency (ms)', key: 'p50', direction: 'min', format: plain },
940
+ { label: 'P99 latency (ms)', key: 'p99', direction: 'min', format: plain },
941
+ { label: 'Errors', key: 'errors', direction: 'min', format: plain },
942
+ ]);
943
+
944
+ const delta = pctDelta(scenario.pgserve, scenario.pgserveV1);
945
+ if (delta !== null) {
946
+ md += `**pgserve v2 vs 1.2.0:** ${delta}% throughput delta.\n\n`;
1274
947
  }
1275
948
  }
1276
949
 
@@ -1284,34 +957,32 @@ function generateReport(results, vectorResults = []) {
1284
957
  md += `### ${scenario.name}\n\n`;
1285
958
  md += `${scenario.description}\n\n`;
1286
959
 
1287
- const { pglite, postgres, pgserve, pgserveRam, k } = scenario;
1288
-
1289
- md += '```\n';
1290
- md += '┌─────────────────┬──────────┬──────────┬──────────┬─────────────┐\n';
1291
- md += '│ Metric │ PGlite │ PostgreSQL│ pgserve │ pgserve RAM │\n';
1292
- md += '├─────────────────┼──────────┼──────────┼──────────┼─────────────┤\n';
1293
-
1294
- const pad = (s, n) => String(s).padEnd(n);
1295
- const val = (r, key) => r.skipped ? 'N/A' : r[key];
1296
- const recallVal = (r) => r.skipped ? 'N/A' : `${r.recall}%`;
960
+ const rows = [
961
+ { name: 'PostgreSQL', data: scenario.postgres },
962
+ { name: 'pgserve 1.2.0', data: scenario.pgserveV1 },
963
+ { name: 'pgserve v2', data: scenario.pgserve },
964
+ ];
965
+ if (scenario.pgserveRam && !scenario.pgserveRam.skipped) {
966
+ rows.push({ name: 'pgserve v2 RAM', data: scenario.pgserveRam });
967
+ }
1297
968
 
1298
- md += `│ Recall@${String(k || 10).padEnd(8)}│ ${pad(recallVal(pglite), 8)} │ ${pad(recallVal(postgres), 9)} │ ${pad(recallVal(pgserve), 8)} │ ${pad(recallVal(pgserveRam), 11)} │\n`;
1299
- md += `│ Throughput (qps)│ ${pad(val(pglite, 'throughput'), 8)} ${pad(val(postgres, 'throughput'), 9)} │ ${pad(val(pgserve, 'throughput'), 8)} ${pad(val(pgserveRam, 'throughput'), 11)} │\n`;
1300
- md += `│ P50 latency (ms)│ ${pad(val(pglite, 'p50'), 8)} │ ${pad(val(postgres, 'p50'), 9)} │ ${pad(val(pgserve, 'p50'), 8)} ${pad(val(pgserveRam, 'p50'), 11)} │\n`;
1301
- md += `│ P99 latency (ms)│ ${pad(val(pglite, 'p99'), 8)} │ ${pad(val(postgres, 'p99'), 9)} │ ${pad(val(pgserve, 'p99'), 8)} ${pad(val(pgserveRam, 'p99'), 11)} │\n`;
1302
- md += `│ Errors │ ${pad(val(pglite, 'errors'), 8)} │ ${pad(val(postgres, 'errors'), 9)} │ ${pad(val(pgserve, 'errors'), 8)} ${pad(val(pgserveRam, 'errors'), 11)} │\n`;
1303
- md += '└─────────────────┴──────────┴──────────┴──────────┴─────────────┘\n';
1304
- md += '```\n\n';
969
+ md += renderMetricTable(rows, [
970
+ { label: `Recall@${scenario.k || 10}`, key: 'recall', direction: 'max', format: percent },
971
+ { label: 'Throughput (qps)', key: 'throughput', direction: 'max', format: plain },
972
+ { label: 'P50 latency (ms)', key: 'p50', direction: 'min', format: plain },
973
+ { label: 'P99 latency (ms)', key: 'p99', direction: 'min', format: plain },
974
+ { label: 'Errors', key: 'errors', direction: 'min', format: plain },
975
+ ]);
1305
976
 
1306
977
  // Find winner among non-skipped (considering both recall and throughput)
1307
978
  const candidates = {};
1308
- if (!pglite.skipped) candidates.pglite = { recall: parseFloat(pglite.recall), qps: pglite.throughput };
1309
- if (!postgres.skipped) candidates.postgres = { recall: parseFloat(postgres.recall), qps: postgres.throughput };
1310
- if (!pgserve.skipped) candidates.pgserve = { recall: parseFloat(pgserve.recall), qps: pgserve.throughput };
1311
- if (pgserveRam && !pgserveRam.skipped) candidates.pgserveRam = { recall: parseFloat(pgserveRam.recall), qps: pgserveRam.throughput };
979
+ if (!scenario.postgres.skipped) candidates.postgres = { recall: parseFloat(scenario.postgres.recall), qps: scenario.postgres.throughput };
980
+ if (!scenario.pgserveV1.skipped) candidates.pgserveV1 = { recall: parseFloat(scenario.pgserveV1.recall), qps: scenario.pgserveV1.throughput };
981
+ if (!scenario.pgserve.skipped) candidates.pgserve = { recall: parseFloat(scenario.pgserve.recall), qps: scenario.pgserve.throughput };
982
+ if (scenario.pgserveRam && !scenario.pgserveRam.skipped) candidates.pgserveRam = { recall: parseFloat(scenario.pgserveRam.recall), qps: scenario.pgserveRam.throughput };
1312
983
 
1313
984
  if (Object.keys(candidates).length > 0) {
1314
- const nameMap = { pglite: 'PGlite', postgres: 'PostgreSQL', pgserve: 'pgserve', pgserveRam: 'pgserve RAM' };
985
+ const nameMap = { postgres: 'PostgreSQL', pgserveV1: 'pgserve 1.2.0', pgserve: 'pgserve v2', pgserveRam: 'pgserve v2 RAM' };
1315
986
  // Winner = highest QPS among those with 100% recall, otherwise highest recall
1316
987
  const perfect = Object.entries(candidates).filter(([, v]) => v.recall === 100);
1317
988
  let winnerKey;
@@ -1323,6 +994,11 @@ function generateReport(results, vectorResults = []) {
1323
994
  md += `**${nameMap[winnerKey]} wins** (${candidates[winnerKey].recall}% recall @ ${candidates[winnerKey].qps} qps)\n\n`;
1324
995
  }
1325
996
  }
997
+
998
+ const delta = pctDelta(scenario.pgserve, scenario.pgserveV1);
999
+ if (delta !== null) {
1000
+ md += `**pgserve v2 vs 1.2.0:** ${delta}% throughput delta.\n\n`;
1001
+ }
1326
1002
  }
1327
1003
  }
1328
1004
 
@@ -1366,9 +1042,9 @@ Options:
1366
1042
  --help, -h Show this help
1367
1043
 
1368
1044
  Vector benchmarks require:
1369
- - PGLite: Built-in pgvector support
1045
+ - PostgreSQL: Built-in pgvector support
1370
1046
  - PostgreSQL: Docker image pgvector/pgvector:pg17
1371
- - pgserve: Not yet supported (marked as skipped)
1047
+ - pgserve 1.2.0 and v2: --pgvector support
1372
1048
  `);
1373
1049
  process.exit(0);
1374
1050
  }
@@ -1401,8 +1077,8 @@ Vector benchmarks require:
1401
1077
  section(scenario.name, scenario.description);
1402
1078
 
1403
1079
  const sqlite = await benchmarkSQLite(scenario);
1404
- const pglite = await benchmarkPGlite(scenario);
1405
1080
  const postgres = await benchmarkPostgreSQL(scenario);
1081
+ const pgserveV1 = await benchmarkPgserveV1(scenario);
1406
1082
  const pgserve = await benchmarkPgserve(scenario, false); // disk mode
1407
1083
  const pgserveRam = canUseRam
1408
1084
  ? await benchmarkPgserve(scenario, true) // RAM mode
@@ -1412,8 +1088,8 @@ Vector benchmarks require:
1412
1088
  name: scenario.name,
1413
1089
  description: scenario.description,
1414
1090
  sqlite,
1415
- pglite,
1416
1091
  postgres,
1092
+ pgserveV1,
1417
1093
  pgserve,
1418
1094
  pgserveRam
1419
1095
  });
@@ -1422,11 +1098,11 @@ Vector benchmarks require:
1422
1098
  console.log(`\n ${C.bold}Results:${C.reset}`);
1423
1099
  console.log(` ${C.dim}──────────────────────────────────────────${C.reset}`);
1424
1100
  console.log(` ${C.yellow}🔸${C.reset} SQLite: ${C.bold}${sqlite.throughput}${C.reset} qps, P50=${sqlite.p50}ms, P99=${sqlite.p99}ms`);
1425
- console.log(` ${C.blue}🔹${C.reset} PGlite: ${C.bold}${pglite.throughput}${C.reset} qps, P50=${pglite.p50}ms, P99=${pglite.p99}ms`);
1426
1101
  console.log(` ${C.cyan}🔷${C.reset} PostgreSQL: ${postgres.skipped ? `${C.dim}SKIPPED${C.reset}` : `${C.bold}${postgres.throughput}${C.reset} qps, P50=${postgres.p50}ms, P99=${postgres.p99}ms`}`);
1427
- console.log(` ${C.green}🚀${C.reset} pgserve: ${C.bold}${pgserve.throughput}${C.reset} qps, P50=${pgserve.p50}ms, P99=${pgserve.p99}ms`);
1102
+ console.log(` ${C.blue}🧭${C.reset} pgserve 1.2.0: ${pgserveV1.skipped ? `${C.dim}SKIPPED${C.reset}` : `${C.bold}${pgserveV1.throughput}${C.reset} qps, P50=${pgserveV1.p50}ms, P99=${pgserveV1.p99}ms`}`);
1103
+ console.log(` ${C.green}🚀${C.reset} pgserve v2: ${C.bold}${pgserve.throughput}${C.reset} qps, P50=${pgserve.p50}ms, P99=${pgserve.p99}ms`);
1428
1104
  if (canUseRam) {
1429
- console.log(` ${C.magenta}⚡${C.reset} pgserve (RAM): ${C.bold}${pgserveRam.throughput}${C.reset} qps, P50=${pgserveRam.p50}ms, P99=${pgserveRam.p99}ms`);
1105
+ console.log(` ${C.magenta}⚡${C.reset} pgserve v2 RAM: ${C.bold}${pgserveRam.throughput}${C.reset} qps, P50=${pgserveRam.p50}ms, P99=${pgserveRam.p99}ms`);
1430
1106
  }
1431
1107
  }
1432
1108
  }
@@ -1466,8 +1142,8 @@ Vector benchmarks require:
1466
1142
  }
1467
1143
 
1468
1144
  // Run benchmarks
1469
- const pglite = await benchmarkPGliteVector(scenario, embeddings, queryVectors, groundTruth);
1470
1145
  const postgres = await benchmarkPostgreSQLVector(scenario, embeddings, queryVectors, groundTruth);
1146
+ const pgserveV1 = await benchmarkPgserveV1Vector(scenario, embeddings, queryVectors, groundTruth);
1471
1147
  const pgserve = await benchmarkPgserveVector(scenario, embeddings, queryVectors, groundTruth, false);
1472
1148
  const pgserveRam = canUseRam
1473
1149
  ? await benchmarkPgserveVector(scenario, embeddings, queryVectors, groundTruth, true)
@@ -1478,8 +1154,8 @@ Vector benchmarks require:
1478
1154
  description: scenario.description,
1479
1155
  type: scenario.type,
1480
1156
  k: scenario.k,
1481
- pglite,
1482
1157
  postgres,
1158
+ pgserveV1,
1483
1159
  pgserve,
1484
1160
  pgserveRam
1485
1161
  });
@@ -1495,11 +1171,11 @@ Vector benchmarks require:
1495
1171
  }
1496
1172
  return ` ${color}${icon}${C.reset} ${name.padEnd(14)} Recall: ${C.bold}${r.recall}%${C.reset}, ${C.bold}${r.throughput}${C.reset} qps, P50=${r.p50}ms`;
1497
1173
  };
1498
- console.log(formatResult(pglite, '🔹', C.blue, 'PGlite:'));
1499
1174
  console.log(formatResult(postgres, '🔷', C.cyan, 'PostgreSQL:'));
1500
- console.log(formatResult(pgserve, '🚀', C.green, 'pgserve:'));
1175
+ console.log(formatResult(pgserveV1, '🧭', C.blue, 'pgserve 1.2.0:'));
1176
+ console.log(formatResult(pgserve, '🚀', C.green, 'pgserve v2:'));
1501
1177
  if (canUseRam) {
1502
- console.log(formatResult(pgserveRam, '⚡', C.magenta, 'pgserve (RAM):'));
1178
+ console.log(formatResult(pgserveRam, '⚡', C.magenta, 'pgserve v2 RAM:'));
1503
1179
  }
1504
1180
  }
1505
1181
  }