pgserve 2.0.0 → 2.0.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.github/workflows/ci.yml +2 -2
- package/.github/workflows/release.yml +1 -1
- package/.github/workflows/version.yml +2 -2
- package/Makefile +12 -12
- package/README.md +30 -9
- package/bin/pgserve-wrapper.cjs +3 -3
- package/bin/{pglite-server.js → postgres-server.js} +5 -0
- package/bun.lock +0 -3
- package/ecosystem.config.cjs +3 -3
- package/knip.json +1 -1
- package/package.json +4 -5
- package/scripts/test-bun-self-heal.sh +10 -10
- package/src/daemon.js +1 -1
- package/src/fingerprint.js +30 -4
- package/src/index.js +20 -0
- package/src/postgres.js +34 -9
- package/src/sdk.js +137 -0
- package/tests/benchmarks/runner.js +430 -754
- package/tests/daemon-fingerprint-integration.test.js +5 -3
- package/tests/daemon-pr24-regression.test.js +1 -4
- package/tests/fingerprint.test.js +14 -0
- package/tests/sdk.test.js +71 -0
|
@@ -2,15 +2,14 @@
|
|
|
2
2
|
|
|
3
3
|
/**
|
|
4
4
|
* Benchmark Runner
|
|
5
|
-
* Compares SQLite,
|
|
5
|
+
* Compares SQLite, PostgreSQL, and pgserve performance
|
|
6
6
|
*
|
|
7
7
|
* 100% Bun-native: Uses bun:sqlite instead of better-sqlite3
|
|
8
8
|
*/
|
|
9
9
|
|
|
10
10
|
import { Database } from 'bun:sqlite';
|
|
11
|
-
import { PGlite } from '@electric-sql/pglite';
|
|
12
|
-
import { vector } from '@electric-sql/pglite/vector';
|
|
13
11
|
import { startMultiTenantServer } from '../../src/index.js';
|
|
12
|
+
import { spawn } from 'child_process';
|
|
14
13
|
import fs from 'fs';
|
|
15
14
|
import path from 'path';
|
|
16
15
|
import os from 'os';
|
|
@@ -18,6 +17,9 @@ import pg from 'pg';
|
|
|
18
17
|
import { loadEmbeddings, generateQueryVectors, formatPgVector, getEmbeddingsPath, getGroundTruth, calculateRecall } from './vector-generator.js';
|
|
19
18
|
|
|
20
19
|
const { Pool } = pg;
|
|
20
|
+
const LEGACY_PGSERVE_VERSION = '1.2.0';
|
|
21
|
+
const LEGACY_PGSERVE_SPEC = `pgserve@${LEGACY_PGSERVE_VERSION}`;
|
|
22
|
+
const NPX_BIN = process.platform === 'win32' ? 'npx.cmd' : 'npx';
|
|
21
23
|
|
|
22
24
|
// ============================================================================
|
|
23
25
|
// ANSI Colors and Visual Utilities (stress-test style)
|
|
@@ -43,7 +45,7 @@ function banner() {
|
|
|
43
45
|
${C.cyan}${C.bold}╔════════════════════════════════════════════════════════════════╗
|
|
44
46
|
║ pgserve UNIFIED BENCHMARK SUITE ║
|
|
45
47
|
║ ║
|
|
46
|
-
║ Comparing: SQLite │
|
|
48
|
+
║ Comparing: SQLite │ PostgreSQL │ pgserve 1.2.0 │ pgserve v2 ║
|
|
47
49
|
╚════════════════════════════════════════════════════════════════╝${C.reset}
|
|
48
50
|
`);
|
|
49
51
|
}
|
|
@@ -104,10 +106,10 @@ ${C.cyan}${C.bold}
|
|
|
104
106
|
// Note: recallCount tracks only SEARCH scenarios (INSERT has 'N/A' recall)
|
|
105
107
|
const engines = {
|
|
106
108
|
sqlite: { name: 'SQLite', crudQps: 0, vecQps: 0, vecRecall: 0, p50: 0, p99: 0, errors: 0, count: 0, vecCount: 0, recallCount: 0 },
|
|
107
|
-
pglite: { name: 'PGlite', crudQps: 0, vecQps: 0, vecRecall: 0, p50: 0, p99: 0, errors: 0, count: 0, vecCount: 0, recallCount: 0 },
|
|
108
109
|
postgres: { name: 'PostgreSQL', crudQps: 0, vecQps: 0, vecRecall: 0, p50: 0, p99: 0, errors: 0, count: 0, vecCount: 0, recallCount: 0, skipped: false },
|
|
109
|
-
|
|
110
|
-
|
|
110
|
+
pgserveV1: { name: 'pgserve 1.2.0', crudQps: 0, vecQps: 0, vecRecall: 0, p50: 0, p99: 0, errors: 0, count: 0, vecCount: 0, recallCount: 0 },
|
|
111
|
+
pgserve: { name: 'pgserve v2', crudQps: 0, vecQps: 0, vecRecall: 0, p50: 0, p99: 0, errors: 0, count: 0, vecCount: 0, recallCount: 0 },
|
|
112
|
+
pgserveRam: { name: 'pgserve v2 RAM', crudQps: 0, vecQps: 0, vecRecall: 0, p50: 0, p99: 0, errors: 0, count: 0, vecCount: 0, recallCount: 0 },
|
|
111
113
|
};
|
|
112
114
|
|
|
113
115
|
// Aggregate CRUD results
|
|
@@ -176,7 +178,7 @@ ${C.cyan}${C.bold}
|
|
|
176
178
|
}
|
|
177
179
|
|
|
178
180
|
// Print each engine row
|
|
179
|
-
const engineOrder = ['sqlite', '
|
|
181
|
+
const engineOrder = ['sqlite', 'postgres', 'pgserveV1', 'pgserve'];
|
|
180
182
|
if (canUseRam) engineOrder.push('pgserveRam');
|
|
181
183
|
|
|
182
184
|
let maxScore = 0;
|
|
@@ -221,7 +223,7 @@ ${C.magenta}${C.bold}╔══════════════════
|
|
|
221
223
|
`);
|
|
222
224
|
}
|
|
223
225
|
|
|
224
|
-
// Global error handlers (suppress expected
|
|
226
|
+
// Global error handlers (suppress expected PostgreSQL WASM ExitStatus errors)
|
|
225
227
|
process.on('unhandledRejection', (reason, promise) => {
|
|
226
228
|
if (reason && reason.name === 'ExitStatus') return;
|
|
227
229
|
console.error('Unhandled Promise Rejection:', reason);
|
|
@@ -370,54 +372,85 @@ class Metrics {
|
|
|
370
372
|
}
|
|
371
373
|
}
|
|
372
374
|
|
|
373
|
-
|
|
374
|
-
|
|
375
|
-
|
|
376
|
-
async function benchmarkSQLite(scenario) {
|
|
377
|
-
console.log(' 🔸 Running SQLite benchmark...');
|
|
375
|
+
function skippedCrud(reason, errors = 1) {
|
|
376
|
+
return { throughput: 0, p50: 0, p99: 0, errors, lockTimeouts: 0, totalOps: 0, skipped: true, reason };
|
|
377
|
+
}
|
|
378
378
|
|
|
379
|
-
|
|
380
|
-
|
|
379
|
+
function skippedVector(reason, errors = 1) {
|
|
380
|
+
return { throughput: 0, recall: 'N/A', p50: 0, p99: 0, errors, totalOps: 0, skipped: true, reason };
|
|
381
|
+
}
|
|
381
382
|
|
|
382
|
-
|
|
383
|
+
async function openPgPool({ port, database, max = 20, timeoutMs = 30_000 }) {
|
|
384
|
+
const pool = new Pool({
|
|
385
|
+
host: '127.0.0.1',
|
|
386
|
+
port,
|
|
387
|
+
database,
|
|
388
|
+
user: 'postgres',
|
|
389
|
+
password: 'postgres',
|
|
390
|
+
max,
|
|
391
|
+
connectionTimeoutMillis: 1000
|
|
392
|
+
});
|
|
383
393
|
|
|
384
|
-
|
|
385
|
-
|
|
386
|
-
|
|
387
|
-
|
|
394
|
+
const deadline = Date.now() + timeoutMs;
|
|
395
|
+
let lastError;
|
|
396
|
+
while (Date.now() < deadline) {
|
|
397
|
+
try {
|
|
398
|
+
await pool.query('SELECT 1');
|
|
399
|
+
return pool;
|
|
400
|
+
} catch (error) {
|
|
401
|
+
lastError = error;
|
|
402
|
+
await new Promise(resolve => setTimeout(resolve, 500));
|
|
403
|
+
}
|
|
404
|
+
}
|
|
405
|
+
|
|
406
|
+
await pool.end().catch(() => {});
|
|
407
|
+
throw lastError || new Error(`PostgreSQL did not become ready on port ${port}`);
|
|
408
|
+
}
|
|
409
|
+
|
|
410
|
+
async function runCrudScenarioOnPool(pool, scenario) {
|
|
411
|
+
await pool.query(`
|
|
412
|
+
DROP TABLE IF EXISTS bench_messages;
|
|
413
|
+
CREATE TABLE bench_messages (
|
|
414
|
+
id SERIAL PRIMARY KEY,
|
|
388
415
|
content TEXT,
|
|
389
|
-
timestamp
|
|
416
|
+
timestamp BIGINT
|
|
390
417
|
)
|
|
391
418
|
`);
|
|
392
419
|
|
|
393
420
|
const metrics = new Metrics();
|
|
394
421
|
metrics.start();
|
|
395
422
|
|
|
396
|
-
// Run operations
|
|
397
423
|
for (const op of scenario.operations) {
|
|
398
424
|
if (op.type === 'INSERT') {
|
|
399
425
|
const concurrent = op.concurrent || 1;
|
|
400
426
|
const perThread = Math.floor(op.count / concurrent);
|
|
401
427
|
|
|
428
|
+
const promises = [];
|
|
402
429
|
for (let i = 0; i < concurrent; i++) {
|
|
403
|
-
|
|
404
|
-
|
|
405
|
-
|
|
406
|
-
|
|
407
|
-
|
|
408
|
-
|
|
409
|
-
|
|
410
|
-
|
|
411
|
-
|
|
412
|
-
|
|
413
|
-
|
|
414
|
-
|
|
430
|
+
promises.push(
|
|
431
|
+
(async () => {
|
|
432
|
+
for (let j = 0; j < perThread; j++) {
|
|
433
|
+
const start = Date.now();
|
|
434
|
+
try {
|
|
435
|
+
await pool.query(
|
|
436
|
+
'INSERT INTO bench_messages (content, timestamp) VALUES ($1, $2)',
|
|
437
|
+
[`Message ${i}-${j}`, Date.now()]
|
|
438
|
+
);
|
|
439
|
+
metrics.addLatency(Date.now() - start);
|
|
440
|
+
} catch (error) {
|
|
441
|
+
metrics.addError(error);
|
|
442
|
+
}
|
|
443
|
+
}
|
|
444
|
+
})()
|
|
445
|
+
);
|
|
415
446
|
}
|
|
447
|
+
|
|
448
|
+
await Promise.all(promises);
|
|
416
449
|
} else if (op.type === 'SELECT') {
|
|
417
450
|
for (let i = 0; i < op.count; i++) {
|
|
418
451
|
const start = Date.now();
|
|
419
452
|
try {
|
|
420
|
-
|
|
453
|
+
await pool.query('SELECT * FROM bench_messages LIMIT 10');
|
|
421
454
|
metrics.addLatency(Date.now() - start);
|
|
422
455
|
} catch (error) {
|
|
423
456
|
metrics.addError(error);
|
|
@@ -427,9 +460,9 @@ async function benchmarkSQLite(scenario) {
|
|
|
427
460
|
for (let i = 0; i < op.count; i++) {
|
|
428
461
|
const start = Date.now();
|
|
429
462
|
try {
|
|
430
|
-
|
|
431
|
-
|
|
432
|
-
(i % 100) + 1
|
|
463
|
+
await pool.query(
|
|
464
|
+
'UPDATE bench_messages SET content = $1 WHERE id = $2',
|
|
465
|
+
[`Updated ${i}`, (i % 100) + 1]
|
|
433
466
|
);
|
|
434
467
|
metrics.addLatency(Date.now() - start);
|
|
435
468
|
} catch (error) {
|
|
@@ -440,57 +473,220 @@ async function benchmarkSQLite(scenario) {
|
|
|
440
473
|
}
|
|
441
474
|
|
|
442
475
|
metrics.end();
|
|
443
|
-
|
|
444
|
-
|
|
476
|
+
await pool.query('DROP TABLE IF EXISTS bench_messages');
|
|
445
477
|
return metrics.getReport();
|
|
446
478
|
}
|
|
447
479
|
|
|
480
|
+
async function runVectorScenarioOnPool(pool, scenario, embeddings, queryVectors, groundTruth) {
|
|
481
|
+
await pool.query('CREATE EXTENSION IF NOT EXISTS vector');
|
|
482
|
+
|
|
483
|
+
await pool.query(`
|
|
484
|
+
DROP TABLE IF EXISTS embeddings;
|
|
485
|
+
CREATE TABLE embeddings (
|
|
486
|
+
id INTEGER PRIMARY KEY,
|
|
487
|
+
vector vector(${scenario.dimension})
|
|
488
|
+
)
|
|
489
|
+
`);
|
|
490
|
+
|
|
491
|
+
if (scenario.type === 'INSERT') {
|
|
492
|
+
console.log(` Inserting ${scenario.insertCount} vectors (measured)...`);
|
|
493
|
+
const latencies = [];
|
|
494
|
+
|
|
495
|
+
for (let i = 0; i < scenario.insertCount; i++) {
|
|
496
|
+
const vec = formatPgVector(embeddings.vectors[i]);
|
|
497
|
+
const start = performance.now();
|
|
498
|
+
await pool.query('INSERT INTO embeddings (id, vector) VALUES ($1, $2::vector)', [i + 1, vec]);
|
|
499
|
+
latencies.push(performance.now() - start);
|
|
500
|
+
}
|
|
501
|
+
|
|
502
|
+
const totalTime = latencies.reduce((a, b) => a + b, 0);
|
|
503
|
+
const qps = Math.round((scenario.insertCount / totalTime) * 1000);
|
|
504
|
+
latencies.sort((a, b) => a - b);
|
|
505
|
+
const p50 = latencies[Math.floor(latencies.length * 0.5)]?.toFixed(1) || 0;
|
|
506
|
+
const p99 = latencies[Math.floor(latencies.length * 0.99)]?.toFixed(1) || 0;
|
|
507
|
+
|
|
508
|
+
await pool.query('DROP TABLE IF EXISTS embeddings');
|
|
509
|
+
|
|
510
|
+
return {
|
|
511
|
+
throughput: qps,
|
|
512
|
+
recall: 'N/A',
|
|
513
|
+
p50: parseFloat(p50),
|
|
514
|
+
p99: parseFloat(p99),
|
|
515
|
+
errors: 0,
|
|
516
|
+
totalOps: scenario.insertCount,
|
|
517
|
+
skipped: false
|
|
518
|
+
};
|
|
519
|
+
}
|
|
520
|
+
|
|
521
|
+
console.log(' Inserting vectors...');
|
|
522
|
+
for (let i = 0; i < embeddings.vectors.length; i++) {
|
|
523
|
+
const vec = formatPgVector(embeddings.vectors[i]);
|
|
524
|
+
await pool.query('INSERT INTO embeddings (id, vector) VALUES ($1, $2::vector)', [i + 1, vec]);
|
|
525
|
+
}
|
|
526
|
+
|
|
527
|
+
console.log(' Warming up...');
|
|
528
|
+
for (let i = 0; i < scenario.warmupQueries; i++) {
|
|
529
|
+
const queryVec = formatPgVector(queryVectors[i % queryVectors.length]);
|
|
530
|
+
await pool.query('SELECT id FROM embeddings ORDER BY vector <-> $1::vector LIMIT $2', [queryVec, scenario.k]);
|
|
531
|
+
}
|
|
532
|
+
|
|
533
|
+
console.log(' Running measured queries...');
|
|
534
|
+
const latencies = [];
|
|
535
|
+
const approximateResults = [];
|
|
536
|
+
|
|
537
|
+
for (let i = 0; i < scenario.queryCount; i++) {
|
|
538
|
+
const queryVec = formatPgVector(queryVectors[i]);
|
|
539
|
+
const start = performance.now();
|
|
540
|
+
const result = await pool.query(
|
|
541
|
+
'SELECT id FROM embeddings ORDER BY vector <-> $1::vector LIMIT $2',
|
|
542
|
+
[queryVec, scenario.k]
|
|
543
|
+
);
|
|
544
|
+
latencies.push(performance.now() - start);
|
|
545
|
+
approximateResults.push(result.rows.map(r => r.id));
|
|
546
|
+
}
|
|
547
|
+
|
|
548
|
+
const { recall } = calculateRecall(approximateResults, groundTruth, scenario.k);
|
|
549
|
+
const totalTime = latencies.reduce((a, b) => a + b, 0);
|
|
550
|
+
const qps = Math.round((scenario.queryCount / totalTime) * 1000);
|
|
551
|
+
latencies.sort((a, b) => a - b);
|
|
552
|
+
const p50 = latencies[Math.floor(latencies.length * 0.5)]?.toFixed(1) || 0;
|
|
553
|
+
const p99 = latencies[Math.floor(latencies.length * 0.99)]?.toFixed(1) || 0;
|
|
554
|
+
|
|
555
|
+
await pool.query('DROP TABLE IF EXISTS embeddings');
|
|
556
|
+
|
|
557
|
+
return {
|
|
558
|
+
throughput: qps,
|
|
559
|
+
recall: (recall * 100).toFixed(1),
|
|
560
|
+
p50: parseFloat(p50),
|
|
561
|
+
p99: parseFloat(p99),
|
|
562
|
+
errors: 0,
|
|
563
|
+
totalOps: scenario.queryCount,
|
|
564
|
+
skipped: false
|
|
565
|
+
};
|
|
566
|
+
}
|
|
567
|
+
|
|
568
|
+
async function startLegacyPgserve({ port, enablePgvector = false }) {
|
|
569
|
+
const dataDir = path.join(RESULTS_DIR, `pgserve-${LEGACY_PGSERVE_VERSION}-port-${port}`);
|
|
570
|
+
fs.rmSync(dataDir, { recursive: true, force: true });
|
|
571
|
+
|
|
572
|
+
const args = [
|
|
573
|
+
'-y',
|
|
574
|
+
LEGACY_PGSERVE_SPEC,
|
|
575
|
+
'--port',
|
|
576
|
+
String(port),
|
|
577
|
+
'--host',
|
|
578
|
+
'127.0.0.1',
|
|
579
|
+
'--data',
|
|
580
|
+
dataDir,
|
|
581
|
+
'--log',
|
|
582
|
+
'error',
|
|
583
|
+
'--no-stats',
|
|
584
|
+
'--no-cluster',
|
|
585
|
+
];
|
|
586
|
+
if (enablePgvector) args.push('--pgvector');
|
|
587
|
+
|
|
588
|
+
const tail = [];
|
|
589
|
+
const child = spawn(NPX_BIN, args, {
|
|
590
|
+
stdio: ['ignore', 'pipe', 'pipe'],
|
|
591
|
+
env: { ...process.env, NO_COLOR: '1' },
|
|
592
|
+
});
|
|
593
|
+
|
|
594
|
+
const append = (chunk) => {
|
|
595
|
+
tail.push(String(chunk));
|
|
596
|
+
while (tail.join('').length > 4000) tail.shift();
|
|
597
|
+
};
|
|
598
|
+
child.stdout.on('data', append);
|
|
599
|
+
child.stderr.on('data', append);
|
|
600
|
+
|
|
601
|
+
let exited = false;
|
|
602
|
+
child.once('exit', () => {
|
|
603
|
+
exited = true;
|
|
604
|
+
});
|
|
605
|
+
|
|
606
|
+
try {
|
|
607
|
+
const pool = await openPgPool({ port, database: 'bench_test', timeoutMs: 60_000 });
|
|
608
|
+
await pool.end();
|
|
609
|
+
} catch (error) {
|
|
610
|
+
await stopChildProcess(child);
|
|
611
|
+
const output = tail.join('').trim();
|
|
612
|
+
const detail = output ? `; output: ${output}` : '';
|
|
613
|
+
throw new Error(`${LEGACY_PGSERVE_SPEC} failed to become ready: ${error.message}${detail}`);
|
|
614
|
+
}
|
|
615
|
+
|
|
616
|
+
return {
|
|
617
|
+
async stop() {
|
|
618
|
+
if (!exited) await stopChildProcess(child);
|
|
619
|
+
fs.rmSync(dataDir, { recursive: true, force: true });
|
|
620
|
+
},
|
|
621
|
+
};
|
|
622
|
+
}
|
|
623
|
+
|
|
624
|
+
async function stopChildProcess(child) {
|
|
625
|
+
if (child.exitCode !== null || child.signalCode !== null) return;
|
|
626
|
+
|
|
627
|
+
child.kill('SIGTERM');
|
|
628
|
+
const exited = await new Promise((resolve) => {
|
|
629
|
+
const timer = setTimeout(() => resolve(false), 3000);
|
|
630
|
+
child.once('exit', () => {
|
|
631
|
+
clearTimeout(timer);
|
|
632
|
+
resolve(true);
|
|
633
|
+
});
|
|
634
|
+
});
|
|
635
|
+
|
|
636
|
+
if (!exited) {
|
|
637
|
+
child.kill('SIGKILL');
|
|
638
|
+
await new Promise((resolve) => child.once('exit', resolve));
|
|
639
|
+
}
|
|
640
|
+
}
|
|
641
|
+
|
|
448
642
|
/**
|
|
449
|
-
*
|
|
643
|
+
* SQLite Benchmark
|
|
450
644
|
*/
|
|
451
|
-
async function
|
|
452
|
-
console.log('
|
|
645
|
+
async function benchmarkSQLite(scenario) {
|
|
646
|
+
console.log(' 🔸 Running SQLite benchmark...');
|
|
453
647
|
|
|
454
|
-
const
|
|
455
|
-
if (fs.existsSync(
|
|
456
|
-
fs.rmSync(dataDir, { recursive: true });
|
|
457
|
-
}
|
|
648
|
+
const dbPath = path.join(RESULTS_DIR, 'sqlite-bench.db');
|
|
649
|
+
if (fs.existsSync(dbPath)) fs.unlinkSync(dbPath);
|
|
458
650
|
|
|
459
|
-
const db = new
|
|
651
|
+
const db = new Database(dbPath);
|
|
460
652
|
|
|
461
653
|
// Setup schema
|
|
462
|
-
|
|
654
|
+
db.exec(`
|
|
463
655
|
CREATE TABLE IF NOT EXISTS messages (
|
|
464
|
-
id
|
|
656
|
+
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
465
657
|
content TEXT,
|
|
466
|
-
timestamp
|
|
658
|
+
timestamp INTEGER
|
|
467
659
|
)
|
|
468
660
|
`);
|
|
469
661
|
|
|
470
662
|
const metrics = new Metrics();
|
|
471
663
|
metrics.start();
|
|
472
664
|
|
|
473
|
-
// Run operations
|
|
665
|
+
// Run operations
|
|
474
666
|
for (const op of scenario.operations) {
|
|
475
667
|
if (op.type === 'INSERT') {
|
|
476
|
-
const
|
|
477
|
-
|
|
478
|
-
|
|
479
|
-
|
|
480
|
-
|
|
481
|
-
|
|
482
|
-
|
|
483
|
-
|
|
484
|
-
|
|
485
|
-
|
|
486
|
-
|
|
668
|
+
const concurrent = op.concurrent || 1;
|
|
669
|
+
const perThread = Math.floor(op.count / concurrent);
|
|
670
|
+
|
|
671
|
+
for (let i = 0; i < concurrent; i++) {
|
|
672
|
+
for (let j = 0; j < perThread; j++) {
|
|
673
|
+
const start = Date.now();
|
|
674
|
+
try {
|
|
675
|
+
db.prepare('INSERT INTO messages (content, timestamp) VALUES (?, ?)').run(
|
|
676
|
+
`Message ${i}-${j}`,
|
|
677
|
+
Date.now()
|
|
678
|
+
);
|
|
679
|
+
metrics.addLatency(Date.now() - start);
|
|
680
|
+
} catch (error) {
|
|
681
|
+
metrics.addError(error);
|
|
682
|
+
}
|
|
487
683
|
}
|
|
488
684
|
}
|
|
489
685
|
} else if (op.type === 'SELECT') {
|
|
490
686
|
for (let i = 0; i < op.count; i++) {
|
|
491
687
|
const start = Date.now();
|
|
492
688
|
try {
|
|
493
|
-
|
|
689
|
+
db.prepare('SELECT * FROM messages LIMIT 10').all();
|
|
494
690
|
metrics.addLatency(Date.now() - start);
|
|
495
691
|
} catch (error) {
|
|
496
692
|
metrics.addError(error);
|
|
@@ -500,9 +696,9 @@ async function benchmarkPGlite(scenario) {
|
|
|
500
696
|
for (let i = 0; i < op.count; i++) {
|
|
501
697
|
const start = Date.now();
|
|
502
698
|
try {
|
|
503
|
-
|
|
504
|
-
|
|
505
|
-
|
|
699
|
+
db.prepare('UPDATE messages SET content = ? WHERE id = ?').run(
|
|
700
|
+
`Updated ${i}`,
|
|
701
|
+
(i % 100) + 1
|
|
506
702
|
);
|
|
507
703
|
metrics.addLatency(Date.now() - start);
|
|
508
704
|
} catch (error) {
|
|
@@ -513,12 +709,7 @@ async function benchmarkPGlite(scenario) {
|
|
|
513
709
|
}
|
|
514
710
|
|
|
515
711
|
metrics.end();
|
|
516
|
-
|
|
517
|
-
try {
|
|
518
|
-
await db.close();
|
|
519
|
-
} catch (e) {
|
|
520
|
-
// Ignore ExitStatus errors from WASM cleanup
|
|
521
|
-
}
|
|
712
|
+
db.close();
|
|
522
713
|
|
|
523
714
|
return metrics.getReport();
|
|
524
715
|
}
|
|
@@ -529,92 +720,36 @@ async function benchmarkPGlite(scenario) {
|
|
|
529
720
|
async function benchmarkPostgreSQL(scenario) {
|
|
530
721
|
console.log(' 🔷 Running PostgreSQL Server benchmark...');
|
|
531
722
|
|
|
532
|
-
|
|
533
|
-
...POSTGRES_CONFIG,
|
|
534
|
-
max: 20
|
|
535
|
-
});
|
|
536
|
-
|
|
723
|
+
let pool;
|
|
537
724
|
try {
|
|
538
|
-
|
|
539
|
-
await pool
|
|
540
|
-
|
|
541
|
-
|
|
542
|
-
|
|
543
|
-
|
|
544
|
-
|
|
545
|
-
|
|
546
|
-
|
|
547
|
-
timestamp BIGINT
|
|
548
|
-
)
|
|
549
|
-
`);
|
|
550
|
-
|
|
551
|
-
const metrics = new Metrics();
|
|
552
|
-
metrics.start();
|
|
553
|
-
|
|
554
|
-
// Run operations
|
|
555
|
-
for (const op of scenario.operations) {
|
|
556
|
-
if (op.type === 'INSERT') {
|
|
557
|
-
const concurrent = op.concurrent || 1;
|
|
558
|
-
const perThread = Math.floor(op.count / concurrent);
|
|
559
|
-
|
|
560
|
-
const promises = [];
|
|
561
|
-
for (let i = 0; i < concurrent; i++) {
|
|
562
|
-
promises.push(
|
|
563
|
-
(async () => {
|
|
564
|
-
for (let j = 0; j < perThread; j++) {
|
|
565
|
-
const start = Date.now();
|
|
566
|
-
try {
|
|
567
|
-
await pool.query(
|
|
568
|
-
'INSERT INTO bench_messages (content, timestamp) VALUES ($1, $2)',
|
|
569
|
-
[`Message ${i}-${j}`, Date.now()]
|
|
570
|
-
);
|
|
571
|
-
metrics.addLatency(Date.now() - start);
|
|
572
|
-
} catch (error) {
|
|
573
|
-
metrics.addError(error);
|
|
574
|
-
}
|
|
575
|
-
}
|
|
576
|
-
})()
|
|
577
|
-
);
|
|
578
|
-
}
|
|
579
|
-
|
|
580
|
-
await Promise.all(promises);
|
|
581
|
-
} else if (op.type === 'SELECT') {
|
|
582
|
-
for (let i = 0; i < op.count; i++) {
|
|
583
|
-
const start = Date.now();
|
|
584
|
-
try {
|
|
585
|
-
await pool.query('SELECT * FROM bench_messages LIMIT 10');
|
|
586
|
-
metrics.addLatency(Date.now() - start);
|
|
587
|
-
} catch (error) {
|
|
588
|
-
metrics.addError(error);
|
|
589
|
-
}
|
|
590
|
-
}
|
|
591
|
-
} else if (op.type === 'UPDATE') {
|
|
592
|
-
for (let i = 0; i < op.count; i++) {
|
|
593
|
-
const start = Date.now();
|
|
594
|
-
try {
|
|
595
|
-
await pool.query(
|
|
596
|
-
'UPDATE bench_messages SET content = $1 WHERE id = $2',
|
|
597
|
-
[`Updated ${i}`, (i % 100) + 1]
|
|
598
|
-
);
|
|
599
|
-
metrics.addLatency(Date.now() - start);
|
|
600
|
-
} catch (error) {
|
|
601
|
-
metrics.addError(error);
|
|
602
|
-
}
|
|
603
|
-
}
|
|
604
|
-
}
|
|
605
|
-
}
|
|
606
|
-
|
|
607
|
-
metrics.end();
|
|
725
|
+
pool = await openPgPool({ ...POSTGRES_CONFIG });
|
|
726
|
+
return await runCrudScenarioOnPool(pool, scenario);
|
|
727
|
+
} catch (error) {
|
|
728
|
+
console.error(' PostgreSQL benchmark skipped:', error.message);
|
|
729
|
+
return skippedCrud(error.message, 0);
|
|
730
|
+
} finally {
|
|
731
|
+
await pool?.end().catch(() => {});
|
|
732
|
+
}
|
|
733
|
+
}
|
|
608
734
|
|
|
609
|
-
|
|
610
|
-
|
|
611
|
-
|
|
735
|
+
/**
|
|
736
|
+
* pgserve 1.2.0 Benchmark (published npm package)
|
|
737
|
+
*/
|
|
738
|
+
async function benchmarkPgserveV1(scenario) {
|
|
739
|
+
console.log(` 🧭 Running ${LEGACY_PGSERVE_SPEC} benchmark...`);
|
|
612
740
|
|
|
613
|
-
|
|
741
|
+
let legacy;
|
|
742
|
+
let pool;
|
|
743
|
+
try {
|
|
744
|
+
legacy = await startLegacyPgserve({ port: 18431 });
|
|
745
|
+
pool = await openPgPool({ port: 18431, database: 'bench_test' });
|
|
746
|
+
return await runCrudScenarioOnPool(pool, scenario);
|
|
614
747
|
} catch (error) {
|
|
615
|
-
console.error(
|
|
616
|
-
|
|
617
|
-
|
|
748
|
+
console.error(` ${LEGACY_PGSERVE_SPEC} benchmark skipped:`, error.message);
|
|
749
|
+
return skippedCrud(error.message);
|
|
750
|
+
} finally {
|
|
751
|
+
await pool?.end().catch(() => {});
|
|
752
|
+
await legacy?.stop().catch(() => {});
|
|
618
753
|
}
|
|
619
754
|
}
|
|
620
755
|
|
|
@@ -625,127 +760,27 @@ async function benchmarkPostgreSQL(scenario) {
|
|
|
625
760
|
*/
|
|
626
761
|
async function benchmarkPgserve(scenario, useRam = false) {
|
|
627
762
|
const mode = useRam ? 'RAM' : 'disk';
|
|
628
|
-
console.log(` 🚀 Running pgserve (${mode}) benchmark...`);
|
|
763
|
+
console.log(` 🚀 Running pgserve v2 (${mode}) benchmark...`);
|
|
629
764
|
|
|
630
765
|
let server;
|
|
766
|
+
let pool;
|
|
631
767
|
try {
|
|
632
768
|
// Start pgserve in memory mode (optionally with RAM storage)
|
|
769
|
+
const port = useRam ? 18433 : 18432;
|
|
633
770
|
server = await startMultiTenantServer({
|
|
634
|
-
port
|
|
771
|
+
port,
|
|
635
772
|
logLevel: 'error',
|
|
636
773
|
useRam
|
|
637
774
|
});
|
|
638
775
|
|
|
639
|
-
|
|
640
|
-
await
|
|
641
|
-
|
|
642
|
-
const port = useRam ? 18433 : 18432;
|
|
643
|
-
const pool = new Pool({
|
|
644
|
-
host: 'localhost',
|
|
645
|
-
port,
|
|
646
|
-
database: 'bench_test',
|
|
647
|
-
user: 'postgres',
|
|
648
|
-
password: 'postgres',
|
|
649
|
-
max: 20,
|
|
650
|
-
connectionTimeoutMillis: 30000
|
|
651
|
-
});
|
|
652
|
-
|
|
653
|
-
// Wait for connection with retries
|
|
654
|
-
let connected = false;
|
|
655
|
-
for (let i = 0; i < 10; i++) {
|
|
656
|
-
try {
|
|
657
|
-
await pool.query('SELECT 1');
|
|
658
|
-
connected = true;
|
|
659
|
-
break;
|
|
660
|
-
} catch (error) {
|
|
661
|
-
if (i === 9) throw error;
|
|
662
|
-
await new Promise(resolve => setTimeout(resolve, 1000));
|
|
663
|
-
}
|
|
664
|
-
}
|
|
665
|
-
|
|
666
|
-
if (!connected) {
|
|
667
|
-
throw new Error('Failed to connect to pgserve');
|
|
668
|
-
}
|
|
669
|
-
|
|
670
|
-
// Setup schema
|
|
671
|
-
await pool.query(`
|
|
672
|
-
DROP TABLE IF EXISTS bench_messages;
|
|
673
|
-
CREATE TABLE bench_messages (
|
|
674
|
-
id SERIAL PRIMARY KEY,
|
|
675
|
-
content TEXT,
|
|
676
|
-
timestamp BIGINT
|
|
677
|
-
)
|
|
678
|
-
`);
|
|
679
|
-
|
|
680
|
-
const metrics = new Metrics();
|
|
681
|
-
metrics.start();
|
|
682
|
-
|
|
683
|
-
// Run operations (TRUE concurrent - pgserve handles this natively)
|
|
684
|
-
for (const op of scenario.operations) {
|
|
685
|
-
if (op.type === 'INSERT') {
|
|
686
|
-
const concurrent = op.concurrent || 1;
|
|
687
|
-
const perThread = Math.floor(op.count / concurrent);
|
|
688
|
-
|
|
689
|
-
const promises = [];
|
|
690
|
-
for (let i = 0; i < concurrent; i++) {
|
|
691
|
-
promises.push(
|
|
692
|
-
(async () => {
|
|
693
|
-
for (let j = 0; j < perThread; j++) {
|
|
694
|
-
const start = Date.now();
|
|
695
|
-
try {
|
|
696
|
-
await pool.query(
|
|
697
|
-
'INSERT INTO bench_messages (content, timestamp) VALUES ($1, $2)',
|
|
698
|
-
[`Message ${i}-${j}`, Date.now()]
|
|
699
|
-
);
|
|
700
|
-
metrics.addLatency(Date.now() - start);
|
|
701
|
-
} catch (error) {
|
|
702
|
-
metrics.addError(error);
|
|
703
|
-
}
|
|
704
|
-
}
|
|
705
|
-
})()
|
|
706
|
-
);
|
|
707
|
-
}
|
|
708
|
-
|
|
709
|
-
await Promise.all(promises);
|
|
710
|
-
} else if (op.type === 'SELECT') {
|
|
711
|
-
for (let i = 0; i < op.count; i++) {
|
|
712
|
-
const start = Date.now();
|
|
713
|
-
try {
|
|
714
|
-
await pool.query('SELECT * FROM bench_messages LIMIT 10');
|
|
715
|
-
metrics.addLatency(Date.now() - start);
|
|
716
|
-
} catch (error) {
|
|
717
|
-
metrics.addError(error);
|
|
718
|
-
}
|
|
719
|
-
}
|
|
720
|
-
} else if (op.type === 'UPDATE') {
|
|
721
|
-
for (let i = 0; i < op.count; i++) {
|
|
722
|
-
const start = Date.now();
|
|
723
|
-
try {
|
|
724
|
-
await pool.query(
|
|
725
|
-
'UPDATE bench_messages SET content = $1 WHERE id = $2',
|
|
726
|
-
[`Updated ${i}`, (i % 100) + 1]
|
|
727
|
-
);
|
|
728
|
-
metrics.addLatency(Date.now() - start);
|
|
729
|
-
} catch (error) {
|
|
730
|
-
metrics.addError(error);
|
|
731
|
-
}
|
|
732
|
-
}
|
|
733
|
-
}
|
|
734
|
-
}
|
|
735
|
-
|
|
736
|
-
metrics.end();
|
|
737
|
-
|
|
738
|
-
// Cleanup
|
|
739
|
-
await pool.end();
|
|
740
|
-
await server.stop();
|
|
741
|
-
|
|
742
|
-
return metrics.getReport();
|
|
776
|
+
pool = await openPgPool({ port, database: 'bench_test' });
|
|
777
|
+
return await runCrudScenarioOnPool(pool, scenario);
|
|
743
778
|
} catch (error) {
|
|
744
|
-
console.error(` pgserve (${mode}) benchmark failed:`, error.message);
|
|
745
|
-
|
|
746
|
-
|
|
747
|
-
}
|
|
748
|
-
|
|
779
|
+
console.error(` pgserve v2 (${mode}) benchmark failed:`, error.message);
|
|
780
|
+
return skippedCrud(error.message);
|
|
781
|
+
} finally {
|
|
782
|
+
await pool?.end().catch(() => {});
|
|
783
|
+
await server?.stop().catch(() => {});
|
|
749
784
|
}
|
|
750
785
|
}
|
|
751
786
|
|
|
@@ -754,247 +789,42 @@ async function benchmarkPgserve(scenario, useRam = false) {
|
|
|
754
789
|
// ============================================================================
|
|
755
790
|
|
|
756
791
|
/**
|
|
757
|
-
*
|
|
792
|
+
* PostgreSQL Server Vector Benchmark
|
|
758
793
|
* Supports both INSERT and SEARCH scenarios
|
|
759
794
|
*/
|
|
760
|
-
async function
|
|
761
|
-
console.log('
|
|
762
|
-
|
|
763
|
-
const dataDir = path.join(RESULTS_DIR, 'pglite-vector-bench');
|
|
764
|
-
if (fs.existsSync(dataDir)) {
|
|
765
|
-
fs.rmSync(dataDir, { recursive: true });
|
|
766
|
-
}
|
|
795
|
+
async function benchmarkPostgreSQLVector(scenario, embeddings, queryVectors, groundTruth) {
|
|
796
|
+
console.log(' 🔷 Running PostgreSQL vector benchmark...');
|
|
767
797
|
|
|
798
|
+
let pool;
|
|
768
799
|
try {
|
|
769
|
-
|
|
770
|
-
|
|
771
|
-
await db.exec('CREATE EXTENSION IF NOT EXISTS vector');
|
|
772
|
-
|
|
773
|
-
// Setup schema
|
|
774
|
-
await db.exec(`
|
|
775
|
-
DROP TABLE IF EXISTS embeddings;
|
|
776
|
-
CREATE TABLE embeddings (
|
|
777
|
-
id INTEGER PRIMARY KEY,
|
|
778
|
-
vector vector(${scenario.dimension})
|
|
779
|
-
)
|
|
780
|
-
`);
|
|
781
|
-
|
|
782
|
-
// INSERT scenario - measure insert speed
|
|
783
|
-
if (scenario.type === 'INSERT') {
|
|
784
|
-
console.log(` Inserting ${scenario.insertCount} vectors (measured)...`);
|
|
785
|
-
const latencies = [];
|
|
786
|
-
|
|
787
|
-
for (let i = 0; i < scenario.insertCount; i++) {
|
|
788
|
-
const vec = formatPgVector(embeddings.vectors[i]);
|
|
789
|
-
const start = performance.now();
|
|
790
|
-
await db.query('INSERT INTO embeddings (id, vector) VALUES ($1, $2::vector)', [i + 1, vec]);
|
|
791
|
-
latencies.push(performance.now() - start);
|
|
792
|
-
}
|
|
793
|
-
|
|
794
|
-
const totalTime = latencies.reduce((a, b) => a + b, 0);
|
|
795
|
-
const qps = Math.round((scenario.insertCount / totalTime) * 1000);
|
|
796
|
-
latencies.sort((a, b) => a - b);
|
|
797
|
-
const p50 = latencies[Math.floor(latencies.length * 0.5)]?.toFixed(1) || 0;
|
|
798
|
-
const p99 = latencies[Math.floor(latencies.length * 0.99)]?.toFixed(1) || 0;
|
|
799
|
-
|
|
800
|
-
try { await db.close(); } catch { /* ignore */ }
|
|
801
|
-
|
|
802
|
-
return {
|
|
803
|
-
throughput: qps,
|
|
804
|
-
recall: 'N/A',
|
|
805
|
-
p50: parseFloat(p50),
|
|
806
|
-
p99: parseFloat(p99),
|
|
807
|
-
errors: 0,
|
|
808
|
-
totalOps: scenario.insertCount,
|
|
809
|
-
skipped: false
|
|
810
|
-
};
|
|
811
|
-
}
|
|
812
|
-
|
|
813
|
-
// SEARCH scenario - measure recall and QPS
|
|
814
|
-
// Phase 1: Insert all vectors (not measured)
|
|
815
|
-
console.log(' Inserting vectors...');
|
|
816
|
-
for (let i = 0; i < embeddings.vectors.length; i++) {
|
|
817
|
-
const vec = formatPgVector(embeddings.vectors[i]);
|
|
818
|
-
await db.query('INSERT INTO embeddings (id, vector) VALUES ($1, $2::vector)', [i + 1, vec]);
|
|
819
|
-
}
|
|
820
|
-
|
|
821
|
-
// Phase 2: Warm-up queries (not measured)
|
|
822
|
-
console.log(' Warming up...');
|
|
823
|
-
for (let i = 0; i < scenario.warmupQueries; i++) {
|
|
824
|
-
const queryVec = formatPgVector(queryVectors[i % queryVectors.length]);
|
|
825
|
-
await db.query(`SELECT id FROM embeddings ORDER BY vector <-> $1::vector LIMIT $2`, [queryVec, scenario.k]);
|
|
826
|
-
}
|
|
827
|
-
|
|
828
|
-
// Phase 3: Measured queries with recall tracking
|
|
829
|
-
console.log(' Running measured queries...');
|
|
830
|
-
const latencies = [];
|
|
831
|
-
const approximateResults = [];
|
|
832
|
-
|
|
833
|
-
for (let i = 0; i < scenario.queryCount; i++) {
|
|
834
|
-
const queryVec = formatPgVector(queryVectors[i]);
|
|
835
|
-
const start = performance.now();
|
|
836
|
-
const result = await db.query(
|
|
837
|
-
`SELECT id FROM embeddings ORDER BY vector <-> $1::vector LIMIT $2`,
|
|
838
|
-
[queryVec, scenario.k]
|
|
839
|
-
);
|
|
840
|
-
latencies.push(performance.now() - start);
|
|
841
|
-
|
|
842
|
-
// Collect IDs for recall calculation
|
|
843
|
-
approximateResults.push(result.rows.map(r => r.id));
|
|
844
|
-
}
|
|
845
|
-
|
|
846
|
-
// Calculate recall
|
|
847
|
-
const { recall } = calculateRecall(approximateResults, groundTruth, scenario.k);
|
|
848
|
-
|
|
849
|
-
// Calculate metrics
|
|
850
|
-
const totalTime = latencies.reduce((a, b) => a + b, 0);
|
|
851
|
-
const qps = Math.round((scenario.queryCount / totalTime) * 1000);
|
|
852
|
-
latencies.sort((a, b) => a - b);
|
|
853
|
-
const p50 = latencies[Math.floor(latencies.length * 0.5)]?.toFixed(1) || 0;
|
|
854
|
-
const p99 = latencies[Math.floor(latencies.length * 0.99)]?.toFixed(1) || 0;
|
|
855
|
-
|
|
856
|
-
try { await db.close(); } catch { /* ignore */ }
|
|
857
|
-
|
|
858
|
-
return {
|
|
859
|
-
throughput: qps,
|
|
860
|
-
recall: (recall * 100).toFixed(1), // as percentage
|
|
861
|
-
p50: parseFloat(p50),
|
|
862
|
-
p99: parseFloat(p99),
|
|
863
|
-
errors: 0,
|
|
864
|
-
totalOps: scenario.queryCount,
|
|
865
|
-
skipped: false
|
|
866
|
-
};
|
|
800
|
+
pool = await openPgPool({ ...POSTGRES_CONFIG });
|
|
801
|
+
return await runVectorScenarioOnPool(pool, scenario, embeddings, queryVectors, groundTruth);
|
|
867
802
|
} catch (error) {
|
|
868
|
-
console.error('
|
|
869
|
-
return
|
|
803
|
+
console.error(' PostgreSQL vector benchmark skipped:', error.message);
|
|
804
|
+
return skippedVector(error.message, 0);
|
|
805
|
+
} finally {
|
|
806
|
+
await pool?.end().catch(() => {});
|
|
870
807
|
}
|
|
871
808
|
}
|
|
872
809
|
|
|
873
810
|
/**
|
|
874
|
-
*
|
|
875
|
-
* Supports both INSERT and SEARCH scenarios
|
|
811
|
+
* pgserve 1.2.0 Vector Benchmark (published npm package)
|
|
876
812
|
*/
|
|
877
|
-
async function
|
|
878
|
-
console.log(
|
|
879
|
-
|
|
880
|
-
const pool = new Pool({
|
|
881
|
-
...POSTGRES_CONFIG,
|
|
882
|
-
max: 20
|
|
883
|
-
});
|
|
813
|
+
async function benchmarkPgserveV1Vector(scenario, embeddings, queryVectors, groundTruth) {
|
|
814
|
+
console.log(` 🧭 Running ${LEGACY_PGSERVE_SPEC} vector benchmark...`);
|
|
884
815
|
|
|
816
|
+
let legacy;
|
|
817
|
+
let pool;
|
|
885
818
|
try {
|
|
886
|
-
|
|
887
|
-
await
|
|
888
|
-
|
|
889
|
-
// Check if pgvector is available
|
|
890
|
-
try {
|
|
891
|
-
await pool.query('CREATE EXTENSION IF NOT EXISTS vector');
|
|
892
|
-
} catch (e) {
|
|
893
|
-
console.error(' pgvector extension not available. Use pgvector/pgvector:pg17 Docker image.');
|
|
894
|
-
await pool.end();
|
|
895
|
-
return { throughput: 0, recall: 'N/A', p50: 0, p99: 0, errors: 1, totalOps: 0, skipped: true };
|
|
896
|
-
}
|
|
897
|
-
|
|
898
|
-
// Setup schema
|
|
899
|
-
await pool.query(`
|
|
900
|
-
DROP TABLE IF EXISTS embeddings;
|
|
901
|
-
CREATE TABLE embeddings (
|
|
902
|
-
id INTEGER PRIMARY KEY,
|
|
903
|
-
vector vector(${scenario.dimension})
|
|
904
|
-
)
|
|
905
|
-
`);
|
|
906
|
-
|
|
907
|
-
// INSERT scenario - measure insert speed
|
|
908
|
-
if (scenario.type === 'INSERT') {
|
|
909
|
-
console.log(` Inserting ${scenario.insertCount} vectors (measured)...`);
|
|
910
|
-
const latencies = [];
|
|
911
|
-
|
|
912
|
-
for (let i = 0; i < scenario.insertCount; i++) {
|
|
913
|
-
const vec = formatPgVector(embeddings.vectors[i]);
|
|
914
|
-
const start = performance.now();
|
|
915
|
-
await pool.query('INSERT INTO embeddings (id, vector) VALUES ($1, $2::vector)', [i + 1, vec]);
|
|
916
|
-
latencies.push(performance.now() - start);
|
|
917
|
-
}
|
|
918
|
-
|
|
919
|
-
const totalTime = latencies.reduce((a, b) => a + b, 0);
|
|
920
|
-
const qps = Math.round((scenario.insertCount / totalTime) * 1000);
|
|
921
|
-
latencies.sort((a, b) => a - b);
|
|
922
|
-
const p50 = latencies[Math.floor(latencies.length * 0.5)]?.toFixed(1) || 0;
|
|
923
|
-
const p99 = latencies[Math.floor(latencies.length * 0.99)]?.toFixed(1) || 0;
|
|
924
|
-
|
|
925
|
-
await pool.query('DROP TABLE IF EXISTS embeddings');
|
|
926
|
-
await pool.end();
|
|
927
|
-
|
|
928
|
-
return {
|
|
929
|
-
throughput: qps,
|
|
930
|
-
recall: 'N/A',
|
|
931
|
-
p50: parseFloat(p50),
|
|
932
|
-
p99: parseFloat(p99),
|
|
933
|
-
errors: 0,
|
|
934
|
-
totalOps: scenario.insertCount,
|
|
935
|
-
skipped: false
|
|
936
|
-
};
|
|
937
|
-
}
|
|
938
|
-
|
|
939
|
-
// SEARCH scenario - Phase 1: Insert all vectors (not measured)
|
|
940
|
-
console.log(' Inserting vectors...');
|
|
941
|
-
for (let i = 0; i < embeddings.vectors.length; i++) {
|
|
942
|
-
const vec = formatPgVector(embeddings.vectors[i]);
|
|
943
|
-
await pool.query('INSERT INTO embeddings (id, vector) VALUES ($1, $2::vector)', [i + 1, vec]);
|
|
944
|
-
}
|
|
945
|
-
|
|
946
|
-
// Phase 2: Warm-up queries (not measured)
|
|
947
|
-
console.log(' Warming up...');
|
|
948
|
-
for (let i = 0; i < scenario.warmupQueries; i++) {
|
|
949
|
-
const queryVec = formatPgVector(queryVectors[i % queryVectors.length]);
|
|
950
|
-
await pool.query(`SELECT id FROM embeddings ORDER BY vector <-> $1::vector LIMIT $2`, [queryVec, scenario.k]);
|
|
951
|
-
}
|
|
952
|
-
|
|
953
|
-
// Phase 3: Measured queries with recall tracking
|
|
954
|
-
console.log(' Running measured queries...');
|
|
955
|
-
const latencies = [];
|
|
956
|
-
const approximateResults = [];
|
|
957
|
-
|
|
958
|
-
for (let i = 0; i < scenario.queryCount; i++) {
|
|
959
|
-
const queryVec = formatPgVector(queryVectors[i]);
|
|
960
|
-
const start = performance.now();
|
|
961
|
-
const result = await pool.query(
|
|
962
|
-
`SELECT id FROM embeddings ORDER BY vector <-> $1::vector LIMIT $2`,
|
|
963
|
-
[queryVec, scenario.k]
|
|
964
|
-
);
|
|
965
|
-
latencies.push(performance.now() - start);
|
|
966
|
-
|
|
967
|
-
// Collect IDs for recall calculation
|
|
968
|
-
approximateResults.push(result.rows.map(r => r.id));
|
|
969
|
-
}
|
|
970
|
-
|
|
971
|
-
// Calculate recall
|
|
972
|
-
const { recall } = calculateRecall(approximateResults, groundTruth, scenario.k);
|
|
973
|
-
|
|
974
|
-
// Calculate metrics
|
|
975
|
-
const totalTime = latencies.reduce((a, b) => a + b, 0);
|
|
976
|
-
const qps = Math.round((scenario.queryCount / totalTime) * 1000);
|
|
977
|
-
latencies.sort((a, b) => a - b);
|
|
978
|
-
const p50 = latencies[Math.floor(latencies.length * 0.5)]?.toFixed(1) || 0;
|
|
979
|
-
const p99 = latencies[Math.floor(latencies.length * 0.99)]?.toFixed(1) || 0;
|
|
980
|
-
|
|
981
|
-
// Cleanup
|
|
982
|
-
await pool.query('DROP TABLE IF EXISTS embeddings');
|
|
983
|
-
await pool.end();
|
|
984
|
-
|
|
985
|
-
return {
|
|
986
|
-
throughput: qps,
|
|
987
|
-
recall: (recall * 100).toFixed(1), // as percentage
|
|
988
|
-
p50: parseFloat(p50),
|
|
989
|
-
p99: parseFloat(p99),
|
|
990
|
-
errors: 0,
|
|
991
|
-
totalOps: scenario.queryCount,
|
|
992
|
-
skipped: false
|
|
993
|
-
};
|
|
819
|
+
legacy = await startLegacyPgserve({ port: 18434, enablePgvector: true });
|
|
820
|
+
pool = await openPgPool({ port: 18434, database: 'vector_bench' });
|
|
821
|
+
return await runVectorScenarioOnPool(pool, scenario, embeddings, queryVectors, groundTruth);
|
|
994
822
|
} catch (error) {
|
|
995
|
-
console.error(
|
|
996
|
-
|
|
997
|
-
|
|
823
|
+
console.error(` ${LEGACY_PGSERVE_SPEC} vector benchmark skipped:`, error.message);
|
|
824
|
+
return skippedVector(error.message);
|
|
825
|
+
} finally {
|
|
826
|
+
await pool?.end().catch(() => {});
|
|
827
|
+
await legacy?.stop().catch(() => {});
|
|
998
828
|
}
|
|
999
829
|
}
|
|
1000
830
|
|
|
@@ -1004,162 +834,28 @@ async function benchmarkPostgreSQLVector(scenario, embeddings, queryVectors, gro
|
|
|
1004
834
|
*/
|
|
1005
835
|
async function benchmarkPgserveVector(scenario, embeddings, queryVectors, groundTruth, useRam = false) {
|
|
1006
836
|
const mode = useRam ? 'RAM' : 'disk';
|
|
1007
|
-
console.log(` 🚀 Running pgserve (${mode}) vector benchmark...`);
|
|
837
|
+
console.log(` 🚀 Running pgserve v2 (${mode}) vector benchmark...`);
|
|
1008
838
|
|
|
1009
839
|
let server;
|
|
840
|
+
let pool;
|
|
1010
841
|
try {
|
|
1011
842
|
// Start pgserve (use different ports for vector benchmarks to avoid conflicts)
|
|
1012
|
-
const port = useRam ?
|
|
843
|
+
const port = useRam ? 18436 : 18435;
|
|
1013
844
|
server = await startMultiTenantServer({
|
|
1014
845
|
port,
|
|
1015
846
|
logLevel: 'error',
|
|
1016
|
-
useRam
|
|
847
|
+
useRam,
|
|
848
|
+
enablePgvector: true
|
|
1017
849
|
});
|
|
1018
850
|
|
|
1019
|
-
|
|
1020
|
-
await
|
|
1021
|
-
|
|
1022
|
-
const pool = new Pool({
|
|
1023
|
-
host: 'localhost',
|
|
1024
|
-
port,
|
|
1025
|
-
database: 'vector_bench',
|
|
1026
|
-
user: 'postgres',
|
|
1027
|
-
password: 'postgres',
|
|
1028
|
-
max: 20,
|
|
1029
|
-
connectionTimeoutMillis: 30000
|
|
1030
|
-
});
|
|
1031
|
-
|
|
1032
|
-
// Wait for connection with retries
|
|
1033
|
-
let connected = false;
|
|
1034
|
-
for (let i = 0; i < 10; i++) {
|
|
1035
|
-
try {
|
|
1036
|
-
await pool.query('SELECT 1');
|
|
1037
|
-
connected = true;
|
|
1038
|
-
break;
|
|
1039
|
-
} catch (error) {
|
|
1040
|
-
if (i === 9) throw error;
|
|
1041
|
-
await new Promise(resolve => setTimeout(resolve, 1000));
|
|
1042
|
-
}
|
|
1043
|
-
}
|
|
1044
|
-
|
|
1045
|
-
if (!connected) {
|
|
1046
|
-
throw new Error('Failed to connect to pgserve');
|
|
1047
|
-
}
|
|
1048
|
-
|
|
1049
|
-
// Enable pgvector extension
|
|
1050
|
-
try {
|
|
1051
|
-
await pool.query('CREATE EXTENSION IF NOT EXISTS vector');
|
|
1052
|
-
} catch (e) {
|
|
1053
|
-
console.error(` pgvector extension not available in pgserve. Install pgvector files to ~/.pgserve/bin/<platform>/`);
|
|
1054
|
-
await pool.end();
|
|
1055
|
-
await server.stop();
|
|
1056
|
-
return { throughput: 0, recall: 'N/A', p50: 0, p99: 0, errors: 0, totalOps: 0, skipped: true, reason: 'pgvector not installed' };
|
|
1057
|
-
}
|
|
1058
|
-
|
|
1059
|
-
// Setup schema
|
|
1060
|
-
await pool.query(`
|
|
1061
|
-
DROP TABLE IF EXISTS embeddings;
|
|
1062
|
-
CREATE TABLE embeddings (
|
|
1063
|
-
id INTEGER PRIMARY KEY,
|
|
1064
|
-
vector vector(${scenario.dimension})
|
|
1065
|
-
)
|
|
1066
|
-
`);
|
|
1067
|
-
|
|
1068
|
-
// INSERT scenario - measure insert speed
|
|
1069
|
-
if (scenario.type === 'INSERT') {
|
|
1070
|
-
console.log(` Inserting ${scenario.insertCount} vectors (measured)...`);
|
|
1071
|
-
const latencies = [];
|
|
1072
|
-
|
|
1073
|
-
for (let i = 0; i < scenario.insertCount; i++) {
|
|
1074
|
-
const vec = formatPgVector(embeddings.vectors[i]);
|
|
1075
|
-
const start = performance.now();
|
|
1076
|
-
await pool.query('INSERT INTO embeddings (id, vector) VALUES ($1, $2::vector)', [i + 1, vec]);
|
|
1077
|
-
latencies.push(performance.now() - start);
|
|
1078
|
-
}
|
|
1079
|
-
|
|
1080
|
-
const totalTime = latencies.reduce((a, b) => a + b, 0);
|
|
1081
|
-
const qps = Math.round((scenario.insertCount / totalTime) * 1000);
|
|
1082
|
-
latencies.sort((a, b) => a - b);
|
|
1083
|
-
const p50 = latencies[Math.floor(latencies.length * 0.5)]?.toFixed(1) || 0;
|
|
1084
|
-
const p99 = latencies[Math.floor(latencies.length * 0.99)]?.toFixed(1) || 0;
|
|
1085
|
-
|
|
1086
|
-
await pool.query('DROP TABLE IF EXISTS embeddings');
|
|
1087
|
-
await pool.end();
|
|
1088
|
-
await server.stop();
|
|
1089
|
-
|
|
1090
|
-
return {
|
|
1091
|
-
throughput: qps,
|
|
1092
|
-
recall: 'N/A',
|
|
1093
|
-
p50: parseFloat(p50),
|
|
1094
|
-
p99: parseFloat(p99),
|
|
1095
|
-
errors: 0,
|
|
1096
|
-
totalOps: scenario.insertCount,
|
|
1097
|
-
skipped: false
|
|
1098
|
-
};
|
|
1099
|
-
}
|
|
1100
|
-
|
|
1101
|
-
// SEARCH scenario - Phase 1: Insert all vectors (not measured)
|
|
1102
|
-
console.log(' Inserting vectors...');
|
|
1103
|
-
for (let i = 0; i < embeddings.vectors.length; i++) {
|
|
1104
|
-
const vec = formatPgVector(embeddings.vectors[i]);
|
|
1105
|
-
await pool.query('INSERT INTO embeddings (id, vector) VALUES ($1, $2::vector)', [i + 1, vec]);
|
|
1106
|
-
}
|
|
1107
|
-
|
|
1108
|
-
// Phase 2: Warm-up queries (not measured)
|
|
1109
|
-
console.log(' Warming up...');
|
|
1110
|
-
for (let i = 0; i < scenario.warmupQueries; i++) {
|
|
1111
|
-
const queryVec = formatPgVector(queryVectors[i % queryVectors.length]);
|
|
1112
|
-
await pool.query(`SELECT id FROM embeddings ORDER BY vector <-> $1::vector LIMIT $2`, [queryVec, scenario.k]);
|
|
1113
|
-
}
|
|
1114
|
-
|
|
1115
|
-
// Phase 3: Measured queries with recall tracking
|
|
1116
|
-
console.log(' Running measured queries...');
|
|
1117
|
-
const latencies = [];
|
|
1118
|
-
const approximateResults = [];
|
|
1119
|
-
|
|
1120
|
-
for (let i = 0; i < scenario.queryCount; i++) {
|
|
1121
|
-
const queryVec = formatPgVector(queryVectors[i]);
|
|
1122
|
-
const start = performance.now();
|
|
1123
|
-
const result = await pool.query(
|
|
1124
|
-
`SELECT id FROM embeddings ORDER BY vector <-> $1::vector LIMIT $2`,
|
|
1125
|
-
[queryVec, scenario.k]
|
|
1126
|
-
);
|
|
1127
|
-
latencies.push(performance.now() - start);
|
|
1128
|
-
|
|
1129
|
-
// Collect IDs for recall calculation
|
|
1130
|
-
approximateResults.push(result.rows.map(r => r.id));
|
|
1131
|
-
}
|
|
1132
|
-
|
|
1133
|
-
// Calculate recall
|
|
1134
|
-
const { recall } = calculateRecall(approximateResults, groundTruth, scenario.k);
|
|
1135
|
-
|
|
1136
|
-
// Calculate metrics
|
|
1137
|
-
const totalTime = latencies.reduce((a, b) => a + b, 0);
|
|
1138
|
-
const qps = Math.round((scenario.queryCount / totalTime) * 1000);
|
|
1139
|
-
latencies.sort((a, b) => a - b);
|
|
1140
|
-
const p50 = latencies[Math.floor(latencies.length * 0.5)]?.toFixed(1) || 0;
|
|
1141
|
-
const p99 = latencies[Math.floor(latencies.length * 0.99)]?.toFixed(1) || 0;
|
|
1142
|
-
|
|
1143
|
-
// Cleanup
|
|
1144
|
-
await pool.query('DROP TABLE IF EXISTS embeddings');
|
|
1145
|
-
await pool.end();
|
|
1146
|
-
await server.stop();
|
|
1147
|
-
|
|
1148
|
-
return {
|
|
1149
|
-
throughput: qps,
|
|
1150
|
-
recall: (recall * 100).toFixed(1), // as percentage
|
|
1151
|
-
p50: parseFloat(p50),
|
|
1152
|
-
p99: parseFloat(p99),
|
|
1153
|
-
errors: 0,
|
|
1154
|
-
totalOps: scenario.queryCount,
|
|
1155
|
-
skipped: false
|
|
1156
|
-
};
|
|
851
|
+
pool = await openPgPool({ port, database: 'vector_bench' });
|
|
852
|
+
return await runVectorScenarioOnPool(pool, scenario, embeddings, queryVectors, groundTruth);
|
|
1157
853
|
} catch (error) {
|
|
1158
|
-
console.error(` pgserve (${mode}) vector benchmark failed:`, error.message);
|
|
1159
|
-
|
|
1160
|
-
|
|
1161
|
-
}
|
|
1162
|
-
|
|
854
|
+
console.error(` pgserve v2 (${mode}) vector benchmark failed:`, error.message);
|
|
855
|
+
return skippedVector(error.message, 0);
|
|
856
|
+
} finally {
|
|
857
|
+
await pool?.end().catch(() => {});
|
|
858
|
+
await server?.stop().catch(() => {});
|
|
1163
859
|
}
|
|
1164
860
|
}
|
|
1165
861
|
|
|
@@ -1188,89 +884,66 @@ function generateReport(results, vectorResults = []) {
|
|
|
1188
884
|
md += 'psql postgresql://localhost:5432/mydb\n';
|
|
1189
885
|
md += '```\n\n';
|
|
1190
886
|
|
|
887
|
+
const metricValue = (data, key) => {
|
|
888
|
+
if (!data || data.skipped) return 'N/A';
|
|
889
|
+
const value = data[key];
|
|
890
|
+
return value === undefined || value === null ? 'N/A' : value;
|
|
891
|
+
};
|
|
892
|
+
const metricNumber = (data, key) => {
|
|
893
|
+
if (!data || data.skipped) return null;
|
|
894
|
+
const value = Number.parseFloat(data[key]);
|
|
895
|
+
return Number.isFinite(value) ? value : null;
|
|
896
|
+
};
|
|
897
|
+
const winnerName = (rows, key, direction) => {
|
|
898
|
+
let winner = null;
|
|
899
|
+
for (const row of rows) {
|
|
900
|
+
const value = metricNumber(row.data, key);
|
|
901
|
+
if (value === null) continue;
|
|
902
|
+
if (!winner || (direction === 'max' ? value > winner.value : value < winner.value)) {
|
|
903
|
+
winner = { name: row.name, value };
|
|
904
|
+
}
|
|
905
|
+
}
|
|
906
|
+
return winner?.name || 'N/A';
|
|
907
|
+
};
|
|
908
|
+
const pctDelta = (current, baseline) => {
|
|
909
|
+
if (!current || !baseline || current.skipped || baseline.skipped || baseline.throughput <= 0) return null;
|
|
910
|
+
return ((current.throughput / baseline.throughput - 1) * 100).toFixed(1);
|
|
911
|
+
};
|
|
912
|
+
const renderMetricTable = (rows, metrics) => {
|
|
913
|
+
let table = `| Metric | ${rows.map(r => r.name).join(' | ')} | Winner |\n`;
|
|
914
|
+
table += `| --- | ${rows.map(() => '---:').join(' | ')} | --- |\n`;
|
|
915
|
+
for (const metric of metrics) {
|
|
916
|
+
table += `| ${metric.label} | ${rows.map(r => metric.format(metricValue(r.data, metric.key))).join(' | ')} | ${winnerName(rows, metric.key, metric.direction)} |\n`;
|
|
917
|
+
}
|
|
918
|
+
return `${table}\n`;
|
|
919
|
+
};
|
|
920
|
+
const plain = (value) => String(value);
|
|
921
|
+
const percent = (value) => value === 'N/A' ? value : `${value}%`;
|
|
922
|
+
|
|
1191
923
|
for (const scenario of results) {
|
|
1192
924
|
md += `## ${scenario.name}\n\n`;
|
|
1193
925
|
md += `${scenario.description}\n\n`;
|
|
1194
926
|
|
|
1195
|
-
const
|
|
1196
|
-
|
|
1197
|
-
|
|
1198
|
-
|
|
1199
|
-
|
|
1200
|
-
|
|
1201
|
-
|
|
1202
|
-
|
|
1203
|
-
|
|
1204
|
-
|
|
1205
|
-
|
|
1206
|
-
|
|
1207
|
-
|
|
1208
|
-
|
|
1209
|
-
|
|
1210
|
-
|
|
1211
|
-
|
|
1212
|
-
|
|
1213
|
-
|
|
1214
|
-
|
|
1215
|
-
const nameMap = { sqlite: 'SQLite', pglite: 'PGlite', postgres: 'PostgreSQL', pgserve: 'pgserve', pgserveRam: 'pgserve RAM' };
|
|
1216
|
-
|
|
1217
|
-
const pad = (s, n) => String(s).padEnd(n);
|
|
1218
|
-
|
|
1219
|
-
md += `│ Throughput (qps)│ ${pad(sqlite.throughput, 8)} │ ${pad(pglite.throughput, 8)} │ ${pad(postgres.throughput, 9)} │ ${pad(pgserve.throughput, 8)} │ ${pad(pgserveRam.throughput, 11)} │ ${pad(nameMap[getMaxKey(throughputs)], 11)} │\n`;
|
|
1220
|
-
md += `│ P50 latency (ms)│ ${pad(sqlite.p50, 8)} │ ${pad(pglite.p50, 8)} │ ${pad(postgres.p50, 9)} │ ${pad(pgserve.p50, 8)} │ ${pad(pgserveRam.p50, 11)} │ ${pad(nameMap[getMinKey(p50s)], 11)} │\n`;
|
|
1221
|
-
md += `│ P99 latency (ms)│ ${pad(sqlite.p99, 8)} │ ${pad(pglite.p99, 8)} │ ${pad(postgres.p99, 9)} │ ${pad(pgserve.p99, 8)} │ ${pad(pgserveRam.p99, 11)} │ ${pad(nameMap[getMinKey(p99s)], 11)} │\n`;
|
|
1222
|
-
md += `│ Errors │ ${pad(sqlite.errors, 8)} │ ${pad(pglite.errors, 8)} │ ${pad(postgres.errors, 9)} │ ${pad(pgserve.errors, 8)} │ ${pad(pgserveRam.errors, 11)} │ ${pad(nameMap[getMinErrorKey(errors)], 11)} │\n`;
|
|
1223
|
-
md += '└─────────────────┴──────────┴──────────┴──────────┴──────────┴─────────────┴─────────────┘\n';
|
|
1224
|
-
md += '```\n\n';
|
|
1225
|
-
|
|
1226
|
-
// Analysis with RAM comparison
|
|
1227
|
-
const winner = nameMap[getMaxKey(throughputs)];
|
|
1228
|
-
if (winner === 'pgserve RAM') {
|
|
1229
|
-
const vsDisk = pgserve.throughput > 0 ? ((pgserveRam.throughput / pgserve.throughput - 1) * 100).toFixed(1) : 'N/A';
|
|
1230
|
-
const vsPGlite = pglite.throughput > 0 ? ((pgserveRam.throughput / pglite.throughput - 1) * 100).toFixed(1) : 'N/A';
|
|
1231
|
-
md += `**pgserve RAM wins!** ${vsDisk}% faster than disk mode, ${vsPGlite}% faster than PGlite.\n\n`;
|
|
1232
|
-
} else if (winner === 'pgserve') {
|
|
1233
|
-
const vsPGlite = pglite.throughput > 0 ? ((pgserve.throughput / pglite.throughput - 1) * 100).toFixed(1) : 'N/A';
|
|
1234
|
-
md += `**pgserve wins!** ${vsPGlite}% faster than PGlite for concurrent workloads.\n\n`;
|
|
1235
|
-
} else {
|
|
1236
|
-
md += `**${winner} wins** this scenario.\n\n`;
|
|
1237
|
-
}
|
|
1238
|
-
} else {
|
|
1239
|
-
// Original table without RAM column
|
|
1240
|
-
md += '```\n';
|
|
1241
|
-
md += '┌─────────────────┬──────────┬──────────┬──────────┬──────────┬──────────┐\n';
|
|
1242
|
-
md += '│ Metric │ SQLite │ PGlite │ PostgreSQL│ pgserve │ Winner │\n';
|
|
1243
|
-
md += '├─────────────────┼──────────┼──────────┼──────────┼──────────┼──────────┤\n';
|
|
1244
|
-
|
|
1245
|
-
// Find winners
|
|
1246
|
-
const throughputs = { sqlite: sqlite.throughput, pglite: pglite.throughput, postgres: postgres.throughput, pgserve: pgserve.throughput };
|
|
1247
|
-
const p50s = { sqlite: sqlite.p50, pglite: pglite.p50, postgres: postgres.p50, pgserve: pgserve.p50 };
|
|
1248
|
-
const p99s = { sqlite: sqlite.p99, pglite: pglite.p99, postgres: postgres.p99, pgserve: pgserve.p99 };
|
|
1249
|
-
const errors = { sqlite: sqlite.errors, pglite: pglite.errors, postgres: postgres.errors, pgserve: pgserve.errors };
|
|
1250
|
-
|
|
1251
|
-
const getMaxKey = (obj) => Object.entries(obj).reduce((a, b) => a[1] > b[1] ? a : b)[0];
|
|
1252
|
-
const getMinKey = (obj) => Object.entries(obj).filter(([k,v]) => v > 0 || k === 'sqlite').reduce((a, b) => a[1] < b[1] ? a : b)[0];
|
|
1253
|
-
const getMinErrorKey = (obj) => Object.entries(obj).reduce((a, b) => a[1] <= b[1] ? a : b)[0];
|
|
1254
|
-
|
|
1255
|
-
const nameMap = { sqlite: 'SQLite', pglite: 'PGlite', postgres: 'PostgreSQL', pgserve: 'pgserve' };
|
|
1256
|
-
|
|
1257
|
-
const pad = (s, n) => String(s).padEnd(n);
|
|
1258
|
-
|
|
1259
|
-
md += `│ Throughput (qps)│ ${pad(sqlite.throughput, 8)} │ ${pad(pglite.throughput, 8)} │ ${pad(postgres.throughput, 9)} │ ${pad(pgserve.throughput, 8)} │ ${pad(nameMap[getMaxKey(throughputs)], 8)} │\n`;
|
|
1260
|
-
md += `│ P50 latency (ms)│ ${pad(sqlite.p50, 8)} │ ${pad(pglite.p50, 8)} │ ${pad(postgres.p50, 9)} │ ${pad(pgserve.p50, 8)} │ ${pad(nameMap[getMinKey(p50s)], 8)} │\n`;
|
|
1261
|
-
md += `│ P99 latency (ms)│ ${pad(sqlite.p99, 8)} │ ${pad(pglite.p99, 8)} │ ${pad(postgres.p99, 9)} │ ${pad(pgserve.p99, 8)} │ ${pad(nameMap[getMinKey(p99s)], 8)} │\n`;
|
|
1262
|
-
md += `│ Errors │ ${pad(sqlite.errors, 8)} │ ${pad(pglite.errors, 8)} │ ${pad(postgres.errors, 9)} │ ${pad(pgserve.errors, 8)} │ ${pad(nameMap[getMinErrorKey(errors)], 8)} │\n`;
|
|
1263
|
-
md += '└─────────────────┴──────────┴──────────┴──────────┴──────────┴──────────┘\n';
|
|
1264
|
-
md += '```\n\n';
|
|
1265
|
-
|
|
1266
|
-
// Analysis
|
|
1267
|
-
const winner = nameMap[getMaxKey(throughputs)];
|
|
1268
|
-
if (winner === 'pgserve') {
|
|
1269
|
-
const vsPGlite = pglite.throughput > 0 ? ((pgserve.throughput / pglite.throughput - 1) * 100).toFixed(1) : 'N/A';
|
|
1270
|
-
md += `**pgserve wins!** ${vsPGlite}% faster than PGlite for concurrent workloads.\n\n`;
|
|
1271
|
-
} else {
|
|
1272
|
-
md += `**${winner} wins** this scenario.\n\n`;
|
|
1273
|
-
}
|
|
927
|
+
const rows = [
|
|
928
|
+
{ name: 'SQLite', data: scenario.sqlite },
|
|
929
|
+
{ name: 'PostgreSQL', data: scenario.postgres },
|
|
930
|
+
{ name: 'pgserve 1.2.0', data: scenario.pgserveV1 },
|
|
931
|
+
{ name: 'pgserve v2', data: scenario.pgserve },
|
|
932
|
+
];
|
|
933
|
+
if (scenario.pgserveRam && !scenario.pgserveRam.skipped) {
|
|
934
|
+
rows.push({ name: 'pgserve v2 RAM', data: scenario.pgserveRam });
|
|
935
|
+
}
|
|
936
|
+
|
|
937
|
+
md += renderMetricTable(rows, [
|
|
938
|
+
{ label: 'Throughput (qps)', key: 'throughput', direction: 'max', format: plain },
|
|
939
|
+
{ label: 'P50 latency (ms)', key: 'p50', direction: 'min', format: plain },
|
|
940
|
+
{ label: 'P99 latency (ms)', key: 'p99', direction: 'min', format: plain },
|
|
941
|
+
{ label: 'Errors', key: 'errors', direction: 'min', format: plain },
|
|
942
|
+
]);
|
|
943
|
+
|
|
944
|
+
const delta = pctDelta(scenario.pgserve, scenario.pgserveV1);
|
|
945
|
+
if (delta !== null) {
|
|
946
|
+
md += `**pgserve v2 vs 1.2.0:** ${delta}% throughput delta.\n\n`;
|
|
1274
947
|
}
|
|
1275
948
|
}
|
|
1276
949
|
|
|
@@ -1284,34 +957,32 @@ function generateReport(results, vectorResults = []) {
|
|
|
1284
957
|
md += `### ${scenario.name}\n\n`;
|
|
1285
958
|
md += `${scenario.description}\n\n`;
|
|
1286
959
|
|
|
1287
|
-
const
|
|
1288
|
-
|
|
1289
|
-
|
|
1290
|
-
|
|
1291
|
-
|
|
1292
|
-
|
|
1293
|
-
|
|
1294
|
-
|
|
1295
|
-
const val = (r, key) => r.skipped ? 'N/A' : r[key];
|
|
1296
|
-
const recallVal = (r) => r.skipped ? 'N/A' : `${r.recall}%`;
|
|
960
|
+
const rows = [
|
|
961
|
+
{ name: 'PostgreSQL', data: scenario.postgres },
|
|
962
|
+
{ name: 'pgserve 1.2.0', data: scenario.pgserveV1 },
|
|
963
|
+
{ name: 'pgserve v2', data: scenario.pgserve },
|
|
964
|
+
];
|
|
965
|
+
if (scenario.pgserveRam && !scenario.pgserveRam.skipped) {
|
|
966
|
+
rows.push({ name: 'pgserve v2 RAM', data: scenario.pgserveRam });
|
|
967
|
+
}
|
|
1297
968
|
|
|
1298
|
-
md +=
|
|
1299
|
-
|
|
1300
|
-
|
|
1301
|
-
|
|
1302
|
-
|
|
1303
|
-
|
|
1304
|
-
|
|
969
|
+
md += renderMetricTable(rows, [
|
|
970
|
+
{ label: `Recall@${scenario.k || 10}`, key: 'recall', direction: 'max', format: percent },
|
|
971
|
+
{ label: 'Throughput (qps)', key: 'throughput', direction: 'max', format: plain },
|
|
972
|
+
{ label: 'P50 latency (ms)', key: 'p50', direction: 'min', format: plain },
|
|
973
|
+
{ label: 'P99 latency (ms)', key: 'p99', direction: 'min', format: plain },
|
|
974
|
+
{ label: 'Errors', key: 'errors', direction: 'min', format: plain },
|
|
975
|
+
]);
|
|
1305
976
|
|
|
1306
977
|
// Find winner among non-skipped (considering both recall and throughput)
|
|
1307
978
|
const candidates = {};
|
|
1308
|
-
if (!
|
|
1309
|
-
if (!
|
|
1310
|
-
if (!pgserve.skipped) candidates.pgserve = { recall: parseFloat(pgserve.recall), qps: pgserve.throughput };
|
|
1311
|
-
if (pgserveRam && !pgserveRam.skipped) candidates.pgserveRam = { recall: parseFloat(pgserveRam.recall), qps: pgserveRam.throughput };
|
|
979
|
+
if (!scenario.postgres.skipped) candidates.postgres = { recall: parseFloat(scenario.postgres.recall), qps: scenario.postgres.throughput };
|
|
980
|
+
if (!scenario.pgserveV1.skipped) candidates.pgserveV1 = { recall: parseFloat(scenario.pgserveV1.recall), qps: scenario.pgserveV1.throughput };
|
|
981
|
+
if (!scenario.pgserve.skipped) candidates.pgserve = { recall: parseFloat(scenario.pgserve.recall), qps: scenario.pgserve.throughput };
|
|
982
|
+
if (scenario.pgserveRam && !scenario.pgserveRam.skipped) candidates.pgserveRam = { recall: parseFloat(scenario.pgserveRam.recall), qps: scenario.pgserveRam.throughput };
|
|
1312
983
|
|
|
1313
984
|
if (Object.keys(candidates).length > 0) {
|
|
1314
|
-
const nameMap = {
|
|
985
|
+
const nameMap = { postgres: 'PostgreSQL', pgserveV1: 'pgserve 1.2.0', pgserve: 'pgserve v2', pgserveRam: 'pgserve v2 RAM' };
|
|
1315
986
|
// Winner = highest QPS among those with 100% recall, otherwise highest recall
|
|
1316
987
|
const perfect = Object.entries(candidates).filter(([, v]) => v.recall === 100);
|
|
1317
988
|
let winnerKey;
|
|
@@ -1323,6 +994,11 @@ function generateReport(results, vectorResults = []) {
|
|
|
1323
994
|
md += `**${nameMap[winnerKey]} wins** (${candidates[winnerKey].recall}% recall @ ${candidates[winnerKey].qps} qps)\n\n`;
|
|
1324
995
|
}
|
|
1325
996
|
}
|
|
997
|
+
|
|
998
|
+
const delta = pctDelta(scenario.pgserve, scenario.pgserveV1);
|
|
999
|
+
if (delta !== null) {
|
|
1000
|
+
md += `**pgserve v2 vs 1.2.0:** ${delta}% throughput delta.\n\n`;
|
|
1001
|
+
}
|
|
1326
1002
|
}
|
|
1327
1003
|
}
|
|
1328
1004
|
|
|
@@ -1366,9 +1042,9 @@ Options:
|
|
|
1366
1042
|
--help, -h Show this help
|
|
1367
1043
|
|
|
1368
1044
|
Vector benchmarks require:
|
|
1369
|
-
-
|
|
1045
|
+
- PostgreSQL: Built-in pgvector support
|
|
1370
1046
|
- PostgreSQL: Docker image pgvector/pgvector:pg17
|
|
1371
|
-
- pgserve
|
|
1047
|
+
- pgserve 1.2.0 and v2: --pgvector support
|
|
1372
1048
|
`);
|
|
1373
1049
|
process.exit(0);
|
|
1374
1050
|
}
|
|
@@ -1401,8 +1077,8 @@ Vector benchmarks require:
|
|
|
1401
1077
|
section(scenario.name, scenario.description);
|
|
1402
1078
|
|
|
1403
1079
|
const sqlite = await benchmarkSQLite(scenario);
|
|
1404
|
-
const pglite = await benchmarkPGlite(scenario);
|
|
1405
1080
|
const postgres = await benchmarkPostgreSQL(scenario);
|
|
1081
|
+
const pgserveV1 = await benchmarkPgserveV1(scenario);
|
|
1406
1082
|
const pgserve = await benchmarkPgserve(scenario, false); // disk mode
|
|
1407
1083
|
const pgserveRam = canUseRam
|
|
1408
1084
|
? await benchmarkPgserve(scenario, true) // RAM mode
|
|
@@ -1412,8 +1088,8 @@ Vector benchmarks require:
|
|
|
1412
1088
|
name: scenario.name,
|
|
1413
1089
|
description: scenario.description,
|
|
1414
1090
|
sqlite,
|
|
1415
|
-
pglite,
|
|
1416
1091
|
postgres,
|
|
1092
|
+
pgserveV1,
|
|
1417
1093
|
pgserve,
|
|
1418
1094
|
pgserveRam
|
|
1419
1095
|
});
|
|
@@ -1422,11 +1098,11 @@ Vector benchmarks require:
|
|
|
1422
1098
|
console.log(`\n ${C.bold}Results:${C.reset}`);
|
|
1423
1099
|
console.log(` ${C.dim}──────────────────────────────────────────${C.reset}`);
|
|
1424
1100
|
console.log(` ${C.yellow}🔸${C.reset} SQLite: ${C.bold}${sqlite.throughput}${C.reset} qps, P50=${sqlite.p50}ms, P99=${sqlite.p99}ms`);
|
|
1425
|
-
console.log(` ${C.blue}🔹${C.reset} PGlite: ${C.bold}${pglite.throughput}${C.reset} qps, P50=${pglite.p50}ms, P99=${pglite.p99}ms`);
|
|
1426
1101
|
console.log(` ${C.cyan}🔷${C.reset} PostgreSQL: ${postgres.skipped ? `${C.dim}SKIPPED${C.reset}` : `${C.bold}${postgres.throughput}${C.reset} qps, P50=${postgres.p50}ms, P99=${postgres.p99}ms`}`);
|
|
1427
|
-
console.log(` ${C.
|
|
1102
|
+
console.log(` ${C.blue}🧭${C.reset} pgserve 1.2.0: ${pgserveV1.skipped ? `${C.dim}SKIPPED${C.reset}` : `${C.bold}${pgserveV1.throughput}${C.reset} qps, P50=${pgserveV1.p50}ms, P99=${pgserveV1.p99}ms`}`);
|
|
1103
|
+
console.log(` ${C.green}🚀${C.reset} pgserve v2: ${C.bold}${pgserve.throughput}${C.reset} qps, P50=${pgserve.p50}ms, P99=${pgserve.p99}ms`);
|
|
1428
1104
|
if (canUseRam) {
|
|
1429
|
-
console.log(` ${C.magenta}⚡${C.reset} pgserve
|
|
1105
|
+
console.log(` ${C.magenta}⚡${C.reset} pgserve v2 RAM: ${C.bold}${pgserveRam.throughput}${C.reset} qps, P50=${pgserveRam.p50}ms, P99=${pgserveRam.p99}ms`);
|
|
1430
1106
|
}
|
|
1431
1107
|
}
|
|
1432
1108
|
}
|
|
@@ -1466,8 +1142,8 @@ Vector benchmarks require:
|
|
|
1466
1142
|
}
|
|
1467
1143
|
|
|
1468
1144
|
// Run benchmarks
|
|
1469
|
-
const pglite = await benchmarkPGliteVector(scenario, embeddings, queryVectors, groundTruth);
|
|
1470
1145
|
const postgres = await benchmarkPostgreSQLVector(scenario, embeddings, queryVectors, groundTruth);
|
|
1146
|
+
const pgserveV1 = await benchmarkPgserveV1Vector(scenario, embeddings, queryVectors, groundTruth);
|
|
1471
1147
|
const pgserve = await benchmarkPgserveVector(scenario, embeddings, queryVectors, groundTruth, false);
|
|
1472
1148
|
const pgserveRam = canUseRam
|
|
1473
1149
|
? await benchmarkPgserveVector(scenario, embeddings, queryVectors, groundTruth, true)
|
|
@@ -1478,8 +1154,8 @@ Vector benchmarks require:
|
|
|
1478
1154
|
description: scenario.description,
|
|
1479
1155
|
type: scenario.type,
|
|
1480
1156
|
k: scenario.k,
|
|
1481
|
-
pglite,
|
|
1482
1157
|
postgres,
|
|
1158
|
+
pgserveV1,
|
|
1483
1159
|
pgserve,
|
|
1484
1160
|
pgserveRam
|
|
1485
1161
|
});
|
|
@@ -1495,11 +1171,11 @@ Vector benchmarks require:
|
|
|
1495
1171
|
}
|
|
1496
1172
|
return ` ${color}${icon}${C.reset} ${name.padEnd(14)} Recall: ${C.bold}${r.recall}%${C.reset}, ${C.bold}${r.throughput}${C.reset} qps, P50=${r.p50}ms`;
|
|
1497
1173
|
};
|
|
1498
|
-
console.log(formatResult(pglite, '🔹', C.blue, 'PGlite:'));
|
|
1499
1174
|
console.log(formatResult(postgres, '🔷', C.cyan, 'PostgreSQL:'));
|
|
1500
|
-
console.log(formatResult(
|
|
1175
|
+
console.log(formatResult(pgserveV1, '🧭', C.blue, 'pgserve 1.2.0:'));
|
|
1176
|
+
console.log(formatResult(pgserve, '🚀', C.green, 'pgserve v2:'));
|
|
1501
1177
|
if (canUseRam) {
|
|
1502
|
-
console.log(formatResult(pgserveRam, '⚡', C.magenta, 'pgserve
|
|
1178
|
+
console.log(formatResult(pgserveRam, '⚡', C.magenta, 'pgserve v2 RAM:'));
|
|
1503
1179
|
}
|
|
1504
1180
|
}
|
|
1505
1181
|
}
|