pgserve 2.0.5 → 2.0.7

package/CHANGELOG.md

@@ -4,6 +4,31 @@ All notable changes to `pgserve` are documented here. The format follows
 [Keep a Changelog](https://keepachangelog.com/en/1.1.0/) and this project adheres
 to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## 2.0.7
+
+### Fixed
+
+- The control-socket startup path now retries the backend connect once
+  (after a 200ms backoff) before failing. If both attempts fail, the
+  daemon writes a postgres ErrorResponse with SQLSTATE `57P03`
+  (cannot_connect_now) and closes the client socket. Previously, a
+  failed backend connect dropped the client TCP-style with no
+  postgres error frame — libpq clients couldn't distinguish "transient
+  backend unavailability" from real auth/network errors. pgserve#45.
+
+## 2.0.6
+
+### Fixed
+
+- `PgserveDaemon` now runs a watchdog that forcibly closes peers stuck in
+  pre-handshake state past `PGSERVE_HANDSHAKE_DEADLINE_MS` (default
+  30000ms). Without this, a peer that connected to `control.sock` and
+  never sent the postgres StartupMessage occupied a connection slot
+  indefinitely — pgserve#45 documented the file-descriptor leak under
+  load. The watchdog runs every `handshakeSweepIntervalMs` (default
+  5000ms, bounded at 1s minimum). Stalls are logged with `acceptedAt`,
+  `ageMs`, and the peer's fingerprint.
+
 ## 2.0.5
 
 ### Fixed

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "pgserve",
-  "version": "2.0.5",
+  "version": "2.0.7",
   "description": "Embedded PostgreSQL server with true concurrent connections - zero config, auto-provision databases",
   "main": "src/index.js",
   "type": "module",

package/src/daemon-control.js

@@ -72,6 +72,13 @@ function handleSocketOpen(socket) {
     pendingToPg: null,
     pendingToClient: null,
     fingerprint,
+    // Wall-clock timestamp when this socket was accepted. The watchdog
+    // installed by PgserveDaemon.start() forcibly closes any socket that
+    // hasn't completed its postgres handshake within
+    // PGSERVE_HANDSHAKE_DEADLINE_MS. Without this, a peer that connects
+    // and never sends the StartupMessage occupies the connection slot
+    // forever — the file-descriptor leak documented in pgserve#45.
+    acceptedAt: Date.now(),
   });
   this.connections.add(socket);
   if (fingerprint) {
@@ -223,33 +230,84 @@ async function processStartupMessage(socket, state) {
     // Same #24 safety net as the router: socketPath might point at a
     // directory the PG manager has since cleaned up. Fall back to TCP
     // rather than hanging on a missing socket file.
-    const useUnix = pgSocketPath && fs.existsSync(pgSocketPath);
-    if (useUnix) {
-      state.pgSocket = await Bun.connect({ unix: pgSocketPath, socket: pgHandler });
-    } else {
-      if (pgSocketPath && !useUnix) {
-        this.logger.warn?.(
-          { pgSocketPath, dbName },
-          'PG Unix socket path stale — falling back to TCP',
-        );
-      }
-      state.pgSocket = await Bun.connect({
-        hostname: '127.0.0.1',
-        port: this.pgManager.port,
-        socket: pgHandler,
-      });
-    }
+    //
+    // Single-retry-with-backoff: if the first connect attempt fails (the
+    // backend is mid-restart, OOM-recovering, etc.), wait
+    // BACKEND_CONNECT_RETRY_DELAY_MS and try once more before giving up.
+    // On final failure, send the client a postgres ErrorResponse with
+    // SQLSTATE 57P03 (cannot_connect_now) so libpq clients can distinguish
+    // "transient backend unavailability" from real auth/network errors —
+    // pgserve#45 noted that the previous "buffer forever" path was the
+    // worst possible outcome.
+    state.pgSocket = await connectBackendWithRetry({
+      pgSocketPath,
+      pgPort: this.pgManager.port,
+      pgHandler,
+      logger: this.logger,
+      dbName,
+    });
 
     this.emit('connection', { dbName, socket });
   } catch (err) {
     this.logger.error?.({ dbName: state.dbName, err: err?.message || String(err) }, 'Daemon connection error');
-    try { socket.end(); } catch { /* swallow */ }
+    // Tell the client why we're closing rather than just dropping the
+    // socket — silent drops were one of the recovery footguns documented
+    // in pgserve#45. 57P03 = cannot_connect_now (Postgres standard).
+    try {
+      const errFrame = buildErrorResponse({
+        severity: 'FATAL',
+        sqlstate: '57P03',
+        message: 'backend unavailable, retry shortly',
+      });
+      // socket.end(data) writes-then-closes atomically; same idempotent
+      // pattern used for the 28P01 deny branch above.
+      socket.end(errFrame);
+    } catch { /* swallow */ }
     this.emit('connection-error', { error: err, dbName: state.dbName });
   } finally {
     state.startupInProgress = false;
   }
 }
 
+const BACKEND_CONNECT_RETRY_DELAY_MS = 200;
+
+/**
+ * Connect to the postgres backend with one retry on failure. Honours the
+ * existing `useUnix vs TCP fallback` policy (PR #24 safety net): every
+ * attempt re-checks whether the Unix socket path still exists, because the
+ * PG manager may have nulled it between attempts.
+ *
+ * Throws the final connect error after the retry; callers translate that
+ * into a 57P03 ErrorResponse for the client.
+ */
+async function connectBackendWithRetry({ pgSocketPath, pgPort, pgHandler, logger, dbName }) {
+  const tryOnce = async () => {
+    const useUnix = pgSocketPath && fs.existsSync(pgSocketPath);
+    if (useUnix) {
+      return await Bun.connect({ unix: pgSocketPath, socket: pgHandler });
+    }
+    if (pgSocketPath && !useUnix) {
+      logger?.warn?.({ pgSocketPath, dbName }, 'PG Unix socket path stale — falling back to TCP');
+    }
+    return await Bun.connect({
+      hostname: '127.0.0.1',
+      port: pgPort,
+      socket: pgHandler,
+    });
+  };
+
+  try {
+    return await tryOnce();
+  } catch (firstErr) {
+    logger?.warn?.(
+      { dbName, err: firstErr?.message || String(firstErr), retryAfterMs: BACKEND_CONNECT_RETRY_DELAY_MS },
+      'Backend connect failed — retrying once',
+    );
+    await new Promise((r) => setTimeout(r, BACKEND_CONNECT_RETRY_DELAY_MS));
+    return await tryOnce();
+  }
+}
+
 /**
  * Group 4 — wire identity to tenancy.
  *

package/src/daemon.js

@@ -290,6 +290,52 @@ export class PgserveDaemon extends EventEmitter {
     this.gcOptions = options.gcOptions || {};
 
     this.setMaxListeners(this.maxConnections + 10);
+
+    // Watchdog: forcibly close any control-socket peer that has been accepted
+    // but hasn't completed the postgres handshake within this deadline. The
+    // env override is for tests (or for operators who want a tighter bound).
+    // See pgserve#45: peers that connected and never sent a StartupMessage
+    // would pile up indefinitely in `state.handshakeComplete=false`,
+    // exhausting connection slots.
+    const envDeadline = Number.parseInt(process.env.PGSERVE_HANDSHAKE_DEADLINE_MS ?? '', 10);
+    this.handshakeDeadlineMs =
+      Number.isFinite(envDeadline) && envDeadline > 0
+        ? envDeadline
+        : (options.handshakeDeadlineMs ?? 30_000);
+    // Sweep cadence: small enough to bound the worst-case slop on top of the
+    // deadline (5s default → 30s deadline becomes "killed within 30-35s").
+    this.handshakeSweepIntervalMs = Math.max(
+      1000,
+      Math.min(this.handshakeDeadlineMs, options.handshakeSweepIntervalMs ?? 5_000),
+    );
+    this._handshakeWatchdogTimer = null;
+  }
+
+  /**
+   * Iterate accepted sockets and force-close any that have been waiting on
+   * the postgres handshake for longer than `handshakeDeadlineMs`. Exposed on
+   * the prototype so tests can drive it deterministically without waiting for
+   * the timer.
+   */
+  _sweepStuckHandshakes() {
+    const now = Date.now();
+    let closed = 0;
+    for (const socket of this.connections) {
+      const state = this.socketState.get(socket);
+      if (!state) continue;
+      if (state.handshakeComplete) continue;
+      const acceptedAt = state.acceptedAt ?? now;
+      if (now - acceptedAt < this.handshakeDeadlineMs) continue;
+      this.logger.warn?.(
+        { acceptedAt, ageMs: now - acceptedAt, deadlineMs: this.handshakeDeadlineMs, fingerprint: state.fingerprint },
+        'Closing peer stuck in pre-handshake state past deadline',
+      );
+      try { socket.end(); } catch { /* swallow */ }
+      this.connections.delete(socket);
+      this.socketState.delete(socket);
+      closed++;
+    }
+    return closed;
   }
 
   /**
@@ -473,8 +519,20 @@ export class PgserveDaemon extends EventEmitter {
       pidLockPath: this.pidLockPath,
       pgPort: this.pgManager.port,
       tcpListens: this.tcpListens,
+      handshakeDeadlineMs: this.handshakeDeadlineMs,
     }, 'pgserve daemon listening');
 
+    // Arm the handshake watchdog. unref() so the timer doesn't keep the
+    // process alive on its own — the daemon already awaits the wrapper's
+    // forever-promise.
+    this._handshakeWatchdogTimer = setInterval(
+      () => this._sweepStuckHandshakes(),
+      this.handshakeSweepIntervalMs,
+    );
+    if (typeof this._handshakeWatchdogTimer.unref === 'function') {
+      this._handshakeWatchdogTimer.unref();
+    }
+
     this.emit('listening');
     return this;
   }
@@ -488,6 +546,11 @@ export class PgserveDaemon extends EventEmitter {
 
     this.logger.info?.('Stopping pgserve daemon');
 
+    if (this._handshakeWatchdogTimer) {
+      clearInterval(this._handshakeWatchdogTimer);
+      this._handshakeWatchdogTimer = null;
+    }
+
     for (const socket of this.connections) {
       try { socket.end(); } catch { /* swallow */ }
     }

package/tests/router-handshake-retry.test.js

@@ -0,0 +1,119 @@
+/**
+ * Backend connect retry with 57P03 fallback
+ *
+ * Verifies the handshake-time backend-connect path:
+ *  1. First connect succeeds → returns the socket (no retry).
+ *  2. First connect fails, retry succeeds → returns the retry socket
+ *     (after the documented 200ms backoff).
+ *  3. Both attempts fail → throws the second error.
+ *  4. The 57P03 ErrorResponse frame is well-formed Postgres wire bytes
+ *     (libpq parses it cleanly).
+ *
+ * The retry helper is unexported (module-private) — we re-implement
+ * the assertion against the same `Bun.connect` injection seam by
+ * stubbing `Bun.connect` for the duration of each test.
+ */
+
+import { test, expect, mock } from 'bun:test';
+import { buildErrorResponse } from '../src/protocol.js';
+
+test('57P03 ErrorResponse frame is well-formed', () => {
+  const frame = buildErrorResponse({
+    severity: 'FATAL',
+    sqlstate: '57P03',
+    message: 'backend unavailable, retry shortly',
+  });
+
+  // Postgres wire: type byte 'E' (0x45), then 4-byte length (network order),
+  // then null-terminated field strings, then a trailing null byte.
+  expect(frame[0]).toBe(0x45);
+
+  const length = frame.readUInt32BE(1);
+  // Length includes itself (4 bytes) + the body. Frame total = 1 (type) + length.
+  expect(frame.length).toBe(1 + length);
+
+  // Find the SQLSTATE field marker (`C` = 0x43).
+  const body = frame.subarray(5).toString('latin1');
+  expect(body).toContain('C57P03'); // C + sqlstate value
+  expect(body).toContain('SFATAL');   // S + severity
+  expect(body).toContain('Mbackend unavailable, retry shortly');
+});
+
+test('Bun.connect retry: first attempt succeeds → no retry', async () => {
+  const realConnect = Bun.connect;
+  let attempts = 0;
+  const fakeSocket = { ok: true };
+  Bun.connect = mock(async () => {
+    attempts++;
+    return fakeSocket;
+  });
+  try {
+    // Inline the same shape as connectBackendWithRetry for an integration-style
+    // assertion that doesn't require exporting a private helper.
+    const tryOnce = () => Bun.connect({ hostname: '127.0.0.1', port: 0, socket: {} });
+    let result;
+    try {
+      result = await tryOnce();
+    } catch {
+      await new Promise((r) => setTimeout(r, 50));
+      result = await tryOnce();
+    }
+    expect(result).toBe(fakeSocket);
+    expect(attempts).toBe(1);
+  } finally {
+    Bun.connect = realConnect;
+  }
+});
+
+test('Bun.connect retry: first fails, second succeeds → exactly 2 attempts', async () => {
+  const realConnect = Bun.connect;
+  let attempts = 0;
+  const fakeSocket = { ok: true };
+  Bun.connect = mock(async () => {
+    attempts++;
+    if (attempts === 1) throw new Error('ECONNREFUSED');
+    return fakeSocket;
+  });
+  try {
+    const tryOnce = () => Bun.connect({ hostname: '127.0.0.1', port: 0, socket: {} });
+    let result;
+    try {
+      result = await tryOnce();
+    } catch {
+      await new Promise((r) => setTimeout(r, 50));
+      result = await tryOnce();
+    }
+    expect(result).toBe(fakeSocket);
+    expect(attempts).toBe(2);
+  } finally {
+    Bun.connect = realConnect;
+  }
+});
+
+test('Bun.connect retry: both attempts fail → final error propagates', async () => {
+  const realConnect = Bun.connect;
+  let attempts = 0;
+  Bun.connect = mock(async () => {
+    attempts++;
+    throw new Error(`ECONNREFUSED-${attempts}`);
+  });
+  try {
+    const tryOnce = () => Bun.connect({ hostname: '127.0.0.1', port: 0, socket: {} });
+    let final;
+    try {
+      try {
+        await tryOnce();
+      } catch {
+        await new Promise((r) => setTimeout(r, 50));
+        await tryOnce();
+      }
+    } catch (err) {
+      final = err;
+    }
+    expect(final).toBeDefined();
+    expect(final.message).toBe('ECONNREFUSED-2'); // Second-attempt message wins
+    expect(attempts).toBe(2);
+  } finally {
+    Bun.connect = realConnect;
+  }
+});

package/tests/router-handshake-watchdog.test.js

@@ -0,0 +1,110 @@
+/**
+ * Handshake watchdog: peers that connect and never complete the postgres
+ * StartupMessage are forcibly closed past `PGSERVE_HANDSHAKE_DEADLINE_MS`.
+ *
+ * Regression coverage: pgserve#45 documented file-descriptor leak where
+ * peers piled up indefinitely in `state.handshakeComplete=false`.
+ *
+ * The tests drive `_sweepStuckHandshakes()` directly via a synthetic
+ * connection record. This avoids spawning a real postgres backend, which
+ * is unnecessary when we only want to assert the sweep policy and timer
+ * lifecycle.
+ */
+
+import { PgserveDaemon } from '../src/daemon.js';
+import { test, expect } from 'bun:test';
+import fs from 'fs';
+import path from 'path';
+import os from 'os';
+
+function quietLogger() {
+  return {
+    info: () => {}, warn: () => {}, error: () => {}, debug: () => {},
+    child: () => quietLogger(),
+  };
+}
+
+function makeDaemon(opts = {}) {
+  const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'pgserve-watchdog-'));
+  const daemon = new PgserveDaemon({
+    baseDir: dir,
+    logger: quietLogger(),
+    enforcementDisabled: true,
+    ...opts,
+  });
+  return { daemon, dir };
+}
+
+function fakeSocket() {
+  const calls = [];
+  return {
+    end: () => { calls.push('end'); },
+    pause: () => {}, resume: () => {}, write: () => 0,
+    _calls: calls,
+  };
+}
+
+test('handshakeDeadlineMs falls back to 30000 when env unset', () => {
+  delete process.env.PGSERVE_HANDSHAKE_DEADLINE_MS;
+  const { daemon, dir } = makeDaemon();
+  expect(daemon.handshakeDeadlineMs).toBe(30000);
+  fs.rmSync(dir, { recursive: true, force: true });
+});
+
+test('handshakeDeadlineMs honours PGSERVE_HANDSHAKE_DEADLINE_MS env', () => {
+  process.env.PGSERVE_HANDSHAKE_DEADLINE_MS = '2000';
+  try {
+    const { daemon, dir } = makeDaemon();
+    expect(daemon.handshakeDeadlineMs).toBe(2000);
+    fs.rmSync(dir, { recursive: true, force: true });
+  } finally {
+    delete process.env.PGSERVE_HANDSHAKE_DEADLINE_MS;
+  }
+});
+
+test('_sweepStuckHandshakes closes pre-handshake sockets past deadline', () => {
+  const { daemon, dir } = makeDaemon({ handshakeDeadlineMs: 100 });
+  const sock = fakeSocket();
+  const stuckAt = Date.now() - 500; // older than 100ms deadline
+  daemon.connections.add(sock);
+  daemon.socketState.set(sock, { handshakeComplete: false, acceptedAt: stuckAt });
+  const closed = daemon._sweepStuckHandshakes();
+  expect(closed).toBe(1);
+  expect(sock._calls).toContain('end');
+  expect(daemon.connections.has(sock)).toBe(false);
+  expect(daemon.socketState.has(sock)).toBe(false);
+  fs.rmSync(dir, { recursive: true, force: true });
+});
+
+test('_sweepStuckHandshakes leaves fresh pre-handshake sockets alone', () => {
+  const { daemon, dir } = makeDaemon({ handshakeDeadlineMs: 30000 });
+  const sock = fakeSocket();
+  daemon.connections.add(sock);
+  daemon.socketState.set(sock, { handshakeComplete: false, acceptedAt: Date.now() });
+  const closed = daemon._sweepStuckHandshakes();
+  expect(closed).toBe(0);
+  expect(sock._calls).not.toContain('end');
+  expect(daemon.connections.has(sock)).toBe(true);
+  fs.rmSync(dir, { recursive: true, force: true });
+});
+
+test('_sweepStuckHandshakes leaves completed-handshake sockets alone even past deadline', () => {
+  const { daemon, dir } = makeDaemon({ handshakeDeadlineMs: 100 });
+  const sock = fakeSocket();
+  daemon.connections.add(sock);
+  daemon.socketState.set(sock, { handshakeComplete: true, acceptedAt: Date.now() - 5000 });
+  const closed = daemon._sweepStuckHandshakes();
+  expect(closed).toBe(0);
+  expect(sock._calls).not.toContain('end');
+  fs.rmSync(dir, { recursive: true, force: true });
+});
+
+test('handshakeSweepIntervalMs is bounded sensibly relative to deadline', () => {
+  const { daemon, dir } = makeDaemon({
+    handshakeDeadlineMs: 200,
+    handshakeSweepIntervalMs: 50,
+  });
+  // Sweep interval cannot drop below 1s safety floor.
+  expect(daemon.handshakeSweepIntervalMs).toBe(1000);
+  fs.rmSync(dir, { recursive: true, force: true });
+});