pgserve 2.3.0 → 2.4.0

package/src/stats-collector.js

@@ -1,453 +0,0 @@
-/**
- * Stats Collector - Centralized statistics gathering for pgserve
- *
- * Aggregates stats from:
- * - Router (active connections)
- * - PostgreSQL Manager (databases, storage)
- * - PostgreSQL internals (pg_stat_activity, pg_stat_database)
- * - Process (memory, uptime)
- */
-
-// CPU sampling threshold - avoid noise from too-frequent calls
-const CPU_SAMPLE_MIN_INTERVAL_MS = 100;
-
-// /proc/diskstats minimum fields per device line
-// Fields: major minor device reads rd_merged rd_sectors rd_ms writes wr_merged wr_sectors wr_ms io_in_progress io_ms weighted_io_ms
-const PROC_DISKSTATS_MIN_FIELDS = 14;
-
-export class StatsCollector {
-  constructor(options = {}) {
-    this.pgManager = options.pgManager;
-    this.router = options.router;
-    this.clusterStats = options.clusterStats; // Function that returns cluster stats
-    this.logger = options.logger;
-
-    // Override values for cluster mode where router is null
-    this.serverPort = options.port;
-    this.serverHost = options.host;
-
-    // Cache to avoid over-querying (configurable for different monitoring needs)
-    this.cache = null;
-    this.cacheTime = 0;
-    this.cacheTTL = options.cacheTTL || 1000; // Default 1s cache
-
-    // CPU tracking
-    this.lastCpuUsage = process.cpuUsage();
-    this.lastCpuTime = Date.now();
-
-    // Disk I/O tracking (Linux)
-    this.lastDiskStats = null;
-    this.lastDiskTime = 0;
-
-    // pgvector stats cache (longer TTL since it requires cross-DB queries)
-    this.pgvectorCache = null;
-    this.pgvectorCacheTime = 0;
-    this.pgvectorCacheTTL = 5000; // 5s cache for pgvector stats
-
-    // Connection pool reuse for pgvector stats (avoids TCP handshake overhead per collection)
-    this.pgvectorDbPools = new Map(); // dbName -> SQL pool
-    this.pgvectorExtCache = new Map(); // dbName -> boolean (vector extension exists)
-  }
-
-  /**
-   * Collect all available stats
-   * @returns {Promise<StatsSnapshot>}
-   */
-  async collect() {
-    // Return cached if recent
-    if (this.cache && Date.now() - this.cacheTime < this.cacheTTL) {
-      return this.cache;
-    }
-
-    const pgStats = this.pgManager?.getStats?.() || {};
-    const routerStats = this.router?.getStats?.() || {};
-    // Prefer clusterStats (cluster mode aggregates from all workers)
-    // Fall back to routerStats (single-process mode)
-    const clusterStats = this.clusterStats?.() || null;
-
-    const snapshot = {
-      timestamp: Date.now(),
-      uptime: process.uptime(),
-
-      // Connection stats - cluster mode has aggregated stats, single-process uses router directly
-      connections: {
-        active: clusterStats?.connections?.active ?? routerStats.activeConnections ?? 0,
-        totalConnected: clusterStats?.connections?.totalConnected ?? 0,
-        totalDisconnected: clusterStats?.connections?.totalDisconnected ?? 0,
-        max: this.router?.maxConnections || 1000
-      },
-
-      // Server config
-      server: {
-        port: this.serverPort || routerStats.port || this.router?.port || 8432,
-        host: this.serverHost || routerStats.host || this.router?.host || '127.0.0.1',
-        pgPort: routerStats.pgPort || pgStats.port || 0,
-        memoryMode: this.router?.memoryMode ?? !pgStats.persistent,
-        useRam: this.pgManager?.useRam || false
-      },
-
-      // PostgreSQL manager stats
-      postgres: {
-        port: pgStats.port,
-        databases: pgStats.databases || [],
-        databaseDir: pgStats.databaseDir,
-        socketDir: pgStats.socketDir,
-        socketPath: pgStats.socketPath,
-        persistent: pgStats.persistent
-      },
-
-      // Cluster stats (if in cluster mode)
-      cluster: clusterStats ? {
-        workers: clusterStats.workers,
-        pids: clusterStats.pids,
-        workerStats: clusterStats.workerStats || {}
-      } : null,
-
-      // PostgreSQL internals (pg_stat_*)
-      internals: await this.collectPgStats(),
-
-      // pgvector stats (if enabled)
-      pgvector: await this.collectPgvectorStats(),
-
-      // Process stats
-      process: {
-        pid: process.pid,
-        memory: process.memoryUsage(),
-        cpu: this.getCpuUsage()
-      },
-
-      // System stats (Linux)
-      system: await this.getSystemStats()
-    };
-
-    this.cache = snapshot;
-    this.cacheTime = Date.now();
-    return snapshot;
-  }
-
-  /**
-   * Get CPU usage percentage
-   */
-  getCpuUsage() {
-    const now = Date.now();
-    const elapsed = now - this.lastCpuTime;
-    if (elapsed < CPU_SAMPLE_MIN_INTERVAL_MS) return this.lastCpuPercent || 0;
-
-    const cpuUsage = process.cpuUsage(this.lastCpuUsage);
-    const totalMicros = cpuUsage.user + cpuUsage.system;
-    const elapsedMicros = elapsed * 1000; // Convert ms to microseconds
-
-    // CPU percentage (can be > 100% on multi-core)
-    const percent = (totalMicros / elapsedMicros) * 100;
-
-    this.lastCpuUsage = process.cpuUsage();
-    this.lastCpuTime = now;
-    this.lastCpuPercent = percent;
-
-    // Return raw percentage - can exceed 100% on multi-core systems
-    // Consumers (dashboard, etc.) can format/cap as needed for display
-    return percent;
-  }
-
-  /**
-   * Get system stats (Linux-specific)
-   */
-  async getSystemStats() {
-    const stats = {
-      loadAvg: null,
-      diskIO: null
-    };
-
-    try {
-      // Load average (works on Linux/macOS)
-      const os = await import('os');
-      const loadAvg = os.loadavg();
-      stats.loadAvg = {
-        '1m': loadAvg[0],
-        '5m': loadAvg[1],
-        '15m': loadAvg[2]
-      };
-
-      // Disk I/O stats (Linux only via /proc/diskstats)
-      if (process.platform === 'linux') {
-        const fs = await import('fs/promises');
-        try {
-          const diskstats = await fs.readFile('/proc/diskstats', 'utf8');
-          const now = Date.now();
-
-          // Parse diskstats - find main disk (sda, nvme0n1, vda, etc.)
-          let readSectors = 0;
-          let writeSectors = 0;
-          let readOps = 0;
-          let writeOps = 0;
-
-          for (const line of diskstats.split('\n')) {
-            const parts = line.trim().split(/\s+/);
-            if (parts.length < PROC_DISKSTATS_MIN_FIELDS) continue;
-
-            const device = parts[2];
-            // Match main disks (sda, sdb, nvme0n1, vda, etc.) but not partitions
-            if (/^(sd[a-z]|nvme\d+n\d+|vd[a-z])$/.test(device)) {
-              readOps += parseInt(parts[3]) || 0;
-              readSectors += parseInt(parts[5]) || 0;
-              writeOps += parseInt(parts[7]) || 0;
-              writeSectors += parseInt(parts[9]) || 0;
-            }
-          }
-
-          if (this.lastDiskStats && this.lastDiskTime) {
-            const elapsed = (now - this.lastDiskTime) / 1000; // seconds
-            if (elapsed > 0) {
-              const readDiff = readSectors - this.lastDiskStats.readSectors;
-              const writeDiff = writeSectors - this.lastDiskStats.writeSectors;
-              const readOpsDiff = readOps - this.lastDiskStats.readOps;
-              const writeOpsDiff = writeOps - this.lastDiskStats.writeOps;
-
-              // Sectors are typically 512 bytes
-              stats.diskIO = {
-                readMBps: ((readDiff * 512) / (1024 * 1024)) / elapsed,
-                writeMBps: ((writeDiff * 512) / (1024 * 1024)) / elapsed,
-                readIOPS: readOpsDiff / elapsed,
-                writeIOPS: writeOpsDiff / elapsed
-              };
-            }
-          }
-
-          this.lastDiskStats = { readSectors, writeSectors, readOps, writeOps };
-          this.lastDiskTime = now;
-        } catch (err) {
-          // /proc/diskstats not available (normal on non-Linux or restricted environments)
-          this.logger?.debug?.({ err: err.message }, 'Could not read /proc/diskstats');
-        }
-      }
-    } catch (err) {
-      // OS module or stats not available (normal on some platforms)
-      this.logger?.debug?.({ err: err.message }, 'Could not collect system stats');
-    }
-
-    return stats;
-  }
-
-  /**
-   * Query PostgreSQL internal statistics
-   */
-  async collectPgStats() {
-    // Get admin pool from pgManager
-    const adminPool = this.pgManager?.adminPool;
-    if (!adminPool) return null;
-
-    try {
-      // Query pg_stat_activity for connection details
-      const activity = await adminPool`
-        SELECT
-          count(*) FILTER (WHERE state = 'active') as active_queries,
-          count(*) FILTER (WHERE state = 'idle') as idle_connections,
-          count(*) as total_connections
-        FROM pg_stat_activity
-        WHERE datname IS NOT NULL
-      `;
-
-      // Query pg_stat_database for DB-level stats
-      const dbStats = await adminPool`
-        SELECT
-          datname,
-          numbackends,
-          xact_commit,
-          xact_rollback,
-          blks_read,
-          blks_hit,
-          tup_returned,
-          tup_fetched,
-          tup_inserted,
-          tup_updated,
-          tup_deleted
-        FROM pg_stat_database
-        WHERE datname NOT IN ('template0', 'template1')
-        ORDER BY numbackends DESC
-        LIMIT 10
-      `;
-
-      return {
-        activity: activity[0] || {},
-        databases: dbStats || []
-      };
-    } catch (err) {
-      this.logger?.debug?.({ err: err.message }, 'Failed to collect pg_stat_*');
-      return null;
-    }
-  }
-
-  /**
-   * Collect pgvector statistics across all databases
-   * Uses separate cache with longer TTL due to cross-DB query overhead
-   */
-  async collectPgvectorStats() {
-    // Return cached if recent
-    if (this.pgvectorCache && Date.now() - this.pgvectorCacheTime < this.pgvectorCacheTTL) {
-      return this.pgvectorCache;
-    }
-
-    const adminPool = this.pgManager?.adminPool;
-    if (!adminPool) return null;
-
-    // Query PostgreSQL directly for databases (pgManager.getStats().databases may be empty in cluster mode)
-    let databases = [];
-    try {
-      const dbResult = await adminPool`
-        SELECT datname FROM pg_database
-        WHERE datname NOT IN ('template0', 'template1', 'postgres')
-      `;
-      databases = dbResult.map(row => row.datname);
-    } catch (err) {
-      this.logger?.debug?.({ err: err.message }, 'Failed to query databases for pgvector stats');
-      return null;
-    }
-
-    if (databases.length === 0) return null;
-
-    try {
-      let totalTables = 0;
-      let totalRows = 0;
-      const dimensions = new Set();
-      let dbsWithVectors = 0;
-
-      // Query each database for vector columns (parallel for performance)
-      const allDbStats = await Promise.all(
-        databases.map(dbName => this.queryDatabaseVectorStats(dbName))
-      );
-
-      for (const dbStats of allDbStats) {
-        if (dbStats && dbStats.tableCount > 0) {
-          dbsWithVectors++;
-          totalTables += dbStats.tableCount;
-          totalRows += dbStats.rowCount;
-          dbStats.dimensions.forEach(d => dimensions.add(d));
-        }
-      }
-
-      // Only return stats if vectors exist
-      if (totalTables === 0) {
-        this.pgvectorCache = null;
-        this.pgvectorCacheTime = Date.now();
-        return null;
-      }
-
-      const stats = {
-        enabled: true,
-        databases: dbsWithVectors,
-        tableCount: totalTables,
-        totalRows: totalRows,
-        dimensions: [...dimensions].sort((a, b) => b - a).join(', ') || null
-      };
-
-      this.pgvectorCache = stats;
-      this.pgvectorCacheTime = Date.now();
-      return stats;
-    } catch (err) {
-      this.logger?.debug?.({ err: err.message }, 'Failed to collect pgvector stats');
-      return null;
-    }
-  }
-
-  /**
-   * Query vector column stats for a specific database
-   * @param {string} dbName - Database name to query
-   * @returns {Promise<{tableCount: number, rowCount: number, dimensions: number[]}>}
-   */
-  async queryDatabaseVectorStats(dbName) {
-    try {
-      // Reuse connection pool (avoids TCP handshake + auth overhead per collection)
-      let dbPool = this.pgvectorDbPools.get(dbName);
-      if (!dbPool) {
-        const { SQL } = await import('bun');
-        dbPool = new SQL({
-          hostname: '127.0.0.1',
-          port: this.pgManager?.port || 5433,
-          database: dbName,
-          username: 'postgres',
-          password: 'postgres',
-          max: 2, // Allow some concurrency
-          idleTimeout: 30, // Keep alive longer for reuse
-          connectionTimeout: 3,
-        });
-        this.pgvectorDbPools.set(dbName, dbPool);
-      }
-
-      // Check cached extension status (only query once per DB)
-      if (!this.pgvectorExtCache.has(dbName)) {
-        const extCheck = await dbPool`
-          SELECT 1 FROM pg_extension WHERE extname = 'vector'
-        `;
-        this.pgvectorExtCache.set(dbName, extCheck.length > 0);
-      }
-
-      if (!this.pgvectorExtCache.get(dbName)) {
-        return { tableCount: 0, rowCount: 0, dimensions: [] };
-      }
-
-      // Single query: get tables, dimensions, and row counts via pg_class.reltuples
-      // Note: reltuples is an estimate (updated by ANALYZE/VACUUM), not exact COUNT(*)
-      // This is acceptable for dashboard stats with 5s cache - avoids expensive full table scans
-      // Trade-off: ~50-100x faster, but counts may be stale after bulk INSERT/DELETE until ANALYZE
-      const vectorInfo = await dbPool`
-        SELECT
-          c.relname as table_name,
-          a.atttypmod as dimensions,
-          GREATEST(c.reltuples, 0)::bigint AS row_count
-        FROM pg_attribute a
-        JOIN pg_class c ON a.attrelid = c.oid
-        JOIN pg_type t ON a.atttypid = t.oid
-        WHERE t.typname = 'vector'
-          AND c.relkind = 'r'
-          AND a.attnum > 0
-          AND NOT a.attisdropped
-      `;
-
-      if (vectorInfo.length === 0) {
-        return { tableCount: 0, rowCount: 0, dimensions: [] };
-      }
-
-      // Aggregate results from single query
-      const tableRows = new Map();
-      const dims = [];
-
-      for (const row of vectorInfo) {
-        if (!tableRows.has(row.table_name)) {
-          tableRows.set(row.table_name, Number(row.row_count || 0));
-        }
-        if (row.dimensions > 0) {
-          dims.push(row.dimensions);
-        }
-      }
-
-      const totalRows = Array.from(tableRows.values()).reduce((sum, count) => sum + count, 0);
-
-      return {
-        tableCount: tableRows.size,
-        rowCount: totalRows,
-        dimensions: dims
-      };
-    } catch (err) {
-      this.logger?.debug?.({ dbName, err: err.message }, 'Failed to query database vector stats');
-      // Invalidate caches on error (DB might have been dropped)
-      this.pgvectorExtCache.delete(dbName);
-      const pool = this.pgvectorDbPools.get(dbName);
-      if (pool) {
-        this.pgvectorDbPools.delete(dbName);
-        await pool.close().catch(() => {});
-      }
-      return { tableCount: 0, rowCount: 0, dimensions: [] };
-    }
-  }
-
-  /**
-   * Close all pgvector database pools (called on shutdown)
-   */
-  async closePgvectorPools() {
-    for (const [_dbName, pool] of this.pgvectorDbPools) {
-      await pool.close().catch(() => {});
-    }
-    this.pgvectorDbPools.clear();
-    this.pgvectorExtCache.clear();
-  }
-}