ts-server-lib 0.0.48

package/db/TSMongo.js

@@ -0,0 +1,516 @@
+"use strict";
+/**
+ * TSMongo – MongoDB connection (pooling, health, index registration).
+ * Static API like TSRedis. Use TSMongo.connect(), TSMongo.getDatabase(), etc.
+ */
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.TSMongo = exports.DEFAULT_MONGO_CONFIG = exports.ReadPreference = exports.ObjectId = exports.Db = exports.MongoClient = void 0;
+const mongodb_1 = require("mongodb");
+// Re-export MongoDB types so consumers (core-service, microservices) import from
+// 'ts-server-lib/db/TSMongo.js' instead of 'mongodb' directly — single resolution
+// path, no duplicate mongodb installs across the monorepo.
+var mongodb_2 = require("mongodb");
+Object.defineProperty(exports, "MongoClient", { enumerable: true, get: function () { return mongodb_2.MongoClient; } });
+Object.defineProperty(exports, "Db", { enumerable: true, get: function () { return mongodb_2.Db; } });
+Object.defineProperty(exports, "ObjectId", { enumerable: true, get: function () { return mongodb_2.ObjectId; } });
+Object.defineProperty(exports, "ReadPreference", { enumerable: true, get: function () { return mongodb_2.ReadPreference; } });
+const ERR_PREFIX = 'TSMongo:ERROR';
+const WARN_PREFIX = 'TSMongo:WARN';
+let client = null;
+let db = null;
+/**
+ * In-flight connect() attempt, if any. `connectImpl` has no other synchronization around the
+ * module-level `client`/`db`/`baseUri` singleton, so concurrent callers (e.g. several services'
+ * e2e suites calling `setConfig` around the same time) would otherwise each construct their own
+ * `MongoClient` and race to overwrite these variables mid-`await`, leaving later reads (`db =
+ * client.db(...)`) pointing at whichever concurrent call's client happened to still be mutating
+ * things last. Serializing behind this promise makes concurrent connect() calls await the same
+ * attempt instead of racing.
+ */
+let connectingPromise = null;
+let _eventHandler = null;
+/**
+ * Register a structured event handler for TSMongo lifecycle events.
+ *
+ * Call once at service startup before `TSMongo.connect()`. Pass `null` to
+ * remove a previously registered handler and revert to console fallback.
+ *
+ * @example
+ * TSMongo.setEventHandler((evt) => {
+ *   logger[evt.level]('TSMongo event', { operation: 'core.database', event: evt.event, ...evt.data });
+ * });
+ */
+function setEventHandlerImpl(handler) {
+    _eventHandler = handler;
+}
+function emitEvent(evt) {
+    if (_eventHandler) {
+        _eventHandler(evt);
+        return;
+    }
+    const prefix = evt.level === 'error' ? ERR_PREFIX : WARN_PREFIX;
+    const logFn = evt.level === 'error' ? console.error : console.warn;
+    logFn(prefix, `[${evt.event}]`, evt.message, evt.data);
+}
+/** Tracks the highest utilization threshold currently crossed; prevents repeated emissions. */
+let _lastUtilizationThreshold = 0;
+// Queue-depth threshold tracking — fires EARLIER than utilization-based warnings under
+// the real failure mode (queue overflow with pool at moderate utilization). Hysteresis:
+// "50" = waitQueueSize >= maxPoolSize * 0.5 (warning); "100" = >= maxPoolSize (critical).
+let _lastQueueThreshold = 0;
+let connectionPoolStats = {
+    totalConnections: 0,
+    checkedOut: 0,
+    availableConnections: 0,
+    waitQueueSize: 0,
+    maxPoolSize: 200,
+    minPoolSize: 10,
+    totalCheckouts: 0,
+    totalCheckins: 0,
+    connectionCreated: 0,
+    connectionClosed: 0,
+    waitQueueTimeouts: 0,
+    lastWaitQueueTimeout: null
+};
+exports.DEFAULT_MONGO_CONFIG = {
+    maxPoolSize: 200,
+    minPoolSize: 10,
+    maxIdleTimeMS: 30000,
+    waitQueueTimeoutMS: 10000,
+    maxWaitingRequests: 500,
+    connectTimeoutMS: 10000,
+    socketTimeoutMS: 45000,
+    serverSelectionTimeoutMS: 30000,
+    readPreference: 'nearest',
+    writeConcern: 'majority',
+    retryWrites: true,
+    retryReads: true
+};
+let baseUri = null;
+function getServerKey(uriObj) {
+    return `${uriObj.protocol}//${uriObj.host}`;
+}
+function resetPoolStats() {
+    connectionPoolStats = {
+        ...connectionPoolStats,
+        totalConnections: 0,
+        checkedOut: 0,
+        availableConnections: 0,
+        waitQueueSize: 0,
+        totalCheckouts: 0,
+        totalCheckins: 0,
+        connectionCreated: 0,
+        connectionClosed: 0,
+        waitQueueTimeouts: 0,
+        lastWaitQueueTimeout: null
+    };
+    _lastUtilizationThreshold = 0;
+    _lastQueueThreshold = 0;
+}
+// eslint-disable-next-line max-lines-per-function
+function attachClientEventHandlers(client, _opts) {
+    client.on('connectionPoolClosed', () => resetPoolStats());
+    client.on('connectionCreated', () => {
+        connectionPoolStats.totalConnections++;
+        connectionPoolStats.connectionCreated++;
+        connectionPoolStats.availableConnections = connectionPoolStats.totalConnections - connectionPoolStats.checkedOut;
+    });
+    client.on('connectionClosed', () => {
+        connectionPoolStats.totalConnections = Math.max(0, connectionPoolStats.totalConnections - 1);
+        connectionPoolStats.connectionClosed++;
+        connectionPoolStats.availableConnections = connectionPoolStats.totalConnections - connectionPoolStats.checkedOut;
+    });
+    client.on('connectionCheckedOut', () => {
+        connectionPoolStats.checkedOut++;
+        connectionPoolStats.totalCheckouts++;
+        connectionPoolStats.availableConnections = connectionPoolStats.totalConnections - connectionPoolStats.checkedOut;
+        connectionPoolStats.waitQueueSize = Math.max(0, connectionPoolStats.waitQueueSize - 1);
+        // Hysteresis: reset queue threshold tracking when the queue drains.
+        //   queueRatio < 0.25  → fully reset (re-warn on next ≥0.5 crossing)
+        //   queueRatio < 0.75 with critical previously → downgrade critical→warning
+        if (_lastQueueThreshold > 0) {
+            const maxPool = connectionPoolStats.maxPoolSize;
+            const queueRatio = maxPool > 0 ? connectionPoolStats.waitQueueSize / maxPool : 0;
+            if (queueRatio < 0.25) {
+                _lastQueueThreshold = 0;
+            }
+            else if (_lastQueueThreshold === 100 && queueRatio < 0.75) {
+                _lastQueueThreshold = 50;
+            }
+        }
+        // Emit utilization threshold events once per crossing (hysteresis resets in checkedIn).
+        //
+        // F6a: `effectiveDemandPercent = (checkedOut + waitQueueSize) / maxPool * 100` — the real
+        // demand signal (can exceed 100 under bursty load). `utilizationPercent` is checkedOut-only.
+        const maxPool = connectionPoolStats.maxPoolSize;
+        const util = maxPool > 0 ? (connectionPoolStats.checkedOut / maxPool) * 100 : 0;
+        const effectiveDemand = maxPool > 0
+            ? ((connectionPoolStats.checkedOut + connectionPoolStats.waitQueueSize) / maxPool) * 100
+            : 0;
+        const utilizationData = {
+            utilizationPercent: Math.round(util),
+            effectiveDemandPercent: Math.round(effectiveDemand),
+            checkedOut: connectionPoolStats.checkedOut,
+            maxPoolSize: maxPool,
+            availableConnections: connectionPoolStats.availableConnections,
+            waitQueueSize: connectionPoolStats.waitQueueSize
+        };
+        if (effectiveDemand >= 95 && _lastUtilizationThreshold < 95) {
+            _lastUtilizationThreshold = 95;
+            emitEvent({
+                level: 'error',
+                event: 'pool.utilization_critical',
+                message: `Pool utilization critical: ${Math.round(effectiveDemand)}% effective (${Math.round(util)}% checkedOut + ${connectionPoolStats.waitQueueSize} queued)`,
+                timestamp: new Date().toISOString(),
+                data: utilizationData
+            });
+        }
+        else if (effectiveDemand >= 80 && _lastUtilizationThreshold < 80) {
+            _lastUtilizationThreshold = 80;
+            emitEvent({
+                level: 'warn',
+                event: 'pool.utilization_warning',
+                message: `Pool utilization warning: ${Math.round(effectiveDemand)}% effective (${Math.round(util)}% checkedOut + ${connectionPoolStats.waitQueueSize} queued)`,
+                timestamp: new Date().toISOString(),
+                data: utilizationData
+            });
+        }
+    });
+    client.on('connectionCheckedIn', () => {
+        connectionPoolStats.checkedOut = Math.max(0, connectionPoolStats.checkedOut - 1);
+        connectionPoolStats.totalCheckins++;
+        connectionPoolStats.availableConnections = connectionPoolStats.totalConnections - connectionPoolStats.checkedOut;
+        // Hysteresis: reset threshold tracking when effective demand cools below 70%.
+        // Downgrade critical→warning when effective demand falls back below 85%.
+        // Uses `effectiveDemand` (checkedOut + waitQueueSize) for symmetry with the alert side —
+        // otherwise a long-draining waitQueue with low checkedOut would prematurely reset and
+        // re-fire on the next spike.
+        if (_lastUtilizationThreshold > 0) {
+            const maxPool = connectionPoolStats.maxPoolSize;
+            const effectiveDemand = maxPool > 0
+                ? ((connectionPoolStats.checkedOut + connectionPoolStats.waitQueueSize) / maxPool) * 100
+                : 0;
+            if (effectiveDemand < 70) {
+                _lastUtilizationThreshold = 0;
+            }
+            else if (_lastUtilizationThreshold === 95 && effectiveDemand < 85) {
+                _lastUtilizationThreshold = 80;
+            }
+        }
+    });
+    client.on('connectionCheckOutStarted', () => {
+        connectionPoolStats.waitQueueSize++;
+        // Queue-depth early-warning — fires BEFORE the utilization-based warnings above
+        // because queue can overflow while pool sits at moderate utilization (the actual
+        // failure mode we observe under bursty load: pool at 70% checkedOut but waitQueue
+        // grows past maxPoolSize and triggers waitQueueTimeoutMS). Hysteresis resets when
+        // the queue drains (handled in connectionCheckedOut / connectionCheckOutFailed).
+        const maxPool = connectionPoolStats.maxPoolSize;
+        if (maxPool > 0) {
+            const queue = connectionPoolStats.waitQueueSize;
+            const queueRatio = queue / maxPool;
+            const queueData = {
+                waitQueueSize: queue,
+                maxPoolSize: maxPool,
+                checkedOut: connectionPoolStats.checkedOut,
+                utilizationPercent: Math.round((connectionPoolStats.checkedOut / maxPool) * 100),
+                // F6a: effective demand includes the wait queue. Dashboards should chart this — the
+                // checkedOut-only `utilizationPercent` can sit at 50% while real demand is 158%.
+                effectiveDemandPercent: Math.round(((connectionPoolStats.checkedOut + queue) / maxPool) * 100)
+            };
+            if (queueRatio >= 1.0 && _lastQueueThreshold < 100) {
+                _lastQueueThreshold = 100;
+                emitEvent({
+                    level: 'error',
+                    event: 'pool.queue_critical',
+                    message: `Pool waitQueue critical: ${queue} queued (${Math.round(queueRatio * 100)}% of pool)`,
+                    timestamp: new Date().toISOString(),
+                    data: queueData
+                });
+            }
+            else if (queueRatio >= 0.5 && _lastQueueThreshold < 50) {
+                _lastQueueThreshold = 50;
+                emitEvent({
+                    level: 'warn',
+                    event: 'pool.queue_warning',
+                    message: `Pool waitQueue warning: ${queue} queued (${Math.round(queueRatio * 100)}% of pool)`,
+                    timestamp: new Date().toISOString(),
+                    data: queueData
+                });
+            }
+        }
+    });
+    client.on('connectionCheckOutFailed', (event) => {
+        connectionPoolStats.waitQueueSize = Math.max(0, connectionPoolStats.waitQueueSize - 1);
+        // Same queue-threshold hysteresis as connectionCheckedOut — keeps the reset path
+        // symmetric across success and timeout decrements.
+        if (_lastQueueThreshold > 0) {
+            const maxPool = connectionPoolStats.maxPoolSize;
+            const queueRatio = maxPool > 0 ? connectionPoolStats.waitQueueSize / maxPool : 0;
+            if (queueRatio < 0.25) {
+                _lastQueueThreshold = 0;
+            }
+            else if (_lastQueueThreshold === 100 && queueRatio < 0.75) {
+                _lastQueueThreshold = 50;
+            }
+        }
+        if (event.reason === 'timeout') {
+            connectionPoolStats.waitQueueTimeouts++;
+            connectionPoolStats.lastWaitQueueTimeout = new Date();
+            emitEvent({
+                level: 'error',
+                event: 'pool.checkout_timeout',
+                message: 'pool exhausted - checkout timeout',
+                timestamp: new Date().toISOString(),
+                data: {
+                    waitQueueTimeouts: connectionPoolStats.waitQueueTimeouts,
+                    checkedOut: connectionPoolStats.checkedOut,
+                    maxPoolSize: connectionPoolStats.maxPoolSize,
+                    waitQueueSize: connectionPoolStats.waitQueueSize,
+                    utilizationPercent: connectionPoolStats.maxPoolSize > 0
+                        ? Math.round((connectionPoolStats.checkedOut / connectionPoolStats.maxPoolSize) * 100)
+                        : 0,
+                    // F6a: see the connectionCheckedOut handler — effective demand is the only metric
+                    // that surfaces "queue blew past the pool" at a glance.
+                    effectiveDemandPercent: connectionPoolStats.maxPoolSize > 0
+                        ? Math.round(((connectionPoolStats.checkedOut + connectionPoolStats.waitQueueSize) / connectionPoolStats.maxPoolSize) * 100)
+                        : 0
+                }
+            });
+        }
+    });
+}
+const customIndexes = new Map();
+function registerIndexesImpl(collection, indexes) {
+    customIndexes.set(collection, indexes);
+}
+// ═══════════════════════════════════════════════════════════════════
+// TSMongo – static API
+// ═══════════════════════════════════════════════════════════════════
+class TSMongo {
+    static connect = connectImpl;
+    static getDatabase = getDatabaseImpl;
+    static getClient = getClientImpl;
+    static close = closeImpl;
+    static checkHealth = checkHealthImpl;
+    static registerIndexes = registerIndexesImpl;
+    static getConnectionPoolStats = getConnectionPoolStatsImpl;
+    static getPoolHealthStatus = getPoolHealthStatusImpl;
+    static getDatabaseStats = getDatabaseStatsImpl;
+    static setEventHandler = setEventHandlerImpl;
+}
+exports.TSMongo = TSMongo;
+async function ensureIndexes(database) {
+    if (customIndexes.size === 0)
+        return;
+    try {
+        const collections = await database.listCollections().toArray();
+        const collNames = collections.map((c) => c.name);
+        for (const [collName, indexes] of customIndexes) {
+            if (collNames.includes(collName)) {
+                await database.collection(collName).createIndexes(indexes);
+            }
+        }
+    }
+    catch (error) {
+        const message = error instanceof Error ? error.message : String(error);
+        emitEvent({
+            level: 'error',
+            event: 'index.ensure_failed',
+            message: `ensure indexes failed: ${message}`,
+            timestamp: new Date().toISOString(),
+            data: { error: message }
+        });
+    }
+}
+const READ_PREF_MAP = {
+    primary: mongodb_1.ReadPreference.PRIMARY,
+    secondary: mongodb_1.ReadPreference.SECONDARY_PREFERRED,
+    secondaryPreferred: mongodb_1.ReadPreference.SECONDARY_PREFERRED,
+    nearest: mongodb_1.ReadPreference.NEAREST
+};
+function applyEnvOverrides(cfg) {
+    if (process.env.MONGO_MAX_POOL_SIZE) {
+        cfg.maxPoolSize = parseInt(process.env.MONGO_MAX_POOL_SIZE, 10) || cfg.maxPoolSize;
+    }
+    if (process.env.MONGO_MIN_POOL_SIZE) {
+        cfg.minPoolSize = parseInt(process.env.MONGO_MIN_POOL_SIZE, 10) || cfg.minPoolSize;
+    }
+    if (process.env.MONGO_READ_PREFERENCE && process.env.MONGO_READ_PREFERENCE in READ_PREF_MAP) {
+        cfg.readPreference = process.env.MONGO_READ_PREFERENCE;
+    }
+}
+function resolveDbName(config, pathOrUri) {
+    let dbName = config.dbName || pathOrUri || 'default';
+    if (dbName.includes('?'))
+        dbName = dbName.split('?')[0];
+    return dbName.trim();
+}
+async function tryReuseConnection(uriObj, config) {
+    if (!client || !baseUri || getServerKey(new URL(baseUri)) !== getServerKey(uriObj)) {
+        return null;
+    }
+    try {
+        await client.db('admin').command({ ping: 1 });
+        const explicitPath = uriObj.pathname.slice(1).replace(/\/$/, '');
+        if (explicitPath || config.dbName) {
+            const dbName = resolveDbName(config, explicitPath);
+            db = client.db(dbName);
+        }
+        return db;
+    }
+    catch {
+        client = null;
+        db = null;
+        baseUri = null;
+        return null;
+    }
+}
+async function connectImpl(uri, config = {}) {
+    const uriObj = new URL(uri);
+    const reused = await tryReuseConnection(uriObj, config);
+    if (reused)
+        return reused;
+    // Another connect() is already establishing a client (possibly for a different server —
+    // tryReuseConnection above already ruled out reusing it). Wait for it to settle, then
+    // re-run from the top: either it connected to the same server we want (reuse succeeds) or
+    // it didn't and we fall through to start our own attempt, but never two at once.
+    if (connectingPromise) {
+        await connectingPromise.catch(() => undefined);
+        return connectImpl(uri, config);
+    }
+    const doConnect = async () => {
+        const currentBaseUri = `${uriObj.protocol}//${uriObj.host}${uriObj.search || ''}`;
+        const cfg = { ...exports.DEFAULT_MONGO_CONFIG, ...config };
+        applyEnvOverrides(cfg);
+        const isLocalhost = uriObj.hostname === 'localhost' || uriObj.hostname === '127.0.0.1';
+        let finalUri = uri;
+        if (isLocalhost) {
+            if (!uri.includes('directConnection=')) {
+                finalUri = `${uri}${uri.includes('?') ? '&' : '?'}directConnection=true`;
+            }
+            if (finalUri.includes('replicaSet=')) {
+                finalUri = finalUri.replace(/[?&]replicaSet=[^&]*/, '');
+            }
+        }
+        const clientOptions = {
+            maxPoolSize: cfg.maxPoolSize,
+            minPoolSize: cfg.minPoolSize,
+            maxIdleTimeMS: cfg.maxIdleTimeMS,
+            waitQueueTimeoutMS: cfg.waitQueueTimeoutMS,
+            connectTimeoutMS: cfg.connectTimeoutMS,
+            socketTimeoutMS: cfg.socketTimeoutMS,
+            serverSelectionTimeoutMS: cfg.serverSelectionTimeoutMS,
+            readPreference: READ_PREF_MAP[cfg.readPreference || 'nearest'],
+            writeConcern: new mongodb_1.WriteConcern((cfg.writeConcern ?? 'majority')),
+            retryWrites: cfg.retryWrites,
+            retryReads: cfg.retryReads,
+            monitorCommands: config.monitorCommands ?? true,
+            ...(isLocalhost && { directConnection: true })
+        };
+        if (config.compressors?.length) {
+            clientOptions.compressors = config.compressors;
+        }
+        // Build and connect using a LOCAL client/db reference throughout — only publish to the
+        // shared module-level `client`/`db`/`baseUri` once the whole sequence has succeeded, so a
+        // concurrent reader can never observe a client whose `.connect()`/`ensureIndexes()` is still
+        // in flight (see the comment on `connectingPromise` above).
+        const newClient = new mongodb_1.MongoClient(finalUri, clientOptions);
+        attachClientEventHandlers(newClient, { monitorCommands: config.monitorCommands ?? true });
+        await newClient.connect();
+        const dbName = resolveDbName(config, new URL(finalUri).pathname.slice(1));
+        const newDb = newClient.db(dbName);
+        await ensureIndexes(newDb);
+        baseUri = currentBaseUri;
+        client = newClient;
+        db = newDb;
+        connectionPoolStats.maxPoolSize = cfg.maxPoolSize;
+        connectionPoolStats.minPoolSize = cfg.minPoolSize;
+        return newDb;
+    };
+    connectingPromise = doConnect();
+    try {
+        return await connectingPromise;
+    }
+    finally {
+        connectingPromise = null;
+    }
+}
+function getDatabaseImpl() {
+    if (!db)
+        throw new Error('Database not connected');
+    return db;
+}
+function getClientImpl() {
+    if (!client)
+        throw new Error('Database not connected');
+    return client;
+}
+async function closeImpl() {
+    if (client) {
+        await client.close();
+        client = null;
+        db = null;
+        baseUri = null;
+        resetPoolStats();
+    }
+}
+function getConnectionPoolStatsImpl() {
+    return {
+        ...connectionPoolStats,
+        availableConnections: connectionPoolStats.totalConnections - connectionPoolStats.checkedOut
+    };
+}
+function getPoolHealthStatusImpl() {
+    const utilization = connectionPoolStats.maxPoolSize > 0 ? (connectionPoolStats.checkedOut / connectionPoolStats.maxPoolSize) * 100 : 0;
+    const recentTimeout = connectionPoolStats.lastWaitQueueTimeout &&
+        Date.now() - connectionPoolStats.lastWaitQueueTimeout.getTime() < 60000;
+    if (recentTimeout || utilization >= 95) {
+        return {
+            status: 'critical',
+            utilizationPercent: Math.round(utilization),
+            message: recentTimeout
+                ? `Pool exhausted - ${connectionPoolStats.waitQueueTimeouts} timeout(s)`
+                : 'Pool nearly exhausted (>95%)'
+        };
+    }
+    if (utilization >= 80) {
+        return { status: 'warning', utilizationPercent: Math.round(utilization), message: 'High pool utilization (>80%)' };
+    }
+    return { status: 'healthy', utilizationPercent: Math.round(utilization), message: 'Pool healthy' };
+}
+async function checkHealthImpl() {
+    if (!db || !client)
+        return { healthy: false, latencyMs: -1, connections: 0, checkedOut: 0 };
+    const start = Date.now();
+    try {
+        await db.command({ ping: 1 });
+        return {
+            healthy: true,
+            latencyMs: Date.now() - start,
+            connections: connectionPoolStats.totalConnections,
+            checkedOut: connectionPoolStats.checkedOut
+        };
+    }
+    catch {
+        return { healthy: false, latencyMs: -1, connections: 0, checkedOut: 0 };
+    }
+}
+async function getDatabaseStatsImpl() {
+    if (!db)
+        return {};
+    try {
+        const stats = await db.stats();
+        return {
+            collections: stats.collections,
+            objects: stats.objects,
+            dataSize: stats.dataSize,
+            storageSize: stats.storageSize,
+            indexes: stats.indexes,
+            indexSize: stats.indexSize
+        };
+    }
+    catch {
+        return {};
+    }
+}