specmem-hardwicksoftware 3.5.99 → 3.6.1

package/dist/index.js

@@ -286,6 +286,12 @@ class LocalEmbeddingProvider {
     socketConnected = false;
     socketReconnecting = false;
     pendingRequests = new Map();
+    // FIX Issue #1: Track active sockets to prevent FD leaks over 24h+ sessions
+    // Each entry: { socket, createdAt, label }
+    activeSockets = new Set();
+    _socketCleanupInterval = null;
+    // FIX Issue #6: Track pending request ages for stale cleanup
+    _pendingCleanupInterval = null;
     // Timeout values - now centralized in config/embeddingTimeouts.ts
     // Set SPECMEM_EMBEDDING_TIMEOUT (in seconds) to control ALL timeouts at once
     // See config/embeddingTimeouts.ts for full documentation
@@ -327,6 +333,131 @@ class LocalEmbeddingProvider {
         }
         // Initialize persistent socket connection
         this.initPersistentSocket();
+        // FIX Issue #1: Start periodic socket FD cleanup
+        this._startSocketCleanup();
+        // FIX Issue #6: Start periodic stale pendingRequests cleanup
+        this._startPendingRequestsCleanup();
+    }
+    /**
+     * FIX Issue #1: Track a socket in the active set for FD leak prevention.
+     * Every socket created via createConnection() MUST be registered here.
+     */
+    _trackSocket(socket, label) {
+        const entry = { socket, createdAt: Date.now(), label };
+        this.activeSockets.add(entry);
+        // Auto-remove from tracking when socket is fully closed/destroyed
+        const removeFromTracking = () => {
+            this.activeSockets.delete(entry);
+        };
+        socket.once('close', removeFromTracking);
+        // If socket is already destroyed, remove immediately
+        if (socket.destroyed) {
+            this.activeSockets.delete(entry);
+        }
+        return entry;
+    }
+    /**
+     * FIX Issue #1: Periodic cleanup of leaked socket FDs.
+     * Destroys sockets that have been open longer than SPECMEM_SOCKET_MAX_AGE_MS.
+     * Runs every SPECMEM_SOCKET_CLEANUP_INTERVAL_MS (default 5min).
+     */
+    _startSocketCleanup() {
+        if (this._socketCleanupInterval) {
+            clearInterval(this._socketCleanupInterval);
+        }
+        const cleanupIntervalMs = parseInt(process.env['SPECMEM_SOCKET_CLEANUP_INTERVAL_MS'] || '300000', 10);
+        const maxAgeMs = parseInt(process.env['SPECMEM_SOCKET_MAX_AGE_MS'] || '60000', 10);
+        this._socketCleanupInterval = setInterval(() => {
+            const now = Date.now();
+            let cleaned = 0;
+            for (const entry of this.activeSockets) {
+                const age = now - entry.createdAt;
+                if (age > maxAgeMs) {
+                    try {
+                        if (!entry.socket.destroyed) {
+                            entry.socket.destroy();
+                            cleaned++;
+                            logger.debug({
+                                label: entry.label,
+                                ageMs: age,
+                                maxAgeMs
+                            }, 'LocalEmbeddingProvider: cleaned up stale socket (FD leak prevention)');
+                        }
+                    }
+                    catch (e) {
+                        // Ignore destroy errors during cleanup
+                    }
+                    this.activeSockets.delete(entry);
+                }
+            }
+            if (cleaned > 0) {
+                logger.info({
+                    cleaned,
+                    remaining: this.activeSockets.size,
+                    maxAgeMs,
+                    cleanupIntervalMs
+                }, 'LocalEmbeddingProvider: periodic socket FD cleanup completed');
+            }
+        }, cleanupIntervalMs);
+        // Don't let cleanup interval prevent process exit
+        if (this._socketCleanupInterval.unref) {
+            this._socketCleanupInterval.unref();
+        }
+        logger.debug({
+            cleanupIntervalMs,
+            maxAgeMs
+        }, 'LocalEmbeddingProvider: socket FD cleanup started');
+    }
+    /**
+     * FIX Issue #6: Periodic cleanup of stale pendingRequests entries.
+     * Rejects and removes requests older than SPECMEM_PENDING_REQUEST_MAX_AGE_MS.
+     * Runs every SPECMEM_PENDING_CLEANUP_INTERVAL_MS (default 60s).
+     */
+    _startPendingRequestsCleanup() {
+        if (this._pendingCleanupInterval) {
+            clearInterval(this._pendingCleanupInterval);
+        }
+        const cleanupIntervalMs = parseInt(process.env['SPECMEM_PENDING_CLEANUP_INTERVAL_MS'] || '60000', 10);
+        const maxAgeMs = parseInt(process.env['SPECMEM_PENDING_REQUEST_MAX_AGE_MS'] || '120000', 10);
+        this._pendingCleanupInterval = setInterval(() => {
+            const now = Date.now();
+            let cleaned = 0;
+            for (const [requestId, pending] of this.pendingRequests) {
+                const age = now - (pending.createdAt || 0);
+                if (age > maxAgeMs) {
+                    clearTimeout(pending.timeout);
+                    this.pendingRequests.delete(requestId);
+                    cleaned++;
+                    try {
+                        pending.reject(new Error(
+                            `Stale pending request cleaned up after ${Math.round(age / 1000)}s ` +
+                            `(max age: ${Math.round(maxAgeMs / 1000)}s). ` +
+                            `Request ID: ${requestId}. ` +
+                            `Increase SPECMEM_PENDING_REQUEST_MAX_AGE_MS if embeddings are legitimately slow.`
+                        ));
+                    }
+                    catch (e) {
+                        // Ignore rejection errors (promise may already be settled)
+                    }
+                }
+            }
+            if (cleaned > 0) {
+                logger.warn({
+                    cleaned,
+                    remaining: this.pendingRequests.size,
+                    maxAgeMs,
+                    cleanupIntervalMs
+                }, 'LocalEmbeddingProvider: cleaned up stale pending requests (memory leak prevention)');
+            }
+        }, cleanupIntervalMs);
+        // Don't let cleanup interval prevent process exit
+        if (this._pendingCleanupInterval.unref) {
+            this._pendingCleanupInterval.unref();
+        }
+        logger.debug({
+            cleanupIntervalMs,
+            maxAgeMs
+        }, 'LocalEmbeddingProvider: pending requests cleanup started');
     }
     /**
      * Initialize the persistent socket connection
@@ -392,6 +523,8 @@ class LocalEmbeddingProvider {
         logger.debug({ socketPath: this.sandboxSocketPath }, 'LocalEmbeddingProvider: initializing persistent socket');
         try {
             this.persistentSocket = createConnection(this.sandboxSocketPath);
+            // FIX Issue #1: Track persistent socket for FD leak prevention
+            this._trackSocket(this.persistentSocket, 'persistent');
             __debugLog('[EMBEDDING DEBUG]', Date.now(), 'INIT_PERSISTENT_SOCKET_CONNECTION_CREATED', {
                 elapsedMs: Date.now() - methodStart
             });
@@ -580,6 +713,18 @@ class LocalEmbeddingProvider {
             pending.reject(new Error('Socket reset by user'));
         }
         this.pendingRequests.clear();
+        // FIX Issue #1: Destroy all tracked active sockets on reset
+        for (const entry of this.activeSockets) {
+            try {
+                if (!entry.socket.destroyed) {
+                    entry.socket.destroy();
+                }
+            }
+            catch (e) {
+                // Ignore cleanup errors
+            }
+        }
+        this.activeSockets.clear();
         // Re-detect socket path and reinitialize
         this.sandboxSocketPath = getEmbeddingSocketPath();
         this.initPersistentSocket();
@@ -1243,6 +1388,41 @@ class LocalEmbeddingProvider {
             }
             catch { /* proceed to spawn */ }
         }
+        // ═══════════════════════════════════════════════════════════════════════════
+        // ORPHAN KILL: Kill any existing Frankenstein process BEFORE spawning new one
+        // This prevents multiple processes fighting over the same socket
+        // ═══════════════════════════════════════════════════════════════════════════
+        const pidFile = join(socketDir, 'embedding.pid');
+        try {
+            if (existsSync(pidFile)) {
+                const pidContent = readFileSync(pidFile, 'utf8').trim();
+                const oldPid = parseInt(pidContent.split(':')[0], 10);
+                if (oldPid && !isNaN(oldPid)) {
+                    try {
+                        process.kill(oldPid, 0); // Check if alive
+                        logger.info({ pid: oldPid }, '[LocalEmbeddingProvider] Killing existing embedding process before respawn');
+                        process.kill(oldPid, 'SIGTERM');
+                        // Give it 2s to die gracefully
+                        const killWaitMs = parseInt(process.env['SPECMEM_ORPHAN_KILL_WAIT_MS'] || '2000', 10);
+                        await new Promise(r => setTimeout(r, killWaitMs));
+                        try {
+                            process.kill(oldPid, 0); // Still alive?
+                            process.kill(oldPid, 'SIGKILL');
+                            logger.warn({ pid: oldPid }, '[LocalEmbeddingProvider] Force killed stubborn embedding process');
+                        } catch { /* dead - good */ }
+                    } catch { /* not running - fine */ }
+                }
+            }
+        } catch (pidErr) {
+            logger.debug({ error: pidErr }, '[LocalEmbeddingProvider] PID file cleanup failed (non-fatal)');
+        }
+        // Also remove stale socket file to prevent bind failures
+        if (existsSync(socketPath)) {
+            try {
+                unlinkSync(socketPath);
+                logger.debug('[LocalEmbeddingProvider] Removed stale socket before respawn');
+            } catch { /* ignore */ }
+        }
         // Find the embedding server script
         // Try multiple locations: SPECMEM_PKG env, relative to this file, project specmem dir
         // Docker location via env var (defaults to /opt/specmem if SPECMEM_DOCKER_PATH set)
@@ -1769,9 +1949,22 @@ class LocalEmbeddingProvider {
         return new Promise((resolve, reject) => {
             const socketPath = getEmbeddingSocketPath();
             const socket = createConnection(socketPath);
+            // FIX Issue #1: Track socket for FD leak prevention
+            this._trackSocket(socket, `batch-${texts.length}`);
             let buffer = '';
             let resolved = false;
             const startTime = Date.now();
+            // FIX Issue #1: Ensure socket is destroyed on all exit paths
+            const ensureSocketCleanup = () => {
+                try {
+                    if (!socket.destroyed) {
+                        socket.destroy();
+                    }
+                }
+                catch (e) {
+                    // Ignore cleanup errors
+                }
+            };
             // Longer timeout for batches - scale with batch size
             const baseTimeout = this.getAdaptiveTimeout();
             const timeoutMs = Math.min(baseTimeout * Math.ceil(texts.length / 10), 300000); // Max 5 min
@@ -1783,7 +1976,7 @@ class LocalEmbeddingProvider {
             let timeout = setTimeout(() => {
                 if (!resolved) {
                     resolved = true;
-                    socket.destroy();
+                    ensureSocketCleanup();
                     reject(new Error(`Batch embedding timeout after ${Math.round(timeoutMs / 1000)}s for ${texts.length} texts`));
                 }
             }, timeoutMs);
@@ -1803,7 +1996,7 @@ class LocalEmbeddingProvider {
                 timeout = setTimeout(() => {
                     if (!resolved) {
                         resolved = true;
-                        socket.destroy();
+                        ensureSocketCleanup();
                         reject(new Error(`Batch embedding idle timeout for ${texts.length} texts`));
                     }
                 }, timeoutMs);
@@ -1830,7 +2023,7 @@ class LocalEmbeddingProvider {
                         // Got actual response - resolve or reject
                         clearTimeout(timeout);
                         resolved = true;
-                        socket.destroy();
+                        ensureSocketCleanup();
                         const responseTime = Date.now() - startTime;
                         this.recordResponseTime(responseTime / texts.length); // Per-text average
                         if (response.error) {
@@ -1852,7 +2045,7 @@ class LocalEmbeddingProvider {
                     catch (err) {
                         clearTimeout(timeout);
                         resolved = true;
-                        socket.destroy();
+                        ensureSocketCleanup();
                         reject(new Error(`Failed to parse batch embedding response: ${err}`));
                     }
                 }
@@ -1861,6 +2054,7 @@ class LocalEmbeddingProvider {
                 if (!resolved) {
                     resolved = true;
                     clearTimeout(timeout);
+                    ensureSocketCleanup();
                     reject(err);
                 }
             });
@@ -2161,6 +2355,8 @@ class LocalEmbeddingProvider {
             try {
                 __debugLog('[EMBEDDING DEBUG]', Date.now(), 'WARMING_SOCKET', { socketPath });
                 this.warmSocket = createConnection(socketPath);
+                // FIX Issue #1: Track warm socket for FD leak prevention
+                this._trackSocket(this.warmSocket, 'warm');
                 this.warmSocketPath = socketPath;
                 this.warmSocket.on('connect', () => {
                     __debugLog('[EMBEDDING DEBUG]', Date.now(), 'WARM_SOCKET_CONNECTED', { socketPath });
@@ -2400,10 +2596,23 @@ class LocalEmbeddingProvider {
     generateWithDirectSocketAttempt(text, socketPath, attempt) {
         return new Promise((resolve, reject) => {
             const socket = createConnection(socketPath);
+            // FIX Issue #1: Track socket for FD leak prevention
+            this._trackSocket(socket, `direct-attempt-${attempt}`);
             let buffer = '';
             let resolved = false;
             const startTime = Date.now();
             const timeoutMs = this.getAdaptiveTimeout();
+            // FIX Issue #1: Ensure socket is destroyed on all exit paths
+            const ensureSocketCleanup = () => {
+                try {
+                    if (!socket.destroyed) {
+                        socket.destroy();
+                    }
+                }
+                catch (e) {
+                    // Ignore cleanup errors
+                }
+            };
             __debugLog('[EMBEDDING DEBUG]', Date.now(), 'DIRECT_SOCKET_CONNECTING', {
                 socketPath,
                 attempt,
@@ -2414,7 +2623,7 @@ class LocalEmbeddingProvider {
             let timeout = setTimeout(() => {
                 if (!resolved) {
                     resolved = true;
-                    socket.destroy();
+                    ensureSocketCleanup(); // FIX Issue #1: Use cleanup helper
                     __debugLog('[EMBEDDING DEBUG]', Date.now(), 'DIRECT_SOCKET_INITIAL_TIMEOUT', {
                         socketPath,
                         attempt,
@@ -2443,7 +2652,7 @@ class LocalEmbeddingProvider {
                 timeout = setTimeout(() => {
                     if (!resolved) {
                         resolved = true;
-                        socket.destroy();
+                        ensureSocketCleanup(); // FIX Issue #1: Use cleanup helper
                         __debugLog('[EMBEDDING DEBUG]', Date.now(), 'DIRECT_SOCKET_IDLE_TIMEOUT', {
                             socketPath,
                             attempt,
@@ -2500,14 +2709,14 @@ class LocalEmbeddingProvider {
                         else {
                             reject(new Error(`Invalid response from embedding service (socket: ${socketPath})`));
                         }
-                        socket.end();
+                        ensureSocketCleanup(); // FIX Issue #1: destroy instead of end
                         return;
                     }
                     catch (err) {
                         clearTimeout(timeout);
                         resolved = true;
                         reject(new Error(`Failed to parse embedding response: ${err instanceof Error ? err.message : err} (socket: ${socketPath})`));
-                        socket.end();
+                        ensureSocketCleanup(); // FIX Issue #1: destroy instead of end
                         return;
                     }
                 }
@@ -2521,6 +2730,7 @@ class LocalEmbeddingProvider {
                 });
                 if (!resolved) {
                     resolved = true;
+                    ensureSocketCleanup(); // FIX Issue #1: Ensure socket destroyed on error
                     reject(new Error(`Socket error: ${err.message} (socket: ${socketPath})`));
                 }
             });
@@ -2698,7 +2908,8 @@ class LocalEmbeddingProvider {
                     resolve(embedding);
                 },
                 reject,
-                timeout
+                timeout,
+                createdAt: Date.now() // FIX Issue #6: Track creation time for stale cleanup
             });
             // Send request with ID so we can match responses
             const request = JSON.stringify({ type: 'embed', text, requestId }) + '\n';
@@ -2765,15 +2976,28 @@ class LocalEmbeddingProvider {
     generateWithNewSocketAttempt(text, attempt) {
         return new Promise((resolve, reject) => {
             const socket = createConnection(this.sandboxSocketPath);
+            // FIX Issue #1: Track socket for FD leak prevention
+            this._trackSocket(socket, `new-attempt-${attempt}`);
             let buffer = '';
             let resolved = false;
             const startTime = Date.now();
             const timeoutMs = this.getAdaptiveTimeout();
+            // FIX Issue #1: Ensure socket is destroyed on all exit paths
+            const ensureSocketCleanup = () => {
+                try {
+                    if (!socket.destroyed) {
+                        socket.destroy();
+                    }
+                }
+                catch (e) {
+                    // Ignore cleanup errors
+                }
+            };
             // IDLE-BASED TIMEOUT: Resets on any data received
             let timeout = setTimeout(() => {
                 if (!resolved) {
                     resolved = true;
-                    socket.destroy();
+                    ensureSocketCleanup();
                     reject(new Error(`Embedding idle timeout after ${Math.round(timeoutMs / 1000)}s of no activity ` +
                         `(attempt ${attempt}/${LocalEmbeddingProvider.SOCKET_MAX_RETRIES}). ` +
                         `Socket: ${this.sandboxSocketPath}. ` +
@@ -2791,7 +3015,7 @@ class LocalEmbeddingProvider {
                 timeout = setTimeout(() => {
                     if (!resolved) {
                         resolved = true;
-                        socket.destroy();
+                        ensureSocketCleanup();
                         reject(new Error(`Embedding idle timeout after ${Math.round(timeoutMs / 1000)}s of no activity ` +
                             `(attempt ${attempt}/${LocalEmbeddingProvider.SOCKET_MAX_RETRIES}). ` +
                             `Socket: ${this.sandboxSocketPath}. ` +
@@ -2825,14 +3049,14 @@ class LocalEmbeddingProvider {
                         else {
                             reject(new Error(`Invalid response from embedding service (socket: ${this.sandboxSocketPath})`));
                         }
-                        socket.end();
+                        ensureSocketCleanup(); // FIX Issue #1: destroy instead of end
                         return;
                     }
                     catch (err) {
                         clearTimeout(timeout);
                         resolved = true;
                         reject(new Error(`Failed to parse embedding response: ${err instanceof Error ? err.message : err} (socket: ${this.sandboxSocketPath})`));
-                        socket.end();
+                        ensureSocketCleanup(); // FIX Issue #1: destroy instead of end
                         return;
                     }
                 }
@@ -2841,6 +3065,7 @@ class LocalEmbeddingProvider {
                 clearTimeout(timeout);
                 if (!resolved) {
                     resolved = true;
+                    ensureSocketCleanup(); // FIX Issue #1: Ensure socket destroyed on error
                     reject(new Error(`Socket error: ${err.message} (socket: ${this.sandboxSocketPath})`));
                 }
             });

package/dist/installer/firstRun.js

@@ -141,8 +141,8 @@ SPECMEM_DB_PASSWORD=specmem_westayunprofessional
 SPECMEM_COORDINATION_ENABLED=true
 SPECMEM_COORDINATION_PORT=3001
 
-# Watcher Configuration (OPTIONAL)
-SPECMEM_WATCHER_ENABLED=false
+# Watcher Configuration
+SPECMEM_WATCHER_ENABLED=true
 SPECMEM_WATCHER_ROOT_PATH=
 SPECMEM_WATCHER_IGNORE_PATH=