specmem-hardwicksoftware 3.7.7 → 3.7.9

package/dist/mcp/embeddingServerManager.js

@@ -189,23 +189,55 @@ export class EmbeddingServerManager extends EventEmitter {
             logger.info({ count: runningServers.length, pids: runningServers.map(s => s.pid) },
                 '[EmbeddingServerManager] FIX 1B: Found existing embedding server processes, checking health');
             // Test if any existing process is healthy via socket
+            // CRITICAL FIX: Retry health check up to 15s to give freshly-spawned servers time to warm up
+            // This prevents MCP bootstrap from killing servers that init just spawned
             if (existsSync(this.socketPath)) {
-                const healthResult = await this.healthCheck();
-                if (healthResult.success) {
-                    logger.info({
-                        pids: runningServers.map(s => s.pid),
-                        responseTimeMs: healthResult.responseTimeMs,
-                    }, '[EmbeddingServerManager] FIX 1B: Existing server is healthy - reusing instead of spawning new');
-                    this.isRunning = true;
-                    this.startTime = Date.now();
-                    this.consecutiveFailures = 0;
-                    return true;
+                for (let attempt = 0; attempt < 6; attempt++) {
+                    const healthResult = await this.healthCheck();
+                    if (healthResult.success) {
+                        logger.info({
+                            pids: runningServers.map(s => s.pid),
+                            responseTimeMs: healthResult.responseTimeMs,
+                            attempt,
+                        }, '[EmbeddingServerManager] FIX 1B: Existing server is healthy - reusing instead of spawning new');
+                        this.isRunning = true;
+                        this.startTime = Date.now();
+                        this.consecutiveFailures = 0;
+                        return true;
+                    }
+                    if (attempt < 5) {
+                        logger.debug({ attempt }, '[EmbeddingServerManager] FIX 1B: Health check failed, waiting 3s before retry...');
+                        await this.sleep(3000);
+                        // Re-check socket still exists
+                        if (!existsSync(this.socketPath)) break;
+                    }
+                }
+            } else {
+                // Socket doesn't exist yet but processes are running - wait for socket to appear
+                logger.debug('[EmbeddingServerManager] FIX 1B: No socket yet, waiting up to 15s for server to create it');
+                for (let i = 0; i < 15; i++) {
+                    await this.sleep(1000);
+                    if (existsSync(this.socketPath)) {
+                        const healthResult = await this.healthCheck();
+                        if (healthResult.success) {
+                            logger.info({
+                                pids: runningServers.map(s => s.pid),
+                                responseTimeMs: healthResult.responseTimeMs,
+                            }, '[EmbeddingServerManager] FIX 1B: Server became healthy after waiting - reusing');
+                            this.isRunning = true;
+                            this.startTime = Date.now();
+                            this.consecutiveFailures = 0;
+                            return true;
+                        }
+                    }
                 }
             }
-            // Existing processes found but unhealthy - kill them all before starting fresh
-            logger.warn({ count: runningServers.length },
-                '[EmbeddingServerManager] FIX 1B: Existing processes are unhealthy, killing all before fresh start');
-            for (const server of runningServers) {
+            // Existing processes found but unhealthy after retries - kill only THIS project's before starting fresh
+            // PROJECT ISOLATION: Filter to only processes belonging to this project
+            const thisProjectServers = runningServers.filter(s => this._isProcessForThisProject(s.pid));
+            logger.warn({ count: thisProjectServers.length, totalFound: runningServers.length },
+                '[EmbeddingServerManager] FIX 1B: Existing processes are unhealthy, killing this project\'s before fresh start');
+            for (const server of thisProjectServers) {
                 try {
                     process.kill(server.pid, 'SIGTERM');
                     logger.info({ pid: server.pid }, '[EmbeddingServerManager] FIX 1B: Sent SIGTERM to unhealthy server');
@@ -217,7 +249,7 @@ export class EmbeddingServerManager extends EventEmitter {
             // Wait 2 seconds for processes to terminate
             await this.sleep(2000);
             // Force kill any survivors
-            for (const server of runningServers) {
+            for (const server of thisProjectServers) {
                 try {
                     process.kill(server.pid, 0); // Check if alive
                     process.kill(server.pid, 'SIGKILL');
@@ -367,19 +399,27 @@ export class EmbeddingServerManager extends EventEmitter {
                 }
             }
             // 2. Kill via pgrep as fallback (catches processes without PID files)
+            // PROJECT ISOLATION: Only kill processes belonging to THIS project
             try {
                 const { execSync: execSyncLocal } = await import('child_process');
                 const pids = execSyncLocal(`pgrep -f "frankenstein-embeddings.py" 2>/dev/null || true`, { encoding: 'utf8' }).trim().split('\n').filter(Boolean);
+                let killedAny = false;
                 for (const pidStr of pids) {
                     const pid = parseInt(pidStr, 10);
                     if (pid && pid !== process.pid) {
+                        // PROJECT ISOLATION: Skip processes belonging to other projects
+                        if (!this._isProcessForThisProject(pid)) {
+                            logger.debug({ pid }, '[EmbeddingServerManager] Pre-spawn: Skipping process belonging to another project');
+                            continue;
+                        }
                         try {
                             process.kill(pid, 'SIGTERM');
+                            killedAny = true;
                             logger.info({ pid }, '[EmbeddingServerManager] Killed orphan frankenstein process (pgrep)');
                         } catch { /* already dead */ }
                     }
                 }
-                if (pids.filter(Boolean).length > 0) {
+                if (killedAny) {
                     await this.sleep(killWaitMs);
                 }
             } catch { /* pgrep not available or no matches */ }
@@ -841,18 +881,20 @@ export class EmbeddingServerManager extends EventEmitter {
         }
         // Also try killing by PID file (in case process reference was lost)
         await this.killByPidFile();
-        // FIX: Kill ALL running embedding server processes (orphans from previous sessions/subagents)
+        // FIX: Kill remaining embedding server processes for THIS project (orphans from previous sessions/subagents)
+        // PROJECT ISOLATION: Only kill processes belonging to this project, not other projects
         const allServers = this.findRunningEmbeddingServers();
-        if (allServers.length > 0) {
-            logger.info({ count: allServers.length, pids: allServers.map(s => s.pid) },
-                '[EmbeddingServerManager] Killing all remaining embedding server processes');
-            for (const server of allServers) {
+        const thisProjectStopServers = allServers.filter(s => this._isProcessForThisProject(s.pid));
+        if (thisProjectStopServers.length > 0) {
+            logger.info({ count: thisProjectStopServers.length, totalFound: allServers.length, pids: thisProjectStopServers.map(s => s.pid) },
+                '[EmbeddingServerManager] Killing remaining embedding server processes for THIS project');
+            for (const server of thisProjectStopServers) {
                 try {
                     process.kill(server.pid, 'SIGTERM');
                 } catch { /* already dead */ }
             }
             await this.sleep(2000);
-            for (const server of allServers) {
+            for (const server of thisProjectStopServers) {
                 try {
                     process.kill(server.pid, 0);
                     process.kill(server.pid, 'SIGKILL');
@@ -1410,7 +1452,7 @@ export class EmbeddingServerManager extends EventEmitter {
             // CRITICAL FIX: Also search for PID files in ANY project path
             // This catches project-specific socket directories like /newServer/specmem/sockets/
             try {
-                const allProjectPids = execSync(`find / -path "*/specmem/sockets/embedding.pid" -o -path "*/.specmem/*/sockets/embedding.pid" 2>/dev/null | xargs cat 2>/dev/null || true`, { encoding: 'utf8', timeout: 5000 }).trim();
+                const allProjectPids = execSync(`find /home /root /opt /srv /var /tmp -maxdepth 6 -path "*/specmem/sockets/embedding.pid" -o -path "*/.specmem/*/sockets/embedding.pid" 2>/dev/null | xargs cat 2>/dev/null || true`, { encoding: 'utf8', timeout: 5000 }).trim();
                 for (const line of allProjectPids.split('\n')) {
                     if (!line.trim())
                         continue;
@@ -1533,6 +1575,34 @@ export class EmbeddingServerManager extends EventEmitter {
             return null;
         }
     }
+    /**
+     * Check if a running process belongs to THIS project instance.
+     * Reads /proc/PID/environ and checks SPECMEM_PROJECT_PATH, SPECMEM_SOCKET_PATH,
+     * and SPECMEM_EMBEDDING_SOCKET for a match against this project's paths.
+     * Returns true if the process belongs to this project, false otherwise.
+     * Returns false on any error (permission denied, process gone, etc.)
+     */
+    _isProcessForThisProject(pid) {
+        try {
+            const environPath = `/proc/${pid}/environ`;
+            if (!existsSync(environPath)) {
+                return false;
+            }
+            const environ = readFileSync(environPath, 'utf8');
+            const envVars = environ.split('\0');
+            const projectPath = this.projectPath || process.cwd();
+            const socketPath = this.socketPath;
+            for (const v of envVars) {
+                if (v.startsWith('SPECMEM_PROJECT_PATH=') && v.includes(projectPath)) return true;
+                if (v.startsWith('SPECMEM_SOCKET_PATH=') && v.includes(projectPath)) return true;
+                if (v.startsWith('SPECMEM_EMBEDDING_SOCKET=') && socketPath && v.includes(socketPath)) return true;
+            }
+            return false;
+        }
+        catch {
+            return false;
+        }
+    }
     /**
      * Get process info for an orphaned process (no PID file)
      */
@@ -1957,19 +2027,23 @@ export class EmbeddingServerManager extends EventEmitter {
                 }
             }
             // FIX 4: Duplicate process detection during health monitoring
-            // Check for multiple embedding server processes and kill extras
+            // Check for multiple embedding server processes FOR THIS PROJECT and kill extras
+            // PROJECT ISOLATION: Filter to only this project's processes before killing duplicates
             try {
                 const runningServers = this.findRunningEmbeddingServers();
-                if (runningServers.length > 1) {
+                // PROJECT ISOLATION: Only consider processes belonging to this project
+                const thisProjectHealthServers = runningServers.filter(s => this._isProcessForThisProject(s.pid));
+                if (thisProjectHealthServers.length > 1) {
                     logger.error({
-                        count: runningServers.length,
-                        pids: runningServers.map(s => s.pid),
-                    }, '[EmbeddingServerManager] CRITICAL: Multiple embedding server processes detected!');
+                        count: thisProjectHealthServers.length,
+                        totalSystemWide: runningServers.length,
+                        pids: thisProjectHealthServers.map(s => s.pid),
+                    }, '[EmbeddingServerManager] CRITICAL: Multiple embedding server processes detected for THIS project!');
                     // Determine the legitimate PID from PID file
                     const legitimatePid = this.readPidFile();
                     if (legitimatePid) {
-                        // Kill all processes that are NOT the legitimate one
-                        for (const server of runningServers) {
+                        // Kill all THIS PROJECT's processes that are NOT the legitimate one
+                        for (const server of thisProjectHealthServers) {
                             if (server.pid !== legitimatePid) {
                                 try {
                                     logger.warn({ pid: server.pid, legitimatePid },
@@ -1994,12 +2068,12 @@ export class EmbeddingServerManager extends EventEmitter {
                         }
                     }
                     else {
-                        // No PID file - keep the first one, kill the rest
+                        // No PID file - keep the first one, kill the rest (only this project's processes)
                         logger.warn('[EmbeddingServerManager] FIX 4: No PID file found, keeping oldest process');
-                        for (let i = 1; i < runningServers.length; i++) {
+                        for (let i = 1; i < thisProjectHealthServers.length; i++) {
                             try {
-                                process.kill(runningServers[i].pid, 'SIGTERM');
-                                logger.warn({ pid: runningServers[i].pid },
+                                process.kill(thisProjectHealthServers[i].pid, 'SIGTERM');
+                                logger.warn({ pid: thisProjectHealthServers[i].pid },
                                     '[EmbeddingServerManager] FIX 4: Killing extra duplicate process');
                             }
                             catch { /* ignore */ }

package/embedding-sandbox/frankenstein-embeddings.py

@@ -150,7 +150,7 @@ import signal
 import sys
 
 # Fix BrokenPipeError when parent process dies - ignore SIGPIPE
-signal.signal(signal.SIGPIPE, signal.SIG_DFL)
+signal.signal(signal.SIGPIPE, signal.SIG_IGN)
 
 def _safe_print(msg, file=None):
     """Print that ignores BrokenPipeError when parent dies"""

package/embedding-sandbox/warm-start.sh

@@ -132,8 +132,19 @@ cleanup_stale() {
 
     # If socket exists but container not running, remove orphan socket
     if [ -S "$SOCKET_PATH" ] && [ "$(get_container_state)" != "running" ] && [ "$(get_container_state)" != "paused" ]; then
-        log "Removing orphaned socket"
-        rm -f "$SOCKET_PATH"
+        # Check if a native Python embedding server owns this socket before removing
+        NATIVE_PID=""
+        PID_FILE="$(dirname "$SOCKET_PATH")/embedding.pid"
+        if [ -f "$PID_FILE" ]; then
+            NATIVE_PID=$(cut -d: -f1 "$PID_FILE" 2>/dev/null)
+        fi
+        if [ -n "$NATIVE_PID" ] && kill -0 "$NATIVE_PID" 2>/dev/null; then
+            log "Socket owned by native Python server (PID $NATIVE_PID) - NOT removing"
+        else
+            log "Removing orphaned socket (no Docker container, no native server)"
+            rm -f "$SOCKET_PATH"
+            [ -f "$PID_FILE" ] && rm -f "$PID_FILE"
+        fi
     fi
 }
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "specmem-hardwicksoftware",
-  "version": "3.7.7",
+  "version": "3.7.9",
   "type": "module",
   "description": "Persistent memory system for coding sessions - semantic search with pgvector, token compression, team coordination, file watching. Needs root: installs system-wide hooks, manages docker/PostgreSQL, writes global configs, handles screen sessions. justcalljon.pro",
   "main": "dist/index.js",

package/scripts/specmem-init.cjs

@@ -3087,6 +3087,39 @@ async function indexCodebase(projectPath, ui, embeddingResult) {
   const serverRunning = embeddingResult?.serverRunning || false;
   const socketExists = activeSocketPath !== null;
 
+  // Helper: Kill any existing embedding server for this project before spawning a new one
+  function killExistingEmbeddingServer(projectPath) {
+    const pidFile = path.join(projectPath, 'specmem', 'sockets', 'embedding.pid');
+    try {
+      if (!fs.existsSync(pidFile)) return false;
+      const content = fs.readFileSync(pidFile, 'utf8').trim();
+      const pid = parseInt(content.split(':')[0], 10);
+      if (!pid || isNaN(pid)) return false;
+      // Check if process is alive
+      try { process.kill(pid, 0); } catch {
+        // Process dead, clean up PID file
+        try { fs.unlinkSync(pidFile); } catch {}
+        return false;
+      }
+      // Process alive — kill it gracefully
+      initLog(`[EMBED] Killing existing embedding server PID ${pid} before respawn`);
+      process.kill(pid, 'SIGTERM');
+      // Wait up to 3 seconds for it to die
+      for (let i = 0; i < 6; i++) {
+        const { execSync } = require('child_process');
+        try { execSync(`sleep 0.5`); } catch {}
+        try { process.kill(pid, 0); } catch { return true; } // Dead
+      }
+      // Force kill if still alive
+      try { process.kill(pid, 'SIGKILL'); } catch {}
+      try { fs.unlinkSync(pidFile); } catch {}
+      return true;
+    } catch (e) {
+      initLog(`[EMBED] PID file check error: ${e.message}`);
+      return false;
+    }
+  }
+
   // CRITICAL: If no embedding server, START IT NOW - never skip embeddings!
   if (!serverRunning && !socketExists) {
     ui.setStatus('Starting embedding server...');
@@ -3116,6 +3149,9 @@ async function indexCodebase(projectPath, ui, embeddingResult) {
       const socketsDir = path.join(projectPath, 'specmem', 'sockets');
       safeMkdir(socketsDir);
 
+      // Kill any existing embedding server before spawning a new one
+      killExistingEmbeddingServer(projectPath);
+
       // Clean up stale socket
       if (fs.existsSync(projectSocketPath)) {
         try {
@@ -3238,6 +3274,9 @@ async function indexCodebase(projectPath, ui, embeddingResult) {
       initLog('Attempting socket recovery - cleaning stale socket and restarting server...');
       ui.setStatus('Recovering embedding server...');
 
+      // Kill any existing embedding server before recovery
+      killExistingEmbeddingServer(projectPath);
+
       // Clean up the stale socket file
       try {
         if (fs.existsSync(projectSocketPath)) {
@@ -3516,6 +3555,10 @@ async function indexCodebase(projectPath, ui, embeddingResult) {
   let consecutiveEmbeddingFailures = 0;
   const MAX_CONSECUTIVE_FAILURES = 3;
 
+  // Respawn backoff for revalidateSocket() — prevents rapid-fire restarts
+  let lastRevalidateTime = 0;
+  let revalidateBackoffMs = 1000; // Start at 1s, doubles each time up to 30s
+
   // FIX: Categorize embedding errors for better debugging and error tracking
   function categorizeEmbeddingError(error) {
     const msg = (error && error.message) ? error.message.toLowerCase() : '';
@@ -3598,6 +3641,16 @@ async function indexCodebase(projectPath, ui, embeddingResult) {
 
   // Revalidate socket path after failures (may have been restarted externally)
   async function revalidateSocket() {
+    // Backoff: prevent rapid-fire revalidation/respawn loops
+    const now = Date.now();
+    const timeSinceLastRevalidate = now - lastRevalidateTime;
+    if (timeSinceLastRevalidate < revalidateBackoffMs) {
+      initLog(`[EMBED] Revalidation throttled (${revalidateBackoffMs}ms backoff, ${timeSinceLastRevalidate}ms elapsed)`);
+      return false;
+    }
+    lastRevalidateTime = now;
+    revalidateBackoffMs = Math.min(revalidateBackoffMs * 2, 30000);
+
     const projectSocketPath = path.join(projectPath, 'specmem', 'sockets', 'embeddings.sock');
     const projDirName = path.basename(projectPath).replace(/[^a-zA-Z0-9_-]/g, '_');
     const sharedSocketPath = path.join(os.homedir(), '.specmem', projDirName, 'sockets', 'embeddings.sock');
@@ -3609,6 +3662,7 @@ async function indexCodebase(projectPath, ui, embeddingResult) {
       if (healthy) {
         initLog(`Revalidated socket: using project socket at ${projectSocketPath}`);
         consecutiveEmbeddingFailures = 0;
+        revalidateBackoffMs = 1000;
         return true;
       }
     }
@@ -3620,6 +3674,7 @@ async function indexCodebase(projectPath, ui, embeddingResult) {
       if (healthy) {
         initLog(`Revalidated socket: using shared socket at ${sharedSocketPath}`);
         consecutiveEmbeddingFailures = 0;
+        revalidateBackoffMs = 1000;
         return true;
       }
     }
@@ -3633,6 +3688,7 @@ async function indexCodebase(projectPath, ui, embeddingResult) {
         if (healthy) {
           initLog(`Revalidated socket: using /tmp shared socket at ${tmpSocket}`);
           consecutiveEmbeddingFailures = 0;
+          revalidateBackoffMs = 1000;
           return true;
         }
       }
@@ -3654,6 +3710,8 @@ async function indexCodebase(projectPath, ui, embeddingResult) {
       if (embeddingScript) {
         const socketsDir = path.join(projectPath, 'specmem', 'sockets');
         if (!fs.existsSync(socketsDir)) fs.mkdirSync(socketsDir, { recursive: true });
+        // Kill any existing embedding server before respawn
+        killExistingEmbeddingServer(projectPath);
         // Clean stale socket
         if (fs.existsSync(projectSocketPath)) {
           try { fs.unlinkSync(projectSocketPath); } catch { /* ignore */ }
@@ -3681,6 +3739,7 @@ async function indexCodebase(projectPath, ui, embeddingResult) {
             if (healthy) {
               initLog(`Embedding server auto-restarted successfully, socket at ${projectSocketPath}`);
               consecutiveEmbeddingFailures = 0;
+              revalidateBackoffMs = 1000;
               return true;
             }
           }
@@ -4858,6 +4917,9 @@ async function extractSessions(projectPath, ui, embeddingResult = null) {
       const socketsDir = path.join(projectPath, 'specmem', 'sockets');
       safeMkdir(socketsDir);
 
+      // Kill any existing embedding server before spawning for session extraction
+      killExistingEmbeddingServer(projectPath);
+
       // Clean up stale socket
       if (fs.existsSync(projectSocketPath)) {
         try {