specmem-hardwicksoftware 3.7.6 → 3.7.8

package/dist/mcp/embeddingServerManager.js

@@ -202,10 +202,12 @@ export class EmbeddingServerManager extends EventEmitter {
                     return true;
                 }
             }
-            // Existing processes found but unhealthy - kill them all before starting fresh
-            logger.warn({ count: runningServers.length },
-                '[EmbeddingServerManager] FIX 1B: Existing processes are unhealthy, killing all before fresh start');
-            for (const server of runningServers) {
+            // Existing processes found but unhealthy - kill only THIS project's before starting fresh
+            // PROJECT ISOLATION: Filter to only processes belonging to this project
+            const thisProjectServers = runningServers.filter(s => this._isProcessForThisProject(s.pid));
+            logger.warn({ count: thisProjectServers.length, totalFound: runningServers.length },
+                '[EmbeddingServerManager] FIX 1B: Existing processes are unhealthy, killing this project\'s before fresh start');
+            for (const server of thisProjectServers) {
                 try {
                     process.kill(server.pid, 'SIGTERM');
                     logger.info({ pid: server.pid }, '[EmbeddingServerManager] FIX 1B: Sent SIGTERM to unhealthy server');
@@ -217,7 +219,7 @@ export class EmbeddingServerManager extends EventEmitter {
             // Wait 2 seconds for processes to terminate
             await this.sleep(2000);
             // Force kill any survivors
-            for (const server of runningServers) {
+            for (const server of thisProjectServers) {
                 try {
                     process.kill(server.pid, 0); // Check if alive
                     process.kill(server.pid, 'SIGKILL');
@@ -367,19 +369,27 @@ export class EmbeddingServerManager extends EventEmitter {
                 }
             }
             // 2. Kill via pgrep as fallback (catches processes without PID files)
+            // PROJECT ISOLATION: Only kill processes belonging to THIS project
             try {
                 const { execSync: execSyncLocal } = await import('child_process');
                 const pids = execSyncLocal(`pgrep -f "frankenstein-embeddings.py" 2>/dev/null || true`, { encoding: 'utf8' }).trim().split('\n').filter(Boolean);
+                let killedAny = false;
                 for (const pidStr of pids) {
                     const pid = parseInt(pidStr, 10);
                     if (pid && pid !== process.pid) {
+                        // PROJECT ISOLATION: Skip processes belonging to other projects
+                        if (!this._isProcessForThisProject(pid)) {
+                            logger.debug({ pid }, '[EmbeddingServerManager] Pre-spawn: Skipping process belonging to another project');
+                            continue;
+                        }
                         try {
                             process.kill(pid, 'SIGTERM');
+                            killedAny = true;
                             logger.info({ pid }, '[EmbeddingServerManager] Killed orphan frankenstein process (pgrep)');
                         } catch { /* already dead */ }
                     }
                 }
-                if (pids.filter(Boolean).length > 0) {
+                if (killedAny) {
                     await this.sleep(killWaitMs);
                 }
             } catch { /* pgrep not available or no matches */ }
@@ -841,18 +851,20 @@ export class EmbeddingServerManager extends EventEmitter {
         }
         // Also try killing by PID file (in case process reference was lost)
         await this.killByPidFile();
-        // FIX: Kill ALL running embedding server processes (orphans from previous sessions/subagents)
+        // FIX: Kill remaining embedding server processes for THIS project (orphans from previous sessions/subagents)
+        // PROJECT ISOLATION: Only kill processes belonging to this project, not other projects
         const allServers = this.findRunningEmbeddingServers();
-        if (allServers.length > 0) {
-            logger.info({ count: allServers.length, pids: allServers.map(s => s.pid) },
-                '[EmbeddingServerManager] Killing all remaining embedding server processes');
-            for (const server of allServers) {
+        const thisProjectStopServers = allServers.filter(s => this._isProcessForThisProject(s.pid));
+        if (thisProjectStopServers.length > 0) {
+            logger.info({ count: thisProjectStopServers.length, totalFound: allServers.length, pids: thisProjectStopServers.map(s => s.pid) },
+                '[EmbeddingServerManager] Killing remaining embedding server processes for THIS project');
+            for (const server of thisProjectStopServers) {
                 try {
                     process.kill(server.pid, 'SIGTERM');
                 } catch { /* already dead */ }
             }
             await this.sleep(2000);
-            for (const server of allServers) {
+            for (const server of thisProjectStopServers) {
                 try {
                     process.kill(server.pid, 0);
                     process.kill(server.pid, 'SIGKILL');
@@ -1533,6 +1545,34 @@ export class EmbeddingServerManager extends EventEmitter {
             return null;
         }
     }
+    /**
+     * Check if a running process belongs to THIS project instance.
+     * Reads /proc/PID/environ and checks SPECMEM_PROJECT_PATH, SPECMEM_SOCKET_PATH,
+     * and SPECMEM_EMBEDDING_SOCKET for a match against this project's paths.
+     * Returns true if the process belongs to this project, false otherwise.
+     * Returns false on any error (permission denied, process gone, etc.)
+     */
+    _isProcessForThisProject(pid) {
+        try {
+            const environPath = `/proc/${pid}/environ`;
+            if (!existsSync(environPath)) {
+                return false;
+            }
+            const environ = readFileSync(environPath, 'utf8');
+            const envVars = environ.split('\0');
+            const projectPath = this.projectPath || process.cwd();
+            const socketPath = this.socketPath;
+            for (const v of envVars) {
+                if (v.startsWith('SPECMEM_PROJECT_PATH=') && v.includes(projectPath)) return true;
+                if (v.startsWith('SPECMEM_SOCKET_PATH=') && v.includes(projectPath)) return true;
+                if (v.startsWith('SPECMEM_EMBEDDING_SOCKET=') && socketPath && v.includes(socketPath)) return true;
+            }
+            return false;
+        }
+        catch {
+            return false;
+        }
+    }
     /**
      * Get process info for an orphaned process (no PID file)
      */
@@ -1957,19 +1997,23 @@ export class EmbeddingServerManager extends EventEmitter {
                 }
             }
             // FIX 4: Duplicate process detection during health monitoring
-            // Check for multiple embedding server processes and kill extras
+            // Check for multiple embedding server processes FOR THIS PROJECT and kill extras
+            // PROJECT ISOLATION: Filter to only this project's processes before killing duplicates
             try {
                 const runningServers = this.findRunningEmbeddingServers();
-                if (runningServers.length > 1) {
+                // PROJECT ISOLATION: Only consider processes belonging to this project
+                const thisProjectHealthServers = runningServers.filter(s => this._isProcessForThisProject(s.pid));
+                if (thisProjectHealthServers.length > 1) {
                     logger.error({
-                        count: runningServers.length,
-                        pids: runningServers.map(s => s.pid),
-                    }, '[EmbeddingServerManager] CRITICAL: Multiple embedding server processes detected!');
+                        count: thisProjectHealthServers.length,
+                        totalSystemWide: runningServers.length,
+                        pids: thisProjectHealthServers.map(s => s.pid),
+                    }, '[EmbeddingServerManager] CRITICAL: Multiple embedding server processes detected for THIS project!');
                     // Determine the legitimate PID from PID file
                     const legitimatePid = this.readPidFile();
                     if (legitimatePid) {
-                        // Kill all processes that are NOT the legitimate one
-                        for (const server of runningServers) {
+                        // Kill all THIS PROJECT's processes that are NOT the legitimate one
+                        for (const server of thisProjectHealthServers) {
                             if (server.pid !== legitimatePid) {
                                 try {
                                     logger.warn({ pid: server.pid, legitimatePid },
@@ -1994,12 +2038,12 @@ export class EmbeddingServerManager extends EventEmitter {
                         }
                     }
                     else {
-                        // No PID file - keep the first one, kill the rest
+                        // No PID file - keep the first one, kill the rest (only this project's processes)
                         logger.warn('[EmbeddingServerManager] FIX 4: No PID file found, keeping oldest process');
-                        for (let i = 1; i < runningServers.length; i++) {
+                        for (let i = 1; i < thisProjectHealthServers.length; i++) {
                             try {
-                                process.kill(runningServers[i].pid, 'SIGTERM');
-                                logger.warn({ pid: runningServers[i].pid },
+                                process.kill(thisProjectHealthServers[i].pid, 'SIGTERM');
+                                logger.warn({ pid: thisProjectHealthServers[i].pid },
                                     '[EmbeddingServerManager] FIX 4: Killing extra duplicate process');
                             }
                             catch { /* ignore */ }

package/embedding-sandbox/frankenstein-embeddings.py

@@ -150,7 +150,7 @@ import signal
 import sys
 
 # Fix BrokenPipeError when parent process dies - ignore SIGPIPE
-signal.signal(signal.SIGPIPE, signal.SIG_DFL)
+signal.signal(signal.SIGPIPE, signal.SIG_IGN)
 
 def _safe_print(msg, file=None):
     """Print that ignores BrokenPipeError when parent dies"""
@@ -4014,9 +4014,23 @@ def main():
 
     # Signal handling for graceful shutdown
     import signal
+    import traceback
     def handle_signal(signum, frame):
         sig_name = signal.Signals(signum).name
+        # DEBUG: Log who sent the signal for troubleshooting
+        my_pid = os.getpid()
+        my_ppid = os.getppid()
         print(f"\n⚡ Received {sig_name} - shutting down gracefully...", file=sys.stderr)
+        print(f"   DEBUG: my_pid={my_pid}, my_ppid={my_ppid}", file=sys.stderr)
+        # Try to identify caller via /proc
+        try:
+            with open(f'/proc/{my_ppid}/cmdline', 'r') as f:
+                parent_cmd = f.read().replace('\x00', ' ').strip()
+            print(f"   DEBUG: parent_cmd={parent_cmd[:200]}", file=sys.stderr)
+        except Exception as e:
+            print(f"   DEBUG: could not read parent cmdline: {e}", file=sys.stderr)
+        print(f"   DEBUG: stack trace:", file=sys.stderr)
+        traceback.print_stack(frame, file=sys.stderr)
         server.shutdown_requested = True
         # Stop QQMS v2 drain thread if enabled
         if qqms_v2_instance:

package/embedding-sandbox/warm-start.sh

@@ -132,8 +132,19 @@ cleanup_stale() {
 
     # If socket exists but container not running, remove orphan socket
     if [ -S "$SOCKET_PATH" ] && [ "$(get_container_state)" != "running" ] && [ "$(get_container_state)" != "paused" ]; then
-        log "Removing orphaned socket"
-        rm -f "$SOCKET_PATH"
+        # Check if a native Python embedding server owns this socket before removing
+        NATIVE_PID=""
+        PID_FILE="$(dirname "$SOCKET_PATH")/embedding.pid"
+        if [ -f "$PID_FILE" ]; then
+            NATIVE_PID=$(cut -d: -f1 "$PID_FILE" 2>/dev/null)
+        fi
+        if [ -n "$NATIVE_PID" ] && kill -0 "$NATIVE_PID" 2>/dev/null; then
+            log "Socket owned by native Python server (PID $NATIVE_PID) - NOT removing"
+        else
+            log "Removing orphaned socket (no Docker container, no native server)"
+            rm -f "$SOCKET_PATH"
+            [ -f "$PID_FILE" ] && rm -f "$PID_FILE"
+        fi
     fi
 }
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "specmem-hardwicksoftware",
-  "version": "3.7.6",
+  "version": "3.7.8",
   "type": "module",
   "description": "Persistent memory system for coding sessions - semantic search with pgvector, token compression, team coordination, file watching. Needs root: installs system-wide hooks, manages docker/PostgreSQL, writes global configs, handles screen sessions. justcalljon.pro",
   "main": "dist/index.js",

package/scripts/specmem-init.cjs

@@ -3087,6 +3087,39 @@ async function indexCodebase(projectPath, ui, embeddingResult) {
   const serverRunning = embeddingResult?.serverRunning || false;
   const socketExists = activeSocketPath !== null;
 
+  // Helper: Kill any existing embedding server for this project before spawning a new one
+  function killExistingEmbeddingServer(projectPath) {
+    const pidFile = path.join(projectPath, 'specmem', 'sockets', 'embedding.pid');
+    try {
+      if (!fs.existsSync(pidFile)) return false;
+      const content = fs.readFileSync(pidFile, 'utf8').trim();
+      const pid = parseInt(content.split(':')[0], 10);
+      if (!pid || isNaN(pid)) return false;
+      // Check if process is alive
+      try { process.kill(pid, 0); } catch {
+        // Process dead, clean up PID file
+        try { fs.unlinkSync(pidFile); } catch {}
+        return false;
+      }
+      // Process alive — kill it gracefully
+      initLog(`[EMBED] Killing existing embedding server PID ${pid} before respawn`);
+      process.kill(pid, 'SIGTERM');
+      // Wait up to 3 seconds for it to die
+      for (let i = 0; i < 6; i++) {
+        const { execSync } = require('child_process');
+        try { execSync(`sleep 0.5`); } catch {}
+        try { process.kill(pid, 0); } catch { return true; } // Dead
+      }
+      // Force kill if still alive
+      try { process.kill(pid, 'SIGKILL'); } catch {}
+      try { fs.unlinkSync(pidFile); } catch {}
+      return true;
+    } catch (e) {
+      initLog(`[EMBED] PID file check error: ${e.message}`);
+      return false;
+    }
+  }
+
   // CRITICAL: If no embedding server, START IT NOW - never skip embeddings!
   if (!serverRunning && !socketExists) {
     ui.setStatus('Starting embedding server...');
@@ -3116,6 +3149,9 @@ async function indexCodebase(projectPath, ui, embeddingResult) {
       const socketsDir = path.join(projectPath, 'specmem', 'sockets');
       safeMkdir(socketsDir);
 
+      // Kill any existing embedding server before spawning a new one
+      killExistingEmbeddingServer(projectPath);
+
       // Clean up stale socket
       if (fs.existsSync(projectSocketPath)) {
         try {
@@ -3238,6 +3274,9 @@ async function indexCodebase(projectPath, ui, embeddingResult) {
       initLog('Attempting socket recovery - cleaning stale socket and restarting server...');
       ui.setStatus('Recovering embedding server...');
 
+      // Kill any existing embedding server before recovery
+      killExistingEmbeddingServer(projectPath);
+
       // Clean up the stale socket file
       try {
         if (fs.existsSync(projectSocketPath)) {
@@ -3516,6 +3555,10 @@ async function indexCodebase(projectPath, ui, embeddingResult) {
   let consecutiveEmbeddingFailures = 0;
   const MAX_CONSECUTIVE_FAILURES = 3;
 
+  // Respawn backoff for revalidateSocket() — prevents rapid-fire restarts
+  let lastRevalidateTime = 0;
+  let revalidateBackoffMs = 1000; // Start at 1s, doubles each time up to 30s
+
   // FIX: Categorize embedding errors for better debugging and error tracking
   function categorizeEmbeddingError(error) {
     const msg = (error && error.message) ? error.message.toLowerCase() : '';
@@ -3598,6 +3641,16 @@ async function indexCodebase(projectPath, ui, embeddingResult) {
 
   // Revalidate socket path after failures (may have been restarted externally)
   async function revalidateSocket() {
+    // Backoff: prevent rapid-fire revalidation/respawn loops
+    const now = Date.now();
+    const timeSinceLastRevalidate = now - lastRevalidateTime;
+    if (timeSinceLastRevalidate < revalidateBackoffMs) {
+      initLog(`[EMBED] Revalidation throttled (${revalidateBackoffMs}ms backoff, ${timeSinceLastRevalidate}ms elapsed)`);
+      return false;
+    }
+    lastRevalidateTime = now;
+    revalidateBackoffMs = Math.min(revalidateBackoffMs * 2, 30000);
+
     const projectSocketPath = path.join(projectPath, 'specmem', 'sockets', 'embeddings.sock');
     const projDirName = path.basename(projectPath).replace(/[^a-zA-Z0-9_-]/g, '_');
     const sharedSocketPath = path.join(os.homedir(), '.specmem', projDirName, 'sockets', 'embeddings.sock');
@@ -3609,6 +3662,7 @@ async function indexCodebase(projectPath, ui, embeddingResult) {
       if (healthy) {
         initLog(`Revalidated socket: using project socket at ${projectSocketPath}`);
         consecutiveEmbeddingFailures = 0;
+        revalidateBackoffMs = 1000;
         return true;
       }
     }
@@ -3620,6 +3674,7 @@ async function indexCodebase(projectPath, ui, embeddingResult) {
       if (healthy) {
         initLog(`Revalidated socket: using shared socket at ${sharedSocketPath}`);
         consecutiveEmbeddingFailures = 0;
+        revalidateBackoffMs = 1000;
         return true;
       }
     }
@@ -3633,6 +3688,7 @@ async function indexCodebase(projectPath, ui, embeddingResult) {
         if (healthy) {
           initLog(`Revalidated socket: using /tmp shared socket at ${tmpSocket}`);
           consecutiveEmbeddingFailures = 0;
+          revalidateBackoffMs = 1000;
           return true;
         }
       }
@@ -3654,20 +3710,25 @@ async function indexCodebase(projectPath, ui, embeddingResult) {
       if (embeddingScript) {
         const socketsDir = path.join(projectPath, 'specmem', 'sockets');
         if (!fs.existsSync(socketsDir)) fs.mkdirSync(socketsDir, { recursive: true });
+        // Kill any existing embedding server before respawn
+        killExistingEmbeddingServer(projectPath);
         // Clean stale socket
         if (fs.existsSync(projectSocketPath)) {
           try { fs.unlinkSync(projectSocketPath); } catch { /* ignore */ }
         }
         const pythonPath = getPythonPath();
+        // CRITICAL: Use file descriptor stdio, NOT pipes. Piped stdio causes SIGTERM
+        // when parent event loop is under heavy load during batch indexing.
+        const revalLogPath = path.join(projectPath, 'specmem', 'sockets', 'embedding-autostart.log');
+        const revalLogFd = fs.openSync(revalLogPath, 'a');
         const proc = spawn(pythonPath, [embeddingScript], {
           cwd: path.dirname(embeddingScript),
           env: { ...process.env, SPECMEM_SOCKET_PATH: projectSocketPath, SPECMEM_PROJECT_PATH: projectPath },
           detached: true,
-          stdio: ['ignore', 'pipe', 'pipe']
+          stdio: ['ignore', revalLogFd, revalLogFd]
         });
-        proc.stdout.on('data', () => {});
-        proc.stderr.on('data', () => {});
         proc.on('error', () => {});
+        fs.closeSync(revalLogFd);
         proc.unref();
         // Wait up to 15s for socket to appear
         for (let i = 0; i < 30; i++) {
@@ -3678,6 +3739,7 @@ async function indexCodebase(projectPath, ui, embeddingResult) {
             if (healthy) {
               initLog(`Embedding server auto-restarted successfully, socket at ${projectSocketPath}`);
               consecutiveEmbeddingFailures = 0;
+              revalidateBackoffMs = 1000;
               return true;
             }
           }
@@ -4855,6 +4917,9 @@ async function extractSessions(projectPath, ui, embeddingResult = null) {
       const socketsDir = path.join(projectPath, 'specmem', 'sockets');
       safeMkdir(socketsDir);
 
+      // Kill any existing embedding server before spawning for session extraction
+      killExistingEmbeddingServer(projectPath);
+
       // Clean up stale socket
       if (fs.existsSync(projectSocketPath)) {
         try {
@@ -4867,6 +4932,10 @@ async function extractSessions(projectPath, ui, embeddingResult = null) {
 
       // Task #22 fix: Use getPythonPath() instead of hardcoded 'python3'
       const pythonPath = getPythonPath();
+      // CRITICAL: Use file descriptor stdio, NOT pipes. Piped stdio causes SIGTERM
+      // when parent event loop is under heavy load during batch indexing.
+      const sessEmbedLogPath = path.join(projectPath, 'specmem', 'sockets', 'embedding-autostart.log');
+      const sessEmbedLogFd = fs.openSync(sessEmbedLogPath, 'a');
       const embeddingProcess = spawn(pythonPath, [embeddingScript], {
         cwd: path.dirname(embeddingScript),
         env: {
@@ -4875,7 +4944,7 @@ async function extractSessions(projectPath, ui, embeddingResult = null) {
           SPECMEM_PROJECT_PATH: projectPath
         },
         detached: true,
-        stdio: ['ignore', 'pipe', 'pipe']
+        stdio: ['ignore', sessEmbedLogFd, sessEmbedLogFd]
       });
 
       // error handler BEFORE unref - prevents silent spawn failures
@@ -4883,14 +4952,7 @@ async function extractSessions(projectPath, ui, embeddingResult = null) {
         ui.setSubStatus('Embedding spawn error: ' + err.message);
       });
 
-      // CRITICAL: Consume stdout/stderr so Python startup banner doesn't leak to terminal
-      embeddingProcess.stdout.on('data', (chunk) => {
-        initLog('[EMBED-STDOUT] ' + chunk.toString().trim());
-      });
-      embeddingProcess.stderr.on('data', (chunk) => {
-        initLog('[EMBED-STDERR] ' + chunk.toString().trim());
-      });
-
+      fs.closeSync(sessEmbedLogFd);
       embeddingProcess.unref();
 
       // Wait for socket to appear (up to 30s)