snow-flow 2.8.9 → 2.9.4

package/dist/utils/mcp-process-manager.js

@@ -0,0 +1,220 @@
+"use strict";
+/**
+ * MCP Process Manager
+ * Prevents excessive MCP server spawning and manages resource limits
+ */
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.MCPProcessManager = void 0;
+const child_process_1 = require("child_process");
+const logger_js_1 = require("./logger.js");
+const logger = new logger_js_1.Logger('MCPProcessManager');
+class MCPProcessManager {
+    constructor() {
+        this.MAX_MCP_SERVERS = parseInt(process.env.SNOW_MAX_MCP_SERVERS || '10');
+        this.MAX_MEMORY_MB = parseInt(process.env.SNOW_MCP_MEMORY_LIMIT || '1500');
+        this.CLEANUP_INTERVAL = 60000; // 1 minute
+        // Start periodic cleanup
+        this.startPeriodicCleanup();
+    }
+    static getInstance() {
+        if (!MCPProcessManager.instance) {
+            MCPProcessManager.instance = new MCPProcessManager();
+        }
+        return MCPProcessManager.instance;
+    }
+    /**
+     * Check if we can spawn a new MCP server
+     */
+    canSpawnServer() {
+        const status = this.getSystemStatus();
+        if (status.processCount >= this.MAX_MCP_SERVERS) {
+            logger.warn(`❌ Cannot spawn: Already at max servers (${status.processCount}/${this.MAX_MCP_SERVERS})`);
+            return false;
+        }
+        if (status.memoryUsageMB > this.MAX_MEMORY_MB) {
+            logger.warn(`❌ Cannot spawn: Memory limit exceeded (${status.memoryUsageMB}MB > ${this.MAX_MEMORY_MB}MB)`);
+            return false;
+        }
+        return true;
+    }
+    /**
+     * Get current MCP system status
+     */
+    getSystemStatus() {
+        try {
+            // Count MCP processes
+            const psOutput = (0, child_process_1.execSync)('ps aux | grep -E "mcp|servicenow.*mcp" | grep -v grep', {
+                encoding: 'utf8'
+            }).trim();
+            if (!psOutput) {
+                return {
+                    processCount: 0,
+                    memoryUsageMB: 0,
+                    processes: []
+                };
+            }
+            const lines = psOutput.split('\n');
+            const processes = [];
+            let totalMemory = 0;
+            for (const line of lines) {
+                const parts = line.split(/\s+/);
+                if (parts.length > 10) {
+                    const pid = parseInt(parts[1]);
+                    const memory = Math.round(parseInt(parts[5]) / 1024); // Convert KB to MB
+                    const name = parts.slice(10).join(' ');
+                    processes.push({ pid, memory, name });
+                    totalMemory += memory;
+                }
+            }
+            return {
+                processCount: processes.length,
+                memoryUsageMB: totalMemory,
+                processes
+            };
+        }
+        catch (error) {
+            // No MCP processes found
+            return {
+                processCount: 0,
+                memoryUsageMB: 0,
+                processes: []
+            };
+        }
+    }
+    /**
+     * Kill duplicate MCP servers (keep only the newest)
+     */
+    killDuplicates() {
+        const status = this.getSystemStatus();
+        // Group processes by server type
+        const serverGroups = new Map();
+        for (const proc of status.processes) {
+            // Extract server type from process name
+            const match = proc.name.match(/servicenow-([^-]+)-mcp\.js/);
+            if (match) {
+                const serverType = match[1];
+                if (!serverGroups.has(serverType)) {
+                    serverGroups.set(serverType, []);
+                }
+                serverGroups.get(serverType).push({
+                    pid: proc.pid,
+                    memory: proc.memory
+                });
+            }
+        }
+        // Kill duplicates (keep the first one)
+        for (const [serverType, procs] of serverGroups) {
+            if (procs.length > 1) {
+                logger.warn(`Found ${procs.length} instances of ${serverType}-mcp, killing duplicates...`);
+                // Sort by PID (older PIDs first) and keep the first one
+                procs.sort((a, b) => a.pid - b.pid);
+                for (let i = 1; i < procs.length; i++) {
+                    try {
+                        process.kill(procs[i].pid, 'SIGTERM');
+                        logger.info(`Killed duplicate ${serverType}-mcp (PID: ${procs[i].pid})`);
+                    }
+                    catch (error) {
+                        // Process might already be dead
+                    }
+                }
+            }
+        }
+    }
+    /**
+     * Kill all MCP servers
+     */
+    killAll() {
+        try {
+            (0, child_process_1.execSync)('pkill -f mcp', { encoding: 'utf8' });
+            logger.info('✅ Killed all MCP processes');
+        }
+        catch (error) {
+            // pkill returns non-zero if no processes found
+        }
+    }
+    /**
+     * Clean up excessive resources
+     */
+    cleanup() {
+        const status = this.getSystemStatus();
+        if (status.processCount > this.MAX_MCP_SERVERS) {
+            logger.warn(`🧹 Cleaning up excessive MCP servers (${status.processCount} > ${this.MAX_MCP_SERVERS})`);
+            this.killDuplicates();
+        }
+        if (status.memoryUsageMB > this.MAX_MEMORY_MB) {
+            logger.warn(`🧹 Memory usage too high (${status.memoryUsageMB}MB), killing oldest processes...`);
+            // Sort by memory usage and kill the highest consumers
+            const sorted = status.processes.sort((a, b) => b.memory - a.memory);
+            let memoryFreed = 0;
+            for (const proc of sorted) {
+                if (status.memoryUsageMB - memoryFreed <= this.MAX_MEMORY_MB * 0.8) {
+                    break; // Stop when we're at 80% of limit
+                }
+                try {
+                    process.kill(proc.pid, 'SIGTERM');
+                    memoryFreed += proc.memory;
+                    logger.info(`Killed high-memory process (PID: ${proc.pid}, ${proc.memory}MB)`);
+                }
+                catch (error) {
+                    // Process might already be dead
+                }
+            }
+        }
+    }
+    /**
+     * Start periodic cleanup
+     */
+    startPeriodicCleanup() {
+        // Clear existing timer
+        if (this.cleanupTimer) {
+            clearInterval(this.cleanupTimer);
+        }
+        // Run cleanup periodically
+        this.cleanupTimer = setInterval(() => {
+            const status = this.getSystemStatus();
+            if (status.processCount > this.MAX_MCP_SERVERS * 0.8 ||
+                status.memoryUsageMB > this.MAX_MEMORY_MB * 0.8) {
+                logger.info('🔄 Running periodic MCP cleanup...');
+                this.cleanup();
+            }
+        }, this.CLEANUP_INTERVAL);
+        // Don't block process exit
+        this.cleanupTimer.unref();
+    }
+    /**
+     * Stop periodic cleanup
+     */
+    stopPeriodicCleanup() {
+        if (this.cleanupTimer) {
+            clearInterval(this.cleanupTimer);
+            this.cleanupTimer = undefined;
+        }
+    }
+    /**
+     * Get resource usage summary
+     */
+    getResourceSummary() {
+        const status = this.getSystemStatus();
+        return `MCP Resources:
+  Processes: ${status.processCount}/${this.MAX_MCP_SERVERS} (${Math.round(status.processCount / this.MAX_MCP_SERVERS * 100)}%)
+  Memory: ${status.memoryUsageMB}MB/${this.MAX_MEMORY_MB}MB (${Math.round(status.memoryUsageMB / this.MAX_MEMORY_MB * 100)}%)
+  Status: ${this.getHealthStatus()}`;
+    }
+    /**
+     * Get health status
+     */
+    getHealthStatus() {
+        const status = this.getSystemStatus();
+        const processPercent = status.processCount / this.MAX_MCP_SERVERS;
+        const memoryPercent = status.memoryUsageMB / this.MAX_MEMORY_MB;
+        if (processPercent > 0.9 || memoryPercent > 0.9) {
+            return 'critical';
+        }
+        if (processPercent > 0.7 || memoryPercent > 0.7) {
+            return 'warning';
+        }
+        return 'healthy';
+    }
+}
+exports.MCPProcessManager = MCPProcessManager;
+//# sourceMappingURL=mcp-process-manager.js.map

package/dist/utils/mcp-server-manager.js

@@ -14,6 +14,7 @@ const events_1 = require("events");
 const os_1 = __importDefault(require("os"));
 const unified_auth_store_js_1 = require("./unified-auth-store.js");
 const mcp_singleton_lock_js_1 = require("./mcp-singleton-lock.js");
+const mcp_process_manager_js_1 = require("./mcp-process-manager.js");
 class MCPServerManager extends events_1.EventEmitter {
     constructor(configPath) {
         super();
@@ -148,6 +149,16 @@ class MCPServerManager extends events_1.EventEmitter {
         if (server.status === 'running') {
             return true; // Already running
         }
+        // Check if we can spawn a new server
+        const processManager = mcp_process_manager_js_1.MCPProcessManager.getInstance();
+        if (!processManager.canSpawnServer()) {
+            // Try cleanup first
+            processManager.cleanup();
+            // Check again after cleanup
+            if (!processManager.canSpawnServer()) {
+                throw new Error('Cannot spawn server: resource limits exceeded. Too many MCP processes running.');
+            }
+        }
         server.status = 'starting';
         this.emit('serverStarting', name);
         try {
@@ -252,15 +263,20 @@ class MCPServerManager extends events_1.EventEmitter {
         try {
             // Try graceful shutdown first
             server.process.kill('SIGTERM');
-            // Wait for graceful shutdown
+            // Wait for graceful shutdown with shorter timeout to prevent hanging
             await new Promise((resolve) => {
                 const timeout = setTimeout(() => {
                     // Force kill if graceful shutdown fails
                     if (server.process) {
-                        server.process.kill('SIGKILL');
+                        try {
+                            server.process.kill('SIGKILL');
+                        }
+                        catch (e) {
+                            // Process might already be dead
+                        }
                     }
                     resolve(undefined);
-                }, 5000);
+                }, 2000); // Reduced from 5000ms to 2000ms to prevent hanging
                 server.process?.on('exit', () => {
                     clearTimeout(timeout);
                     resolve(undefined);
@@ -287,12 +303,31 @@ class MCPServerManager extends events_1.EventEmitter {
         if (!singletonLock.acquire()) {
             throw new Error('❌ MCP servers already running. Cannot start duplicate instances.');
         }
-        console.log('✅ Starting all MCP servers (singleton protected)...');
-        const promises = Array.from(this.servers.keys()).map(name => this.startServer(name).catch(error => {
-            console.error(`Failed to start server '${name}':`, error);
-            return false;
-        }));
-        await Promise.all(promises);
+        // Clean up any existing duplicates first
+        const processManager = mcp_process_manager_js_1.MCPProcessManager.getInstance();
+        processManager.killDuplicates();
+        console.log('✅ Starting all MCP servers (singleton protected with resource limits)...');
+        console.log(processManager.getResourceSummary());
+        // Start servers sequentially with delay to avoid resource spikes
+        let started = 0;
+        for (const name of Array.from(this.servers.keys())) {
+            try {
+                // Check resource limits before each start
+                if (!processManager.canSpawnServer()) {
+                    console.warn(`⚠️ Skipping ${name} - resource limits reached`);
+                    continue;
+                }
+                await this.startServer(name);
+                started++;
+                // Small delay between starts to avoid CPU spike
+                await new Promise(resolve => setTimeout(resolve, 200));
+            }
+            catch (error) {
+                console.error(`Failed to start server '${name}':`, error);
+            }
+        }
+        console.log(`✅ Started ${started}/${this.servers.size} MCP servers`);
+        console.log(processManager.getResourceSummary());
     }
     /**
      * Stop all running MCP servers
@@ -303,6 +338,12 @@ class MCPServerManager extends events_1.EventEmitter {
             return false;
         }));
         await Promise.all(promises);
+        // Release singleton lock after stopping all servers
+        const singletonLock = (0, mcp_singleton_lock_js_1.getMCPSingletonLock)();
+        if (singletonLock.isAcquired()) {
+            singletonLock.release();
+            console.log('✅ Released MCP singleton lock after stopping all servers');
+        }
     }
     /**
      * Get status of a specific server

package/dist/utils/mcp-singleton-lock.d.ts

@@ -15,6 +15,10 @@ export declare class MCPSingletonLock {
      * Release the singleton lock
      */
     release(): void;
+    /**
+     * Release the singleton lock without blocking (for graceful shutdown)
+     */
+    releaseAsync(): Promise<void>;
     /**
      * Check if lock is currently held by this process
      */

package/dist/utils/mcp-singleton-lock.js

@@ -71,6 +71,18 @@ class MCPSingletonLock {
             }
         }
     }
+    /**
+     * Release the singleton lock without blocking (for graceful shutdown)
+     */
+    releaseAsync() {
+        return new Promise((resolve) => {
+            // Use setImmediate to avoid blocking
+            setImmediate(() => {
+                this.release();
+                resolve();
+            });
+        });
+    }
     /**
      * Check if lock is currently held by this process
      */
@@ -82,20 +94,35 @@ class MCPSingletonLock {
      */
     setupCleanupHandlers() {
         const cleanup = () => {
-            this.release();
+            // Use non-blocking release to prevent hanging during shutdown
+            setImmediate(() => {
+                this.release();
+            });
         };
-        process.on('exit', cleanup);
-        process.on('SIGINT', cleanup);
-        process.on('SIGTERM', cleanup);
-        process.on('uncaughtException', (error) => {
+        // Remove existing handlers to prevent duplicate registrations
+        process.removeAllListeners('exit');
+        process.removeAllListeners('SIGINT');
+        process.removeAllListeners('SIGTERM');
+        process.once('exit', cleanup);
+        process.once('SIGINT', () => {
+            cleanup();
+            // Allow graceful exit after cleanup
+            setTimeout(() => process.exit(0), 100);
+        });
+        process.once('SIGTERM', () => {
+            cleanup();
+            // Allow graceful exit after cleanup
+            setTimeout(() => process.exit(0), 100);
+        });
+        process.once('uncaughtException', (error) => {
             logger.error('Uncaught exception, releasing MCP lock:', error);
             cleanup();
-            process.exit(1);
+            setTimeout(() => process.exit(1), 100);
         });
-        process.on('unhandledRejection', (reason) => {
+        process.once('unhandledRejection', (reason) => {
             logger.error('Unhandled rejection, releasing MCP lock:', reason);
             cleanup();
-            process.exit(1);
+            setTimeout(() => process.exit(1), 100);
         });
     }
     /**

package/dist/utils/ml-data-fetcher.d.ts

@@ -0,0 +1,66 @@
+/**
+ * Smart ML Data Fetcher
+ * Handles intelligent data fetching for ML training with batching and field discovery
+ * Prevents token limit errors and optimizes data retrieval
+ */
+export interface MLDataFetchOptions {
+    table: string;
+    query?: string;
+    totalSamples?: number;
+    batchSize?: number;
+    fields?: string[];
+    discoverFields?: boolean;
+    includeContent?: boolean;
+}
+export interface FieldDiscoveryResult {
+    allFields: string[];
+    recommendedFields: string[];
+    sampleData: any[];
+}
+export interface BatchFetchResult {
+    data: any[];
+    totalFetched: number;
+    batchesProcessed: number;
+    fields: string[];
+}
+export declare class MLDataFetcher {
+    private operationsMCP;
+    constructor(operationsMCP: any);
+    /**
+     * Smart fetch with automatic field discovery and batching
+     */
+    smartFetch(options: MLDataFetchOptions): Promise<BatchFetchResult>;
+    /**
+     * Get total count of records matching query
+     */
+    private getRecordCount;
+    /**
+     * Discover available fields by sampling a few records
+     */
+    private discoverFields;
+    /**
+     * Fetch a single batch of data
+     */
+    private fetchBatch;
+    /**
+     * Extract data from MCP tool result
+     */
+    private extractDataFromResult;
+    /**
+     * Calculate optimal batch size based on data characteristics
+     */
+    private calculateOptimalBatchSize;
+    /**
+     * Select appropriate fields for ML training
+     */
+    private selectMLFields;
+    /**
+     * Get default fields for a table type
+     */
+    private getDefaultFields;
+    /**
+     * Get key fields that should always be included
+     */
+    private getKeyFields;
+}
+//# sourceMappingURL=ml-data-fetcher.d.ts.map