snow-flow 2.9.0 → 2.9.4

package/dist/utils/mcp-server-manager.js

@@ -14,6 +14,7 @@ const events_1 = require("events");
 const os_1 = __importDefault(require("os"));
 const unified_auth_store_js_1 = require("./unified-auth-store.js");
 const mcp_singleton_lock_js_1 = require("./mcp-singleton-lock.js");
+const mcp_process_manager_js_1 = require("./mcp-process-manager.js");
 class MCPServerManager extends events_1.EventEmitter {
     constructor(configPath) {
         super();
@@ -148,6 +149,16 @@ class MCPServerManager extends events_1.EventEmitter {
         if (server.status === 'running') {
             return true; // Already running
         }
+        // Check if we can spawn a new server
+        const processManager = mcp_process_manager_js_1.MCPProcessManager.getInstance();
+        if (!processManager.canSpawnServer()) {
+            // Try cleanup first
+            processManager.cleanup();
+            // Check again after cleanup
+            if (!processManager.canSpawnServer()) {
+                throw new Error('Cannot spawn server: resource limits exceeded. Too many MCP processes running.');
+            }
+        }
         server.status = 'starting';
         this.emit('serverStarting', name);
         try {
@@ -252,15 +263,20 @@ class MCPServerManager extends events_1.EventEmitter {
         try {
             // Try graceful shutdown first
             server.process.kill('SIGTERM');
-            // Wait for graceful shutdown
+            // Wait for graceful shutdown with shorter timeout to prevent hanging
             await new Promise((resolve) => {
                 const timeout = setTimeout(() => {
                     // Force kill if graceful shutdown fails
                     if (server.process) {
-                        server.process.kill('SIGKILL');
+                        try {
+                            server.process.kill('SIGKILL');
+                        }
+                        catch (e) {
+                            // Process might already be dead
+                        }
                     }
                     resolve(undefined);
-                }, 5000);
+                }, 2000); // Reduced from 5000ms to 2000ms to prevent hanging
                 server.process?.on('exit', () => {
                     clearTimeout(timeout);
                     resolve(undefined);
@@ -287,12 +303,31 @@ class MCPServerManager extends events_1.EventEmitter {
         if (!singletonLock.acquire()) {
             throw new Error('❌ MCP servers already running. Cannot start duplicate instances.');
         }
-        console.log('✅ Starting all MCP servers (singleton protected)...');
-        const promises = Array.from(this.servers.keys()).map(name => this.startServer(name).catch(error => {
-            console.error(`Failed to start server '${name}':`, error);
-            return false;
-        }));
-        await Promise.all(promises);
+        // Clean up any existing duplicates first
+        const processManager = mcp_process_manager_js_1.MCPProcessManager.getInstance();
+        processManager.killDuplicates();
+        console.log('✅ Starting all MCP servers (singleton protected with resource limits)...');
+        console.log(processManager.getResourceSummary());
+        // Start servers sequentially with delay to avoid resource spikes
+        let started = 0;
+        for (const name of Array.from(this.servers.keys())) {
+            try {
+                // Check resource limits before each start
+                if (!processManager.canSpawnServer()) {
+                    console.warn(`⚠️ Skipping ${name} - resource limits reached`);
+                    continue;
+                }
+                await this.startServer(name);
+                started++;
+                // Small delay between starts to avoid CPU spike
+                await new Promise(resolve => setTimeout(resolve, 200));
+            }
+            catch (error) {
+                console.error(`Failed to start server '${name}':`, error);
+            }
+        }
+        console.log(`✅ Started ${started}/${this.servers.size} MCP servers`);
+        console.log(processManager.getResourceSummary());
     }
     /**
      * Stop all running MCP servers
@@ -303,6 +338,12 @@ class MCPServerManager extends events_1.EventEmitter {
             return false;
         }));
         await Promise.all(promises);
+        // Release singleton lock after stopping all servers
+        const singletonLock = (0, mcp_singleton_lock_js_1.getMCPSingletonLock)();
+        if (singletonLock.isAcquired()) {
+            singletonLock.release();
+            console.log('✅ Released MCP singleton lock after stopping all servers');
+        }
     }
     /**
      * Get status of a specific server

package/dist/utils/mcp-singleton-lock.d.ts

@@ -15,6 +15,10 @@ export declare class MCPSingletonLock {
      * Release the singleton lock
      */
     release(): void;
+    /**
+     * Release the singleton lock without blocking (for graceful shutdown)
+     */
+    releaseAsync(): Promise<void>;
     /**
      * Check if lock is currently held by this process
      */

package/dist/utils/mcp-singleton-lock.js

@@ -71,6 +71,18 @@ class MCPSingletonLock {
             }
         }
     }
+    /**
+     * Release the singleton lock without blocking (for graceful shutdown)
+     */
+    releaseAsync() {
+        return new Promise((resolve) => {
+            // Use setImmediate to avoid blocking
+            setImmediate(() => {
+                this.release();
+                resolve();
+            });
+        });
+    }
     /**
      * Check if lock is currently held by this process
      */
@@ -82,20 +94,35 @@ class MCPSingletonLock {
      */
     setupCleanupHandlers() {
         const cleanup = () => {
-            this.release();
+            // Use non-blocking release to prevent hanging during shutdown
+            setImmediate(() => {
+                this.release();
+            });
         };
-        process.on('exit', cleanup);
-        process.on('SIGINT', cleanup);
-        process.on('SIGTERM', cleanup);
-        process.on('uncaughtException', (error) => {
+        // Remove existing handlers to prevent duplicate registrations
+        process.removeAllListeners('exit');
+        process.removeAllListeners('SIGINT');
+        process.removeAllListeners('SIGTERM');
+        process.once('exit', cleanup);
+        process.once('SIGINT', () => {
+            cleanup();
+            // Allow graceful exit after cleanup
+            setTimeout(() => process.exit(0), 100);
+        });
+        process.once('SIGTERM', () => {
+            cleanup();
+            // Allow graceful exit after cleanup
+            setTimeout(() => process.exit(0), 100);
+        });
+        process.once('uncaughtException', (error) => {
             logger.error('Uncaught exception, releasing MCP lock:', error);
             cleanup();
-            process.exit(1);
+            setTimeout(() => process.exit(1), 100);
         });
-        process.on('unhandledRejection', (reason) => {
+        process.once('unhandledRejection', (reason) => {
             logger.error('Unhandled rejection, releasing MCP lock:', reason);
             cleanup();
-            process.exit(1);
+            setTimeout(() => process.exit(1), 100);
         });
     }
     /**

package/dist/utils/ml-data-fetcher.d.ts

@@ -0,0 +1,66 @@
+/**
+ * Smart ML Data Fetcher
+ * Handles intelligent data fetching for ML training with batching and field discovery
+ * Prevents token limit errors and optimizes data retrieval
+ */
+export interface MLDataFetchOptions {
+    table: string;
+    query?: string;
+    totalSamples?: number;
+    batchSize?: number;
+    fields?: string[];
+    discoverFields?: boolean;
+    includeContent?: boolean;
+}
+export interface FieldDiscoveryResult {
+    allFields: string[];
+    recommendedFields: string[];
+    sampleData: any[];
+}
+export interface BatchFetchResult {
+    data: any[];
+    totalFetched: number;
+    batchesProcessed: number;
+    fields: string[];
+}
+export declare class MLDataFetcher {
+    private operationsMCP;
+    constructor(operationsMCP: any);
+    /**
+     * Smart fetch with automatic field discovery and batching
+     */
+    smartFetch(options: MLDataFetchOptions): Promise<BatchFetchResult>;
+    /**
+     * Get total count of records matching query
+     */
+    private getRecordCount;
+    /**
+     * Discover available fields by sampling a few records
+     */
+    private discoverFields;
+    /**
+     * Fetch a single batch of data
+     */
+    private fetchBatch;
+    /**
+     * Extract data from MCP tool result
+     */
+    private extractDataFromResult;
+    /**
+     * Calculate optimal batch size based on data characteristics
+     */
+    private calculateOptimalBatchSize;
+    /**
+     * Select appropriate fields for ML training
+     */
+    private selectMLFields;
+    /**
+     * Get default fields for a table type
+     */
+    private getDefaultFields;
+    /**
+     * Get key fields that should always be included
+     */
+    private getKeyFields;
+}
+//# sourceMappingURL=ml-data-fetcher.d.ts.map

package/dist/utils/ml-data-fetcher.js

@@ -0,0 +1,288 @@
+"use strict";
+/**
+ * Smart ML Data Fetcher
+ * Handles intelligent data fetching for ML training with batching and field discovery
+ * Prevents token limit errors and optimizes data retrieval
+ */
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.MLDataFetcher = void 0;
+const logger_js_1 = require("./logger.js");
+const logger = new logger_js_1.Logger('MLDataFetcher');
+class MLDataFetcher {
+    constructor(operationsMCP) {
+        this.operationsMCP = operationsMCP;
+    }
+    /**
+     * Smart fetch with automatic field discovery and batching
+     */
+    async smartFetch(options) {
+        const { table, query = '', totalSamples = 1000, batchSize = 100, fields, discoverFields = true, includeContent = true } = options;
+        logger.info(`🧠 Smart ML data fetch for ${table} - Target: ${totalSamples} samples`);
+        // Step 1: Get total count
+        const countResult = await this.getRecordCount(table, query);
+        logger.info(`📊 Total available records: ${countResult}`);
+        // Step 2: Discover fields if needed
+        let fieldsToFetch = fields;
+        if (!fieldsToFetch && discoverFields) {
+            const discovery = await this.discoverFields(table, query);
+            fieldsToFetch = discovery.recommendedFields;
+            logger.info(`🔍 Discovered ${discovery.allFields.length} fields, using ${fieldsToFetch.length} for ML`);
+        }
+        // Step 3: Calculate optimal batching strategy
+        const actualTotal = Math.min(totalSamples, countResult);
+        const optimalBatchSize = this.calculateOptimalBatchSize(actualTotal, batchSize, fieldsToFetch?.length || 10);
+        const numBatches = Math.ceil(actualTotal / optimalBatchSize);
+        logger.info(`📦 Fetching ${actualTotal} records in ${numBatches} batches of ${optimalBatchSize}`);
+        // Step 4: Fetch data in batches
+        const allData = [];
+        for (let batch = 0; batch < numBatches; batch++) {
+            const offset = batch * optimalBatchSize;
+            const limit = Math.min(optimalBatchSize, actualTotal - offset);
+            logger.info(`  Batch ${batch + 1}/${numBatches}: Fetching ${limit} records (offset: ${offset})`);
+            try {
+                const batchData = await this.fetchBatch(table, query, limit, offset, fieldsToFetch, includeContent);
+                allData.push(...batchData);
+                // Small delay between batches to avoid overwhelming the API
+                if (batch < numBatches - 1) {
+                    await new Promise(resolve => setTimeout(resolve, 200)); // Increased from 100ms to 200ms
+                }
+            }
+            catch (error) {
+                if (error.message?.includes('exceeds maximum allowed tokens')) {
+                    // Reduce batch size and retry
+                    logger.warn(`⚠️ Token limit hit, reducing batch size and retrying...`);
+                    const smallerBatchSize = Math.floor(optimalBatchSize / 2);
+                    return this.smartFetch({
+                        ...options,
+                        batchSize: smallerBatchSize
+                    });
+                }
+                throw error;
+            }
+        }
+        return {
+            data: allData,
+            totalFetched: allData.length,
+            batchesProcessed: numBatches,
+            fields: fieldsToFetch || []
+        };
+    }
+    /**
+     * Get total count of records matching query
+     */
+    async getRecordCount(table, query) {
+        try {
+            const result = await this.operationsMCP.handleTool('snow_query_table', {
+                table,
+                query,
+                limit: 1,
+                include_content: false // Count only, no data
+            });
+            // Extract count from result
+            if (result?.content?.[0]?.text) {
+                const text = result.content[0].text;
+                const match = text.match(/Found (\d+) .* records/);
+                if (match) {
+                    return parseInt(match[1], 10);
+                }
+            }
+            return 0;
+        }
+        catch (error) {
+            logger.error('Failed to get record count:', error);
+            return 0;
+        }
+    }
+    /**
+     * Discover available fields by sampling a few records
+     */
+    async discoverFields(table, query) {
+        logger.info('🔍 Discovering fields from sample records...');
+        try {
+            // Fetch just 3 records with all fields to discover schema
+            const result = await this.operationsMCP.handleTool('snow_query_table', {
+                table,
+                query,
+                limit: 3,
+                include_content: true
+                // No fields specified = get all fields
+            });
+            const sampleData = this.extractDataFromResult(result);
+            if (sampleData.length === 0) {
+                logger.warn('No sample data available for field discovery');
+                return {
+                    allFields: [],
+                    recommendedFields: this.getDefaultFields(table),
+                    sampleData: []
+                };
+            }
+            // Extract all field names from sample
+            const allFields = Object.keys(sampleData[0] || {});
+            // Recommend fields for ML (exclude system fields and large text fields)
+            const recommendedFields = this.selectMLFields(allFields, sampleData, table);
+            return {
+                allFields,
+                recommendedFields,
+                sampleData
+            };
+        }
+        catch (error) {
+            logger.error('Field discovery failed:', error);
+            return {
+                allFields: [],
+                recommendedFields: this.getDefaultFields(table),
+                sampleData: []
+            };
+        }
+    }
+    /**
+     * Fetch a single batch of data
+     */
+    async fetchBatch(table, query, limit, offset, fields, includeContent = true) {
+        try {
+            // Build query with offset
+            const offsetQuery = query ? `${query}^ORDERBY${offset}` : `ORDERBY${offset}`;
+            const result = await this.operationsMCP.handleTool('snow_query_table', {
+                table,
+                query: offsetQuery,
+                limit,
+                fields,
+                include_content: includeContent
+            });
+            return this.extractDataFromResult(result);
+        }
+        catch (error) {
+            logger.error(`Failed to fetch batch (offset: ${offset}, limit: ${limit}):`, error);
+            return [];
+        }
+    }
+    /**
+     * Extract data from MCP tool result
+     */
+    extractDataFromResult(result) {
+        if (!result?.content?.[0]?.text) {
+            return [];
+        }
+        try {
+            const text = result.content[0].text;
+            // Try to parse as JSON first
+            if (text.includes('[') && text.includes(']')) {
+                const jsonMatch = text.match(/\[[\s\S]*\]/);
+                if (jsonMatch) {
+                    return JSON.parse(jsonMatch[0]);
+                }
+            }
+            // Try to extract from formatted output
+            const lines = text.split('\n');
+            const data = [];
+            let currentRecord = null;
+            for (const line of lines) {
+                if (line.includes('number:') || line.includes('sys_id:')) {
+                    if (currentRecord) {
+                        data.push(currentRecord);
+                    }
+                    currentRecord = {};
+                }
+                if (currentRecord && line.includes(':')) {
+                    const [key, ...valueParts] = line.split(':');
+                    const cleanKey = key.trim().replace(/^[-\s]+/, '');
+                    const value = valueParts.join(':').trim();
+                    if (cleanKey && value) {
+                        currentRecord[cleanKey] = value;
+                    }
+                }
+            }
+            if (currentRecord && Object.keys(currentRecord).length > 0) {
+                data.push(currentRecord);
+            }
+            return data;
+        }
+        catch (error) {
+            logger.error('Failed to extract data from result:', error);
+            return [];
+        }
+    }
+    /**
+     * Calculate optimal batch size based on data characteristics
+     */
+    calculateOptimalBatchSize(totalRecords, requestedBatchSize, numFields) {
+        // Estimate tokens per record (rough approximation)
+        const avgTokensPerField = 10; // Conservative estimate
+        const tokensPerRecord = numFields * avgTokensPerField;
+        const maxTokensPerBatch = 20000; // Leave buffer below 25000 limit
+        // Calculate max records per batch based on token limit
+        const maxRecordsPerBatch = Math.floor(maxTokensPerBatch / tokensPerRecord);
+        // Use the smaller of requested batch size and calculated max
+        const optimalSize = Math.min(requestedBatchSize, maxRecordsPerBatch);
+        // Ensure at least 10 records per batch but not more than total
+        return Math.max(10, Math.min(optimalSize, totalRecords));
+    }
+    /**
+     * Select appropriate fields for ML training
+     */
+    selectMLFields(allFields, sampleData, table) {
+        const excluded = new Set([
+            'sys_id', 'sys_created_on', 'sys_created_by', 'sys_updated_on', 'sys_updated_by',
+            'sys_mod_count', 'sys_tags', 'sys_package', 'sys_policy', 'sys_scope',
+            'sys_domain', 'sys_domain_path', 'sys_class_name'
+        ]);
+        const mlFields = allFields.filter(field => {
+            // Exclude system fields
+            if (excluded.has(field))
+                return false;
+            // Check if field has useful data in samples
+            const hasData = sampleData.some(record => {
+                const value = record[field];
+                return value && value !== 'null' && value !== '';
+            });
+            return hasData;
+        });
+        // Always include key fields for the table type
+        const keyFields = this.getKeyFields(table);
+        const combinedFields = [...new Set([...keyFields, ...mlFields])];
+        // Limit to 20 most relevant fields to avoid token issues
+        return combinedFields.slice(0, 20);
+    }
+    /**
+     * Get default fields for a table type
+     */
+    getDefaultFields(table) {
+        const fieldMap = {
+            incident: [
+                'number', 'short_description', 'description', 'category', 'subcategory',
+                'priority', 'urgency', 'impact', 'state', 'assignment_group',
+                'assigned_to', 'caller_id', 'opened_at', 'resolved_at'
+            ],
+            change_request: [
+                'number', 'short_description', 'description', 'type', 'category',
+                'priority', 'risk', 'impact', 'state', 'assignment_group',
+                'assigned_to', 'requested_by', 'start_date', 'end_date'
+            ],
+            problem: [
+                'number', 'short_description', 'description', 'category', 'subcategory',
+                'priority', 'urgency', 'impact', 'state', 'assignment_group',
+                'assigned_to', 'opened_at', 'known_error'
+            ],
+            sc_request: [
+                'number', 'short_description', 'description', 'request_state', 'approval',
+                'requested_for', 'requested_by', 'assignment_group', 'assigned_to',
+                'opened_at', 'closed_at'
+            ]
+        };
+        return fieldMap[table] || ['number', 'short_description', 'state', 'priority'];
+    }
+    /**
+     * Get key fields that should always be included
+     */
+    getKeyFields(table) {
+        const keyFieldMap = {
+            incident: ['number', 'short_description', 'category', 'priority', 'state'],
+            change_request: ['number', 'short_description', 'type', 'risk', 'state'],
+            problem: ['number', 'short_description', 'category', 'priority', 'state'],
+            sc_request: ['number', 'short_description', 'request_state', 'approval']
+        };
+        return keyFieldMap[table] || ['number', 'short_description', 'state'];
+    }
+}
+exports.MLDataFetcher = MLDataFetcher;
+//# sourceMappingURL=ml-data-fetcher.js.map

package/dist/utils/servicenow-client.js

@@ -13,10 +13,10 @@ const axios_1 = __importDefault(require("axios"));
 const https_1 = __importDefault(require("https"));
 const snow_oauth_1 = require("./snow-oauth");
 const action_type_cache_1 = require("./action-type-cache");
-const snow_flow_config_js_1 = require("../config/snow-flow-config.js");
 const widget_template_generator_js_1 = require("./widget-template-generator.js");
 const logger_1 = require("./logger");
 const unified_auth_store_js_1 = require("./unified-auth-store.js");
+const timeout_manager_js_1 = require("./timeout-manager.js");
 class ServiceNowClient {
     constructor() {
         this.credentials = null;
@@ -68,8 +68,10 @@ class ServiceNowClient {
             maxVersion: 'TLSv1.3',
             minVersion: 'TLSv1.2'
         });
+        // Use intelligent timeout based on operation type (default to TABLE_QUERY)
+        const defaultTimeout = (0, timeout_manager_js_1.getTimeoutConfig)(timeout_manager_js_1.OperationType.TABLE_QUERY).baseTimeout;
         this.client = axios_1.default.create({
-            timeout: snow_flow_config_js_1.snowFlowConfig.servicenow.timeout,
+            timeout: parseInt(process.env.SNOW_API_TIMEOUT || String(defaultTimeout)),
             headers: {
                 'Content-Type': 'application/json',
                 'Accept': 'application/json'
@@ -979,21 +981,33 @@ class ServiceNowClient {
     async searchRecords(table, query, limit = 10) {
         try {
             await this.ensureAuthenticated();
-            const response = await this.client.get(`${this.getBaseUrl()}/api/now/table/${table}`, {
-                params: {
-                    sysparm_query: query,
-                    sysparm_limit: limit
-                }
+            // Detect operation type for intelligent timeout
+            const operationType = (0, timeout_manager_js_1.detectOperationType)({
+                action: 'query',
+                table,
+                limit
             });
+            // Use retry wrapper with intelligent timeout
+            const result = await (0, timeout_manager_js_1.withRetry)(async () => {
+                const response = await this.client.get(`${this.getBaseUrl()}/api/now/table/${table}`, {
+                    params: {
+                        sysparm_query: query,
+                        sysparm_limit: limit
+                    },
+                    // Override timeout for this specific request
+                    timeout: (0, timeout_manager_js_1.getTimeoutConfig)(operationType).baseTimeout
+                });
+                return response;
+            }, operationType, `Search ${table} (${limit} records)`);
             return {
                 success: true,
                 data: {
-                    result: response.data.result || []
+                    result: result.data.result || []
                 }
             };
         }
         catch (error) {
-            console.error(`Failed to search records in ${table}:`, error);
+            this.logger.error(`Failed to search records in ${table}:`, error);
             return {
                 success: false,
                 error: error instanceof Error ? error.message : String(error)
@@ -1006,22 +1020,34 @@ class ServiceNowClient {
     async searchRecordsWithOffset(table, query, limit = 10, offset = 0) {
         try {
             await this.ensureAuthenticated();
-            const response = await this.client.get(`${this.getBaseUrl()}/api/now/table/${table}`, {
-                params: {
-                    sysparm_query: query,
-                    sysparm_limit: limit,
-                    sysparm_offset: offset
-                }
+            // Detect operation type for intelligent timeout
+            const operationType = (0, timeout_manager_js_1.detectOperationType)({
+                action: 'query',
+                table,
+                limit
             });
+            // Use retry wrapper with intelligent timeout
+            const result = await (0, timeout_manager_js_1.withRetry)(async () => {
+                const response = await this.client.get(`${this.getBaseUrl()}/api/now/table/${table}`, {
+                    params: {
+                        sysparm_query: query,
+                        sysparm_limit: limit,
+                        sysparm_offset: offset
+                    },
+                    // Override timeout for this specific request
+                    timeout: (0, timeout_manager_js_1.getTimeoutConfig)(operationType).baseTimeout
+                });
+                return response;
+            }, operationType, `Search ${table} with offset ${offset}`);
             return {
                 success: true,
                 data: {
-                    result: response.data.result || []
+                    result: result.data.result || []
                 }
             };
         }
         catch (error) {
-            console.error(`Failed to search records in ${table} with offset ${offset}:`, error);
+            this.logger.error(`Failed to search records in ${table} with offset ${offset}:`, error);
             return {
                 success: false,
                 error: error instanceof Error ? error.message : String(error)