snow-flow 1.3.22 → 1.3.24

package/dist/mcp/base-mcp-server.js

@@ -26,6 +26,8 @@ class BaseMCPServer {
         this.tools = new Map();
         // Performance tracking
         this.toolMetrics = new Map();
+        // 🔴 SNOW-003 FIX: Circuit breaker implementation
+        this.circuitBreakers = new Map();
         /**
          * Tool handlers map
          */
@@ -159,48 +161,214 @@ class BaseMCPServer {
         }
     }
     /**
-     * Execute tool with retry logic
+     * 🔴 SNOW-003 FIX: Enhanced retry logic with intelligent backoff and circuit breaker
+     * Addresses the 19% failure rate with better retry strategies and failure prevention
      */
     async executeWithRetry(toolName, args, attempt = 1) {
-        const maxRetries = 3;
-        const backoffMs = 1000 * Math.pow(2, attempt - 1); // Exponential backoff
+        // 🔴 CRITICAL: Increased retries from 3 to 6 for better resilience
+        const maxRetries = 6;
+        // 🔴 CRITICAL: Intelligent backoff based on error type
+        const backoffMs = this.calculateBackoff(attempt, toolName);
+        // 🔴 CRITICAL: Dynamic timeout based on tool complexity
+        const timeout = this.calculateTimeout(toolName);
         try {
             // Get tool handler
             const handler = this.getToolHandler(toolName);
             if (!handler) {
                 throw new types_js_1.McpError(types_js_1.ErrorCode.MethodNotFound, `Unknown tool: ${toolName}`);
             }
-            // Execute with timeout
-            const timeout = 30000; // 30 seconds
+            // 🔴 CRITICAL: Memory usage check before execution
+            if (attempt === 1) {
+                await this.checkMemoryUsage();
+            }
+            // Execute with dynamic timeout
             const result = await Promise.race([
                 handler(args),
-                new Promise((_, reject) => setTimeout(() => reject(new Error('Tool execution timeout')), timeout)),
+                new Promise((_, reject) => setTimeout(() => reject(new Error(`Tool execution timeout after ${timeout}ms`)), timeout)),
             ]);
+            // 🔴 SUCCESS: Reset circuit breaker on success
+            this.resetCircuitBreaker(toolName);
             return result;
         }
         catch (error) {
-            this.logger.error(`Tool ${toolName} execution failed (attempt ${attempt}):`, error);
-            // Check if retryable
-            if (attempt < maxRetries && this.isRetryableError(error)) {
-                this.logger.info(`Retrying ${toolName} after ${backoffMs}ms...`);
+            this.logger.error(`🔴 Tool ${toolName} execution failed (attempt ${attempt}/${maxRetries}):`, error);
+            // 🔴 CRITICAL: Update circuit breaker
+            this.updateCircuitBreaker(toolName, error);
+            // Check if retryable and within limits
+            if (attempt < maxRetries && this.isRetryableError(error) && !this.isCircuitBreakerOpen(toolName)) {
+                this.logger.info(`🔄 Retrying ${toolName} after ${backoffMs}ms (attempt ${attempt + 1}/${maxRetries})...`);
                 await new Promise(resolve => setTimeout(resolve, backoffMs));
                 return this.executeWithRetry(toolName, args, attempt + 1);
             }
-            // Final failure
-            throw new types_js_1.McpError(types_js_1.ErrorCode.InternalError, `Tool execution failed: ${error instanceof Error ? error.message : 'Unknown error'}`);
+            // 🔴 FINAL FAILURE: Enhanced error reporting
+            const errorMessage = this.createEnhancedErrorMessage(toolName, error, attempt, maxRetries);
+            throw new types_js_1.McpError(types_js_1.ErrorCode.InternalError, errorMessage);
         }
     }
     /**
-     * Check if error is retryable
+     * 🔴 SNOW-003 FIX: Calculate intelligent backoff based on error type and attempt
+     */
+    calculateBackoff(attempt, toolName) {
+        // Base exponential backoff: 1s, 2s, 4s, 8s, 16s, 32s
+        const baseBackoff = 1000 * Math.pow(2, attempt - 1);
+        // Add jitter to prevent thundering herd (±25%)
+        const jitter = baseBackoff * 0.25 * (Math.random() - 0.5);
+        // Cap maximum backoff at 30 seconds
+        const maxBackoff = 30000;
+        return Math.min(baseBackoff + jitter, maxBackoff);
+    }
+    /**
+     * 🔴 SNOW-003 FIX: Calculate dynamic timeout based on tool complexity
+     */
+    calculateTimeout(toolName) {
+        // Tool-specific timeouts based on complexity
+        const timeoutMap = {
+            'snow_create_flow': 90000, // Flow creation: 90s
+            'snow_deploy': 120000, // Deployment: 2 minutes
+            'snow_comprehensive_search': 45000, // Search: 45s
+            'snow_find_artifact': 30000, // Find: 30s
+            'snow_validate_live_connection': 15000, // Validation: 15s
+        };
+        // Default timeout for unknown tools
+        return timeoutMap[toolName] || 60000; // 60s default (increased from 30s)
+    }
+    updateCircuitBreaker(toolName, error) {
+        const breaker = this.circuitBreakers.get(toolName) || { failures: 0, lastFailure: 0, isOpen: false };
+        breaker.failures++;
+        breaker.lastFailure = Date.now();
+        // Open circuit breaker after 5 failures within 5 minutes
+        if (breaker.failures >= 5 && (Date.now() - breaker.lastFailure) < 300000) {
+            breaker.isOpen = true;
+            this.logger.warn(`🚨 Circuit breaker opened for ${toolName} due to repeated failures`);
+        }
+        this.circuitBreakers.set(toolName, breaker);
+    }
+    isCircuitBreakerOpen(toolName) {
+        const breaker = this.circuitBreakers.get(toolName);
+        if (!breaker || !breaker.isOpen)
+            return false;
+        // Auto-reset circuit breaker after 10 minutes
+        if (Date.now() - breaker.lastFailure > 600000) {
+            breaker.isOpen = false;
+            breaker.failures = 0;
+            this.circuitBreakers.set(toolName, breaker);
+            this.logger.info(`✅ Circuit breaker reset for ${toolName}`);
+            return false;
+        }
+        return true;
+    }
+    resetCircuitBreaker(toolName) {
+        const breaker = this.circuitBreakers.get(toolName);
+        if (breaker) {
+            breaker.failures = 0;
+            breaker.isOpen = false;
+            this.circuitBreakers.set(toolName, breaker);
+        }
+    }
+    /**
+     * 🔴 SNOW-003 FIX: Memory usage monitoring to prevent memory exhaustion failures
+     */
+    async checkMemoryUsage() {
+        try {
+            const memUsage = process.memoryUsage();
+            const heapUsedMB = Math.round(memUsage.heapUsed / 1024 / 1024);
+            const heapTotalMB = Math.round(memUsage.heapTotal / 1024 / 1024);
+            // Log memory usage if high (>200MB)
+            if (heapUsedMB > 200) {
+                this.logger.warn(`⚠️ High memory usage: ${heapUsedMB}MB heap used, ${heapTotalMB}MB total`);
+            }
+            // Trigger garbage collection if memory usage is very high (>500MB)
+            if (heapUsedMB > 500 && global.gc) {
+                this.logger.info('🧹 Triggering garbage collection due to high memory usage');
+                global.gc();
+            }
+            // Fail fast if memory usage is critical (>800MB)
+            if (heapUsedMB > 800) {
+                throw new Error(`Critical memory usage: ${heapUsedMB}MB. Operation aborted to prevent system instability.`);
+            }
+        }
+        catch (error) {
+            this.logger.warn('Could not check memory usage:', error);
+        }
+    }
+    /**
+     * 🔴 SNOW-003 FIX: Enhanced error message with troubleshooting guidance
+     */
+    createEnhancedErrorMessage(toolName, error, attempts, maxRetries) {
+        const baseMessage = `Tool '${toolName}' failed after ${attempts}/${maxRetries} attempts`;
+        const errorDetail = error instanceof Error ? error.message : String(error);
+        let troubleshooting = '';
+        // Add specific troubleshooting based on error type
+        if (error.response?.status === 401) {
+            troubleshooting = '\n💡 Authentication issue: Run "snow-flow auth login" to re-authenticate';
+        }
+        else if (error.response?.status === 403) {
+            troubleshooting = '\n💡 Permission issue: Check ServiceNow user permissions and OAuth scopes';
+        }
+        else if (error.response?.status >= 500) {
+            troubleshooting = '\n💡 ServiceNow server issue: Try again later or contact ServiceNow administrator';
+        }
+        else if (errorDetail.includes('timeout')) {
+            troubleshooting = '\n💡 Timeout issue: ServiceNow instance may be slow - try again later';
+        }
+        else if (errorDetail.includes('network') || errorDetail.includes('connection')) {
+            troubleshooting = '\n💡 Network issue: Check internet connection and ServiceNow instance availability';
+        }
+        return `${baseMessage}: ${errorDetail}${troubleshooting}`;
+    }
+    /**
+     * 🔴 SNOW-003 FIX: Enhanced error classification for ServiceNow specific errors
+     * Addresses the 19% failure rate by properly categorizing retryable errors
      */
     isRetryableError(error) {
         if (error instanceof Error) {
             const message = error.message.toLowerCase();
-            return (message.includes('timeout') ||
+            // 🔴 CRITICAL: ServiceNow specific retryable errors
+            const serviceNowRetryable = (message.includes('timeout') ||
                 message.includes('econnreset') ||
                 message.includes('socket hang up') ||
                 message.includes('enotfound') ||
-                message.includes('rate limit'));
+                message.includes('rate limit') ||
+                message.includes('service unavailable') ||
+                message.includes('bad gateway') ||
+                message.includes('gateway timeout') ||
+                message.includes('connection refused') ||
+                message.includes('network error') ||
+                message.includes('dns lookup failed') ||
+                message.includes('connect etimedout') ||
+                message.includes('index not available') ||
+                message.includes('search index updating') ||
+                message.includes('temporary failure') ||
+                message.includes('server is busy') ||
+                message.includes('database lock') ||
+                message.includes('deadlock detected'));
+            // 🔴 CRITICAL: HTTP status code based retry logic
+            if (error.response?.status) {
+                const status = error.response.status;
+                const httpRetryable = (status === 429 || // Rate limit
+                    status === 502 || // Bad Gateway 
+                    status === 503 || // Service Unavailable
+                    status === 504 || // Gateway Timeout
+                    status === 507 || // Insufficient Storage
+                    status === 520 || // CloudFlare unknown error
+                    status === 521 || // Web server is down
+                    status === 522 || // Connection timed out
+                    status === 523 || // Origin is unreachable
+                    status === 524 // A timeout occurred
+                );
+                // 401 is retryable only once (for token refresh)
+                const authRetryable = status === 401 && !error.config?._retry;
+                return httpRetryable || authRetryable;
+            }
+            return serviceNowRetryable;
+        }
+        // Handle specific error types
+        if (error.code) {
+            const retryableCodes = [
+                'ECONNRESET', 'ENOTFOUND', 'ECONNREFUSED', 'ETIMEDOUT',
+                'ESOCKETTIMEDOUT', 'EHOSTUNREACH', 'EPIPE', 'EAI_AGAIN'
+            ];
+            return retryableCodes.includes(error.code);
         }
         return false;
     }

package/dist/mcp/servicenow-deployment-mcp.js

@@ -789,6 +789,56 @@ class ServiceNowDeploymentMCP {
                     };
                     let troubleshootingSteps = '';
                     if (is403Error(directError) || is403Error(tableError)) {
+                        // CRITICAL FIX: Check if widget was actually created despite 403 error
+                        this.logger.info('403 error detected, verifying if widget was actually created...');
+                        const verificationResult = await this.verifyWidgetInServiceNow(args.name);
+                        if (verificationResult.exists) {
+                            // Widget was created successfully despite 403 error!
+                            this.logger.info('🎉 Widget verification SUCCESS: Widget exists despite 403 error', {
+                                widgetName: args.name,
+                                sys_id: verificationResult.sys_id,
+                                completenessScore: verificationResult.completenessScore
+                            });
+                            // Format successful response similar to normal deployment
+                            const credentials = await this.oauth.loadCredentials();
+                            const widgetUrl = credentials?.instance ?
+                                `https://${credentials.instance}/sp_config?id=widget_editor&sys_id=${verificationResult.sys_id}` :
+                                'ServiceNow instance URL not available';
+                            return {
+                                content: [
+                                    {
+                                        type: 'text',
+                                        text: `✅ Widget deployed successfully! (Despite 403 error)
+              
+🎯 Widget Details:
+- Name: ${args.name}
+- Title: ${verificationResult.title || args.title}
+- Sys ID: ${verificationResult.sys_id}
+- Deployment Method: ${deploymentMethod} (with error recovery)
+- Verification: ✅ Confirmed (${verificationResult.completenessScore}/100 complete)
+
+📦 Update Set:
+- Name: ${updateSetName}
+- ID: ${updateSetId || 'None'}
+- Status: ${updateSetId ? '✅ Tracked' : '⚠️ Not tracked'}
+
+🔗 Direct Links:
+- Widget Editor: ${widgetUrl}
+- Service Portal Designer: https://${credentials?.instance}/sp_config?id=designer
+
+🔧 Note: Widget was created successfully despite receiving a 403 error. This is a known issue with ServiceNow permissions that has been automatically resolved.
+
+⚡ **Ready for Testing**
+Your widget has been deployed and is ready for testing in Service Portal.`
+                                    }
+                                ]
+                            };
+                        }
+                        // Widget was NOT created, continue with error handling
+                        this.logger.warn('Widget verification failed: Widget does not exist after deployment attempts', {
+                            widgetName: args.name,
+                            verificationDetails: verificationResult.debugInfo
+                        });
                         // Run authentication diagnostics automatically on 403 errors
                         this.logger.info('403 error detected, running automatic authentication diagnostics...');
                         let diagnosticsResult = '';
@@ -2079,15 +2129,44 @@ ${sessionSummary.statusCounts.pending > 0 ? '- 📋 Complete pending deployments
             if (!data || typeof data !== 'object') {
                 throw new Error('Invalid diagnostics data received from ServiceNow');
             }
-            // Format test results
-            const testResults = Object.entries(tests).map(([name, result]) => `**${name}:** ${result?.status || 'Unknown'}
-  - ${result?.description || 'No description'}
-  ${result?.error ? `- Error: ${result.error}` : ''}
-  ${result?.http_status ? `- HTTP Status: ${result.http_status}` : ''}`).join('\n\n');
-            // Format recommendations
-            const recommendationText = args.include_recommendations !== false && recommendations.length > 0
-                ? `\n\n**🔧 Troubleshooting Recommendations:**\n${recommendations.map((rec) => `- ${rec}`).join('\n')}`
-                : '';
+            // Format test results with enhanced null safety
+            let testResults = 'No test results available';
+            if (tests && typeof tests === 'object') {
+                try {
+                    const entries = Object.entries(tests);
+                    if (entries.length > 0) {
+                        testResults = entries.map(([name, result]) => {
+                            // CRITICAL: Extra null checks for each property
+                            const status = result?.status || 'Unknown';
+                            const description = result?.description || 'No description';
+                            const error = result?.error && typeof result.error === 'string' ? `- Error: ${result.error}` : '';
+                            const httpStatus = result?.http_status && typeof result.http_status === 'number' ? `- HTTP Status: ${result.http_status}` : '';
+                            return `**${name}:** ${status}
+  - ${description}
+  ${error}
+  ${httpStatus}`;
+                        }).join('\n\n');
+                    }
+                }
+                catch (testFormatError) {
+                    this.logger.error('Error formatting test results:', testFormatError);
+                    testResults = '❌ Error formatting test results - check ServiceNow connection';
+                }
+            }
+            // Format recommendations with null safety
+            let recommendationText = '';
+            if (args.include_recommendations !== false && Array.isArray(recommendations) && recommendations.length > 0) {
+                try {
+                    const validRecommendations = recommendations.filter(rec => rec && typeof rec === 'string');
+                    if (validRecommendations.length > 0) {
+                        recommendationText = `\n\n**🔧 Troubleshooting Recommendations:**\n${validRecommendations.map((rec) => `- ${rec}`).join('\n')}`;
+                    }
+                }
+                catch (recFormatError) {
+                    this.logger.error('Error formatting recommendations:', recFormatError);
+                    recommendationText = '\n\n**🔧 Troubleshooting Recommendations:**\n- Unable to format recommendations due to error';
+                }
+            }
             // Generate URL fix recommendation if we detect the trailing slash issue
             const urlFixRecommendation = data.instance_url && typeof data.instance_url === 'string' && data.instance_url.includes('//')
                 ? '\n\n**🚨 CRITICAL URL ISSUE DETECTED:**\n- Your SNOW_INSTANCE in .env has a trailing slash\n- This causes malformed URLs like https://instance.com//api/\n- Remove the trailing slash from SNOW_INSTANCE=your-instance.com/'
@@ -5506,6 +5585,176 @@ Use individual deployment tools like \`snow_deploy_${args.type}\` with manual co
 
 **Error Details**: ${error.message || error}`;
     }
+    /**
+     * Verify widget exists in ServiceNow with comprehensive retry logic
+     * Addresses the critical false negative bug where widgets show 403 errors but are actually created
+     */
+    async verifyWidgetInServiceNow(widgetName) {
+        const maxRetries = 5;
+        const baseDelay = 2000; // Start with 2 seconds
+        for (let attempt = 1; attempt <= maxRetries; attempt++) {
+            try {
+                // Progressive delay - ServiceNow needs time to process
+                if (attempt > 1) {
+                    const delay = baseDelay * attempt; // 2s, 4s, 6s, 8s, 10s
+                    this.logger.info(`Waiting ${delay}ms before widget verification attempt ${attempt}/${maxRetries}`);
+                    await this.sleep(delay);
+                }
+                this.logger.info(`Widget verification attempt ${attempt}/${maxRetries}`, { widgetName });
+                // Multi-table verification approach (similar to flow verification)
+                const [mainWidgetCheck, widgetSearchCheck] = await Promise.allSettled([
+                    // Check 1: Direct sp_widget table search by name
+                    this.client.searchRecords('sp_widget', `name=${widgetName}`, 1),
+                    // Check 2: Broader search with ID field
+                    this.client.searchRecords('sp_widget', `name=${widgetName}^ORid=${widgetName}`, 5)
+                ]);
+                let mainWidget = null;
+                let searchResults = null;
+                let verificationDetails = {
+                    attempt,
+                    mainWidgetCheck: 'pending',
+                    widgetSearchCheck: 'pending',
+                    totalFound: 0
+                };
+                // Process main widget check
+                if (mainWidgetCheck.status === 'fulfilled' && mainWidgetCheck.value.success) {
+                    mainWidget = mainWidgetCheck.value.data?.[0];
+                    verificationDetails.mainWidgetCheck = mainWidget ? 'found' : 'not_found';
+                    verificationDetails.totalFound = mainWidgetCheck.value.data?.length || 0;
+                }
+                else {
+                    verificationDetails.mainWidgetCheck = `failed: ${mainWidgetCheck.status === 'rejected' ? mainWidgetCheck.reason : 'unknown error'}`;
+                }
+                // Process widget search check
+                if (widgetSearchCheck.status === 'fulfilled' && widgetSearchCheck.value.success) {
+                    searchResults = widgetSearchCheck.value.data || [];
+                    verificationDetails.widgetSearchCheck = `found_${searchResults.length}`;
+                    // Use search results if main check didn't find anything
+                    if (!mainWidget && searchResults.length > 0) {
+                        mainWidget = searchResults[0];
+                        verificationDetails.totalFound = searchResults.length;
+                    }
+                }
+                else {
+                    verificationDetails.widgetSearchCheck = `failed: ${widgetSearchCheck.status === 'rejected' ? widgetSearchCheck.reason : 'unknown error'}`;
+                }
+                // Calculate completeness score
+                let completenessScore = 0;
+                if (mainWidget) {
+                    completenessScore += mainWidget.sys_id ? 25 : 0;
+                    completenessScore += mainWidget.name ? 25 : 0;
+                    completenessScore += mainWidget.title ? 25 : 0;
+                    completenessScore += mainWidget.template ? 25 : 0;
+                }
+                if (mainWidget && completenessScore >= 75) {
+                    // Widget found and appears complete
+                    this.logger.info(`✅ Widget verification SUCCESS on attempt ${attempt}`, {
+                        widgetName,
+                        sys_id: mainWidget.sys_id,
+                        completenessScore,
+                        totalRetries: attempt
+                    });
+                    return {
+                        exists: true,
+                        sys_id: mainWidget.sys_id,
+                        name: mainWidget.name,
+                        title: mainWidget.title,
+                        completenessScore,
+                        attempt,
+                        verificationDetails,
+                        debugInfo: {
+                            foundVia: verificationDetails.mainWidgetCheck === 'found' ? 'main_check' : 'search_check',
+                            totalFound: verificationDetails.totalFound,
+                            retriesNeeded: attempt
+                        }
+                    };
+                }
+                else if (mainWidget && completenessScore < 75) {
+                    // Widget found but incomplete - might still be processing
+                    this.logger.warn(`⚠️ Widget found but incomplete on attempt ${attempt}`, {
+                        widgetName,
+                        sys_id: mainWidget.sys_id,
+                        completenessScore,
+                        remainingRetries: maxRetries - attempt
+                    });
+                    if (attempt === maxRetries) {
+                        // Last attempt - return what we have
+                        return {
+                            exists: true,
+                            sys_id: mainWidget.sys_id,
+                            name: mainWidget.name,
+                            title: mainWidget.title,
+                            completenessScore,
+                            attempt,
+                            verificationDetails,
+                            debugInfo: {
+                                warning: 'Widget exists but appears incomplete',
+                                foundVia: verificationDetails.mainWidgetCheck === 'found' ? 'main_check' : 'search_check',
+                                totalFound: verificationDetails.totalFound,
+                                retriesNeeded: attempt
+                            }
+                        };
+                    }
+                }
+                else {
+                    // Widget not found
+                    this.logger.warn(`❌ Widget not found on attempt ${attempt}`, {
+                        widgetName,
+                        verificationDetails,
+                        remainingRetries: maxRetries - attempt
+                    });
+                    if (attempt === maxRetries) {
+                        // Final attempt failed
+                        return {
+                            exists: false,
+                            attempt,
+                            verificationDetails,
+                            debugInfo: {
+                                finalAttempt: true,
+                                allChecks: {
+                                    mainWidgetCheck: verificationDetails.mainWidgetCheck,
+                                    widgetSearchCheck: verificationDetails.widgetSearchCheck
+                                },
+                                totalRetries: maxRetries
+                            }
+                        };
+                    }
+                }
+            }
+            catch (verificationError) {
+                this.logger.error(`Widget verification attempt ${attempt} failed`, {
+                    widgetName,
+                    error: verificationError instanceof Error ? verificationError.message : String(verificationError),
+                    remainingRetries: maxRetries - attempt
+                });
+                if (attempt === maxRetries) {
+                    // Final attempt - return failure with error details
+                    return {
+                        exists: false,
+                        attempt,
+                        error: verificationError instanceof Error ? verificationError.message : String(verificationError),
+                        debugInfo: {
+                            finalAttempt: true,
+                            verificationError: true,
+                            totalRetries: maxRetries
+                        }
+                    };
+                }
+            }
+        }
+        // Should not reach here, but safety fallback
+        return {
+            exists: false,
+            attempt: maxRetries,
+            debugInfo: { unexpectedFallback: true }
+        };
+    }
+    /**
+     * Sleep utility for retry delays
+     */
+    sleep(ms) {
+        return new Promise(resolve => setTimeout(resolve, ms));
+    }
     /**
      * Format successful deployment response
      */