snow-flow 8.3.2 → 8.4.0

package/scripts/bulk-optimize-tools.js

@@ -0,0 +1,486 @@
+#!/usr/bin/env node
+/**
+ * Bulk MCP Tool Optimization Script
+ *
+ * Optimizes ALL Snow-Flow MCP tools:
+ * - Compresses verbose descriptions → concise
+ * - Adds categorization metadata
+ * - Calculates token savings
+ * - Validates changes
+ */
+
+const fs = require('fs');
+const path = require('path');
+
+// ============================================================================
+// CONFIGURATION
+// ============================================================================
+
+const TOOL_DIRS = [
+  'dist/mcp/servicenow-mcp-unified/tools',
+  'dist/mcp/snow-flow/tools'
+];
+
+const DRY_RUN = process.argv.includes('--dry-run');
+const VERBOSE = process.argv.includes('--verbose');
+
+// Description optimization patterns
+const OPTIMIZATIONS = {
+  // Remove verbose phrases
+  removes: [
+    /Executes all \d+ steps automatically:/gi,
+    /with automatic/gi,
+    /comprehensive/gi,
+    /advanced/gi,
+    /flexible/gi,
+    /powerful/gi,
+    /complete/gi,
+    /full support for/gi,
+    /provides capability to/gi,
+    /allows you to/gi,
+    /enables you to/gi,
+    /enables/gi
+  ],
+
+  // Shorten common phrases
+  replaces: [
+    ['Create a new', 'Create'],
+    ['Create an', 'Create'],
+    ['Update an existing', 'Update'],
+    ['Update the', 'Update'],
+    ['Delete an existing', 'Delete'],
+    ['Delete the', 'Delete'],
+    ['Query for', 'Query'],
+    ['Search for', 'Search'],
+    ['Retrieve information about', 'Get'],
+    ['Retrieve', 'Get'],
+    ['with filtering and pagination', ': filter, paginate'],
+    ['with pagination and filtering', ': filter, paginate'],
+    ['with role-based access control', 'with RBAC'],
+    ['Now Experience Framework', 'UX'],
+    ['UI Builder', 'UIB'],
+    ['Service Portal', 'SP'],
+    ['Configuration Management Database', 'CMDB'],
+    ['Predictive Intelligence', 'PI'],
+    ['Performance Analytics', 'PA'],
+    ['and configuration', ''],
+    [' and ', ', ']
+  ]
+};
+
+// Property description shortcuts
+const PROPERTY_SHORTCUTS = {
+  'Table name to query': 'Table name',
+  'Table name to create record in': 'Table name',
+  'Table name to update': 'Table name',
+  'Encoded query string': 'Encoded query',
+  'Fields to return': 'Fields',
+  'Maximum number of records': 'Max records',
+  'Number of records to skip': 'Records to skip',
+  'Field to order by': 'Order by',
+  'Return display values': 'Display values',
+  'Workspace name': 'Name',
+  'Page name': 'Name',
+  'Component name': 'Name',
+  'Description of the': 'Description',
+  'System ID of the': 'System ID'
+};
+
+// Metadata taxonomy by directory
+const CATEGORY_TAXONOMY = {
+  'operations': {
+    category: 'core-operations',
+    subcategory: 'general',
+    complexity: 'beginner',
+    frequency: 'very-high'
+  },
+  'ui-builder': {
+    category: 'ui-frameworks',
+    subcategory: 'ui-builder',
+    complexity: 'intermediate',
+    frequency: 'high'
+  },
+  'workspace': {
+    category: 'ui-frameworks',
+    subcategory: 'workspace',
+    complexity: 'intermediate',
+    frequency: 'high'
+  },
+  'service-portal': {
+    category: 'ui-frameworks',
+    subcategory: 'service-portal',
+    complexity: 'intermediate',
+    frequency: 'medium'
+  },
+  'update-sets': {
+    category: 'development',
+    subcategory: 'update-sets',
+    complexity: 'beginner',
+    frequency: 'very-high'
+  },
+  'deployment': {
+    category: 'development',
+    subcategory: 'deployment',
+    complexity: 'intermediate',
+    frequency: 'high'
+  },
+  'development': {
+    category: 'development',
+    subcategory: 'platform',
+    complexity: 'intermediate',
+    frequency: 'high'
+  },
+  'automation': {
+    category: 'automation',
+    subcategory: 'script-execution',
+    complexity: 'intermediate',
+    frequency: 'high'
+  },
+  'integration': {
+    category: 'integration',
+    subcategory: 'rest-soap',
+    complexity: 'advanced',
+    frequency: 'medium'
+  },
+  'cmdb': {
+    category: 'cmdb',
+    subcategory: 'ci-management',
+    complexity: 'intermediate',
+    frequency: 'medium'
+  },
+  'knowledge': {
+    category: 'itsm',
+    subcategory: 'knowledge',
+    complexity: 'beginner',
+    frequency: 'medium'
+  },
+  'change': {
+    category: 'itsm',
+    subcategory: 'change',
+    complexity: 'intermediate',
+    frequency: 'high'
+  },
+  'catalog': {
+    category: 'itsm',
+    subcategory: 'catalog',
+    complexity: 'intermediate',
+    frequency: 'medium'
+  },
+  'predictive-intelligence': {
+    category: 'ml-analytics',
+    subcategory: 'predictive-intelligence',
+    complexity: 'advanced',
+    frequency: 'medium'
+  },
+  'performance-analytics': {
+    category: 'ml-analytics',
+    subcategory: 'performance-analytics',
+    complexity: 'intermediate',
+    frequency: 'medium'
+  },
+  'ai-ml': {
+    category: 'ml-analytics',
+    subcategory: 'machine-learning',
+    complexity: 'advanced',
+    frequency: 'low'
+  },
+  'flow-designer': {
+    category: 'automation',
+    subcategory: 'flow-designer',
+    complexity: 'intermediate',
+    frequency: 'medium'
+  },
+  'local-sync': {
+    category: 'development',
+    subcategory: 'local-sync',
+    complexity: 'intermediate',
+    frequency: 'high'
+  },
+  'system-properties': {
+    category: 'core-operations',
+    subcategory: 'properties',
+    complexity: 'beginner',
+    frequency: 'high'
+  }
+};
+
+// ============================================================================
+// UTILITY FUNCTIONS
+// ============================================================================
+
+function estimateTokens(text) {
+  // Rough estimation: 1 token ≈ 4 characters
+  return Math.ceil(text.length / 4);
+}
+
+function optimizeDescription(desc) {
+  let optimized = desc;
+
+  // Apply removals
+  OPTIMIZATIONS.removes.forEach(pattern => {
+    optimized = optimized.replace(pattern, '');
+  });
+
+  // Apply replacements
+  OPTIMIZATIONS.replaces.forEach(([from, to]) => {
+    optimized = optimized.replace(new RegExp(from, 'gi'), to);
+  });
+
+  // Clean up whitespace
+  optimized = optimized.replace(/\s+/g, ' ').trim();
+  optimized = optimized.replace(/,\s*,/g, ',');
+  optimized = optimized.replace(/:\s*,/g, ':');
+
+  return optimized;
+}
+
+function optimizePropertyDesc(desc) {
+  let optimized = desc;
+
+  // Try exact matches first
+  for (const [from, to] of Object.entries(PROPERTY_SHORTCUTS)) {
+    if (desc.includes(from)) {
+      optimized = optimized.replace(from, to);
+    }
+  }
+
+  // Remove common redundancies
+  optimized = optimized
+    .replace(/\(e\.g\.,.*?\)/g, '')
+    .replace(/\(default:.*?\)/g, '')
+    .replace(/\(optional\)/g, '')
+    .trim();
+
+  return optimized;
+}
+
+function inferMetadata(filePath) {
+  // Extract directory name
+  const parts = filePath.split(path.sep);
+  const toolsIndex = parts.findIndex(p => p === 'tools');
+  const dirName = parts[toolsIndex + 1] || 'general';
+
+  const metadata = CATEGORY_TAXONOMY[dirName] || {
+    category: 'advanced',
+    subcategory: 'specialized',
+    complexity: 'intermediate',
+    frequency: 'low'
+  };
+
+  return {
+    ...metadata,
+    use_cases: [dirName.replace(/-/g, '_')]
+  };
+}
+
+// ============================================================================
+// FILE PROCESSING
+// ============================================================================
+
+function processToolFile(filePath) {
+  try {
+    let content = fs.readFileSync(filePath, 'utf-8');
+
+    // Skip index files and non-tool files
+    if (filePath.includes('index.js') || !content.includes('toolDefinition')) {
+      return null;
+    }
+
+    // Extract toolDefinition
+    const toolDefStart = content.indexOf('exports.toolDefinition = {');
+    const toolDefEnd = content.indexOf('};', toolDefStart);
+
+    if (toolDefStart === -1 || toolDefEnd === -1) {
+      if (VERBOSE) console.warn(`⚠️  Could not find toolDefinition in ${filePath}`);
+      return null;
+    }
+
+    const toolDefSection = content.substring(toolDefStart, toolDefEnd + 2);
+
+    // Extract current description
+    const descMatch = toolDefSection.match(/description:\s*['"](.*?)['"]/);
+    if (!descMatch) {
+      if (VERBOSE) console.warn(`⚠️  No description found in ${filePath}`);
+      return null;
+    }
+
+    const originalDesc = descMatch[1];
+    const optimizedDesc = optimizeDescription(originalDesc);
+
+    // Calculate token savings
+    const originalTokens = estimateTokens(originalDesc);
+    const optimizedTokens = estimateTokens(optimizedDesc);
+    const savings = originalTokens - optimizedTokens;
+
+    // Get metadata
+    const metadata = inferMetadata(filePath);
+
+    // Build new toolDefinition with metadata
+    let newToolDef = toolDefSection.replace(
+      /description:\s*['"].*?['"]/,
+      `description: '${optimizedDesc}',\n    // Metadata for tool discovery (not sent to LLM)\n    category: '${metadata.category}',\n    subcategory: '${metadata.subcategory}',\n    use_cases: ${JSON.stringify(metadata.use_cases)},\n    complexity: '${metadata.complexity}',\n    frequency: '${metadata.frequency}'`
+    );
+
+    // Optimize property descriptions
+    newToolDef = newToolDef.replace(/description:\s*['"]([^'"]+)['"]/g, (match, desc) => {
+      if (match.includes('category:') || match.includes('subcategory:')) {
+        return match; // Don't optimize metadata
+      }
+      const optimized = optimizePropertyDesc(desc);
+      return `description: '${optimized}'`;
+    });
+
+    // Replace in content
+    const newContent = content.substring(0, toolDefStart) + newToolDef + content.substring(toolDefEnd + 2);
+
+    // Write back
+    if (!DRY_RUN) {
+      fs.writeFileSync(filePath, newContent, 'utf-8');
+    }
+
+    return {
+      file: path.relative(process.cwd(), filePath),
+      originalDesc,
+      optimizedDesc,
+      originalTokens,
+      optimizedTokens,
+      savings,
+      metadata
+    };
+
+  } catch (error) {
+    console.error(`❌ Error processing ${filePath}:`, error.message);
+    return null;
+  }
+}
+
+function findToolFiles(dir) {
+  const files = [];
+
+  function traverse(currentDir) {
+    try {
+      const entries = fs.readdirSync(currentDir, { withFileTypes: true });
+
+      for (const entry of entries) {
+        const fullPath = path.join(currentDir, entry.name);
+
+        if (entry.isDirectory()) {
+          traverse(fullPath);
+        } else if (entry.isFile() && entry.name.endsWith('.js') && !entry.name.endsWith('.map.js')) {
+          files.push(fullPath);
+        }
+      }
+    } catch (error) {
+      console.warn(`⚠️  Could not read directory ${currentDir}:`, error.message);
+    }
+  }
+
+  traverse(dir);
+  return files;
+}
+
+// ============================================================================
+// MAIN EXECUTION
+// ============================================================================
+
+async function main() {
+  console.log('🚀 Starting Bulk MCP Tool Optimization...\n');
+  console.log(`Mode: ${DRY_RUN ? 'DRY RUN (no files will be modified)' : 'LIVE (files will be modified)'}\n`);
+
+  const results = [];
+  let totalOriginalTokens = 0;
+  let totalOptimizedTokens = 0;
+  let totalFiles = 0;
+
+  for (const toolDir of TOOL_DIRS) {
+    const fullPath = path.join(process.cwd(), toolDir);
+
+    if (!fs.existsSync(fullPath)) {
+      console.warn(`⚠️  Directory not found: ${toolDir}`);
+      continue;
+    }
+
+    console.log(`📁 Processing: ${toolDir}`);
+    const files = findToolFiles(fullPath);
+    console.log(`   Found ${files.length} JavaScript files`);
+
+    let processed = 0;
+    for (const file of files) {
+      const result = processToolFile(file);
+      if (result) {
+        results.push(result);
+        totalOriginalTokens += result.originalTokens;
+        totalOptimizedTokens += result.optimizedTokens;
+        processed++;
+
+        if (VERBOSE) {
+          console.log(`   ✓ ${result.file} (${result.savings} tokens saved)`);
+        }
+      }
+    }
+
+    totalFiles += processed;
+    console.log(`   ✅ Processed ${processed} tool files\n`);
+  }
+
+  // Print summary
+  console.log('='.repeat(80));
+  console.log('📊 OPTIMIZATION SUMMARY');
+  console.log('='.repeat(80));
+  console.log(`Total tools optimized: ${totalFiles}`);
+  console.log(`Original description tokens: ${totalOriginalTokens.toLocaleString()}`);
+  console.log(`Optimized description tokens: ${totalOptimizedTokens.toLocaleString()}`);
+  console.log(`Total savings: ${(totalOriginalTokens - totalOptimizedTokens).toLocaleString()} tokens`);
+  console.log(`Reduction: ${((1 - totalOptimizedTokens / totalOriginalTokens) * 100).toFixed(1)}%`);
+  console.log('='.repeat(80));
+
+  // Category breakdown
+  const byCategory = {};
+  results.forEach(r => {
+    const cat = r.metadata.category;
+    if (!byCategory[cat]) {
+      byCategory[cat] = { count: 0, savings: 0 };
+    }
+    byCategory[cat].count++;
+    byCategory[cat].savings += r.savings;
+  });
+
+  console.log('\n📈 BY CATEGORY:');
+  Object.entries(byCategory)
+    .sort((a, b) => b[1].savings - a[1].savings)
+    .forEach(([cat, data]) => {
+      console.log(`   ${cat.padEnd(25)} ${data.count.toString().padStart(3)} tools, ${data.savings.toString().padStart(5)} tokens saved`);
+    });
+
+  // Save detailed report
+  const report = {
+    timestamp: new Date().toISOString(),
+    mode: DRY_RUN ? 'dry-run' : 'live',
+    summary: {
+      totalTools: totalFiles,
+      originalTokens: totalOriginalTokens,
+      optimizedTokens: totalOptimizedTokens,
+      savings: totalOriginalTokens - totalOptimizedTokens,
+      reductionPercent: parseFloat(((1 - totalOptimizedTokens / totalOriginalTokens) * 100).toFixed(1))
+    },
+    byCategory,
+    results: results.map(r => ({
+      file: r.file,
+      category: r.metadata.category,
+      originalDesc: r.originalDesc,
+      optimizedDesc: r.optimizedDesc,
+      savings: r.savings
+    }))
+  };
+
+  const reportPath = path.join(process.cwd(), 'docs/optimization-report.json');
+  fs.writeFileSync(reportPath, JSON.stringify(report, null, 2));
+  console.log(`\n✅ Detailed report saved to: ${reportPath}`);
+
+  if (DRY_RUN) {
+    console.log('\n⚠️  DRY RUN MODE: No files were modified. Run without --dry-run to apply changes.');
+  }
+}
+
+// Run
+main().catch(console.error);

package/scripts/cleanup-mcp-servers.js

@@ -0,0 +1,115 @@
+#!/usr/bin/env node
+
+/**
+ * MCP Server Cleanup Script
+ * Prevents duplicate MCP servers and memory exhaustion
+ */
+
+const { exec } = require('child_process');
+const { promisify } = require('util');
+const execAsync = promisify(exec);
+
+async function cleanupMCPServers() {
+  console.log('🧹 Cleaning up MCP servers...\n');
+  
+  try {
+    // 1. Check for running MCP processes
+    const { stdout: psOutput } = await execAsync('ps aux | grep -E "node.*mcp" | grep -v grep | wc -l');
+    const processCount = parseInt(psOutput.trim());
+    
+    if (processCount > 0) {
+      console.log(`⚠️  Found ${processCount} running MCP processes`);
+      console.log('🔪 Killing all MCP processes...');
+      
+      // Kill all MCP processes
+      await execAsync('pkill -f "node.*mcp"').catch(() => {
+        // Ignore errors if no processes found
+      });
+      
+      // Wait for processes to terminate
+      await new Promise(resolve => setTimeout(resolve, 2000));
+      
+      console.log('✅ All MCP processes terminated\n');
+    } else {
+      console.log('✅ No MCP processes running\n');
+    }
+    
+    // 2. Clean up any stale lock files
+    console.log('🔒 Cleaning up lock files...');
+    await execAsync('rm -f /tmp/mcp-*.lock').catch(() => {});
+    await execAsync('rm -f ~/.claude/mcp-*.lock').catch(() => {});
+    await execAsync('rm -f ~/.claude/mcp-servers.lock').catch(() => {});
+    
+    // 3. Check memory usage
+    const { stdout: memOutput } = await execAsync('ps aux | grep node | awk \'{sum+=$6} END {print sum/1024}\'');
+    const nodeMemoryMB = parseFloat(memOutput.trim());
+    
+    console.log(`📊 Node.js memory usage: ${nodeMemoryMB.toFixed(2)} MB\n`);
+    
+    if (nodeMemoryMB > 1000) {
+      console.log('⚠️  High memory usage detected!');
+      console.log('💡 Recommendation: Restart your terminal or run "killall node"\n');
+    }
+    
+    // 4. Create singleton lock mechanism
+    console.log('🔐 Setting up singleton lock mechanism...');
+    const lockScript = `
+const fs = require('fs');
+const path = require('path');
+
+const lockFile = path.join(process.env.HOME, '.claude', 'mcp-singleton.lock');
+const pid = process.pid;
+
+// Check if lock exists
+if (fs.existsSync(lockFile)) {
+  const existingPid = fs.readFileSync(lockFile, 'utf8');
+  try {
+    // Check if process is still running
+    process.kill(existingPid, 0);
+    console.error('MCP servers already running with PID:', existingPid);
+    process.exit(1);
+  } catch (e) {
+    // Process not running, remove stale lock
+    fs.unlinkSync(lockFile);
+  }
+}
+
+// Create lock
+fs.mkdirSync(path.dirname(lockFile), { recursive: true });
+fs.writeFileSync(lockFile, pid.toString());
+
+// Clean up on exit
+process.on('exit', () => {
+  try {
+    fs.unlinkSync(lockFile);
+  } catch (e) {}
+});
+`;
+    
+    // Save singleton script
+    const fs = require('fs');
+    const path = require('path');
+    const singletonPath = path.join(__dirname, '..', 'dist', 'mcp', 'singleton-check.js');
+    fs.writeFileSync(singletonPath, lockScript);
+    console.log('✅ Singleton mechanism created\n');
+    
+    // 5. Report status
+    console.log('📋 Cleanup Summary:');
+    console.log('─'.repeat(40));
+    console.log('✅ All MCP processes terminated');
+    console.log('✅ Lock files cleaned');
+    console.log('✅ Singleton mechanism installed');
+    console.log(`✅ Memory usage: ${nodeMemoryMB.toFixed(2)} MB`);
+    console.log('\n🎯 Next Steps:');
+    console.log('1. Run "npm run build" to rebuild');
+    console.log('2. Use "snow-flow mcp start" to start servers properly');
+    console.log('3. Monitor with "snow-flow mcp status"');
+    
+  } catch (error) {
+    console.error('❌ Cleanup failed:', error.message);
+    process.exit(1);
+  }
+}
+
+// Run cleanup
+cleanupMCPServers().catch(console.error);

package/scripts/generate-mcp-config.js

@@ -0,0 +1,45 @@
+#!/usr/bin/env node
+
+const fs = require('fs');
+const path = require('path');
+const dotenv = require('dotenv');
+
+// Load environment variables
+const envPath = path.join(__dirname, '..', '.env');
+if (fs.existsSync(envPath)) {
+  dotenv.config({ path: envPath });
+} else {
+  console.error('❌ .env file not found!');
+  process.exit(1);
+}
+
+// Read template
+const templatePath = path.join(__dirname, '..', '.mcp.json.template');
+const template = JSON.parse(fs.readFileSync(templatePath, 'utf-8'));
+
+// Process template - replace variables
+const config = JSON.parse(JSON.stringify(template)
+  .replace(/{{PROJECT_ROOT}}/g, path.resolve(__dirname, '..'))
+  .replace(/{{SNOW_INSTANCE}}/g, process.env.SNOW_INSTANCE || '')
+  .replace(/{{SNOW_CLIENT_ID}}/g, process.env.SNOW_CLIENT_ID || '')
+  .replace(/{{SNOW_CLIENT_SECRET}}/g, process.env.SNOW_CLIENT_SECRET || '')
+  .replace(/{{SNOW_DEPLOYMENT_TIMEOUT}}/g, process.env.SNOW_DEPLOYMENT_TIMEOUT || '300000')
+  .replace(/{{MCP_DEPLOYMENT_TIMEOUT}}/g, process.env.MCP_DEPLOYMENT_TIMEOUT || '360000')
+  .replace(/{{SNOW_FLOW_ENV}}/g, process.env.SNOW_FLOW_ENV || 'production')
+);
+
+// Write config
+const outputPath = path.join(__dirname, '..', '.mcp.json');
+fs.writeFileSync(outputPath, JSON.stringify(config, null, 2));
+
+console.log('✅ Generated .mcp.json with all ServiceNow MCP servers');
+console.log(`📁 Location: ${outputPath}`);
+console.log(`🔧 Servers configured: ${Object.keys(config.servers).length}`);
+console.log('\nServers:');
+Object.keys(config.servers).forEach(server => {
+  console.log(`  ✓ ${server}`);
+});
+
+console.log('\n💡 Next steps:');
+console.log('1. Restart Claude Code to load the new configuration');
+console.log('2. Run /doctor again to verify everything is working');