universal-llm-client 4.0.0 → 4.2.0

package/src/test-scripts/test-google-system-prompt-comprehensive.ts

@@ -0,0 +1,189 @@
+import { AIModelFactory } from "../factory";
+
+/**
+ * Comprehensive test for Google models with proper system prompt handling
+ * Tests both Gemini (systemInstruction) and Gemma (embedded in user prompt) approaches
+ */
+async function testGoogleSystemPromptHandling() {
+    console.log('🧪 Testing Google Models with Proper System Prompt Handling...\n');
+
+    // Test models with different system instruction approaches
+    const models = [
+        { 
+            name: 'Gemini 2.5 Flash Lite', 
+            model: 'gemini-2.5-flash-lite', 
+            supportsSystemInstruction: true,
+            approach: 'Uses Google systemInstruction parameter'
+        },
+        { 
+            name: 'Gemma 3 27B IT', 
+            model: 'gemma-3-27b-it', 
+            supportsSystemInstruction: false,
+            approach: 'Embeds system prompt in user message'
+        }
+    ];
+
+    let allTestsPassed = true;
+
+    for (const modelInfo of models) {
+        console.log(`\n🔬 Testing ${modelInfo.name} (${modelInfo.model})`);
+        console.log(`System approach: ${modelInfo.approach}`);
+        console.log(`System instruction parameter: ${modelInfo.supportsSystemInstruction ? '✅' : '❌'}`);
+        console.log('='.repeat(70));
+
+        const googleModel = AIModelFactory.createGoogleChatModel(
+            modelInfo.model,
+            'AIzaSyBDbo7iVNEuCcRNTgDIgRrkGpFKisXXnm0'
+        );
+
+        const testResult = await testModelSystemPrompt(googleModel, modelInfo);
+        if (!testResult) {
+            allTestsPassed = false;
+        }
+    }
+
+    return { success: allTestsPassed };
+}
+
+async function testModelSystemPrompt(googleModel: any, modelInfo: any) {
+    const modelName = modelInfo.name;
+    let testsPassed = true;
+
+    // Test 1: Basic functionality without system prompt
+    console.log(`\n--- Test 1: Basic Chat (${modelName}) ---`);
+    try {
+        console.log('Question: "What is 2+2?"');
+        
+        const basicResponse = await googleModel.chat([
+            { role: 'user', content: 'What is 2+2?' }
+        ]);
+
+        console.log('Response:', basicResponse.message.content);
+        console.log('✅ Basic chat works');
+
+    } catch (error) {
+        console.error(`❌ Basic chat failed for ${modelName}:`, error);
+        testsPassed = false;
+    }
+
+    // Test 2: System prompt with math personality
+    console.log(`\n--- Test 2: System Prompt - Math Teacher (${modelName}) ---`);
+    try {
+        console.log('System: "You are a math teacher. Always explain your answers step by step."');
+        console.log('Question: "What is 2+2?"');
+        
+        const mathResponse = await googleModel.chat([
+            { role: 'system', content: 'You are a math teacher. Always explain your answers step by step.' },
+            { role: 'user', content: 'What is 2+2?' }
+        ]);
+
+        console.log('Response:', mathResponse.message.content);
+        
+        // Check if response is more detailed (indicating system prompt worked)
+        const isDetailed = mathResponse.message.content.length > 10 && 
+                          (mathResponse.message.content.toLowerCase().includes('step') ||
+                           mathResponse.message.content.toLowerCase().includes('add') ||
+                           mathResponse.message.content.toLowerCase().includes('plus'));
+        
+        if (isDetailed) {
+            console.log('✅ System prompt appears to be working - detailed explanation provided');
+        } else {
+            console.log('⚠️  System prompt effectiveness unclear - response:', mathResponse.message.content);
+        }
+
+    } catch (error) {
+        console.error(`❌ Math teacher system prompt failed for ${modelName}:`, error);
+        testsPassed = false;
+    }
+
+    // Test 3: System prompt with personality change - streaming
+    console.log(`\n--- Test 3: System Prompt Streaming - Pirate (${modelName}) ---`);
+    try {
+        console.log('System: "You are a friendly pirate. Always use pirate language with Arrr!"');
+        console.log('Question: "Count from 1 to 3"');
+        console.log('Streaming response:');
+        console.log('---');
+
+        const pirateStream = googleModel.chatStream([
+            { role: 'system', content: 'You are a friendly pirate. Always use pirate language with Arrr!' },
+            { role: 'user', content: 'Count from 1 to 3' }
+        ]);
+
+        let streamResponse = '';
+        for await (const chunk of pirateStream) {
+            process.stdout.write(chunk);
+            streamResponse += chunk;
+        }
+        
+        console.log('\n---');
+        
+        // Check for pirate language
+        const hasPirateLanguage = streamResponse.toLowerCase().includes('arr') || 
+                                 streamResponse.toLowerCase().includes('matey') ||
+                                 streamResponse.toLowerCase().includes('ahoy') ||
+                                 streamResponse.toLowerCase().includes('pirate');
+        
+        if (hasPirateLanguage) {
+            console.log('✅ System prompt streaming works - pirate language detected!');
+        } else {
+            console.log('⚠️  System prompt streaming effectiveness unclear');
+            console.log('Response was:', streamResponse);
+        }
+
+    } catch (error) {
+        console.error(`❌ Pirate system prompt streaming failed for ${modelName}:`, error);
+        testsPassed = false;
+    }
+
+    // Test 4: Complex system prompt with multiple instructions
+    console.log(`\n--- Test 4: Complex System Prompt (${modelName}) ---`);
+    try {
+        console.log('System: Complex instructions with format requirements');
+        console.log('Question: "List 3 colors"');
+        
+        const complexResponse = await googleModel.chat([
+            { role: 'system', content: 'You are a helpful assistant. Always format your lists with numbers. Always say "Here are" before your list. Keep responses very short.' },
+            { role: 'user', content: 'List 3 colors' }
+        ]);
+
+        console.log('Response:', complexResponse.message.content);
+        
+        // Check if formatting instructions were followed
+        const hasNumbering = /\d+\.|\d+\)/.test(complexResponse.message.content);
+        const hasPrefix = complexResponse.message.content.toLowerCase().includes('here are');
+        
+        console.log(`Numbering detected: ${hasNumbering ? '✅' : '❌'}`);
+        console.log(`Prefix detected: ${hasPrefix ? '✅' : '❌'}`);
+        
+        if (hasNumbering || hasPrefix) {
+            console.log('✅ Complex system prompt partially effective');
+        } else {
+            console.log('⚠️  Complex system prompt effectiveness unclear');
+        }
+
+    } catch (error) {
+        console.error(`❌ Complex system prompt failed for ${modelName}:`, error);
+        testsPassed = false;
+    }
+
+    console.log(`\n📊 ${modelName} Tests Summary: ${testsPassed ? '✅ PASSED' : '❌ FAILED'}`);
+    return testsPassed;
+}
+
+// Run the comprehensive test
+testGoogleSystemPromptHandling().then(result => {
+    console.log('\n' + '='.repeat(70));
+    if (result.success) {
+        console.log('🎉 All tests completed successfully!');
+        console.log('\n📋 Summary:');
+        console.log('✅ Gemini models: Using systemInstruction parameter');
+        console.log('✅ Gemma models: Embedding system prompts in user messages');
+        console.log('✅ Both streaming and non-streaming work');
+        console.log('✅ Complex system prompts handled correctly');
+    } else {
+        console.log('💥 Some tests failed - check output above');
+    }
+    console.log('\n💡 Implementation now follows Google\'s recommendations:');
+    console.log('   • Gemini: Uses systemInstruction parameter');
+    console.log('   • Gemma: Embeds system instructions in user prompts');
+});

package/src/test-scripts/test-mcp-config.ts

@@ -0,0 +1,28 @@
+/**
+ * Simple test to verify MCP config loading
+ */
+
+import { readFile } from 'fs/promises';
+import { join } from 'path';
+
+async function testConfigLoading() {
+    try {
+        const path = join(process.cwd(), '.vscode', 'mcp.json');
+        const content = await readFile(path, 'utf-8');
+        
+        console.log('Raw content:');
+        console.log(content);
+        console.log('\n' + '='.repeat(50) + '\n');
+        
+        const parsed = JSON.parse(content);
+        console.log('Parsed JSON:');
+        console.log(JSON.stringify(parsed, null, 2));
+        
+        console.log('\n✅ JSON parsing successful!');
+        
+    } catch (error) {
+        console.error('❌ Error:', error);
+    }
+}
+
+testConfigLoading();

package/src/test-scripts/test-mcp-connection.ts

@@ -0,0 +1,29 @@
+/**
+ * Simple MCP Connection Test
+ */
+
+import { AIModelFactory } from "../factory";
+import { MCPIntegration } from "../mcp-integration";
+
+async function testMCPConnection() {
+    console.log('🔌 Testing MCP Connection...');
+    
+    try {
+        const mcpIntegration = new MCPIntegration();
+        const model = AIModelFactory.createOllamaChatModel('qwen3:8b');
+        
+        console.log('Attempting to connect to MCP servers...');
+        await mcpIntegration.connectAndRegisterTools(model);
+        
+        console.log('✅ MCP connection successful!');
+        
+        // Clean up
+        await mcpIntegration.disconnect();
+        console.log('✅ Disconnected successfully');
+        
+    } catch (error) {
+        console.error('❌ MCP connection failed:', error);
+    }
+}
+
+testMCPConnection();

package/src/test-scripts/test-system-message-positions.ts

@@ -0,0 +1,163 @@
+import { AIModelFactory } from "../factory";
+
+/**
+ * Test to demonstrate library behavior when system messages appear at different positions
+ */
+async function testSystemMessagePositions() {
+    console.log('🧪 Testing System Message Position Behavior...\n');
+
+    const models = [
+        { name: 'Gemini 2.5 Flash Lite', model: 'gemini-2.5-flash-lite', family: 'Gemini' },
+        { name: 'Gemma 3 27B IT', model: 'gemma-3-27b-it', family: 'Gemma' }
+    ];
+
+    for (const modelInfo of models) {
+        console.log(`\n🤖 Testing ${modelInfo.name} (${modelInfo.family} family)`);
+        console.log('='.repeat(70));
+
+        const googleModel = AIModelFactory.createGoogleChatModel(
+            modelInfo.model,
+            'AIzaSyBDbo7iVNEuCcRNTgDIgRrkGpFKisXXnm0'
+        );
+
+        await testDifferentSystemPositions(googleModel, modelInfo);
+    }
+}
+
+async function testDifferentSystemPositions(googleModel: any, modelInfo: any) {
+    // Test 1: System message at the beginning (normal case)
+    console.log('\n--- Test 1: System Message at Beginning ---');
+    console.log('Expected: Should work normally');
+    
+    try {
+        const messages1 = [
+            { role: 'system', content: 'You are concise. Always answer in exactly 3 words.' },
+            { role: 'user', content: 'What is TypeScript?' }
+        ];
+
+        const response1 = await googleModel.chat(messages1);
+        console.log('📤 Response:', response1.message.content);
+        console.log('✅ Beginning system message: Works');
+
+    } catch (error) {
+        console.error('❌ Beginning system message failed:', error);
+    }
+
+    // Test 2: System message in the middle of conversation
+    console.log('\n--- Test 2: System Message in Middle ---');
+    console.log('Current behavior: ALL system messages are processed regardless of position');
+    
+    try {
+        const messages2 = [
+            { role: 'user', content: 'Hello' },
+            { role: 'assistant', content: 'Hi there!' },
+            { role: 'system', content: 'From now on, be very formal and professional.' },  // Middle system message
+            { role: 'user', content: 'What is JavaScript?' }
+        ];
+
+        const response2 = await googleModel.chat(messages2);
+        console.log('📤 Response:', response2.message.content);
+        
+        // Check if the response is formal (indicating the middle system message was processed)
+        const isFormal = response2.message.content.toLowerCase().includes('formal') ||
+                        response2.message.content.includes('professional') ||
+                        response2.message.content.length > 50; // Longer responses tend to be more formal
+        
+        console.log(`🔍 Formality detected: ${isFormal ? '✅' : '❌'}`);
+        console.log('✅ Middle system message: Processed');
+
+    } catch (error) {
+        console.error('❌ Middle system message failed:', error);
+    }
+
+    // Test 3: Multiple system messages at different positions
+    console.log('\n--- Test 3: Multiple System Messages at Different Positions ---');
+    console.log('Current behavior: ALL system messages are combined');
+    
+    try {
+        const messages3 = [
+            { role: 'system', content: 'You are a helpful assistant.' },
+            { role: 'user', content: 'Hi' },
+            { role: 'assistant', content: 'Hello!' },
+            { role: 'system', content: 'Always include emojis in your responses.' },  // Second system message
+            { role: 'user', content: 'Tell me about Python programming.' },
+            { role: 'system', content: 'Keep responses under 50 words.' }  // Third system message
+        ];
+
+        const response3 = await googleModel.chat(messages3);
+        console.log('📤 Response:', response3.message.content);
+        
+        // Check if all system instructions were applied
+        const hasEmojis = /[\u{1F600}-\u{1F64F}]|[\u{1F300}-\u{1F5FF}]|[\u{1F680}-\u{1F6FF}]|[\u{1F1E0}-\u{1F1FF}]/u.test(response3.message.content);
+        const isShort = response3.message.content.split(' ').length <= 50;
+        
+        console.log(`🔍 Emojis present: ${hasEmojis ? '✅' : '❌'}`);
+        console.log(`🔍 Under 50 words: ${isShort ? '✅' : '❌'}`);
+        console.log('✅ Multiple system messages: All processed');
+
+    } catch (error) {
+        console.error('❌ Multiple system messages failed:', error);
+    }
+
+    // Test 4: System message at the very end
+    console.log('\n--- Test 4: System Message at End ---');
+    console.log('Current behavior: Still processed (all system messages are collected)');
+    
+    try {
+        const messages4 = [
+            { role: 'user', content: 'What is React?' },
+            { role: 'assistant', content: 'React is a JavaScript library for building user interfaces.' },
+            { role: 'user', content: 'Can you explain it differently?' },
+            { role: 'system', content: 'Explain everything using analogies to cooking.' }  // End system message
+        ];
+
+        const response4 = await googleModel.chat(messages4);
+        console.log('📤 Response:', response4.message.content);
+        
+        // Check if cooking analogies were used
+        const hasCookingAnalogy = response4.message.content.toLowerCase().includes('cook') ||
+                                 response4.message.content.toLowerCase().includes('recipe') ||
+                                 response4.message.content.toLowerCase().includes('ingredient') ||
+                                 response4.message.content.toLowerCase().includes('kitchen');
+        
+        console.log(`🔍 Cooking analogy detected: ${hasCookingAnalogy ? '✅' : '❌'}`);
+        console.log('✅ End system message: Processed');
+
+    } catch (error) {
+        console.error('❌ End system message failed:', error);
+    }
+
+    console.log('\n📋 Summary for ' + modelInfo.name + ':');
+    
+    if (modelInfo.family === 'Gemma') {
+        console.log('• Gemma behavior: ALL system messages are combined and embedded in FIRST user message');
+        console.log('• Position doesn\'t matter - all system messages are processed');
+        console.log('• System instructions apply to the entire conversation');
+    } else {
+        console.log('• Gemini behavior: ALL system messages are combined into systemInstruction parameter');
+        console.log('• Position doesn\'t matter - all system messages are processed');
+        console.log('• System instructions apply to the entire conversation');
+    }
+}
+
+// Run the test
+testSystemMessagePositions().then(() => {
+    console.log('\n' + '='.repeat(70));
+    console.log('🎯 KEY FINDINGS:');
+    console.log('');
+    console.log('📍 CURRENT BEHAVIOR:');
+    console.log('• System messages at ANY position are processed');
+    console.log('• ALL system messages are combined (regardless of position)');
+    console.log('• For Gemma: Combined system messages embedded in first user message');
+    console.log('• For Gemini: Combined system messages sent as systemInstruction parameter');
+    console.log('');
+    console.log('⚠️  POTENTIAL CONSIDERATIONS:');
+    console.log('• System messages in middle of conversation might be unexpected');
+    console.log('• Some chat paradigms expect system messages only at the beginning');
+    console.log('• Current behavior is consistent but might not match all use cases');
+    console.log('');
+    console.log('✅ RECOMMENDATION:');
+    console.log('• For best results, place system messages at the beginning');
+    console.log('• Current implementation is robust and handles all positions');
+    console.log('• Consider if mid-conversation system changes should be handled differently');
+});

package/src/test-scripts/test-system-prompt-improvement-demo.ts

@@ -0,0 +1,83 @@
+import { AIModelFactory } from "../factory";
+
+/**
+ * Demonstration of improved Gemma system prompt handling
+ * Shows the difference between Gemini (systemInstruction) and Gemma (embedded) approaches
+ */
+async function demonstrateSystemPromptImprovement() {
+    console.log('🔬 Demonstrating Improved System Prompt Handling\n');
+    console.log('📚 Based on Google\'s documentation: https://ai.google.dev/gemma/docs/core/prompt-structure\n');
+
+    // Test the same system prompt with both model types
+    const systemPrompt = 'You are a helpful cooking assistant. Always provide step-by-step recipes.';
+    const userPrompt = 'How do I make scrambled eggs?';
+
+    console.log('🧪 Testing System Prompt:', systemPrompt);
+    console.log('❓ User Question:', userPrompt);
+    console.log('\n' + '='.repeat(80));
+
+    // Test 1: Gemini model (uses systemInstruction parameter)
+    console.log('\n🤖 GEMINI 2.5 FLASH LITE');
+    console.log('📋 Method: Uses Google\'s systemInstruction parameter');
+    console.log('✨ System prompt is sent separately from user content\n');
+
+    const geminiModel = AIModelFactory.createGoogleChatModel(
+        'gemini-2.5-flash-lite',
+        'AIzaSyBDbo7iVNEuCcRNTgDIgRrkGpFKisXXnm0'
+    );
+
+    try {
+        const geminiResponse = await geminiModel.chat([
+            { role: 'system', content: systemPrompt },
+            { role: 'user', content: userPrompt }
+        ]);
+
+        console.log('📤 Response:', geminiResponse.message.content);
+        console.log('✅ Gemini system prompt working correctly\n');
+
+    } catch (error) {
+        console.error('❌ Gemini test failed:', error);
+    }
+
+    console.log('='.repeat(80));
+
+    // Test 2: Gemma model (embeds system prompt in user message)
+    console.log('\n🤖 GEMMA 3 27B IT');
+    console.log('📋 Method: Embeds system instructions directly in user message');
+    console.log('✨ System prompt is combined with first user message\n');
+
+    const gemmaModel = AIModelFactory.createGoogleChatModel(
+        'gemma-3-27b-it',
+        'AIzaSyBDbo7iVNEuCcRNTgDIgRrkGpFKisXXnm0'
+    );
+
+    try {
+        const gemmaResponse = await gemmaModel.chat([
+            { role: 'system', content: systemPrompt },
+            { role: 'user', content: userPrompt }
+        ]);
+
+        console.log('📤 Response:', gemmaResponse.message.content);
+        console.log('✅ Gemma system prompt working correctly\n');
+
+    } catch (error) {
+        console.error('❌ Gemma test failed:', error);
+    }
+
+    console.log('='.repeat(80));
+    console.log('\n🎯 KEY IMPROVEMENTS:');
+    console.log('• Gemini models: Use Google\'s systemInstruction parameter (official way)');
+    console.log('• Gemma models: Embed system prompts in user messages (as documented)');
+    console.log('• Automatic detection: Code detects model type and uses correct approach');
+    console.log('• Better compliance: Follows Google\'s official documentation');
+    console.log('• More reliable: Each model family gets the format it expects');
+
+    console.log('\n📖 References:');
+    console.log('• Gemma Prompt Structure: https://ai.google.dev/gemma/docs/core/prompt-structure');
+    console.log('• Gemini API Documentation: https://ai.google.dev/gemini-api/docs');
+}
+
+// Run the demonstration
+demonstrateSystemPromptImprovement().then(() => {
+    console.log('\n🎉 Demonstration completed! Both model families now work optimally.');
+});