universal-llm-client 4.2.0 → 4.5.0

package/src/test-scripts/test-system-message-positions.ts

@@ -1,163 +1,163 @@
-import { AIModelFactory } from "../factory";
-
-/**
- * Test to demonstrate library behavior when system messages appear at different positions
- */
-async function testSystemMessagePositions() {
-    console.log('🧪 Testing System Message Position Behavior...\n');
-
-    const models = [
-        { name: 'Gemini 2.5 Flash Lite', model: 'gemini-2.5-flash-lite', family: 'Gemini' },
-        { name: 'Gemma 3 27B IT', model: 'gemma-3-27b-it', family: 'Gemma' }
-    ];
-
-    for (const modelInfo of models) {
-        console.log(`\n🤖 Testing ${modelInfo.name} (${modelInfo.family} family)`);
-        console.log('='.repeat(70));
-
-        const googleModel = AIModelFactory.createGoogleChatModel(
-            modelInfo.model,
-            'AIzaSyBDbo7iVNEuCcRNTgDIgRrkGpFKisXXnm0'
-        );
-
-        await testDifferentSystemPositions(googleModel, modelInfo);
-    }
-}
-
-async function testDifferentSystemPositions(googleModel: any, modelInfo: any) {
-    // Test 1: System message at the beginning (normal case)
-    console.log('\n--- Test 1: System Message at Beginning ---');
-    console.log('Expected: Should work normally');
-    
-    try {
-        const messages1 = [
-            { role: 'system', content: 'You are concise. Always answer in exactly 3 words.' },
-            { role: 'user', content: 'What is TypeScript?' }
-        ];
-
-        const response1 = await googleModel.chat(messages1);
-        console.log('📤 Response:', response1.message.content);
-        console.log('✅ Beginning system message: Works');
-
-    } catch (error) {
-        console.error('❌ Beginning system message failed:', error);
-    }
-
-    // Test 2: System message in the middle of conversation
-    console.log('\n--- Test 2: System Message in Middle ---');
-    console.log('Current behavior: ALL system messages are processed regardless of position');
-    
-    try {
-        const messages2 = [
-            { role: 'user', content: 'Hello' },
-            { role: 'assistant', content: 'Hi there!' },
-            { role: 'system', content: 'From now on, be very formal and professional.' },  // Middle system message
-            { role: 'user', content: 'What is JavaScript?' }
-        ];
-
-        const response2 = await googleModel.chat(messages2);
-        console.log('📤 Response:', response2.message.content);
-        
-        // Check if the response is formal (indicating the middle system message was processed)
-        const isFormal = response2.message.content.toLowerCase().includes('formal') ||
-                        response2.message.content.includes('professional') ||
-                        response2.message.content.length > 50; // Longer responses tend to be more formal
-        
-        console.log(`🔍 Formality detected: ${isFormal ? '✅' : '❌'}`);
-        console.log('✅ Middle system message: Processed');
-
-    } catch (error) {
-        console.error('❌ Middle system message failed:', error);
-    }
-
-    // Test 3: Multiple system messages at different positions
-    console.log('\n--- Test 3: Multiple System Messages at Different Positions ---');
-    console.log('Current behavior: ALL system messages are combined');
-    
-    try {
-        const messages3 = [
-            { role: 'system', content: 'You are a helpful assistant.' },
-            { role: 'user', content: 'Hi' },
-            { role: 'assistant', content: 'Hello!' },
-            { role: 'system', content: 'Always include emojis in your responses.' },  // Second system message
-            { role: 'user', content: 'Tell me about Python programming.' },
-            { role: 'system', content: 'Keep responses under 50 words.' }  // Third system message
-        ];
-
-        const response3 = await googleModel.chat(messages3);
-        console.log('📤 Response:', response3.message.content);
-        
-        // Check if all system instructions were applied
-        const hasEmojis = /[\u{1F600}-\u{1F64F}]|[\u{1F300}-\u{1F5FF}]|[\u{1F680}-\u{1F6FF}]|[\u{1F1E0}-\u{1F1FF}]/u.test(response3.message.content);
-        const isShort = response3.message.content.split(' ').length <= 50;
-        
-        console.log(`🔍 Emojis present: ${hasEmojis ? '✅' : '❌'}`);
-        console.log(`🔍 Under 50 words: ${isShort ? '✅' : '❌'}`);
-        console.log('✅ Multiple system messages: All processed');
-
-    } catch (error) {
-        console.error('❌ Multiple system messages failed:', error);
-    }
-
-    // Test 4: System message at the very end
-    console.log('\n--- Test 4: System Message at End ---');
-    console.log('Current behavior: Still processed (all system messages are collected)');
-    
-    try {
-        const messages4 = [
-            { role: 'user', content: 'What is React?' },
-            { role: 'assistant', content: 'React is a JavaScript library for building user interfaces.' },
-            { role: 'user', content: 'Can you explain it differently?' },
-            { role: 'system', content: 'Explain everything using analogies to cooking.' }  // End system message
-        ];
-
-        const response4 = await googleModel.chat(messages4);
-        console.log('📤 Response:', response4.message.content);
-        
-        // Check if cooking analogies were used
-        const hasCookingAnalogy = response4.message.content.toLowerCase().includes('cook') ||
-                                 response4.message.content.toLowerCase().includes('recipe') ||
-                                 response4.message.content.toLowerCase().includes('ingredient') ||
-                                 response4.message.content.toLowerCase().includes('kitchen');
-        
-        console.log(`🔍 Cooking analogy detected: ${hasCookingAnalogy ? '✅' : '❌'}`);
-        console.log('✅ End system message: Processed');
-
-    } catch (error) {
-        console.error('❌ End system message failed:', error);
-    }
-
-    console.log('\n📋 Summary for ' + modelInfo.name + ':');
-    
-    if (modelInfo.family === 'Gemma') {
-        console.log('• Gemma behavior: ALL system messages are combined and embedded in FIRST user message');
-        console.log('• Position doesn\'t matter - all system messages are processed');
-        console.log('• System instructions apply to the entire conversation');
-    } else {
-        console.log('• Gemini behavior: ALL system messages are combined into systemInstruction parameter');
-        console.log('• Position doesn\'t matter - all system messages are processed');
-        console.log('• System instructions apply to the entire conversation');
-    }
-}
-
-// Run the test
-testSystemMessagePositions().then(() => {
-    console.log('\n' + '='.repeat(70));
-    console.log('🎯 KEY FINDINGS:');
-    console.log('');
-    console.log('📍 CURRENT BEHAVIOR:');
-    console.log('• System messages at ANY position are processed');
-    console.log('• ALL system messages are combined (regardless of position)');
-    console.log('• For Gemma: Combined system messages embedded in first user message');
-    console.log('• For Gemini: Combined system messages sent as systemInstruction parameter');
-    console.log('');
-    console.log('⚠️  POTENTIAL CONSIDERATIONS:');
-    console.log('• System messages in middle of conversation might be unexpected');
-    console.log('• Some chat paradigms expect system messages only at the beginning');
-    console.log('• Current behavior is consistent but might not match all use cases');
-    console.log('');
-    console.log('✅ RECOMMENDATION:');
-    console.log('• For best results, place system messages at the beginning');
-    console.log('• Current implementation is robust and handles all positions');
-    console.log('• Consider if mid-conversation system changes should be handled differently');
-});
+import { AIModelFactory } from "../factory";
+
+/**
+ * Test to demonstrate library behavior when system messages appear at different positions
+ */
+async function testSystemMessagePositions() {
+    console.log('🧪 Testing System Message Position Behavior...\n');
+
+    const models = [
+        { name: 'Gemini 2.5 Flash Lite', model: 'gemini-2.5-flash-lite', family: 'Gemini' },
+        { name: 'Gemma 3 27B IT', model: 'gemma-3-27b-it', family: 'Gemma' }
+    ];
+
+    for (const modelInfo of models) {
+        console.log(`\n🤖 Testing ${modelInfo.name} (${modelInfo.family} family)`);
+        console.log('='.repeat(70));
+
+        const googleModel = AIModelFactory.createGoogleChatModel(
+            modelInfo.model,
+            (process.env.GOOGLE_API_KEY ?? '')
+        );
+
+        await testDifferentSystemPositions(googleModel, modelInfo);
+    }
+}
+
+async function testDifferentSystemPositions(googleModel: any, modelInfo: any) {
+    // Test 1: System message at the beginning (normal case)
+    console.log('\n--- Test 1: System Message at Beginning ---');
+    console.log('Expected: Should work normally');
+    
+    try {
+        const messages1 = [
+            { role: 'system', content: 'You are concise. Always answer in exactly 3 words.' },
+            { role: 'user', content: 'What is TypeScript?' }
+        ];
+
+        const response1 = await googleModel.chat(messages1);
+        console.log('📤 Response:', response1.message.content);
+        console.log('✅ Beginning system message: Works');
+
+    } catch (error) {
+        console.error('❌ Beginning system message failed:', error);
+    }
+
+    // Test 2: System message in the middle of conversation
+    console.log('\n--- Test 2: System Message in Middle ---');
+    console.log('Current behavior: ALL system messages are processed regardless of position');
+    
+    try {
+        const messages2 = [
+            { role: 'user', content: 'Hello' },
+            { role: 'assistant', content: 'Hi there!' },
+            { role: 'system', content: 'From now on, be very formal and professional.' },  // Middle system message
+            { role: 'user', content: 'What is JavaScript?' }
+        ];
+
+        const response2 = await googleModel.chat(messages2);
+        console.log('📤 Response:', response2.message.content);
+        
+        // Check if the response is formal (indicating the middle system message was processed)
+        const isFormal = response2.message.content.toLowerCase().includes('formal') ||
+                        response2.message.content.includes('professional') ||
+                        response2.message.content.length > 50; // Longer responses tend to be more formal
+        
+        console.log(`🔍 Formality detected: ${isFormal ? '✅' : '❌'}`);
+        console.log('✅ Middle system message: Processed');
+
+    } catch (error) {
+        console.error('❌ Middle system message failed:', error);
+    }
+
+    // Test 3: Multiple system messages at different positions
+    console.log('\n--- Test 3: Multiple System Messages at Different Positions ---');
+    console.log('Current behavior: ALL system messages are combined');
+    
+    try {
+        const messages3 = [
+            { role: 'system', content: 'You are a helpful assistant.' },
+            { role: 'user', content: 'Hi' },
+            { role: 'assistant', content: 'Hello!' },
+            { role: 'system', content: 'Always include emojis in your responses.' },  // Second system message
+            { role: 'user', content: 'Tell me about Python programming.' },
+            { role: 'system', content: 'Keep responses under 50 words.' }  // Third system message
+        ];
+
+        const response3 = await googleModel.chat(messages3);
+        console.log('📤 Response:', response3.message.content);
+        
+        // Check if all system instructions were applied
+        const hasEmojis = /[\u{1F600}-\u{1F64F}]|[\u{1F300}-\u{1F5FF}]|[\u{1F680}-\u{1F6FF}]|[\u{1F1E0}-\u{1F1FF}]/u.test(response3.message.content);
+        const isShort = response3.message.content.split(' ').length <= 50;
+        
+        console.log(`🔍 Emojis present: ${hasEmojis ? '✅' : '❌'}`);
+        console.log(`🔍 Under 50 words: ${isShort ? '✅' : '❌'}`);
+        console.log('✅ Multiple system messages: All processed');
+
+    } catch (error) {
+        console.error('❌ Multiple system messages failed:', error);
+    }
+
+    // Test 4: System message at the very end
+    console.log('\n--- Test 4: System Message at End ---');
+    console.log('Current behavior: Still processed (all system messages are collected)');
+    
+    try {
+        const messages4 = [
+            { role: 'user', content: 'What is React?' },
+            { role: 'assistant', content: 'React is a JavaScript library for building user interfaces.' },
+            { role: 'user', content: 'Can you explain it differently?' },
+            { role: 'system', content: 'Explain everything using analogies to cooking.' }  // End system message
+        ];
+
+        const response4 = await googleModel.chat(messages4);
+        console.log('📤 Response:', response4.message.content);
+        
+        // Check if cooking analogies were used
+        const hasCookingAnalogy = response4.message.content.toLowerCase().includes('cook') ||
+                                 response4.message.content.toLowerCase().includes('recipe') ||
+                                 response4.message.content.toLowerCase().includes('ingredient') ||
+                                 response4.message.content.toLowerCase().includes('kitchen');
+        
+        console.log(`🔍 Cooking analogy detected: ${hasCookingAnalogy ? '✅' : '❌'}`);
+        console.log('✅ End system message: Processed');
+
+    } catch (error) {
+        console.error('❌ End system message failed:', error);
+    }
+
+    console.log('\n📋 Summary for ' + modelInfo.name + ':');
+    
+    if (modelInfo.family === 'Gemma') {
+        console.log('• Gemma behavior: ALL system messages are combined and embedded in FIRST user message');
+        console.log('• Position doesn\'t matter - all system messages are processed');
+        console.log('• System instructions apply to the entire conversation');
+    } else {
+        console.log('• Gemini behavior: ALL system messages are combined into systemInstruction parameter');
+        console.log('• Position doesn\'t matter - all system messages are processed');
+        console.log('• System instructions apply to the entire conversation');
+    }
+}
+
+// Run the test
+testSystemMessagePositions().then(() => {
+    console.log('\n' + '='.repeat(70));
+    console.log('🎯 KEY FINDINGS:');
+    console.log('');
+    console.log('📍 CURRENT BEHAVIOR:');
+    console.log('• System messages at ANY position are processed');
+    console.log('• ALL system messages are combined (regardless of position)');
+    console.log('• For Gemma: Combined system messages embedded in first user message');
+    console.log('• For Gemini: Combined system messages sent as systemInstruction parameter');
+    console.log('');
+    console.log('⚠️  POTENTIAL CONSIDERATIONS:');
+    console.log('• System messages in middle of conversation might be unexpected');
+    console.log('• Some chat paradigms expect system messages only at the beginning');
+    console.log('• Current behavior is consistent but might not match all use cases');
+    console.log('');
+    console.log('✅ RECOMMENDATION:');
+    console.log('• For best results, place system messages at the beginning');
+    console.log('• Current implementation is robust and handles all positions');
+    console.log('• Consider if mid-conversation system changes should be handled differently');
+});

package/src/test-scripts/test-system-prompt-improvement-demo.ts

@@ -1,83 +1,83 @@
-import { AIModelFactory } from "../factory";
-
-/**
- * Demonstration of improved Gemma system prompt handling
- * Shows the difference between Gemini (systemInstruction) and Gemma (embedded) approaches
- */
-async function demonstrateSystemPromptImprovement() {
-    console.log('🔬 Demonstrating Improved System Prompt Handling\n');
-    console.log('📚 Based on Google\'s documentation: https://ai.google.dev/gemma/docs/core/prompt-structure\n');
-
-    // Test the same system prompt with both model types
-    const systemPrompt = 'You are a helpful cooking assistant. Always provide step-by-step recipes.';
-    const userPrompt = 'How do I make scrambled eggs?';
-
-    console.log('🧪 Testing System Prompt:', systemPrompt);
-    console.log('❓ User Question:', userPrompt);
-    console.log('\n' + '='.repeat(80));
-
-    // Test 1: Gemini model (uses systemInstruction parameter)
-    console.log('\n🤖 GEMINI 2.5 FLASH LITE');
-    console.log('📋 Method: Uses Google\'s systemInstruction parameter');
-    console.log('✨ System prompt is sent separately from user content\n');
-
-    const geminiModel = AIModelFactory.createGoogleChatModel(
-        'gemini-2.5-flash-lite',
-        'AIzaSyBDbo7iVNEuCcRNTgDIgRrkGpFKisXXnm0'
-    );
-
-    try {
-        const geminiResponse = await geminiModel.chat([
-            { role: 'system', content: systemPrompt },
-            { role: 'user', content: userPrompt }
-        ]);
-
-        console.log('📤 Response:', geminiResponse.message.content);
-        console.log('✅ Gemini system prompt working correctly\n');
-
-    } catch (error) {
-        console.error('❌ Gemini test failed:', error);
-    }
-
-    console.log('='.repeat(80));
-
-    // Test 2: Gemma model (embeds system prompt in user message)
-    console.log('\n🤖 GEMMA 3 27B IT');
-    console.log('📋 Method: Embeds system instructions directly in user message');
-    console.log('✨ System prompt is combined with first user message\n');
-
-    const gemmaModel = AIModelFactory.createGoogleChatModel(
-        'gemma-3-27b-it',
-        'AIzaSyBDbo7iVNEuCcRNTgDIgRrkGpFKisXXnm0'
-    );
-
-    try {
-        const gemmaResponse = await gemmaModel.chat([
-            { role: 'system', content: systemPrompt },
-            { role: 'user', content: userPrompt }
-        ]);
-
-        console.log('📤 Response:', gemmaResponse.message.content);
-        console.log('✅ Gemma system prompt working correctly\n');
-
-    } catch (error) {
-        console.error('❌ Gemma test failed:', error);
-    }
-
-    console.log('='.repeat(80));
-    console.log('\n🎯 KEY IMPROVEMENTS:');
-    console.log('• Gemini models: Use Google\'s systemInstruction parameter (official way)');
-    console.log('• Gemma models: Embed system prompts in user messages (as documented)');
-    console.log('• Automatic detection: Code detects model type and uses correct approach');
-    console.log('• Better compliance: Follows Google\'s official documentation');
-    console.log('• More reliable: Each model family gets the format it expects');
-
-    console.log('\n📖 References:');
-    console.log('• Gemma Prompt Structure: https://ai.google.dev/gemma/docs/core/prompt-structure');
-    console.log('• Gemini API Documentation: https://ai.google.dev/gemini-api/docs');
-}
-
-// Run the demonstration
-demonstrateSystemPromptImprovement().then(() => {
-    console.log('\n🎉 Demonstration completed! Both model families now work optimally.');
-});
+import { AIModelFactory } from "../factory";
+
+/**
+ * Demonstration of improved Gemma system prompt handling
+ * Shows the difference between Gemini (systemInstruction) and Gemma (embedded) approaches
+ */
+async function demonstrateSystemPromptImprovement() {
+    console.log('🔬 Demonstrating Improved System Prompt Handling\n');
+    console.log('📚 Based on Google\'s documentation: https://ai.google.dev/gemma/docs/core/prompt-structure\n');
+
+    // Test the same system prompt with both model types
+    const systemPrompt = 'You are a helpful cooking assistant. Always provide step-by-step recipes.';
+    const userPrompt = 'How do I make scrambled eggs?';
+
+    console.log('🧪 Testing System Prompt:', systemPrompt);
+    console.log('❓ User Question:', userPrompt);
+    console.log('\n' + '='.repeat(80));
+
+    // Test 1: Gemini model (uses systemInstruction parameter)
+    console.log('\n🤖 GEMINI 2.5 FLASH LITE');
+    console.log('📋 Method: Uses Google\'s systemInstruction parameter');
+    console.log('✨ System prompt is sent separately from user content\n');
+
+    const geminiModel = AIModelFactory.createGoogleChatModel(
+        'gemini-2.5-flash-lite',
+        (process.env.GOOGLE_API_KEY ?? '')
+    );
+
+    try {
+        const geminiResponse = await geminiModel.chat([
+            { role: 'system', content: systemPrompt },
+            { role: 'user', content: userPrompt }
+        ]);
+
+        console.log('📤 Response:', geminiResponse.message.content);
+        console.log('✅ Gemini system prompt working correctly\n');
+
+    } catch (error) {
+        console.error('❌ Gemini test failed:', error);
+    }
+
+    console.log('='.repeat(80));
+
+    // Test 2: Gemma model (embeds system prompt in user message)
+    console.log('\n🤖 GEMMA 3 27B IT');
+    console.log('📋 Method: Embeds system instructions directly in user message');
+    console.log('✨ System prompt is combined with first user message\n');
+
+    const gemmaModel = AIModelFactory.createGoogleChatModel(
+        'gemma-3-27b-it',
+        (process.env.GOOGLE_API_KEY ?? '')
+    );
+
+    try {
+        const gemmaResponse = await gemmaModel.chat([
+            { role: 'system', content: systemPrompt },
+            { role: 'user', content: userPrompt }
+        ]);
+
+        console.log('📤 Response:', gemmaResponse.message.content);
+        console.log('✅ Gemma system prompt working correctly\n');
+
+    } catch (error) {
+        console.error('❌ Gemma test failed:', error);
+    }
+
+    console.log('='.repeat(80));
+    console.log('\n🎯 KEY IMPROVEMENTS:');
+    console.log('• Gemini models: Use Google\'s systemInstruction parameter (official way)');
+    console.log('• Gemma models: Embed system prompts in user messages (as documented)');
+    console.log('• Automatic detection: Code detects model type and uses correct approach');
+    console.log('• Better compliance: Follows Google\'s official documentation');
+    console.log('• More reliable: Each model family gets the format it expects');
+
+    console.log('\n📖 References:');
+    console.log('• Gemma Prompt Structure: https://ai.google.dev/gemma/docs/core/prompt-structure');
+    console.log('• Gemini API Documentation: https://ai.google.dev/gemini-api/docs');
+}
+
+// Run the demonstration
+demonstrateSystemPromptImprovement().then(() => {
+    console.log('\n🎉 Demonstration completed! Both model families now work optimally.');
+});