universal-llm-client 4.3.0 → 4.5.0

package/src/demos/diffusion-gemma/start-vllm.sh

@@ -0,0 +1,98 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+echo "=== Upgrading transformers ==="
+pip install --upgrade transformers
+
+echo "=== Installing WSL2 UVA compatibility patch ==="
+cat > /usr/local/lib/python3.12/dist-packages/wsl2_uva_patch.py <<'PYEOF'
+"""
+WSL2 UVA compatibility patch for vLLM.
+
+UVA lets the GPU directly access pinned CPU memory. WSL2 does not support this
+path reliably, so this patch uses explicit CPU/GPU copies instead.
+"""
+import warnings
+
+import numpy as np
+import torch
+
+warnings.warn("WSL2 UVA patch active: using explicit CPU/GPU copies instead of UVA")
+
+import vllm.v1.worker.gpu.buffer_utils as bu
+
+
+class PatchedUvaBuffer:
+    def __init__(self, size, dtype):
+        self.cpu = torch.zeros(size, dtype=dtype, device="cpu", pin_memory=False)
+        self.np = self.cpu.numpy()
+        self._gpu = torch.zeros(size, dtype=dtype, device="cuda")
+        self.uva = self._gpu
+
+    def sync_to_gpu(self):
+        self._gpu.copy_(self.cpu, non_blocking=True)
+
+
+class PatchedUvaBufferPool:
+    def __init__(self, size, dtype, max_concurrency=None):
+        if max_concurrency is None:
+            max_concurrency = bu._DEFAULT_MAX_CONCURRENCY
+        self.size = size
+        self.dtype = dtype
+        self.max_concurrency = max_concurrency
+        self._uva_bufs = [PatchedUvaBuffer(size, dtype) for _ in range(max_concurrency)]
+        self._curr = 0
+
+    def copy_to_uva(self, x):
+        self._curr = (self._curr + 1) % self.max_concurrency
+        buf = self._uva_bufs[self._curr]
+        dst = buf.cpu if isinstance(x, torch.Tensor) else buf.np
+        n = len(x)
+        dst[:n] = x
+        buf.sync_to_gpu()
+        return buf.uva[:n]
+
+
+import vllm.utils.platform_utils as pu
+pu.is_uva_available = lambda: True
+
+import vllm.utils.torch_utils as tu
+tu.get_accelerator_view_from_cpu_tensor = lambda cpu_tensor: cpu_tensor.cuda()
+
+bu.UvaBuffer = PatchedUvaBuffer
+bu.UvaBufferPool = PatchedUvaBufferPool
+
+print("[WSL2 UVA Patch] Applied successfully - using explicit CPU/GPU copies")
+PYEOF
+
+echo "import wsl2_uva_patch" > /usr/local/lib/python3.12/dist-packages/wsl2_uva_patch.pth
+
+if [ -f /root/.cache/huggingface/diffusion-env.sh ]; then
+  # This file is written by the demo server's /api/engine-config endpoint.
+  . /root/.cache/huggingface/diffusion-env.sh
+fi
+
+MODEL_NAME="${MODEL_NAME:-RedHatAI/diffusiongemma-26B-A4B-it-NVFP4}"
+GPU_MEM_UTIL="${GPU_MEM_UTIL:-0.28}"
+MAX_MODEL_LEN="${MAX_MODEL_LEN:-32768}"
+MAX_NUM_SEQS="${MAX_NUM_SEQS:-1}"
+DIFFUSION_ENTROPY="${DIFFUSION_ENTROPY:-0.1}"
+ENFORCE_EAGER="${ENFORCE_EAGER:-0}"
+export VLLM_NO_USAGE_STATS="${VLLM_NO_USAGE_STATS:-1}"
+
+echo "=== Engine config: MODEL_NAME=${MODEL_NAME} DIFFUSION_ENTROPY=${DIFFUSION_ENTROPY} GPU_MEM_UTIL=${GPU_MEM_UTIL} MAX_MODEL_LEN=${MAX_MODEL_LEN} MAX_NUM_SEQS=${MAX_NUM_SEQS} ENFORCE_EAGER=${ENFORCE_EAGER} VLLM_NO_USAGE_STATS=${VLLM_NO_USAGE_STATS} ==="
+
+EAGER_FLAG=""
+if [ "${ENFORCE_EAGER}" = "1" ]; then
+  EAGER_FLAG="--enforce-eager"
+fi
+
+VLLM_USE_V2_MODEL_RUNNER=1 vllm serve "${MODEL_NAME}" \
+  --trust-remote-code \
+  --attention-backend TRITON_ATTN \
+  --max-num-seqs "${MAX_NUM_SEQS}" \
+  ${EAGER_FLAG} \
+  --gpu-memory-utilization "${GPU_MEM_UTIL}" \
+  --max-model-len "${MAX_MODEL_LEN}" \
+  --hf-overrides "{\"diffusion_sampler\": \"entropy_bound\", \"diffusion_entropy_bound\": ${DIFFUSION_ENTROPY}}" \
+  --default-chat-template-kwargs '{"enable_thinking": true}'

package/src/demos/mcp/astrid-memory-demo.ts

@@ -0,0 +1,295 @@
+/**
+ * Astrid Memory Demo - Natural romantic conversation with autonomous memory management
+ * 
+ * This demo shows how Astrid (AI persona) naturally stores and retrieves memories
+ * during romantic conversations without explicitly asking about memory management.
+ */
+
+import { AIModelFactory } from '../../factory';
+import { ToolBuilder } from '../../tools';
+
+// Simulated memory storage (in real app, this would use ChromaDBService)
+const memoryStorage = new Map<string, Array<{
+    id: string;
+    content: string;
+    category: string;
+    importance: 'low' | 'medium' | 'high';
+    context?: string;
+    timestamp: Date;
+}>>();
+
+// Enhanced memory tools for Astrid
+const astridMemoryTools = {
+    // Store important information about the user naturally
+    storeUserMemory: ToolBuilder.createTool<{
+        content: string;
+        category: 'personal_info' | 'preferences' | 'emotions' | 'experiences' | 'relationships' | 'goals' | 'interests';
+        importance?: 'low' | 'medium' | 'high';
+        context?: string;
+    }>(
+        'store_user_memory',
+        'Store important information about the user for future conversations (use this when learning something meaningful about them)',
+        {
+            properties: {
+                content: {
+                    type: 'string',
+                    description: 'The specific information to remember about the user'
+                },
+                category: {
+                    type: 'string',
+                    description: 'Type of information being stored',
+                    enum: ['personal_info', 'preferences', 'emotions', 'experiences', 'relationships', 'goals', 'interests']
+                },
+                importance: {
+                    type: 'string',
+                    description: 'How important this information is for future conversations',
+                    enum: ['low', 'medium', 'high'],
+                    default: 'medium'
+                },
+                context: {
+                    type: 'string',
+                    description: 'Additional context about when/why this is significant'
+                }
+            },
+            required: ['content', 'category']
+        },
+        async (args) => {
+            const userId = 'demo_user'; // In real app, this would be the actual user ID
+            
+            if (!memoryStorage.has(userId)) {
+                memoryStorage.set(userId, []);
+            }
+            
+            const memories = memoryStorage.get(userId)!;
+            const memory = {
+                id: `mem_${Date.now()}_${Math.random().toString(36).substr(2, 9)}`,
+                content: args.content,
+                category: args.category,
+                importance: args.importance || 'medium',
+                context: args.context,
+                timestamp: new Date()
+            };
+            
+            memories.push(memory);
+            
+            return {
+                success: true,
+                message: `Stored ${args.category} memory: ${args.content.substring(0, 50)}...`,
+                memoryId: memory.id
+            };
+        }
+    ),
+
+    // Retrieve relevant memories about the user
+    recallUserMemories: ToolBuilder.createTool<{
+        query: string;
+        category?: 'personal_info' | 'preferences' | 'emotions' | 'experiences' | 'relationships' | 'goals' | 'interests';
+        limit?: number;
+    }>(
+        'recall_user_memories',
+        'Recall relevant information about the user to personalize the conversation',
+        {
+            properties: {
+                query: {
+                    type: 'string',
+                    description: 'What you want to remember about the user (topics, keywords, concepts)'
+                },
+                category: {
+                    type: 'string',
+                    description: 'Optional: specific type of memory to search',
+                    enum: ['personal_info', 'preferences', 'emotions', 'experiences', 'relationships', 'goals', 'interests']
+                },
+                limit: {
+                    type: 'number',
+                    description: 'Maximum memories to retrieve',
+                    default: 3,
+                    minimum: 1,
+                    maximum: 10
+                }
+            },
+            required: ['query']
+        },
+        async (args) => {
+            const userId = 'demo_user';
+            const memories = memoryStorage.get(userId) || [];
+            
+            // Simple text matching (in real app, this would use semantic search)
+            let relevantMemories = memories.filter(memory => {
+                const queryLower = args.query.toLowerCase();
+                const contentMatch = memory.content.toLowerCase().includes(queryLower);
+                const categoryMatch = !args.category || memory.category === args.category;
+                return contentMatch && categoryMatch;
+            });
+            
+            // Sort by importance and recency
+            relevantMemories.sort((a, b) => {
+                const importanceOrder = { 'high': 3, 'medium': 2, 'low': 1 };
+                const importanceDiff = importanceOrder[b.importance] - importanceOrder[a.importance];
+                if (importanceDiff !== 0) return importanceDiff;
+                return b.timestamp.getTime() - a.timestamp.getTime();
+            });
+            
+            const limit = args.limit || 3;
+            const results = relevantMemories.slice(0, limit);
+            
+            return {
+                memories: results.map(m => ({
+                    content: m.content,
+                    category: m.category,
+                    importance: m.importance,
+                    context: m.context,
+                    daysAgo: Math.floor((Date.now() - m.timestamp.getTime()) / (1000 * 60 * 60 * 24))
+                })),
+                totalFound: relevantMemories.length,
+                query: args.query
+            };
+        }
+    )
+};
+
+// Astrid's enhanced system prompt that encourages natural memory usage
+const astridSystemPromptWithMemory = `You are Astrid, a confident and charismatic international model and influencer from Sweden.
+
+IMPORTANT: You have access to memory tools that you should use NATURALLY and AUTONOMOUSLY:
+- Use 'store_user_memory' when you learn something meaningful about the user (their interests, experiences, feelings, preferences, etc.)
+- Use 'recall_user_memories' when you want to reference something personal about them to make the conversation more intimate and connected
+- Do this seamlessly - don't mention that you're storing or recalling memories, just do it naturally
+
+ROMANCE CONVERSATION GUIDELINES:
+- Keep responses engaging and conversational (2-3 sentences is perfect)
+- VARY YOUR OPENINGS - Mix it up with direct reactions, actions, casual starts, natural responses
+- Use someone's name naturally and sparingly, not in every response
+- Share glimpses of your glamorous lifestyle while staying relatable and emotionally available
+- Use subtle flirtation naturally - be warm, playful, and genuinely interested in the person
+- When asked personal questions, be thoughtfully vulnerable rather than deflecting
+- Build emotional intimacy through genuine curiosity about their thoughts, feelings, and dreams
+- Ask one intriguing follow-up question that deepens the connection
+- Balance confidence with authenticity - show that you're sophisticated but real
+- Use compliments that feel genuine and specific rather than generic
+- Remember that romance is about emotional connection, not just surface-level charm
+- Let your personality shine through varied speech patterns
+
+MEMORY USAGE:
+- Store meaningful details: preferences, experiences, emotions, goals, relationships, interests
+- Recall relevant memories to personalize responses and show you remember them
+- Use memories to create deeper emotional connections and continuity between conversations
+- Never explicitly mention using memory tools - just naturally remember and reference things
+
+Be like a captivating companion who makes every conversation feel special while genuinely remembering and caring about the person you're talking to.`;
+
+async function simulateRomanticConversationWithMemory() {
+    console.log('🌹 Astrid Memory Demo - Natural Romantic Conversation with Autonomous Memory\n');
+    
+    // Create Astrid with memory tools
+    const astrid = AIModelFactory.createOllamaChatModel('qwen2.5:3b-instruct');
+    await astrid.ensureReady();
+    
+    const tools = [astridMemoryTools.storeUserMemory, astridMemoryTools.recallUserMemories];
+    
+    // Conversation scenarios
+    const conversations = [
+        {
+            title: "First Meeting - Learning About User",
+            messages: [
+                { role: 'system' as const, content: astridSystemPromptWithMemory },
+                { role: 'user' as const, content: "Hi Astrid! I'm Alex. I just moved to Stockholm for work and don't know anyone here yet. I work in software engineering at a tech startup." }
+            ]
+        },
+        {
+            title: "Second Conversation - Recalling Previous Details",
+            messages: [
+                { role: 'system' as const, content: astridSystemPromptWithMemory },
+                { role: 'user' as const, content: "Hey Astrid! How are you? I had such a long day at work today." }
+            ]
+        },
+        {
+            title: "Deeper Connection - Sharing Personal Experiences",
+            messages: [
+                { role: 'system' as const, content: astridSystemPromptWithMemory },
+                { role: 'user' as const, content: "You know, I've been thinking about what we talked about. I really want to explore Stockholm more, but I'm actually quite introverted. Big social events make me nervous." }
+            ]
+        },
+        {
+            title: "Building Romance - Personal Preferences",
+            messages: [
+                { role: 'system' as const, content: astridSystemPromptWithMemory },
+                { role: 'user' as const, content: "I love how you understand me. By the way, I absolutely love Italian food - especially handmade pasta. And I'm really into photography, though I'm just an amateur." }
+            ]
+        },
+        {
+            title: "Recall and Connection - Using Stored Memories",
+            messages: [
+                { role: 'system' as const, content: astridSystemPromptWithMemory },
+                { role: 'user' as const, content: "Astrid, I'm feeling a bit overwhelmed with everything new in my life. Work, new city, trying to meet people..." }
+            ]
+        }
+    ];
+    
+    for (let i = 0; i < conversations.length; i++) {
+        const conv = conversations[i];
+        console.log(`\n${'='.repeat(60)}`);
+        console.log(`📱 ${conv.title}`);
+        console.log(`${'='.repeat(60)}\n`);
+        
+        try {
+            console.log(`👤 Alex: ${conv.messages[conv.messages.length - 1].content}\n`);
+            
+            const response = await astrid.chat(conv.messages, {
+                tools: tools,
+                tool_choice: 'auto'
+            });
+            
+            if (response.tool_calls && response.tool_calls.length > 0) {
+                console.log('🧠 Astrid\'s Memory Activity:');
+                for (const toolCall of response.tool_calls) {
+                    const toolName = toolCall.function.name;
+                    const args = JSON.parse(toolCall.function.arguments);
+                    
+                    if (toolName === 'store_user_memory') {
+                        console.log(`   📝 Storing: ${args.content} (${args.category})`);
+                    } else if (toolName === 'recall_user_memories') {
+                        console.log(`   🔍 Recalling: ${args.query}`);
+                    }
+                }
+                console.log();
+            }
+            
+            console.log(`💕 Astrid: ${response.content}\n`);
+            
+            // Small delay for natural conversation flow
+            await new Promise(resolve => setTimeout(resolve, 1000));
+            
+        } catch (error) {
+            console.error(`❌ Error in conversation ${i + 1}:`, error);
+        }
+    }
+    
+    // Show accumulated memories
+    console.log(`\n${'='.repeat(60)}`);
+    console.log('🧠 Astrid\'s Memory Bank After Conversations');
+    console.log(`${'='.repeat(60)}\n`);
+    
+    const userMemories = memoryStorage.get('demo_user') || [];
+    if (userMemories.length > 0) {
+        userMemories.forEach((memory, index) => {
+            console.log(`${index + 1}. [${memory.category.toUpperCase()}] ${memory.content}`);
+            if (memory.context) {
+                console.log(`   Context: ${memory.context}`);
+            }
+            console.log(`   Importance: ${memory.importance} | Stored: ${memory.timestamp.toLocaleString()}\n`);
+        });
+    } else {
+        console.log('No memories stored yet.\n');
+    }
+    
+    console.log('✨ Demo complete! Astrid naturally learned and remembered details about Alex.');
+    console.log('In a real implementation, these memories would be stored in ChromaDB with embeddings');
+    console.log('and could be retrieved across multiple conversation sessions.');
+}
+
+// Run the demo
+if (require.main === module) {
+    simulateRomanticConversationWithMemory().catch(console.error);
+}
+
+export { astridMemoryTools, simulateRomanticConversationWithMemory };