universal-llm-client 4.3.0 → 4.5.0

package/src/test-scripts/test-tool-calling.ts

@@ -0,0 +1,231 @@
+/**
+ * Test tool calling functionality with free local models
+ */
+
+import { AIModelFactory, ToolBuilder } from '../index';
+
+async function testToolCallingLocal() {
+    console.log('🛠️  Testing Universal LLM Client Tool Calling with Local Models\n');
+
+    // Create models for testing (using models with good tool calling support)
+    const models = {
+        // Test with Ollama - qwen3:8b has excellent tool calling support
+        ollama: AIModelFactory.createOllamaChatModelWithTools('qwen3:8b'),
+        // Test with LM Studio - qwen/qwen3-8b for tool calling
+        lmstudio: AIModelFactory.createOpenAIChatModelWithTools('qwen/qwen3-8b', 'http://localhost:1234/v1')
+    };
+
+    console.log('📋 Test 1: Basic Calculator Tool\n');
+    
+    for (const [provider, model] of Object.entries(models)) {
+        console.log(`\n🔧 Testing ${provider} (checking if server is available):`);
+        
+        try {
+            await model.ensureReady();
+            
+            const response = await model.chat([
+                { role: 'user', content: 'What is 25 * 4 + 10? Please use the calculator tool to solve this mathematically.' }
+            ], {}, { 
+                tool_choice: 'auto' 
+            });
+
+            console.log(`Response: ${response.content}`);
+            
+            if (response.tool_calls) {
+                console.log(`🔨 Tool calls made:`, response.tool_calls.length);
+                for (const toolCall of response.tool_calls) {
+                    console.log(`  - ${toolCall.function.name}: ${toolCall.function.arguments}`);
+                }
+            } else {
+                console.log('ℹ️  No tool calls made - model may have calculated directly');
+            }
+        } catch (error) {
+            if ((error as Error).message.includes('ECONNREFUSED') || (error as Error).message.includes('fetch failed')) {
+                console.error(`❌ ${provider} server not running - skipping tests for this provider`);
+            } else {
+                console.error(`❌ Error with ${provider}:`, (error as Error).message);
+            }
+        }
+    }
+
+    // Test automatic tool execution with calculator
+    console.log('\n\n📋 Test 2: Automatic Tool Execution\n');
+    
+    const ollamaModel = models.ollama;
+    
+    try {
+        console.log('🔧 Testing automatic tool execution with Ollama:');
+        
+        const response = await ollamaModel.chatWithTools([
+            { 
+                role: 'user', 
+                content: 'Calculate 15 * 8 + 32, then tell me what time it is right now' 
+            }
+        ]);
+
+        console.log('✅ Final response:', response.content);
+    } catch (error) {
+        console.error('❌ Error:', (error as Error).message);
+    }
+
+    // Test multiple tools
+    console.log('\n\n📋 Test 3: Multiple Tool Usage\n');
+    
+    try {
+        console.log('🔧 Testing multiple tools with Ollama:');
+        
+        // Register additional useful tools
+        const randomNumberTool = ToolBuilder.createTool<{ min: number; max: number; count?: number }>(
+            'generate_random_numbers',
+            'Generate random numbers within a range',
+            {
+                properties: {
+                    min: { type: 'number', description: 'Minimum value' },
+                    max: { type: 'number', description: 'Maximum value' },
+                    count: { type: 'number', description: 'How many numbers to generate', default: 1 }
+                },
+                required: ['min', 'max']
+            },
+            (args) => {
+                const count = args.count || 1;
+                const numbers: any[] = [];
+                for (let i = 0; i < count; i++) {
+                    numbers.push(Math.floor(Math.random() * (args.max - args.min + 1)) + args.min);
+                }
+                return { numbers, count: numbers.length };
+            }
+        );
+
+        const textTool = ToolBuilder.createTool<{ text: string; operation: 'uppercase' | 'lowercase' | 'reverse' }>(
+            'text_transform',
+            'Transform text in various ways',
+            {
+                properties: {
+                    text: { type: 'string', description: 'Text to transform' },
+                    operation: { 
+                        type: 'string', 
+                        enum: ['uppercase', 'lowercase', 'reverse'],
+                        description: 'Type of transformation' 
+                    }
+                },
+                required: ['text', 'operation']
+            },
+            (args) => {
+                let result = args.text;
+                switch (args.operation) {
+                    case 'uppercase':
+                        result = args.text.toUpperCase();
+                        break;
+                    case 'lowercase':
+                        result = args.text.toLowerCase();
+                        break;
+                    case 'reverse':
+                        result = args.text.split('').reverse().join('');
+                        break;
+                }
+                return { original: args.text, transformed: result, operation: args.operation };
+            }
+        );
+
+        ollamaModel.registerTools([randomNumberTool, textTool]);
+        
+        const response = await ollamaModel.chatWithTools([
+            { 
+                role: 'user', 
+                content: 'First calculate 100 / 4, then generate 3 random numbers between 1 and 10, and finally convert the text "hello world" to uppercase' 
+            }
+        ]);
+
+        console.log('✅ Final response:', response.content);
+    } catch (error) {
+        console.error('❌ Error:', (error as Error).message);
+    }
+
+    // Test custom tool
+    console.log('\n\n📋 Test 4: Custom Tool Registration\n');
+    
+    try {
+        // Register a custom tool
+        const customTool = ToolBuilder.createTool<{ city: string; country?: string }>(
+            'get_city_info',
+            'Get information about a city',
+            {
+                properties: {
+                    city: { type: 'string', description: 'Name of the city' },
+                    country: { type: 'string', description: 'Country the city is in' }
+                },
+                required: ['city']
+            },
+            (args) => ({
+                city: args.city,
+                country: args.country || 'Unknown',
+                population: Math.floor(Math.random() * 10000000) + 100000,
+                weather: 'Sunny',
+                timezone: 'UTC+0',
+                founded: Math.floor(Math.random() * 2000) + 1
+            })
+        );
+
+        ollamaModel.registerTool(
+            customTool.name,
+            customTool.description,
+            customTool.parameters,
+            customTool.handler
+        );
+
+        console.log('🔧 Testing custom tool:');
+        
+        const response = await ollamaModel.chatWithTools([
+            { 
+                role: 'user', 
+                content: 'Can you get information about Paris, France?' 
+            }
+        ]);
+
+        console.log('✅ Custom tool response:', response.content);
+    } catch (error) {
+        console.error('❌ Error:', (error as Error).message);
+    }
+
+    // Test tool choice options
+    console.log('\n\n📋 Test 5: Tool Choice Control\n');
+    
+    try {
+        console.log('🔧 Testing forced tool usage:');
+        
+        const response = await ollamaModel.chat([
+            { role: 'user', content: 'Tell me about the weather today' }
+        ], {}, { 
+            tool_choice: { type: 'function', function: { name: 'get_current_time' } }
+        });
+
+        console.log('✅ Forced tool response:', response.content);
+        if (response.tool_calls) {
+            console.log('🔨 Tool used:', response.tool_calls[0]?.function.name);
+        }
+    } catch (error) {
+        console.error('❌ Error:', (error as Error).message);
+    }
+
+    // Clean up
+    Object.values(models).forEach(model => model.dispose());
+    
+    console.log('\n✅ Tool calling tests completed!');
+}
+
+// Add error handling for the main test
+async function runTests() {
+    try {
+        await testToolCallingLocal();
+    } catch (error) {
+        console.error('💥 Test suite failed:', (error as Error).message);
+        console.error(error);
+    }
+}
+
+// Run the tests
+if (require.main === module) {
+    runTests();
+}
+
+export { testToolCallingLocal };

package/src/test-scripts/test-vllm-qwen36.ts

@@ -0,0 +1,256 @@
+/**
+ * vLLM + Qwen3.6 (NVFP4) compatibility test for universal-llm-client.
+ *
+ * Exercises the OpenAI-compatible provider against a local vLLM server running
+ * nvidia/Qwen3.6-35B-A3B-NVFP4, with special attention to REASONING handling.
+ *
+ * Run (server must be up on :8000):
+ *   bun run src/test-scripts/test-vllm-qwen36.ts
+ *
+ * Env overrides:
+ *   VLLM_URL   (default http://localhost:8000)
+ *   VLLM_MODEL (default qwen3.6-nvfp4)
+ */
+
+import { AIModel } from '../index.js';
+import type { DecodedEvent } from '../stream-decoder.js';
+import type { LLMChatResponse } from '../interfaces.js';
+
+const URL = process.env.VLLM_URL ?? 'http://localhost:8000';
+const MODEL = process.env.VLLM_MODEL ?? 'qwen3.6-nvfp4';
+
+// ---------------------------------------------------------------------------
+// tiny test harness
+// ---------------------------------------------------------------------------
+type Status = 'PASS' | 'FAIL' | 'PARTIAL';
+const results: { name: string; status: Status; note: string }[] = [];
+function record(name: string, status: Status, note = '') {
+    results.push({ name, status, note });
+    const icon = status === 'PASS' ? '✅' : status === 'PARTIAL' ? '🟡' : '❌';
+    console.log(`\n${icon} ${name} — ${status}${note ? `\n   ${note}` : ''}`);
+}
+function section(title: string) {
+    console.log(`\n${'━'.repeat(70)}\n${title}\n${'━'.repeat(70)}`);
+}
+
+/** Drain a chatStream generator, collecting events AND the final return value. */
+async function drainStream(
+    gen: AsyncGenerator<DecodedEvent, LLMChatResponse | void, unknown>,
+): Promise<{ events: DecodedEvent[]; result: LLMChatResponse | void }> {
+    const events: DecodedEvent[] = [];
+    let result: LLMChatResponse | void;
+    while (true) {
+        const { value, done } = await gen.next();
+        if (done) { result = value as LLMChatResponse | void; break; }
+        events.push(value);
+    }
+    return { events, result };
+}
+
+/** Raw OpenAI call straight to vLLM — ground truth for what the server emits. */
+async function rawChat(body: Record<string, unknown>): Promise<any> {
+    const res = await fetch(`${URL}/v1/chat/completions`, {
+        method: 'POST',
+        headers: { 'Content-Type': 'application/json' },
+        body: JSON.stringify({ model: MODEL, ...body }),
+    });
+    return res.json();
+}
+
+const REASON_PROMPT =
+    'A farmer has 17 sheep. All but 9 run away. Then he buys 5 more. How many sheep does he have? Think it through, then give the number.';
+
+// ---------------------------------------------------------------------------
+
+async function main() {
+    console.log(`vLLM compatibility test\n  url   = ${URL}\n  model = ${MODEL}`);
+
+    const model = new AIModel({
+        model: MODEL,
+        thinking: true, // intent flag (no-op for the openai provider, but documents intent)
+        timeout: 120_000,
+        providers: [{ type: 'openai', url: URL, apiKey: 'EMPTY' }],
+    });
+
+    // ----- 1. Connectivity / model discovery --------------------------------
+    section('1. Connectivity & model discovery');
+    try {
+        const models = await model.getModels();
+        console.log('   /v1/models ->', models);
+        if (models.includes(MODEL)) record('Model discovery', 'PASS', `served model "${MODEL}" is listed`);
+        else record('Model discovery', 'PARTIAL', `server reachable but "${MODEL}" not in ${JSON.stringify(models)}`);
+    } catch (e) {
+        record('Model discovery', 'FAIL', `cannot reach server: ${(e as Error).message}`);
+        console.log('\nAborting — server unreachable.');
+        printSummary();
+        return;
+    }
+
+    // ----- 2. Basic chat (non-streaming) ------------------------------------
+    section('2. Basic chat (non-streaming)');
+    try {
+        const r = await model.chat(
+            [{ role: 'user', content: 'In one short sentence, what is the capital of Japan?' }],
+            { temperature: 0, maxTokens: 256 },
+        );
+        const content = r.message.content?.trim() ?? '';
+        console.log('   content :', JSON.stringify(content));
+        console.log('   usage   :', JSON.stringify(r.usage));
+        if (r.usage?.tokensPerSecond) {
+            console.log(`   stats   : ${r.usage.tokensPerSecond.toFixed(1)} tok/s over ${r.usage.durationMs}ms wall-clock`);
+        }
+        if (content.toLowerCase().includes('tokyo')) record('Basic chat', 'PASS', 'correct, clean answer (reasoning stripped server-side)');
+        else if (content.length > 0) record('Basic chat', 'PARTIAL', 'got content but expected "Tokyo"');
+        else record('Basic chat', 'FAIL', 'empty content (model spent budget reasoning — see reasoning section)');
+    } catch (e) {
+        record('Basic chat', 'FAIL', (e as Error).message);
+    }
+
+    // ----- 3. Streaming -----------------------------------------------------
+    section('3. Streaming (chatStream)');
+    try {
+        const { events, result } = await drainStream(
+            model.chatStream(
+                [{ role: 'user', content: 'List three colors, comma separated.' }],
+                // Generous budget: Qwen3.6 thinks first, so a small cap is spent
+                // entirely on reasoning before any answer tokens are produced.
+                { temperature: 0, maxTokens: 1024 },
+            ),
+        );
+        const textEvents = events.filter(e => e.type === 'text').length;
+        const thinkingEvents = events.filter(e => e.type === 'thinking').length;
+        const finalContent = (result && 'message' in result ? result.message.content : '')?.trim() ?? '';
+        console.log(`   events: ${events.length} (text=${textEvents}, thinking=${thinkingEvents})`);
+        console.log('   final content:', JSON.stringify(finalContent));
+        if (textEvents > 0 && finalContent.length > 0) record('Streaming', 'PASS', `${textEvents} text deltas streamed, final content assembled`);
+        else record('Streaming', 'PARTIAL', 'stream completed but content was empty');
+    } catch (e) {
+        record('Streaming', 'FAIL', (e as Error).message);
+    }
+
+    // ----- 4. REASONING (the focus) -----------------------------------------
+    section('4. Reasoning exposure (Qwen3.6 thinking)');
+    try {
+        // 4a. Ground truth: what does vLLM actually send?
+        const raw = await rawChat({
+            messages: [{ role: 'user', content: REASON_PROMPT }],
+            max_tokens: 800,
+            temperature: 0,
+        });
+        const rawMsg = raw?.choices?.[0]?.message ?? {};
+        const serverReasoning: string = rawMsg.reasoning ?? rawMsg.reasoning_content ?? '';
+        const serverContent: string = rawMsg.content ?? '';
+        console.log(`   [server raw] reasoning field: ${serverReasoning.length} chars; content field: ${serverContent.length} chars`);
+        if (serverReasoning) console.log(`   [server raw] reasoning preview: ${JSON.stringify(serverReasoning.slice(0, 120))}…`);
+        console.log(`   [server raw] content        : ${JSON.stringify(serverContent.slice(0, 120))}`);
+
+        // 4b. What the client surfaces (non-streaming)
+        const r = await model.chat([{ role: 'user', content: REASON_PROMPT }], { temperature: 0, maxTokens: 800 });
+        const clientReasoning = r.reasoning ?? '';
+        const clientContent = r.message.content ?? '';
+        console.log(`   [client chat] .reasoning: ${clientReasoning.length} chars; .content: ${clientContent.length} chars`);
+
+        // 4c. What the client surfaces (streaming — uses StandardChatDecoder <think> parser)
+        const { result } = await drainStream(model.chatStream([{ role: 'user', content: REASON_PROMPT }], { temperature: 0, maxTokens: 800 }));
+        const streamReasoning = (result && 'reasoning' in result ? result.reasoning : '') ?? '';
+        console.log(`   [client stream] .reasoning: ${streamReasoning.length} chars`);
+
+        const serverHasReasoning = serverReasoning.length > 50;
+        const clientExposes = clientReasoning.length > 0 || streamReasoning.length > 0;
+        const contentClean = !clientContent.includes('<think>');
+
+        if (serverHasReasoning && clientExposes) {
+            record('Reasoning exposure', 'PASS', 'client surfaces the reasoning trace via .reasoning');
+        } else if (serverHasReasoning && !clientExposes && contentClean) {
+            record('Reasoning exposure', 'PARTIAL',
+                'Server emits reasoning in a separate `reasoning` field; client returns CLEAN answers but does NOT expose the trace ' +
+                '(the openai provider reads `content`/`delta.content` only, never `reasoning`/`reasoning_content`). ' +
+                'Fix: read `message.reasoning`/`delta.reasoning` in providers/openai.ts.');
+        } else if (clientContent.includes('<think>')) {
+            record('Reasoning exposure', 'PARTIAL', 'Reasoning leaks into content as <think> tags (run server WITHOUT --reasoning-parser, then streaming separates it).');
+        } else {
+            record('Reasoning exposure', 'FAIL', 'No reasoning surfaced anywhere.');
+        }
+    } catch (e) {
+        record('Reasoning exposure', 'FAIL', (e as Error).message);
+    }
+
+    // ----- 5. Tool calling --------------------------------------------------
+    section('5. Tool calling (chatWithTools)');
+    try {
+        let toolHit = false;
+        model.registerTool(
+            'multiply',
+            'Multiply two integers and return the product',
+            { type: 'object', properties: { a: { type: 'number' }, b: { type: 'number' } }, required: ['a', 'b'] },
+            async (args: any) => { toolHit = true; return { product: args.a * args.b }; },
+        );
+        const r = await model.chatWithTools(
+            [{ role: 'user', content: 'Use the multiply tool to compute 17 times 23, then state the result.' }],
+            { temperature: 0, maxTokens: 1024, maxIterations: 3 },
+        );
+        const trace = r.toolExecutions ?? [];
+        const content = r.message.content ?? '';
+        console.log('   toolExecutions:', JSON.stringify(trace));
+        console.log('   content  :', JSON.stringify(content.slice(0, 160)));
+        if (toolHit && content.includes('391')) record('Tool calling', 'PASS', `tool executed (${trace.length} trace entr${trace.length === 1 ? 'y' : 'ies'}), answer 391 returned`);
+        else if (toolHit) record('Tool calling', 'PARTIAL', 'tool fired but final answer missing 391');
+        else record('Tool calling', 'PARTIAL', 'tool NOT invoked — vLLM likely needs `--enable-auto-tool-choice --tool-call-parser hermes`');
+    } catch (e) {
+        record('Tool calling', 'FAIL', (e as Error).message);
+    }
+
+    // ----- 6. Structured output (JSON schema / guided decoding) --------------
+    section('6. Structured output (response_format json_schema)');
+    try {
+        const r = await model.chat(
+            [{ role: 'user', content: 'Give the capital and population (millions, integer) of France.' }],
+            {
+                temperature: 0,
+                maxTokens: 1024,
+                // Unified thinking flag — now wired through the openai provider to
+                // vLLM's chat_template_kwargs.enable_thinking. Disable thinking so
+                // guided decoding emits the object directly.
+                thinking: false,
+                jsonSchema: {
+                    type: 'object',
+                    properties: { capital: { type: 'string' }, population_millions: { type: 'number' } },
+                    required: ['capital', 'population_millions'],
+                    additionalProperties: false,
+                },
+                name: 'CountryFact',
+            } as any,
+        );
+        const content = r.message.content ?? '';
+        const structured = (r as any).structured;
+        console.log('   content   :', JSON.stringify(content.slice(0, 200)));
+        console.log('   structured:', JSON.stringify(structured));
+        let parsed: any = structured;
+        if (!parsed) { try { parsed = JSON.parse(content); } catch { /* ignore */ } }
+        if (parsed && typeof parsed.capital === 'string' && typeof parsed.population_millions === 'number') {
+            record('Structured output', 'PASS', `valid JSON: capital=${parsed.capital}`);
+        } else if (parsed) {
+            record('Structured output', 'PARTIAL', 'JSON parsed but schema fields missing/mistyped');
+        } else {
+            record('Structured output', 'FAIL', 'response was not valid JSON');
+        }
+    } catch (e) {
+        record('Structured output', 'FAIL', (e as Error).message);
+    }
+
+    await model.dispose();
+    printSummary();
+}
+
+function printSummary() {
+    section('SUMMARY');
+    const pad = Math.max(...results.map(r => r.name.length));
+    for (const r of results) {
+        const icon = r.status === 'PASS' ? '✅' : r.status === 'PARTIAL' ? '🟡' : '❌';
+        console.log(`${icon} ${r.name.padEnd(pad)}  ${r.status}`);
+    }
+    const pass = results.filter(r => r.status === 'PASS').length;
+    console.log(`\n${pass}/${results.length} PASS`);
+}
+
+main().catch(e => { console.error('FATAL', e); process.exit(1); });