universal-llm-client 4.5.0 → 4.5.1

package/src/demos/diffusion-gemma/docker-compose.yml

@@ -1,29 +0,0 @@
-services:
-  diffusiongemma:
-    container_name: diffusiongemma
-    image: ${VLLM_IMAGE:-vllm/vllm-openai:gemma}
-    entrypoint: ["bash", "-lc", "/start-vllm.sh"]
-    ipc: host
-    shm_size: 2gb
-    ports:
-      - "${VLLM_PORT:-8000}:8000"
-    environment:
-      MODEL_NAME: ${MODEL_NAME:-RedHatAI/diffusiongemma-26B-A4B-it-NVFP4}
-      GPU_MEM_UTIL: ${GPU_MEM_UTIL:-0.28}
-      MAX_MODEL_LEN: ${MAX_MODEL_LEN:-32768}
-      MAX_NUM_SEQS: ${MAX_NUM_SEQS:-1}
-      DIFFUSION_ENTROPY: ${DIFFUSION_ENTROPY:-0.1}
-      ENFORCE_EAGER: ${ENFORCE_EAGER:-0}
-      VLLM_NO_USAGE_STATS: ${VLLM_NO_USAGE_STATS:-1}
-      NVIDIA_VISIBLE_DEVICES: ${NVIDIA_VISIBLE_DEVICES:-all}
-      NVIDIA_DRIVER_CAPABILITIES: compute,utility
-    volumes:
-      - ./start-vllm.sh:/start-vllm.sh:ro
-      - ./.cache/huggingface:/root/.cache/huggingface
-    deploy:
-      resources:
-        reservations:
-          devices:
-            - driver: nvidia
-              count: all
-              capabilities: [gpu]

package/src/demos/diffusion-gemma/probe-stream.ts

@@ -1,51 +0,0 @@
-/**
- * Probe the raw vLLM SSE stream to understand chunk arrival patterns.
- * Logs: chunk index, ms since start, gap since last chunk, content length, field, preview.
- */
-
-const res = await fetch('http://localhost:3333/api/stream-raw', {
-    method: 'POST',
-    headers: { 'Content-Type': 'application/json' },
-    body: JSON.stringify({
-        prompt: 'Write a short poem about the stars at night.',
-        maxTokens: 512,
-    }),
-});
-
-if (!res.ok || !res.body) {
-    console.error('HTTP', res.status, res.statusText);
-    process.exit(1);
-}
-
-const reader = res.body.getReader();
-const decoder = new TextDecoder();
-let buffer = '';
-const t0 = performance.now();
-let last = t0;
-let i = 0;
-let total = 0;
-
-while (true) {
-    const { done, value } = await reader.read();
-    if (done) break;
-    buffer += decoder.decode(value, { stream: true });
-    const lines = buffer.split('\n');
-    buffer = lines.pop() ?? '';
-    for (const line of lines) {
-        if (!line.startsWith('data: ') || line === 'data: [DONE]') continue;
-        let chunk: any;
-        try { chunk = JSON.parse(line.slice(6)); } catch { continue; }
-        const delta = chunk.choices?.[0]?.delta;
-        if (!delta) continue;
-        const field = delta.reasoning_content != null ? 'reasoning' : delta.content != null ? 'content' : '?';
-        const text: string = delta.reasoning_content ?? delta.content ?? '';
-        const now = performance.now();
-        total += text.length;
-        const extraKeys = Object.keys(delta).filter(k => !['content', 'reasoning_content', 'role', 'tool_calls'].includes(k));
-        console.log(
-            `#${String(i++).padStart(3)}  t=${(now - t0).toFixed(0).padStart(6)}ms  gap=${(now - last).toFixed(1).padStart(7)}ms  len=${String(text.length).padStart(4)}  ${field.padEnd(9)}  ${JSON.stringify(text.slice(0, 60))}${extraKeys.length ? '  extra=' + JSON.stringify(extraKeys) : ''}`,
-        );
-        last = now;
-    }
-}
-console.log(`\nTotal: ${i} chunks, ${total} chars, ${(performance.now() - t0).toFixed(0)}ms`);

package/src/demos/diffusion-gemma/probe-tools.ts

@@ -1,55 +0,0 @@
-/**
- * Probe the full native tool-calling loop against vLLM, step by step,
- * printing raw wire content (skip_special_tokens: false throughout).
- */
-
-const MODEL = process.env.MODEL_NAME ?? 'RedHatAI/diffusiongemma-26B-A4B-it-NVFP4';
-const VLLM = process.env.VLLM_URL ?? 'http://localhost:8000';
-
-const tools = [{
-    type: 'function',
-    function: {
-        name: 'get_weather',
-        description: 'Get current weather for a city',
-        parameters: {
-            type: 'object',
-            properties: {
-                city: { type: 'string', description: 'City name' },
-                unit: { type: 'string', enum: ['celsius', 'fahrenheit'] },
-            },
-            required: ['city'],
-        },
-    },
-}];
-
-async function post(messages: unknown[], withTools: boolean): Promise<string> {
-    const res = await fetch(`${VLLM}/v1/chat/completions`, {
-        method: 'POST',
-        headers: { 'Content-Type': 'application/json' },
-        body: JSON.stringify({
-            model: MODEL,
-            messages,
-            max_tokens: 1024,
-            skip_special_tokens: false,
-            ...(withTools ? { tools, tool_choice: 'none' } : {}),
-        }),
-    });
-    const d = await res.json() as any;
-    return d.choices?.[0]?.message?.content ?? JSON.stringify(d).slice(0, 300);
-}
-
-const followUp = [
-    { role: 'user', content: 'What is the weather in Paris right now, in celsius?' },
-    {
-        role: 'assistant', content: '', tool_calls: [{
-            id: 'call_x', type: 'function',
-            function: { name: 'get_weather', arguments: JSON.stringify({ city: 'Paris', unit: 'celsius' }) },
-        }],
-    },
-    { role: 'tool', tool_call_id: 'call_x', content: JSON.stringify({ temp_c: 18, condition: 'partly cloudy' }) },
-];
-
-console.log('A) follow-up WITH tools+choice none:');
-console.log('   ', JSON.stringify(await post(followUp, true)).slice(0, 500));
-console.log('B) follow-up WITHOUT tools:');
-console.log('   ', JSON.stringify(await post(followUp, false)).slice(0, 500));