universal-llm-client 4.3.0 → 4.5.0

package/src/demos/diffusion-gemma/docker-compose.yml

@@ -0,0 +1,29 @@
+services:
+  diffusiongemma:
+    container_name: diffusiongemma
+    image: ${VLLM_IMAGE:-vllm/vllm-openai:gemma}
+    entrypoint: ["bash", "-lc", "/start-vllm.sh"]
+    ipc: host
+    shm_size: 2gb
+    ports:
+      - "${VLLM_PORT:-8000}:8000"
+    environment:
+      MODEL_NAME: ${MODEL_NAME:-RedHatAI/diffusiongemma-26B-A4B-it-NVFP4}
+      GPU_MEM_UTIL: ${GPU_MEM_UTIL:-0.28}
+      MAX_MODEL_LEN: ${MAX_MODEL_LEN:-32768}
+      MAX_NUM_SEQS: ${MAX_NUM_SEQS:-1}
+      DIFFUSION_ENTROPY: ${DIFFUSION_ENTROPY:-0.1}
+      ENFORCE_EAGER: ${ENFORCE_EAGER:-0}
+      VLLM_NO_USAGE_STATS: ${VLLM_NO_USAGE_STATS:-1}
+      NVIDIA_VISIBLE_DEVICES: ${NVIDIA_VISIBLE_DEVICES:-all}
+      NVIDIA_DRIVER_CAPABILITIES: compute,utility
+    volumes:
+      - ./start-vllm.sh:/start-vllm.sh:ro
+      - ./.cache/huggingface:/root/.cache/huggingface
+    deploy:
+      resources:
+        reservations:
+          devices:
+            - driver: nvidia
+              count: all
+              capabilities: [gpu]

package/src/demos/diffusion-gemma/probe-stream.ts

@@ -0,0 +1,51 @@
+/**
+ * Probe the raw vLLM SSE stream to understand chunk arrival patterns.
+ * Logs: chunk index, ms since start, gap since last chunk, content length, field, preview.
+ */
+
+const res = await fetch('http://localhost:3333/api/stream-raw', {
+    method: 'POST',
+    headers: { 'Content-Type': 'application/json' },
+    body: JSON.stringify({
+        prompt: 'Write a short poem about the stars at night.',
+        maxTokens: 512,
+    }),
+});
+
+if (!res.ok || !res.body) {
+    console.error('HTTP', res.status, res.statusText);
+    process.exit(1);
+}
+
+const reader = res.body.getReader();
+const decoder = new TextDecoder();
+let buffer = '';
+const t0 = performance.now();
+let last = t0;
+let i = 0;
+let total = 0;
+
+while (true) {
+    const { done, value } = await reader.read();
+    if (done) break;
+    buffer += decoder.decode(value, { stream: true });
+    const lines = buffer.split('\n');
+    buffer = lines.pop() ?? '';
+    for (const line of lines) {
+        if (!line.startsWith('data: ') || line === 'data: [DONE]') continue;
+        let chunk: any;
+        try { chunk = JSON.parse(line.slice(6)); } catch { continue; }
+        const delta = chunk.choices?.[0]?.delta;
+        if (!delta) continue;
+        const field = delta.reasoning_content != null ? 'reasoning' : delta.content != null ? 'content' : '?';
+        const text: string = delta.reasoning_content ?? delta.content ?? '';
+        const now = performance.now();
+        total += text.length;
+        const extraKeys = Object.keys(delta).filter(k => !['content', 'reasoning_content', 'role', 'tool_calls'].includes(k));
+        console.log(
+            `#${String(i++).padStart(3)}  t=${(now - t0).toFixed(0).padStart(6)}ms  gap=${(now - last).toFixed(1).padStart(7)}ms  len=${String(text.length).padStart(4)}  ${field.padEnd(9)}  ${JSON.stringify(text.slice(0, 60))}${extraKeys.length ? '  extra=' + JSON.stringify(extraKeys) : ''}`,
+        );
+        last = now;
+    }
+}
+console.log(`\nTotal: ${i} chunks, ${total} chars, ${(performance.now() - t0).toFixed(0)}ms`);

package/src/demos/diffusion-gemma/probe-tools.ts

@@ -0,0 +1,55 @@
+/**
+ * Probe the full native tool-calling loop against vLLM, step by step,
+ * printing raw wire content (skip_special_tokens: false throughout).
+ */
+
+const MODEL = process.env.MODEL_NAME ?? 'RedHatAI/diffusiongemma-26B-A4B-it-NVFP4';
+const VLLM = process.env.VLLM_URL ?? 'http://localhost:8000';
+
+const tools = [{
+    type: 'function',
+    function: {
+        name: 'get_weather',
+        description: 'Get current weather for a city',
+        parameters: {
+            type: 'object',
+            properties: {
+                city: { type: 'string', description: 'City name' },
+                unit: { type: 'string', enum: ['celsius', 'fahrenheit'] },
+            },
+            required: ['city'],
+        },
+    },
+}];
+
+async function post(messages: unknown[], withTools: boolean): Promise<string> {
+    const res = await fetch(`${VLLM}/v1/chat/completions`, {
+        method: 'POST',
+        headers: { 'Content-Type': 'application/json' },
+        body: JSON.stringify({
+            model: MODEL,
+            messages,
+            max_tokens: 1024,
+            skip_special_tokens: false,
+            ...(withTools ? { tools, tool_choice: 'none' } : {}),
+        }),
+    });
+    const d = await res.json() as any;
+    return d.choices?.[0]?.message?.content ?? JSON.stringify(d).slice(0, 300);
+}
+
+const followUp = [
+    { role: 'user', content: 'What is the weather in Paris right now, in celsius?' },
+    {
+        role: 'assistant', content: '', tool_calls: [{
+            id: 'call_x', type: 'function',
+            function: { name: 'get_weather', arguments: JSON.stringify({ city: 'Paris', unit: 'celsius' }) },
+        }],
+    },
+    { role: 'tool', tool_call_id: 'call_x', content: JSON.stringify({ temp_c: 18, condition: 'partly cloudy' }) },
+];
+
+console.log('A) follow-up WITH tools+choice none:');
+console.log('   ', JSON.stringify(await post(followUp, true)).slice(0, 500));
+console.log('B) follow-up WITHOUT tools:');
+console.log('   ', JSON.stringify(await post(followUp, false)).slice(0, 500));