universal-llm-client 4.2.0 → 4.5.0

package/src/debug/debug-google-streaming.ts

@@ -6,7 +6,7 @@ import {request} from 'undici';
 async function debugGoogleStreaming() {
     console.log('🔍 Debugging Google Generative AI Streaming...\n');
 
-    const apiKey = 'AIzaSyBDbo7iVNEuCcRNTgDIgRrkGpFKisXXnm0';
+    const apiKey = (process.env.GOOGLE_API_KEY ?? '');
     const model = 'gemma-3-4b-it';
     const endpoint = `https://generativelanguage.googleapis.com/v1beta/models/${model}:streamGenerateContent`;
 

package/src/demos/basic/universal-llm-examples.ts

@@ -25,7 +25,7 @@ export async function createAIApplicationExample() {
         },
         google: {
             chatModel: 'gemma-3-4b-it',
-            apiKey: 'AIzaSyBDbo7iVNEuCcRNTgDIgRrkGpFKisXXnm0'
+            apiKey: (process.env.GOOGLE_API_KEY ?? '')
         }
     });
 
@@ -36,7 +36,7 @@ export async function createAIApplicationExample() {
     // Method 3: Google-specific setup
     const googleChatModel = AIModelFactory.createGoogleChatModel(
         'gemma-3-4b-it',
-        'AIzaSyBDbo7iVNEuCcRNTgDIgRrkGpFKisXXnm0'
+        (process.env.GOOGLE_API_KEY ?? '')
     );
 
     // Example usage patterns:
@@ -113,7 +113,7 @@ export async function testGoogleAPI() {
 
     const googleModel = AIModelFactory.createGoogleChatModel(
         'gemma-3-4b-it',
-        'AIzaSyBDbo7iVNEuCcRNTgDIgRrkGpFKisXXnm0'
+        (process.env.GOOGLE_API_KEY ?? '')
     );
 
     try {

package/src/demos/diffusion-gemma/.env

@@ -0,0 +1,29 @@
+# Optional docker compose overrides for the DiffusionGemma vLLM backend.
+#
+# Start from packages/universal-llm-client:
+#   docker compose --env-file src/demos/diffusion-gemma/.env -f src/demos/diffusion-gemma/docker-compose.yml up -d
+
+# Public vLLM image to run. If a future nightly regresses DiffusionGemma support,
+# set this to a known-good local or registry tag.
+VLLM_IMAGE=vllm/vllm-openai:gemma
+
+# Host port for the OpenAI-compatible vLLM API.
+VLLM_PORT=18000
+
+VLLM_URL=http://localhost:18000
+
+# DiffusionGemma model served by vLLM.
+MODEL_NAME=RedHatAI/diffusiongemma-26B-A4B-it-NVFP4
+
+# Single-user local serving defaults. Tune for your GPU.
+GPU_MEM_UTIL=0.28
+MAX_MODEL_LEN=32768
+MAX_NUM_SEQS=1
+DIFFUSION_ENTROPY=0.1
+
+# Set to 1 only for CUDA graph / torch.compile debugging.
+ENFORCE_EAGER=0
+
+# Disable vLLM telemetry. In WSL this avoids a py-cpuinfo JSONDecodeError in
+# vLLM's background usage-reporting thread during engine startup/reload.
+VLLM_NO_USAGE_STATS=1

package/src/demos/diffusion-gemma/.env.example

@@ -0,0 +1,27 @@
+# Optional docker compose overrides for the DiffusionGemma vLLM backend.
+#
+# Start from packages/universal-llm-client:
+#   docker compose --env-file src/demos/diffusion-gemma/.env -f src/demos/diffusion-gemma/docker-compose.yml up -d
+
+# Public vLLM image to run. If a future nightly regresses DiffusionGemma support,
+# set this to a known-good local or registry tag.
+VLLM_IMAGE=vllm/vllm-openai:gemma
+
+# Host port for the OpenAI-compatible vLLM API.
+VLLM_PORT=8000
+
+# DiffusionGemma model served by vLLM.
+MODEL_NAME=RedHatAI/diffusiongemma-26B-A4B-it-NVFP4
+
+# Single-user local serving defaults. Tune for your GPU.
+GPU_MEM_UTIL=0.28
+MAX_MODEL_LEN=32768
+MAX_NUM_SEQS=1
+DIFFUSION_ENTROPY=0.1
+
+# Set to 1 only for CUDA graph / torch.compile debugging.
+ENFORCE_EAGER=0
+
+# Disable vLLM telemetry. In WSL this avoids a py-cpuinfo JSONDecodeError in
+# vLLM's background usage-reporting thread during engine startup/reload.
+VLLM_NO_USAGE_STATS=1

package/src/demos/diffusion-gemma/CLAUDE.md

@@ -0,0 +1,95 @@
+# DiffusionGemma demo — test harness + "Signal from Noise" canvas
+
+Standalone Bun server exercising `universal-llm-client` against DiffusionGemma
+(a discrete diffusion LM served by vLLM).
+
+## Run
+
+```bash
+bun run demo:diffusion-gemma:engine  # starts vLLM via demo-local docker compose
+bun run demo:diffusion-gemma         # starts the Bun demo server
+```
+
+- Demo server: **http://localhost:3333** (`/` test harness, `/canvas` diffusion chat UI)
+- vLLM upstream: `VLLM_URL` env, default `http://localhost:8000`
+- Model: `MODEL_NAME` env, default `RedHatAI/diffusiongemma-26B-A4B-it-NVFP4`
+- vLLM is started via `src/demos/diffusion-gemma/docker-compose.yml` and
+  `src/demos/diffusion-gemma/start-vllm.sh` — includes a WSL2 UVA patch and
+  the `entropy_bound` diffusion sampler. Runs as docker container
+  `diffusiongemma` (script is bind-mounted as `/start-vllm.sh`, so edits apply
+  on `docker restart diffusiongemma`). The script also sources
+  `src/demos/diffusion-gemma/.cache/huggingface/diffusion-env.sh`
+  (host-writable through the HF-cache bind mount) — that's how
+  `/api/engine-config` changes settings without recreating the container.
+- **Tuned for single-user local serving** (env-overridable in the start script):
+  `GPU_MEM_UTIL` (default 0.28 ≈ 27 GiB — without caps vLLM grabbed ~88 GiB:
+  69 GiB KV cache for the native 262k context, measured <0.5% used),
+  `MAX_MODEL_LEN` (32768), `MAX_NUM_SEQS` (1), `DIFFUSION_ENTROPY` (0.1),
+  `ENFORCE_EAGER` (0). Weights are 17.4 GiB.
+- **Never re-add `--enforce-eager` casually:** it disables CUDA graphs AND
+  torch.compile and cost 2.2× throughput (387 → 841 tok/s avg, peak 1002,
+  steady-state ~644 on long runs). Set `ENFORCE_EAGER=1` only to debug
+  WSL2/Blackwell graph-capture issues. Entropy 0.1→0.2 measured ≈ no speed
+  change (745–845 tok/s) — the dial trades quality, not meaningful speed,
+  at these settings.
+
+## Routes
+
+| Route | What |
+| ----- | ---- |
+| `/` | Test harness UI (chat + compatibility tests via universal-llm-client) |
+| `/canvas` | "Signal from Noise" — cinematic chat UI replaying the diffusion process |
+| `/api/chat` | Chat via universal-llm-client (`messages`, `stream`, `maxTokens`, `temperature`) |
+| `/api/stream-raw` | Direct vLLM SSE proxy preserving chunk timing (`messages` or `prompt`, `maxTokens`, `thinking:false` to disable the thought channel). Always sets `skip_special_tokens:false` so channel markers survive. |
+| `/api/engine-config` | GET current entropy; POST `{entropy}` writes the env file + `docker restart`s the engine (~2–4 min; UI polls `/api/health`) |
+| `/api/health` | Pings vLLM `/v1/models` |
+
+## Native protocol (no server-side parsers!)
+
+This vLLM build has **no reasoning parser and no tool-call parser module** —
+request-level `tools` with auto choice 400s. Everything is client-side, against
+the chat template's native markers (visible only with `skip_special_tokens:false`):
+
+- Reasoning: `<|channel>thought\n …<channel|>answer`. The canvas splits this
+  with a streaming state machine (partial markers carried across chunks) and
+  renders reasoning as a collapsible amber channel above the answer surface.
+- **Canvas reading view:** the mono token surface is the animation; when a
+  reply settles it fades into a rendered-markdown view (zero-dep renderer in
+  the inner script — headings/lists/code/bold/links, all input HTML-escaped
+  first; backticks via `String.fromCharCode(96)` because literal backticks
+  would terminate the outer template literal). Replay/scrub swaps back to the
+  token surface. Root font scales with viewport (`clamp` on `html`) for
+  screen-recording legibility. Max-tokens select goes to 16k (default 4k);
+  `finish_reason:'length'` shows an amber "⚠ capped" warning in phase+footer.
+- Tool calls: `<|tool_call>call:name{k:<|"|>v<|"|>,n:3}<tool_call|>` — pseudo-JSON
+  args (bare keys, `<|"|>` quote token). Send `tools` + `tool_choice:'none'`
+  (declarations still get rendered into the template); history tool turns go as
+  standard structured `tool_calls` + `role:'tool'` messages (template renders
+  them natively).
+- All of this is implemented for the library in `src/gemma-diffusion.ts` and
+  wired into the OpenAI provider (auto-detected by model name; override with
+  `LLMClientOptions.gemmaNativeProtocol`). `chatWithTools` works end-to-end.
+  Tests: `src/tests/gemma-diffusion.test.ts`. Probes: `probe-stream.ts`
+  (chunk timing), `probe-tools.ts` (tool-loop wire format).
+
+## Things that bite
+
+- **`canvas.ts` is one giant TS template literal.** Backslash escapes inside the
+  inner `<script>` are eaten by the outer literal (`/\S+/` silently becomes
+  `/S+/`). The inner script is written with ZERO backslashes — newlines via
+  `String.fromCharCode(10)`, tokenizing via charCode scans. Keep it that way.
+- **No hot reload.** `CANVAS_HTML` is bundled at startup — restart the server
+  after editing `canvas.ts` (kill the bun process on :3333, start again).
+- **Don't name a top-level browser var `history`** — `window.history` is
+  unshadowable; the conversation array is called `convo`.
+- **Stream shape (measured):** the vLLM OpenAI stream emits ~1KB bursts, one per
+  finished 256-token diffusion block, every ~0.8–1.2s. There is no per-denoise-step
+  state in the stream; `/canvas` animates each block's reveal during the real
+  compute window of the next block. `probe-stream.ts` logs chunk timing.
+- **The model emits stray unbalanced `<channel|>` closers** occasionally —
+  the parser strips them (`RESIDUAL_SPECIAL` in gemma-diffusion.ts), and it
+  sometimes puts the whole final answer inside the thought channel on
+  post-tool turns.
+- **Entropy is engine-level** (`hf_overrides` read once at model init in
+  vLLM's `diffusion_gemma.py`); per-request `vllm_xargs` is accepted but
+  ignored. Hence the reload-based `/api/engine-config`.

package/src/demos/diffusion-gemma/README.md

@@ -0,0 +1,59 @@
+# DiffusionGemma demo
+
+Standalone Bun demo for testing `universal-llm-client` against DiffusionGemma
+served by vLLM's OpenAI-compatible API.
+
+## Run the backend
+
+From `packages/universal-llm-client`:
+
+```bash
+docker compose -f src/demos/diffusion-gemma/docker-compose.yml up -d
+```
+
+The compose file runs a `diffusiongemma` container on `localhost:8000`, mounts a
+demo-local Hugging Face cache at `src/demos/diffusion-gemma/.cache/huggingface`,
+and bind-mounts `start-vllm.sh` as the container entrypoint.
+
+If you already have an older hand-created `diffusiongemma` container, remove it
+before switching to the demo compose file:
+
+```bash
+docker rm -f diffusiongemma
+```
+
+Optional overrides:
+
+```bash
+cp src/demos/diffusion-gemma/.env.example src/demos/diffusion-gemma/.env
+docker compose --env-file src/demos/diffusion-gemma/.env -f src/demos/diffusion-gemma/docker-compose.yml up -d
+```
+
+Useful knobs are `VLLM_IMAGE`, `GPU_MEM_UTIL`, `MAX_MODEL_LEN`,
+`DIFFUSION_ENTROPY`, `ENFORCE_EAGER`, and `VLLM_NO_USAGE_STATS`.
+
+## Run the demo UI
+
+```bash
+bun run src/demos/diffusion-gemma/server.ts
+```
+
+- Harness: <http://localhost:3333/>
+- Canvas: <http://localhost:3333/canvas>
+- vLLM API: <http://localhost:8000/v1/models>
+
+## Notes
+
+- The prior BentoKit setup did not use a `docker-compose.yml`; it was a direct
+  Docker container using a repo-root `scripts/diffusiongemma-start.sh` bind
+  mount. This demo now carries its own compose file and startup script.
+- The default image is `vllm/vllm-openai:gemma`, the vLLM image line that
+  includes DiffusionGemma support. Set `VLLM_IMAGE` if you need to test another
+  local or registry image.
+- The first startup can take several minutes while vLLM loads and compiles the
+  model. Poll `docker logs -f diffusiongemma` or `/api/health` from the demo UI.
+- The `/api/engine-config` endpoint writes `diffusion-env.sh` into the mounted
+  Hugging Face cache and restarts the `diffusiongemma` container.
+- `VLLM_NO_USAGE_STATS=1` is enabled by default because this vLLM image can hit
+  a non-fatal `py-cpuinfo` `JSONDecodeError` in its background usage-reporting
+  thread under WSL during startup/reload.