universal-llm-client 4.5.1 → 4.5.2

package/CHANGELOG.md

@@ -5,6 +5,12 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [4.5.2] - 2026-06-28
+
+### Fixed
+
+- **`chat_template_kwargs` no longer sent to hosted OpenAI-compatible gateways** — the unified `thinking` flag mapped to vLLM's `chat_template_kwargs.enable_thinking` for *every* OpenAI-compatible endpoint except official `api.openai.com`. Hosted gateways that expose an OpenAI-compatible surface but validate the request body strictly — **Cerebras**, Groq, Fireworks, Together, Mistral, DeepSeek, OpenRouter, xAI — reject the unknown field with `HTTP 400 ("wrong_api_format")`, which broke every call that set `thinking` (including `thinking: false`). The OpenAI-compatible provider now gates this vLLM/SGLang extension behind a `supportsChatTemplateKwargs(url)` strict-host check (`src/thinking.ts`), so it is only sent to endpoints not on the strict-host list (i.e. self-hosted vLLM/Qwen). Self-hosted servers (localhost / custom hosts) are unaffected. Verified live against Cerebras (`gemma-4-31b`).
+
 ## [4.5.1] - 2026-06-14
 
 ### Added

package/dist/providers/ollama.js

@@ -120,6 +120,17 @@ export class OllamaClient extends BaseLLMClient {
         }
         // Ollama `think` is on/off (no levels); default on for thinking models.
         body['think'] = resolveThinking(options?.thinking, this.options.thinking)?.enabled ?? true;
+        // Handle structured output via format parameter — same as chat(). Without
+        // this, streaming structured output is unconstrained and the model can emit
+        // malformed JSON that only fails at the final parse.
+        const schemaOptions = this.extractSchemaOptions(options);
+        if (schemaOptions) {
+            body['format'] = this.buildFormatParameter(schemaOptions);
+        }
+        else if (options?.responseFormat) {
+            // Legacy json_object mode - map to Ollama's "json" format
+            body['format'] = 'json';
+        }
         const start = Date.now();
         this.auditor.record({
             timestamp: start,

package/dist/providers/openai.js

@@ -5,9 +5,9 @@
  * Works with: OpenAI, OpenRouter, LM Studio, LlamaCpp, vLLM, Groq, Together.
  */
 import { BaseLLMClient } from '../client.js';
-import { resolveThinking, isOpenAIReasoningModel } from '../thinking.js';
+import { resolveThinking, isOpenAIReasoningModel, supportsChatTemplateKwargs } from '../thinking.js';
 import { httpRequest, httpStream, parseSSE, buildHeaders } from '../http.js';
-import { StandardChatDecoder } from '../stream-decoder.js';
+import { createDecoder, StandardChatDecoder } from '../stream-decoder.js';
 import { normalizeJsonSchema, getJsonSchemaFromConfig, } from '../structured-output.js';
 import { isGemmaDiffusionModel, parseGemmaDiffusionOutput } from '../gemma-diffusion.js';
 const VLLM_AUTO_TOOL_CHOICE_HINT = 'vLLM rejected automatic tool choice. Retrying with text-level tool calling. To use native tool_calls, start vLLM with --enable-auto-tool-choice and --tool-call-parser <parser>.';
@@ -409,10 +409,18 @@ export class OpenAICompatibleClient extends BaseLLMClient {
             provider: 'openai',
             model: this.options.model,
         });
-        // In gemma-native mode the decoder classifies thought-channel content,
-        // so we yield ITS events (thinking vs text) instead of the raw deltas.
+        // In gemma-native mode, or when a caller selects an explicit decoder,
+        // the decoder classifies content into typed events instead of exposing
+        // raw deltas with protocol tags still attached.
         const decoderEvents = [];
-        const decoder = new StandardChatDecoder(this.gemmaNative ? e => decoderEvents.push(e) : () => { });
+        const decoderOption = options?.decoder;
+        const decoderInstanceProvided = Boolean(decoderOption && typeof decoderOption === 'object');
+        const yieldDecoderEvents = !decoderInstanceProvided && (this.gemmaNative || typeof decoderOption === 'string');
+        const decoder = typeof decoderOption === 'string'
+            ? createDecoder(decoderOption, e => decoderEvents.push(e))
+            : decoderInstanceProvided
+                ? decoderOption
+                : new StandardChatDecoder(this.gemmaNative ? e => decoderEvents.push(e) : () => { });
         // Track accumulated tool calls across chunks
         const toolCallAccum = new Map();
         let activeBody = body;
@@ -451,7 +459,7 @@ export class OpenAICompatibleClient extends BaseLLMClient {
                         }
                         if (delta.content) {
                             decoder.push(delta.content);
-                            if (this.gemmaNative) {
+                            if (yieldDecoderEvents) {
                                 while (decoderEvents.length)
                                     yield decoderEvents.shift();
                             }
@@ -512,7 +520,7 @@ export class OpenAICompatibleClient extends BaseLLMClient {
             }
         }
         decoder.flush();
-        if (this.gemmaNative) {
+        if (yieldDecoderEvents) {
             while (decoderEvents.length)
                 yield decoderEvents.shift();
         }
@@ -653,16 +661,16 @@ export class OpenAICompatibleClient extends BaseLLMClient {
         // A user-supplied value (via parameters) always wins.
         const thinking = resolveThinking(options?.thinking, this.options.thinking);
         if (thinking) {
-            const isOfficialOpenAI = (this.options.url ?? '').includes('api.openai.com');
             if (isOpenAIReasoningModel(this.options.model)) {
                 if (params['reasoning_effort'] === undefined) {
                     params['reasoning_effort'] = thinking.enabled ? (thinking.level ?? 'medium') : 'minimal';
                 }
             }
-            else if (!isOfficialOpenAI) {
-                // `chat_template_kwargs` is a vLLM/Qwen extension. Official OpenAI
-                // rejects unknown body fields (and gpt-4o has no thinking toggle),
-                // so only send it to self-hosted / compatible gateways.
+            else if (supportsChatTemplateKwargs(this.options.url)) {
+                // `chat_template_kwargs` is a self-hosted vLLM/Qwen extension.
+                // Official OpenAI and hosted OpenAI-compatible gateways (Cerebras,
+                // Groq, Fireworks, …) reject unknown body fields with HTTP 400,
+                // so only send it to endpoints not on the strict-host list.
                 const existing = params['chat_template_kwargs'] ?? {};
                 params['chat_template_kwargs'] = { enable_thinking: thinking.enabled, ...existing };
             }

package/dist/thinking.d.ts

@@ -23,6 +23,18 @@ export interface ResolvedThinking {
 export declare function resolveThinking(perCall: boolean | ThinkingLevel | undefined, config: boolean | ThinkingLevel | undefined): ResolvedThinking | undefined;
 /** Heuristic: OpenAI reasoning models use `reasoning_effort` (o-series, GPT-5). */
 export declare function isOpenAIReasoningModel(model: string): boolean;
+/**
+ * Whether an OpenAI-compatible endpoint accepts the vLLM/SGLang
+ * `chat_template_kwargs` extension (used to toggle `enable_thinking`).
+ *
+ * This is a self-hosted-server feature. Commercial hosted gateways that expose
+ * an OpenAI-compatible surface (OpenAI, Cerebras, Groq, Fireworks, Together,
+ * Mistral, DeepSeek, OpenRouter, …) reject unknown body fields, so we only send
+ * it to endpoints not on the strict list (assumed self-hosted vLLM/Qwen). When
+ * the URL is unknown/empty we default to permissive (self-hosted) behavior to
+ * preserve the prior contract for local servers.
+ */
+export declare function supportsChatTemplateKwargs(url: string | undefined): boolean;
 /**
  * Gemini 2.5 `thinkingBudget` for a level. 0 disables, -1 is dynamic, and the
  * Flash range is 0–24576. A bare `true` (no level) maps to dynamic (-1).

package/dist/thinking.js

@@ -24,6 +24,40 @@ export function resolveThinking(perCall, config) {
 export function isOpenAIReasoningModel(model) {
     return /^(o\d|gpt-5)/i.test(model);
 }
+/**
+ * Hosted, OpenAI-*compatible* gateways that validate the request body strictly
+ * and reject unknown fields with HTTP 400 — just like official OpenAI. The
+ * `chat_template_kwargs` knob (below) is a self-hosted vLLM/SGLang extension and
+ * must NOT be sent to these. Matched as case-insensitive host substrings.
+ */
+const STRICT_OPENAI_COMPAT_HOSTS = [
+    'api.openai.com',
+    'api.cerebras.ai',
+    'api.groq.com',
+    'api.fireworks.ai',
+    'api.together.xyz',
+    'api.together.ai',
+    'api.mistral.ai',
+    'api.deepseek.com',
+    'api.perplexity.ai',
+    'openrouter.ai',
+    'api.x.ai',
+];
+/**
+ * Whether an OpenAI-compatible endpoint accepts the vLLM/SGLang
+ * `chat_template_kwargs` extension (used to toggle `enable_thinking`).
+ *
+ * This is a self-hosted-server feature. Commercial hosted gateways that expose
+ * an OpenAI-compatible surface (OpenAI, Cerebras, Groq, Fireworks, Together,
+ * Mistral, DeepSeek, OpenRouter, …) reject unknown body fields, so we only send
+ * it to endpoints not on the strict list (assumed self-hosted vLLM/Qwen). When
+ * the URL is unknown/empty we default to permissive (self-hosted) behavior to
+ * preserve the prior contract for local servers.
+ */
+export function supportsChatTemplateKwargs(url) {
+    const u = (url ?? '').toLowerCase();
+    return !STRICT_OPENAI_COMPAT_HOSTS.some((host) => u.includes(host));
+}
 /**
  * Gemini 2.5 `thinkingBudget` for a level. 0 disables, -1 is dynamic, and the
  * Flash range is 0–24576. A bare `true` (no level) maps to dynamic (-1).

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "universal-llm-client",
-  "version": "4.5.1",
+  "version": "4.5.2",
   "type": "module",
   "description": "A universal LLM client with transparent provider failover, streaming tool execution, pluggable reasoning, and native observability.",
   "main": "./dist/index.js",