universal-llm-client 4.3.0 → 4.5.0

package/src/gemma-diffusion.ts

@@ -0,0 +1,167 @@
+/**
+ * DiffusionGemma (vLLM) native-protocol adapter.
+ *
+ * Trimmed vLLM builds that serve DiffusionGemma ship with NO reasoning parser
+ * and NO tool-call parser module, and they reject OpenAI-style `tools` unless
+ * `--tool-call-parser` is configured. Everything therefore has to be handled
+ * client-side, against the model's native channel format (visible only when
+ * the request sets `skip_special_tokens: false`):
+ *
+ *   <|channel>thought ...reasoning... <channel|>          reasoning channel
+ *   <|tool_call>call:name{k:<|"|>v<|"|>,n:3}<tool_call|>  tool call
+ *
+ * Tool-call arguments are NOT JSON: keys are bare, strings are wrapped in the
+ * <|"|> quote token, numbers/booleans are bare (see the model's
+ * chat_template.jinja `format_argument` macro). `gemmaArgsToJson` converts
+ * that into a standard JSON string.
+ *
+ * Request-side protocol (implemented in the OpenAI provider):
+ *   - always send `skip_special_tokens: false`
+ *   - send `tools` with `tool_choice: 'none'` — vLLM still renders the
+ *     declarations into the chat template, it just skips its (absent) parser
+ *   - send history tool turns structurally (assistant `tool_calls` +
+ *     `role: 'tool'` messages) — the chat template renders them natively
+ */
+
+import { extractGemmaThoughtChannels } from './gemma-channel.js';
+
+export interface GemmaParsedToolCall {
+    readonly name: string;
+    /** JSON-encoded arguments object, ready for LLMToolCall.function.arguments */
+    readonly argumentsJson: string;
+}
+
+export interface GemmaDiffusionParsed {
+    /** Final answer with reasoning, tool-call blocks and special tokens removed */
+    readonly content: string;
+    readonly reasoning: string;
+    readonly toolCalls: readonly GemmaParsedToolCall[];
+}
+
+/** Models that speak this native protocol when served by vLLM. */
+export function isGemmaDiffusionModel(model: string): boolean {
+    return /diffusion[-_]?gemma/i.test(model);
+}
+
+const TOOL_CALL_BLOCK = /<\|tool_call>\s*call:([a-zA-Z0-9_.-]+)\s*\{([\s\S]*?)\}\s*<tool_call\|>/g;
+
+/**
+ * Residual control tokens that may leak into text output — including stray
+ * unbalanced channel markers (the model occasionally emits an extra
+ * <channel|> closer mid-answer).
+ */
+const RESIDUAL_SPECIAL = /<\|?(?:turn|think|image|audio|video|tool_response|tool_call|tool|channel)\b[^>]*?\|?>|<(?:turn|channel|tool_response|tool_call|tool)\|>/g;
+
+const QUOTE_TOKEN = '<|"|>';
+
+/**
+ * Convert the Gemma template's pseudo-JSON argument syntax to a JSON string.
+ * Lenient by design: bare words that aren't numbers/booleans become strings,
+ * since the model occasionally omits the quote token.
+ */
+export function gemmaArgsToJson(body: string): string {
+    // Argument bodies arrive without their outer braces (the regex strips them)
+    const src = `{${body}}`;
+    let i = 0;
+    const n = src.length;
+
+    function skipWs(): void {
+        while (i < n && /\s/.test(src[i]!)) i++;
+    }
+
+    function parseQuoted(): string {
+        // positioned at the start of QUOTE_TOKEN
+        i += QUOTE_TOKEN.length;
+        const end = src.indexOf(QUOTE_TOKEN, i);
+        const raw = end === -1 ? src.slice(i) : src.slice(i, end);
+        i = end === -1 ? n : end + QUOTE_TOKEN.length;
+        return raw;
+    }
+
+    function parseBare(stops: string): string {
+        const start = i;
+        while (i < n && !stops.includes(src[i]!) && !src.startsWith(QUOTE_TOKEN, i)) i++;
+        return src.slice(start, i).trim();
+    }
+
+    function parseValue(): string {
+        skipWs();
+        if (src.startsWith(QUOTE_TOKEN, i)) return JSON.stringify(parseQuoted());
+        const c = src[i];
+        if (c === '{') return parseObject();
+        if (c === '[') return parseArray();
+        const bare = parseBare(',}]');
+        if (/^-?\d+(\.\d+)?([eE][+-]?\d+)?$/.test(bare)) return bare;
+        if (bare === 'true' || bare === 'false' || bare === 'null') return bare;
+        return JSON.stringify(bare);
+    }
+
+    function parseObject(): string {
+        i++; // consume {
+        const parts: string[] = [];
+        skipWs();
+        while (i < n && src[i] !== '}') {
+            skipWs();
+            const key = src.startsWith(QUOTE_TOKEN, i) ? parseQuoted() : parseBare(':');
+            skipWs();
+            if (src[i] === ':') i++;
+            const value = parseValue();
+            parts.push(`${JSON.stringify(key.trim())}:${value}`);
+            skipWs();
+            if (src[i] === ',') i++;
+            skipWs();
+        }
+        i++; // consume }
+        return `{${parts.join(',')}}`;
+    }
+
+    function parseArray(): string {
+        i++; // consume [
+        const parts: string[] = [];
+        skipWs();
+        while (i < n && src[i] !== ']') {
+            parts.push(parseValue());
+            skipWs();
+            if (src[i] === ',') i++;
+            skipWs();
+        }
+        i++; // consume ]
+        return `[${parts.join(',')}]`;
+    }
+
+    skipWs();
+    return parseObject();
+}
+
+/**
+ * Parse a complete raw DiffusionGemma output into reasoning, tool calls and
+ * clean answer text.
+ */
+export function parseGemmaDiffusionOutput(raw: string): GemmaDiffusionParsed {
+    if (!raw) return { content: raw, reasoning: '', toolCalls: [] };
+
+    const toolCalls: GemmaParsedToolCall[] = [];
+    let text = raw.replace(TOOL_CALL_BLOCK, (_m, name: string, args: string) => {
+        toolCalls.push({ name, argumentsJson: gemmaArgsToJson(args) });
+        return '';
+    });
+
+    const channels = extractGemmaThoughtChannels(text);
+    text = channels.content;
+
+    // Unterminated thought channel (model hit max_tokens mid-reasoning)
+    let reasoning = channels.reasoning;
+    const danglingThought = text.match(/<\|channel>\s*thought\s*\r?\n?([\s\S]*)$/i);
+    if (danglingThought) {
+        reasoning = reasoning ? `${reasoning}\n\n${danglingThought[1]!.trim()}` : danglingThought[1]!.trim();
+        text = text.slice(0, danglingThought.index);
+    }
+
+    text = text.replace(RESIDUAL_SPECIAL, '');
+
+    return {
+        content: text.trim(),
+        reasoning,
+        toolCalls,
+    };
+}

package/src/http.ts

@@ -0,0 +1,261 @@
+/**
+ * Universal LLM Client v3 — HTTP Utilities
+ *
+ * Zero-dependency HTTP layer using native fetch.
+ * Works on Node 22+, Bun, Deno, and browsers.
+ */
+
+import type { LLMClientOptions } from './interfaces.js';
+
+// ============================================================================
+// Types
+// ============================================================================
+
+export interface HttpRequestOptions {
+    method?: 'GET' | 'POST';
+    headers?: Record<string, string>;
+    body?: unknown;
+    timeout?: number;
+    signal?: AbortSignal;
+}
+
+export interface HttpResponse<T = unknown> {
+    ok: boolean;
+    status: number;
+    data: T;
+    /** Response headers (when available) */
+    headers?: Headers;
+}
+
+// ============================================================================
+// HTTP Request
+// ============================================================================
+
+/**
+ * Make an HTTP request with timeout and error handling.
+ * Uses native fetch (available in all target runtimes).
+ */
+export async function httpRequest<T = unknown>(
+    url: string,
+    options: HttpRequestOptions = {},
+): Promise<HttpResponse<T>> {
+    const { method = 'GET', headers = {}, body, timeout = 30000, signal } = options;
+
+    const controller = new AbortController();
+    const timeoutId = setTimeout(() => controller.abort(), timeout);
+
+    // Combine external signal with timeout
+    const combinedSignal = signal
+        ? AbortSignal.any([signal, controller.signal])
+        : controller.signal;
+
+    try {
+        const response = await fetch(url, {
+            method,
+            headers: {
+                'Content-Type': 'application/json',
+                ...headers,
+            },
+            body: body ? JSON.stringify(body) : undefined,
+            signal: combinedSignal,
+        });
+
+        clearTimeout(timeoutId);
+
+        if (!response.ok) {
+            const errorText = await response.text().catch(() => 'Unknown error');
+            throw new Error(`HTTP ${response.status}: ${errorText}`);
+        }
+
+        const data = (await response.json()) as T;
+
+        return {
+            ok: response.ok,
+            status: response.status,
+            data,
+            headers: response.headers,
+        };
+    } catch (error) {
+        clearTimeout(timeoutId);
+
+        if (error instanceof Error && error.name === 'AbortError') {
+            throw new Error(`Request timeout after ${timeout}ms: ${url}`);
+        }
+        throw error;
+    }
+}
+
+// ============================================================================
+// Streaming HTTP
+// ============================================================================
+
+/**
+ * Make a streaming HTTP request.
+ * Yields raw string chunks as they arrive.
+ */
+export async function* httpStream(
+    url: string,
+    options: HttpRequestOptions = {},
+): AsyncGenerator<string, void, unknown> {
+    const { method = 'POST', headers = {}, body, timeout = 120000, signal } = options;
+
+    const controller = new AbortController();
+    const timeoutId = setTimeout(() => controller.abort(), timeout);
+
+    const combinedSignal = signal
+        ? AbortSignal.any([signal, controller.signal])
+        : controller.signal;
+
+    try {
+        const response = await fetch(url, {
+            method,
+            headers: {
+                'Content-Type': 'application/json',
+                ...headers,
+            },
+            body: body ? JSON.stringify(body) : undefined,
+            signal: combinedSignal,
+        });
+
+        clearTimeout(timeoutId);
+
+        if (!response.ok) {
+            const errorText = await response.text().catch(() => 'Unknown error');
+            throw new Error(`HTTP ${response.status}: ${errorText}`);
+        }
+
+        if (!response.body) {
+            throw new Error('No response body for streaming');
+        }
+
+        const reader = response.body.getReader();
+        const decoder = new TextDecoder();
+
+        try {
+            while (true) {
+                const { done, value } = await reader.read();
+                if (done) break;
+                yield decoder.decode(value, { stream: true });
+            }
+        } finally {
+            reader.releaseLock();
+        }
+    } catch (error) {
+        clearTimeout(timeoutId);
+
+        if (error instanceof Error && error.name === 'AbortError') {
+            throw new Error(`Stream timeout after ${timeout}ms: ${url}`);
+        }
+        throw error;
+    }
+}
+
+// ============================================================================
+// Protocol Parsers
+// ============================================================================
+
+/**
+ * Parse NDJSON (newline-delimited JSON) stream.
+ * Used by Ollama's streaming API.
+ */
+export async function* parseNDJSON<T = unknown>(
+    stream: AsyncGenerator<string>,
+): AsyncGenerator<T, void, unknown> {
+    let buffer = '';
+
+    for await (const chunk of stream) {
+        buffer += chunk;
+        const lines = buffer.split('\n');
+        buffer = lines.pop() ?? '';
+
+        for (const line of lines) {
+            const trimmed = line.trim();
+            if (!trimmed) continue;
+
+            try {
+                yield JSON.parse(trimmed) as T;
+            } catch {
+                // Skip invalid JSON lines
+            }
+        }
+    }
+
+    // Handle remaining buffer
+    if (buffer.trim()) {
+        try {
+            yield JSON.parse(buffer) as T;
+        } catch {
+            // Skip invalid JSON
+        }
+    }
+}
+
+/**
+ * Parse Server-Sent Events stream.
+ * Used by OpenAI-compatible APIs and LlamaCpp/vLLM.
+ */
+export async function* parseSSE(
+    stream: AsyncGenerator<string>,
+): AsyncGenerator<{ event?: string; data: string }, void, unknown> {
+    let buffer = '';
+
+    for await (const chunk of stream) {
+        buffer += chunk;
+
+        // Split by double newlines (SSE event delimiter)
+        const events = buffer.split('\n\n');
+        buffer = events.pop() ?? '';
+
+        for (const event of events) {
+            const lines = event.split('\n');
+            let eventType: string | undefined;
+            const dataLines: string[] = [];
+
+            for (const line of lines) {
+                if (line.startsWith('event:')) {
+                    eventType = line.slice(6).trim();
+                } else if (line.startsWith('data:')) {
+                    dataLines.push(line.slice(5).trim());
+                }
+            }
+
+            const data = dataLines.join('\n');
+            if (data && data !== '[DONE]') {
+                yield { event: eventType, data };
+            }
+        }
+    }
+}
+
+// ============================================================================
+// Header Utilities
+// ============================================================================
+
+/**
+ * Build standard headers for LLM API requests.
+ * Merges any provider-specific extraHeaders (from ProviderConfig) on top.
+ * Provider clients can still fully override (e.g. Anthropic uses x-api-key).
+ *
+ * Respects authHeader / authPrefix from config for Azure-style or gateway auth.
+ */
+export function buildHeaders(options: LLMClientOptions): Record<string, string> {
+    const headers: Record<string, string> = {
+        'Content-Type': 'application/json',
+    };
+
+    if (options.apiKey) {
+        const headerName = options.authHeader || 'Authorization';
+        // Sensible default prefix: Bearer for Authorization, nothing for api-key / x-api-key etc.
+        const defaultPrefix = headerName.toLowerCase() === 'authorization' ? 'Bearer ' : '';
+        const prefix = options.authPrefix !== undefined ? options.authPrefix : defaultPrefix;
+        headers[headerName] = `${prefix}${options.apiKey}`.trim();
+    }
+
+    // Merge provider-specific extras (e.g. Azure 'api-key', custom gateway headers).
+    // Later entries win on conflicts, allowing complete override of auth.
+    if (options.extraHeaders) {
+        Object.assign(headers, options.extraHeaders);
+    }
+
+    return headers;
+}

package/src/index.ts

@@ -0,0 +1,180 @@
+/**
+ * Universal LLM Client v3
+ *
+ * A universal LLM client with transparent provider failover,
+ * streaming tool execution, pluggable reasoning, and native observability.
+ *
+ * @module universal-llm-client
+ */
+
+// ============================================================================
+// Public API — The Universal Client
+// ============================================================================
+
+export { AIModel } from './ai-model.js';
+
+// ============================================================================
+// Types & Interfaces
+// ============================================================================
+
+export {
+    // Enums
+    AIModelApiType,
+    AIModelType,
+    // Config
+    type AIModelConfig,
+    type ProviderConfig,
+    type LLMClientOptions,
+    // Messages
+    type LLMChatMessage,
+    type LLMMessageContent,
+    type LLMContentPart,
+    type LLMTextContent,
+    type LLMImageContent,
+    type LLMAudioContent,
+    // Responses
+    type LLMChatResponse,
+    type TokenUsageInfo,
+    // Tools
+    type LLMToolCall,
+    type LLMToolDefinition,
+    type LLMFunction,
+    type ToolHandler,
+    type ToolExecutionResult,
+    type ToolRegistry,
+    type ToolRegistryEntry,
+    // Options
+    type ChatOptions,
+    type ResponseFormat,
+    type OutputOptions,
+    // Thinking / reasoning
+    type ThinkingLevel,
+    // Deep Research (Gemini)
+    type DeepResearchOptions,
+    type DeepResearchResult,
+    type DeepResearchStep,
+    type DeepResearchEvent,
+    // Model info
+    type ModelMetadata,
+    // Helpers
+    textContent,
+    imageContent,
+    multimodalMessage,
+    extractTextContent,
+    hasImages,
+    audioContent,
+    hasAudio,
+} from './interfaces.js';
+
+// ============================================================================
+// Observability
+// ============================================================================
+
+export {
+    type Auditor,
+    type AuditEvent,
+    type AuditEventType,
+    NoopAuditor,
+    ConsoleAuditor,
+    BufferedAuditor,
+} from './auditor.js';
+
+// ============================================================================
+// Stream Decoding
+// ============================================================================
+
+export {
+    type StreamDecoder,
+    type DecodedEvent,
+    type DecoderCallback,
+    type DecoderType,
+    type DecoderOptions,
+    type DecoderFactory,
+    createDecoder,
+    registerDecoder,
+    getRegisteredDecoders,
+    PassthroughDecoder,
+    StandardChatDecoder,
+    InterleavedReasoningDecoder,
+} from './stream-decoder.js';
+
+// ============================================================================
+// Tool Utilities
+// ============================================================================
+
+export {
+    ToolBuilder,
+    ToolExecutor,
+    createTimeTool,
+    createRandomNumberTool,
+} from './tools.js';
+
+// ============================================================================
+// HTTP Utilities (for advanced use cases)
+// ============================================================================
+
+export {
+    httpRequest,
+    httpStream,
+    parseNDJSON,
+    parseSSE,
+    buildHeaders,
+    type HttpRequestOptions,
+    type HttpResponse,
+} from './http.js';
+
+// ============================================================================
+// DiffusionGemma Native Protocol (vLLM without server-side parsers)
+// ============================================================================
+
+export {
+    isGemmaDiffusionModel,
+    parseGemmaDiffusionOutput,
+    gemmaArgsToJson,
+    type GemmaDiffusionParsed,
+    type GemmaParsedToolCall,
+} from './gemma-diffusion.js';
+
+// ============================================================================
+// MCP Integration
+// ============================================================================
+
+export {
+    MCPToolBridge,
+    type MCPBridgeConfig,
+    type MCPServerConfig,
+    type MCPTool,
+} from './mcp.js';
+
+// ============================================================================
+// Structured Output
+// ============================================================================
+
+export {
+    StructuredOutputError,
+    type StructuredOutputErrorOptions,
+    type StructuredOutputOptions,
+    type StructuredOutputResult,
+    type StructuredOutputSuccess,
+    type StructuredOutputFailure,
+    type JSONSchema,
+    type SchemaProvider,
+    type ProviderSchema,
+    type SchemaConfig,
+    isStructuredOutputSuccess,
+    isStructuredOutputFailure,
+    // Schema conversion utilities
+    normalizeJsonSchema,
+    convertToProviderSchema,
+    stripUnsupportedFeatures,
+    getJsonSchema,
+    getJsonSchemaFromConfig,
+    // Validation functions
+    parseStructured,
+    tryParseStructured,
+    validateStructuredOutput,
+    stripJsonFences,
+    // Streaming parser
+    StreamingJsonParser,
+    type StreamingStructuredResult,
+} from './structured-output.js';