theorum 0.1.15 → 1.1.3

package/esm/src/providers/google/live/framing.js

@@ -0,0 +1,552 @@
+/**
+ * Pure protocol framing & serialization for Google Gemini Live WebSocket API (`BidiGenerateContent`).
+ *
+ * All functions are pure data transformations with no network I/O.
+ *
+ * @module
+ */
+import { TheorumError } from '../../../guardrails/error.js';
+import { groundingFromEvent } from '../../../kernel/engine/delta.js';
+import { isMediaRefPart } from '../../../kernel/interaction-parts.js';
+import { getTool } from '../../../kernel/tools/registry.js';
+import { base64ToBytes, bytesToBase64, wrapPcmAsWav } from '../../shared/pcm.js';
+import { parseToolArgumentsObject } from '../../shared/tool-args.js';
+import { GEMINI_LIVE_WS_URL } from '../urls.js';
+import { toGeminiOpenApiSchema } from './openapi-schema.js';
+/** Construct authenticated WebSocket URL for Gemini Live API. */
+export function buildGeminiLiveWebSocketUrl(apiKey) {
+    return `${GEMINI_LIVE_WS_URL}?key=${encodeURIComponent(apiKey)}`;
+}
+/**
+ * Live tools are always declared non-blocking: the kernel session already runs
+ * tool execution asynchronously and honours `toolCallCancellation`, so the model
+ * is free to keep speaking while a call is in flight.
+ */
+const LIVE_FUNCTION_BEHAVIOR = 'NON_BLOCKING';
+export function wireFunctionDeclaration(decl) {
+    const parameters = toGeminiOpenApiSchema(decl.parameters);
+    return {
+        name: decl.name,
+        description: decl.description,
+        behavior: LIVE_FUNCTION_BEHAVIOR,
+        parameters: parameters && typeof parameters === 'object'
+            ? parameters
+            : { type: 'OBJECT', properties: {} },
+    };
+}
+export function wireLiveTools(req) {
+    const functionDeclarations = [];
+    for (const id of req.builtins) {
+        const entry = getTool(id);
+        if (entry?.type === 'builtin' && entry.wire.live) {
+            functionDeclarations.push({
+                name: id,
+                description: entry.description,
+                behavior: LIVE_FUNCTION_BEHAVIOR,
+            });
+        }
+    }
+    for (const decl of req.wireTools ?? []) {
+        functionDeclarations.push(wireFunctionDeclaration(decl));
+    }
+    if (functionDeclarations.length === 0) {
+        return [];
+    }
+    return [{ functionDeclarations }];
+}
+function buildLiveGenerationConfig(req) {
+    const generationConfig = {
+        responseModalities: ['AUDIO'],
+        temperature: req.temperature,
+        maxOutputTokens: req.maxOutputTokens,
+    };
+    if (req.live?.voice) {
+        generationConfig.speechConfig = {
+            voiceConfig: {
+                prebuiltVoiceConfig: {
+                    voiceName: req.live.voice,
+                },
+            },
+        };
+    }
+    if (req.thinking && req.thinking !== 'none') {
+        generationConfig.thinkingConfig = {
+            thinkingLevel: req.thinking,
+        };
+    }
+    return generationConfig;
+}
+function normalizeStartSensitivity(val) {
+    return val?.includes('HIGH') ? 'START_SENSITIVITY_HIGH' : 'START_SENSITIVITY_LOW';
+}
+function normalizeEndSensitivity(val) {
+    return val?.includes('HIGH') ? 'END_SENSITIVITY_HIGH' : 'END_SENSITIVITY_LOW';
+}
+function buildLiveRealtimeInputConfig(vad) {
+    const automaticActivityDetection = {};
+    if (vad.startSensitivity !== undefined) {
+        automaticActivityDetection.startOfSpeechSensitivity = normalizeStartSensitivity(vad.startSensitivity);
+    }
+    if (vad.endSensitivity !== undefined) {
+        automaticActivityDetection.endOfSpeechSensitivity = normalizeEndSensitivity(vad.endSensitivity);
+    }
+    if (vad.prefixPaddingMs !== undefined) {
+        automaticActivityDetection.prefixPaddingMs = vad.prefixPaddingMs;
+    }
+    if (vad.silenceDurationMs !== undefined) {
+        automaticActivityDetection.silenceDurationMs = vad.silenceDurationMs;
+    }
+    const config = {};
+    if (vad.activityHandling !== undefined) {
+        config.activityHandling = vad.activityHandling;
+    }
+    if (Object.keys(automaticActivityDetection).length > 0) {
+        config.automaticActivityDetection = automaticActivityDetection;
+    }
+    return Object.keys(config).length > 0 ? config : undefined;
+}
+function buildLiveSessionResumption(req) {
+    if (req.sessionResumptionHandle) {
+        return { handle: req.sessionResumptionHandle };
+    }
+    if (req.live?.sessionResumption) {
+        return {};
+    }
+    return undefined;
+}
+function liveModelName(apiId) {
+    return apiId.startsWith('models/') ? apiId : `models/${apiId}`;
+}
+function applyLiveOptionalFeatures(live, setup) {
+    if (live?.transcription?.input) {
+        setup.inputAudioTranscription = {};
+    }
+    if (live?.transcription?.output) {
+        setup.outputAudioTranscription = {};
+    }
+    if (live?.proactiveAudio === true) {
+        setup.proactivity = { proactiveAudio: true };
+    }
+}
+/** Build the initial `setup` message sent once immediately after WebSocket open. */
+export function buildGeminiLiveSetupMessage(req) {
+    const live = req.live;
+    const realtimeInputConfig = live?.vad ? buildLiveRealtimeInputConfig(live.vad) : undefined;
+    const tools = wireLiveTools(req);
+    const sessionResumption = buildLiveSessionResumption(req);
+    // Only opt into initial-history gating when we actually have history to seed.
+    // With `initialHistoryInClientContent: true`, Gemini waits for clientContent
+    // after setupComplete and will not start realtime generation until that lands —
+    // empty sessions (e.g. Th30) would hang forever if this were always set.
+    const seedInitialHistory = Boolean(req.history && req.history.length > 0);
+    const setup = {
+        model: liveModelName(req.apiId),
+        generationConfig: buildLiveGenerationConfig(req),
+        systemInstruction: {
+            parts: [{ text: req.system }],
+        },
+        ...(tools.length > 0 ? { tools } : {}),
+        ...(sessionResumption ? { sessionResumption } : {}),
+        ...(live?.contextCompression === 'slidingWindow'
+            ? { contextWindowCompression: { slidingWindow: {} } }
+            : {}),
+        ...(seedInitialHistory ? { historyConfig: { initialHistoryInClientContent: true } } : {}),
+        ...(realtimeInputConfig ? { realtimeInputConfig } : {}),
+    };
+    applyLiveOptionalFeatures(live, setup);
+    return { setup };
+}
+/** Live carries inline bytes only — provider file references are rejected until support is verified. */
+function inlineMediaPart(part) {
+    if (isMediaRefPart(part)) {
+        throw new TheorumError('media references are not supported on geminiLive');
+    }
+    return part;
+}
+function inlineData(part) {
+    const inline = inlineMediaPart(part);
+    return { mimeType: inline.mimeType, data: inline.data };
+}
+/** Format a single history message into a Google turn object. */
+function historyTurnToGoogleTurn(msg) {
+    const role = msg.role === 'assistant' ? 'model' : 'user';
+    const parts = [];
+    if (msg.content) {
+        parts.push({ text: msg.content });
+    }
+    for (const part of msg.parts ?? []) {
+        if (part.type === 'text') {
+            parts.push({ text: part.text });
+        }
+        else {
+            parts.push({ inlineData: inlineData(part) });
+        }
+    }
+    return { role, parts };
+}
+/** Build the `clientContent` message used for seeding conversation history before realtime streaming. */
+export function buildGeminiLiveClientContent(history) {
+    if (!history || history.length === 0) {
+        return null;
+    }
+    return {
+        clientContent: {
+            turns: history.map(historyTurnToGoogleTurn),
+            turnComplete: true,
+        },
+    };
+}
+/** Build a `realtimeInput` message for streaming audio, video, or text chunks. */
+export function buildGeminiLiveRealtimeInput(input) {
+    if (input.type === 'text') {
+        return {
+            realtimeInput: {
+                text: input.text,
+            },
+        };
+    }
+    const part = inlineMediaPart(input);
+    if (part.type === 'audio') {
+        return {
+            realtimeInput: {
+                audio: {
+                    mimeType: part.mimeType.includes('rate=') ? part.mimeType : 'audio/pcm;rate=16000',
+                    data: part.data,
+                },
+            },
+        };
+    }
+    // Image / video frame
+    return {
+        realtimeInput: {
+            video: {
+                mimeType: part.mimeType || 'image/jpeg',
+                data: part.data,
+            },
+        },
+    };
+}
+/** Build a `realtimeInput` message with text. */
+export function buildGeminiLiveRealtimeText(text) {
+    return {
+        realtimeInput: {
+            text,
+        },
+    };
+}
+/** Build the Gemini Live `response` struct for a function result. */
+function liveFunctionResponsePayload(output) {
+    if (typeof output === 'object' &&
+        output !== null &&
+        'error' in output &&
+        typeof output.error === 'string') {
+        return { error: output.error };
+    }
+    return { result: output };
+}
+/** Build a `toolResponse` message returning the execution result of a tool call. */
+export function buildGeminiLiveToolResponse(id, name, output) {
+    return buildGeminiLiveToolResponses([{ id, name, output }]);
+}
+/** Build a batched `toolResponse` for one or more function results. */
+export function buildGeminiLiveToolResponses(responses) {
+    return {
+        toolResponse: {
+            functionResponses: responses.map(({ id, name, output }) => ({
+                id,
+                name,
+                response: liveFunctionResponsePayload(output),
+            })),
+        },
+    };
+}
+export function parseGeminiLiveMessage(raw) {
+    if (typeof raw === 'object' && raw !== null && !Array.isArray(raw)) {
+        if (raw instanceof ArrayBuffer || raw instanceof Uint8Array) {
+            const text = new TextDecoder().decode(raw);
+            return parseGeminiLiveMessage(text);
+        }
+    }
+    if (typeof raw !== 'string' || !raw.trim()) {
+        return { ok: false, reason: 'empty' };
+    }
+    try {
+        const parsed = JSON.parse(raw);
+        if (parsed && typeof parsed === 'object' && !Array.isArray(parsed)) {
+            return { ok: true, value: parsed };
+        }
+        return { ok: false, reason: 'malformed' };
+    }
+    catch {
+        return { ok: false, reason: 'malformed' };
+    }
+}
+export function parseFunctionArguments(raw) {
+    return parseToolArgumentsObject(raw);
+}
+function readTokenCount(metadata, camelKey, snakeKey) {
+    const val = metadata[camelKey] ?? metadata[snakeKey];
+    return typeof val === 'number' ? val : 0;
+}
+export function extractLiveUsageTokens(metadata) {
+    const prompt = readTokenCount(metadata, 'promptTokenCount', 'prompt_token_count');
+    const output = readTokenCount(metadata, 'responseTokenCount', 'response_token_count');
+    const thinking = readTokenCount(metadata, 'thoughtsTokenCount', 'thoughts_token_count');
+    const rawTotal = metadata.totalTokenCount ?? metadata.total_token_count;
+    const total = typeof rawTotal === 'number' ? rawTotal : prompt + output;
+    if (prompt === 0 && output === 0 && total === 0) {
+        return undefined;
+    }
+    return {
+        input: prompt,
+        output,
+        thinking: thinking > 0 ? thinking : undefined,
+        total,
+    };
+}
+function foldSessionUpdate(message, events) {
+    const sessionUpdate = message.sessionResumptionUpdate;
+    if (!sessionUpdate || typeof sessionUpdate !== 'object')
+        return;
+    const hasHandle = typeof sessionUpdate.newHandle === 'string' && sessionUpdate.newHandle.length > 0;
+    const hasResumable = typeof sessionUpdate.resumable === 'boolean';
+    if (!hasHandle && !hasResumable)
+        return;
+    events.push({
+        type: 'evidence',
+        ...(hasHandle ? { sessionResumptionHandle: sessionUpdate.newHandle } : {}),
+        evidence: {
+            provider: 'google',
+            kind: 'session_resumption',
+            resumable: hasResumable ? sessionUpdate.resumable : hasHandle,
+            raw: sessionUpdate,
+        },
+    });
+}
+function foldToolCalls(message, events) {
+    const toolCall = message.toolCall;
+    if (!toolCall?.functionCalls || !Array.isArray(toolCall.functionCalls))
+        return;
+    for (const call of toolCall.functionCalls) {
+        if (!call.name)
+            continue;
+        const parsed = parseFunctionArguments(call.args);
+        if (!parsed.ok) {
+            events.push({
+                type: 'tool',
+                tool: {
+                    id: call.id,
+                    name: call.name,
+                    arguments: {},
+                    phase: 'error',
+                    failure: {
+                        code: 'malformed_arguments',
+                        message: parsed.error,
+                        details: { raw: parsed.raw },
+                    },
+                },
+            });
+            continue;
+        }
+        events.push({
+            type: 'tool',
+            tool: {
+                id: call.id,
+                name: call.name,
+                arguments: parsed.value,
+            },
+        });
+    }
+}
+function foldToolCancellations(message, events) {
+    const cancellation = message.toolCallCancellation;
+    const ids = cancellation?.ids;
+    if (!Array.isArray(ids))
+        return;
+    for (const id of ids) {
+        if (typeof id !== 'string' || id.length === 0)
+            continue;
+        events.push({
+            type: 'tool',
+            tool: {
+                id,
+                name: '',
+                phase: 'cancel',
+            },
+        });
+    }
+}
+/** Parse Gemini goAway.timeLeft (seconds number, "10s", duration string) → ms when known. */
+export function parseGoAwayTimeLeftMs(timeLeft) {
+    if (typeof timeLeft === 'number' && Number.isFinite(timeLeft) && timeLeft >= 0) {
+        return Math.round(timeLeft * 1000);
+    }
+    if (typeof timeLeft !== 'string')
+        return undefined;
+    const trimmed = timeLeft.trim();
+    if (!trimmed)
+        return undefined;
+    const seconds = Number(trimmed);
+    if (Number.isFinite(seconds) && seconds >= 0) {
+        return Math.round(seconds * 1000);
+    }
+    const match = /^(\d+(?:\.\d+)?)\s*s$/i.exec(trimmed);
+    if (match?.[1]) {
+        const s = Number(match[1]);
+        if (Number.isFinite(s) && s >= 0)
+            return Math.round(s * 1000);
+    }
+    return undefined;
+}
+function foldGoAway(message, events) {
+    const goAway = message.goAway;
+    if (!goAway || typeof goAway !== 'object')
+        return;
+    const timeLeftMs = parseGoAwayTimeLeftMs(goAway.timeLeft);
+    events.push({
+        type: 'session',
+        session: {
+            kind: 'closing_soon',
+            ...(timeLeftMs !== undefined ? { timeLeftMs } : {}),
+        },
+    });
+}
+function foldModelPart(part, events) {
+    if (part.text) {
+        if (part.thought) {
+            events.push({ type: 'thought', text: part.text });
+        }
+        else {
+            events.push({ type: 'text', text: part.text });
+        }
+    }
+    if (part.inlineData?.data) {
+        const mime = part.inlineData.mimeType ?? 'audio/pcm;rate=24000';
+        if (mime.startsWith('audio/pcm') ||
+            mime.startsWith('audio/raw') ||
+            mime.startsWith('audio/l16')) {
+            const wav = wrapPcmAsWav(base64ToBytes(part.inlineData.data), 24000);
+            events.push({
+                type: 'media',
+                media: { mimeType: 'audio/wav', data: bytesToBase64(wav) },
+            });
+        }
+        else {
+            events.push({
+                type: 'media',
+                media: { mimeType: mime, data: part.inlineData.data },
+            });
+        }
+    }
+}
+function foldTranscription(text, kind, events, interim) {
+    if (!text)
+        return;
+    events.push({
+        type: 'evidence',
+        text,
+        evidence: {
+            provider: 'google',
+            kind,
+            ...(interim ? { interim: true } : {}),
+        },
+    });
+}
+function foldLiveGrounding(serverContent, events) {
+    const groundingEvent = groundingFromEvent({
+        groundingMetadata: serverContent.groundingMetadata ?? serverContent.grounding_metadata,
+    });
+    if (groundingEvent) {
+        events.push(groundingEvent);
+    }
+    const urlContext = serverContent.urlContextMetadata ?? serverContent.url_context_metadata;
+    if (urlContext && typeof urlContext === 'object') {
+        events.push({
+            type: 'evidence',
+            evidence: {
+                provider: 'google',
+                kind: 'url_context',
+                raw: urlContext,
+            },
+        });
+    }
+}
+/** Read the server-side `interactionStatus` when the message carries one. */
+export function readLiveInteractionStatus(message) {
+    const raw = message.interactionStatus ?? message.interaction_status;
+    if (raw === 'IN_PROGRESS' || raw === 'IDLE')
+        return raw;
+    return undefined;
+}
+function foldInteractionStatus(message, events) {
+    const status = readLiveInteractionStatus(message);
+    if (!status)
+        return;
+    events.push({
+        type: 'session',
+        session: { kind: status === 'IDLE' ? 'idle' : 'working' },
+    });
+}
+function foldServerContent(message, events) {
+    const serverContent = message.serverContent;
+    if (!serverContent || typeof serverContent !== 'object')
+        return;
+    if (serverContent.interrupted === true) {
+        events.push({
+            type: 'done',
+            interrupted: true,
+            stop: { kind: 'interrupted' },
+        });
+    }
+    if (serverContent.waitingForInput === true || serverContent.waiting_for_input === true) {
+        events.push({
+            type: 'session',
+            session: { kind: 'waiting_for_input' },
+        });
+    }
+    if (serverContent.generationComplete === true || serverContent.generation_complete === true) {
+        events.push({
+            type: 'done',
+            stop: { kind: 'generation_complete' },
+        });
+    }
+    const inputTranscription = serverContent.inputTranscription;
+    foldTranscription(inputTranscription?.text, 'input_transcription', events);
+    const interimInput = serverContent.interimInputTranscription;
+    foldTranscription(interimInput?.text, 'input_transcription', events, true);
+    const outputTranscription = serverContent.outputTranscription;
+    foldTranscription(outputTranscription?.text, 'output_transcription', events);
+    const modelTurn = serverContent.modelTurn;
+    for (const part of modelTurn?.parts ?? []) {
+        foldModelPart(part, events);
+    }
+    foldLiveGrounding(serverContent, events);
+    if (serverContent.turnComplete === true || serverContent.turn_complete === true) {
+        events.push({ type: 'session', session: { kind: 'turn_complete' } });
+    }
+}
+function foldUsageMetadata(message, events) {
+    const usageMetadata = message.usageMetadata;
+    if (usageMetadata) {
+        const tokens = extractLiveUsageTokens(usageMetadata);
+        if (tokens) {
+            events.push({ type: 'tokens', tokens });
+        }
+    }
+}
+/**
+ * Fold a raw `BidiGenerateContentServerMessage` into normalized `TurnEvent` items.
+ */
+export function foldGeminiLiveServerMessage(message) {
+    if (!message || typeof message !== 'object')
+        return [];
+    const events = [];
+    foldGoAway(message, events);
+    foldSessionUpdate(message, events);
+    foldToolCalls(message, events);
+    foldToolCancellations(message, events);
+    foldServerContent(message, events);
+    foldInteractionStatus(message, events);
+    foldUsageMetadata(message, events);
+    return events;
+}

package/esm/src/providers/google/live/openapi-schema.d.ts

@@ -0,0 +1,6 @@
+/**
+ * Gemini Live / OpenAPI Schema 3.0 style type uppercasing for function parameters.
+ * JSON Schema uses lowercase `object`/`string`; Gemini Live expects `OBJECT`/`STRING`.
+ */
+/** Convert JSON-Schema-ish parameters to Gemini Live OpenAPI Schema shape. */
+export declare function toGeminiOpenApiSchema(value: unknown): unknown;

package/esm/src/providers/google/live/openapi-schema.js

@@ -0,0 +1,46 @@
+/**
+ * Gemini Live / OpenAPI Schema 3.0 style type uppercasing for function parameters.
+ * JSON Schema uses lowercase `object`/`string`; Gemini Live expects `OBJECT`/`STRING`.
+ */
+function convertGeminiSchemaType(value) {
+    if (typeof value === 'string')
+        return { type: value.toUpperCase() };
+    if (!Array.isArray(value))
+        return { type: toGeminiOpenApiSchema(value) };
+    const schemaTypes = value.filter((item) => typeof item === 'string');
+    const nonNullTypes = schemaTypes.filter((item) => item !== 'null');
+    const converted = {};
+    if (schemaTypes.includes('null'))
+        converted.nullable = true;
+    const [onlyType] = nonNullTypes;
+    if (onlyType !== undefined && nonNullTypes.length === 1)
+        converted.type = onlyType.toUpperCase();
+    if (nonNullTypes.length > 1) {
+        converted.anyOf = nonNullTypes.map((item) => ({ type: item.toUpperCase() }));
+    }
+    return converted;
+}
+function convertGeminiSchemaObject(obj) {
+    const result = {};
+    for (const [key, value] of Object.entries(obj)) {
+        if (key === 'type') {
+            Object.assign(result, convertGeminiSchemaType(value));
+        }
+        else if (key === 'enum' && Array.isArray(value)) {
+            result.enum = value.filter((item) => item !== null);
+        }
+        else {
+            result[key] = toGeminiOpenApiSchema(value);
+        }
+    }
+    return result;
+}
+/** Convert JSON-Schema-ish parameters to Gemini Live OpenAPI Schema shape. */
+export function toGeminiOpenApiSchema(value) {
+    if (Array.isArray(value))
+        return value.map(toGeminiOpenApiSchema);
+    if (value !== null && typeof value === 'object') {
+        return convertGeminiSchemaObject(value);
+    }
+    return value;
+}

package/esm/src/providers/google/live/session.d.ts

@@ -0,0 +1,25 @@
+/**
+ * Google Gemini Live session transport — long-lived BidiGenerateContent WebSocket.
+ *
+ * `turnComplete` is a conversational turn boundary, not session teardown.
+ *
+ * @module
+ */
+import type { ProviderCompleteRequest } from '../../../kernel/types.js';
+import { type GeminiTransport } from '../keys.js';
+import { type SessionQueueItem } from './stream.js';
+export interface GoogleLiveConnection {
+    /** Send a framed JSON payload upstream. */
+    send(payload: string): void;
+    /** Drain session batches until the socket closes or errors. */
+    batches(): AsyncGenerator<SessionQueueItem>;
+    close(code?: number, reason?: string): void;
+}
+export type OpenLiveWebSocket = (url: string) => Promise<WebSocket>;
+/**
+ * Open a long-lived Gemini Live WebSocket after setup handshake.
+ * Callers own send / batch drain / close — typically via `runSession`.
+ *
+ * @param openWebSocket Host override for Cloudflare fetch-upgrade (etc.).
+ */
+export declare function openGoogleLiveSession(req: ProviderCompleteRequest, transport: GeminiTransport, openWebSocket?: OpenLiveWebSocket): Promise<GoogleLiveConnection>;