realtime-voice-agents 2.0.0

package/dist/gemini.mjs

@@ -0,0 +1,395 @@
+import { a as noopLogger, n as deepMerge, t as BaseRealtimeProvider } from "./BaseRealtimeProvider-BQigr5mB.mjs";
+import { t as resolveApiKey } from "./env-DUwUWTsg.mjs";
+import { n as OutboundTranscoder, t as InboundTranscoder } from "./transcode-CsAp97G9.mjs";
+//#region src/providers/gemini/GeminiLiveProvider.ts
+/**
+* Gemini Live provider via the official `@google/genai` SDK (optional peer).
+*
+* Unlike the OpenAI-compatible path, Gemini speaks PCM (16 kHz in, 24 kHz
+* out), so this provider owns a stateful transcoding pair: μ-law↔PCM with
+* inter-chunk resampler state — the engine stays μ-law-only.
+*
+* Gemini specifics handled here:
+* - turn ids are synthesized (`gturn_N`) — the wire has no response ids
+* - `serverContent.interrupted` is the barge-in signal (maps to
+*   userSpeechStarted + responseDone; the server already cancelled itself)
+* - session resumption handles are captured continuously and replayed on the
+*   next connect, so reconnects restore context server-side
+* - `goAway` (imminent disconnect, e.g. the session time limit) is surfaced
+*   so the session can reconnect proactively
+* - tool results go through the proper `toolResponse` message
+*/
+const GEMINI_INPUT_MIME = "audio/pcm;rate=16000";
+var GeminiLiveProvider = class extends BaseRealtimeProvider {
+	name = "gemini";
+	capabilities = {
+		truncate: false,
+		sessionUpdate: false,
+		voiceChangeMidSession: false,
+		transcodeRequired: true,
+		resumption: true,
+		agentTranscriptDeltas: true,
+		vadInterruptControl: false
+	};
+	config;
+	logger;
+	session = null;
+	sessionInit = null;
+	ready = false;
+	intentionalClose = false;
+	inbound = new InboundTranscoder(16e3);
+	outbound = new OutboundTranscoder(24e3);
+	turnCounter = 0;
+	currentTurnId = null;
+	userTranscriptBuffer = "";
+	agentTranscriptBuffer = "";
+	resumptionHandle = null;
+	resumed = false;
+	constructor(config, logger = noopLogger) {
+		super();
+		this.config = config;
+		this.logger = logger;
+	}
+	get isConnected() {
+		return this.ready && this.session !== null;
+	}
+	/** True when the last connect restored server-side context via a handle. */
+	get didResume() {
+		return this.resumed;
+	}
+	async connect(init) {
+		if (this.session) await this.close();
+		this.sessionInit = init;
+		this.intentionalClose = false;
+		this.ready = false;
+		this.resumed = false;
+		this.inbound.reset();
+		this.outbound.reset();
+		const connector = this.config.connector ?? await this.defaultConnector();
+		const handle = init.freshSession ? void 0 : init.resumptionHandle ?? this.resumptionHandle ?? void 0;
+		if (init.freshSession) this.resumptionHandle = null;
+		let setupResolve;
+		let rejectSetup;
+		const setupDone = new Promise((resolve, reject) => {
+			setupResolve = resolve;
+			rejectSetup = reject;
+		});
+		let setupError = null;
+		setupDone.catch(() => {});
+		const setupReject = (error) => {
+			setupError ??= error;
+			rejectSetup(error);
+		};
+		const timeoutMs = this.config.connectTimeoutMs ?? 1e4;
+		const timer = setTimeout(() => setupReject(/* @__PURE__ */ new Error(`gemini setup not complete within ${timeoutMs}ms`)), timeoutMs);
+		timer.unref?.();
+		try {
+			try {
+				this.session = await connector({
+					model: this.config.model,
+					config: this.buildConfig(init, handle),
+					callbacks: {
+						onopen: () => this.emit("open"),
+						onmessage: (message) => {
+							if (message?.setupComplete !== void 0) {
+								this.ready = true;
+								if (handle) this.resumed = true;
+								setupResolve();
+								return;
+							}
+							this.handleMessage(message);
+						},
+						onerror: (error) => {
+							const err = error instanceof Error ? error : new Error(String(error?.message ?? error));
+							if (!this.ready) setupReject(err);
+							else this.emit("error", err);
+						},
+						onclose: (event) => {
+							const wasReady = this.ready;
+							this.ready = false;
+							this.session = null;
+							if (!wasReady) {
+								setupReject(/* @__PURE__ */ new Error(`gemini closed during setup (${event?.code} ${event?.reason ?? ""})`));
+								return;
+							}
+							this.emit("close", {
+								code: event?.code,
+								reason: event?.reason,
+								retriable: !this.intentionalClose && !isNonRetriableClose(event?.code)
+							});
+						}
+					}
+				});
+			} catch (error) {
+				throw setupError ?? (error instanceof Error ? error : new Error(String(error)));
+			}
+			await setupDone;
+		} finally {
+			clearTimeout(timer);
+		}
+	}
+	async defaultConnector() {
+		let genai;
+		try {
+			genai = await import("@google/genai");
+		} catch (error) {
+			throw new Error("the '@google/genai' package is required for the Gemini provider — npm install @google/genai", { cause: error });
+		}
+		const { GoogleGenAI } = genai;
+		const ai = this.config.vertex ? new GoogleGenAI({
+			vertexai: true,
+			project: this.config.vertex.project,
+			location: this.config.vertex.location
+		}) : new GoogleGenAI({ apiKey: this.config.apiKey });
+		return (params) => ai.live.connect({
+			model: params.model,
+			config: params.config,
+			callbacks: params.callbacks
+		});
+	}
+	buildConfig(init, resumptionHandle) {
+		const transcription = this.config.transcription;
+		const vad = init.vad !== void 0 ? init.vad : this.config.defaultVad;
+		const config = {
+			responseModalities: ["AUDIO"],
+			systemInstruction: init.instructions,
+			speechConfig: {
+				voiceConfig: { prebuiltVoiceConfig: { voiceName: init.voice ?? this.config.voice ?? "Aoede" } },
+				...this.config.languageCode ? { languageCode: this.config.languageCode } : {}
+			},
+			...init.tools?.length ? { tools: [{ functionDeclarations: init.tools.map((tool) => ({
+				name: tool.name,
+				description: tool.description ?? "",
+				parameters: tool.parameters
+			})) }] } : {},
+			realtimeInputConfig: { automaticActivityDetection: buildGeminiVad(vad) },
+			...transcription === false ? {} : {
+				...transcription?.input !== false ? { inputAudioTranscription: {} } : {},
+				...transcription?.output !== false ? { outputAudioTranscription: {} } : {}
+			},
+			...this.config.resumption !== false ? { sessionResumption: { ...resumptionHandle ? { handle: resumptionHandle } : {} } } : {},
+			...this.config.contextWindowCompression ? { contextWindowCompression: { slidingWindow: {} } } : {}
+		};
+		return deepMerge(deepMerge(config, init.providerOptions), this.config.extraConfig);
+	}
+	async close() {
+		this.intentionalClose = true;
+		this.ready = false;
+		const session = this.session;
+		this.session = null;
+		if (!session) return;
+		try {
+			session.close();
+		} catch {}
+	}
+	sendAudio(base64Mulaw) {
+		if (!this.isConnected || !this.session) return;
+		const pcm = this.inbound.process(base64Mulaw);
+		if (pcm.length === 0) return;
+		this.session.sendRealtimeInput({ audio: {
+			data: pcm,
+			mimeType: GEMINI_INPUT_MIME
+		} });
+	}
+	sendText(text, options = {}) {
+		if (!this.session) return;
+		const role = options.role === "assistant" ? "model" : "user";
+		const content = options.role === "system" ? `(system note) ${text}` : text;
+		this.session.sendClientContent({
+			turns: [{
+				role,
+				parts: [{ text: content }]
+			}],
+			turnComplete: options.triggerResponse !== false
+		});
+	}
+	sendToolResult(callId, output, options = {}) {
+		if (!this.session) return;
+		const response = output !== null && typeof output === "object" && !Array.isArray(output) ? output : { result: output };
+		this.session.sendToolResponse({ functionResponses: [{
+			id: callId,
+			name: this.toolNameForCall(callId),
+			response
+		}] });
+	}
+	createResponse(options = {}) {
+		if (!this.session) return;
+		const text = options.instructions ? `(instruction) ${options.instructions}` : "(instruction) Continue the conversation now — speak to the caller.";
+		this.session.sendClientContent({
+			turns: [{
+				role: "user",
+				parts: [{ text }]
+			}],
+			turnComplete: true
+		});
+	}
+	async updateSession(_patch) {
+		this.logger.warn("gemini: updateSession ignored (no mid-session config on Live API)");
+	}
+	handleMessage(message) {
+		const serverContent = message?.serverContent;
+		if (message?.toolCall?.functionCalls) {
+			for (const call of message.toolCall.functionCalls) {
+				const id = call.id ?? `gcall_${++this.turnCounter}`;
+				this.callNames.set(id, call.name);
+				this.emit("toolCall", {
+					id,
+					name: call.name,
+					argumentsJson: JSON.stringify(call.args ?? {}),
+					responseId: this.currentTurnId ?? void 0
+				});
+			}
+			return;
+		}
+		if (message?.sessionResumptionUpdate) {
+			const update = message.sessionResumptionUpdate;
+			if (update.resumable && update.newHandle) {
+				this.resumptionHandle = update.newHandle;
+				this.emit("resumptionUpdate", update.newHandle);
+			}
+			return;
+		}
+		if (message?.goAway) {
+			const timeLeftMs = parseGoAwayTime(message.goAway.timeLeft);
+			this.logger.info("gemini goAway received", { timeLeftMs });
+			this.emit("goAway", { timeLeftMs });
+			return;
+		}
+		if (message?.usageMetadata) {
+			const usage = message.usageMetadata;
+			this.emit("usage", {
+				inputTokens: usage.promptTokenCount ?? 0,
+				outputTokens: usage.responseTokenCount ?? usage.candidatesTokenCount ?? 0,
+				totalTokens: usage.totalTokenCount ?? 0,
+				raw: usage
+			});
+		}
+		if (!serverContent) return;
+		if (serverContent.interrupted) {
+			const turnId = this.currentTurnId;
+			this.emit("userSpeechStarted");
+			if (turnId) {
+				this.flushAgentTranscript(turnId);
+				this.currentTurnId = null;
+				this.emit("responseDone", { responseId: turnId });
+			}
+			return;
+		}
+		if (serverContent.inputTranscription?.text) this.userTranscriptBuffer += serverContent.inputTranscription.text;
+		if (serverContent.outputTranscription?.text) {
+			const turnId = this.ensureTurn();
+			this.agentTranscriptBuffer += serverContent.outputTranscription.text;
+			this.emit("agentTranscriptDelta", {
+				responseId: turnId,
+				delta: serverContent.outputTranscription.text
+			});
+		}
+		const parts = serverContent.modelTurn?.parts ?? [];
+		for (const part of parts) if (part.inlineData?.data) {
+			const turnId = this.ensureTurn();
+			const mulaw = this.outbound.process(part.inlineData.data);
+			if (mulaw.length > 0) this.emit("audio", {
+				base64Mulaw: mulaw,
+				responseId: turnId
+			});
+		}
+		if (serverContent.turnComplete) {
+			this.flushUserTranscript();
+			const turnId = this.currentTurnId;
+			if (turnId) {
+				this.flushAgentTranscript(turnId);
+				this.currentTurnId = null;
+				this.emit("responseDone", { responseId: turnId });
+			}
+		}
+	}
+	callNames = /* @__PURE__ */ new Map();
+	toolNameForCall(callId) {
+		const name = this.callNames.get(callId) ?? callId;
+		this.callNames.delete(callId);
+		return name;
+	}
+	ensureTurn() {
+		if (!this.currentTurnId) {
+			this.currentTurnId = `gturn_${++this.turnCounter}`;
+			this.flushUserTranscript();
+			this.emit("responseStarted", { responseId: this.currentTurnId });
+		}
+		return this.currentTurnId;
+	}
+	flushUserTranscript() {
+		const text = this.userTranscriptBuffer.trim();
+		this.userTranscriptBuffer = "";
+		if (text) this.emit("userTranscript", { text });
+	}
+	flushAgentTranscript(turnId) {
+		const text = this.agentTranscriptBuffer.trim();
+		this.agentTranscriptBuffer = "";
+		if (text) this.emit("agentTranscript", {
+			responseId: turnId,
+			text
+		});
+	}
+};
+/** Protocol/auth/policy closes where retrying cannot help. */
+function isNonRetriableClose(code) {
+	if (code === void 0) return false;
+	return code === 1002 || code === 1003 || code === 1007 || code === 1008 || code >= 4400 && code < 4500;
+}
+function buildGeminiVad(vad) {
+	if (vad === null) return { disabled: true };
+	return {
+		disabled: false,
+		...vad?.startSensitivity ? { startOfSpeechSensitivity: `START_SENSITIVITY_${vad.startSensitivity.toUpperCase()}` } : {},
+		...vad?.endSensitivity ? { endOfSpeechSensitivity: `END_SENSITIVITY_${vad.endSensitivity.toUpperCase()}` } : {},
+		...vad?.prefixPaddingMs !== void 0 ? { prefixPaddingMs: vad.prefixPaddingMs } : {},
+		...vad?.silenceDurationMs !== void 0 ? { silenceDurationMs: vad.silenceDurationMs } : {}
+	};
+}
+function parseGoAwayTime(timeLeft) {
+	if (typeof timeLeft === "number") return timeLeft;
+	if (typeof timeLeft === "string") {
+		const seconds = Number.parseFloat(timeLeft.replace(/s$/, ""));
+		if (!Number.isNaN(seconds)) return Math.round(seconds * 1e3);
+	}
+}
+//#endregion
+//#region src/gemini.ts
+/** Google Gemini Live provider — `realtime-voice-agents/gemini`. */
+/** Env vars checked (in order) when no explicit apiKey is passed. */
+const GEMINI_KEY_ENV_VARS = [
+	"GOOGLE_API_KEY",
+	"GEMINI_API_KEY",
+	"GOOGLE_GENAI_API_KEY"
+];
+const GEMINI_DEFAULT_MODEL = "gemini-2.5-flash-native-audio-preview-12-2025";
+const GEMINI_DEFAULT_VOICE = "Aoede";
+const GEMINI_VOICES = [
+	"Aoede",
+	"Charon",
+	"Fenrir",
+	"Kore",
+	"Puck"
+];
+/** Create a Gemini Live provider factory for the bridge. */
+function geminiLive(options = {}) {
+	const apiKey = resolveApiKey(options.apiKey, GEMINI_KEY_ENV_VARS);
+	if (!apiKey && !options.vertex && !options.connector) throw new Error(`geminiLive: credentials missing (pass apiKey, set one of ${GEMINI_KEY_ENV_VARS.join("/")}, or configure vertex)`);
+	const config = {
+		apiKey,
+		vertex: options.vertex,
+		model: options.model ?? "gemini-2.5-flash-native-audio-preview-12-2025",
+		voice: options.voice ?? "Aoede",
+		languageCode: options.languageCode,
+		defaultVad: options.vad,
+		transcription: options.transcription,
+		resumption: options.resumption,
+		contextWindowCompression: options.contextWindowCompression,
+		extraConfig: options.config,
+		connectTimeoutMs: options.connectTimeoutMs,
+		connector: options.connector
+	};
+	return ({ logger }) => new GeminiLiveProvider(config, logger);
+}
+//#endregion
+export { GEMINI_DEFAULT_MODEL, GEMINI_DEFAULT_VOICE, GEMINI_KEY_ENV_VARS, GEMINI_VOICES, GeminiLiveProvider, geminiLive };