livekit-plugins-google 1.0.16__py3-none-any.whl → 1.0.18__py3-none-any.whl

livekit/plugins/google/beta/realtime/realtime_api.py

@@ -1,12 +1,15 @@
 from __future__ import annotations
 
 import asyncio
+import contextlib
 import json
 import os
 import weakref
+from collections.abc import Iterator
 from dataclasses import dataclass
 
 from google import genai
+from google.genai.live import AsyncSession
 from google.genai.types import (
     AudioTranscriptionConfig,
     Blob,
@@ -23,6 +26,7 @@ from google.genai.types import (
     Modality,
     Part,
     PrebuiltVoiceConfig,
+    SessionResumptionConfig,
     SpeechConfig,
     Tool,
     UsageMetadata,
@@ -31,15 +35,16 @@ from google.genai.types import (
 from livekit import rtc
 from livekit.agents import llm, utils
 from livekit.agents.types import NOT_GIVEN, NotGivenOr
-from livekit.agents.utils import images, is_given
+from livekit.agents.utils import audio as audio_utils, images, is_given
+from livekit.plugins.google.beta.realtime.api_proto import ClientEvents, LiveAPIModels, Voice
 
 from ...log import logger
 from ...utils import _build_gemini_fnc, get_tool_results_for_realtime, to_chat_ctx
-from .api_proto import ClientEvents, LiveAPIModels, Voice
 
 INPUT_AUDIO_SAMPLE_RATE = 16000
+INPUT_AUDIO_CHANNELS = 1
 OUTPUT_AUDIO_SAMPLE_RATE = 24000
-NUM_CHANNELS = 1
+OUTPUT_AUDIO_CHANNELS = 1
 
 DEFAULT_ENCODE_OPTIONS = images.EncodeOptions(
     format="JPEG",
@@ -59,6 +64,7 @@ class _RealtimeOptions:
     model: LiveAPIModels | str
     api_key: str | None
     voice: Voice | str
+    language: NotGivenOr[str]
     response_modalities: NotGivenOr[list[Modality]]
     vertexai: bool
     project: str | None
@@ -98,6 +104,7 @@ class RealtimeModel(llm.RealtimeModel):
         model: LiveAPIModels | str = "gemini-2.0-flash-live-001",
         api_key: NotGivenOr[str] = NOT_GIVEN,
         voice: Voice | str = "Puck",
+        language: NotGivenOr[str] = NOT_GIVEN,
         modalities: NotGivenOr[list[Modality]] = NOT_GIVEN,
         vertexai: bool = False,
         project: NotGivenOr[str] = NOT_GIVEN,
@@ -126,8 +133,9 @@ class RealtimeModel(llm.RealtimeModel):
             instructions (str, optional): Initial system instructions for the model. Defaults to "".
             api_key (str, optional): Google Gemini API key. If None, will attempt to read from the environment variable GOOGLE_API_KEY.
             modalities (list[Modality], optional): Modalities to use, such as ["TEXT", "AUDIO"]. Defaults to ["AUDIO"].
-            model (str, optional): The name of the model to use. Defaults to "gemini-2.0-flash-exp".
+            model (str, optional): The name of the model to use. Defaults to "gemini-2.0-flash-live-001".
             voice (api_proto.Voice, optional): Voice setting for audio outputs. Defaults to "Puck".
+            language (str, optional): The language(BCP-47 Code) to use for the API. supported languages - https://ai.google.dev/gemini-api/docs/live#supported-languages
             temperature (float, optional): Sampling temperature for response generation. Defaults to 0.8.
             vertexai (bool, optional): Whether to use VertexAI for the API. Defaults to False.
                 project (str, optional): The project id to use for the API. Defaults to None. (for vertexai)
@@ -147,20 +155,20 @@ class RealtimeModel(llm.RealtimeModel):
             capabilities=llm.RealtimeCapabilities(
                 message_truncation=False,
                 turn_detection=True,
-                user_transcription=False,
+                user_transcription=is_given(input_audio_transcription),
             )
         )
 
         gemini_api_key = api_key if is_given(api_key) else os.environ.get("GOOGLE_API_KEY")
         gcp_project = project if is_given(project) else os.environ.get("GOOGLE_CLOUD_PROJECT")
         gcp_location = location if is_given(location) else os.environ.get("GOOGLE_CLOUD_LOCATION")
+
         if vertexai:
             if not gcp_project or not gcp_location:
                 raise ValueError(
                     "Project and location are required for VertexAI either via project and location or GOOGLE_CLOUD_PROJECT and GOOGLE_CLOUD_LOCATION environment variables"  # noqa: E501
                 )
             gemini_api_key = None  # VertexAI does not require an API key
-
         else:
             gcp_project = None
             gcp_location = None
@@ -192,6 +200,7 @@ class RealtimeModel(llm.RealtimeModel):
             instructions=instructions,
             input_audio_transcription=input_audio_transcription,
             output_audio_transcription=output_audio_transcription,
+            language=language,
         )
 
         self._sessions = weakref.WeakSet[RealtimeSession]()
@@ -213,7 +222,8 @@ class RealtimeModel(llm.RealtimeModel):
         for sess in self._sessions:
             sess.update_options(voice=self._opts.voice, temperature=self._opts.temperature)
 
-    async def aclose(self) -> None: ...
+    async def aclose(self) -> None:
+        pass
 
 
 class RealtimeSession(llm.RealtimeSession):
@@ -221,138 +231,164 @@ class RealtimeSession(llm.RealtimeSession):
         super().__init__(realtime_model)
         self._opts = realtime_model._opts
         self._tools = llm.ToolContext.empty()
+        self._gemini_declarations: list[FunctionDeclaration] = []
         self._chat_ctx = llm.ChatContext.empty()
         self._msg_ch = utils.aio.Chan[ClientEvents]()
-        self._gemini_tools: list[Tool] = []
+        self._input_resampler: rtc.AudioResampler | None = None
+
+        # 50ms chunks
+        self._bstream = audio_utils.AudioByteStream(
+            INPUT_AUDIO_SAMPLE_RATE,
+            INPUT_AUDIO_CHANNELS,
+            samples_per_channel=INPUT_AUDIO_SAMPLE_RATE // 20,
+        )
+
         self._client = genai.Client(
             api_key=self._opts.api_key,
             vertexai=self._opts.vertexai,
             project=self._opts.project,
             location=self._opts.location,
         )
+
         self._main_atask = asyncio.create_task(self._main_task(), name="gemini-realtime-session")
 
         self._current_generation: _ResponseGeneration | None = None
-
-        self._is_interrupted = False
-        self._active_response_id = None
-        self._session = None
-        self._update_chat_ctx_lock = asyncio.Lock()
-        self._update_fnc_ctx_lock = asyncio.Lock()
+        self._active_session: AsyncSession | None = None
+        # indicates if the underlying session should end
+        self._session_should_close = asyncio.Event()
         self._response_created_futures: dict[str, asyncio.Future[llm.GenerationCreatedEvent]] = {}
-        self._pending_generation_event_id = None
+        self._pending_generation_fut: asyncio.Future[llm.GenerationCreatedEvent] | None = None
 
-        self._reconnect_event = asyncio.Event()
-        self._session_lock = asyncio.Lock()
-        self._gemini_close_task: asyncio.Task | None = None
+        self._session_resumption_handle: str | None = None
 
-    def _schedule_gemini_session_close(self) -> None:
-        if self._session is not None:
-            self._gemini_close_task = asyncio.create_task(self._close_gemini_session())
+        self._update_lock = asyncio.Lock()
+        self._session_lock = asyncio.Lock()
 
-    async def _close_gemini_session(self) -> None:
+    async def _close_active_session(self) -> None:
         async with self._session_lock:
-            if self._session:
+            if self._active_session:
                 try:
-                    await self._session.close()
+                    await self._active_session.close()
+                except Exception as e:
+                    logger.warning(f"error closing Gemini session: {e}")
                 finally:
-                    self._session = None
+                    self._active_session = None
 
-    def update_options(
+    def _mark_restart_needed(self):
+        if not self._session_should_close.is_set():
+            self._session_should_close.set()
+            # reset the msg_ch, do not send messages from previous session
+            self._msg_ch = utils.aio.Chan[ClientEvents]()
+
+    async def update_options(
         self,
         *,
         voice: NotGivenOr[str] = NOT_GIVEN,
-        tool_choice: NotGivenOr[llm.ToolChoice | None] = NOT_GIVEN,
         temperature: NotGivenOr[float] = NOT_GIVEN,
+        tool_choice: NotGivenOr[llm.ToolChoice | None] = NOT_GIVEN,
     ) -> None:
-        if is_given(voice):
-            self._opts.voice = voice
+        async with self._update_lock:
+            should_restart = False
+            if is_given(voice) and self._opts.voice != voice:
+                self._opts.voice = voice
+                should_restart = True
 
-        if is_given(temperature):
-            self._opts.temperature = temperature
+            if is_given(temperature) and self._opts.temperature != temperature:
+                self._opts.temperature = temperature if is_given(temperature) else NOT_GIVEN
+                should_restart = True
 
-        if self._session:
-            logger.warning("Updating options; triggering Gemini session reconnect.")
-            self._reconnect_event.set()
-            self._schedule_gemini_session_close()
+            if should_restart:
+                self._mark_restart_needed()
 
     async def update_instructions(self, instructions: str) -> None:
-        self._opts.instructions = instructions
-        if self._session:
-            logger.warning("Updating instructions; triggering Gemini session reconnect.")
-            self._reconnect_event.set()
-            self._schedule_gemini_session_close()
+        async with self._update_lock:
+            if not is_given(self._opts.instructions) or self._opts.instructions != instructions:
+                self._opts.instructions = instructions
+                self._mark_restart_needed()
 
     async def update_chat_ctx(self, chat_ctx: llm.ChatContext) -> None:
-        async with self._update_chat_ctx_lock:
-            self._chat_ctx = chat_ctx
+        async with self._update_lock:
+            self._chat_ctx = chat_ctx.copy()
             turns, _ = to_chat_ctx(self._chat_ctx, id(self), ignore_functions=True)
             tool_results = get_tool_results_for_realtime(self._chat_ctx)
+            # TODO(dz): need to compute delta and then either append or recreate session
             if turns:
-                self._msg_ch.send_nowait(LiveClientContent(turns=turns, turn_complete=False))
+                self._send_client_event(LiveClientContent(turns=turns, turn_complete=False))
             if tool_results:
-                self._msg_ch.send_nowait(tool_results)
+                self._send_client_event(tool_results)
 
     async def update_tools(self, tools: list[llm.FunctionTool]) -> None:
-        async with self._update_fnc_ctx_lock:
-            retained_tools: list[llm.FunctionTool] = []
-            gemini_function_declarations: list[FunctionDeclaration] = []
-
-            for tool in tools:
-                gemini_function = _build_gemini_fnc(tool)
-                gemini_function_declarations.append(gemini_function)
-                retained_tools.append(tool)
-
-            self._tools = llm.ToolContext(retained_tools)
-            self._gemini_tools = [Tool(function_declarations=gemini_function_declarations)]
-            if self._session and gemini_function_declarations:
-                logger.warning("Updating tools; triggering Gemini session reconnect.")
-                self._reconnect_event.set()
-                self._schedule_gemini_session_close()
+        async with self._update_lock:
+            new_declarations: list[FunctionDeclaration] = [
+                _build_gemini_fnc(tool) for tool in tools
+            ]
+            current_tool_names = {f.name for f in self._gemini_declarations}
+            new_tool_names = {f.name for f in new_declarations}
+
+            if current_tool_names != new_tool_names:
+                self._gemini_declarations = new_declarations
+                self._tools = llm.ToolContext(tools)
+                self._mark_restart_needed()
 
     @property
     def chat_ctx(self) -> llm.ChatContext:
-        return self._chat_ctx
+        return self._chat_ctx.copy()
 
     @property
     def tools(self) -> llm.ToolContext:
-        return self._tools
+        return self._tools.copy()
 
     def push_audio(self, frame: rtc.AudioFrame) -> None:
-        self.push_media(frame.data.tobytes(), "audio/pcm")
+        for f in self._resample_audio(frame):
+            for nf in self._bstream.write(f.data.tobytes()):
+                realtime_input = LiveClientRealtimeInput(
+                    media_chunks=[Blob(data=nf.data.tobytes(), mime_type="audio/pcm")]
+                )
+                self._send_client_event(realtime_input)
 
     def push_video(self, frame: rtc.VideoFrame) -> None:
         encoded_data = images.encode(frame, DEFAULT_ENCODE_OPTIONS)
-        self.push_media(encoded_data, "image/jpeg")
-
-    def push_media(self, bytes: bytes, mime_type: str) -> None:
         realtime_input = LiveClientRealtimeInput(
-            media_chunks=[Blob(data=bytes, mime_type=mime_type)]
+            media_chunks=[Blob(data=encoded_data, mime_type="image/jpeg")]
         )
-        self._msg_ch.send_nowait(realtime_input)
+        self._send_client_event(realtime_input)
+
+    def _send_client_event(self, event: ClientEvents) -> None:
+        with contextlib.suppress(utils.aio.channel.ChanClosed):
+            self._msg_ch.send_nowait(event)
 
     def generate_reply(
         self, *, instructions: NotGivenOr[str] = NOT_GIVEN
     ) -> asyncio.Future[llm.GenerationCreatedEvent]:
-        fut = asyncio.Future()
+        if self._pending_generation_fut and not self._pending_generation_fut.done():
+            logger.warning(
+                "generate_reply called while another generation is pending, cancelling previous."
+            )
+            self._pending_generation_fut.cancel("Superseded by new generate_reply call")
 
-        event_id = utils.shortuuid("gemini-response-")
-        self._response_created_futures[event_id] = fut
-        self._pending_generation_event_id = event_id
+        fut = asyncio.Future()
+        self._pending_generation_fut = fut
 
-        instructions_content = instructions if is_given(instructions) else "."
-        ctx = [Content(parts=[Part(text=instructions_content)], role="user")]
-        self._msg_ch.send_nowait(LiveClientContent(turns=ctx, turn_complete=True))
+        # Gemini requires the last message to end with user's turn
+        # so we need to add a placeholder user turn in order to trigger a new generation
+        event = LiveClientContent(turns=[], turn_complete=True)
+        if is_given(instructions):
+            event.turns.append(Content(parts=[Part(text=instructions)], role="model"))
+        event.turns.append(Content(parts=[Part(text=".")], role="user"))
+        self._send_client_event(event)
 
         def _on_timeout() -> None:
-            if event_id in self._response_created_futures and not fut.done():
-                fut.set_exception(llm.RealtimeError("generate_reply timed out."))
-                self._response_created_futures.pop(event_id, None)
-                if self._pending_generation_event_id == event_id:
-                    self._pending_generation_event_id = None
+            if not fut.done():
+                fut.set_exception(
+                    llm.RealtimeError(
+                        "generate_reply timed out waiting for generation_created event."
+                    )
+                )
+                if self._pending_generation_fut is fut:
+                    self._pending_generation_fut = None
 
-        handle = asyncio.get_event_loop().call_later(5.0, _on_timeout)
-        fut.add_done_callback(lambda _: handle.cancel())
+        timeout_handle = asyncio.get_event_loop().call_later(5.0, _on_timeout)
+        fut.add_done_callback(lambda _: timeout_handle.cancel())
 
         return fut
 
@@ -360,133 +396,206 @@ class RealtimeSession(llm.RealtimeSession):
         pass
 
     def truncate(self, *, message_id: str, audio_end_ms: int) -> None:
+        logger.warning("truncate is not supported by the Google Realtime API.")
         pass
 
     async def aclose(self) -> None:
         self._msg_ch.close()
-
-        for fut in self._response_created_futures.values():
-            if not fut.done():
-                fut.set_exception(llm.RealtimeError("Session closed"))
+        self._session_should_close.set()
 
         if self._main_atask:
             await utils.aio.cancel_and_wait(self._main_atask)
 
-        if self._gemini_close_task:
-            await utils.aio.cancel_and_wait(self._gemini_close_task)
+        await self._close_active_session()
+
+        if self._pending_generation_fut and not self._pending_generation_fut.done():
+            self._pending_generation_fut.cancel("Session closed")
+
+        for fut in self._response_created_futures.values():
+            if not fut.done():
+                fut.set_exception(llm.RealtimeError("Session closed before response created"))
+        self._response_created_futures.clear()
+
+        if self._current_generation:
+            self._finalize_response(closed=True)
 
     @utils.log_exceptions(logger=logger)
     async def _main_task(self):
-        while True:
-            config = LiveConnectConfig(
-                response_modalities=self._opts.response_modalities
-                if is_given(self._opts.response_modalities)
-                else [Modality.AUDIO],
-                generation_config=GenerationConfig(
-                    candidate_count=self._opts.candidate_count,
-                    temperature=self._opts.temperature
-                    if is_given(self._opts.temperature)
-                    else None,
-                    max_output_tokens=self._opts.max_output_tokens
-                    if is_given(self._opts.max_output_tokens)
-                    else None,
-                    top_p=self._opts.top_p if is_given(self._opts.top_p) else None,
-                    top_k=self._opts.top_k if is_given(self._opts.top_k) else None,
-                    presence_penalty=self._opts.presence_penalty
-                    if is_given(self._opts.presence_penalty)
-                    else None,
-                    frequency_penalty=self._opts.frequency_penalty
-                    if is_given(self._opts.frequency_penalty)
-                    else None,
-                ),
-                system_instruction=Content(parts=[Part(text=self._opts.instructions)])
-                if is_given(self._opts.instructions)
-                else None,
-                speech_config=SpeechConfig(
-                    voice_config=VoiceConfig(
-                        prebuilt_voice_config=PrebuiltVoiceConfig(voice_name=self._opts.voice)
+        while not self._msg_ch.closed:
+            # previous session might not be closed yet, we'll do it here.
+            await self._close_active_session()
+
+            self._session_should_close.clear()
+            config = self._build_connect_config()
+            session = None
+            try:
+                logger.debug("connecting to Gemini Realtime API...")
+                async with self._client.aio.live.connect(
+                    model=self._opts.model, config=config
+                ) as session:
+                    async with self._session_lock:
+                        self._active_session = session
+
+                    # queue up existing chat context
+                    send_task = asyncio.create_task(
+                        self._send_task(session), name="gemini-realtime-send"
+                    )
+                    recv_task = asyncio.create_task(
+                        self._recv_task(session), name="gemini-realtime-recv"
+                    )
+                    restart_wait_task = asyncio.create_task(
+                        self._session_should_close.wait(), name="gemini-restart-wait"
                     )
-                ),
-                tools=self._gemini_tools,
-                input_audio_transcription=self._opts.input_audio_transcription,
-                output_audio_transcription=self._opts.output_audio_transcription,
-            )
-
-            async with self._client.aio.live.connect(
-                model=self._opts.model, config=config
-            ) as session:
-                async with self._session_lock:
-                    self._session = session
-
-                @utils.log_exceptions(logger=logger)
-                async def _send_task():
-                    async for msg in self._msg_ch:
-                        if isinstance(msg, LiveClientContent):
-                            await session.send(input=msg, end_of_turn=True)
-                        else:
-                            await session.send(input=msg)
-                    await session.send(input=".", end_of_turn=True)
-
-                @utils.log_exceptions(logger=logger)
-                async def _recv_task():
-                    while True:
-                        async for response in session.receive():
-                            if self._active_response_id is None:
-                                self._start_new_generation()
-                            if response.setup_complete:
-                                logger.info("connection established with gemini live api server")
-                            if response.server_content:
-                                self._handle_server_content(response.server_content)
-                            if response.tool_call:
-                                self._handle_tool_calls(response.tool_call)
-                            if response.tool_call_cancellation:
-                                self._handle_tool_call_cancellation(response.tool_call_cancellation)
-                            if response.usage_metadata:
-                                self._handle_usage_metadata(response.usage_metadata)
-                            if response.go_away:
-                                self._handle_go_away(response.go_away)
-
-                send_task = asyncio.create_task(_send_task(), name="gemini-realtime-send")
-                recv_task = asyncio.create_task(_recv_task(), name="gemini-realtime-recv")
-                reconnect_task = asyncio.create_task(
-                    self._reconnect_event.wait(), name="reconnect-wait"
-                )
 
-                try:
-                    done, _ = await asyncio.wait(
-                        [send_task, recv_task, reconnect_task],
+                    done, pending = await asyncio.wait(
+                        [send_task, recv_task, restart_wait_task],
                         return_when=asyncio.FIRST_COMPLETED,
                     )
+
                     for task in done:
-                        if task != reconnect_task:
-                            task.result()
+                        if task is not restart_wait_task and task.exception():
+                            logger.error(f"error in task {task.get_name()}: {task.exception()}")
+                            raise task.exception() or Exception(f"{task.get_name()} failed")
 
-                    if reconnect_task not in done:
+                    if restart_wait_task not in done and self._msg_ch.closed:
                         break
 
-                    self._reconnect_event.clear()
-                finally:
-                    await utils.aio.cancel_and_wait(send_task, recv_task, reconnect_task)
+                    for task in pending:
+                        await utils.aio.cancel_and_wait(task)
+
+            except asyncio.CancelledError:
+                break
+            except Exception as e:
+                logger.error(f"Gemini Realtime API error: {e}", exc_info=e)
+                if not self._msg_ch.closed:
+                    logger.info("attempting to reconnect after 1 seconds...")
+                    await asyncio.sleep(1)
+            finally:
+                await self._close_active_session()
+
+    async def _send_task(self, session: AsyncSession):
+        try:
+            async for msg in self._msg_ch:
+                async with self._session_lock:
+                    if self._session_should_close.is_set() or (
+                        not self._active_session or self._active_session != session
+                    ):
+                        break
+
+                if isinstance(msg, LiveClientContent):
+                    await session.send(input=msg)
+                else:
+                    await session.send(input=msg)
+        except Exception as e:
+            if not self._session_should_close.is_set():
+                logger.error(f"error in send task: {e}", exc_info=e)
+                self._mark_restart_needed()
+        finally:
+            logger.debug("send task finished.")
+
+    async def _recv_task(self, session: AsyncSession):
+        try:
+            while True:
+                async with self._session_lock:
+                    if self._session_should_close.is_set() or (
+                        not self._active_session or self._active_session != session
+                    ):
+                        logger.debug("receive task: Session changed or closed, stopping receive.")
+                        break
+
+                async for response in session.receive():
+                    if not self._current_generation and (
+                        response.server_content or response.tool_call
+                    ):
+                        self._start_new_generation()
+
+                    if response.session_resumption_update:
+                        if (
+                            response.session_resumption_update.resumable
+                            and response.session_resumption_update.new_handle
+                        ):
+                            self._session_resumption_handle = (
+                                response.session_resumption_update.new_handle
+                            )
+
+                    if response.server_content:
+                        self._handle_server_content(response.server_content)
+                    if response.tool_call:
+                        self._handle_tool_calls(response.tool_call)
+                    if response.tool_call_cancellation:
+                        self._handle_tool_call_cancellation(response.tool_call_cancellation)
+                    if response.usage_metadata:
+                        self._handle_usage_metadata(response.usage_metadata)
+                    if response.go_away:
+                        self._handle_go_away(response.go_away)
+
+                # TODO(dz): a server-side turn is complete
+        except Exception as e:
+            if not self._session_should_close.is_set():
+                logger.error(f"error in receive task: {e}", exc_info=e)
+                self._mark_restart_needed()
+        finally:
+            self._finalize_response(closed=True)
+
+    def _build_connect_config(self) -> LiveConnectConfig:
+        temp = self._opts.temperature if is_given(self._opts.temperature) else None
+
+        return LiveConnectConfig(
+            response_modalities=self._opts.response_modalities
+            if is_given(self._opts.response_modalities)
+            else [Modality.AUDIO],
+            generation_config=GenerationConfig(
+                candidate_count=self._opts.candidate_count,
+                temperature=temp,
+                max_output_tokens=self._opts.max_output_tokens
+                if is_given(self._opts.max_output_tokens)
+                else None,
+                top_p=self._opts.top_p if is_given(self._opts.top_p) else None,
+                top_k=self._opts.top_k if is_given(self._opts.top_k) else None,
+                presence_penalty=self._opts.presence_penalty
+                if is_given(self._opts.presence_penalty)
+                else None,
+                frequency_penalty=self._opts.frequency_penalty
+                if is_given(self._opts.frequency_penalty)
+                else None,
+            ),
+            system_instruction=Content(parts=[Part(text=self._opts.instructions)])
+            if is_given(self._opts.instructions)
+            else None,
+            speech_config=SpeechConfig(
+                voice_config=VoiceConfig(
+                    prebuilt_voice_config=PrebuiltVoiceConfig(voice_name=self._opts.voice)
+                ),
+                language_code=self._opts.language if is_given(self._opts.language) else None,
+            ),
+            tools=[Tool(function_declarations=self._gemini_declarations)],
+            input_audio_transcription=self._opts.input_audio_transcription,
+            output_audio_transcription=self._opts.output_audio_transcription,
+            session_resumption=SessionResumptionConfig(handle=self._session_resumption_handle),
+        )
 
     def _start_new_generation(self):
-        self._is_interrupted = False
-        self._active_response_id = utils.shortuuid("gemini-turn-")
+        if self._current_generation:
+            logger.warning("starting new generation while another is active. Finalizing previous.")
+            self._finalize_response(closed=True)
+
+        response_id = utils.shortuuid("gemini-turn-")
         self._current_generation = _ResponseGeneration(
             message_ch=utils.aio.Chan[llm.MessageGeneration](),
             function_ch=utils.aio.Chan[llm.FunctionCall](),
             messages={},
         )
 
-        # We'll assume each chunk belongs to a single message ID self._active_response_id
         item_generation = _MessageGeneration(
-            message_id=self._active_response_id,
+            message_id=response_id,
             text_ch=utils.aio.Chan[str](),
             audio_ch=utils.aio.Chan[rtc.AudioFrame](),
         )
+        self._current_generation.messages[response_id] = item_generation
 
         self._current_generation.message_ch.send_nowait(
             llm.MessageGeneration(
-                message_id=self._active_response_id,
+                message_id=response_id,
                 text_stream=item_generation.text_ch,
                 audio_stream=item_generation.audio_ch,
             )
@@ -498,84 +607,92 @@ class RealtimeSession(llm.RealtimeSession):
             user_initiated=False,
         )
 
-        # Resolve any pending future from generate_reply()
-        if self._pending_generation_event_id and (
-            fut := self._response_created_futures.pop(self._pending_generation_event_id, None)
-        ):
-            fut.set_result(generation_event)
+        if self._pending_generation_fut and not self._pending_generation_fut.done():
+            generation_event.user_initiated = True
+            self._pending_generation_fut.set_result(generation_event)
+            self._pending_generation_fut = None
 
-        self._pending_generation_event_id = None
         self.emit("generation_created", generation_event)
 
-        self._current_generation.messages[self._active_response_id] = item_generation
-
     def _handle_server_content(self, server_content: LiveServerContent):
-        if not self._current_generation or not self._active_response_id:
-            logger.warning(
-                "gemini-realtime-session: No active response ID, skipping server content"
-            )
+        if not self._current_generation:
+            logger.warning("received server content but no active generation.")
             return
 
-        item_generation = self._current_generation.messages[self._active_response_id]
+        response_id = list(self._current_generation.messages.keys())[0]
+        item_generation = self._current_generation.messages[response_id]
 
-        model_turn = server_content.model_turn
-        if model_turn:
+        if model_turn := server_content.model_turn:
             for part in model_turn.parts:
                 if part.text:
                     item_generation.text_ch.send_nowait(part.text)
                 if part.inline_data:
                     frame_data = part.inline_data.data
-                    frame = rtc.AudioFrame(
-                        data=frame_data,
-                        sample_rate=OUTPUT_AUDIO_SAMPLE_RATE,
-                        num_channels=NUM_CHANNELS,
-                        samples_per_channel=len(frame_data) // 2,
-                    )
-                    item_generation.audio_ch.send_nowait(frame)
-        input_transcription = server_content.input_transcription
-        if input_transcription and input_transcription.text:
-            self.emit(
-                "input_audio_transcription_completed",
-                llm.InputTranscriptionCompleted(
-                    item_id=self._active_response_id, transcript=input_transcription.text
-                ),
-            )
-        output_transcription = server_content.output_transcription
-        if output_transcription and output_transcription.text:
-            item_generation.text_ch.send_nowait(output_transcription.text)
+                    try:
+                        frame = rtc.AudioFrame(
+                            data=frame_data,
+                            sample_rate=OUTPUT_AUDIO_SAMPLE_RATE,
+                            num_channels=OUTPUT_AUDIO_CHANNELS,
+                            samples_per_channel=len(frame_data) // (2 * OUTPUT_AUDIO_CHANNELS),
+                        )
+                        item_generation.audio_ch.send_nowait(frame)
+                    except ValueError as e:
+                        logger.error(f"Error creating audio frame from Gemini data: {e}")
+
+        if input_transcription := server_content.input_transcription:
+            if input_transcription.text:
+                self.emit(
+                    "input_audio_transcription_completed",
+                    llm.InputTranscriptionCompleted(
+                        item_id=response_id, transcript=input_transcription.text
+                    ),
+                )
+                self._handle_input_speech_started()
+
+        if output_transcription := server_content.output_transcription:
+            if output_transcription.text:
+                item_generation.text_ch.send_nowait(output_transcription.text)
+
         if server_content.interrupted:
-            self._finalize_response()
+            self._finalize_response(interrupted=True)
             self._handle_input_speech_started()
 
         if server_content.turn_complete:
             self._finalize_response()
 
-    def _finalize_response(self) -> None:
+    def _finalize_response(self, interrupted: bool = False, closed: bool = False) -> None:
         if not self._current_generation:
             return
 
-        for item_generation in self._current_generation.messages.values():
-            item_generation.text_ch.close()
-            item_generation.audio_ch.close()
-
-        self._current_generation.function_ch.close()
-        self._current_generation.message_ch.close()
+        gen = self._current_generation
         self._current_generation = None
-        self._is_interrupted = True
-        self._active_response_id = None
+
+        for item_generation in gen.messages.values():
+            if not item_generation.text_ch.closed:
+                item_generation.text_ch.close()
+            if not item_generation.audio_ch.closed:
+                item_generation.audio_ch.close()
+
+        gen.function_ch.close()
+        gen.message_ch.close()
 
     def _handle_input_speech_started(self):
         self.emit("input_speech_started", llm.InputSpeechStartedEvent())
 
     def _handle_tool_calls(self, tool_call: LiveServerToolCall):
         if not self._current_generation:
+            logger.warning("received tool call but no active generation.")
             return
+
+        gen = self._current_generation
         for fnc_call in tool_call.function_calls:
-            self._current_generation.function_ch.send_nowait(
+            arguments = json.dumps(fnc_call.args)
+
+            gen.function_ch.send_nowait(
                 llm.FunctionCall(
-                    call_id=fnc_call.id or "",
+                    call_id=fnc_call.id or utils.shortuuid("fnc-call-"),
                     name=fnc_call.name,
-                    arguments=json.dumps(fnc_call.args),
+                    arguments=arguments,
                 )
             )
         self._finalize_response()
@@ -584,28 +701,45 @@ class RealtimeSession(llm.RealtimeSession):
         self, tool_call_cancellation: LiveServerToolCallCancellation
     ):
         logger.warning(
-            "function call cancelled",
-            extra={
-                "function_call_ids": tool_call_cancellation.ids,
-            },
+            "server cancelled tool calls",
+            extra={"function_call_ids": tool_call_cancellation.ids},
         )
-        self.emit("function_calls_cancelled", tool_call_cancellation.ids)
 
     def _handle_usage_metadata(self, usage_metadata: UsageMetadata):
-        # todo: handle metrics
-        logger.info("Usage metadata", extra={"usage_metadata": usage_metadata})
+        # TODO: handle metrics
+        logger.debug("usage metadata", extra={"usage_metadata": usage_metadata})
 
     def _handle_go_away(self, go_away: LiveServerGoAway):
-        # should we reconnect?
         logger.warning(
-            f"gemini live api server will soon disconnect. time left: {go_away.time_left}"
+            f"Gemini server indicates disconnection soon. Time left: {go_away.time_left}"
         )
+        # TODO(dz): this isn't a seamless reconnection just yet
+        self._session_should_close.set()
 
     def commit_audio(self) -> None:
-        raise NotImplementedError("commit_audio_buffer is not supported yet")
+        pass
 
     def clear_audio(self) -> None:
-        raise NotImplementedError("clear_audio is not supported yet")
+        self._bstream.clear()
 
-    def server_vad_enabled(self) -> bool:
-        return True
+    def _resample_audio(self, frame: rtc.AudioFrame) -> Iterator[rtc.AudioFrame]:
+        if self._input_resampler:
+            if frame.sample_rate != self._input_resampler._input_rate:
+                # input audio changed to a different sample rate
+                self._input_resampler = None
+
+        if self._input_resampler is None and (
+            frame.sample_rate != INPUT_AUDIO_SAMPLE_RATE
+            or frame.num_channels != INPUT_AUDIO_CHANNELS
+        ):
+            self._input_resampler = rtc.AudioResampler(
+                input_rate=frame.sample_rate,
+                output_rate=INPUT_AUDIO_SAMPLE_RATE,
+                num_channels=INPUT_AUDIO_CHANNELS,
+            )
+
+        if self._input_resampler:
+            # TODO(long): flush the resampler when the input source is changed
+            yield from self._input_resampler.push(frame)
+        else:
+            yield frame

livekit/plugins/google/tts.py

@@ -105,7 +105,7 @@ class TTS(tts.TTS):
         self._opts = _TTSOptions(
             voice=voice_params,
             audio_config=texttospeech.AudioConfig(
-                audio_encoding=texttospeech.AudioEncoding.OGG_OPUS,
+                audio_encoding=texttospeech.AudioEncoding.PCM,
                 sample_rate_hertz=sample_rate,
                 pitch=pitch,
                 effects_profile_id=effects_profile_id,
@@ -132,11 +132,11 @@ class TTS(tts.TTS):
         """  # noqa: E501
         params = {}
         if is_given(language):
-            params["language"] = language
+            params["language_code"] = str(language)
         if is_given(gender):
-            params["gender"] = gender
+            params["ssml_gender"] = _gender_from_str(str(gender))
         if is_given(voice_name):
-            params["voice_name"] = voice_name
+            params["name"] = voice_name
 
         if params:
             self._opts.voice = texttospeech.VoiceSelectionParams(**params)

livekit/plugins/google/utils.py

@@ -28,7 +28,7 @@ def get_tool_results_for_realtime(chat_ctx: llm.ChatContext) -> types.LiveClient
                 types.FunctionResponse(
                     id=msg.call_id,
                     name=msg.name,
-                    response={"text": msg.output},
+                    response={"output": msg.output},
                 )
             )
     return (
@@ -99,9 +99,11 @@ def to_chat_ctx(
     if current_role is not None and parts:
         turns.append(types.Content(role=current_role, parts=parts))
 
-    if not turns:
-        # if no turns, add a user message with a placeholder
-        turns = [types.Content(role="user", parts=[types.Part(text=".")])]
+    # # Gemini requires the last message to end with user's turn before they can generate
+    # # currently not used because to_chat_ctx should not be used to force a new generation
+    # if current_role != "user":
+    #     turns.append(types.Content(role="user", parts=[types.Part(text=".")]))
+
     return turns, system_instruction
 
 

livekit/plugins/google/version.py

@@ -12,4 +12,4 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-__version__ = "1.0.16"
+__version__ = "1.0.18"

{livekit_plugins_google-1.0.16.dist-info → livekit_plugins_google-1.0.18.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: livekit-plugins-google
-Version: 1.0.16
+Version: 1.0.18
 Summary: Agent Framework plugin for services from Google Cloud
 Project-URL: Documentation, https://docs.livekit.io
 Project-URL: Website, https://livekit.io/
@@ -21,8 +21,8 @@ Requires-Python: >=3.9.0
 Requires-Dist: google-auth<3,>=2
 Requires-Dist: google-cloud-speech<3,>=2
 Requires-Dist: google-cloud-texttospeech<3,>=2
-Requires-Dist: google-genai>=1.10.0
-Requires-Dist: livekit-agents>=1.0.16
+Requires-Dist: google-genai>=1.12.1
+Requires-Dist: livekit-agents>=1.0.18
 Description-Content-Type: text/markdown
 
 # LiveKit Plugins Google

{livekit_plugins_google-1.0.16.dist-info → livekit_plugins_google-1.0.18.dist-info}/RECORD

@@ -4,13 +4,13 @@ livekit/plugins/google/log.py,sha256=GI3YWN5YzrafnUccljzPRS_ZALkMNk1i21IRnTl2vNA
 livekit/plugins/google/models.py,sha256=SGjAumdDK97NNLwMFcqZdKR68f1NoGB2Rk1UP2-imG0,1457
 livekit/plugins/google/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 livekit/plugins/google/stt.py,sha256=AG_lh2fuuduJi0jFbA_QKFXLJ6NUdF1W_FfkLUJML_Q,22413
-livekit/plugins/google/tts.py,sha256=xhINokqY8UutXn85N-cbzq68eptbM6TTtIXmLktE_RM,9004
-livekit/plugins/google/utils.py,sha256=pbLSOAdQxInWhgI2Yhsrr9KvgvpFXYDdU2yx2p03pFg,9437
-livekit/plugins/google/version.py,sha256=JrpH7xxAlurLX7a-QPfPkangHuYRj8VFBapC1AR9TZs,601
+livekit/plugins/google/tts.py,sha256=fmQwW9a1kPsEsrTvIo8fqw479RxWEx0SIc3oTVaj41U,9031
+livekit/plugins/google/utils.py,sha256=TjjTwMbdJdxr3bZjUXxs-J_fipTTM00goW2-d9KWX6w,9582
+livekit/plugins/google/version.py,sha256=cnPu9FVKZV9tFmmz7lEvftrO3B_nWJVFghi3j6UcJLs,601
 livekit/plugins/google/beta/__init__.py,sha256=AxRYc7NGG62Tv1MmcZVCDHNvlhbC86hM-_yP01Qb28k,47
 livekit/plugins/google/beta/realtime/__init__.py,sha256=_fW2NMN22F-hnQ4xAJ_g5lPbR7CvM_xXzSWlUQY-E-U,188
 livekit/plugins/google/beta/realtime/api_proto.py,sha256=Fyrejs3SG0EjOPCCFLEnWXKEUxCff47PMWk2VsKJm5E,594
-livekit/plugins/google/beta/realtime/realtime_api.py,sha256=HvPYyQXC9OodWaDNxbRt1UAJ8IVdXZGK-PsIEr7UwbY,25078
-livekit_plugins_google-1.0.16.dist-info/METADATA,sha256=x7Ugs0szCq26O8ASP_7tjBzTUeMKReQ7K3Ao_MRf2DU,3492
-livekit_plugins_google-1.0.16.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-livekit_plugins_google-1.0.16.dist-info/RECORD,,
+livekit/plugins/google/beta/realtime/realtime_api.py,sha256=sXp2oHnTlHrAp5wFmcXj0bRtQKixBYedfbufcbjVHxk,30897
+livekit_plugins_google-1.0.18.dist-info/METADATA,sha256=Vqt0FoqibcKzX_jFXlyFkn-mT7iPC16JlH61VS0fbuw,3492
+livekit_plugins_google-1.0.18.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+livekit_plugins_google-1.0.18.dist-info/RECORD,,