livekit-plugins-google 1.0.20__py3-none-any.whl → 1.0.22__py3-none-any.whl

livekit/plugins/google/beta/realtime/realtime_api.py

@@ -4,14 +4,16 @@ import asyncio
 import contextlib
 import json
 import os
+import time
 import weakref
 from collections.abc import Iterator
-from dataclasses import dataclass
+from dataclasses import dataclass, field
 
 from google import genai
 from google.genai.live import AsyncSession
 from google.genai.types import (
     AudioTranscriptionConfig,
+    AutomaticActivityDetection,
     Blob,
     Content,
     FunctionDeclaration,
@@ -25,8 +27,10 @@ from google.genai.types import (
     LiveServerToolCall,
     LiveServerToolCallCancellation,
     Modality,
+    ModalityTokenCount,
     Part,
     PrebuiltVoiceConfig,
+    RealtimeInputConfig,
     SessionResumptionConfig,
     SpeechConfig,
     Tool,
@@ -35,19 +39,20 @@ from google.genai.types import (
 )
 from livekit import rtc
 from livekit.agents import llm, utils
+from livekit.agents.metrics import RealtimeModelMetrics
 from livekit.agents.types import NOT_GIVEN, NotGivenOr
 from livekit.agents.utils import audio as audio_utils, images, is_given
 from livekit.plugins.google.beta.realtime.api_proto import ClientEvents, LiveAPIModels, Voice
 
 from ...log import logger
-from ...utils import _build_gemini_fnc, get_tool_results_for_realtime, to_chat_ctx
+from ...utils import get_tool_results_for_realtime, to_chat_ctx, to_fnc_ctx
 
 INPUT_AUDIO_SAMPLE_RATE = 16000
 INPUT_AUDIO_CHANNELS = 1
 OUTPUT_AUDIO_SAMPLE_RATE = 24000
 OUTPUT_AUDIO_CHANNELS = 1
 
-DEFAULT_ENCODE_OPTIONS = images.EncodeOptions(
+DEFAULT_IMAGE_ENCODE_OPTIONS = images.EncodeOptions(
     format="JPEG",
     quality=75,
     resize_options=images.ResizeOptions(width=1024, height=1024, strategy="scale_aspect_fit"),
@@ -80,13 +85,7 @@ class _RealtimeOptions:
     instructions: NotGivenOr[str]
     input_audio_transcription: AudioTranscriptionConfig | None
     output_audio_transcription: AudioTranscriptionConfig | None
-
-
-@dataclass
-class _MessageGeneration:
-    message_id: str
-    text_ch: utils.aio.Chan[str]
-    audio_ch: utils.aio.Chan[rtc.AudioFrame]
+    image_encode_options: NotGivenOr[images.EncodeOptions]
 
 
 @dataclass
@@ -94,7 +93,19 @@ class _ResponseGeneration:
     message_ch: utils.aio.Chan[llm.MessageGeneration]
     function_ch: utils.aio.Chan[llm.FunctionCall]
 
-    messages: dict[str, _MessageGeneration]
+    response_id: str
+    text_ch: utils.aio.Chan[str]
+    audio_ch: utils.aio.Chan[rtc.AudioFrame]
+    input_transcription: str = ""
+
+    _created_timestamp: float = field(default_factory=time.time)
+    """The timestamp when the generation is created"""
+    _first_token_timestamp: float | None = None
+    """The timestamp when the first audio token is received"""
+    _completed_timestamp: float | None = None
+    """The timestamp when the generation is completed"""
+    _done: bool = False
+    """Whether the generation is done (set when the turn is complete)"""
 
 
 class RealtimeModel(llm.RealtimeModel):
@@ -107,7 +118,7 @@ class RealtimeModel(llm.RealtimeModel):
         voice: Voice | str = "Puck",
         language: NotGivenOr[str] = NOT_GIVEN,
         modalities: NotGivenOr[list[Modality]] = NOT_GIVEN,
-        vertexai: bool = False,
+        vertexai: NotGivenOr[bool] = NOT_GIVEN,
         project: NotGivenOr[str] = NOT_GIVEN,
         location: NotGivenOr[str] = NOT_GIVEN,
         candidate_count: int = 1,
@@ -119,12 +130,13 @@ class RealtimeModel(llm.RealtimeModel):
         frequency_penalty: NotGivenOr[float] = NOT_GIVEN,
         input_audio_transcription: NotGivenOr[AudioTranscriptionConfig | None] = NOT_GIVEN,
         output_audio_transcription: NotGivenOr[AudioTranscriptionConfig | None] = NOT_GIVEN,
+        image_encode_options: NotGivenOr[images.EncodeOptions] = NOT_GIVEN,
     ) -> None:
         """
         Initializes a RealtimeModel instance for interacting with Google's Realtime API.
 
         Environment Requirements:
-        - For VertexAI: Set the `GOOGLE_APPLICATION_CREDENTIALS` environment variable to the path of the service account key file.
+        - For VertexAI: Set the `GOOGLE_APPLICATION_CREDENTIALS` environment variable to the path of the service account key file or use any of the other Google Cloud auth methods.
         The Google Cloud project and location can be set via `project` and `location` arguments or the environment variables
         `GOOGLE_CLOUD_PROJECT` and `GOOGLE_CLOUD_LOCATION`. By default, the project is inferred from the service account key file,
         and the location defaults to "us-central1".
@@ -148,15 +160,22 @@ class RealtimeModel(llm.RealtimeModel):
             frequency_penalty (float, optional): The frequency penalty for response generation
             input_audio_transcription (AudioTranscriptionConfig | None, optional): The configuration for input audio transcription. Defaults to None.)
             output_audio_transcription (AudioTranscriptionConfig | None, optional): The configuration for output audio transcription. Defaults to AudioTranscriptionConfig().
+            image_encode_options (images.EncodeOptions, optional): The configuration for image encoding. Defaults to DEFAULT_ENCODE_OPTIONS.
 
         Raises:
             ValueError: If the API key is required but not found.
         """  # noqa: E501
+        if not is_given(input_audio_transcription):
+            input_audio_transcription = AudioTranscriptionConfig()
+        if not is_given(output_audio_transcription):
+            output_audio_transcription = AudioTranscriptionConfig()
+
         super().__init__(
             capabilities=llm.RealtimeCapabilities(
                 message_truncation=False,
                 turn_detection=True,
-                user_transcription=is_given(input_audio_transcription),
+                user_transcription=input_audio_transcription is not None,
+                auto_tool_reply_generation=True,
             )
         )
 
@@ -173,8 +192,13 @@ class RealtimeModel(llm.RealtimeModel):
             if is_given(location)
             else os.environ.get("GOOGLE_CLOUD_LOCATION") or "us-central1"
         )
+        use_vertexai = (
+            vertexai
+            if is_given(vertexai)
+            else os.environ.get("GOOGLE_GENAI_USE_VERTEXAI", "0").lower() in ["true", "1"]
+        )
 
-        if vertexai:
+        if use_vertexai:
             if not gcp_project or not gcp_location:
                 raise ValueError(
                     "Project is required for VertexAI via project kwarg or GOOGLE_CLOUD_PROJECT environment variable"  # noqa: E501
@@ -188,17 +212,12 @@ class RealtimeModel(llm.RealtimeModel):
                     "API key is required for Google API either via api_key or GOOGLE_API_KEY environment variable"  # noqa: E501
                 )
 
-        if not is_given(input_audio_transcription):
-            input_audio_transcription = None
-        if not is_given(output_audio_transcription):
-            output_audio_transcription = AudioTranscriptionConfig()
-
         self._opts = _RealtimeOptions(
             model=model,
             api_key=gemini_api_key,
             voice=voice,
             response_modalities=modalities,
-            vertexai=vertexai,
+            vertexai=use_vertexai,
             project=gcp_project,
             location=gcp_location,
             candidate_count=candidate_count,
@@ -212,6 +231,7 @@ class RealtimeModel(llm.RealtimeModel):
             input_audio_transcription=input_audio_transcription,
             output_audio_transcription=output_audio_transcription,
             language=language,
+            image_encode_options=image_encode_options,
         )
 
         self._sessions = weakref.WeakSet[RealtimeSession]()
@@ -272,7 +292,6 @@ class RealtimeSession(llm.RealtimeSession):
 
         self._session_resumption_handle: str | None = None
 
-        self._update_lock = asyncio.Lock()
         self._session_lock = asyncio.Lock()
 
     async def _close_active_session(self) -> None:
@@ -291,57 +310,59 @@ class RealtimeSession(llm.RealtimeSession):
             # reset the msg_ch, do not send messages from previous session
             self._msg_ch = utils.aio.Chan[ClientEvents]()
 
-    async def update_options(
+    def update_options(
         self,
         *,
         voice: NotGivenOr[str] = NOT_GIVEN,
         temperature: NotGivenOr[float] = NOT_GIVEN,
         tool_choice: NotGivenOr[llm.ToolChoice | None] = NOT_GIVEN,
     ) -> None:
-        async with self._update_lock:
-            should_restart = False
-            if is_given(voice) and self._opts.voice != voice:
-                self._opts.voice = voice
-                should_restart = True
+        should_restart = False
+        if is_given(voice) and self._opts.voice != voice:
+            self._opts.voice = voice
+            should_restart = True
 
-            if is_given(temperature) and self._opts.temperature != temperature:
-                self._opts.temperature = temperature if is_given(temperature) else NOT_GIVEN
-                should_restart = True
+        if is_given(temperature) and self._opts.temperature != temperature:
+            self._opts.temperature = temperature if is_given(temperature) else NOT_GIVEN
+            should_restart = True
 
-            if should_restart:
-                self._mark_restart_needed()
+        if should_restart:
+            self._mark_restart_needed()
 
     async def update_instructions(self, instructions: str) -> None:
-        async with self._update_lock:
-            if not is_given(self._opts.instructions) or self._opts.instructions != instructions:
-                self._opts.instructions = instructions
-                self._mark_restart_needed()
+        if not is_given(self._opts.instructions) or self._opts.instructions != instructions:
+            self._opts.instructions = instructions
+            self._mark_restart_needed()
 
     async def update_chat_ctx(self, chat_ctx: llm.ChatContext) -> None:
-        async with self._update_lock:
-            self._chat_ctx = chat_ctx.copy()
-            turns, _ = to_chat_ctx(self._chat_ctx, id(self), ignore_functions=True)
-            tool_results = get_tool_results_for_realtime(
-                self._chat_ctx, vertexai=self._opts.vertexai
-            )
-            # TODO(dz): need to compute delta and then either append or recreate session
+        diff_ops = llm.utils.compute_chat_ctx_diff(self._chat_ctx, chat_ctx)
+
+        if diff_ops.to_remove:
+            logger.warning("Gemini Live does not support removing messages")
+
+        append_ctx = llm.ChatContext.empty()
+        for _, item_id in diff_ops.to_create:
+            item = chat_ctx.get_by_id(item_id)
+            if item:
+                append_ctx.items.append(item)
+
+        if append_ctx.items:
+            turns, _ = to_chat_ctx(append_ctx, id(self), ignore_functions=True)
+            tool_results = get_tool_results_for_realtime(append_ctx, vertexai=self._opts.vertexai)
             if turns:
                 self._send_client_event(LiveClientContent(turns=turns, turn_complete=False))
             if tool_results:
                 self._send_client_event(tool_results)
 
     async def update_tools(self, tools: list[llm.FunctionTool]) -> None:
-        async with self._update_lock:
-            new_declarations: list[FunctionDeclaration] = [
-                _build_gemini_fnc(tool) for tool in tools
-            ]
-            current_tool_names = {f.name for f in self._gemini_declarations}
-            new_tool_names = {f.name for f in new_declarations}
-
-            if current_tool_names != new_tool_names:
-                self._gemini_declarations = new_declarations
-                self._tools = llm.ToolContext(tools)
-                self._mark_restart_needed()
+        new_declarations: list[FunctionDeclaration] = to_fnc_ctx(tools)
+        current_tool_names = {f.name for f in self._gemini_declarations}
+        new_tool_names = {f.name for f in new_declarations}
+
+        if current_tool_names != new_tool_names:
+            self._gemini_declarations = new_declarations
+            self._tools = llm.ToolContext(tools)
+            self._mark_restart_needed()
 
     @property
     def chat_ctx(self) -> llm.ChatContext:
@@ -360,7 +381,9 @@ class RealtimeSession(llm.RealtimeSession):
                 self._send_client_event(realtime_input)
 
     def push_video(self, frame: rtc.VideoFrame) -> None:
-        encoded_data = images.encode(frame, DEFAULT_ENCODE_OPTIONS)
+        encoded_data = images.encode(
+            frame, self._opts.image_encode_options or DEFAULT_IMAGE_ENCODE_OPTIONS
+        )
         realtime_input = LiveClientRealtimeInput(
             media_chunks=[Blob(data=encoded_data, mime_type="image/jpeg")]
         )
@@ -430,7 +453,7 @@ class RealtimeSession(llm.RealtimeSession):
         self._response_created_futures.clear()
 
         if self._current_generation:
-            self._finalize_response(closed=True)
+            self._mark_current_generation_done()
 
     @utils.log_exceptions(logger=logger)
     async def _main_task(self):
@@ -524,7 +547,7 @@ class RealtimeSession(llm.RealtimeSession):
                         break
 
                 async for response in session.receive():
-                    if not self._current_generation and (
+                    if (not self._current_generation or self._current_generation._done) and (
                         response.server_content or response.tool_call
                     ):
                         self._start_new_generation()
@@ -555,7 +578,7 @@ class RealtimeSession(llm.RealtimeSession):
                 logger.error(f"error in receive task: {e}", exc_info=e)
                 self._mark_restart_needed()
         finally:
-            self._finalize_response(closed=True)
+            self._mark_current_generation_done()
 
     def _build_connect_config(self) -> LiveConnectConfig:
         temp = self._opts.temperature if is_given(self._opts.temperature) else None
@@ -592,32 +615,31 @@ class RealtimeSession(llm.RealtimeSession):
             input_audio_transcription=self._opts.input_audio_transcription,
             output_audio_transcription=self._opts.output_audio_transcription,
             session_resumption=SessionResumptionConfig(handle=self._session_resumption_handle),
+            realtime_input_config=RealtimeInputConfig(
+                automatic_activity_detection=AutomaticActivityDetection(),
+            ),
         )
 
     def _start_new_generation(self):
-        if self._current_generation:
+        if self._current_generation and not self._current_generation._done:
             logger.warning("starting new generation while another is active. Finalizing previous.")
-            self._finalize_response(closed=True)
+            self._mark_current_generation_done()
 
         response_id = utils.shortuuid("gemini-turn-")
         self._current_generation = _ResponseGeneration(
             message_ch=utils.aio.Chan[llm.MessageGeneration](),
             function_ch=utils.aio.Chan[llm.FunctionCall](),
-            messages={},
-        )
-
-        item_generation = _MessageGeneration(
-            message_id=response_id,
+            response_id=response_id,
             text_ch=utils.aio.Chan[str](),
             audio_ch=utils.aio.Chan[rtc.AudioFrame](),
+            _created_timestamp=time.time(),
         )
-        self._current_generation.messages[response_id] = item_generation
 
         self._current_generation.message_ch.send_nowait(
             llm.MessageGeneration(
                 message_id=response_id,
-                text_stream=item_generation.text_ch,
-                audio_stream=item_generation.audio_ch,
+                text_stream=self._current_generation.text_ch,
+                audio_stream=self._current_generation.audio_ch,
             )
         )
 
@@ -635,18 +657,18 @@ class RealtimeSession(llm.RealtimeSession):
         self.emit("generation_created", generation_event)
 
     def _handle_server_content(self, server_content: LiveServerContent):
-        if not self._current_generation:
+        current_gen = self._current_generation
+        if not current_gen:
             logger.warning("received server content but no active generation.")
             return
 
-        response_id = list(self._current_generation.messages.keys())[0]
-        item_generation = self._current_generation.messages[response_id]
-
         if model_turn := server_content.model_turn:
             for part in model_turn.parts:
                 if part.text:
-                    item_generation.text_ch.send_nowait(part.text)
+                    current_gen.text_ch.send_nowait(part.text)
                 if part.inline_data:
+                    if not current_gen._first_token_timestamp:
+                        current_gen._first_token_timestamp = time.time()
                     frame_data = part.inline_data.data
                     try:
                         frame = rtc.AudioFrame(
@@ -655,46 +677,65 @@ class RealtimeSession(llm.RealtimeSession):
                             num_channels=OUTPUT_AUDIO_CHANNELS,
                             samples_per_channel=len(frame_data) // (2 * OUTPUT_AUDIO_CHANNELS),
                         )
-                        item_generation.audio_ch.send_nowait(frame)
+                        current_gen.audio_ch.send_nowait(frame)
                     except ValueError as e:
                         logger.error(f"Error creating audio frame from Gemini data: {e}")
 
         if input_transcription := server_content.input_transcription:
-            if input_transcription.text:
+            text = input_transcription.text
+            if text:
+                if current_gen.input_transcription == "":
+                    # gemini would start with a space, which doesn't make sense
+                    # at beginning of the transcript
+                    text = text.lstrip()
+                current_gen.input_transcription += text
                 self.emit(
                     "input_audio_transcription_completed",
                     llm.InputTranscriptionCompleted(
-                        item_id=response_id, transcript=input_transcription.text
+                        item_id=current_gen.response_id,
+                        transcript=current_gen.input_transcription,
+                        is_final=False,
                     ),
                 )
-                self._handle_input_speech_started()
 
         if output_transcription := server_content.output_transcription:
-            if output_transcription.text:
-                item_generation.text_ch.send_nowait(output_transcription.text)
+            text = output_transcription.text
+            if text:
+                current_gen.text_ch.send_nowait(text)
+
+        if server_content.generation_complete:
+            # The only way we'd know that the transcription is complete is by when they are
+            # done with generation
+            if current_gen.input_transcription:
+                self.emit(
+                    "input_audio_transcription_completed",
+                    llm.InputTranscriptionCompleted(
+                        item_id=current_gen.response_id,
+                        transcript=current_gen.input_transcription,
+                        is_final=True,
+                    ),
+                )
+            current_gen._completed_timestamp = time.time()
 
         if server_content.interrupted:
-            self._finalize_response(interrupted=True)
             self._handle_input_speech_started()
 
         if server_content.turn_complete:
-            self._finalize_response()
+            self._mark_current_generation_done()
 
-    def _finalize_response(self, interrupted: bool = False, closed: bool = False) -> None:
+    def _mark_current_generation_done(self) -> None:
         if not self._current_generation:
             return
 
         gen = self._current_generation
-        self._current_generation = None
-
-        for item_generation in gen.messages.values():
-            if not item_generation.text_ch.closed:
-                item_generation.text_ch.close()
-            if not item_generation.audio_ch.closed:
-                item_generation.audio_ch.close()
+        if not gen.text_ch.closed:
+            gen.text_ch.close()
+        if not gen.audio_ch.closed:
+            gen.audio_ch.close()
 
         gen.function_ch.close()
         gen.message_ch.close()
+        gen._done = True
 
     def _handle_input_speech_started(self):
         self.emit("input_speech_started", llm.InputSpeechStartedEvent())
@@ -715,7 +756,7 @@ class RealtimeSession(llm.RealtimeSession):
                     arguments=arguments,
                 )
             )
-        self._finalize_response()
+        self._mark_current_generation_done()
 
     def _handle_tool_call_cancellation(
         self, tool_call_cancellation: LiveServerToolCallCancellation
@@ -726,8 +767,62 @@ class RealtimeSession(llm.RealtimeSession):
         )
 
     def _handle_usage_metadata(self, usage_metadata: UsageMetadata):
-        # TODO: handle metrics
-        logger.debug("usage metadata", extra={"usage_metadata": usage_metadata})
+        current_gen = self._current_generation
+        if not current_gen:
+            logger.warning("no active generation to report metrics for")
+            return
+
+        ttft = (
+            current_gen._first_token_timestamp - current_gen._created_timestamp
+            if current_gen._first_token_timestamp
+            else -1
+        )
+        duration = (
+            current_gen._completed_timestamp or time.time()
+        ) - current_gen._created_timestamp
+
+        def _token_details_map(
+            token_details: list[ModalityTokenCount] | None,
+        ) -> dict[Modality, int]:
+            token_details_map = {"audio_tokens": 0, "text_tokens": 0, "image_tokens": 0}
+            if not token_details:
+                return token_details_map
+
+            for token_detail in token_details:
+                if token_detail.modality == Modality.AUDIO:
+                    token_details_map["audio_tokens"] += token_detail.token_count
+                elif token_detail.modality == Modality.TEXT:
+                    token_details_map["text_tokens"] += token_detail.token_count
+                elif token_detail.modality == Modality.IMAGE:
+                    token_details_map["image_tokens"] += token_detail.token_count
+            return token_details_map
+
+        metrics = RealtimeModelMetrics(
+            label=self._realtime_model._label,
+            request_id=current_gen.response_id,
+            timestamp=current_gen._created_timestamp,
+            duration=duration,
+            ttft=ttft,
+            cancelled=False,
+            input_tokens=usage_metadata.prompt_token_count or 0,
+            output_tokens=usage_metadata.response_token_count or 0,
+            total_tokens=usage_metadata.total_token_count or 0,
+            tokens_per_second=(usage_metadata.response_token_count or 0) / duration,
+            input_token_details=RealtimeModelMetrics.InputTokenDetails(
+                **_token_details_map(usage_metadata.prompt_tokens_details),
+                cached_tokens=sum(
+                    token_detail.token_count or 0
+                    for token_detail in usage_metadata.cache_tokens_details or []
+                ),
+                cached_tokens_details=RealtimeModelMetrics.CachedTokenDetails(
+                    **_token_details_map(usage_metadata.cache_tokens_details),
+                ),
+            ),
+            output_token_details=RealtimeModelMetrics.OutputTokenDetails(
+                **_token_details_map(usage_metadata.response_tokens_details),
+            ),
+        )
+        self.emit("metrics_collected", metrics)
 
     def _handle_go_away(self, go_away: LiveServerGoAway):
         logger.warning(

livekit/plugins/google/llm.py

@@ -62,7 +62,7 @@ class LLM(llm.LLM):
         *,
         model: ChatModels | str = "gemini-2.0-flash-001",
         api_key: NotGivenOr[str] = NOT_GIVEN,
-        vertexai: NotGivenOr[bool] = False,
+        vertexai: NotGivenOr[bool] = NOT_GIVEN,
         project: NotGivenOr[str] = NOT_GIVEN,
         location: NotGivenOr[str] = NOT_GIVEN,
         temperature: NotGivenOr[float] = NOT_GIVEN,
@@ -78,7 +78,7 @@ class LLM(llm.LLM):
         Create a new instance of Google GenAI LLM.
 
         Environment Requirements:
-        - For VertexAI: Set the `GOOGLE_APPLICATION_CREDENTIALS` environment variable to the path of the service account key file.
+        - For VertexAI: Set the `GOOGLE_APPLICATION_CREDENTIALS` environment variable to the path of the service account key file or use any of the other Google Cloud auth methods.
         The Google Cloud project and location can be set via `project` and `location` arguments or the environment variables
         `GOOGLE_CLOUD_PROJECT` and `GOOGLE_CLOUD_LOCATION`. By default, the project is inferred from the service account key file,
         and the location defaults to "us-central1".
@@ -87,9 +87,9 @@ class LLM(llm.LLM):
         Args:
             model (ChatModels | str, optional): The model name to use. Defaults to "gemini-2.0-flash-001".
             api_key (str, optional): The API key for Google Gemini. If not provided, it attempts to read from the `GOOGLE_API_KEY` environment variable.
-            vertexai (bool, optional): Whether to use VertexAI. Defaults to False.
-            project (str, optional): The Google Cloud project to use (only for VertexAI). Defaults to None.
-            location (str, optional): The location to use for VertexAI API requests. Defaults value is "us-central1".
+            vertexai (bool, optional): Whether to use VertexAI. If not provided, it attempts to read from the `GOOGLE_GENAI_USE_VERTEXAI` environment variable. Defaults to False.
+                project (str, optional): The Google Cloud project to use (only for VertexAI). Defaults to None.
+                location (str, optional): The location to use for VertexAI API requests. Defaults value is "us-central1".
             temperature (float, optional): Sampling temperature for response generation. Defaults to 0.8.
             max_output_tokens (int, optional): Maximum number of tokens to generate in the output. Defaults to None.
             top_p (float, optional): The nucleus sampling probability for response generation. Defaults to None.
@@ -101,15 +101,19 @@ class LLM(llm.LLM):
         """  # noqa: E501
         super().__init__()
         gcp_project = project if is_given(project) else os.environ.get("GOOGLE_CLOUD_PROJECT")
-        gcp_location = location if is_given(location) else os.environ.get("GOOGLE_CLOUD_LOCATION")
+        gcp_location = (
+            location
+            if is_given(location)
+            else os.environ.get("GOOGLE_CLOUD_LOCATION") or "us-central1"
+        )
+        use_vertexai = (
+            vertexai
+            if is_given(vertexai)
+            else os.environ.get("GOOGLE_GENAI_USE_VERTEXAI", "0").lower() in ["true", "1"]
+        )
         gemini_api_key = api_key if is_given(api_key) else os.environ.get("GOOGLE_API_KEY")
-        _gac = os.environ.get("GOOGLE_APPLICATION_CREDENTIALS")
-        if _gac is None:
-            logger.warning(
-                "`GOOGLE_APPLICATION_CREDENTIALS` environment variable is not set. please set it to the path of the service account key file. Otherwise, use any of the other Google Cloud auth methods."  # noqa: E501
-            )
 
-        if is_given(vertexai) and vertexai:
+        if use_vertexai:
             if not gcp_project:
                 _, gcp_project = default_async(
                     scopes=["https://www.googleapis.com/auth/cloud-platform"]
@@ -144,7 +148,7 @@ class LLM(llm.LLM):
             model=model,
             temperature=temperature,
             tool_choice=tool_choice,
-            vertexai=vertexai,
+            vertexai=use_vertexai,
             project=project,
             location=location,
             max_output_tokens=max_output_tokens,
@@ -156,7 +160,7 @@ class LLM(llm.LLM):
         )
         self._client = genai.Client(
             api_key=gemini_api_key,
-            vertexai=is_given(vertexai) and vertexai,
+            vertexai=use_vertexai,
             project=gcp_project,
             location=gcp_location,
         )
@@ -325,6 +329,7 @@ class LLMStream(llm.LLMStream):
                             usage=llm.CompletionUsage(
                                 completion_tokens=usage.candidates_token_count or 0,
                                 prompt_tokens=usage.prompt_token_count or 0,
+                                prompt_cached_tokens=usage.cached_content_token_count or 0,
                                 total_tokens=usage.total_token_count or 0,
                             ),
                         )

livekit/plugins/google/models.py

@@ -95,6 +95,8 @@ SpeechLanguages = Literal[
 Gender = Literal["male", "female", "neutral"]
 
 ChatModels = Literal[
+    "gemini-2.5-pro-preview-05-06",
+    "gemini-2.5-flash-preview-04-17",
     "gemini-2.0-flash-001",
     "gemini-2.0-flash-lite-preview-02-05",
     "gemini-2.0-pro-exp-02-05",

livekit/plugins/google/tts.py

@@ -14,6 +14,8 @@
 
 from __future__ import annotations
 
+import asyncio
+import weakref
 from dataclasses import dataclass
 
 from google.api_core.client_options import ClientOptions
@@ -25,6 +27,7 @@ from livekit.agents import (
     APIConnectOptions,
     APIStatusError,
     APITimeoutError,
+    tokenize,
     tts,
     utils,
 )
@@ -35,13 +38,21 @@ from livekit.agents.types import (
 )
 from livekit.agents.utils import is_given
 
+from .log import logger
 from .models import Gender, SpeechLanguages
 
+BUFFERED_WORDS_COUNT = 8
+NUM_CHANNELS = 1
+DEFAULT_VOICE_NAME = "en-US-Chirp3-HD-Charon"
+DEFAULT_LANGUAGE = "en-US"
+DEFAULT_GENDER = "neutral"
+
 
 @dataclass
 class _TTSOptions:
     voice: texttospeech.VoiceSelectionParams
     audio_config: texttospeech.AudioConfig
+    tokenizer: tokenize.SentenceTokenizer
 
 
 class TTS(tts.TTS):
@@ -59,6 +70,8 @@ class TTS(tts.TTS):
         audio_encoding: texttospeech.AudioEncoding = texttospeech.AudioEncoding.PCM,
         credentials_info: NotGivenOr[dict] = NOT_GIVEN,
         credentials_file: NotGivenOr[str] = NOT_GIVEN,
+        tokenizer: NotGivenOr[tokenize.SentenceTokenizer] = NOT_GIVEN,
+        use_streaming: NotGivenOr[bool] = NOT_GIVEN,
     ) -> None:
         """
         Create a new instance of Google TTS.
@@ -78,12 +91,14 @@ class TTS(tts.TTS):
             speaking_rate (float, optional): Speed of speech. Default is 1.0.
             credentials_info (dict, optional): Dictionary containing Google Cloud credentials. Default is None.
             credentials_file (str, optional): Path to the Google Cloud credentials JSON file. Default is None.
+            tokenizer (tokenize.SentenceTokenizer, optional): Tokenizer for the TTS. Default is a basic sentence tokenizer.
+            use_streaming (bool, optional): Whether to use streaming synthesis. Default is True.
         """  # noqa: E501
+        if not is_given(use_streaming):
+            use_streaming = True
 
         super().__init__(
-            capabilities=tts.TTSCapabilities(
-                streaming=False,
-            ),
+            capabilities=tts.TTSCapabilities(streaming=use_streaming),
             sample_rate=sample_rate,
             num_channels=1,
         )
@@ -93,15 +108,17 @@ class TTS(tts.TTS):
         self._credentials_file = credentials_file
         self._location = location
 
-        lang = language if is_given(language) else "en-US"
-        ssml_gender = _gender_from_str("neutral" if not is_given(gender) else gender)
-        name = "" if not is_given(voice_name) else voice_name
+        lang = language if is_given(language) else DEFAULT_LANGUAGE
+        ssml_gender = _gender_from_str(DEFAULT_GENDER if not is_given(gender) else gender)
+        name = DEFAULT_VOICE_NAME if not is_given(voice_name) else voice_name
 
         voice_params = texttospeech.VoiceSelectionParams(
             name=name,
             language_code=lang,
             ssml_gender=ssml_gender,
         )
+        if not is_given(tokenizer):
+            tokenizer = tokenize.basic.SentenceTokenizer(min_sentence_len=BUFFERED_WORDS_COUNT)
 
         self._opts = _TTSOptions(
             voice=voice_params,
@@ -112,7 +129,9 @@ class TTS(tts.TTS):
                 effects_profile_id=effects_profile_id,
                 speaking_rate=speaking_rate,
             ),
+            tokenizer=tokenizer,
         )
+        self._streams = weakref.WeakSet[SynthesizeStream]()
 
     def update_options(
         self,
@@ -168,6 +187,18 @@ class TTS(tts.TTS):
         assert self._client is not None
         return self._client
 
+    def stream(
+        self, *, conn_options: APIConnectOptions = DEFAULT_API_CONNECT_OPTIONS
+    ) -> SynthesizeStream:
+        stream = SynthesizeStream(
+            tts=self,
+            opts=self._opts,
+            client=self._ensure_client(),
+            conn_options=conn_options,
+        )
+        self._streams.add(stream)
+        return stream
+
     def synthesize(
         self,
         text: str,
@@ -182,6 +213,12 @@ class TTS(tts.TTS):
             client=self._ensure_client(),
         )
 
+    async def aclose(self) -> None:
+        for stream in list(self._streams):
+            await stream.aclose()
+        self._streams.clear()
+        await super().aclose()
+
 
 class ChunkedStream(tts.ChunkedStream):
     def __init__(
@@ -230,8 +267,105 @@ class ChunkedStream(tts.ChunkedStream):
             raise APITimeoutError() from None
         except GoogleAPICallError as e:
             raise APIStatusError(
-                e.message, status_code=e.code or -1, request_id=None, body=None
-            ) from None
+                f"{e.message} {e.details}", status_code=e.code or -1, request_id=None, body=None
+            ) from e
+        except Exception as e:
+            raise APIConnectionError() from e
+
+
+class SynthesizeStream(tts.SynthesizeStream):
+    def __init__(
+        self,
+        *,
+        tts: TTS,
+        opts: _TTSOptions,
+        client: texttospeech.TextToSpeechAsyncClient,
+        conn_options: APIConnectOptions,
+    ):
+        super().__init__(tts=tts, conn_options=conn_options)
+        self._opts, self._client = opts, client
+        self._segments_ch = utils.aio.Chan[tokenize.SentenceStream]()
+
+    async def _run(self) -> None:
+        request_id = utils.shortuuid()
+
+        @utils.log_exceptions(logger=logger)
+        async def _tokenize_input():
+            input_stream = None
+            async for input in self._input_ch:
+                if isinstance(input, str):
+                    if input_stream is None:
+                        input_stream = self._opts.tokenizer.stream()
+                        self._segments_ch.send_nowait(input_stream)
+                    input_stream.push_text(input)
+                elif isinstance(input, self._FlushSentinel):
+                    if input_stream:
+                        input_stream.end_input()
+                    input_stream = None
+            self._segments_ch.close()
+
+        @utils.log_exceptions(logger=logger)
+        async def _run_segments():
+            async for input_stream in self._segments_ch:
+                await self._run_stream(input_stream, request_id)
+
+        tasks = [
+            asyncio.create_task(_tokenize_input()),
+            asyncio.create_task(_run_segments()),
+        ]
+        try:
+            await asyncio.gather(*tasks)
+        except Exception as e:
+            raise APIConnectionError() from e
+
+    async def _run_stream(self, input_stream, request_id):
+        streaming_config = texttospeech.StreamingSynthesizeConfig(
+            voice=self._opts.voice,
+            streaming_audio_config=texttospeech.StreamingAudioConfig(
+                audio_encoding=texttospeech.AudioEncoding.PCM
+            ),
+        )
+        emitter = tts.SynthesizedAudioEmitter(event_ch=self._event_ch, request_id=request_id)
+        audio_bstream = utils.audio.AudioByteStream(
+            sample_rate=self._opts.audio_config.sample_rate_hertz,
+            num_channels=NUM_CHANNELS,
+        )
+
+        @utils.log_exceptions(logger=logger)
+        async def input_generator():
+            try:
+                yield texttospeech.StreamingSynthesizeRequest(streaming_config=streaming_config)
+                async for input in input_stream:
+                    self._mark_started()
+                    yield texttospeech.StreamingSynthesizeRequest(
+                        input=texttospeech.StreamingSynthesisInput(text=input.token)
+                    )
+
+            except Exception:
+                logger.exception("an error occurred while streaming input to google TTS")
+
+        try:
+            stream = await self._client.streaming_synthesize(
+                input_generator(),
+                timeout=self._conn_options.timeout,
+            )
+            async for resp in stream:
+                for frame in audio_bstream.write(resp.audio_content):
+                    emitter.push(frame)
+
+            for frame in audio_bstream.flush():
+                emitter.push(frame)
+            emitter.flush()
+        except DeadlineExceeded as e:
+            logger.debug(f"google tts deadline exceeded: {e}")
+            pass
+        except GoogleAPICallError as e:
+            raise APIStatusError(
+                f"{e.message} {e.details}",
+                status_code=e.code or -1,
+                request_id=request_id,
+                body=None,
+            ) from e
         except Exception as e:
             raise APIConnectionError() from e
 

livekit/plugins/google/utils.py

@@ -9,15 +9,31 @@ from pydantic import TypeAdapter
 
 from google.genai import types
 from livekit.agents import llm
-from livekit.agents.llm import FunctionTool, utils as llm_utils
+from livekit.agents.llm import utils as llm_utils
+from livekit.agents.llm.tool_context import (
+    FunctionTool,
+    RawFunctionTool,
+    get_raw_function_info,
+    is_function_tool,
+    is_raw_function_tool,
+)
 
 from .log import logger
 
 __all__ = ["to_chat_ctx", "to_fnc_ctx"]
 
 
-def to_fnc_ctx(fncs: list[FunctionTool]) -> list[types.FunctionDeclaration]:
-    return [_build_gemini_fnc(fnc) for fnc in fncs]
+def to_fnc_ctx(fncs: list[FunctionTool | RawFunctionTool]) -> list[types.FunctionDeclaration]:
+    tools: list[types.FunctionDeclaration] = []
+    for fnc in fncs:
+        if is_raw_function_tool(fnc):
+            info = get_raw_function_info(fnc)
+            tools.append(types.FunctionDeclaration(**info.raw_schema))
+
+        elif is_function_tool(fnc):
+            tools.append(_build_gemini_fnc(fnc))
+
+    return tools
 
 
 def get_tool_results_for_realtime(

livekit/plugins/google/version.py

@@ -12,4 +12,4 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-__version__ = "1.0.20"
+__version__ = "1.0.22"

{livekit_plugins_google-1.0.20.dist-info → livekit_plugins_google-1.0.22.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: livekit-plugins-google
-Version: 1.0.20
+Version: 1.0.22
 Summary: Agent Framework plugin for services from Google Cloud
 Project-URL: Documentation, https://docs.livekit.io
 Project-URL: Website, https://livekit.io/
@@ -20,9 +20,9 @@ Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
 Requires-Python: >=3.9.0
 Requires-Dist: google-auth<3,>=2
 Requires-Dist: google-cloud-speech<3,>=2
-Requires-Dist: google-cloud-texttospeech<3,>=2
-Requires-Dist: google-genai>=1.12.1
-Requires-Dist: livekit-agents>=1.0.20
+Requires-Dist: google-cloud-texttospeech<3,>=2.24
+Requires-Dist: google-genai>=1.14.0
+Requires-Dist: livekit-agents>=1.0.22
 Description-Content-Type: text/markdown
 
 # Google AI plugin for LiveKit Agents

livekit_plugins_google-1.0.22.dist-info/RECORD

@@ -0,0 +1,16 @@
+livekit/plugins/google/__init__.py,sha256=xain2qUzU-YWhYWsLBkW8Q-szV-htpnzHTqymMPo-j0,1364
+livekit/plugins/google/llm.py,sha256=Kr9qeBZ5Dd0WCCBR_-gM3WWsVRZPCSteK8NpBsg2C5Y,16304
+livekit/plugins/google/log.py,sha256=GI3YWN5YzrafnUccljzPRS_ZALkMNk1i21IRnTl2vNA,69
+livekit/plugins/google/models.py,sha256=maGlEM3hK4-5hMnH9UQMJewA7BZMrnStsFLBNoNVySg,1531
+livekit/plugins/google/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+livekit/plugins/google/stt.py,sha256=2jk-1fHiBT8UW_n3CZsIEdMp2iBnUAlTnmefdUd8rAM,23620
+livekit/plugins/google/tts.py,sha256=FfhNfGtW8drmYDDfLLZDjaIp2GvNiIdoovgtZq4t_l8,14211
+livekit/plugins/google/utils.py,sha256=UBAbddYk7G8Nojg6bSC7_xN2pdl9qhs86HGhKYFuf9M,10509
+livekit/plugins/google/version.py,sha256=-8dkOE2vDSF9WN8VoBrSwU2sb5YBGFuwPnSQXQ-uaYM,601
+livekit/plugins/google/beta/__init__.py,sha256=5PnoG3Ux24bjzMSzmTeSVljE9EINivGcbWUEV6egGnM,216
+livekit/plugins/google/beta/realtime/__init__.py,sha256=_fW2NMN22F-hnQ4xAJ_g5lPbR7CvM_xXzSWlUQY-E-U,188
+livekit/plugins/google/beta/realtime/api_proto.py,sha256=Fyrejs3SG0EjOPCCFLEnWXKEUxCff47PMWk2VsKJm5E,594
+livekit/plugins/google/beta/realtime/realtime_api.py,sha256=yYB5fKXl_aaMH_ZSpfUlfOTUg4eRqqRENLTZhZMfBMc,36253
+livekit_plugins_google-1.0.22.dist-info/METADATA,sha256=S4bQZr4NhWrAI6vyJi299sh5lsD5eVMNfxvN9__xAMY,1908
+livekit_plugins_google-1.0.22.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+livekit_plugins_google-1.0.22.dist-info/RECORD,,

livekit_plugins_google-1.0.20.dist-info/RECORD

@@ -1,16 +0,0 @@
-livekit/plugins/google/__init__.py,sha256=xain2qUzU-YWhYWsLBkW8Q-szV-htpnzHTqymMPo-j0,1364
-livekit/plugins/google/llm.py,sha256=m_lRoUw4RIO1d-LtNYugl99LUNcA1y4NQ17wX7Vv5j0,16189
-livekit/plugins/google/log.py,sha256=GI3YWN5YzrafnUccljzPRS_ZALkMNk1i21IRnTl2vNA,69
-livekit/plugins/google/models.py,sha256=SGjAumdDK97NNLwMFcqZdKR68f1NoGB2Rk1UP2-imG0,1457
-livekit/plugins/google/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-livekit/plugins/google/stt.py,sha256=2jk-1fHiBT8UW_n3CZsIEdMp2iBnUAlTnmefdUd8rAM,23620
-livekit/plugins/google/tts.py,sha256=29R0ieV5sRPBf5Yi0SPFQk7ZZMbELF30bIL9K_j_Wcg,9100
-livekit/plugins/google/utils.py,sha256=zPzmnR-Rs2I87mT_k5S-PVbbuJMH8S-Hp5QcM4wv8vA,10067
-livekit/plugins/google/version.py,sha256=t4KmPVTpEy1pOJ2GRCA-GNJfCQq_-zHNDBxGj4GKfVk,601
-livekit/plugins/google/beta/__init__.py,sha256=5PnoG3Ux24bjzMSzmTeSVljE9EINivGcbWUEV6egGnM,216
-livekit/plugins/google/beta/realtime/__init__.py,sha256=_fW2NMN22F-hnQ4xAJ_g5lPbR7CvM_xXzSWlUQY-E-U,188
-livekit/plugins/google/beta/realtime/api_proto.py,sha256=Fyrejs3SG0EjOPCCFLEnWXKEUxCff47PMWk2VsKJm5E,594
-livekit/plugins/google/beta/realtime/realtime_api.py,sha256=K_YD2CND3PMGV7c3gJY2UdReeLfsOPtIWDys5EU2T_A,31699
-livekit_plugins_google-1.0.20.dist-info/METADATA,sha256=govmSaj6few3t11vreVNKlH9Ki2YzbRGnN3b3il2f20,1905
-livekit_plugins_google-1.0.20.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-livekit_plugins_google-1.0.20.dist-info/RECORD,,