livekit-plugins-google 1.2.9__py3-none-any.whl → 1.2.12__py3-none-any.whl

livekit/plugins/google/beta/gemini_tts.py

@@ -148,6 +148,17 @@ class TTS(tts.TTS):
             location=gcp_location,
         )
 
+    @property
+    def model(self) -> str:
+        return self._opts.model
+
+    @property
+    def provider(self) -> str:
+        if self._client.vertexai:
+            return "Vertex AI"
+        else:
+            return "Gemini"
+
     def synthesize(
         self, text: str, *, conn_options: APIConnectOptions = DEFAULT_API_CONNECT_OPTIONS
     ) -> ChunkedStream:

livekit/plugins/google/beta/realtime/realtime_api.py

@@ -10,12 +10,12 @@ from collections.abc import Iterator
 from dataclasses import dataclass, field
 from typing import Literal
 
-from google import genai
-from google.genai import types
+from google.genai import Client as GenAIClient, types
 from google.genai.live import AsyncSession
 from livekit import rtc
 from livekit.agents import APIConnectionError, llm, utils
 from livekit.agents.metrics import RealtimeModelMetrics
+from livekit.agents.metrics.base import Metadata
 from livekit.agents.types import (
     DEFAULT_API_CONNECT_OPTIONS,
     NOT_GIVEN,
@@ -76,6 +76,8 @@ class _RealtimeOptions:
     context_window_compression: NotGivenOr[types.ContextWindowCompressionConfig] = NOT_GIVEN
     api_version: NotGivenOr[str] = NOT_GIVEN
     gemini_tools: NotGivenOr[list[_LLMTool]] = NOT_GIVEN
+    tool_behavior: NotGivenOr[types.Behavior] = NOT_GIVEN
+    tool_response_scheduling: NotGivenOr[types.FunctionResponseScheduling] = NOT_GIVEN
 
 
 @dataclass
@@ -136,6 +138,8 @@ class RealtimeModel(llm.RealtimeModel):
         proactivity: NotGivenOr[bool] = NOT_GIVEN,
         realtime_input_config: NotGivenOr[types.RealtimeInputConfig] = NOT_GIVEN,
         context_window_compression: NotGivenOr[types.ContextWindowCompressionConfig] = NOT_GIVEN,
+        tool_behavior: NotGivenOr[types.Behavior] = NOT_GIVEN,
+        tool_response_scheduling: NotGivenOr[types.FunctionResponseScheduling] = NOT_GIVEN,
         api_version: NotGivenOr[str] = NOT_GIVEN,
         conn_options: APIConnectOptions = DEFAULT_API_CONNECT_OPTIONS,
         http_options: NotGivenOr[types.HttpOptions] = NOT_GIVEN,
@@ -174,6 +178,8 @@ class RealtimeModel(llm.RealtimeModel):
             proactivity (bool, optional): Whether to enable proactive audio. Defaults to False.
             realtime_input_config (RealtimeInputConfig, optional): The configuration for realtime input. Defaults to None.
             context_window_compression (ContextWindowCompressionConfig, optional): The configuration for context window compression. Defaults to None.
+            tool_behavior (Behavior, optional): The behavior for tool call. Default behavior is BLOCK in Gemini Realtime API.
+            tool_response_scheduling (FunctionResponseScheduling, optional): The scheduling for tool response. Default scheduling is WHEN_IDLE.
             conn_options (APIConnectOptions, optional): The configuration for the API connection. Defaults to DEFAULT_API_CONNECT_OPTIONS.
             _gemini_tools (list[LLMTool], optional): Gemini-specific tools to use for the session. This parameter is experimental and may change.
 
@@ -265,12 +271,24 @@ class RealtimeModel(llm.RealtimeModel):
             context_window_compression=context_window_compression,
             api_version=api_version,
             gemini_tools=_gemini_tools,
+            tool_behavior=tool_behavior,
             conn_options=conn_options,
             http_options=http_options,
         )
 
         self._sessions = weakref.WeakSet[RealtimeSession]()
 
+    @property
+    def model(self) -> str:
+        return self._opts.model
+
+    @property
+    def provider(self) -> str:
+        if self._opts.vertexai:
+            return "Vertex AI"
+        else:
+            return "Gemini"
+
     def session(self) -> RealtimeSession:
         sess = RealtimeSession(self)
         self._sessions.add(sess)
@@ -281,6 +299,8 @@ class RealtimeModel(llm.RealtimeModel):
         *,
         voice: NotGivenOr[str] = NOT_GIVEN,
         temperature: NotGivenOr[float] = NOT_GIVEN,
+        tool_behavior: NotGivenOr[types.Behavior] = NOT_GIVEN,
+        tool_response_scheduling: NotGivenOr[types.FunctionResponseScheduling] = NOT_GIVEN,
     ) -> None:
         """
         Update the options for the RealtimeModel.
@@ -296,10 +316,18 @@ class RealtimeModel(llm.RealtimeModel):
         if is_given(temperature):
             self._opts.temperature = temperature
 
+        if is_given(tool_behavior):
+            self._opts.tool_behavior = tool_behavior
+
+        if is_given(tool_response_scheduling):
+            self._opts.tool_response_scheduling = tool_response_scheduling
+
         for sess in self._sessions:
             sess.update_options(
                 voice=self._opts.voice,
                 temperature=self._opts.temperature,
+                tool_behavior=self._opts.tool_behavior,
+                tool_response_scheduling=self._opts.tool_response_scheduling,
             )
 
     async def aclose(self) -> None:
@@ -333,7 +361,7 @@ class RealtimeSession(llm.RealtimeSession):
         if api_version:
             http_options.api_version = api_version
 
-        self._client = genai.Client(
+        self._client = GenAIClient(
             api_key=self._opts.api_key,
             vertexai=self._opts.vertexai,
             project=self._opts.project,
@@ -377,6 +405,8 @@ class RealtimeSession(llm.RealtimeSession):
         voice: NotGivenOr[str] = NOT_GIVEN,
         temperature: NotGivenOr[float] = NOT_GIVEN,
         tool_choice: NotGivenOr[llm.ToolChoice | None] = NOT_GIVEN,
+        tool_behavior: NotGivenOr[types.Behavior] = NOT_GIVEN,
+        tool_response_scheduling: NotGivenOr[types.FunctionResponseScheduling] = NOT_GIVEN,
     ) -> None:
         should_restart = False
         if is_given(voice) and self._opts.voice != voice:
@@ -387,6 +417,20 @@ class RealtimeSession(llm.RealtimeSession):
             self._opts.temperature = temperature if is_given(temperature) else NOT_GIVEN
             should_restart = True
 
+        if is_given(tool_behavior) and self._opts.tool_behavior != tool_behavior:
+            self._opts.tool_behavior = tool_behavior
+            should_restart = True
+
+        if (
+            is_given(tool_response_scheduling)
+            and self._opts.tool_response_scheduling != tool_response_scheduling
+        ):
+            self._opts.tool_response_scheduling = tool_response_scheduling
+            # no need to restart
+
+        if is_given(tool_choice):
+            logger.warning("tool_choice is not supported by the Google Realtime API.")
+
         if should_restart:
             self._mark_restart_needed()
 
@@ -418,7 +462,11 @@ class RealtimeSession(llm.RealtimeSession):
             ).to_provider_format(format="google", inject_dummy_user_message=False)
             # we are not generating, and do not need to inject
             turns = [types.Content.model_validate(turn) for turn in turns_dict]
-            tool_results = get_tool_results_for_realtime(append_ctx, vertexai=self._opts.vertexai)
+            tool_results = get_tool_results_for_realtime(
+                append_ctx,
+                vertexai=self._opts.vertexai,
+                tool_response_scheduling=self._opts.tool_response_scheduling,
+            )
             if turns:
                 self._send_client_event(types.LiveClientContent(turns=turns, turn_complete=False))
             if tool_results:
@@ -430,7 +478,7 @@ class RealtimeSession(llm.RealtimeSession):
 
     async def update_tools(self, tools: list[llm.FunctionTool | llm.RawFunctionTool]) -> None:
         new_declarations: list[types.FunctionDeclaration] = to_fnc_ctx(
-            tools, use_parameters_json_schema=False
+            tools, use_parameters_json_schema=False, tool_behavior=self._opts.tool_behavior
         )
         current_tool_names = {f.name for f in self._gemini_declarations}
         new_tool_names = {f.name for f in new_declarations}
@@ -830,6 +878,7 @@ class RealtimeSession(llm.RealtimeSession):
             message_stream=self._current_generation.message_ch,
             function_stream=self._current_generation.function_ch,
             user_initiated=False,
+            response_id=self._current_generation.response_id,
         )
 
         if self._pending_generation_fut and not self._pending_generation_fut.done():
@@ -1019,7 +1068,7 @@ class RealtimeSession(llm.RealtimeSession):
             return token_details_map
 
         metrics = RealtimeModelMetrics(
-            label=self._realtime_model._label,
+            label=self._realtime_model.label,
             request_id=current_gen.response_id,
             timestamp=current_gen._created_timestamp,
             duration=duration,
@@ -1044,6 +1093,9 @@ class RealtimeSession(llm.RealtimeSession):
             output_token_details=RealtimeModelMetrics.OutputTokenDetails(
                 **_token_details_map(usage_metadata.response_tokens_details),
             ),
+            metadata=Metadata(
+                model_name=self._realtime_model.model, model_provider=self._realtime_model.provider
+            ),
         )
         self.emit("metrics_collected", metrics)
 

livekit/plugins/google/llm.py

@@ -187,6 +187,13 @@ class LLM(llm.LLM):
     def model(self) -> str:
         return self._opts.model
 
+    @property
+    def provider(self) -> str:
+        if self._client.vertexai:
+            return "Vertex AI"
+        else:
+            return "Gemini"
+
     def chat(
         self,
         *,

livekit/plugins/google/stt.py

@@ -70,6 +70,7 @@ class STTOptions:
     spoken_punctuation: bool
     enable_word_time_offsets: bool
     enable_word_confidence: bool
+    enable_voice_activity_events: bool
     model: SpeechModels | str
     sample_rate: int
     min_confidence_threshold: float
@@ -103,6 +104,7 @@ class STT(stt.STT):
         spoken_punctuation: bool = False,
         enable_word_time_offsets: bool = True,
         enable_word_confidence: bool = False,
+        enable_voice_activity_events: bool = False,
         model: SpeechModels | str = "latest_long",
         location: str = "global",
         sample_rate: int = 16000,
@@ -127,6 +129,7 @@ class STT(stt.STT):
             spoken_punctuation(bool): whether to use spoken punctuation (default: False)
             enable_word_time_offsets(bool): whether to enable word time offsets (default: True)
             enable_word_confidence(bool): whether to enable word confidence (default: False)
+            enable_voice_activity_events(bool): whether to enable voice activity events (default: False)
             model(SpeechModels): the model to use for recognition default: "latest_long"
             location(str): the location to use for recognition default: "global"
             sample_rate(int): the sample rate of the audio default: 16000
@@ -168,6 +171,7 @@ class STT(stt.STT):
             spoken_punctuation=spoken_punctuation,
             enable_word_time_offsets=enable_word_time_offsets,
             enable_word_confidence=enable_word_confidence,
+            enable_voice_activity_events=enable_voice_activity_events,
             model=model,
             sample_rate=sample_rate,
             min_confidence_threshold=min_confidence_threshold,
@@ -179,6 +183,14 @@ class STT(stt.STT):
             connect_cb=self._create_client,
         )
 
+    @property
+    def model(self) -> str:
+        return self._config.model
+
+    @property
+    def provider(self) -> str:
+        return "Google Cloud Platform"
+
     async def _create_client(self, timeout: float) -> SpeechAsyncClient:
         # Add support for passing a specific location that matches recognizer
         # see: https://cloud.google.com/speech-to-text/v2/docs/speech-to-text-supported-languages
@@ -507,6 +519,7 @@ class SpeechStream(stt.SpeechStream):
                         ),
                         streaming_features=cloud_speech.StreamingRecognitionFeatures(
                             interim_results=self._config.interim_results,
+                            enable_voice_activity_events=self._config.enable_voice_activity_events,
                         ),
                     )
 

livekit/plugins/google/tts.py

@@ -155,6 +155,14 @@ class TTS(tts.TTS):
         )
         self._streams = weakref.WeakSet[SynthesizeStream]()
 
+    @property
+    def model(self) -> str:
+        return "Chirp3"
+
+    @property
+    def provider(self) -> str:
+        return "Google Cloud Platform"
+
     def update_options(
         self,
         *,

livekit/plugins/google/utils.py

@@ -16,6 +16,8 @@ from livekit.agents.llm.tool_context import (
     is_function_tool,
     is_raw_function_tool,
 )
+from livekit.agents.types import NOT_GIVEN, NotGivenOr
+from livekit.agents.utils import is_given
 
 from .log import logger
 from .tools import _LLMTool
@@ -24,7 +26,10 @@ __all__ = ["to_fnc_ctx"]
 
 
 def to_fnc_ctx(
-    fncs: list[FunctionTool | RawFunctionTool], *, use_parameters_json_schema: bool = True
+    fncs: list[FunctionTool | RawFunctionTool],
+    *,
+    use_parameters_json_schema: bool = True,
+    tool_behavior: NotGivenOr[types.Behavior] = NOT_GIVEN,
 ) -> list[types.FunctionDeclaration]:
     tools: list[types.FunctionDeclaration] = []
     for fnc in fncs:
@@ -43,10 +48,14 @@ def to_fnc_ctx(
                         info.raw_schema.get("parameters", {})
                     )
                 )
+
+            if is_given(tool_behavior):
+                fnc_kwargs["behavior"] = tool_behavior
+
             tools.append(types.FunctionDeclaration(**fnc_kwargs))
 
         elif is_function_tool(fnc):
-            tools.append(_build_gemini_fnc(fnc))
+            tools.append(_build_gemini_fnc(fnc, tool_behavior=tool_behavior))
 
     return tools
 
@@ -88,7 +97,10 @@ def create_tools_config(
 
 
 def get_tool_results_for_realtime(
-    chat_ctx: llm.ChatContext, *, vertexai: bool = False
+    chat_ctx: llm.ChatContext,
+    *,
+    vertexai: bool = False,
+    tool_response_scheduling: NotGivenOr[types.FunctionResponseScheduling] = NOT_GIVEN,
 ) -> types.LiveClientToolResponse | None:
     function_responses: list[types.FunctionResponse] = []
     for msg in chat_ctx.items:
@@ -96,6 +108,9 @@ def get_tool_results_for_realtime(
             res = types.FunctionResponse(
                 name=msg.name,
                 response={"output": msg.output},
+                scheduling=tool_response_scheduling
+                if is_given(tool_response_scheduling)
+                else types.FunctionResponseScheduling.WHEN_IDLE,
             )
             if not vertexai:
                 # vertexai does not support id in FunctionResponse
@@ -109,14 +124,21 @@ def get_tool_results_for_realtime(
     )
 
 
-def _build_gemini_fnc(function_tool: FunctionTool) -> types.FunctionDeclaration:
+def _build_gemini_fnc(
+    function_tool: FunctionTool, *, tool_behavior: NotGivenOr[types.Behavior] = NOT_GIVEN
+) -> types.FunctionDeclaration:
     fnc = llm.utils.build_legacy_openai_schema(function_tool, internally_tagged=True)
     json_schema = _GeminiJsonSchema(fnc["parameters"]).simplify()
-    return types.FunctionDeclaration(
-        name=fnc["name"],
-        description=fnc["description"],
-        parameters=types.Schema.model_validate(json_schema) if json_schema else None,
-    )
+
+    kwargs = {
+        "name": fnc["name"],
+        "description": fnc["description"],
+        "parameters": types.Schema.model_validate(json_schema) if json_schema else None,
+    }
+    if is_given(tool_behavior):
+        kwargs["behavior"] = tool_behavior
+
+    return types.FunctionDeclaration(**kwargs)
 
 
 def to_response_format(response_format: type | dict) -> types.SchemaUnion:

livekit/plugins/google/version.py

@@ -12,4 +12,4 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-__version__ = "1.2.9"
+__version__ = "1.2.12"

{livekit_plugins_google-1.2.9.dist-info → livekit_plugins_google-1.2.12.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: livekit-plugins-google
-Version: 1.2.9
+Version: 1.2.12
 Summary: Agent Framework plugin for services from Google Cloud
 Project-URL: Documentation, https://docs.livekit.io
 Project-URL: Website, https://livekit.io/
@@ -22,7 +22,7 @@ Requires-Dist: google-auth<3,>=2
 Requires-Dist: google-cloud-speech<3,>=2
 Requires-Dist: google-cloud-texttospeech<3,>=2.27
 Requires-Dist: google-genai>=v1.23.0
-Requires-Dist: livekit-agents>=1.2.9
+Requires-Dist: livekit-agents>=1.2.12
 Description-Content-Type: text/markdown
 
 # Google AI plugin for LiveKit Agents

livekit_plugins_google-1.2.12.dist-info/RECORD

@@ -0,0 +1,18 @@
+livekit/plugins/google/__init__.py,sha256=XIyZ-iFnRBpaLtOJgVwojlB-a8GjdDugVFcjBpMEww8,1412
+livekit/plugins/google/llm.py,sha256=u9ZSSkdouPk0018UdiLfgthgTjjLLrXgseX1zrkeg64,18962
+livekit/plugins/google/log.py,sha256=GI3YWN5YzrafnUccljzPRS_ZALkMNk1i21IRnTl2vNA,69
+livekit/plugins/google/models.py,sha256=poOvUBvgpqmmQV5EUQsq0RgNIRAq7nH-_IZIcIfPSBI,2801
+livekit/plugins/google/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+livekit/plugins/google/stt.py,sha256=i99gqXAvYeuhdJ8wh6UlOqLXj6f5_cIni71EwSR4FGw,26467
+livekit/plugins/google/tools.py,sha256=tD5HVDHO5JfUF029Cx3axHMJec0Gxalkl7s1FDgxLzI,259
+livekit/plugins/google/tts.py,sha256=2Ba4HjAc9RWYL3W4Z2586Ir3bYQGdSH2gfxSR7VsyY4,17454
+livekit/plugins/google/utils.py,sha256=tFByjJ357A1WdCPwBQC4JABR9G5kxX0g7_FuWAIxix4,10002
+livekit/plugins/google/version.py,sha256=NhtcZ3HrHC6TVbG58IieJaw1vTkwefziFdDCa5-FYxs,601
+livekit/plugins/google/beta/__init__.py,sha256=RvAUdvEiRN-fe4JrgPcN0Jkw1kZR9wPerGMFVjS1Cc0,270
+livekit/plugins/google/beta/gemini_tts.py,sha256=SpKorOteQ7GYoGWsxV5YPuGeMexoosmtDXQVz_1ZeLA,8743
+livekit/plugins/google/beta/realtime/__init__.py,sha256=_fW2NMN22F-hnQ4xAJ_g5lPbR7CvM_xXzSWlUQY-E-U,188
+livekit/plugins/google/beta/realtime/api_proto.py,sha256=nb_QkVQDEH7h0SKA9vdS3JaL12a6t2Z1ja4SdnxE6a8,814
+livekit/plugins/google/beta/realtime/realtime_api.py,sha256=bvGLk75j6mO870PYLTZh2W3xY5IxuFkjGevltY2BhQA,50294
+livekit_plugins_google-1.2.12.dist-info/METADATA,sha256=K3DT1L2Klv00Qj7Dbg-OQm9cmz_yc70G3NZhg8VZhAY,1909
+livekit_plugins_google-1.2.12.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+livekit_plugins_google-1.2.12.dist-info/RECORD,,

livekit_plugins_google-1.2.9.dist-info/RECORD

@@ -1,18 +0,0 @@
-livekit/plugins/google/__init__.py,sha256=XIyZ-iFnRBpaLtOJgVwojlB-a8GjdDugVFcjBpMEww8,1412
-livekit/plugins/google/llm.py,sha256=aeeGqhbEScbEs-GKp1T8rLocNqmvG4UBj6diekYe4FU,18809
-livekit/plugins/google/log.py,sha256=GI3YWN5YzrafnUccljzPRS_ZALkMNk1i21IRnTl2vNA,69
-livekit/plugins/google/models.py,sha256=poOvUBvgpqmmQV5EUQsq0RgNIRAq7nH-_IZIcIfPSBI,2801
-livekit/plugins/google/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-livekit/plugins/google/stt.py,sha256=gRhVRsfg3BPNkBJGG78QOxEia0mF1jBnI_Ckq1jxqIs,25938
-livekit/plugins/google/tools.py,sha256=tD5HVDHO5JfUF029Cx3axHMJec0Gxalkl7s1FDgxLzI,259
-livekit/plugins/google/tts.py,sha256=LBLP3pEq1iCCgfidpRTtpeoDKYmXh8PKeJf1llAsybQ,17302
-livekit/plugins/google/utils.py,sha256=z0iCP6-hYix3JRm2RM5JOBEJCICehUe5N4FTl-JpXLc,9269
-livekit/plugins/google/version.py,sha256=qBF6bhFO57YNRku03dnWNtaFtkRztcLr4rdWnggtS84,600
-livekit/plugins/google/beta/__init__.py,sha256=RvAUdvEiRN-fe4JrgPcN0Jkw1kZR9wPerGMFVjS1Cc0,270
-livekit/plugins/google/beta/gemini_tts.py,sha256=esWjr0Xf95tl0_AB7MXiFZ_VCORWgcWjzvLvRa3t0FQ,8515
-livekit/plugins/google/beta/realtime/__init__.py,sha256=_fW2NMN22F-hnQ4xAJ_g5lPbR7CvM_xXzSWlUQY-E-U,188
-livekit/plugins/google/beta/realtime/api_proto.py,sha256=nb_QkVQDEH7h0SKA9vdS3JaL12a6t2Z1ja4SdnxE6a8,814
-livekit/plugins/google/beta/realtime/realtime_api.py,sha256=wmQQZB8lwreUxZ2ReMmWOutW1Hc6TiPTm53DBqdOJG0,47830
-livekit_plugins_google-1.2.9.dist-info/METADATA,sha256=77Ltyq9ZFibheExEbFyZbFrs5ATnmEETBHn353cuxUk,1907
-livekit_plugins_google-1.2.9.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-livekit_plugins_google-1.2.9.dist-info/RECORD,,