livekit-plugins-google 1.1.6__py3-none-any.whl → 1.2.0__py3-none-any.whl

livekit/plugins/google/beta/__init__.py

@@ -1,6 +1,7 @@
 from . import realtime
+from .gemini_tts import TTS as GeminiTTS
 
-__all__ = ["realtime"]
+__all__ = ["realtime", "GeminiTTS"]
 
 # Cleanup docs of unexported modules
 _module = dir()

livekit/plugins/google/beta/gemini_tts.py

@@ -0,0 +1,247 @@
+from __future__ import annotations
+
+import os
+from dataclasses import dataclass
+from typing import Literal
+
+from google.genai import Client, types
+from google.genai.errors import APIError, ClientError, ServerError
+from livekit.agents import APIConnectionError, APIStatusError, tts, utils
+from livekit.agents.types import (
+    DEFAULT_API_CONNECT_OPTIONS,
+    NOT_GIVEN,
+    APIConnectOptions,
+    NotGivenOr,
+)
+from livekit.agents.utils import is_given
+
+GEMINI_TTS_MODELS = Literal["gemini-2.5-flash-preview-tts", "gemini-2.5-pro-preview-tts"]
+GEMINI_VOICES = Literal[
+    "Zephyr",
+    "Puck",
+    "Charon",
+    "Kore",
+    "Fenrir",
+    "Leda",
+    "Orus",
+    "Aoede",
+    "Callirrhoe",
+    "Autonoe",
+    "Enceladus",
+    "Iapetus",
+    "Umbriel",
+    "Algieba",
+    "Despina",
+    "Erinome",
+    "Algenib",
+    "Rasalgethi",
+    "Laomedeia",
+    "Achernar",
+    "Alnilam",
+    "Schedar",
+    "Gacrux",
+    "Pulcherrima",
+    "Achird",
+    "Zubenelgenubi",
+    "Vindemiatrix",
+    "Sadachbia",
+    "Sadaltager",
+    "Sulafat",
+]
+
+DEFAULT_MODEL = "gemini-2.5-flash-preview-tts"
+DEFAULT_VOICE = "Kore"
+DEFAULT_SAMPLE_RATE = 24000  # not configurable
+NUM_CHANNELS = 1
+
+DEFAULT_INSTRUCTIONS = "Say the text with a proper tone, don't omit or add any words"
+
+
+@dataclass
+class _TTSOptions:
+    model: GEMINI_TTS_MODELS | str
+    voice_name: GEMINI_VOICES | str
+    vertexai: bool
+    project: str | None
+    location: str | None
+    instructions: str | None
+
+
+class TTS(tts.TTS):
+    def __init__(
+        self,
+        *,
+        model: GEMINI_TTS_MODELS | str = DEFAULT_MODEL,
+        voice_name: GEMINI_VOICES | str = DEFAULT_VOICE,
+        api_key: NotGivenOr[str] = NOT_GIVEN,
+        vertexai: NotGivenOr[bool] = NOT_GIVEN,
+        project: NotGivenOr[str] = NOT_GIVEN,
+        location: NotGivenOr[str] = NOT_GIVEN,
+        instructions: NotGivenOr[str | None] = NOT_GIVEN,
+    ) -> None:
+        """
+        Create a new instance of Gemini TTS.
+
+        Environment Requirements:
+        - For VertexAI: Set the `GOOGLE_APPLICATION_CREDENTIALS` environment variable to the path of the service account key file.
+        - For Google Gemini API: Set the `api_key` argument or the `GOOGLE_API_KEY` environment variable.
+
+        Args:
+            model (str, optional): The Gemini TTS model to use. Defaults to "gemini-2.5-flash-preview-tts".
+            voice_name (str, optional): The voice to use for synthesis. Defaults to "Kore".
+            api_key (str, optional): The API key for Google Gemini. If not provided, it attempts to read from the `GOOGLE_API_KEY` environment variable.
+            vertexai (bool, optional): Whether to use VertexAI. Defaults to False.
+            project (str, optional): The Google Cloud project to use (only for VertexAI).
+            location (str, optional): The location to use for VertexAI API requests. Defaults to "us-central1".
+            instructions (str, optional): Control the style, tone, accent, and pace using prompts. See https://ai.google.dev/gemini-api/docs/speech-generation#controllable
+        """  # noqa: E501
+        super().__init__(
+            capabilities=tts.TTSCapabilities(streaming=False),
+            sample_rate=DEFAULT_SAMPLE_RATE,
+            num_channels=NUM_CHANNELS,
+        )
+
+        gcp_project: str | None = (
+            project if is_given(project) else os.environ.get("GOOGLE_CLOUD_PROJECT")
+        )
+        gcp_location: str | None = (
+            location
+            if is_given(location)
+            else os.environ.get("GOOGLE_CLOUD_LOCATION") or "us-central1"
+        )
+        use_vertexai = (
+            vertexai
+            if is_given(vertexai)
+            else os.environ.get("GOOGLE_GENAI_USE_VERTEXAI", "0").lower() in ["true", "1"]
+        )
+        gemini_api_key = api_key if is_given(api_key) else os.environ.get("GOOGLE_API_KEY")
+
+        if use_vertexai:
+            if not gcp_project:
+                from google.auth._default_async import default_async
+
+                _, gcp_project = default_async(  # type: ignore
+                    scopes=["https://www.googleapis.com/auth/cloud-platform"]
+                )
+            gemini_api_key = None  # VertexAI does not require an API key
+        else:
+            gcp_project = None
+            gcp_location = None
+            if not gemini_api_key:
+                raise ValueError(
+                    "API key is required for Google API either via api_key or GOOGLE_API_KEY environment variable"  # noqa: E501
+                )
+
+        self._opts = _TTSOptions(
+            model=model,
+            voice_name=voice_name,
+            vertexai=use_vertexai,
+            project=gcp_project,
+            location=gcp_location,
+            instructions=instructions if is_given(instructions) else DEFAULT_INSTRUCTIONS,
+        )
+
+        self._client = Client(
+            api_key=gemini_api_key,
+            vertexai=use_vertexai,
+            project=gcp_project,
+            location=gcp_location,
+        )
+
+    def synthesize(
+        self, text: str, *, conn_options: APIConnectOptions = DEFAULT_API_CONNECT_OPTIONS
+    ) -> ChunkedStream:
+        return ChunkedStream(tts=self, input_text=text, conn_options=conn_options)
+
+    def update_options(
+        self,
+        *,
+        voice_name: NotGivenOr[str] = NOT_GIVEN,
+    ) -> None:
+        """
+        Update the TTS options.
+
+        Args:
+            voice_name (str, optional): The voice to use for synthesis.
+        """
+        if is_given(voice_name):
+            self._opts.voice_name = voice_name
+
+
+class ChunkedStream(tts.ChunkedStream):
+    def __init__(self, *, tts: TTS, input_text: str, conn_options: APIConnectOptions) -> None:
+        super().__init__(tts=tts, input_text=input_text, conn_options=conn_options)
+        self._tts: TTS = tts
+
+    async def _run(self, output_emitter: tts.AudioEmitter) -> None:
+        try:
+            config = types.GenerateContentConfig(
+                response_modalities=["AUDIO"],
+                speech_config=types.SpeechConfig(
+                    voice_config=types.VoiceConfig(
+                        prebuilt_voice_config=types.PrebuiltVoiceConfig(
+                            voice_name=self._tts._opts.voice_name,
+                        )
+                    )
+                ),
+            )
+            input_text = self._input_text
+            if self._tts._opts.instructions is not None:
+                input_text = f'{self._tts._opts.instructions}:\n"{input_text}"'
+
+            response = await self._tts._client.aio.models.generate_content(
+                model=self._tts._opts.model,
+                contents=input_text,
+                config=config,
+            )
+
+            output_emitter.initialize(
+                request_id=utils.shortuuid(),
+                sample_rate=self._tts.sample_rate,
+                num_channels=self._tts.num_channels,
+                mime_type="audio/pcm",
+            )
+
+            if (
+                not response.candidates
+                or not (content := response.candidates[0].content)
+                or not content.parts
+            ):
+                raise APIStatusError("No audio content generated")
+
+            for part in content.parts:
+                if (
+                    (inline_data := part.inline_data)
+                    and inline_data.data
+                    and inline_data.mime_type
+                    and inline_data.mime_type.startswith("audio/")
+                ):
+                    # mime_type: audio/L16;codec=pcm;rate=24000
+                    output_emitter.push(inline_data.data)
+
+        except ClientError as e:
+            raise APIStatusError(
+                "gemini tts: client error",
+                status_code=e.code,
+                body=f"{e.message} {e.status}",
+                retryable=False if e.code != 429 else True,
+            ) from e
+        except ServerError as e:
+            raise APIStatusError(
+                "gemini tts: server error",
+                status_code=e.code,
+                body=f"{e.message} {e.status}",
+                retryable=True,
+            ) from e
+        except APIError as e:
+            raise APIStatusError(
+                "gemini tts: api error",
+                status_code=e.code,
+                body=f"{e.message} {e.status}",
+                retryable=True,
+            ) from e
+        except Exception as e:
+            raise APIConnectionError(
+                f"gemini tts: error generating speech {str(e)}",
+                retryable=True,
+            ) from e

livekit/plugins/google/beta/realtime/realtime_api.py

@@ -52,7 +52,7 @@ class _RealtimeOptions:
     api_key: str | None
     voice: Voice | str
     language: NotGivenOr[str]
-    response_modalities: NotGivenOr[list[types.Modality]]
+    response_modalities: list[types.Modality]
     vertexai: bool
     project: str | None
     location: str | None
@@ -192,12 +192,15 @@ class RealtimeModel(llm.RealtimeModel):
         ):
             server_turn_detection = False
 
+        modalities = modalities if is_given(modalities) else [types.Modality.AUDIO]
+
         super().__init__(
             capabilities=llm.RealtimeCapabilities(
                 message_truncation=False,
                 turn_detection=server_turn_detection,
                 user_transcription=input_audio_transcription is not None,
                 auto_tool_reply_generation=True,
+                audio_output=types.Modality.AUDIO in modalities,
             )
         )
 
@@ -538,7 +541,9 @@ class RealtimeSession(llm.RealtimeSession):
             return
         self.start_user_activity()
 
-    def truncate(self, *, message_id: str, audio_end_ms: int) -> None:
+    def truncate(
+        self, *, message_id: str, audio_end_ms: int, audio_transcript: NotGivenOr[str] = NOT_GIVEN
+    ) -> None:
         logger.warning("truncate is not supported by the Google Realtime API.")
         pass
 
@@ -732,9 +737,7 @@ class RealtimeSession(llm.RealtimeSession):
             gemini_tools=self._opts.gemini_tools if is_given(self._opts.gemini_tools) else None,
         )
         conf = types.LiveConnectConfig(
-            response_modalities=self._opts.response_modalities
-            if is_given(self._opts.response_modalities)
-            else [types.Modality.AUDIO],
+            response_modalities=self._opts.response_modalities,
             generation_config=types.GenerationConfig(
                 candidate_count=self._opts.candidate_count,
                 temperature=temp,
@@ -793,6 +796,8 @@ class RealtimeSession(llm.RealtimeSession):
             audio_ch=utils.aio.Chan[rtc.AudioFrame](),
             _created_timestamp=time.time(),
         )
+        if not self._realtime_model.capabilities.audio_output:
+            self._current_generation.audio_ch.close()
 
         self._current_generation.message_ch.send_nowait(
             llm.MessageGeneration(
@@ -937,7 +942,6 @@ class RealtimeSession(llm.RealtimeSession):
                     arguments=arguments,
                 )
             )
-        self._on_final_input_audio_transcription()
         self._mark_current_generation_done()
 
     def _handle_tool_call_cancellation(
@@ -1018,15 +1022,6 @@ class RealtimeSession(llm.RealtimeSession):
         # TODO(dz): this isn't a seamless reconnection just yet
         self._session_should_close.set()
 
-    def _on_final_input_audio_transcription(self) -> None:
-        if (gen := self._current_generation) and gen.input_transcription:
-            self.emit(
-                "input_audio_transcription_completed",
-                llm.InputTranscriptionCompleted(
-                    item_id=gen.response_id, transcript=gen.input_transcription, is_final=True
-                ),
-            )
-
     def commit_audio(self) -> None:
         pass
 

livekit/plugins/google/llm.py

@@ -187,6 +187,10 @@ class LLM(llm.LLM):
             location=gcp_location,
         )
 
+    @property
+    def model(self) -> str:
+        return self._opts.model
+
     def chat(
         self,
         *,

livekit/plugins/google/tts.py

@@ -34,7 +34,6 @@ from livekit.agents.utils import is_given
 from .log import logger
 from .models import Gender, SpeechLanguages
 
-BUFFERED_WORDS_COUNT = 8
 NUM_CHANNELS = 1
 DEFAULT_VOICE_NAME = "en-US-Chirp3-HD-Charon"
 DEFAULT_LANGUAGE = "en-US"
@@ -124,7 +123,7 @@ class TTS(tts.TTS):
             ssml_gender=ssml_gender,
         )
         if not is_given(tokenizer):
-            tokenizer = tokenize.basic.SentenceTokenizer(min_sentence_len=BUFFERED_WORDS_COUNT)
+            tokenizer = tokenize.blingfire.SentenceTokenizer()
 
         pronunciations = None if not is_given(custom_pronunciations) else custom_pronunciations
 

livekit/plugins/google/version.py

@@ -12,4 +12,4 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-__version__ = "1.1.6"
+__version__ = "1.2.0"

{livekit_plugins_google-1.1.6.dist-info → livekit_plugins_google-1.2.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: livekit-plugins-google
-Version: 1.1.6
+Version: 1.2.0
 Summary: Agent Framework plugin for services from Google Cloud
 Project-URL: Documentation, https://docs.livekit.io
 Project-URL: Website, https://livekit.io/
@@ -22,7 +22,7 @@ Requires-Dist: google-auth<3,>=2
 Requires-Dist: google-cloud-speech<3,>=2
 Requires-Dist: google-cloud-texttospeech<3,>=2.27
 Requires-Dist: google-genai>=v1.23.0
-Requires-Dist: livekit-agents>=1.1.6
+Requires-Dist: livekit-agents>=1.2.0
 Description-Content-Type: text/markdown
 
 # Google AI plugin for LiveKit Agents

{livekit_plugins_google-1.1.6.dist-info → livekit_plugins_google-1.2.0.dist-info}/RECORD

@@ -1,17 +1,18 @@
 livekit/plugins/google/__init__.py,sha256=XIyZ-iFnRBpaLtOJgVwojlB-a8GjdDugVFcjBpMEww8,1412
-livekit/plugins/google/llm.py,sha256=PqRQk6E2XfWkTdDrtEtcHjknGZMGbkQgVLr8uTg7F-s,18960
+livekit/plugins/google/llm.py,sha256=VmM-OEDRplHEYEVHh9rq9rI180yP7xvu_JTI2zFolbY,19035
 livekit/plugins/google/log.py,sha256=GI3YWN5YzrafnUccljzPRS_ZALkMNk1i21IRnTl2vNA,69
 livekit/plugins/google/models.py,sha256=hOpfbN_qdQ1ZTpCN9m9dvG2eb6WgQ3KE3WRpIeeM_T0,1569
 livekit/plugins/google/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 livekit/plugins/google/stt.py,sha256=ssDMH5U1vQOLA44XMlovYWIR4UqVtZSge3YFN-zZ7Iw,24696
 livekit/plugins/google/tools.py,sha256=tD5HVDHO5JfUF029Cx3axHMJec0Gxalkl7s1FDgxLzI,259
-livekit/plugins/google/tts.py,sha256=QVM4xcF7WHpbQOZDAhRJrz481iMhO9ACjjqPEdTT4Lw,16277
+livekit/plugins/google/tts.py,sha256=SODcGwteJIpGmuFArVRwuwy49k8-uQXJAIK5wBNiMC8,16219
 livekit/plugins/google/utils.py,sha256=6iihkKx76DDtLiHOoTU2ZXqzupBRY_gN3njpnwdmeqY,8829
-livekit/plugins/google/version.py,sha256=-bNd31cMcYCdhZCIKJ1-jtY4NgZvppVgKyzXAIzQtqM,600
-livekit/plugins/google/beta/__init__.py,sha256=5PnoG3Ux24bjzMSzmTeSVljE9EINivGcbWUEV6egGnM,216
+livekit/plugins/google/version.py,sha256=K1_77sNd00TXwPoLo9GplK9QYgMVj_EQWf1hyvhbWYI,600
+livekit/plugins/google/beta/__init__.py,sha256=RvAUdvEiRN-fe4JrgPcN0Jkw1kZR9wPerGMFVjS1Cc0,270
+livekit/plugins/google/beta/gemini_tts.py,sha256=esWjr0Xf95tl0_AB7MXiFZ_VCORWgcWjzvLvRa3t0FQ,8515
 livekit/plugins/google/beta/realtime/__init__.py,sha256=_fW2NMN22F-hnQ4xAJ_g5lPbR7CvM_xXzSWlUQY-E-U,188
 livekit/plugins/google/beta/realtime/api_proto.py,sha256=cbKmpX32G4gPjF6cxFNzGEDfYX19SK-vWi4Myxb8Yks,777
-livekit/plugins/google/beta/realtime/realtime_api.py,sha256=tlAsTFsumqOavC9JT2SuQi_3eGYygZ3bbS-nEM7ea8Q,46293
-livekit_plugins_google-1.1.6.dist-info/METADATA,sha256=mxBww_GRBSqdB7Djd5iZYZmik-gDD2wtTRS3i0qRbDs,1907
-livekit_plugins_google-1.1.6.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-livekit_plugins_google-1.1.6.dist-info/RECORD,,
+livekit/plugins/google/beta/realtime/realtime_api.py,sha256=LMUOhmv4KWTI6mrLAKnnRdZ72ouug69a_oULGKvoMvw,46073
+livekit_plugins_google-1.2.0.dist-info/METADATA,sha256=Vgf6Wp4oeos1kobtO3Juzs2WTPQBgeXBHKcaF8GRR78,1907
+livekit_plugins_google-1.2.0.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+livekit_plugins_google-1.2.0.dist-info/RECORD,,